Skip to main content

av_harness/
reconciler.rs

1//! Session finalization and periodic idle reconciliation.
2
3use crate::session::{Session, SessionRegistry, Workflow};
4use av_bridge::EventBus;
5use av_core::metrics::Registry;
6use av_core::time::elapsed_us;
7use av_events::StopReason;
8use av_receipts::{Receipt, ReceiptSubject, Signer};
9use serde::Serialize;
10use std::path::PathBuf;
11use std::sync::Arc;
12use std::time::Instant;
13
14/// Hard upper bound on a single ATIF spool file the recovery scan will
15/// buffer into memory. Chosen to exceed the largest reasonable trajectory
16/// (millions of steps, hundreds of MB of reasoning) while capping the
17/// coarsest resource-exhaustion vector: an attacker-crafted trajectory
18/// with a multi-gigabyte `steps[i].message` cannot force recovery to OOM.
19const MAX_ATIF_RECOVERY_BYTES: u64 = 256 * 1024 * 1024;
20
21/// Result of closing a session.
22#[derive(Debug, Clone, Serialize)]
23#[serde(tag = "kind", rename_all = "snake_case")]
24pub enum FinalizeOutcome {
25    /// A signed-workflow receipt was issued.
26    Receipt {
27        /// The issued receipt.
28        receipt: Box<Receipt>,
29    },
30    /// An unsigned ATIF trajectory was persisted.
31    Atif {
32        /// Atomic spool path.
33        path: PathBuf,
34    },
35    /// The session had already been closed.
36    AlreadyClosed,
37}
38
39/// Lifecycle errors.
40#[derive(Debug, thiserror::Error)]
41#[non_exhaustive]
42pub enum FinalizeError {
43    /// Blocking task failed or panicked.
44    #[error("finalization task failed: {0}")]
45    Task(String),
46    /// Receipt issuance failed.
47    #[error("receipt issuance failed: {0}")]
48    Receipt(String),
49    /// ATIF persistence or parsing failed.
50    #[error("ATIF finalization failed: {0}")]
51    Atif(String),
52    /// Promotion is invalid for this session.
53    #[error("promotion refused: {0}")]
54    Promotion(String),
55    /// One or more upstream actions were not captured.
56    #[error("session capture is incomplete; refusing final artifact")]
57    CaptureIncomplete,
58    /// Lifecycle event could not be durably published.
59    #[error("lifecycle event publication failed: {0}")]
60    Bridge(String),
61}
62
63/// Shared asynchronous finalization service.
64#[derive(Clone)]
65pub struct Finalizer {
66    signer: Arc<dyn Signer>,
67    spool_dir: PathBuf,
68    metrics: Arc<Registry>,
69    bridge: Option<Arc<dyn EventBus>>,
70    /// Quota/budget state to clear once a session is sealed (closed sessions
71    /// refuse admission, so their budget counters are dead weight). Optional
72    /// because lifecycle tests construct a Finalizer without one.
73    state_store: Option<Arc<dyn av_state::StateStore>>,
74    recovery_lock: Arc<tokio::sync::Mutex<()>>,
75    /// Per-session lifecycle mutex table. Serialises `close_session`,
76    /// `promote`, and `recover_spooled_sessions` on the *same* session
77    /// id; different sessions proceed concurrently. Replaces the earlier
78    /// single global `lifecycle_lock` which head-of-line-blocked every
79    /// client close behind a long recovery scan or an idle sweep of
80    /// thousands of sessions.
81    lifecycle_locks: Arc<SessionLockTable>,
82    quarantined_sessions: Arc<parking_lot::Mutex<std::collections::HashSet<String>>>,
83    /// Artifacts already warned about during recovery scans, so a corrupt
84    /// file left on disk as evidence does not repeat its warning every tick.
85    /// Round-17 F8 + round-18 F6: bounded by `warn_once` via FIFO
86    /// eviction, not full-clear. FIFO avoids the "clear then all
87    /// 4096 legitimate recurring artifacts re-warn on the same
88    /// tick" log storm the round-17 clear-on-overflow approach
89    /// enabled under a rotating-timestamp attacker.
90    warned_artifacts: Arc<parking_lot::Mutex<WarnedArtifacts>>,
91    journal_key: [u8; 32],
92}
93
94/// FIFO-evicting set used by `warn_once`. Round-18 F6: replaces the
95/// round-17 F8 clear-on-overflow HashSet so a rotating-timestamp
96/// attacker who forces one eviction per tick cannot cause every
97/// legitimate recurring artifact to re-warn together — only ONE
98/// entry evicts per insert-past-cap.
99///
100/// Round-19 F5: the cap is stored on the struct rather than passed
101/// per-insert. Previously callers had to agree on the cap for
102/// every call; a future caller who passed a smaller value would
103/// have shrunk the deque without evicting matching set entries,
104/// silently desyncing the two collections.
105pub(crate) struct WarnedArtifacts {
106    order: std::collections::VecDeque<PathBuf>,
107    set: std::collections::HashSet<PathBuf>,
108    cap: usize,
109}
110
111impl WarnedArtifacts {
112    fn new(cap: usize) -> Self {
113        // Round-20 F6: clamp `cap` to a minimum of 1. Under
114        // `cap: 0` the FIFO oscillated at size 1 (evicting the
115        // one entry on every insert), silently breaking the
116        // "warn once per path per window" contract. A future
117        // caller wiring the cap through `HarnessConfig` and
118        // mistyping the field to `0` would otherwise degrade to
119        // warning-once-for-one-artifact-ever. Clamp closes that
120        // failure mode without a panic path.
121        let cap = cap.max(1);
122        Self {
123            order: std::collections::VecDeque::new(),
124            set: std::collections::HashSet::new(),
125            cap,
126        }
127    }
128
129    fn insert(&mut self, path: PathBuf) -> bool {
130        if self.set.contains(&path) {
131            return false;
132        }
133        if self.order.len() >= self.cap {
134            if let Some(evicted) = self.order.pop_front() {
135                self.set.remove(&evicted);
136            }
137        }
138        self.order.push_back(path.clone());
139        self.set.insert(path);
140        true
141    }
142
143    #[cfg(test)]
144    fn len(&self) -> usize {
145        self.set.len()
146    }
147}
148
149/// Cap on `warned_artifacts` so a rotating-timestamp attacker (or
150/// unbounded orphan churn) cannot leak memory forever. Round-18 F6:
151/// on insert-past-cap, ONE oldest entry evicts (FIFO) — not a full
152/// clear that would let a legitimate 4096-entry working set re-warn
153/// together every tick.
154const WARNED_ARTIFACTS_CAP: usize = 4096;
155
156/// Per-session lifecycle mutex table.
157///
158/// Invariants:
159///   * At most one `close_session` / `promote` / recovery-adopt runs
160///     concurrently for a given session_id.
161///   * Different session_ids proceed concurrently.
162///   * Entries are Arc-refcounted. `SessionLifecycleGuard::drop` opportunistically
163///     removes the entry when this task was the last waiter; readers
164///     that observed the entry before the remove simply create a new
165///     Arc — correctness is preserved because they cannot yet hold
166///     the guard.
167#[derive(Default)]
168pub struct SessionLockTable {
169    inner: dashmap::DashMap<String, Arc<tokio::sync::Mutex<()>>>,
170}
171
172impl SessionLockTable {
173    fn arc_for(&self, session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
174        // `entry().or_insert_with()` is race-free within a shard.
175        self.inner
176            .entry(session_id.to_owned())
177            .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
178            .value()
179            .clone()
180    }
181
182    /// Called after a caller drops its owned guard. Removes the entry
183    /// IFF the map is the last strong reference. `remove_if` runs under
184    /// a shard write lock so the refcount check is atomic w.r.t. `arc_for`.
185    fn try_gc(&self, session_id: &str) {
186        self.inner
187            .remove_if(session_id, |_, arc| Arc::strong_count(arc) == 1);
188    }
189
190    /// Test-only: how many session lock entries are currently resident.
191    #[cfg(test)]
192    pub(crate) fn len(&self) -> usize {
193        self.inner.len()
194    }
195}
196
197/// RAII guard that (a) holds an owned mutex permit and (b) opportunistically
198/// prunes its entry from [`SessionLockTable`] on drop.
199pub struct SessionLifecycleGuard {
200    permit: Option<tokio::sync::OwnedMutexGuard<()>>,
201    table: Arc<SessionLockTable>,
202    session_id: String,
203}
204
205impl Drop for SessionLifecycleGuard {
206    fn drop(&mut self) {
207        // Drop the permit *first* (releasing the mutex) and only then
208        // attempt GC: the strong-count check inside `try_gc` must see
209        // us gone. `Option::take` explicitly sequences the two steps
210        // instead of relying on field-declaration order.
211        drop(self.permit.take());
212        self.table.try_gc(&self.session_id);
213    }
214}
215
216struct CloseClaim<'a> {
217    session: &'a Session,
218    committed: bool,
219}
220
221#[derive(serde::Serialize, serde::Deserialize)]
222struct LifecycleOutbox {
223    session_id: String,
224    kind: String,
225    topic: String,
226    key: String,
227    value: serde_json::Value,
228    ack: Option<av_bridge::PublishAck>,
229}
230
231#[derive(serde::Serialize, serde::Deserialize)]
232struct AtifProvenance {
233    session_id: String,
234    digest: String,
235}
236
237#[derive(serde::Serialize, serde::Deserialize)]
238struct PromotionMarker {
239    session_id: String,
240    trajectory_digest: String,
241}
242
243impl Drop for CloseClaim<'_> {
244    fn drop(&mut self) {
245        if !self.committed {
246            self.session.reset_close();
247        }
248    }
249}
250
251impl Finalizer {
252    /// Access the shared metrics registry so background paths (stream
253    /// abort, worker-side supervision) can bump counters without
254    /// plumbing the registry through every struct field.
255    pub fn metrics(&self) -> &Arc<Registry> {
256        &self.metrics
257    }
258
259    /// Create a finalizer writing unsigned artifacts beneath `spool_dir`.
260    pub fn new(signer: Arc<dyn Signer>, spool_dir: PathBuf, metrics: Arc<Registry>) -> Self {
261        let journal_key = crate::journal::key_from_signer(signer.as_ref());
262        Self {
263            signer,
264            spool_dir,
265            metrics,
266            bridge: None,
267            state_store: None,
268            recovery_lock: Arc::new(tokio::sync::Mutex::new(())),
269            lifecycle_locks: Arc::new(SessionLockTable::default()),
270            quarantined_sessions: Arc::new(parking_lot::Mutex::new(std::collections::HashSet::new())),
271            warned_artifacts: Arc::new(parking_lot::Mutex::new(WarnedArtifacts::new(
272                WARNED_ARTIFACTS_CAP,
273            ))),
274            journal_key,
275        }
276    }
277
278    /// Create a finalizer that also emits receipt events to the Bridge.
279    pub fn with_bridge(
280        signer: Arc<dyn Signer>,
281        spool_dir: PathBuf,
282        metrics: Arc<Registry>,
283        bridge: Arc<dyn EventBus>,
284    ) -> Self {
285        let journal_key = crate::journal::key_from_signer(signer.as_ref());
286        Self {
287            signer,
288            spool_dir,
289            metrics,
290            bridge: Some(bridge),
291            state_store: None,
292            recovery_lock: Arc::new(tokio::sync::Mutex::new(())),
293            lifecycle_locks: Arc::new(SessionLockTable::default()),
294            quarantined_sessions: Arc::new(parking_lot::Mutex::new(std::collections::HashSet::new())),
295            warned_artifacts: Arc::new(parking_lot::Mutex::new(WarnedArtifacts::new(
296                WARNED_ARTIFACTS_CAP,
297            ))),
298            journal_key,
299        }
300    }
301
302    /// Acquire the lifecycle lock scoped to a single session id. Held
303    /// during `close_session` / `promote` / recovery-adopt so a single
304    /// session cannot be finalised twice concurrently; different
305    /// sessions proceed in parallel.
306    async fn acquire_lifecycle(&self, session_id: &str) -> SessionLifecycleGuard {
307        let arc = self.lifecycle_locks.arc_for(session_id);
308        let permit = arc.lock_owned().await;
309        SessionLifecycleGuard {
310            permit: Some(permit),
311            table: Arc::clone(&self.lifecycle_locks),
312            session_id: session_id.to_owned(),
313        }
314    }
315
316    #[cfg(test)]
317    pub(crate) fn lifecycle_locks(&self) -> Arc<SessionLockTable> {
318        Arc::clone(&self.lifecycle_locks)
319    }
320
321    /// Round-17 F8: bounded insert-if-absent for `warned_artifacts`.
322    /// When the tracked set is about to exceed `WARNED_ARTIFACTS_CAP`,
323    /// Round-17 F8 + round-18 F6: bounded insert-if-absent for
324    /// `warned_artifacts`. When the tracked set is about to exceed
325    /// `WARNED_ARTIFACTS_CAP`, ONE oldest entry evicts (FIFO) — the
326    /// round-17 approach cleared the whole set, which under a
327    /// rotating-timestamp attacker meant every legitimate recurring
328    /// artifact re-warned together each tick. FIFO cost per insert
329    /// is O(1). Returns true if this is the first warn for `path`
330    /// in the current window (caller emits the warn only then).
331    fn warn_once(&self, path: PathBuf) -> bool {
332        self.warned_artifacts.lock().insert(path)
333    }
334
335    /// Attach the quota/budget state store whose per-session counters are
336    /// cleared when a session is sealed.
337    #[must_use]
338    pub fn with_state_store(mut self, store: Arc<dyn av_state::StateStore>) -> Self {
339        self.state_store = Some(store);
340        self
341    }
342
343    /// Drop a sealed session's budget counters. Admission gates reject
344    /// closed and capture-failed sessions before any quota check, so the
345    /// counters can never be consulted again; leaving them would grow the
346    /// in-memory state store by a few cells per session forever
347    /// (attacker-chosen session ids make that unbounded).
348    fn clear_budget_state(&self, session_id: &str) {
349        if let Some(store) = self.state_store.as_deref() {
350            store.remove_prefix(&av_state::ActionBudget::session_prefix(session_id));
351        }
352    }
353
354    /// Close exactly once. Receipt signing and ATIF serialization never run on
355    /// the request hot path.
356    #[tracing::instrument(
357        name = "agentvisor.session.close",
358        skip_all,
359        fields(session.id = %session.id, workflow = ?session.workflow)
360    )]
361    pub async fn close_session(
362        &self,
363        session: Arc<Session>,
364        stop_reason: StopReason,
365    ) -> Result<FinalizeOutcome, FinalizeError> {
366        let _lifecycle = self.acquire_lifecycle(&session.id).await;
367        self.close_session_locked(session, stop_reason).await
368    }
369
370    async fn close_session_locked(
371        &self,
372        session: Arc<Session>,
373        stop_reason: StopReason,
374    ) -> Result<FinalizeOutcome, FinalizeError> {
375        let close_guard = session.close_guard();
376        if !session.try_close() {
377            return Ok(FinalizeOutcome::AlreadyClosed);
378        }
379        drop(close_guard);
380        let mut claim = CloseClaim {
381            session: &session,
382            committed: false,
383        };
384        session.wait_for_streams().await;
385        session.wait_for_worker_jobs().await;
386        if session.capture_failed() {
387            self.metrics
388                .counter(
389                    "av_incomplete_sessions_total",
390                    "Sessions refused due to incomplete capture",
391                )
392                .inc();
393            // Seal the session finalized so the idle sweeper's `!is_closed()`
394            // filter skips it — otherwise CloseClaim resets `closed` to 0 and
395            // this branch retries on every idle tick forever.
396            session.mark_artifact_committed();
397            claim.committed = true;
398            self.clear_budget_state(&session.id);
399            return Err(FinalizeError::CaptureIncomplete);
400        }
401        let started = Instant::now();
402        let outcome = match session.workflow {
403            Workflow::Signed => {
404                let subject = {
405                    let chain = session.chain.lock();
406                    ReceiptSubject::EventChain {
407                        chain_head: chain.head_hex(),
408                        event_count: chain.count(),
409                    }
410                };
411                let persisted_receipt = { session.receipt.lock().clone() };
412                let receipt = if let Some(receipt) = persisted_receipt {
413                    self.verify_configured_receipt(&receipt)?;
414                    if receipt.body.subject != subject {
415                        return Err(FinalizeError::Receipt(
416                            "persisted receipt subject does not match reconstructed chain".to_owned(),
417                        ));
418                    }
419                    receipt
420                } else {
421                    let body = session.receipt_body(subject, stop_reason);
422                    let sign_started = Instant::now();
423                    let receipt = Receipt::issue(body, self.signer.as_ref())
424                        .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
425                    self.metrics
426                        .histogram("av_receipt_sign_duration_seconds", "Receipt signing latency")
427                        .observe_us(elapsed_us(sign_started));
428                    self.persist_receipt(&session.id, &receipt).await?;
429                    *session.receipt.lock() = Some(receipt.clone());
430                    receipt
431                };
432                session.mark_artifact_committed();
433                self.emit_receipt_event(&session, &receipt).await?;
434                FinalizeOutcome::Receipt {
435                    receipt: Box::new(receipt),
436                }
437            }
438            Workflow::Unsigned => {
439                let existing_path = { session.atif_path.lock().clone() };
440                let path = if let Some(path) = existing_path {
441                    path
442                } else {
443                    let mut trajectory = session.snapshot_trajectory();
444                    // An unsigned session that captured no steps cannot ever produce a strict-valid
445                    // ATIF; seal it here so the idle sweeper skips it instead of churning forever.
446                    if trajectory.steps.is_empty() {
447                        session.mark_artifact_committed();
448                        claim.committed = true;
449                        self.clear_budget_state(&session.id);
450                        return Err(FinalizeError::Atif(
451                            "cannot finalize an unsigned session with no captured steps".to_owned(),
452                        ));
453                    }
454                    let identity = session.current_identity();
455                    trajectory.agent.extra = Some(serde_json::json!({
456                        "charter": identity.charter,
457                        "instance_uid": identity.instance_uid,
458                        "ttl_remaining_s": identity.ttl_remaining_s,
459                    }));
460                    if let Some(metrics) = trajectory.final_metrics.as_mut() {
461                        metrics.total_prompt_tokens = Some(
462                            session
463                                .totals
464                                .prompt_tokens
465                                .load(std::sync::atomic::Ordering::Acquire),
466                        );
467                        metrics.total_completion_tokens = Some(
468                            session
469                                .totals
470                                .completion_tokens
471                                .load(std::sync::atomic::Ordering::Acquire),
472                        );
473                        metrics.total_cached_tokens = Some(
474                            session
475                                .totals
476                                .cached_tokens
477                                .load(std::sync::atomic::Ordering::Acquire),
478                        );
479                        metrics.total_cost_usd = Some(
480                            session
481                                .totals
482                                .cost_usd_micros
483                                .load(std::sync::atomic::Ordering::Acquire)
484                                as f64
485                                / av_core::units::USD_MICROS_PER_DOLLAR as f64,
486                        );
487                        metrics.extra = Some(serde_json::json!({
488                            "tool_calls": session.totals.tool_calls.load(std::sync::atomic::Ordering::Acquire),
489                            "tool_allowed": session.totals.tool_allowed.load(std::sync::atomic::Ordering::Acquire),
490                            "tool_blocked": session.totals.tool_blocked.load(std::sync::atomic::Ordering::Acquire),
491                            "cost_usd_micros": session.totals.cost_usd_micros.load(std::sync::atomic::Ordering::Acquire),
492                            "stop_reason_id": session.recorded_stop_reason_id(),
493                        }));
494                    }
495                    let name = format!(
496                        "{}.json",
497                        &av_core::digest::sha256_hex(session.id.as_bytes())[..32]
498                    );
499                    let path = self.spool_dir.join(name);
500                    let write_path = path.clone();
501                    tokio::task::spawn_blocking(move || av_atif::write_atomic(&trajectory, &write_path))
502                        .await
503                        .map_err(|error| FinalizeError::Task(error.to_string()))?
504                        .map_err(|error| FinalizeError::Atif(error.to_string()))?;
505                    *session.atif_path.lock() = Some(path.clone());
506                    path
507                };
508                self.ensure_atif_provenance(&path, &session.id).await?;
509                session.mark_artifact_committed();
510                FinalizeOutcome::Atif { path }
511            }
512        };
513        let workflow = session.workflow.as_str();
514        self.emit_bridge_event(
515            &session,
516            av_events::EventClass::Session,
517            serde_json::json!({"action": "closed", "workflow": workflow}),
518            crate::journal::SESSION_CLOSE_OUTBOX_KIND,
519        )
520        .await?;
521        self.remove_step_journal(&session.id).await?;
522        self.remove_lifecycle_outbox(&session.id, crate::journal::RECEIPT_OUTBOX_KIND)
523            .await?;
524        self.remove_lifecycle_outbox(&session.id, crate::journal::SESSION_CLOSE_OUTBOX_KIND)
525            .await?;
526        self.metrics
527            .histogram(
528                "av_session_finalize_duration_seconds",
529                "Session finalization latency",
530            )
531            .observe_us(elapsed_us(started));
532        self.metrics
533            .counter("av_sessions_finalized_total", "Sessions finalized")
534            .inc();
535        claim.committed = true;
536        // Only now — with lifecycle events published and the on-disk journal
537        // removed — may the registry evict this session.
538        session.mark_close_complete();
539        self.clear_budget_state(&session.id);
540        Ok(outcome)
541    }
542
543    /// Promote a persisted unsigned trajectory into a retroactive Receipt.
544    #[tracing::instrument(
545        name = "agentvisor.session.promote",
546        skip_all,
547        fields(session.id = %session.id)
548    )]
549    pub async fn promote(&self, session: Arc<Session>) -> Result<Receipt, FinalizeError> {
550        let _lifecycle = self.acquire_lifecycle(&session.id).await;
551        if session.workflow != Workflow::Unsigned {
552            return session
553                .receipt
554                .lock()
555                .clone()
556                .ok_or_else(|| FinalizeError::Promotion("signed session has no issued receipt".to_owned()));
557        }
558        if !session.is_closed() {
559            self.close_session_locked(Arc::clone(&session), StopReason::SessionClosed)
560                .await?;
561        }
562        let persisted_receipt = { session.receipt.lock().clone() };
563        if session.is_promoted() {
564            let receipt = persisted_receipt.ok_or_else(|| {
565                FinalizeError::Promotion("promoted session has no persisted receipt".to_owned())
566            })?;
567            // Round-27 F3: opportunistically clean up an orphan
568            // `.promote` marker for an already-promoted session. Two
569            // windows used to leak markers indefinitely: (a) a crash
570            // between `finish_promotion()` and `remove_outbox(&marker)`
571            // below, and (b) any recovery-time `promote()` call that
572            // hits the early-return here without ever touching the
573            // marker. `retry_marked_promotions` would then re-read,
574            // re-verify, and re-early-return the same orphan on every
575            // idle tick forever. Best-effort cleanup: any I/O failure
576            // is warned but does not fail the promotion — the caller
577            // still gets its receipt.
578            //
579            // Extract the atif path with an inner scope so the
580            // parking_lot MutexGuard drops before `.await` (an
581            // `atif_path.lock()` temporary living across the await
582            // makes the future !Send).
583            let atif_path_opt: Option<std::path::PathBuf> = { session.atif_path.lock().clone() };
584            if let Some(atif_path) = atif_path_opt {
585                let marker = atif_path.with_extension("promote");
586                if marker.exists() {
587                    if let Err(error) = remove_outbox(&marker).await {
588                        tracing::warn!(
589                            %error,
590                            path = %av_core::fsutil::basename(&marker),
591                            "failed to clean up orphan promotion marker (promotion still succeeded)"
592                        );
593                    }
594                }
595            }
596            return Ok(receipt);
597        }
598        let path =
599            session.atif_path.lock().clone().ok_or_else(|| {
600                FinalizeError::Promotion("session has no persisted ATIF artifact".to_owned())
601            })?;
602        let marker = path.with_extension("promote");
603        if !path.with_extension("atif-auth").exists() {
604            return Err(FinalizeError::Atif(
605                "ATIF artifact has no authenticated provenance".to_owned(),
606            ));
607        }
608        self.ensure_atif_provenance(&path, &session.id).await?;
609        let bytes = read_capped_async(path.clone(), av_core::fsutil::MAX_ATIF_BYTES)
610            .await
611            .map_err(|error| FinalizeError::Atif(error.to_string()))?;
612        let trajectory: av_atif::Trajectory =
613            serde_json::from_slice(&bytes).map_err(|error| FinalizeError::Atif(error.to_string()))?;
614        let issues = av_atif::validate_trajectory(&trajectory, av_atif::Mode::Strict);
615        if !issues.is_empty() {
616            // Round-19 F6: cap the rendered head. An attacker-planted
617            // trajectory can legitimately fit millions of issues
618            // inside MAX_ATIF_BYTES; Debug-formatting all of them
619            // into a `FinalizeError::Atif(String)` amplified
620            // attacker input through every downstream log sink
621            // (tracing::warn → Vector → OTLP → …).
622            const RENDER_ISSUE_HEAD: usize = 16;
623            let total = issues.len();
624            let head: Vec<_> = issues.iter().take(RENDER_ISSUE_HEAD).collect();
625            return Err(FinalizeError::Atif(format!(
626                "strict validation failed ({total} issues, showing first {}): {head:?}",
627                head.len()
628            )));
629        }
630        let trajectory_digest = av_core::digest::sha256_hex(&bytes);
631        let subject = ReceiptSubject::AtifTrajectory {
632            trajectory_digest: trajectory_digest.clone(),
633            step_count: trajectory.steps.len() as u64,
634            retroactive: true,
635        };
636        let marker_payload = PromotionMarker {
637            session_id: session.id.clone(),
638            trajectory_digest,
639        };
640        if marker.exists() {
641            let sealed = read_capped_async(marker.clone(), av_core::fsutil::MAX_CONTROL_BYTES)
642                .await
643                .map_err(|error| FinalizeError::Atif(error.to_string()))?;
644            let actual: PromotionMarker =
645                crate::journal::open(&self.journal_key, "promotion-marker", 0, &sealed)
646                    .map_err(FinalizeError::Atif)?;
647            if actual.session_id != marker_payload.session_id
648                || actual.trajectory_digest != marker_payload.trajectory_digest
649            {
650                return Err(FinalizeError::Atif(
651                    "promotion marker does not match session and trajectory".to_owned(),
652                ));
653            }
654        } else {
655            let sealed = crate::journal::seal(&self.journal_key, "promotion-marker", 0, &marker_payload)
656                .map_err(FinalizeError::Atif)?;
657            persist_marker(&marker, &sealed).await?;
658        }
659        if !session.try_promote() {
660            return Err(FinalizeError::Promotion(
661                "promotion is already in progress".to_owned(),
662            ));
663        }
664        let receipt = if let Some(receipt) = persisted_receipt {
665            if let Err(error) = self.verify_configured_receipt(&receipt) {
666                session.reset_promotion();
667                return Err(error);
668            }
669            if receipt.body.subject != subject {
670                session.reset_promotion();
671                return Err(FinalizeError::Receipt(
672                    "persisted promotion receipt does not match ATIF artifact".to_owned(),
673                ));
674            }
675            receipt
676        } else {
677            let body = session.receipt_body(subject, StopReason::SessionClosed);
678            let issued = Receipt::issue(body, self.signer.as_ref())
679                .map_err(|error| FinalizeError::Receipt(error.to_string()));
680            let receipt = match issued {
681                Ok(receipt) => receipt,
682                Err(error) => {
683                    session.reset_promotion();
684                    return Err(error);
685                }
686            };
687            if let Err(error) = self.persist_receipt(&session.id, &receipt).await {
688                session.reset_promotion();
689                return Err(error);
690            }
691            *session.receipt.lock() = Some(receipt.clone());
692            receipt
693        };
694        if let Err(error) = self.emit_receipt_event(&session, &receipt).await {
695            session.reset_promotion();
696            return Err(error);
697        }
698        session.finish_promotion();
699        remove_outbox(&marker).await?;
700        self.remove_lifecycle_outbox(&session.id, crate::journal::RECEIPT_OUTBOX_KIND)
701            .await?;
702        self.metrics
703            .counter("av_sessions_promoted_total", "Unsigned sessions promoted")
704            .inc();
705        Ok(receipt)
706    }
707
708    /// Recover interrupted sessions from the spool: quarantine sessions with
709    /// incomplete effects, replay lifecycle outboxes, recover signed journal
710    /// sessions, consolidate unsigned step journals, then scan strict ATIF
711    /// artifacts for closed unsigned sessions. Returns the total count of
712    /// recovered sessions (unsigned + signed).
713    #[tracing::instrument(name = "agentvisor.recovery", skip_all)]
714    pub async fn recover_spooled_sessions(
715        &self,
716        sessions: &SessionRegistry,
717        breaker: &av_loopdetect::BreakerConfig,
718    ) -> Result<usize, FinalizeError> {
719        let _recovery = self.recovery_lock.lock().await;
720        // NB: no global lifecycle lock here — per-session locks are
721        // acquired inside the scan loop, right before each candidate is
722        // mutated. Without this change, a large ATIF spool at restart
723        // would block every /v1/close client call for the duration of
724        // the scan.
725        let mut quarantined = crate::worker::inflight_response_sessions(&self.spool_dir, &self.journal_key)
726            .await
727            .map_err(FinalizeError::Atif)?;
728        quarantined.extend(
729            crate::routes::unresolved_tool_sessions(&self.spool_dir, &self.journal_key)
730                .await
731                .map_err(FinalizeError::Atif)?,
732        );
733        // A marker only proves an *abandoned* effect when its session is
734        // not currently active: live sessions legitimately hold markers
735        // for the duration of an upstream call, and a request that merely
736        // straddles a periodic tick must not poison its session as
737        // capture-failed forever.
738        quarantined.retain(|id| sessions.get(id).is_none());
739        if !quarantined.is_empty() {
740            // The markers stay on disk as evidence, so every periodic tick
741            // rediscovers the same set. Warn only about ids not already in
742            // the quarantine — otherwise a single crash would repeat this
743            // warning every tick forever.
744            let mut known = self.quarantined_sessions.lock();
745            let new: Vec<&String> = quarantined.iter().filter(|id| !known.contains(*id)).collect();
746            if !new.is_empty() {
747                tracing::warn!(
748                    sessions = new.len(),
749                    "quarantining sessions with incomplete effects"
750                );
751            }
752            known.extend(quarantined.iter().cloned());
753        }
754        self.replay_lifecycle_outboxes().await?;
755        let signed_recovered = self.recover_signed_journals(sessions, breaker).await?;
756        self.consolidate_step_journals(sessions, breaker).await?;
757        let mut entries = match tokio::fs::read_dir(&self.spool_dir).await {
758            Ok(entries) => entries,
759            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
760            Err(error) => return Err(FinalizeError::Atif(error.to_string())),
761        };
762        let mut recovered = 0usize;
763        while let Some(entry) = entries
764            .next_entry()
765            .await
766            .map_err(|error| FinalizeError::Atif(error.to_string()))?
767        {
768            let path = entry.path();
769            if path.extension().and_then(std::ffi::OsStr::to_str) != Some("json") {
770                continue;
771            }
772            // Live session journals ({hash}.session.json) are handled by
773            // consolidate_step_journals above; they are not ATIF documents,
774            // so parsing them here would only spam misleading warnings
775            // every tick while a session is open.
776            if path
777                .file_name()
778                .and_then(std::ffi::OsStr::to_str)
779                .is_some_and(|name| name.ends_with(".session.json"))
780            {
781                continue;
782            }
783            // Round-44 F1: cheap sidecar-existence check FIRST — before
784            // the 64 MiB read + serde parse + strict validate. Without
785            // this ordering, N sidecar-less files (attacker-planted OR
786            // honest crashes between reconciler.rs:497 `write_atomic` and
787            // :504 `ensure_atif_provenance`) would burn O(N * 64 MiB)
788            // IO per 5 s reconciler tick, missing tick cadence and
789            // starving lifecycle-outbox replay, close completion,
790            // promotion retry, and idle eviction.
791            //
792            // Round-44 F4: quarantine sidecar-less files after the warn
793            // so per-tick cost is bounded to a single stat+rename per
794            // file, regardless of how many the attacker plants or how
795            // long a crash-torn orphan persists. Data cannot be
796            // authenticated without a `journal_key`-signed sidecar (an
797            // attacker-planted trajectory would forge a session's
798            // audit trail if we generated a sidecar from bytes on
799            // recovery), so orphaned trajectories are unrecoverable by
800            // design; renaming to `<name>.corrupt-<uid>` moves them
801            // out of the recovery scan glob (the `.json` extension
802            // filter at :767 rejects the renamed file) while preserving
803            // the bytes for operator forensic inspection.
804            if !path.with_extension("atif-auth").exists() {
805                self.metrics
806                    .counter(
807                        "av_atif_recovery_skipped_total{reason=\"unauthenticated\"}",
808                        "ATIF spool files skipped during recovery",
809                    )
810                    .inc();
811                let mut quarantine = path.clone();
812                let stem = quarantine
813                    .file_name()
814                    .and_then(std::ffi::OsStr::to_str)
815                    .unwrap_or("orphan-atif")
816                    .to_owned();
817                let new_name = format!("{stem}.corrupt-{}", av_core::new_event_uid());
818                quarantine.set_file_name(new_name);
819                match tokio::fs::rename(&path, &quarantine).await {
820                    Ok(()) => {
821                        if self.warn_once(quarantine.clone()) {
822                            tracing::warn!(
823                                original = %av_core::fsutil::basename(&path),
824                                quarantine = %av_core::fsutil::basename(&quarantine),
825                                "quarantined ATIF spool file with no authenticated provenance"
826                            );
827                        }
828                    }
829                    Err(error) => {
830                        if self.warn_once(path.clone()) {
831                            tracing::warn!(
832                                %error,
833                                path = %av_core::fsutil::basename(&path),
834                                "failed to quarantine ATIF spool file with no authenticated provenance; will retry next tick"
835                            );
836                        }
837                    }
838                }
839                continue;
840            }
841            // Bounded read: a hostile ATIF file cannot force recovery to
842            // buffer arbitrary bytes. The size cap catches the coarsest
843            // resource-exhaustion vector; adversarial JSON within the cap
844            // is still handled by serde's default recursion limit + our
845            // Strict validator.
846            let metadata = match tokio::fs::metadata(&path).await {
847                Ok(m) => m,
848                Err(error) => {
849                    tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "skipping ATIF spool file whose metadata is unreadable");
850                    continue;
851                }
852            };
853            if metadata.len() > MAX_ATIF_RECOVERY_BYTES {
854                self.metrics
855                    .counter(
856                        "av_atif_recovery_skipped_total{reason=\"too_large\"}",
857                        "ATIF spool files skipped during recovery",
858                    )
859                    .inc();
860                tracing::warn!(
861                    path = %av_core::fsutil::basename(&path),
862                    size = metadata.len(),
863                    max = MAX_ATIF_RECOVERY_BYTES,
864                    "ignoring oversize ATIF spool file",
865                );
866                continue;
867            }
868            // Round-27 F1: previously any read failure aborted the entire
869            // recovery scan via `?`. One EIO / EACCES on a single spool
870            // file (root-owned test artifact, chattr +i, transient NFS
871            // blip) would head-of-line-block every other session on
872            // every subsequent restart tick. Mirror the warn+continue
873            // discipline the other per-file steps in this scan use
874            // so recovery is per-file.
875            let bytes = match read_capped_async(path.clone(), av_core::fsutil::MAX_ATIF_BYTES).await {
876                Ok(bytes) => bytes,
877                Err(error) => {
878                    self.metrics
879                        .counter(
880                            "av_atif_recovery_skipped_total{reason=\"read_error\"}",
881                            "ATIF spool files skipped during recovery",
882                        )
883                        .inc();
884                    tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "skipping unreadable ATIF spool file");
885                    continue;
886                }
887            };
888            let trajectory: av_atif::Trajectory = match serde_json::from_slice(&bytes) {
889                Ok(trajectory) => trajectory,
890                Err(error) => {
891                    self.metrics
892                        .counter(
893                            "av_atif_recovery_skipped_total{reason=\"invalid_json\"}",
894                            "ATIF spool files skipped during recovery",
895                        )
896                        .inc();
897                    tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "ignoring invalid ATIF spool file");
898                    continue;
899                }
900            };
901            if !av_atif::validate_trajectory(&trajectory, av_atif::Mode::Strict).is_empty() {
902                self.metrics
903                    .counter(
904                        "av_atif_recovery_skipped_total{reason=\"nonconformant\"}",
905                        "ATIF spool files skipped during recovery",
906                    )
907                    .inc();
908                tracing::warn!(path = %av_core::fsutil::basename(&path), "ignoring nonconformant ATIF spool file");
909                continue;
910            }
911            let Some(session_id) = trajectory.session_id.clone() else {
912                continue;
913            };
914            // Round-44 F1: sidecar existence is now checked early
915            // (before the read+parse+validate) at the top of this
916            // loop iteration, so orphan sidecar-less files no longer
917            // reach this point.
918            if let Err(error) = self.ensure_atif_provenance(&path, &session_id).await {
919                self.metrics
920                    .counter(
921                        "av_atif_recovery_skipped_total{reason=\"provenance\"}",
922                        "ATIF spool files skipped during recovery",
923                    )
924                    .inc();
925                if self.warn_once(path.clone()) {
926                    tracing::warn!(
927                        %error,
928                        path = %av_core::fsutil::basename(&path),
929                        "ignoring ATIF spool file whose provenance does not verify"
930                    );
931                }
932                continue;
933            }
934            if sessions.get(&session_id).is_some() {
935                continue;
936            }
937            // Round-42 F1: previously the per-candidate "adopt +
938            // restore receipt" body used bare `?` on the receipt
939            // read/parse/verify calls, so a single corrupt or
940            // tamper-signed receipt on disk would abort recovery of
941            // every OTHER ATIF trajectory in the same tick (HOL
942            // block). Round-41 F1 fixed the same class of bug in
943            // `recover_signed_journals` and `consolidate_step_journals`;
944            // this branch was missed. Mirror the async-block +
945            // outcome-enum wrap so per-candidate errors warn+skip
946            // via the `av_atif_trajectory_recovery_skipped_total`
947            // counter instead of stopping the scan.
948            enum AtifCandidateOutcome {
949                Recovered,
950                Skipped,
951            }
952            let outcome: Result<AtifCandidateOutcome, FinalizeError> = async {
953            // Per-session lifecycle lock: scoped to this candidate only,
954            // released at the end of this loop iteration so the next
955            // candidate proceeds without waiting on all previous ones.
956            let _lifecycle = self.acquire_lifecycle(&session_id).await;
957            let extra = trajectory.agent.extra.as_ref();
958            let instance_uid = extra
959                .and_then(|value| value.get("instance_uid"))
960                .and_then(serde_json::Value::as_str)
961                .unwrap_or("recovered")
962                .to_owned();
963            let charter = extra
964                .and_then(|value| value.get("charter"))
965                .and_then(|value| {
966                    serde_json::from_value::<av_events::CharterFile>(value.clone())
967                        .ok()
968                        .or_else(|| value.as_str().map(Into::into))
969                })
970                .unwrap_or_else(|| "recovered".into());
971            let ttl_remaining_s = extra
972                .and_then(|value| value.get("ttl_remaining_s"))
973                .and_then(serde_json::Value::as_u64);
974            let recovered_session = match sessions.try_insert_recovered(
975                Session::recover_unsigned(
976                    session_id.clone(),
977                    av_events::AgentIdentity {
978                        version: trajectory.agent.version.clone(),
979                        charter,
980                        instance_uid,
981                        ttl_remaining_s,
982                    },
983                    breaker.clone(),
984                    path.clone(),
985                    trajectory.final_metrics.as_ref(),
986                )
987                .map_err(FinalizeError::Atif)?,
988            ) {
989                Ok(inserted) => inserted,
990                Err(_active) => {
991                    tracing::info!(session = %av_core::fsutil::basename(&path), "unsigned recovery skipped: session already active");
992                    return Ok(AtifCandidateOutcome::Skipped);
993                }
994            };
995            let receipt_path = self.receipt_path(&recovered_session.id);
996            // Round-40 F4: distinguish ENOENT from other read
997            // failures (see the twin in recover_signed_journals
998            // for full rationale). An oversize/EACCES/EIO would
999            // previously fold into "no prior receipt" and mint a
1000            // fresh one — silently erasing evidence of the
1001            // original.
1002            match tokio::fs::metadata(&receipt_path).await {
1003                Ok(_) => {
1004                    let bytes = read_capped_async(
1005                        receipt_path.clone(),
1006                        av_core::fsutil::MAX_RECEIPT_BYTES,
1007                    )
1008                    .await
1009                    .map_err(|error| {
1010                        FinalizeError::Receipt(format!("existing receipt unreadable: {error}"))
1011                    })?;
1012                    // Round-16: use the strict deserializer that
1013                    // refuses duplicate keys at any nesting level. A
1014                    // post-compromise attacker who overwrote the
1015                    // on-disk receipt bytes could otherwise smuggle a
1016                    // duplicate `instance_uid` past round-15 F4's
1017                    // top-level guard — the round-15 walker closes
1018                    // that gap uniformly.
1019                    // Round-17 F3: read is bounded by MAX_RECEIPT_BYTES
1020                    // so a hostile plant can no longer OOM the
1021                    // recovery scan on this session.
1022                    let receipt = Receipt::from_json_slice(&bytes)
1023                        .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
1024                    self.verify_configured_receipt(&receipt)?;
1025                    if path.with_extension("promote").exists() {
1026                        recovered_session.restore_pending_receipt(receipt);
1027                    } else {
1028                        recovered_session.restore_receipt(receipt);
1029                    }
1030                }
1031                Err(error) if error.kind() == std::io::ErrorKind::NotFound
1032                    || error.kind() == std::io::ErrorKind::NotADirectory =>
1033                {
1034                    // Fresh recovery — no prior receipt.
1035                }
1036                Err(error) => {
1037                    return Err(FinalizeError::Receipt(format!(
1038                        "existing receipt stat failed: {error}"
1039                    )));
1040                }
1041            }
1042            Ok(AtifCandidateOutcome::Recovered)
1043            }.await;
1044            match outcome {
1045                Ok(AtifCandidateOutcome::Recovered) => recovered += 1,
1046                Ok(AtifCandidateOutcome::Skipped) => {}
1047                Err(error) => {
1048                    self.metrics
1049                        .counter(
1050                            "av_atif_trajectory_recovery_skipped_total",
1051                            "ATIF trajectories skipped during recovery due to per-session errors (round-42 F1)",
1052                        )
1053                        .inc();
1054                    if self.warn_once(path.clone()) {
1055                        tracing::warn!(
1056                            %error,
1057                            path = %av_core::fsutil::basename(&path),
1058                            "skipping ATIF trajectory recovery due to per-session error; other sessions continue"
1059                        );
1060                    }
1061                }
1062            }
1063        }
1064        self.remove_acked_lifecycle_outboxes().await?;
1065        Ok(recovered + signed_recovered)
1066    }
1067
1068    // Round-40 F1 -> round-41: extracted the per-candidate body
1069    // into an inner `async` block whose Err is caught in the outer
1070    // loop and turned into `warn_once + counter + continue`, so a
1071    // single tampered sidecar / corrupted receipt / HMAC-drift
1072    // never head-of-line-blocks recovery of every OTHER signed
1073    // AND unsigned session for the tick.
1074    //
1075    // Round-27 F1 / F2 applied the same discipline to the ATIF-
1076    // spool and promotion-marker paths. Under-load stability
1077    // depends on this being uniform across every recovery path;
1078    // the block's outcome enum makes the "did this candidate
1079    // actually recover" question type-safe.
1080    async fn recover_signed_journals(
1081        &self,
1082        sessions: &SessionRegistry,
1083        breaker: &av_loopdetect::BreakerConfig,
1084    ) -> Result<usize, FinalizeError> {
1085        /// Round-41 F1: per-candidate outcome so the outer loop can
1086        /// distinguish "session recovered" from "candidate wasn't
1087        /// mine / already active / deliberately skipped".
1088        enum SignedCandidateOutcome {
1089            /// The session was materialised and finalized (or set aside
1090            /// as capture-failed for later quarantine). Increment the
1091            /// recovered counter.
1092            Recovered,
1093            /// This candidate wasn't for us (not a signed sidecar,
1094            /// already-active session, journal quarantined, etc.).
1095            /// Not an error; do NOT count as recovered.
1096            Skipped,
1097        }
1098        let mut entries = match tokio::fs::read_dir(&self.spool_dir).await {
1099            Ok(entries) => entries,
1100            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
1101            Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1102        };
1103        let mut recovered = 0usize;
1104        while let Some(entry) = entries
1105            .next_entry()
1106            .await
1107            .map_err(|error| FinalizeError::Atif(error.to_string()))?
1108        {
1109            let metadata_path = entry.path();
1110            let Some(name) = metadata_path.file_name().and_then(std::ffi::OsStr::to_str) else {
1111                continue;
1112            };
1113            let Some(stem) = name.strip_suffix(".session.json") else {
1114                continue;
1115            };
1116            // Round-41 F1: per-candidate body wrapped in an inner
1117            // async block so every `?` and `return Err(...)` inside
1118            // gets caught by the outer match instead of propagating
1119            // up through `recover_spooled_sessions:753` and killing
1120            // the whole recovery scan for the tick. Every prior
1121            // `continue;` becomes `return Ok(Skipped);`; every
1122            // `recovered += 1; continue;` becomes `return Ok(Recovered);`.
1123            let outcome: Result<SignedCandidateOutcome, FinalizeError> = async {
1124            let metadata = self.read_journal_metadata(&metadata_path).await?;
1125            if metadata
1126                .get("journal_version")
1127                .and_then(serde_json::Value::as_u64)
1128                != Some(2)
1129            {
1130                // Round-15 F1: previously returned Err, which
1131                // aborted the whole spool scan via the caller's `?`
1132                // — a single drifted or corrupted sidecar (upgrade
1133                // migration in progress, hostile plant, disk
1134                // bit-rot) blocked recovery of every OTHER session
1135                // on this instance. Warn + skip so unrelated
1136                // sessions still recover; an operator inspecting
1137                // the log can quarantine the specific sidecar.
1138                //
1139                // Round-16 F4: recover_spooled_sessions runs on
1140                // every reconciler tick. Dedup via
1141                // `warned_artifacts` so a persistent hostile plant
1142                // does not produce N warn lines every tick until
1143                // process restart, drowning real signal.
1144                if self.warn_once(metadata_path.clone()) {
1145                    tracing::warn!(
1146                        path = %av_core::fsutil::basename(&metadata_path),
1147                        version = ?metadata.get("journal_version"),
1148                        "sidecar has unsupported journal_version; skipping this session so recovery can proceed for the rest"
1149                    );
1150                }
1151                return Ok(SignedCandidateOutcome::Skipped);
1152            }
1153            if metadata.get("workflow").and_then(serde_json::Value::as_str) != Some(Workflow::Signed.as_str())
1154            {
1155                return Ok(SignedCandidateOutcome::Skipped);
1156            }
1157            let session_id = metadata
1158                .get("session_id")
1159                .and_then(serde_json::Value::as_str)
1160                .ok_or_else(|| FinalizeError::Atif("journal metadata has no session_id".into()))?;
1161            if sessions.get(session_id).is_some() {
1162                return Ok(SignedCandidateOutcome::Skipped);
1163            }
1164            // Per-session lifecycle lock for the signed-recovery path,
1165            // scoped to this candidate only. Released between candidates
1166            // so a client close on session B is not blocked by an
1167            // in-flight recovery-adopt of session A.
1168            let _lifecycle = self.acquire_lifecycle(session_id).await;
1169            let identity: av_events::AgentIdentity = serde_json::from_value(
1170                metadata
1171                    .get("identity")
1172                    .cloned()
1173                    .ok_or_else(|| FinalizeError::Atif("journal metadata has no identity".into()))?,
1174            )
1175            .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1176            let journal_path = self.spool_dir.join(format!("{stem}.events.ndjson"));
1177            let journal = if journal_path.exists() {
1178                read_complete_journal(&journal_path).await?
1179            } else {
1180                Vec::new()
1181            };
1182            if journal.is_empty() {
1183                // Round-14 F5: preserve the sealed metadata sidecar
1184                // when a torn-write journal has been quarantined in
1185                // a prior tick (see `read_complete_journal` at
1186                // ~:1980). Without this, we'd delete the sidecar the
1187                // very next tick, orphaning the `.corrupt-<uid>`
1188                // bytes with no linkage back to session identity.
1189                if quarantine_sibling_exists(&self.spool_dir, stem).await? {
1190                    let quarantine_metadata = self
1191                        .spool_dir
1192                        .join(format!("{stem}.session.json.corrupt-{}", av_core::new_event_uid()));
1193                    match tokio::fs::rename(&metadata_path, &quarantine_metadata).await {
1194                        Ok(()) => tracing::warn!(
1195                            metadata = %av_core::fsutil::basename(&metadata_path),
1196                            quarantine = %av_core::fsutil::basename(&quarantine_metadata),
1197                            "sealed metadata sidecar quarantined alongside its torn signed journal (round-14 F5)"
1198                        ),
1199                        Err(error) => tracing::error!(
1200                            metadata = %av_core::fsutil::basename(&metadata_path),
1201                            %error,
1202                            "failed to quarantine metadata sidecar; leaving in place so a future recovery can try again"
1203                        ),
1204                    }
1205                    return Ok(SignedCandidateOutcome::Skipped);
1206                }
1207                tokio::fs::remove_file(metadata_path.clone())
1208                    .await
1209                    .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1210                return Ok(SignedCandidateOutcome::Skipped);
1211            }
1212            let session = Arc::new(Session::new(
1213                session_id.to_owned(),
1214                Workflow::Signed,
1215                identity,
1216                breaker.clone(),
1217            ));
1218            let mut next_sequence = 0u64;
1219            let mut tool_calls = 0u64;
1220            let mut tool_allowed = 0u64;
1221            let mut tool_blocked = 0u64;
1222            let mut prompt_tokens = 0u64;
1223            let mut completion_tokens = 0u64;
1224            let mut cached_tokens = 0u64;
1225            let mut cost_usd_micros = 0u64;
1226            let mut pending_responses = std::collections::HashSet::new();
1227            let domain = format!("{}:active", session.id);
1228            for (index, line) in journal.into_iter().enumerate() {
1229                let index = u64::try_from(index)
1230                    .map_err(|_| FinalizeError::Atif("active journal index overflow".to_owned()))?;
1231                let record: crate::worker::ActiveJournalRecord =
1232                    crate::journal::open(&self.journal_key, &domain, index, line.as_bytes())
1233                        .map_err(FinalizeError::Atif)?;
1234                let event: av_events::OcsfEvent = serde_json::from_value(record.event.clone())
1235                    .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1236                if event.session_uid != session.id {
1237                    return Err(FinalizeError::Atif(format!(
1238                        "signed journal event belongs to session {:?}, expected {:?}",
1239                        event.session_uid, session.id
1240                    )));
1241                }
1242                if record.atif_step.is_some() || record.identity != event.ai_agent {
1243                    return Err(FinalizeError::Atif(
1244                        "signed active record has inconsistent workflow or identity".to_owned(),
1245                    ));
1246                }
1247                track_response_attempt(&mut pending_responses, record.response_attempt.as_ref())?;
1248                if event.metadata.sequence != index {
1249                    return Err(FinalizeError::Atif(
1250                        "signed event sequence does not match active journal index".to_owned(),
1251                    ));
1252                }
1253                if record.identity.version != session.identity.version
1254                    || record.identity.charter != session.identity.charter
1255                    || record.identity.instance_uid != session.identity.instance_uid
1256                {
1257                    return Err(FinalizeError::Atif(
1258                        "active journal changed the session identity".to_owned(),
1259                    ));
1260                }
1261                session.refresh_identity(&record.identity);
1262                next_sequence = index
1263                    .checked_add(1)
1264                    .ok_or_else(|| FinalizeError::Atif("event sequence overflow".to_owned()))?;
1265                session
1266                    .chain
1267                    .lock()
1268                    .append(&record.event)
1269                    .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
1270                tool_calls = checked_recovery_add(tool_calls, record.tool_calls, "tool calls")?;
1271                tool_allowed = checked_recovery_add(tool_allowed, record.tool_allowed, "allowed tools")?;
1272                tool_blocked = checked_recovery_add(tool_blocked, record.tool_blocked, "blocked tools")?;
1273                prompt_tokens = checked_recovery_add(prompt_tokens, record.prompt_tokens, "prompt tokens")?;
1274                completion_tokens =
1275                    checked_recovery_add(completion_tokens, record.completion_tokens, "completion tokens")?;
1276                cached_tokens = checked_recovery_add(cached_tokens, record.cached_tokens, "cached tokens")?;
1277                cost_usd_micros = checked_recovery_add(cost_usd_micros, record.cost_usd_micros, "cost")?;
1278                if let Some(id) = record.stop_reason_id {
1279                    let reason = av_events::StopReason::from_id(id);
1280                    if reason != av_events::StopReason::Unknown {
1281                        session.record_stop_reason(reason);
1282                    }
1283                }
1284                self.ensure_active_event_published(&session.id, &event, &record.event)
1285                    .await?;
1286            }
1287            if tool_allowed
1288                .checked_add(tool_blocked)
1289                .is_none_or(|classified| classified > tool_calls)
1290            {
1291                return Err(FinalizeError::Atif(
1292                    "signed journal has inconsistent tool accounting".to_owned(),
1293                ));
1294            }
1295            session
1296                .totals
1297                .tool_calls
1298                .store(tool_calls, std::sync::atomic::Ordering::Release);
1299            session
1300                .totals
1301                .tool_allowed
1302                .store(tool_allowed, std::sync::atomic::Ordering::Release);
1303            session
1304                .totals
1305                .tool_blocked
1306                .store(tool_blocked, std::sync::atomic::Ordering::Release);
1307            session
1308                .totals
1309                .prompt_tokens
1310                .store(prompt_tokens, std::sync::atomic::Ordering::Release);
1311            session
1312                .totals
1313                .completion_tokens
1314                .store(completion_tokens, std::sync::atomic::Ordering::Release);
1315            session
1316                .totals
1317                .cached_tokens
1318                .store(cached_tokens, std::sync::atomic::Ordering::Release);
1319            session
1320                .totals
1321                .cost_usd_micros
1322                .store(cost_usd_micros, std::sync::atomic::Ordering::Release);
1323            let inconsistent_responses = !pending_responses.is_empty();
1324            session.restore_next_seq(next_sequence);
1325            let expected_subject = {
1326                let chain = session.chain.lock();
1327                ReceiptSubject::EventChain {
1328                    chain_head: chain.head_hex(),
1329                    event_count: chain.count(),
1330                }
1331            };
1332            let receipt_path = self.receipt_path(&session.id);
1333            // Round-40 F4: distinguish `ENOENT` (fresh recovery, no
1334            // prior receipt to reload) from every other read
1335            // failure (EACCES, EIO, or the round-19 F10 read cap
1336            // firing on a file that grew past MAX_RECEIPT_BYTES).
1337            // The prior `if let Ok(bytes) = ...` folded all
1338            // failures into "no prior receipt" and re-issued a
1339            // fresh receipt over the recovered chain — silently
1340            // destroying evidence that a legitimate receipt had
1341            // already been issued. Under the oversize case, a
1342            // hostile local process could grow receipt.json past
1343            // the cap to erase the operator's receipt on the next
1344            // recovery tick. Now: a genuine ENOENT is a proper
1345            // recovery no-op; any other error is a per-session
1346            // failure that surfaces to the reconciler tick.
1347            match tokio::fs::metadata(&receipt_path).await {
1348                Ok(_) => {
1349                    let bytes = read_capped_async(
1350                        receipt_path.clone(),
1351                        av_core::fsutil::MAX_RECEIPT_BYTES,
1352                    )
1353                    .await
1354                    .map_err(|error| {
1355                        FinalizeError::Receipt(format!(
1356                            "existing receipt unreadable: {error}"
1357                        ))
1358                    })?;
1359                    // Round-16: strict deserializer (see the twin call
1360                    // in the unsigned recovery path above).
1361                    // Round-17 F3: bounded read.
1362                    let receipt = Receipt::from_json_slice(&bytes)
1363                        .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
1364                    self.verify_configured_receipt(&receipt)?;
1365                    if receipt.body.subject != expected_subject {
1366                        return Err(FinalizeError::Receipt(
1367                            "persisted receipt does not attest the recovered signed journal".to_owned(),
1368                        ));
1369                    }
1370                    *session.receipt.lock() = Some(receipt);
1371                }
1372                Err(error) if error.kind() == std::io::ErrorKind::NotFound
1373                    || error.kind() == std::io::ErrorKind::NotADirectory =>
1374                {
1375                    // Fresh recovery — no prior receipt.
1376                    // `NotADirectory` covers the case where a
1377                    // parent component of the receipt path is a
1378                    // file (equivalent to "no receipt at that
1379                    // path" from the receipt's point of view).
1380                }
1381                Err(error) => {
1382                    return Err(FinalizeError::Receipt(format!(
1383                        "existing receipt stat failed: {error}"
1384                    )));
1385                }
1386            }
1387            let unwrapped = Arc::try_unwrap(session)
1388                .map_err(|_| FinalizeError::Task("signed recovery retained session".to_owned()))?;
1389            // Seal the session to new leases before it is ever reachable via
1390            // `sessions.get(id)`. Between `try_insert_recovered` and the
1391            // finalize path's `try_close`, a client request for the same id
1392            // could otherwise take a lease, submit a worker job, and append to
1393            // the recovered chain — permanently diverging it from the persisted
1394            // receipt's subject.event_count (and leaving a wrong-index journal
1395            // entry). `is_closed()` is true whenever `artifact_committed` is
1396            // set, but `try_close` still transitions `closed` 0→1, so
1397            // `close_session_locked` still runs its full finalize body.
1398            //
1399            // NB: the per-session lifecycle lock was already acquired at
1400            // the top of this iteration (see the `let _lifecycle = ...`
1401            // block near the session_id skip check) and covers this
1402            // whole mutation region — a second acquire here would
1403            // deadlock on itself.
1404            unwrapped.mark_artifact_committed();
1405            let session = match sessions.try_insert_recovered(unwrapped) {
1406                Ok(inserted) => inserted,
1407                Err(_active) => {
1408                    tracing::info!(session = %session_id, "signed recovery skipped: session already active");
1409                    return Ok(SignedCandidateOutcome::Skipped);
1410                }
1411            };
1412            if inconsistent_responses {
1413                // Quarantine only after we know the recovered Session was actually installed —
1414                // otherwise a live session with the same id would inherit the capture-failed verdict.
1415                self.quarantined_sessions.lock().insert(session.id.clone());
1416                session.mark_capture_failed();
1417                return Ok(SignedCandidateOutcome::Recovered);
1418            }
1419            if self.quarantined_sessions.lock().contains(&session.id) {
1420                session.mark_capture_failed();
1421                return Ok(SignedCandidateOutcome::Recovered);
1422            }
1423            // Round-42 F3: `mark_artifact_committed()` above sealed the
1424            // session against new leases before `try_insert_recovered`
1425            // to plug the race the seal-before-insert comment above
1426            // describes. But if
1427            // `close_session_locked` then fails transiently (broker
1428            // outage → Bridge; ENOSPC/EIO → Receipt; verify mismatch
1429            // after key rotation → Receipt), the session sits in the
1430            // registry with `artifact_committed = 1` forever:
1431            // `is_closed()` is true so the idle sweeper skips it, and
1432            // the next reconciler tick's registry-hit check
1433            // returns Skipped so signed recovery never re-attempts.
1434            // Remove the half-committed session on transient error so
1435            // the next tick re-reads the still-present journal sidecar
1436            // and re-adopts cleanly. `CaptureIncomplete` is a legit
1437            // sealed-quarantine outcome (close_session_locked already
1438            // set `claim.committed = true`), so leave those installed.
1439            match self
1440                .close_session_locked(Arc::clone(&session), StopReason::SessionClosed)
1441                .await
1442            {
1443                Ok(_) => Ok(SignedCandidateOutcome::Recovered),
1444                Err(FinalizeError::CaptureIncomplete) => {
1445                    Err(FinalizeError::CaptureIncomplete)
1446                }
1447                Err(error) => {
1448                    sessions.remove(&session.id);
1449                    Err(error)
1450                }
1451            }
1452            }.await;
1453            match outcome {
1454                Ok(SignedCandidateOutcome::Recovered) => recovered += 1,
1455                Ok(SignedCandidateOutcome::Skipped) => {}
1456                Err(error) => {
1457                    self.metrics
1458                        .counter(
1459                            "av_signed_recovery_skipped_total",
1460                            "Signed sessions skipped during recovery due to per-session errors (round-41 F1)",
1461                        )
1462                        .inc();
1463                    if self.warn_once(metadata_path.clone()) {
1464                        tracing::warn!(
1465                            %error,
1466                            path = %av_core::fsutil::basename(&metadata_path),
1467                            "skipping signed session recovery due to per-session error; other sessions continue"
1468                        );
1469                    }
1470                }
1471            }
1472        }
1473        Ok(recovered)
1474    }
1475
1476    async fn ensure_active_event_published(
1477        &self,
1478        session_id: &str,
1479        event: &av_events::OcsfEvent,
1480        value: &serde_json::Value,
1481    ) -> Result<(), FinalizeError> {
1482        let topic = event.class_name.topic();
1483        let event_uid = &event.metadata.uid;
1484        if let Some(ack) =
1485            crate::worker::read_broker_ack(&self.spool_dir, session_id, event_uid, &self.journal_key)
1486                .await
1487                .map_err(FinalizeError::Bridge)?
1488        {
1489            if ack.topic != topic {
1490                return Err(FinalizeError::Bridge(
1491                    "broker acknowledgment topic does not match active event".to_owned(),
1492                ));
1493            }
1494            return Ok(());
1495        }
1496        let bridge = self.bridge.as_ref().map(Arc::clone).ok_or_else(|| {
1497            FinalizeError::Bridge("unacknowledged active event has no configured broker".to_owned())
1498        })?;
1499        let topic = topic.to_owned();
1500        let key = event.ai_agent.instance_uid.clone();
1501        let value = value.clone();
1502        let uid = event_uid.clone();
1503        let lookup_bridge = Arc::clone(&bridge);
1504        let lookup_topic = topic.clone();
1505        let lookup_key = key.clone();
1506        let lookup_uid = uid.clone();
1507        if let Some(ack) = tokio::task::spawn_blocking(move || {
1508            lookup_bridge.find_event_by_uid(&lookup_topic, &lookup_key, &lookup_uid)
1509        })
1510        .await
1511        .map_err(|error| FinalizeError::Task(error.to_string()))?
1512        .map_err(|error| FinalizeError::Bridge(error.to_string()))?
1513        {
1514            crate::worker::persist_broker_ack(
1515                &self.spool_dir,
1516                session_id,
1517                event_uid,
1518                &ack,
1519                &self.journal_key,
1520            )
1521            .await
1522            .map_err(FinalizeError::Bridge)?;
1523            return Ok(());
1524        }
1525        let ack = tokio::task::spawn_blocking(move || bridge.publish_idempotent(&topic, &key, &value, &uid))
1526            .await
1527            .map_err(|error| FinalizeError::Task(error.to_string()))?
1528            .map_err(|error| FinalizeError::Bridge(error.to_string()))?;
1529        crate::worker::persist_broker_ack(&self.spool_dir, session_id, event_uid, &ack, &self.journal_key)
1530            .await
1531            .map_err(FinalizeError::Bridge)
1532    }
1533
1534    async fn consolidate_step_journals(
1535        &self,
1536        sessions: &SessionRegistry,
1537        breaker: &av_loopdetect::BreakerConfig,
1538    ) -> Result<(), FinalizeError> {
1539        let mut entries = match tokio::fs::read_dir(&self.spool_dir).await {
1540            Ok(entries) => entries,
1541            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
1542            Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1543        };
1544        while let Some(entry) = entries
1545            .next_entry()
1546            .await
1547            .map_err(|error| FinalizeError::Atif(error.to_string()))?
1548        {
1549            let metadata_path = entry.path();
1550            let Some(name) = metadata_path.file_name().and_then(std::ffi::OsStr::to_str) else {
1551                continue;
1552            };
1553            let Some(stem) = name.strip_suffix(".session.json") else {
1554                continue;
1555            };
1556            // Round-41 F1 (twin of the recover_signed_journals fix):
1557            // wrap the per-candidate body so per-session errors
1558            // warn+continue instead of aborting the whole
1559            // consolidation scan. A single poisoned sidecar or a
1560            // torn events journal used to head-of-line-block
1561            // every OTHER unsigned session on the tick.
1562            let outcome: Result<(), FinalizeError> = async {
1563            let final_path = self.spool_dir.join(format!("{stem}.json"));
1564            let journal_path = self.spool_dir.join(format!("{stem}.events.ndjson"));
1565            let metadata = self.read_journal_metadata(&metadata_path).await?;
1566            if metadata
1567                .get("journal_version")
1568                .and_then(serde_json::Value::as_u64)
1569                != Some(2)
1570            {
1571                // Round-15 F1: same HOL-block fix as the signed
1572                // branch above — one drifted sidecar must not deny
1573                // recovery to unrelated sessions.
1574                // Round-16 F4: dedup via `warned_artifacts`.
1575                if self.warn_once(metadata_path.clone()) {
1576                    tracing::warn!(
1577                        path = %av_core::fsutil::basename(&metadata_path),
1578                        version = ?metadata.get("journal_version"),
1579                        "sidecar has unsupported journal_version; skipping this session so recovery can proceed for the rest"
1580                    );
1581                }
1582                return Ok(());
1583            }
1584            if metadata.get("workflow").and_then(serde_json::Value::as_str) == Some(Workflow::Signed.as_str())
1585            {
1586                return Ok(());
1587            }
1588            let session_id = metadata
1589                .get("session_id")
1590                .and_then(serde_json::Value::as_str)
1591                .ok_or_else(|| FinalizeError::Atif("journal metadata has no session_id".into()))?;
1592            if sessions.get(session_id).is_some() {
1593                return Ok(());
1594            }
1595            // Per-session lifecycle lock for the unsigned-consolidation
1596            // path, scoped to this candidate only. Released between
1597            // candidates so the recovery scan cannot head-of-line-block
1598            // a client-driven close on an unrelated session.
1599            let _lifecycle = self.acquire_lifecycle(session_id).await;
1600            let identity: av_events::AgentIdentity = serde_json::from_value(
1601                metadata
1602                    .get("identity")
1603                    .cloned()
1604                    .ok_or_else(|| FinalizeError::Atif("journal metadata has no identity".into()))?,
1605            )
1606            .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1607            let journal = if journal_path.exists() {
1608                read_complete_journal(&journal_path).await?
1609            } else {
1610                Vec::new()
1611            };
1612            if self.quarantined_sessions.lock().contains(session_id) {
1613                let quarantined = Session::new(
1614                    session_id.to_owned(),
1615                    Workflow::Unsigned,
1616                    identity.clone(),
1617                    breaker.clone(),
1618                );
1619                quarantined.restore_journal_index(
1620                    u64::try_from(journal.len())
1621                        .map_err(|_| FinalizeError::Atif("active journal length overflow".to_owned()))?,
1622                );
1623                quarantined.mark_capture_failed();
1624                // Also seal the session finalized (like the signed-journal
1625                // capture-failed path) so the idle sweeper's `!is_closed()` filter
1626                // skips it. Otherwise every idle tick re-enters
1627                // close_session_locked, hits the capture_failed guard, and
1628                // CloseClaim resets `closed` — an unbounded churn loop.
1629                quarantined.mark_artifact_committed();
1630                sessions.insert_recovered(quarantined);
1631                return Ok(());
1632            }
1633            if journal.is_empty() {
1634                // Round-14 F5: same quarantine-preservation guard as
1635                // the signed branch — don't delete the sealed
1636                // metadata when a torn journal has been quarantined
1637                // in a prior tick.
1638                if quarantine_sibling_exists(&self.spool_dir, stem).await? {
1639                    let quarantine_metadata = self
1640                        .spool_dir
1641                        .join(format!("{stem}.session.json.corrupt-{}", av_core::new_event_uid()));
1642                    match tokio::fs::rename(&metadata_path, &quarantine_metadata).await {
1643                        Ok(()) => tracing::warn!(
1644                            metadata = %av_core::fsutil::basename(&metadata_path),
1645                            quarantine = %av_core::fsutil::basename(&quarantine_metadata),
1646                            "sealed metadata sidecar quarantined alongside its torn unsigned journal (round-14 F5)"
1647                        ),
1648                        Err(error) => tracing::error!(
1649                            metadata = %av_core::fsutil::basename(&metadata_path),
1650                            %error,
1651                            "failed to quarantine metadata sidecar; leaving in place so a future recovery can try again"
1652                        ),
1653                    }
1654                    return Ok(());
1655                }
1656                tokio::fs::remove_file(metadata_path.clone())
1657                    .await
1658                    .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1659                return Ok(());
1660            }
1661            let journal_len = u64::try_from(journal.len())
1662                .map_err(|_| FinalizeError::Atif("active journal length overflow".to_owned()))?;
1663            let agent = av_atif::Agent {
1664                name: "agentvisor-ai-harness".into(),
1665                version: identity.version.clone(),
1666                model_name: None,
1667                tool_definitions: None,
1668                extra: Some(serde_json::json!({
1669                    "charter": identity.charter,
1670                    "instance_uid": identity.instance_uid,
1671                })),
1672            };
1673            let mut builder = av_atif::TrajectoryBuilder::new(agent, Some(session_id.to_owned()));
1674            let domain = format!("{session_id}:active");
1675            let mut latest_identity = identity.clone();
1676            let mut prompt_tokens = 0u64;
1677            let mut completion_tokens = 0u64;
1678            let mut cached_tokens = 0u64;
1679            let mut cost_usd_micros = 0u64;
1680            let mut tool_calls = 0u64;
1681            let mut tool_allowed = 0u64;
1682            let mut tool_blocked = 0u64;
1683            let mut stop_reason_id = None;
1684            let mut pending_responses = std::collections::HashSet::new();
1685            for (index, line) in journal.into_iter().enumerate() {
1686                let index = u64::try_from(index)
1687                    .map_err(|_| FinalizeError::Atif("active journal index overflow".to_owned()))?;
1688                let record: crate::worker::ActiveJournalRecord =
1689                    crate::journal::open(&self.journal_key, &domain, index, line.as_bytes())
1690                        .map_err(FinalizeError::Atif)?;
1691                let event: av_events::OcsfEvent = serde_json::from_value(record.event.clone())
1692                    .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1693                if event.session_uid != session_id || event.ai_agent != record.identity {
1694                    return Err(FinalizeError::Atif(
1695                        "unsigned active record has inconsistent session or identity".to_owned(),
1696                    ));
1697                }
1698                track_response_attempt(&mut pending_responses, record.response_attempt.as_ref())?;
1699                if event.metadata.sequence != index {
1700                    return Err(FinalizeError::Atif(
1701                        "unsigned event sequence does not match active journal index".to_owned(),
1702                    ));
1703                }
1704                if record.identity.version != identity.version
1705                    || record.identity.charter != identity.charter
1706                    || record.identity.instance_uid != identity.instance_uid
1707                {
1708                    return Err(FinalizeError::Atif(
1709                        "active journal changed the unsigned session identity".to_owned(),
1710                    ));
1711                }
1712                self.ensure_active_event_published(session_id, &event, &record.event)
1713                    .await?;
1714                let step = record.atif_step.ok_or_else(|| {
1715                    FinalizeError::Atif("unsigned active record has no ATIF step".to_owned())
1716                })?;
1717                builder
1718                    .push_step(step)
1719                    .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1720                latest_identity = record.identity;
1721                prompt_tokens = checked_recovery_add(prompt_tokens, record.prompt_tokens, "prompt tokens")?;
1722                completion_tokens =
1723                    checked_recovery_add(completion_tokens, record.completion_tokens, "completion tokens")?;
1724                cached_tokens = checked_recovery_add(cached_tokens, record.cached_tokens, "cached tokens")?;
1725                cost_usd_micros = checked_recovery_add(cost_usd_micros, record.cost_usd_micros, "cost")?;
1726                tool_calls = checked_recovery_add(tool_calls, record.tool_calls, "tool calls")?;
1727                tool_allowed = checked_recovery_add(tool_allowed, record.tool_allowed, "allowed tools")?;
1728                tool_blocked = checked_recovery_add(tool_blocked, record.tool_blocked, "blocked tools")?;
1729                if record.stop_reason_id.is_some() {
1730                    stop_reason_id = record.stop_reason_id;
1731                }
1732            }
1733            if tool_allowed
1734                .checked_add(tool_blocked)
1735                .is_none_or(|classified| classified > tool_calls)
1736            {
1737                return Err(FinalizeError::Atif(
1738                    "unsigned journal has inconsistent tool accounting".to_owned(),
1739                ));
1740            }
1741            if !pending_responses.is_empty() {
1742                let quarantined = Session::new(
1743                    session_id.to_owned(),
1744                    Workflow::Unsigned,
1745                    latest_identity,
1746                    breaker.clone(),
1747                );
1748                quarantined.restore_journal_index(journal_len);
1749                quarantined.restore_next_seq(journal_len);
1750                quarantined
1751                    .totals
1752                    .tool_calls
1753                    .store(tool_calls, std::sync::atomic::Ordering::Release);
1754                quarantined
1755                    .totals
1756                    .tool_allowed
1757                    .store(tool_allowed, std::sync::atomic::Ordering::Release);
1758                quarantined
1759                    .totals
1760                    .tool_blocked
1761                    .store(tool_blocked, std::sync::atomic::Ordering::Release);
1762                quarantined
1763                    .totals
1764                    .prompt_tokens
1765                    .store(prompt_tokens, std::sync::atomic::Ordering::Release);
1766                quarantined
1767                    .totals
1768                    .completion_tokens
1769                    .store(completion_tokens, std::sync::atomic::Ordering::Release);
1770                quarantined
1771                    .totals
1772                    .cached_tokens
1773                    .store(cached_tokens, std::sync::atomic::Ordering::Release);
1774                quarantined
1775                    .totals
1776                    .cost_usd_micros
1777                    .store(cost_usd_micros, std::sync::atomic::Ordering::Release);
1778                quarantined.mark_capture_failed();
1779                // Also seal the session finalized so the idle sweeper's
1780                // `!is_closed()` filter skips it — same reasoning as the
1781                // quarantined-already branch above.
1782                quarantined.mark_artifact_committed();
1783                // Quarantine only after we know a fresh session was actually installed —
1784                // a live session with the same id must not inherit this capture-failed verdict.
1785                match sessions.try_insert_recovered(quarantined) {
1786                    Ok(_) => {
1787                        self.quarantined_sessions.lock().insert(session_id.to_owned());
1788                    }
1789                    Err(_active) => {
1790                        tracing::info!(
1791                            session = %session_id,
1792                            "unsigned quarantine skipped: session already active",
1793                        );
1794                    }
1795                }
1796                return Ok(());
1797            }
1798            let mut trajectory = builder.finish();
1799            trajectory.agent.extra = Some(serde_json::json!({
1800                "charter": latest_identity.charter,
1801                "instance_uid": latest_identity.instance_uid,
1802                "ttl_remaining_s": latest_identity.ttl_remaining_s,
1803            }));
1804            if let Some(metrics) = trajectory.final_metrics.as_mut() {
1805                metrics.total_prompt_tokens = Some(prompt_tokens);
1806                metrics.total_completion_tokens = Some(completion_tokens);
1807                metrics.total_cached_tokens = Some(cached_tokens);
1808                metrics.total_cost_usd =
1809                    Some(cost_usd_micros as f64 / av_core::units::USD_MICROS_PER_DOLLAR as f64);
1810                metrics.extra = Some(serde_json::json!({
1811                    "tool_calls": tool_calls,
1812                    "tool_allowed": tool_allowed,
1813                    "tool_blocked": tool_blocked,
1814                    "cost_usd_micros": cost_usd_micros,
1815                    // Round-43 F2: match the close-time serialization at
1816                    // reconciler.rs:488 which writes `u64` (0 when never
1817                    // recorded) via `session.recorded_stop_reason_id()`.
1818                    // Emitting `Option<u8>` here made JSON `null` vs
1819                    // JSON `0`, so the `trajectory != existing`
1820                    // comparison below
1821                    // fired on every session that closed without a
1822                    // terminal stop-reason event (client hangup mid-
1823                    // stream, /close before final assistant message,
1824                    // tool-only session). That mismatch marked the
1825                    // session `av_unsigned_recovery_skipped_total`,
1826                    // left the step-journal on disk uncleaned, and
1827                    // repeated every restart.
1828                    "stop_reason_id": stop_reason_id.map_or(0u64, u64::from),
1829                }));
1830            }
1831            if final_path.exists() {
1832                let existing: av_atif::Trajectory = serde_json::from_slice(
1833                    &read_capped_async(final_path.clone(), av_core::fsutil::MAX_ATIF_BYTES)
1834                        .await
1835                        .map_err(|error| FinalizeError::Atif(error.to_string()))?,
1836                )
1837                .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1838                trajectory.trajectory_id.clone_from(&existing.trajectory_id);
1839                if trajectory != existing {
1840                    return Err(FinalizeError::Atif(
1841                        "persisted ATIF does not match authenticated active journal".to_owned(),
1842                    ));
1843                }
1844            } else {
1845                let write_path = final_path.clone();
1846                tokio::task::spawn_blocking(move || av_atif::write_atomic(&trajectory, &write_path))
1847                    .await
1848                    .map_err(|error| FinalizeError::Task(error.to_string()))?
1849                    .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1850            }
1851            self.ensure_atif_provenance(&final_path, session_id).await?;
1852            self.remove_step_journal(session_id).await?;
1853            Ok(())
1854            }.await;
1855            if let Err(error) = outcome {
1856                self.metrics
1857                    .counter(
1858                        "av_unsigned_recovery_skipped_total",
1859                        // Keep in sync with the pre-registration in pipeline.rs —
1860                        // render() emits one HELP per family.
1861                        "Unsigned step-journal consolidations skipped during recovery due to per-session errors (round-41 F1)",
1862                    )
1863                    .inc();
1864                if self.warn_once(metadata_path.clone()) {
1865                    tracing::warn!(
1866                        %error,
1867                        path = %av_core::fsutil::basename(&metadata_path),
1868                        "skipping unsigned session consolidation due to per-session error; other sessions continue"
1869                    );
1870                }
1871            }
1872        }
1873        Ok(())
1874    }
1875
1876    async fn remove_step_journal(&self, session_id: &str) -> Result<(), FinalizeError> {
1877        let digest = av_core::digest::sha256_hex(session_id.as_bytes());
1878        let stem = digest.get(..32).unwrap_or(&digest).to_owned();
1879        let spool_dir = self.spool_dir.clone();
1880        tokio::task::spawn_blocking(move || -> Result<(), FinalizeError> {
1881            let mut spool_changed = false;
1882            for suffix in ["session.json", "steps.ndjson", "events.ndjson"] {
1883                let path = spool_dir.join(format!("{stem}.{suffix}"));
1884                match std::fs::remove_file(&path) {
1885                    Ok(()) => spool_changed = true,
1886                    Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1887                    Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1888                }
1889            }
1890            if spool_changed {
1891                av_core::fsutil::sync_directory(&spool_dir)
1892                    .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1893            }
1894            let ack_parent = spool_dir.join("broker-acks");
1895            let ack_path = ack_parent.join(&stem);
1896            match std::fs::remove_dir_all(&ack_path) {
1897                Ok(()) => av_core::fsutil::sync_directory(&ack_parent)
1898                    .map_err(|error| FinalizeError::Atif(error.to_string()))?,
1899                Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1900                Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1901            }
1902            Ok(())
1903        })
1904        .await
1905        .map_err(|error| FinalizeError::Task(error.to_string()))?
1906    }
1907
1908    /// Retry every durable promotion marker whose session can be recovered.
1909    pub async fn retry_marked_promotions(&self, sessions: &SessionRegistry) -> Result<usize, FinalizeError> {
1910        let mut entries = match tokio::fs::read_dir(&self.spool_dir).await {
1911            Ok(entries) => entries,
1912            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
1913            Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1914        };
1915        let mut promoted = 0usize;
1916        while let Some(entry) = entries
1917            .next_entry()
1918            .await
1919            .map_err(|error| FinalizeError::Atif(error.to_string()))?
1920        {
1921            let path = entry.path();
1922            if path.extension().and_then(std::ffi::OsStr::to_str) != Some("promote") {
1923                continue;
1924            }
1925            // Round-27 F2: previously any read failure or MAC-verify
1926            // failure aborted the entire retry pass via `?`. One
1927            // unreadable or corrupt `.promote` marker would prevent
1928            // retry of every other pending promotion after a crash.
1929            // Mirror `replay_lifecycle_outboxes`'s warn+continue on
1930            // the same two failure modes; leave the bad marker on
1931            // disk as forensic evidence.
1932            let sealed = match read_capped_async(path.clone(), av_core::fsutil::MAX_CONTROL_BYTES).await {
1933                Ok(bytes) => bytes,
1934                Err(error) => {
1935                    tracing::warn!(
1936                        %error,
1937                        path = %av_core::fsutil::basename(&path),
1938                        "skipping unreadable promotion marker"
1939                    );
1940                    continue;
1941                }
1942            };
1943            let marker: PromotionMarker =
1944                match crate::journal::open(&self.journal_key, "promotion-marker", 0, &sealed) {
1945                    Ok(m) => m,
1946                    Err(error) => {
1947                        tracing::warn!(
1948                            %error,
1949                            path = %av_core::fsutil::basename(&path),
1950                            "skipping unauthenticated promotion marker"
1951                        );
1952                        continue;
1953                    }
1954                };
1955            let Some(session) = sessions.get(&marker.session_id) else {
1956                continue;
1957            };
1958            // Background retry must never force-close a live session that
1959            // happens to share this id — that path belongs to the explicit
1960            // `promote_session` endpoint. See `promote`: any non-closed
1961            // session gets `close_session_locked`-ed on entry.
1962            if !session.is_closed() {
1963                tracing::info!(
1964                    session = %marker.session_id,
1965                    "promotion retry skipped: session is currently active",
1966                );
1967                continue;
1968            }
1969            match self.promote(session).await {
1970                Ok(_) => promoted += 1,
1971                Err(error) => {
1972                    tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "promotion retry failed");
1973                }
1974            }
1975        }
1976        Ok(promoted)
1977    }
1978
1979    fn receipt_path(&self, session_id: &str) -> PathBuf {
1980        self.spool_dir.join("receipts").join(format!(
1981            "{}.json",
1982            &av_core::digest::sha256_hex(session_id.as_bytes())[..32]
1983        ))
1984    }
1985
1986    async fn ensure_atif_provenance(
1987        &self,
1988        path: &std::path::Path,
1989        session_id: &str,
1990    ) -> Result<AtifProvenance, FinalizeError> {
1991        // Round-18: ATIF trajectory read is bounded via MAX_ATIF_BYTES.
1992        // Sibling of round-17 F3 that missed this internal caller.
1993        let bytes = read_capped_async(path.to_path_buf(), av_core::fsutil::MAX_ATIF_BYTES)
1994            .await
1995            .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1996        let expected = AtifProvenance {
1997            session_id: session_id.to_owned(),
1998            digest: av_core::digest::sha256_hex(&bytes),
1999        };
2000        let provenance_path = path.with_extension("atif-auth");
2001        if provenance_path.exists() {
2002            let sealed = read_capped_async(provenance_path.clone(), av_core::fsutil::MAX_CONTROL_BYTES)
2003                .await
2004                .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2005            let actual: AtifProvenance =
2006                crate::journal::open(&self.journal_key, "atif-provenance", 0, &sealed)
2007                    .map_err(FinalizeError::Atif)?;
2008            if actual.session_id != expected.session_id || actual.digest != expected.digest {
2009                return Err(FinalizeError::Atif(
2010                    "ATIF provenance does not match artifact bytes and session".to_owned(),
2011                ));
2012            }
2013            return Ok(actual);
2014        }
2015        let sealed = crate::journal::seal(&self.journal_key, "atif-provenance", 0, &expected)
2016            .map_err(FinalizeError::Atif)?;
2017        persist_marker(&provenance_path, &sealed).await?;
2018        Ok(expected)
2019    }
2020
2021    async fn read_journal_metadata(
2022        &self,
2023        path: &std::path::Path,
2024    ) -> Result<serde_json::Value, FinalizeError> {
2025        // Round-18: bounded via MAX_CONTROL_BYTES — journal metadata
2026        // sidecar is a tiny sealed blob (session_id + identity +
2027        // workflow), so 1 MiB is a generous upper bound.
2028        let bytes = read_capped_async(path.to_path_buf(), av_core::fsutil::MAX_CONTROL_BYTES)
2029            .await
2030            .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2031        crate::journal::open(&self.journal_key, "metadata", 0, &bytes).map_err(FinalizeError::Atif)
2032    }
2033
2034    fn verify_configured_receipt(&self, receipt: &Receipt) -> Result<(), FinalizeError> {
2035        if receipt.body.key_id != self.signer.key_id() {
2036            return Err(FinalizeError::Receipt(format!(
2037                "receipt key {:?} does not match configured key {:?}",
2038                receipt.body.key_id,
2039                self.signer.key_id()
2040            )));
2041        }
2042        let mut keyring = av_receipts::Keyring::new();
2043        keyring
2044            .add_signer(self.signer.as_ref())
2045            .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
2046        receipt
2047            .verify(&keyring)
2048            .map_err(|error| FinalizeError::Receipt(error.to_string()))
2049    }
2050
2051    async fn persist_receipt(&self, session_id: &str, receipt: &Receipt) -> Result<(), FinalizeError> {
2052        let path = self.receipt_path(session_id);
2053        let bytes =
2054            serde_json::to_vec_pretty(receipt).map_err(|error| FinalizeError::Receipt(error.to_string()))?;
2055        tokio::task::spawn_blocking(move || av_core::fsutil::write_atomic(&path, &bytes))
2056            .await
2057            .map_err(|error| FinalizeError::Task(error.to_string()))?
2058            .map_err(|error| FinalizeError::Receipt(error.to_string()))
2059    }
2060
2061    async fn emit_receipt_event(&self, session: &Session, receipt: &Receipt) -> Result<(), FinalizeError> {
2062        self.emit_bridge_event(
2063            session,
2064            av_events::EventClass::Receipt,
2065            serde_json::json!({
2066                "receipt_id": receipt.body.receipt_id,
2067                "key_id": receipt.body.key_id,
2068                "subject": receipt.body.subject,
2069                "receipt": receipt,
2070            }),
2071            crate::journal::RECEIPT_OUTBOX_KIND,
2072        )
2073        .await
2074    }
2075
2076    async fn emit_bridge_event(
2077        &self,
2078        session: &Session,
2079        class: av_events::EventClass,
2080        payload: serde_json::Value,
2081        kind: &str,
2082    ) -> Result<(), FinalizeError> {
2083        let Some(bridge) = self.bridge.as_ref().map(Arc::clone) else {
2084            return Ok(());
2085        };
2086        let path = self.lifecycle_outbox_path(&session.id, kind);
2087        let mut outbox = if path.exists() {
2088            let sealed = read_capped_async(path.clone(), av_core::fsutil::MAX_CONTROL_BYTES)
2089                .await
2090                .map_err(|error| FinalizeError::Bridge(error.to_string()))?;
2091            let outbox: LifecycleOutbox = crate::journal::open(
2092                &self.journal_key,
2093                crate::journal::LIFECYCLE_OUTBOX_DOMAIN,
2094                0,
2095                &sealed,
2096            )
2097            .map_err(FinalizeError::Bridge)?;
2098            if outbox.session_id != session.id || outbox.kind != kind {
2099                return Err(FinalizeError::Bridge(
2100                    "lifecycle outbox does not match its session and kind".to_owned(),
2101                ));
2102            }
2103            // A crash between a prior successful emit and a subsequent one loses the
2104            // in-memory seq advance for this outbox — recovery only restores seq from
2105            // the journal length. Fast-forward past the persisted seq so a following
2106            // lifecycle event (e.g., SESSION_CLOSE after a persisted RECEIPT_OUTBOX)
2107            // cannot land on the same metadata.sequence value.
2108            if let Some(persisted_seq) = outbox
2109                .value
2110                .get("metadata")
2111                .and_then(|metadata| metadata.get("sequence"))
2112                .and_then(serde_json::Value::as_u64)
2113            {
2114                if session.peek_seq() <= persisted_seq {
2115                    session.advance_seq_past(persisted_seq);
2116                }
2117            }
2118            outbox
2119        } else {
2120            // Peek the seq without consuming it; a failed persist_outbox
2121            // below would otherwise burn a seq that recovery expects to see
2122            // at a later journal position, breaking the position-vs-seq
2123            // invariant when reset_close reopens the session.
2124            let event_seq = session.peek_seq();
2125            let event = av_events::OcsfEventBuilder::new(
2126                class,
2127                session.id.clone(),
2128                session.current_identity(),
2129                event_seq,
2130            )
2131            .payload(payload)
2132            .build()
2133            .map_err(|error| FinalizeError::Bridge(error.to_string()))?;
2134            let outbox = LifecycleOutbox {
2135                session_id: session.id.clone(),
2136                kind: kind.to_owned(),
2137                topic: class.topic().to_owned(),
2138                key: session.current_identity().instance_uid,
2139                value: serde_json::to_value(event)
2140                    .map_err(|error| FinalizeError::Bridge(error.to_string()))?,
2141                ack: None,
2142            };
2143            persist_outbox(&path, &outbox, &self.journal_key).await?;
2144            session.advance_seq_past(event_seq);
2145            outbox
2146        };
2147        if outbox.ack.is_some() {
2148            return Ok(());
2149        }
2150        let topic = outbox.topic.clone();
2151        let key = outbox.key.clone();
2152        let value = outbox.value.clone();
2153        let event_uid = lifecycle_event_uid(&value)?;
2154        let ack = match resolve_lifecycle_ack(bridge, topic, key, value, event_uid).await {
2155            Ok(ack) => ack,
2156            Err(error) => {
2157                self.metrics
2158                    .counter(
2159                        "av_lifecycle_event_errors_total",
2160                        "Lifecycle events not published",
2161                    )
2162                    .inc();
2163                return Err(error);
2164            }
2165        };
2166        outbox.ack = Some(ack);
2167        persist_outbox(&path, &outbox, &self.journal_key).await?;
2168        Ok(())
2169    }
2170
2171    async fn replay_lifecycle_outboxes(&self) -> Result<usize, FinalizeError> {
2172        let Some(bridge) = self.bridge.as_ref().map(Arc::clone) else {
2173            return Ok(0);
2174        };
2175        let directory = self.spool_dir.join(crate::spool::OUTBOX);
2176        let mut entries = match tokio::fs::read_dir(&directory).await {
2177            Ok(entries) => entries,
2178            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
2179            Err(error) => return Err(FinalizeError::Bridge(error.to_string())),
2180        };
2181        let mut replayed = 0usize;
2182        while let Some(entry) = entries
2183            .next_entry()
2184            .await
2185            .map_err(|error| FinalizeError::Bridge(error.to_string()))?
2186        {
2187            let path = entry.path();
2188            if path.extension().and_then(std::ffi::OsStr::to_str) != Some("json") {
2189                continue;
2190            }
2191            let sealed = match read_capped_async(path.clone(), av_core::fsutil::MAX_CONTROL_BYTES).await {
2192                Ok(bytes) => bytes,
2193                Err(error) => {
2194                    tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "skipping unreadable outbox file");
2195                    continue;
2196                }
2197            };
2198            let mut outbox: LifecycleOutbox = match crate::journal::open(
2199                &self.journal_key,
2200                crate::journal::LIFECYCLE_OUTBOX_DOMAIN,
2201                0,
2202                &sealed,
2203            ) {
2204                Ok(outbox) => outbox,
2205                // A single corrupt or MAC-failing outbox file must not abort
2206                // the entire scan and stop us from replaying the other
2207                // sessions' outboxes. The bad file stays on disk as
2208                // forensic evidence (`open` does not delete on failure).
2209                Err(error) => {
2210                    tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "skipping malformed outbox");
2211                    continue;
2212                }
2213            };
2214            if path != self.lifecycle_outbox_path(&outbox.session_id, &outbox.kind) {
2215                tracing::warn!(
2216                    path = %av_core::fsutil::basename(&path),
2217                    "skipping outbox whose filename does not match its authenticated session_id/kind"
2218                );
2219                continue;
2220            }
2221            if outbox.ack.is_none() {
2222                let topic = outbox.topic.clone();
2223                let key = outbox.key.clone();
2224                let value = outbox.value.clone();
2225                let event_uid = lifecycle_event_uid(&value)?;
2226                outbox.ack =
2227                    Some(resolve_lifecycle_ack(Arc::clone(&bridge), topic, key, value, event_uid).await?);
2228                persist_outbox(&path, &outbox, &self.journal_key).await?;
2229                replayed = replayed.saturating_add(1);
2230            }
2231        }
2232        Ok(replayed)
2233    }
2234
2235    /// Round-43 F1: complete the finalization tail for sessions
2236    /// where `close_session_locked` marked `artifact_committed = 1`
2237    /// but returned Err before running the tail — typically a
2238    /// transient `emit_bridge_event(SESSION_CLOSE)` publish failure
2239    /// while the broker was unreachable, or a `remove_step_journal`
2240    /// EIO. Round-42 F3 handled the "orphaned in recovery" analogue
2241    /// inside `recover_signed_journals`, but the client
2242    /// `/v1/sessions/{id}/close` route and the idle-sweeper caller
2243    /// (both entering via `Finalizer::close_session`) inherited the
2244    /// same orphan-after-partial-close shape. Rather than removing
2245    /// the session from the registry (which would break the client
2246    /// contract that the id remains queryable), drive the tail to
2247    /// completion here: every step is idempotent
2248    /// (`emit_bridge_event` re-uses an existing outbox,
2249    /// `remove_step_journal` treats ENOENT as success, and
2250    /// `remove_lifecycle_outbox` is `rm -f`). Once
2251    /// `mark_close_complete` fires, `evict_finalized` can reclaim
2252    /// the registry slot.
2253    ///
2254    /// Per-session errors warn+continue via
2255    /// `av_pending_close_completion_failed_total` so one persistently-
2256    /// bad broker for one session does not HOL-block completion of
2257    /// every other pending-close session for the tick.
2258    pub(crate) async fn complete_pending_closes(
2259        &self,
2260        sessions: &SessionRegistry,
2261    ) -> Result<usize, FinalizeError> {
2262        let pending = sessions.pending_close_sessions();
2263        let mut completed = 0usize;
2264        for session in pending {
2265            // Round-44 F3: acquire the per-session lifecycle lock
2266            // before running the finalization tail. Without this the
2267            // sweep could race with a concurrent client `/v1/close`
2268            // (which also enters through `close_session` and holds
2269            // `acquire_lifecycle`), producing:
2270            //   - two parallel `resolve_lifecycle_ack` calls on the
2271            //     same event UID → duplicate SESSION_CLOSE OCSF
2272            //     events on the bridge for one session close,
2273            //   - transient re-creation of a just-deleted outbox
2274            //     file after the client's `remove_lifecycle_outbox`
2275            //     ran (visible to disk snapshots / backups),
2276            //   - split audit trail if a chat request arriving
2277            //     during the sweep triggers `get_or_open` reopen
2278            //     while the client's original `/close` is still
2279            //     `.await`ing on the old Arc.
2280            // The lifecycle lock is the same one every other
2281            // finalize path takes (reconciler.rs:302), so this
2282            // preserves the "close_session_locked is the single
2283            // serialization point for finalization tail work"
2284            // invariant.
2285            let _lifecycle = self.acquire_lifecycle(&session.id).await;
2286            // Re-check state under the lock — a concurrent client
2287            // close may have already driven this session to
2288            // completion between `pending_close_sessions()` and
2289            // this point.
2290            if !session.artifact_committed_flag() || session.close_complete_flag() {
2291                continue;
2292            }
2293            let workflow = session.workflow.as_str();
2294            let outcome: Result<(), FinalizeError> = async {
2295                self.emit_bridge_event(
2296                    &session,
2297                    av_events::EventClass::Session,
2298                    serde_json::json!({"action": "closed", "workflow": workflow}),
2299                    crate::journal::SESSION_CLOSE_OUTBOX_KIND,
2300                )
2301                .await?;
2302                self.remove_step_journal(&session.id).await?;
2303                self.remove_lifecycle_outbox(&session.id, crate::journal::RECEIPT_OUTBOX_KIND)
2304                    .await?;
2305                self.remove_lifecycle_outbox(&session.id, crate::journal::SESSION_CLOSE_OUTBOX_KIND)
2306                    .await?;
2307                session.mark_close_complete();
2308                Ok(())
2309            }
2310            .await;
2311            match outcome {
2312                Ok(()) => completed = completed.saturating_add(1),
2313                Err(error) => {
2314                    self.metrics
2315                        .counter(
2316                            "av_pending_close_completion_failed_total",
2317                            "Pending-close completions that failed to finish their tail (round-43 F1)",
2318                        )
2319                        .inc();
2320                    let key = self.spool_dir.join(format!("pending-close::{}", session.id));
2321                    if self.warn_once(key) {
2322                        tracing::warn!(
2323                            session = %session.id,
2324                            %error,
2325                            "pending-close completion failed; will retry next tick",
2326                        );
2327                    }
2328                }
2329            }
2330        }
2331        Ok(completed)
2332    }
2333
2334    async fn remove_acked_lifecycle_outboxes(&self) -> Result<(), FinalizeError> {
2335        let directory = self.spool_dir.join(crate::spool::OUTBOX);
2336        let mut entries = match tokio::fs::read_dir(&directory).await {
2337            Ok(entries) => entries,
2338            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
2339            Err(error) => return Err(FinalizeError::Bridge(error.to_string())),
2340        };
2341        while let Some(entry) = entries
2342            .next_entry()
2343            .await
2344            .map_err(|error| FinalizeError::Bridge(error.to_string()))?
2345        {
2346            let path = entry.path();
2347            if path.extension().and_then(std::ffi::OsStr::to_str) != Some("json") {
2348                continue;
2349            }
2350            let sealed = read_capped_async(path.clone(), av_core::fsutil::MAX_CONTROL_BYTES)
2351                .await
2352                .map_err(|error| FinalizeError::Bridge(error.to_string()))?;
2353            let outbox: LifecycleOutbox = crate::journal::open(
2354                &self.journal_key,
2355                crate::journal::LIFECYCLE_OUTBOX_DOMAIN,
2356                0,
2357                &sealed,
2358            )
2359            .map_err(FinalizeError::Bridge)?;
2360            if path != self.lifecycle_outbox_path(&outbox.session_id, &outbox.kind) {
2361                return Err(FinalizeError::Bridge(
2362                    "lifecycle outbox path does not match authenticated payload".to_owned(),
2363                ));
2364            }
2365            if outbox.ack.is_some() {
2366                remove_outbox(&path).await?;
2367            }
2368        }
2369        Ok(())
2370    }
2371
2372    fn lifecycle_outbox_path(&self, session_id: &str, kind: &str) -> PathBuf {
2373        let session_hash = &av_core::digest::sha256_hex(session_id.as_bytes())[..32];
2374        self.spool_dir
2375            .join(crate::spool::OUTBOX)
2376            .join(format!("{session_hash}.{kind}.json"))
2377    }
2378
2379    async fn remove_lifecycle_outbox(&self, session_id: &str, kind: &str) -> Result<(), FinalizeError> {
2380        remove_outbox(&self.lifecycle_outbox_path(session_id, kind)).await
2381    }
2382}
2383
2384async fn persist_outbox(
2385    path: &std::path::Path,
2386    outbox: &LifecycleOutbox,
2387    journal_key: &[u8; 32],
2388) -> Result<(), FinalizeError> {
2389    let path = path.to_path_buf();
2390    let bytes = crate::journal::seal(journal_key, crate::journal::LIFECYCLE_OUTBOX_DOMAIN, 0, outbox)
2391        .map_err(FinalizeError::Bridge)?;
2392    tokio::task::spawn_blocking(move || av_core::fsutil::write_atomic(&path, &bytes))
2393        .await
2394        .map_err(|error| FinalizeError::Task(error.to_string()))?
2395        .map_err(|error| FinalizeError::Bridge(error.to_string()))
2396}
2397
2398async fn persist_marker(path: &std::path::Path, bytes: &[u8]) -> Result<(), FinalizeError> {
2399    let path = path.to_path_buf();
2400    let bytes = bytes.to_vec();
2401    tokio::task::spawn_blocking(move || av_core::fsutil::write_atomic(&path, &bytes))
2402        .await
2403        .map_err(|error| FinalizeError::Task(error.to_string()))?
2404        .map_err(|error| FinalizeError::Atif(error.to_string()))
2405}
2406
2407async fn remove_outbox(path: &std::path::Path) -> Result<(), FinalizeError> {
2408    let path = path.to_path_buf();
2409    tokio::task::spawn_blocking(move || -> Result<(), FinalizeError> {
2410        let parent = path
2411            .parent()
2412            .ok_or_else(|| FinalizeError::Bridge("outbox has no parent".to_owned()))?;
2413        match std::fs::remove_file(&path) {
2414            Ok(()) => av_core::fsutil::sync_directory(parent)
2415                .map_err(|error| FinalizeError::Bridge(error.to_string())),
2416            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
2417            Err(error) => Err(FinalizeError::Bridge(error.to_string())),
2418        }
2419    })
2420    .await
2421    .map_err(|error| FinalizeError::Task(error.to_string()))?
2422}
2423
2424/// Start the periodic reconciler tick: spool recovery, promotion retry,
2425/// pending-close completion, idle-session finalization, and
2426/// finalized-session eviction.
2427pub fn spawn_reconciler(
2428    sessions: Arc<SessionRegistry>,
2429    finalizer: Finalizer,
2430    idle_s: u64,
2431    tick_s: u64,
2432    breaker: av_loopdetect::BreakerConfig,
2433    metrics: Arc<Registry>,
2434) -> tokio::task::JoinHandle<()> {
2435    use futures::future::FutureExt as _;
2436    tokio::spawn(async move {
2437        let mut interval = tokio::time::interval(std::time::Duration::from_secs(tick_s.max(1)));
2438        // Skip missed ticks instead of firing them back-to-back. Under
2439        // transient overload (a 5 s tick body that takes 60 s) the
2440        // default `Burst` behaviour would fire 12 immediate consecutive
2441        // ticks, each running the full sweep — turning momentary
2442        // pressure into a stall spiral.
2443        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2444        loop {
2445            interval.tick().await;
2446            let started = Instant::now();
2447            // Wrap the whole tick body in catch_unwind: any panic in
2448            // the reconciler (fs unwrap, JCS overflow, allocator
2449            // failure inside tracing) would otherwise silently kill
2450            // the task; idle sessions would then never finalize until
2451            // the harness restarts. The JWKS refresh loop uses the
2452            // same shape (main.rs).
2453            let outcome = std::panic::AssertUnwindSafe(async {
2454                if let Err(error) = finalizer.recover_spooled_sessions(&sessions, &breaker).await {
2455                    tracing::warn!(%error, "ATIF spool recovery failed");
2456                    metrics
2457                        .counter("av_reconcile_errors_total", "Reconciliation errors")
2458                        .inc();
2459                }
2460                if let Err(error) = finalizer.retry_marked_promotions(&sessions).await {
2461                    tracing::warn!(%error, "durable promotion retry failed");
2462                    metrics
2463                        .counter("av_reconcile_errors_total", "Reconciliation errors")
2464                        .inc();
2465                }
2466                // Round-43 F1: drive the finalization tail forward for
2467                // sessions that got past `mark_artifact_committed` but
2468                // failed the subsequent SESSION_CLOSE emit or journal
2469                // cleanup on a prior tick / client call. Without this
2470                // sweep those sessions accumulate in the registry
2471                // forever and their step journals never get removed.
2472                if let Err(error) = finalizer.complete_pending_closes(&sessions).await {
2473                    tracing::warn!(%error, "pending-close completion sweep failed");
2474                    metrics
2475                        .counter("av_reconcile_errors_total", "Reconciliation errors")
2476                        .inc();
2477                }
2478                for session in sessions.idle_sessions(idle_s) {
2479                    let session_id = session.id.clone();
2480                    if let Err(error) = finalizer.close_session(session, StopReason::SessionClosed).await {
2481                        tracing::warn!(session = %session_id, %error, "idle session finalization failed");
2482                        metrics
2483                            .counter("av_reconcile_errors_total", "Reconciliation errors")
2484                            .inc();
2485                    }
2486                }
2487                let evicted = sessions.evict_finalized(idle_s);
2488                if !evicted.is_empty() {
2489                    tracing::debug!(count = evicted.len(), "evicted finalized signed sessions");
2490                }
2491            })
2492            .catch_unwind()
2493            .await;
2494            if let Err(panic) = outcome {
2495                let msg = panic
2496                    .downcast_ref::<&'static str>()
2497                    .copied()
2498                    .or_else(|| panic.downcast_ref::<String>().map(String::as_str))
2499                    .unwrap_or("panic payload was not a string");
2500                metrics
2501                    .counter(
2502                        "av_reconciler_panics_total",
2503                        "Reconciler tick body panicked; loop supervised via catch_unwind",
2504                    )
2505                    .inc();
2506                tracing::error!(
2507                    panic = %msg,
2508                    "reconciler tick body panicked; continuing on the next tick"
2509                );
2510            }
2511            metrics
2512                .histogram("av_reconcile_duration_seconds", "Idle reconciliation duration")
2513                .observe_us(elapsed_us(started));
2514        }
2515    })
2516}
2517
2518fn checked_recovery_add(current: u64, value: u64, field: &str) -> Result<u64, FinalizeError> {
2519    current
2520        .checked_add(value)
2521        .filter(|total| *total <= av_core::error::JCS_SAFE_MAX)
2522        .ok_or_else(|| FinalizeError::Atif(format!("recovered {field} overflow")))
2523}
2524
2525fn track_response_attempt(
2526    pending: &mut std::collections::HashSet<String>,
2527    attempt: Option<&crate::worker::ResponseAttempt>,
2528) -> Result<(), FinalizeError> {
2529    let Some(attempt) = attempt else {
2530        return Ok(());
2531    };
2532    if attempt.terminal {
2533        if !pending.remove(&attempt.id) {
2534            pending.insert(format!("orphan-terminal:{}", attempt.id));
2535        }
2536    } else if !pending.insert(attempt.id.clone()) {
2537        return Err(FinalizeError::Atif(
2538            "active journal repeats a response attempt id".to_owned(),
2539        ));
2540    }
2541    Ok(())
2542}
2543
2544fn lifecycle_event_uid(value: &serde_json::Value) -> Result<String, FinalizeError> {
2545    value
2546        .get("metadata")
2547        .and_then(|metadata| metadata.get("uid"))
2548        .and_then(serde_json::Value::as_str)
2549        .map(str::to_owned)
2550        .ok_or_else(|| FinalizeError::Bridge("lifecycle event has no metadata UID".to_owned()))
2551}
2552
2553async fn resolve_lifecycle_ack(
2554    bridge: Arc<dyn EventBus>,
2555    topic: String,
2556    key: String,
2557    value: serde_json::Value,
2558    event_uid: String,
2559) -> Result<av_bridge::PublishAck, FinalizeError> {
2560    let lookup_bridge = Arc::clone(&bridge);
2561    let lookup_topic = topic.clone();
2562    let lookup_key = key.clone();
2563    let lookup_uid = event_uid.clone();
2564    if let Some(ack) = tokio::task::spawn_blocking(move || {
2565        lookup_bridge.find_event_by_uid(&lookup_topic, &lookup_key, &lookup_uid)
2566    })
2567    .await
2568    .map_err(|error| FinalizeError::Task(error.to_string()))?
2569    .map_err(|error| FinalizeError::Bridge(error.to_string()))?
2570    {
2571        return Ok(ack);
2572    }
2573    tokio::task::spawn_blocking(move || bridge.publish_idempotent(&topic, &key, &value, &event_uid))
2574        .await
2575        .map_err(|error| FinalizeError::Task(error.to_string()))?
2576        .map_err(|error| FinalizeError::Bridge(error.to_string()))
2577}
2578
2579/// Round-14 F5: check whether `read_complete_journal` has previously
2580/// quarantined this stem's events journal to
2581/// `<stem>.events.ndjson.corrupt-*`. Callers use this to decide
2582/// whether to delete the sealed metadata sidecar when the events
2583/// journal appears empty — if there's a sibling `.corrupt-*` file,
2584/// the "empty" is actually "torn and moved out for post-mortem" and
2585/// the metadata must be preserved (or quarantined itself) rather
2586/// than removed.
2587async fn quarantine_sibling_exists(spool_dir: &std::path::Path, stem: &str) -> Result<bool, FinalizeError> {
2588    let prefix = format!("{stem}.events.ndjson.corrupt-");
2589    let spool_dir = spool_dir.to_path_buf();
2590    tokio::task::spawn_blocking(move || -> Result<bool, FinalizeError> {
2591        let entries = match std::fs::read_dir(&spool_dir) {
2592            Ok(entries) => entries,
2593            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
2594            Err(error) => return Err(FinalizeError::Atif(error.to_string())),
2595        };
2596        for entry in entries {
2597            let entry = entry.map_err(|error| FinalizeError::Atif(error.to_string()))?;
2598            let name = entry.file_name();
2599            if let Some(name) = name.to_str() {
2600                if name.starts_with(&prefix) {
2601                    return Ok(true);
2602                }
2603            }
2604        }
2605        Ok(false)
2606    })
2607    .await
2608    .map_err(|error| FinalizeError::Task(error.to_string()))?
2609}
2610
2611/// Round-17 F3: async wrapper around `av_core::fsutil::read_capped`
2612/// for the reconciler's hot-path reads. A fs-tamper attacker
2613/// (co-scheduled workload, backup restore gone wrong, malicious
2614/// sidecar) can otherwise plant a multi-GB receipt/trajectory and
2615/// OOM the harness on every recovery tick. `spawn_blocking` keeps
2616/// the tokio runtime healthy while `File::open` + `metadata` +
2617/// bounded `read_to_end` run on the blocking pool.
2618async fn read_capped_async(path: std::path::PathBuf, max_bytes: u64) -> Result<Vec<u8>, std::io::Error> {
2619    tokio::task::spawn_blocking(move || av_core::fsutil::read_capped(&path, max_bytes))
2620        .await
2621        .map_err(|e| std::io::Error::other(e.to_string()))?
2622}
2623
2624async fn read_complete_journal(path: &std::path::Path) -> Result<Vec<String>, FinalizeError> {
2625    let path = path.to_path_buf();
2626    tokio::task::spawn_blocking(move || -> Result<Vec<String>, FinalizeError> {
2627        // Round-18: bounded via the shared MAX_ATIF_BYTES so a fs-tamper
2628        // attacker cannot plant a multi-GB journal and OOM the
2629        // recovery scan.
2630        let bytes = av_core::fsutil::read_capped(&path, av_core::fsutil::MAX_ATIF_BYTES)
2631            .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2632        if bytes.is_empty() {
2633            return Ok(Vec::new());
2634        }
2635        let complete_len = if bytes.last() == Some(&b'\n') {
2636            bytes.len()
2637        } else {
2638            bytes
2639                .iter()
2640                .rposition(|byte| *byte == b'\n')
2641                .map_or(0, |index| index + 1)
2642        };
2643        // Round-13 F2: if the file contains NO complete line (no
2644        // newline anywhere), truncating to 0 would silently destroy
2645        // every byte of a torn-write journal — the caller then sees
2646        // an empty journal, deletes the sealed metadata sidecar, and
2647        // an operator investigating "session missing" has no
2648        // evidence at all. Instead, quarantine the file to
2649        // `<path>.corrupt-<uid>` so post-mortem inspection can happen,
2650        // then return an error so the reconciler leaves the session
2651        // in place rather than progressing to metadata deletion.
2652        if complete_len == 0 && !bytes.is_empty() {
2653            let mut quarantine = path.clone();
2654            let stem = quarantine
2655                .file_name()
2656                .and_then(std::ffi::OsStr::to_str)
2657                .unwrap_or("journal");
2658            let new_name = format!("{stem}.corrupt-{}", av_core::new_event_uid());
2659            quarantine.set_file_name(new_name);
2660            let rename_error_message = match std::fs::rename(&path, &quarantine) {
2661                Ok(()) => {
2662                    tracing::error!(
2663                        original = %av_core::fsutil::basename(&path),
2664                        quarantine = %av_core::fsutil::basename(&quarantine),
2665                        bytes = bytes.len(),
2666                        "journal has no complete lines; quarantined for post-mortem instead of silent 0-truncate"
2667                    );
2668                    None
2669                }
2670                Err(rename_error) => {
2671                    tracing::error!(
2672                        path = %av_core::fsutil::basename(&path),
2673                        bytes = bytes.len(),
2674                        error = %rename_error,
2675                        "journal has no complete lines and quarantine rename failed; refusing to truncate"
2676                    );
2677                    Some(rename_error.to_string())
2678                }
2679            };
2680            // Round-14 F6: don't claim the file was quarantined if
2681            // the rename itself failed. Otherwise the operator
2682            // chases a phantom `.corrupt-<uid>` path while the real
2683            // failure (ENOSPC / EACCES / cross-fs rename) sits
2684            // buried in the tracing log.
2685            //
2686            // Round-37 F1: return the file basenames only. This
2687            // FinalizeError::Atif ultimately flows to
2688            // `tracing::warn!(%error, "ATIF spool recovery failed")`
2689            // and `"promotion retry failed"` (grep those literals);
2690            // both then export through
2691            // tracing_opentelemetry -> OTLP -> SIEM. Round-36 F1's
2692            // sweep basenamed the outer tracing fields but missed
2693            // this path leak inside a FinalizeError message body,
2694            // where `#[error("...{0}")]` re-emits the full string.
2695            let name = av_core::fsutil::basename(&path);
2696            let qname = av_core::fsutil::basename(&quarantine);
2697            return Err(FinalizeError::Atif(match rename_error_message {
2698                None => format!(
2699                    "journal {name} contained no complete lines ({} bytes); quarantined at {qname}",
2700                    bytes.len()
2701                ),
2702                Some(rename_error) => format!(
2703                    "journal {name} contained no complete lines ({} bytes); quarantine rename to {qname} failed: {rename_error}; bytes remain at {name}",
2704                    bytes.len()
2705                ),
2706            }));
2707        }
2708        if complete_len < bytes.len() {
2709            tracing::warn!(
2710                path = %av_core::fsutil::basename(&path),
2711                stored = bytes.len(),
2712                keeping = complete_len,
2713                dropping = bytes.len() - complete_len,
2714                "trimming partial trailing line from journal recovery"
2715            );
2716            let file = std::fs::OpenOptions::new()
2717                .write(true)
2718                .open(&path)
2719                .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2720            file.set_len(complete_len as u64)
2721                .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2722            file.sync_all()
2723                .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2724        }
2725        let complete = String::from_utf8(bytes.get(..complete_len).unwrap_or_default().to_vec())
2726            .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2727        Ok(complete
2728            .lines()
2729            .filter(|line| !line.trim().is_empty())
2730            .map(str::to_owned)
2731            .collect())
2732    })
2733    .await
2734    .map_err(|error| FinalizeError::Task(error.to_string()))?
2735}
2736
2737#[cfg(test)]
2738mod tests {
2739    #![allow(
2740        clippy::expect_used,
2741        clippy::indexing_slicing,
2742        clippy::panic,
2743        clippy::unwrap_used
2744    )]
2745
2746    use super::*;
2747    use av_bridge::{BusError, PublishAck, StoredEvent};
2748    use av_events::AgentIdentity;
2749    use av_receipts::Ed25519Signer;
2750
2751    struct FailFirstReceiptBus {
2752        fail: std::sync::atomic::AtomicBool,
2753        attempts: parking_lot::Mutex<Vec<(String, serde_json::Value)>>,
2754    }
2755
2756    impl EventBus for FailFirstReceiptBus {
2757        fn publish(
2758            &self,
2759            topic: &str,
2760            _key: &str,
2761            value: &serde_json::Value,
2762        ) -> Result<PublishAck, BusError> {
2763            self.attempts.lock().push((topic.to_owned(), value.clone()));
2764            if topic == "agent.receipt" && self.fail.swap(false, std::sync::atomic::Ordering::AcqRel) {
2765                return Err(BusError::Backend("injected receipt outage".to_owned()));
2766            }
2767            Ok(PublishAck {
2768                topic: topic.to_owned(),
2769                partition: 0,
2770                offset: self.attempts.lock().len() as u64,
2771            })
2772        }
2773
2774        fn fetch(
2775            &self,
2776            _topic: &str,
2777            _partition: u32,
2778            _offset: u64,
2779            _max: usize,
2780        ) -> Result<Vec<StoredEvent>, BusError> {
2781            Ok(Vec::new())
2782        }
2783
2784        fn partitions(&self, _topic: &str) -> Result<u32, BusError> {
2785            Ok(1)
2786        }
2787
2788        fn topics(&self) -> Vec<String> {
2789            av_events::EventClass::all()
2790                .iter()
2791                .map(|class| class.topic().to_owned())
2792                .collect()
2793        }
2794    }
2795
2796    fn session(workflow: Workflow) -> Arc<Session> {
2797        Arc::new(Session::new(
2798            "lifecycle-session".to_owned(),
2799            workflow,
2800            AgentIdentity {
2801                version: "1".to_owned(),
2802                charter: "test".into(),
2803                instance_uid: "instance-1".to_owned(),
2804                ttl_remaining_s: Some(600),
2805            },
2806            Default::default(),
2807        ))
2808    }
2809
2810    fn finalizer(directory: &std::path::Path) -> Finalizer {
2811        Finalizer::new(
2812            Arc::new(Ed25519Signer::from_seed(&[7; 32])),
2813            directory.to_path_buf(),
2814            Arc::new(Registry::new()),
2815        )
2816    }
2817
2818    /// A lifecycle emit that fails at `persist_outbox` must not have
2819    /// consumed a sequence number — otherwise `reset_close` reopens the
2820    /// session with a burned seq, and the next worker envelope's
2821    /// `next_seq` return would put a mismatched
2822    /// `event.metadata.sequence` at the journal's next byte position,
2823    /// tripping recovery's `sequence != index` check.
2824    #[tokio::test]
2825    async fn emit_bridge_event_persist_failure_does_not_burn_a_seq() {
2826        let directory = tempfile::tempdir().unwrap();
2827        // Sabotage the outbox path: a regular file at `<spool>/outbox` makes
2828        // `create_dir_all` inside `write_atomic` fail for every subsequent
2829        // outbox write.
2830        std::fs::write(directory.path().join(crate::spool::OUTBOX), b"").unwrap();
2831        let bus = Arc::new(FailFirstReceiptBus {
2832            fail: std::sync::atomic::AtomicBool::new(false),
2833            attempts: parking_lot::Mutex::new(Vec::new()),
2834        });
2835        let finalizer = Finalizer::with_bridge(
2836            Arc::new(Ed25519Signer::from_seed(&[7; 32])),
2837            directory.path().to_path_buf(),
2838            Arc::new(Registry::new()),
2839            bus,
2840        );
2841        let session = session(Workflow::Signed);
2842        let seq_before = session.peek_seq();
2843        // Signed close writes the receipt to disk, then tries emit_receipt_event → emit_bridge_event.
2844        // The latter must fail at persist_outbox because <spool>/outbox is a file.
2845        let result = finalizer
2846            .close_session(Arc::clone(&session), StopReason::SessionClosed)
2847            .await;
2848        assert!(
2849            result.is_err(),
2850            "close_session must fail when outbox persist is blocked, got {result:?}",
2851        );
2852        assert_eq!(
2853            session.peek_seq(),
2854            seq_before,
2855            "peek_seq must not advance past a failed persist_outbox — the burned \
2856             seq would misalign the journal on retry",
2857        );
2858    }
2859
2860    /// After a crash between a persisted RECEIPT_OUTBOX (seq = N) and its
2861    /// corresponding SESSION_CLOSE_OUTBOX emit, recovery restores
2862    /// `session.seq` from the journal length, which lags the seq the receipt
2863    /// outbox baked in. `emit_bridge_event` reading the pre-existing outbox
2864    /// must fast-forward the counter so the next lifecycle event does not
2865    /// reuse the seq — bridge consumers rely on unique
2866    /// `metadata.sequence` within a session.
2867    #[tokio::test]
2868    async fn emit_bridge_event_reading_persisted_outbox_advances_seq_past_it() {
2869        let directory = tempfile::tempdir().unwrap();
2870        let bus = Arc::new(FailFirstReceiptBus {
2871            fail: std::sync::atomic::AtomicBool::new(false),
2872            attempts: parking_lot::Mutex::new(Vec::new()),
2873        });
2874        let finalizer = Finalizer::with_bridge(
2875            Arc::new(Ed25519Signer::from_seed(&[7; 32])),
2876            directory.path().to_path_buf(),
2877            Arc::new(Registry::new()),
2878            bus.clone(),
2879        );
2880        let session = session(Workflow::Signed);
2881        // Simulate a crash mid-close: manually persist a RECEIPT_OUTBOX carrying a specific
2882        // seq and an already-set ack so emit_receipt_event will skip the publish path.
2883        let receipt_seq = 42u64;
2884        let receipt_event_uid = av_core::new_event_uid();
2885        let value = serde_json::json!({
2886            "metadata": { "sequence": receipt_seq, "uid": receipt_event_uid },
2887            "topic": av_events::EventClass::Receipt.topic(),
2888        });
2889        let outbox = LifecycleOutbox {
2890            session_id: session.id.clone(),
2891            kind: crate::journal::RECEIPT_OUTBOX_KIND.to_owned(),
2892            topic: av_events::EventClass::Receipt.topic().to_owned(),
2893            key: session.identity.instance_uid.clone(),
2894            value,
2895            ack: Some(av_bridge::PublishAck {
2896                topic: av_events::EventClass::Receipt.topic().to_owned(),
2897                partition: 0,
2898                offset: 1,
2899            }),
2900        };
2901        let outbox_path = finalizer.lifecycle_outbox_path(&session.id, crate::journal::RECEIPT_OUTBOX_KIND);
2902        std::fs::create_dir_all(outbox_path.parent().unwrap()).unwrap();
2903        let sealed = crate::journal::seal(
2904            &finalizer.journal_key,
2905            crate::journal::LIFECYCLE_OUTBOX_DOMAIN,
2906            0,
2907            &outbox,
2908        )
2909        .unwrap();
2910        std::fs::write(&outbox_path, sealed).unwrap();
2911        assert!(
2912            session.peek_seq() < receipt_seq,
2913            "precondition: in-memory seq must trail the persisted outbox seq",
2914        );
2915        finalizer
2916            .close_session(Arc::clone(&session), StopReason::SessionClosed)
2917            .await
2918            .unwrap();
2919        let attempts = bus.attempts.lock();
2920        let session_close = attempts
2921            .iter()
2922            .find(|(topic, _)| topic == av_events::EventClass::Session.topic())
2923            .expect("SESSION_CLOSE lifecycle event must reach the bridge");
2924        let session_close_seq = session_close
2925            .1
2926            .get("metadata")
2927            .and_then(|metadata| metadata.get("sequence"))
2928            .and_then(serde_json::Value::as_u64)
2929            .expect("published event must carry a numeric metadata.sequence");
2930        assert!(
2931            session_close_seq > receipt_seq,
2932            "SESSION_CLOSE seq ({session_close_seq}) must exceed the persisted RECEIPT_OUTBOX seq ({receipt_seq}) — otherwise consumers see duplicate metadata.sequence values within one session",
2933        );
2934    }
2935
2936    /// A signed session inserted into the registry by `recover_signed_journals`
2937    /// must reject new leases from the moment it is visible — otherwise a client
2938    /// request landing between `try_insert_recovered` and the finalize path can
2939    /// submit a worker job that appends to the recovered chain, permanently
2940    /// diverging it from the persisted receipt's `subject.event_count` and
2941    /// leaving a wrong-index entry in the on-disk journal. `CloseClaim::drop`
2942    /// resets `closed` on any finalize error, so pinning the "no leases" state
2943    /// only via `try_close` inside `close_session_locked` is not enough — the
2944    /// session must be sealed before it is ever visible.
2945    #[tokio::test]
2946    async fn recovered_signed_session_rejects_leases_even_when_finalize_errors() {
2947        use av_events::{EventClass, OcsfEventBuilder, StatusId};
2948        use std::io::Write as _;
2949
2950        let directory = tempfile::tempdir().unwrap();
2951        let signer: Arc<dyn Signer> = Arc::new(Ed25519Signer::from_seed(&[29; 32]));
2952        let journal_key = crate::journal::key_from_signer(signer.as_ref());
2953        let session_id = "signed-recovery-lease-guard";
2954        let identity = AgentIdentity {
2955            version: "1".into(),
2956            charter: "test".into(),
2957            instance_uid: "instance-lease-guard".into(),
2958            ttl_remaining_s: Some(600),
2959        };
2960
2961        // Seed the signed session metadata file.
2962        let digest = av_core::digest::sha256_hex(session_id.as_bytes());
2963        let stem = &digest[..32];
2964        let metadata_payload = serde_json::json!({
2965            "journal_version": 2,
2966            "session_id": session_id,
2967            "identity": identity,
2968            "workflow": "signed",
2969        });
2970        let metadata_sealed = crate::journal::seal(&journal_key, "metadata", 0, &metadata_payload).unwrap();
2971        std::fs::write(
2972            directory.path().join(format!("{stem}.session.json")),
2973            &metadata_sealed,
2974        )
2975        .unwrap();
2976
2977        // Seed one signed event in the active journal.
2978        let event = OcsfEventBuilder::new(
2979            EventClass::Compression,
2980            session_id.to_owned(),
2981            identity.clone(),
2982            0,
2983        )
2984        .status(StatusId::Success)
2985        .payload(serde_json::json!({}))
2986        .build()
2987        .unwrap();
2988        let event_uid = event.metadata.uid.clone();
2989        let record = crate::worker::ActiveJournalRecord {
2990            event: serde_json::to_value(&event).unwrap(),
2991            identity: identity.clone(),
2992            atif_step: None,
2993            tool_calls: 0,
2994            tool_allowed: 0,
2995            tool_blocked: 0,
2996            prompt_tokens: 0,
2997            completion_tokens: 0,
2998            cached_tokens: 0,
2999            cost_usd_micros: 0,
3000            stop_reason_id: None,
3001            response_attempt: None,
3002        };
3003        let domain = format!("{session_id}:active");
3004        let sealed = crate::journal::seal(&journal_key, &domain, 0, &record).unwrap();
3005        let mut journal_file =
3006            std::fs::File::create(directory.path().join(format!("{stem}.events.ndjson"))).unwrap();
3007        journal_file.write_all(&sealed).unwrap();
3008        journal_file.write_all(b"\n").unwrap();
3009        drop(journal_file);
3010
3011        // Seed a broker ack so `ensure_active_event_published` short-circuits.
3012        crate::worker::persist_broker_ack(
3013            directory.path(),
3014            session_id,
3015            &event_uid,
3016            &PublishAck {
3017                topic: EventClass::Compression.topic().to_owned(),
3018                partition: 0,
3019                offset: 1,
3020            },
3021            &journal_key,
3022        )
3023        .await
3024        .unwrap();
3025
3026        // Sabotage the receipts directory so that `persist_receipt` inside
3027        // `close_session_locked` fails BEFORE `mark_artifact_committed` runs.
3028        // A regular file at `<spool>/receipts` makes `create_dir_all` fail for
3029        // the receipt write. Because there's no persisted receipt for recovery
3030        // to verify, the finalize path signs a fresh one and only then hits
3031        // the sabotage — reproducing the exact "closed=1 → reset → 0, but
3032        // artifact_committed still 0" window a racing lease can exploit.
3033        std::fs::write(directory.path().join("receipts"), b"").unwrap();
3034
3035        let registry = crate::session::SessionRegistry::new();
3036        let finalizer = Finalizer::with_bridge(
3037            signer,
3038            directory.path().to_path_buf(),
3039            Arc::new(Registry::new()),
3040            Arc::new(FailFirstReceiptBus {
3041                fail: std::sync::atomic::AtomicBool::new(false),
3042                attempts: parking_lot::Mutex::new(Vec::new()),
3043            }),
3044        );
3045        let result = finalizer
3046            .recover_spooled_sessions(&registry, &Default::default())
3047            .await;
3048        // Round-41 F1: per-session finalize errors during signed
3049        // recovery no longer propagate to the outer function's Err
3050        // — they warn+continue so one broken session cannot HOL-
3051        // block every other.
3052        //
3053        // Round-42 F3: transient close errors (Bridge / Receipt /
3054        // Atif / Task) now REMOVE the half-committed session from
3055        // the registry instead of leaving it orphaned with
3056        // `artifact_committed = 1` forever. The security invariant
3057        // this test enforces (post-error the session cannot be
3058        // leased and thus cannot have its chain diverged) still
3059        // holds — even more strongly, because the session is no
3060        // longer in the registry to be looked up at all. The
3061        // journal sidecar remains on disk so the next reconciler
3062        // tick re-adopts cleanly once the transient cause clears.
3063        assert!(
3064            result.is_ok(),
3065            "round-41 F1: per-session errors warn+continue; got {result:?}",
3066        );
3067
3068        assert!(
3069            registry.get(session_id).is_none(),
3070            "round-42 F3: after a transient close error the session must be removed from the registry so the next reconciler tick can re-adopt via the still-present journal sidecar — leaving it in the registry with `is_closed()` = true would starve recovery forever",
3071        );
3072    }
3073
3074    /// Round-41 F1: a corrupt/tampered signed sidecar must NOT
3075    /// head-of-line-block recovery of every OTHER session for the
3076    /// tick. Before this fix, the outer function returned Err on
3077    /// the first poisoned sidecar and skipped every subsequent
3078    /// signed AND unsigned candidate. Round-27 F1 / F2 already
3079    /// applied the warn+continue discipline to the ATIF-spool and
3080    /// promotion-marker paths; this locks in parity for the
3081    /// signed-journal path.
3082    ///
3083    /// Test plan: plant one poisoned metadata sidecar with a
3084    /// filename-shape that recover_signed_journals will accept but
3085    /// content that fails HMAC verification. Assert:
3086    ///   (a) recover_spooled_sessions returns Ok (was Err
3087    ///       pre-round-41),
3088    ///   (b) the poisoned session id is NOT installed into the
3089    ///       registry.
3090    #[tokio::test]
3091    async fn round_41_f1_corrupt_signed_sidecar_does_not_block_other_signed_recovery() {
3092        let directory = tempfile::tempdir().unwrap();
3093        // Plant a poisoned sidecar with the correct filename shape
3094        // but garbage content — read_journal_metadata will fail
3095        // HMAC verification and return Err. Pre-round-41 F1 this
3096        // Err propagated through recover_spooled_sessions and
3097        // aborted every unrelated session's recovery for the tick.
3098        let poison_stem = "poisonpoisonpoisonpoisonpoison32";
3099        std::fs::write(
3100            directory.path().join(format!("{poison_stem}.session.json")),
3101            b"{\"garbage\": true, \"not\": \"a valid sealed metadata\"}",
3102        )
3103        .unwrap();
3104        std::fs::write(
3105            directory.path().join(format!("{poison_stem}.events.ndjson")),
3106            b"{}\n",
3107        )
3108        .unwrap();
3109        let registry = crate::session::SessionRegistry::new();
3110        let finalizer = finalizer(directory.path());
3111        let outcome = finalizer
3112            .recover_spooled_sessions(&registry, &Default::default())
3113            .await;
3114        assert!(
3115            outcome.is_ok(),
3116            "poisoned sidecar must not fail the outer recover_spooled_sessions; got {outcome:?}"
3117        );
3118        assert!(
3119            registry.get(poison_stem).is_none(),
3120            "poisoned session id must NEVER be installed into the registry"
3121        );
3122    }
3123
3124    #[tokio::test]
3125    async fn signed_close_issues_exactly_one_offline_verifiable_receipt() {
3126        let directory = tempfile::tempdir().unwrap();
3127        let finalizer = finalizer(directory.path());
3128        let session = session(Workflow::Signed);
3129
3130        let first = finalizer
3131            .close_session(Arc::clone(&session), StopReason::SessionClosed)
3132            .await
3133            .unwrap();
3134        let FinalizeOutcome::Receipt { receipt } = first else {
3135            panic!("expected receipt")
3136        };
3137        receipt.verify_embedded().unwrap();
3138        assert!(matches!(
3139            finalizer
3140                .close_session(Arc::clone(&session), StopReason::SessionClosed)
3141                .await
3142                .unwrap(),
3143            FinalizeOutcome::AlreadyClosed
3144        ));
3145        assert_eq!(
3146            session.receipt.lock().as_ref().unwrap().body.receipt_id,
3147            receipt.body.receipt_id
3148        );
3149    }
3150
3151    #[tokio::test]
3152    async fn lifecycle_outbox_retries_the_same_receipt_event() {
3153        let directory = tempfile::tempdir().unwrap();
3154        let bus = Arc::new(FailFirstReceiptBus {
3155            fail: std::sync::atomic::AtomicBool::new(true),
3156            attempts: parking_lot::Mutex::new(Vec::new()),
3157        });
3158        let finalizer = Finalizer::with_bridge(
3159            Arc::new(Ed25519Signer::from_seed(&[17; 32])),
3160            directory.path().to_path_buf(),
3161            Arc::new(Registry::new()),
3162            bus.clone(),
3163        );
3164        let session = session(Workflow::Signed);
3165        assert!(matches!(
3166            finalizer
3167                .close_session(Arc::clone(&session), StopReason::SessionClosed)
3168                .await,
3169            Err(FinalizeError::Bridge(_))
3170        ));
3171        assert!(session.is_closed());
3172        assert!(
3173            session.try_lease().is_none(),
3174            "artifact commit must keep admission closed"
3175        );
3176        let receipt_id = session.receipt.lock().as_ref().unwrap().body.receipt_id.clone();
3177        let outcome = finalizer
3178            .close_session(Arc::clone(&session), StopReason::SessionClosed)
3179            .await
3180            .unwrap();
3181        let FinalizeOutcome::Receipt { receipt } = outcome else {
3182            panic!("expected receipt")
3183        };
3184        assert_eq!(receipt.body.receipt_id, receipt_id);
3185        let attempts = bus.attempts.lock();
3186        let receipt_events: Vec<_> = attempts
3187            .iter()
3188            .filter(|(topic, _)| topic == "agent.receipt")
3189            .collect();
3190        assert_eq!(receipt_events.len(), 2);
3191        assert_eq!(
3192            receipt_events[0].1["metadata"]["uid"],
3193            receipt_events[1].1["metadata"]["uid"]
3194        );
3195        assert!(!finalizer.lifecycle_outbox_path(&session.id, "receipt").exists());
3196    }
3197
3198    /// Round-43 F1: fire-and-forget close (no client retry, no idle
3199    /// sweeper hit) that failed at `emit_receipt_event` AFTER
3200    /// `mark_artifact_committed` used to leave the session
3201    /// permanently orphaned in the registry — `is_closed()` is true
3202    /// so the idle sweeper skips it, `close_complete = 0` so
3203    /// `evict_finalized` refuses it, and recovery scans hit the
3204    /// "already in registry" short-circuit. `complete_pending_closes`
3205    /// now drives the finalization tail to completion on the next
3206    /// reconciler tick without any client involvement.
3207    #[tokio::test]
3208    async fn round_43_f1_pending_close_completes_via_reconciler_sweep() {
3209        let directory = tempfile::tempdir().unwrap();
3210        let bus = Arc::new(FailFirstReceiptBus {
3211            fail: std::sync::atomic::AtomicBool::new(true),
3212            attempts: parking_lot::Mutex::new(Vec::new()),
3213        });
3214        let finalizer = Finalizer::with_bridge(
3215            Arc::new(Ed25519Signer::from_seed(&[43; 32])),
3216            directory.path().to_path_buf(),
3217            Arc::new(Registry::new()),
3218            bus.clone(),
3219        );
3220        let sessions = SessionRegistry::new();
3221        let session = session(Workflow::Signed);
3222        // Install into registry so `pending_close_sessions` can see it.
3223        sessions.insert_recovered(Arc::try_unwrap(session.clone()).unwrap_or_else(|arc| {
3224            Session::new(
3225                arc.id.clone(),
3226                arc.workflow,
3227                arc.current_identity(),
3228                Default::default(),
3229            )
3230        }));
3231        // Look up the Arc actually stored in the registry (identity
3232        // mapping after insert_recovered).
3233        let registered = sessions.get(&session.id).unwrap();
3234
3235        // First close attempt: signed workflow persists receipt on
3236        // disk, marks `artifact_committed = 1`, then fails on
3237        // `emit_receipt_event` via the injected bus outage. This is
3238        // the fire-and-forget window — no second `close_session` call
3239        // will follow.
3240        assert!(matches!(
3241            finalizer
3242                .close_session(Arc::clone(&registered), StopReason::SessionClosed)
3243                .await,
3244            Err(FinalizeError::Bridge(_))
3245        ));
3246        assert!(
3247            registered.artifact_committed_flag(),
3248            "signed close must have persisted the receipt and marked artifact_committed before the emit_receipt_event failure",
3249        );
3250        assert!(
3251            !registered.close_complete_flag(),
3252            "close_complete must NOT be set yet — the finalization tail did not run",
3253        );
3254        // Pre-round-43 the session would sit here forever.
3255        let pending = sessions.pending_close_sessions();
3256        assert_eq!(pending.len(), 1, "sweep must see the orphaned session");
3257        assert_eq!(pending[0].id, registered.id);
3258
3259        // Simulate the next reconciler tick — `replay_lifecycle_outboxes`
3260        // publishes the pending receipt event (the bus outage was one-
3261        // shot, so the retry succeeds), then `complete_pending_closes`
3262        // drives the tail to `mark_close_complete`.
3263        finalizer.replay_lifecycle_outboxes().await.unwrap();
3264        let completed = finalizer.complete_pending_closes(&sessions).await.unwrap();
3265        assert_eq!(completed, 1, "sweep must complete the orphan");
3266        assert!(
3267            registered.close_complete_flag(),
3268            "close_complete must be set after the sweep — round-43 F1 invariant",
3269        );
3270        assert!(
3271            sessions.pending_close_sessions().is_empty(),
3272            "no sessions remain in the pending-close set once completed",
3273        );
3274        // Step journal cleanup ran too — no debris on disk.
3275        let digest = av_core::digest::sha256_hex(registered.id.as_bytes());
3276        let stem = digest.get(..32).unwrap();
3277        assert!(
3278            !directory.path().join(format!("{stem}.session.json")).exists(),
3279            "step journal metadata sidecar must be removed by the completion sweep",
3280        );
3281    }
3282
3283    /// Round-44 F1: sidecar-less ATIF files (attacker plants OR
3284    /// honest crash-torn state between `write_atomic` and
3285    /// `ensure_atif_provenance`) must be checked cheaply and
3286    /// quarantined on first sighting — NOT read + parsed +
3287    /// strict-validated on every reconciler tick. Pre-fix, N such
3288    /// files would each burn a 64 MiB read + serde deserialize +
3289    /// strict validate every 5 s, starving the tick cadence and
3290    /// blocking lifecycle-outbox replay, close completion,
3291    /// promotion retry, and idle eviction.
3292    #[tokio::test]
3293    async fn round_44_f1_sidecar_less_atif_is_quarantined_without_reading_bytes() {
3294        let directory = tempfile::tempdir().unwrap();
3295        let finalizer = finalizer(directory.path());
3296        // Plant a `.json` file that would look like an ATIF spool
3297        // artifact but has no `.atif-auth` sidecar. Use bytes that
3298        // would definitely fail to parse — if the fix regresses and
3299        // the parser runs, the test will still pass because the
3300        // `invalid_json` skip branch also does `continue`, but the
3301        // quarantine rename assertion below distinguishes the
3302        // fix from the regression.
3303        let orphan = directory.path().join("hostileplant0000000000000000.json");
3304        std::fs::write(&orphan, b"{not valid json at all - this MUST NOT be parsed").unwrap();
3305
3306        let registry = SessionRegistry::new();
3307        let outcome = finalizer
3308            .recover_spooled_sessions(&registry, &Default::default())
3309            .await;
3310        assert!(
3311            outcome.is_ok(),
3312            "orphan must not fail the outer scan; got {outcome:?}"
3313        );
3314
3315        // The orphan must have been renamed out of the `.json`
3316        // extension so subsequent ticks skip it in O(1). Pre-fix
3317        // it would still be at `hostileplant....json` costing a
3318        // full read+parse per tick.
3319        assert!(
3320            !orphan.exists(),
3321            "sidecar-less ATIF must be quarantined-renamed after first sighting so subsequent ticks don't re-read it",
3322        );
3323        // Some sibling file must exist with the same stem plus a
3324        // `.corrupt-<uid>` suffix — the operator-forensic bytes.
3325        let mut found_quarantine = false;
3326        for entry in std::fs::read_dir(directory.path()).unwrap() {
3327            let entry = entry.unwrap();
3328            if let Some(name) = entry.file_name().to_str() {
3329                if name.contains(".json.corrupt-") {
3330                    found_quarantine = true;
3331                }
3332            }
3333        }
3334        assert!(
3335            found_quarantine,
3336            "quarantined file must be preserved on disk under a `.corrupt-<uid>` name for forensic inspection",
3337        );
3338    }
3339
3340    /// Round-44 F2: the pending-close sweep must NOT touch the
3341    /// empty-unsigned quarantine. That reject path
3342    /// (reconciler.rs:442-449) sets `artifact_committed = 1` but
3343    /// never wrote an ATIF file and never emitted a receipt.
3344    /// Driving the finalization tail on it would emit a spurious
3345    /// SESSION_CLOSE bridge event for a session that has no
3346    /// observable audit event on the wire, AND mark
3347    /// `close_complete = 1` which lets `get_or_open` (reopen=true)
3348    /// silently replace the quarantined Session on the next chat
3349    /// request — losing the incident evidence the reject was
3350    /// designed to preserve.
3351    #[tokio::test]
3352    async fn round_44_f2_empty_unsigned_quarantine_excluded_from_pending_close_sweep() {
3353        let directory = tempfile::tempdir().unwrap();
3354        let finalizer = finalizer(directory.path());
3355        let sessions = SessionRegistry::new();
3356        let empty = session(Workflow::Unsigned);
3357        // Install into the registry so the sweep can see it.
3358        let empty_id = empty.id.clone();
3359        sessions.insert_recovered(Arc::try_unwrap(empty).unwrap_or_else(|arc| {
3360            Session::new(
3361                arc.id.clone(),
3362                arc.workflow,
3363                arc.current_identity(),
3364                Default::default(),
3365            )
3366        }));
3367        let registered = sessions.get(&empty_id).unwrap();
3368
3369        // The empty-unsigned close reject is the code path we're
3370        // simulating. It returns Err after `mark_artifact_committed`
3371        // + `claim.committed = true`, so post-error the session
3372        // has `artifact_committed = 1`, `close_complete = 0`,
3373        // `capture_failed = 0`, and no `atif_path` — the exact
3374        // shape that used to trip the sweep.
3375        let result = finalizer
3376            .close_session(Arc::clone(&registered), StopReason::SessionClosed)
3377            .await;
3378        assert!(
3379            matches!(result, Err(FinalizeError::Atif(_))),
3380            "empty unsigned close must reject; got {result:?}",
3381        );
3382        assert!(
3383            registered.artifact_committed_flag(),
3384            "empty-unsigned reject seals with artifact_committed to stop idle-sweep churn",
3385        );
3386        assert!(
3387            registered.is_empty_unsigned_quarantine(),
3388            "empty-unsigned reject must be recognizable as a quarantine",
3389        );
3390
3391        // Pre-round-44 F2 the sweep would have picked this up.
3392        // Post-fix it must be excluded so no spurious SESSION_CLOSE
3393        // event is emitted and `close_complete` stays 0 (preserving
3394        // the incident evidence — a subsequent get_or_open won't
3395        // replace this session).
3396        let pending = sessions.pending_close_sessions();
3397        assert!(
3398            pending.iter().all(|s| s.id != empty_id),
3399            "empty-unsigned quarantine must be excluded from pending_close_sessions()",
3400        );
3401        let completed = finalizer.complete_pending_closes(&sessions).await.unwrap();
3402        assert_eq!(
3403            completed, 0,
3404            "sweep must NOT complete the empty-unsigned quarantine — evidence would be lost",
3405        );
3406        assert!(
3407            !registered.close_complete_flag(),
3408            "close_complete must remain 0 for the empty-unsigned quarantine so it can NOT be reopened by get_or_open (reopen=true) — preserving the incident record",
3409        );
3410    }
3411
3412    #[tokio::test]
3413    async fn concurrent_close_waits_for_failed_lifecycle_attempt() {
3414        let directory = tempfile::tempdir().unwrap();
3415        let bus = Arc::new(FailFirstReceiptBus {
3416            fail: std::sync::atomic::AtomicBool::new(true),
3417            attempts: parking_lot::Mutex::new(Vec::new()),
3418        });
3419        let finalizer = Finalizer::with_bridge(
3420            Arc::new(Ed25519Signer::from_seed(&[23; 32])),
3421            directory.path().to_path_buf(),
3422            Arc::new(Registry::new()),
3423            bus,
3424        );
3425        let session = session(Workflow::Signed);
3426        let first_finalizer = finalizer.clone();
3427        let first_session = Arc::clone(&session);
3428        let first = tokio::spawn(async move {
3429            first_finalizer
3430                .close_session(first_session, StopReason::SessionClosed)
3431                .await
3432        });
3433        tokio::task::yield_now().await;
3434        let second = finalizer.close_session(session, StopReason::SessionClosed).await;
3435        let first = first.await.unwrap();
3436        assert!(matches!(first, Err(FinalizeError::Bridge(_))));
3437        assert!(matches!(second, Ok(FinalizeOutcome::Receipt { .. })));
3438    }
3439
3440    #[tokio::test]
3441    async fn startup_replays_lifecycle_outbox_without_session_journal() {
3442        let directory = tempfile::tempdir().unwrap();
3443        let bus = Arc::new(FailFirstReceiptBus {
3444            fail: std::sync::atomic::AtomicBool::new(true),
3445            attempts: parking_lot::Mutex::new(Vec::new()),
3446        });
3447        let finalizer = Finalizer::with_bridge(
3448            Arc::new(Ed25519Signer::from_seed(&[19; 32])),
3449            directory.path().to_path_buf(),
3450            Arc::new(Registry::new()),
3451            bus.clone(),
3452        );
3453        let session = session(Workflow::Signed);
3454        assert!(matches!(
3455            finalizer.close_session(session, StopReason::SessionClosed).await,
3456            Err(FinalizeError::Bridge(_))
3457        ));
3458        assert!(finalizer
3459            .lifecycle_outbox_path("lifecycle-session", "receipt")
3460            .exists());
3461
3462        let sessions = SessionRegistry::new();
3463        assert_eq!(
3464            finalizer
3465                .recover_spooled_sessions(&sessions, &Default::default())
3466                .await
3467                .unwrap(),
3468            0
3469        );
3470        assert!(sessions.get("lifecycle-session").is_none());
3471        assert!(!finalizer
3472            .lifecycle_outbox_path("lifecycle-session", "receipt")
3473            .exists());
3474        assert_eq!(
3475            bus.attempts
3476                .lock()
3477                .iter()
3478                .filter(|(topic, _)| topic == "agent.receipt")
3479                .count(),
3480            2
3481        );
3482    }
3483
3484    #[tokio::test]
3485    async fn unsigned_close_and_promotion_are_strict_and_idempotent() {
3486        let directory = tempfile::tempdir().unwrap();
3487        let finalizer = finalizer(directory.path());
3488        let session = session(Workflow::Unsigned);
3489        session
3490            .atif
3491            .lock()
3492            .push_step(av_atif::Step {
3493                step_id: 0,
3494                timestamp: Some(av_core::time::now_iso8601()),
3495                source: av_atif::Source::Agent,
3496                message: serde_json::json!("done"),
3497                reasoning_effort: None,
3498                reasoning_content: None,
3499                model_name: None,
3500                tool_calls: None,
3501                observation: None,
3502                metrics: Some(av_atif::Metrics {
3503                    prompt_tokens: Some(10),
3504                    completion_tokens: Some(2),
3505                    cached_tokens: Some(4),
3506                    cost_usd: Some(0.001),
3507                    logprobs: None,
3508                    completion_token_ids: None,
3509                    prompt_token_ids: None,
3510                    extra: None,
3511                }),
3512                is_copied_context: None,
3513                llm_call_count: Some(1),
3514                extra: None,
3515            })
3516            .unwrap();
3517
3518        let outcome = finalizer
3519            .close_session(Arc::clone(&session), StopReason::SessionClosed)
3520            .await
3521            .unwrap();
3522        let FinalizeOutcome::Atif { path } = outcome else {
3523            panic!("expected ATIF artifact")
3524        };
3525        let value: serde_json::Value = serde_json::from_slice(&tokio::fs::read(path).await.unwrap()).unwrap();
3526        assert!(av_atif::validate_value(&value, av_atif::Mode::Strict).is_empty());
3527
3528        let first = finalizer.promote(Arc::clone(&session)).await.unwrap();
3529        let second = finalizer.promote(Arc::clone(&session)).await.unwrap();
3530        assert_eq!(first.body.receipt_id, second.body.receipt_id);
3531        first.verify_embedded().unwrap();
3532        assert!(matches!(
3533            first.body.subject,
3534            ReceiptSubject::AtifTrajectory {
3535                step_count: 1,
3536                retroactive: true,
3537                ..
3538            }
3539        ));
3540    }
3541
3542    #[tokio::test]
3543    async fn unsigned_restart_preserves_receipt_accounting_and_identity() {
3544        let directory = tempfile::tempdir().unwrap();
3545        let finalizer = finalizer(directory.path());
3546        let original = session(Workflow::Unsigned);
3547        original
3548            .totals
3549            .tool_calls
3550            .store(2, std::sync::atomic::Ordering::Release);
3551        original
3552            .totals
3553            .tool_allowed
3554            .store(1, std::sync::atomic::Ordering::Release);
3555        original
3556            .totals
3557            .tool_blocked
3558            .store(1, std::sync::atomic::Ordering::Release);
3559        original
3560            .totals
3561            .prompt_tokens
3562            .store(17, std::sync::atomic::Ordering::Release);
3563        original
3564            .totals
3565            .completion_tokens
3566            .store(9, std::sync::atomic::Ordering::Release);
3567        original
3568            .totals
3569            .cached_tokens
3570            .store(3, std::sync::atomic::Ordering::Release);
3571        original
3572            .totals
3573            .cost_usd_micros
3574            .store(1_234_567, std::sync::atomic::Ordering::Release);
3575        original.record_stop_reason(StopReason::PolicyBlocked);
3576        original
3577            .atif
3578            .lock()
3579            .push_step(av_atif::Step {
3580                step_id: 0,
3581                timestamp: None,
3582                source: av_atif::Source::User,
3583                message: serde_json::json!("test"),
3584                reasoning_effort: None,
3585                reasoning_content: None,
3586                model_name: None,
3587                tool_calls: None,
3588                observation: None,
3589                metrics: None,
3590                is_copied_context: None,
3591                llm_call_count: None,
3592                extra: None,
3593            })
3594            .unwrap();
3595        finalizer
3596            .close_session(original, StopReason::SessionClosed)
3597            .await
3598            .unwrap();
3599
3600        let registry = SessionRegistry::new();
3601        finalizer
3602            .recover_spooled_sessions(&registry, &Default::default())
3603            .await
3604            .unwrap();
3605        let recovered = registry.get("lifecycle-session").unwrap();
3606        assert_eq!(recovered.current_identity().ttl_remaining_s, Some(600));
3607        let receipt = finalizer.promote(recovered).await.unwrap();
3608        assert_eq!(receipt.body.tool_calls.total, 2);
3609        assert_eq!(receipt.body.tool_calls.allowed, 1);
3610        assert_eq!(receipt.body.tool_calls.blocked, 1);
3611        assert_eq!(receipt.body.cost.prompt_tokens, 17);
3612        assert_eq!(receipt.body.cost.completion_tokens, 9);
3613        assert_eq!(receipt.body.cost.cached_tokens, 3);
3614        assert_eq!(receipt.body.cost.cost_usd_micros, 1_234_567);
3615        assert_eq!(receipt.body.stop_reason_id, StopReason::PolicyBlocked.id());
3616    }
3617
3618    #[tokio::test]
3619    async fn incomplete_capture_never_produces_receipt_or_atif() {
3620        let directory = tempfile::tempdir().unwrap();
3621        let finalizer = finalizer(directory.path());
3622        for workflow in [Workflow::Signed, Workflow::Unsigned] {
3623            let session = session(workflow);
3624            session.mark_capture_failed();
3625            assert!(matches!(
3626                finalizer
3627                    .close_session(Arc::clone(&session), StopReason::SessionClosed)
3628                    .await,
3629                Err(FinalizeError::CaptureIncomplete)
3630            ));
3631            assert!(session.receipt.lock().is_none());
3632            assert!(session.atif_path.lock().is_none());
3633        }
3634    }
3635
3636    /// Regression for the live-session analog of the quarantined-recovery
3637    /// idle-sweep churn (bug 20). A worker job panic sets `capture_failed = 1`
3638    /// on the live session's flag while `closed = 0, artifact_committed = 0`.
3639    /// The idle sweeper's `!is_closed()` filter therefore picks the session
3640    /// up on every tick, `close_session_locked` runs its full body only to
3641    /// hit `if session.capture_failed() { return Err(CaptureIncomplete); }`,
3642    /// `CloseClaim` drops unarmed, `reset_close()` puts `closed` back to 0,
3643    /// and the session churns forever burning CPU, `lifecycle_lock`
3644    /// acquisitions, log noise, and `av_incomplete_sessions_total`. The fix
3645    /// is symmetric with bug 20: on the `CaptureIncomplete` return, mark the
3646    /// session `artifact_committed` and commit the `CloseClaim` so the
3647    /// session is sealed once and the idle sweeper's `!is_closed()` filter
3648    /// skips it forever after.
3649    #[tokio::test]
3650    async fn close_session_seals_capture_failed_session_so_idle_sweep_stops_churning() {
3651        let directory = tempfile::tempdir().unwrap();
3652        let finalizer = finalizer(directory.path());
3653        for workflow in [Workflow::Signed, Workflow::Unsigned] {
3654            let session = session(workflow);
3655            session.mark_capture_failed();
3656            let result = finalizer
3657                .close_session(Arc::clone(&session), StopReason::SessionClosed)
3658                .await;
3659            assert!(matches!(result, Err(FinalizeError::CaptureIncomplete)));
3660            assert!(
3661                session.is_closed(),
3662                "close_session_locked must seal a capture_failed session (mark_artifact_committed + claim.committed = true) on the CaptureIncomplete return path so subsequent idle-sweep passes skip it via the `!is_closed()` filter — otherwise CloseClaim drops unarmed, reset_close puts `closed` back to 0, is_closed() stays false, and the idle sweeper churns forever on this session (workflow: {workflow:?})",
3663            );
3664            assert!(session.receipt.lock().is_none());
3665            assert!(session.atif_path.lock().is_none());
3666        }
3667    }
3668
3669    /// Regression for a third idle-sweep churn shape (analog of bugs 20 and
3670    /// 21): an unsigned session that was opened but never had any events
3671    /// captured. Sessions get opened as a side effect of `get_or_open` inside
3672    /// `prepare_chat` / `intercept_tool`, but the request itself can fail
3673    /// before any worker job is submitted (worker queue full, admission
3674    /// rejected, loop-breaker Open). The session is left in the registry
3675    /// with an empty `atif` (no `push_step` ever ran). When
3676    /// `close_session_locked` reaches its unsigned branch it calls
3677    /// `snapshot_trajectory()`, hands the empty trajectory to
3678    /// `av_atif::write_atomic`, which runs strict validation, which rejects
3679    /// `steps.is_empty()` with "must contain at least one step". The write
3680    /// returns `WriterError::Invalid`, `close_session_locked` returns
3681    /// `Err(FinalizeError::Atif)`, `CloseClaim` drops unarmed,
3682    /// `reset_close()` puts `closed` back to `0`, and the idle sweeper
3683    /// re-enters this exact code path on every tick forever — burning CPU,
3684    /// growing `av_reconcile_errors_total`, and generating warning logs.
3685    /// The fix is analogous to bug 21: detect the terminal condition (empty
3686    /// ATIF cannot ever produce a valid strict artifact) and seal the
3687    /// session so `is_closed()` returns true and the idle sweeper skips it.
3688    #[tokio::test]
3689    async fn close_session_seals_empty_unsigned_session_so_idle_sweep_stops_churning() {
3690        let directory = tempfile::tempdir().unwrap();
3691        let finalizer = finalizer(directory.path());
3692        let session = session(Workflow::Unsigned);
3693        assert_eq!(
3694            session.atif.lock().clone().finish().steps.len(),
3695            0,
3696            "precondition: session has no captured steps",
3697        );
3698        let result = finalizer
3699            .close_session(Arc::clone(&session), StopReason::SessionClosed)
3700            .await;
3701        assert!(
3702            matches!(result, Err(FinalizeError::Atif(_))),
3703            "empty unsigned close must surface an ATIF error to the caller: {result:?}",
3704        );
3705        assert!(
3706            session.is_closed(),
3707            "close_session_locked must seal an empty unsigned session (mark_artifact_committed + claim.committed = true) when write_atomic's strict validation rejects the empty trajectory — otherwise CloseClaim drops unarmed, reset_close puts `closed` back to 0, is_closed() stays false, and the idle sweeper churns forever on this session (write_atomic → validate → \"must contain at least one step\" → Err → reset_close → picked up next tick → repeat)",
3708        );
3709        assert!(
3710            session.atif_path.lock().is_none(),
3711            "no ATIF file was ever produced for the empty session",
3712        );
3713    }
3714
3715    /// An in-flight response marker belonging to a *currently active*
3716    /// session is normal operation (the marker lives for the duration of
3717    /// the upstream call), not evidence of an abandoned effect. A periodic
3718    /// recovery tick that runs while such a request is in flight must not
3719    /// quarantine the session — otherwise any LLM call slower than one
3720    /// reconcile tick would poison its own session as capture-failed and
3721    /// wrongly quarantine the final trajectory at close.
3722    #[tokio::test]
3723    async fn recovery_tick_does_not_quarantine_live_sessions_with_inflight_markers() {
3724        let directory = tempfile::tempdir().unwrap();
3725        let finalizer = finalizer(directory.path());
3726        let live = session(Workflow::Unsigned);
3727        let registry = crate::session::SessionRegistry::new();
3728        let live = registry.insert_recovered(Arc::try_unwrap(live).map_err(|_| ()).unwrap());
3729        crate::worker::create_response_marker(
3730            directory.path(),
3731            &finalizer.journal_key,
3732            &live.id,
3733            "digest".to_owned(),
3734        )
3735        .await
3736        .unwrap();
3737        finalizer
3738            .recover_spooled_sessions(&registry, &Default::default())
3739            .await
3740            .unwrap();
3741        assert!(
3742            !finalizer.quarantined_sessions.lock().contains(&live.id),
3743            "a live session's in-flight marker must not put it in quarantine",
3744        );
3745        assert!(
3746            !live.capture_failed(),
3747            "a recovery tick must not poison a live session mid-request",
3748        );
3749    }
3750
3751    /// The quarantined_sessions set records id-space markers for recoveries
3752    /// that saw inconsistent effects on disk. A live session (client retry
3753    /// under the same id) that shares such an id must NOT inherit the
3754    /// capture-failed verdict on finalize — the verdict belongs to the
3755    /// recovered Session inserted with `mark_capture_failed()`, not to a
3756    /// fresh live Session whose in-memory `capture_failed` flag is still 0.
3757    /// close_session_locked must therefore rely on the per-session flag,
3758    /// not on the process-wide id set.
3759    #[tokio::test]
3760    async fn live_session_with_id_in_quarantine_set_can_still_close() {
3761        let directory = tempfile::tempdir().unwrap();
3762        let finalizer = finalizer(directory.path());
3763        // Simulate a prior recovery pass that added this session id to the set.
3764        finalizer
3765            .quarantined_sessions
3766            .lock()
3767            .insert("lifecycle-session".to_owned());
3768        let live = session(Workflow::Unsigned);
3769        assert!(!live.capture_failed(), "precondition: live session is clean");
3770        // Give the live session a step so its unsigned finalize can succeed.
3771        live.atif
3772            .lock()
3773            .push_step(av_atif::Step {
3774                step_id: 0,
3775                timestamp: None,
3776                source: av_atif::Source::Agent,
3777                message: serde_json::json!("live response"),
3778                reasoning_effort: None,
3779                reasoning_content: None,
3780                model_name: Some("test-model".into()),
3781                tool_calls: None,
3782                observation: None,
3783                metrics: Some(av_atif::Metrics {
3784                    prompt_tokens: Some(1),
3785                    completion_tokens: Some(1),
3786                    cached_tokens: Some(0),
3787                    cost_usd: Some(0.0),
3788                    logprobs: None,
3789                    completion_token_ids: None,
3790                    prompt_token_ids: None,
3791                    extra: None,
3792                }),
3793                is_copied_context: None,
3794                llm_call_count: Some(1),
3795                extra: None,
3796            })
3797            .unwrap();
3798        let outcome = finalizer
3799            .close_session(Arc::clone(&live), StopReason::SessionClosed)
3800            .await
3801            .expect("live session must finalize despite id sharing space with a set entry");
3802        match outcome {
3803            FinalizeOutcome::Atif { .. } => {}
3804            other => panic!("expected FinalizeOutcome::Atif, got {other:?}"),
3805        }
3806        assert!(
3807            !live.capture_failed(),
3808            "close_session must not poison the live session's capture_failed flag",
3809        );
3810    }
3811
3812    /// Recovery must never clobber a live session that shares its id with a
3813    /// stale spool artifact — a client retrying the same session_id after a
3814    /// crash could otherwise have its in-flight session force-closed by the
3815    /// reconciler. Two layers guard against this: the early registry check
3816    /// AND `try_insert_recovered` returning `Err(existing)` at the point of
3817    /// insertion. This test locks the outer invariant.
3818    #[tokio::test]
3819    async fn recovery_does_not_clobber_a_live_session_with_the_same_id() {
3820        let directory = tempfile::tempdir().unwrap();
3821        let finalizer = finalizer(directory.path());
3822        // Produce a valid ATIF artifact on disk under the "lifecycle-session" id.
3823        let closed_session = session(Workflow::Unsigned);
3824        closed_session
3825            .atif
3826            .lock()
3827            .push_step(av_atif::Step {
3828                step_id: 0,
3829                timestamp: None,
3830                source: av_atif::Source::Agent,
3831                message: serde_json::json!("archived response"),
3832                reasoning_effort: None,
3833                reasoning_content: None,
3834                model_name: Some("test-model".into()),
3835                tool_calls: None,
3836                observation: None,
3837                metrics: Some(av_atif::Metrics {
3838                    prompt_tokens: Some(1),
3839                    completion_tokens: Some(1),
3840                    cached_tokens: Some(0),
3841                    cost_usd: Some(0.0),
3842                    logprobs: None,
3843                    completion_token_ids: None,
3844                    prompt_token_ids: None,
3845                    extra: None,
3846                }),
3847                is_copied_context: None,
3848                llm_call_count: Some(1),
3849                extra: None,
3850            })
3851            .unwrap();
3852        finalizer
3853            .close_session(Arc::clone(&closed_session), StopReason::SessionClosed)
3854            .await
3855            .unwrap();
3856        // Now simulate a client retrying under the same session_id.
3857        let registry = SessionRegistry::new();
3858        let live = registry.get_or_open(
3859            "lifecycle-session",
3860            Workflow::Unsigned,
3861            &AgentIdentity {
3862                version: "1".to_owned(),
3863                charter: "test".into(),
3864                instance_uid: "instance-1".to_owned(),
3865                ttl_remaining_s: Some(600),
3866            },
3867            &Default::default(),
3868        );
3869        assert!(!live.is_closed(), "precondition: live session is open");
3870        assert!(live.receipt.lock().is_none(), "precondition: no receipt yet");
3871        assert!(live.atif_path.lock().is_none(), "precondition: no atif path yet");
3872        // Recovery must see the live session and skip the stale artifact.
3873        finalizer
3874            .recover_spooled_sessions(&registry, &Default::default())
3875            .await
3876            .unwrap();
3877        assert!(
3878            !live.is_closed(),
3879            "live session must not be force-closed by recovery"
3880        );
3881        assert!(
3882            live.receipt.lock().is_none(),
3883            "live session's receipt must not be overwritten from the stale artifact",
3884        );
3885        assert!(
3886            live.atif_path.lock().is_none(),
3887            "live session's atif_path must not be reassigned to the stale artifact",
3888        );
3889        assert_eq!(registry.len(), 1, "recovery must not add a duplicate entry");
3890    }
3891
3892    /// A tampered (or unauthenticated) artifact left in the spool must not
3893    /// abort the recovery scan: before this fix, `recover_spooled_sessions`
3894    /// returned `Err` at the first provenance failure, so one corrupt file
3895    /// starved recovery of every *other* session on every tick — and the
3896    /// warn (with no path) repeated forever. Integrity failures now skip
3897    /// the file (it stays on disk as evidence), count a skip metric, and
3898    /// let the rest of the spool recover.
3899    #[tokio::test]
3900    async fn recovery_skips_tampered_artifact_and_still_recovers_the_rest() {
3901        let directory = tempfile::tempdir().unwrap();
3902        let metrics = Arc::new(Registry::new());
3903        let finalizer = Finalizer::new(
3904            Arc::new(Ed25519Signer::from_seed(&[7; 32])),
3905            directory.path().to_path_buf(),
3906            Arc::clone(&metrics),
3907        );
3908        let step = av_atif::Step {
3909            step_id: 0,
3910            timestamp: None,
3911            source: av_atif::Source::Agent,
3912            message: serde_json::json!("archived response"),
3913            reasoning_effort: None,
3914            reasoning_content: None,
3915            model_name: Some("test-model".into()),
3916            tool_calls: None,
3917            observation: None,
3918            metrics: Some(av_atif::Metrics {
3919                prompt_tokens: Some(1),
3920                completion_tokens: Some(1),
3921                cached_tokens: Some(0),
3922                cost_usd: Some(0.0),
3923                logprobs: None,
3924                completion_token_ids: None,
3925                prompt_token_ids: None,
3926                extra: None,
3927            }),
3928            is_copied_context: None,
3929            llm_call_count: Some(1),
3930            extra: None,
3931        };
3932        let identity = AgentIdentity {
3933            version: "1".to_owned(),
3934            charter: "test".into(),
3935            instance_uid: "instance-1".to_owned(),
3936            ttl_remaining_s: Some(600),
3937        };
3938        let mut artifact_paths = Vec::new();
3939        for id in ["tampered-session", "healthy-session"] {
3940            let session = Arc::new(Session::new(
3941                id.to_owned(),
3942                Workflow::Unsigned,
3943                identity.clone(),
3944                Default::default(),
3945            ));
3946            session.atif.lock().push_step(step.clone()).unwrap();
3947            let outcome = finalizer
3948                .close_session(Arc::clone(&session), StopReason::SessionClosed)
3949                .await
3950                .unwrap();
3951            match outcome {
3952                FinalizeOutcome::Atif { path } => artifact_paths.push(path),
3953                other => panic!("expected FinalizeOutcome::Atif, got {other:?}"),
3954            }
3955        }
3956        // Corrupt the first artifact's bytes so its .atif-auth digest no
3957        // longer matches.
3958        let tampered = &artifact_paths[0];
3959        let mut bytes = std::fs::read(tampered).unwrap();
3960        let position = bytes
3961            .windows("archived".len())
3962            .position(|window| window == b"archived")
3963            .expect("artifact must contain the step message");
3964        bytes[position] = b'X';
3965        std::fs::write(tampered, &bytes).unwrap();
3966
3967        let registry = SessionRegistry::new();
3968        finalizer
3969            .recover_spooled_sessions(&registry, &Default::default())
3970            .await
3971            .expect("one tampered artifact must not abort the recovery scan");
3972        assert!(
3973            registry.get("healthy-session").is_some(),
3974            "the healthy artifact must still recover when a sibling is tampered",
3975        );
3976        assert!(
3977            registry.get("tampered-session").is_none(),
3978            "the tampered artifact must not recover a session",
3979        );
3980        assert!(
3981            tampered.exists(),
3982            "the tampered artifact must stay on disk as evidence",
3983        );
3984        let rendered = metrics.render();
3985        assert!(
3986            rendered.contains("av_atif_recovery_skipped_total{reason=\"provenance\"}"),
3987            "skips must be visible to operators via metrics; got: {rendered}",
3988        );
3989    }
3990
3991    /// Round-42 F1: a corrupt on-disk receipt for one ATIF-recovered
3992    /// session must NOT head-of-line-block recovery of every OTHER
3993    /// unsigned session for the tick. Pre-fix, the receipt-restore
3994    /// branch inside `recover_spooled_sessions`' ATIF loop used bare
3995    /// `?` for `read_capped_async`, `Receipt::from_json_slice`, and
3996    /// `verify_configured_receipt`, so a single garbage receipt file
3997    /// aborted the entire scan at the outer function's Err.
3998    ///
3999    /// Round-41 F1 applied the same async-block-with-outcome-enum
4000    /// pattern to `recover_signed_journals` +
4001    /// `consolidate_step_journals`; this test locks in parity for
4002    /// the third recovery loop.
4003    #[tokio::test]
4004    async fn round_42_f1_corrupt_receipt_does_not_block_other_atif_recovery() {
4005        let directory = tempfile::tempdir().unwrap();
4006        let metrics = Arc::new(Registry::new());
4007        let finalizer = Finalizer::new(
4008            Arc::new(Ed25519Signer::from_seed(&[42; 32])),
4009            directory.path().to_path_buf(),
4010            Arc::clone(&metrics),
4011        );
4012        let step = av_atif::Step {
4013            step_id: 0,
4014            timestamp: None,
4015            source: av_atif::Source::Agent,
4016            message: serde_json::json!("archived response"),
4017            reasoning_effort: None,
4018            reasoning_content: None,
4019            model_name: Some("test-model".into()),
4020            tool_calls: None,
4021            observation: None,
4022            metrics: Some(av_atif::Metrics {
4023                prompt_tokens: Some(1),
4024                completion_tokens: Some(1),
4025                cached_tokens: Some(0),
4026                cost_usd: Some(0.0),
4027                logprobs: None,
4028                completion_token_ids: None,
4029                prompt_token_ids: None,
4030                extra: None,
4031            }),
4032            is_copied_context: None,
4033            llm_call_count: Some(1),
4034            extra: None,
4035        };
4036        let identity = AgentIdentity {
4037            version: "1".to_owned(),
4038            charter: "test".into(),
4039            instance_uid: "instance-1".to_owned(),
4040            ttl_remaining_s: Some(600),
4041        };
4042        // Close two unsigned sessions to produce two valid ATIF
4043        // trajectories (with matching .atif-auth sidecars). One will
4044        // receive a poisoned receipt on disk; the other stays clean
4045        // and must still recover.
4046        for id in ["poisoned-receipt-session", "healthy-atif-session"] {
4047            let session = Arc::new(Session::new(
4048                id.to_owned(),
4049                Workflow::Unsigned,
4050                identity.clone(),
4051                Default::default(),
4052            ));
4053            session.atif.lock().push_step(step.clone()).unwrap();
4054            finalizer
4055                .close_session(Arc::clone(&session), StopReason::SessionClosed)
4056                .await
4057                .unwrap();
4058        }
4059
4060        // Plant a garbage receipt file at the exact path
4061        // `recover_spooled_sessions` looks up for the first session.
4062        // `Receipt::from_json_slice` will reject the bytes, which
4063        // pre-round-42 propagated Err out of the outer function and
4064        // aborted the whole scan before the second session was even
4065        // examined.
4066        let poisoned_receipt = directory.path().join("receipts").join(format!(
4067            "{}.json",
4068            &av_core::digest::sha256_hex(b"poisoned-receipt-session")[..32]
4069        ));
4070        std::fs::create_dir_all(poisoned_receipt.parent().unwrap()).unwrap();
4071        std::fs::write(&poisoned_receipt, b"{not valid receipt json").unwrap();
4072
4073        let registry = SessionRegistry::new();
4074        let outcome = finalizer
4075            .recover_spooled_sessions(&registry, &Default::default())
4076            .await;
4077        assert!(
4078            outcome.is_ok(),
4079            "round-42 F1: corrupt receipt must not abort the ATIF recovery scan; got {outcome:?}",
4080        );
4081        assert!(
4082            registry.get("healthy-atif-session").is_some(),
4083            "the healthy ATIF session MUST recover even when a sibling has a poisoned receipt — this is the HOL-block invariant",
4084        );
4085        let rendered = metrics.render();
4086        assert!(
4087            rendered.contains("av_atif_trajectory_recovery_skipped_total"),
4088            "per-session ATIF recovery skips must be visible to operators via a dedicated counter; got: {rendered}",
4089        );
4090    }
4091
4092    /// A background promotion retry must never force-close a session that is
4093    /// currently active. Without this guard, `retry_marked_promotions` would
4094    /// pick up a stale `.promote` marker left by a prior crash, look up the
4095    /// current live session from the registry (client retried under the same
4096    /// session_id), and call `promote()` — which starts by
4097    /// `close_session_locked`-ing any non-closed session. The live session
4098    /// would be prematurely terminated and its ATIF artifact overwritten.
4099    #[tokio::test]
4100    async fn promotion_retry_does_not_force_close_a_live_session() {
4101        let directory = tempfile::tempdir().unwrap();
4102        let finalizer = finalizer(directory.path());
4103        // Produce a valid unsigned artifact so a real promotion marker can point at it.
4104        let closed_session = session(Workflow::Unsigned);
4105        closed_session
4106            .atif
4107            .lock()
4108            .push_step(av_atif::Step {
4109                step_id: 0,
4110                timestamp: None,
4111                source: av_atif::Source::Agent,
4112                message: serde_json::json!("archived response"),
4113                reasoning_effort: None,
4114                reasoning_content: None,
4115                model_name: Some("test-model".into()),
4116                tool_calls: None,
4117                observation: None,
4118                metrics: Some(av_atif::Metrics {
4119                    prompt_tokens: Some(1),
4120                    completion_tokens: Some(1),
4121                    cached_tokens: Some(0),
4122                    cost_usd: Some(0.0),
4123                    logprobs: None,
4124                    completion_token_ids: None,
4125                    prompt_token_ids: None,
4126                    extra: None,
4127                }),
4128                is_copied_context: None,
4129                llm_call_count: Some(1),
4130                extra: None,
4131            })
4132            .unwrap();
4133        let outcome = finalizer
4134            .close_session(Arc::clone(&closed_session), StopReason::SessionClosed)
4135            .await
4136            .unwrap();
4137        let FinalizeOutcome::Atif { path } = outcome else {
4138            panic!("expected ATIF artifact")
4139        };
4140        let trajectory_bytes = tokio::fs::read(&path).await.unwrap();
4141        let promotion_marker = crate::journal::seal(
4142            &finalizer.journal_key,
4143            "promotion-marker",
4144            0,
4145            &PromotionMarker {
4146                session_id: closed_session.id.clone(),
4147                trajectory_digest: av_core::digest::sha256_hex(&trajectory_bytes),
4148            },
4149        )
4150        .unwrap();
4151        tokio::fs::write(path.with_extension("promote"), &promotion_marker)
4152            .await
4153            .unwrap();
4154        // Now simulate a client retrying under the same session_id.
4155        let registry = SessionRegistry::new();
4156        let live = registry.get_or_open(
4157            &closed_session.id,
4158            Workflow::Unsigned,
4159            &AgentIdentity {
4160                version: "1".to_owned(),
4161                charter: "test".into(),
4162                instance_uid: "instance-1".to_owned(),
4163                ttl_remaining_s: Some(600),
4164            },
4165            &Default::default(),
4166        );
4167        // Give the live session a distinct step so its trajectory would
4168        // pass strict validation — this makes the potential overwrite of
4169        // the archived artifact directly observable.
4170        live.atif
4171            .lock()
4172            .push_step(av_atif::Step {
4173                step_id: 0,
4174                timestamp: None,
4175                source: av_atif::Source::Agent,
4176                message: serde_json::json!("live response"),
4177                reasoning_effort: None,
4178                reasoning_content: None,
4179                model_name: Some("test-model".into()),
4180                tool_calls: None,
4181                observation: None,
4182                metrics: Some(av_atif::Metrics {
4183                    prompt_tokens: Some(2),
4184                    completion_tokens: Some(3),
4185                    cached_tokens: Some(0),
4186                    cost_usd: Some(0.0),
4187                    logprobs: None,
4188                    completion_token_ids: None,
4189                    prompt_token_ids: None,
4190                    extra: None,
4191                }),
4192                is_copied_context: None,
4193                llm_call_count: Some(1),
4194                extra: None,
4195            })
4196            .unwrap();
4197        assert!(!live.is_closed(), "precondition: live session is open");
4198        let promoted = finalizer.retry_marked_promotions(&registry).await.unwrap();
4199        assert_eq!(
4200            promoted, 0,
4201            "promotion retry must not count a skipped live session as promoted",
4202        );
4203        assert!(
4204            !live.is_closed(),
4205            "live session must not be force-closed by promotion retry",
4206        );
4207        assert!(
4208            live.receipt.lock().is_none(),
4209            "live session must not receive a receipt from a stale promotion marker",
4210        );
4211        assert!(
4212            live.atif_path.lock().is_none(),
4213            "live session's atif_path must not be set by a background promotion retry — that would mean its trajectory was snapshotted to disk out of band",
4214        );
4215        assert_eq!(
4216            tokio::fs::read(&path).await.unwrap(),
4217            trajectory_bytes,
4218            "the archived ATIF artifact must not be overwritten by a background retry that force-finalized the live session",
4219        );
4220        // The stale marker persists — it will be handled after the live
4221        // session finalizes normally, or expire with the artifact.
4222        assert!(
4223            path.with_extension("promote").exists(),
4224            "the promotion marker must remain on disk for future retries",
4225        );
4226    }
4227
4228    /// Regression for the quarantined-unsigned recovery branch. When a
4229    /// crashed process left an unsigned session's `.session.json` metadata
4230    /// on disk AND the session id was already in the `quarantined_sessions`
4231    /// set (populated on the same pass by `inflight_response_sessions` or
4232    /// `unresolved_tool_sessions`), `consolidate_step_journals` builds a
4233    /// fresh `Session::new` (`closed = 0`, `artifact_committed = 0`),
4234    /// calls `mark_capture_failed()`, and inserts it via
4235    /// `insert_recovered` — but forgets the `mark_artifact_committed()`
4236    /// step that its signed-recovery sibling applies before
4237    /// `try_insert_recovered`. The result is a permanent
4238    /// `is_closed() == false, capture_failed == true` session in the
4239    /// registry: the idle sweeper's `!is_closed()` filter keeps picking
4240    /// it up every tick, `close_session_locked` runs its full body only
4241    /// to hit `if session.capture_failed()` and return
4242    /// `CaptureIncomplete`, `CloseClaim` drops unarmed → `reset_close`
4243    /// puts `closed` back to `0`, and the churn is unbounded: growing
4244    /// `av_incomplete_sessions_total`, growing log noise, wasted lifecycle
4245    /// lock acquisitions, and a session that never leaves the registry.
4246    #[tokio::test]
4247    async fn recovery_marks_quarantined_unsigned_session_finalized_to_stop_idle_sweep_churn() {
4248        let directory = tempfile::tempdir().unwrap();
4249        let finalizer = finalizer(directory.path());
4250        let session_id = "quarantined-unsigned";
4251        let identity = AgentIdentity {
4252            version: "1".into(),
4253            charter: "test".into(),
4254            instance_uid: "instance-1".into(),
4255            ttl_remaining_s: Some(600),
4256        };
4257
4258        let digest = av_core::digest::sha256_hex(session_id.as_bytes());
4259        let stem = &digest[..32];
4260        let metadata_payload = serde_json::json!({
4261            "journal_version": 2,
4262            "session_id": session_id,
4263            "identity": identity,
4264            "workflow": "unsigned",
4265        });
4266        let metadata_sealed =
4267            crate::journal::seal(&finalizer.journal_key, "metadata", 0, &metadata_payload).unwrap();
4268        std::fs::write(
4269            directory.path().join(format!("{stem}.session.json")),
4270            &metadata_sealed,
4271        )
4272        .unwrap();
4273
4274        // Pre-populate the quarantine set — this is what a prior recovery
4275        // pass would do after finding an inflight-response marker or an
4276        // unresolved-tool marker on disk for this session id.
4277        finalizer
4278            .quarantined_sessions
4279            .lock()
4280            .insert(session_id.to_owned());
4281
4282        let registry = SessionRegistry::new();
4283        finalizer
4284            .recover_spooled_sessions(&registry, &Default::default())
4285            .await
4286            .unwrap();
4287
4288        let recovered = registry
4289            .get(session_id)
4290            .expect("quarantined session must be inserted");
4291        assert!(
4292            recovered.capture_failed(),
4293            "quarantined session must carry the capture-failed verdict",
4294        );
4295        assert!(
4296            recovered.is_closed(),
4297            "the quarantined-unsigned recovery branch (consolidate_step_journals) must also mark the session finalized (artifact_committed) so the idle sweeper's `!is_closed()` filter skips it — otherwise every idle tick calls close_session_locked which returns CaptureIncomplete, CloseClaim resets the close, and the session churns forever burning CPU, log noise, and metrics without ever leaving the registry",
4298        );
4299    }
4300
4301    /// A live session's `{stem}.session.json` journal metadata must be
4302    /// invisible to the ATIF spool scan. Both journal consumers skip a
4303    /// session that is still in the registry, so without the scan-side
4304    /// guard, every reconciler tick re-parsed the metadata file as an
4305    /// ATIF document, failed, warned "ignoring invalid ATIF spool file",
4306    /// and inflated the invalid_json skip counter — pure noise while a
4307    /// session was merely open.
4308    #[tokio::test]
4309    async fn atif_scan_ignores_live_session_journal_metadata() {
4310        let directory = tempfile::tempdir().unwrap();
4311        let metrics = Arc::new(Registry::new());
4312        let finalizer = Finalizer::new(
4313            Arc::new(Ed25519Signer::from_seed(&[7; 32])),
4314            directory.path().to_path_buf(),
4315            Arc::clone(&metrics),
4316        );
4317        let session_id = "still-open-session";
4318        let identity = AgentIdentity {
4319            version: "1".into(),
4320            charter: "test".into(),
4321            instance_uid: "instance-1".into(),
4322            ttl_remaining_s: Some(600),
4323        };
4324        let digest = av_core::digest::sha256_hex(session_id.as_bytes());
4325        let stem = &digest[..32];
4326        let metadata_payload = serde_json::json!({
4327            "journal_version": 2,
4328            "session_id": session_id,
4329            "identity": identity,
4330            "workflow": "unsigned",
4331        });
4332        let metadata_sealed =
4333            crate::journal::seal(&finalizer.journal_key, "metadata", 0, &metadata_payload).unwrap();
4334        std::fs::write(
4335            directory.path().join(format!("{stem}.session.json")),
4336            &metadata_sealed,
4337        )
4338        .unwrap();
4339
4340        // The session is live, so consolidate/recover leave its journal alone.
4341        let registry = SessionRegistry::new();
4342        let live = registry.get_or_open(session_id, Workflow::Unsigned, &identity, &Default::default());
4343        assert!(!live.is_closed(), "precondition: session is open");
4344
4345        finalizer
4346            .recover_spooled_sessions(&registry, &Default::default())
4347            .await
4348            .unwrap();
4349
4350        assert!(
4351            !metrics.render().contains("av_atif_recovery_skipped_total"),
4352            "the ATIF scan must skip *.session.json instead of counting it as an invalid spool file",
4353        );
4354        assert!(
4355            directory.path().join(format!("{stem}.session.json")).exists(),
4356            "the live session's journal metadata must survive the pass",
4357        );
4358    }
4359
4360    #[tokio::test]
4361    async fn restart_quarantines_inflight_response_without_stopping_recovery() {
4362        let directory = tempfile::tempdir().unwrap();
4363        let finalizer = finalizer(directory.path());
4364        crate::worker::create_response_marker(
4365            directory.path(),
4366            &finalizer.journal_key,
4367            "uncertain-session",
4368            "request-digest".to_owned(),
4369        )
4370        .await
4371        .unwrap();
4372
4373        let registry = SessionRegistry::new();
4374        assert_eq!(
4375            finalizer
4376                .recover_spooled_sessions(&registry, &Default::default())
4377                .await
4378                .unwrap(),
4379            0
4380        );
4381        assert!(finalizer
4382            .quarantined_sessions
4383            .lock()
4384            .contains("uncertain-session"));
4385        assert!(registry.get("uncertain-session").is_none());
4386    }
4387
4388    #[tokio::test]
4389    async fn restart_recovers_atif_and_retries_marked_promotion() {
4390        let directory = tempfile::tempdir().unwrap();
4391        let first = finalizer(directory.path());
4392        let original = session(Workflow::Unsigned);
4393        original
4394            .atif
4395            .lock()
4396            .push_step(av_atif::Step {
4397                step_id: 0,
4398                timestamp: None,
4399                source: av_atif::Source::Agent,
4400                message: serde_json::json!("recovered response"),
4401                reasoning_effort: None,
4402                reasoning_content: None,
4403                model_name: Some("test-model".into()),
4404                tool_calls: None,
4405                observation: None,
4406                metrics: Some(av_atif::Metrics {
4407                    prompt_tokens: Some(5),
4408                    completion_tokens: Some(2),
4409                    cached_tokens: Some(1),
4410                    cost_usd: Some(0.001),
4411                    logprobs: None,
4412                    completion_token_ids: None,
4413                    prompt_token_ids: None,
4414                    extra: None,
4415                }),
4416                is_copied_context: None,
4417                llm_call_count: Some(1),
4418                extra: None,
4419            })
4420            .unwrap();
4421        let outcome = first
4422            .close_session(Arc::clone(&original), StopReason::SessionClosed)
4423            .await
4424            .unwrap();
4425        let FinalizeOutcome::Atif { path } = outcome else {
4426            panic!("expected ATIF artifact")
4427        };
4428        let trajectory_bytes = tokio::fs::read(&path).await.unwrap();
4429        let promotion_marker = crate::journal::seal(
4430            &first.journal_key,
4431            "promotion-marker",
4432            0,
4433            &PromotionMarker {
4434                session_id: original.id.clone(),
4435                trajectory_digest: av_core::digest::sha256_hex(&trajectory_bytes),
4436            },
4437        )
4438        .unwrap();
4439        tokio::fs::write(path.with_extension("promote"), &promotion_marker)
4440            .await
4441            .unwrap();
4442
4443        let recovered_registry = SessionRegistry::new();
4444        let after_restart = finalizer(directory.path());
4445        assert_eq!(
4446            after_restart
4447                .recover_spooled_sessions(&recovered_registry, &Default::default())
4448                .await
4449                .unwrap(),
4450            1
4451        );
4452        assert_eq!(
4453            after_restart
4454                .retry_marked_promotions(&recovered_registry)
4455                .await
4456                .unwrap(),
4457            1
4458        );
4459        let recovered = recovered_registry.get(&original.id).unwrap();
4460        let receipt = recovered.receipt.lock().clone().unwrap();
4461        receipt.verify_embedded().unwrap();
4462        assert!(recovered.is_promoted());
4463        assert!(!path.with_extension("promote").exists());
4464
4465        tokio::fs::write(path.with_extension("promote"), &promotion_marker)
4466            .await
4467            .unwrap();
4468        let second_registry = SessionRegistry::new();
4469        assert_eq!(
4470            after_restart
4471                .recover_spooled_sessions(&second_registry, &Default::default())
4472                .await
4473                .unwrap(),
4474            1
4475        );
4476        let restored = second_registry.get(&original.id).unwrap();
4477        assert_eq!(
4478            restored.receipt.lock().as_ref().unwrap().body.receipt_id,
4479            receipt.body.receipt_id,
4480            "restart must restore the persisted receipt, not issue a duplicate"
4481        );
4482        assert_eq!(
4483            after_restart
4484                .retry_marked_promotions(&second_registry)
4485                .await
4486                .unwrap(),
4487            1
4488        );
4489        assert!(restored.is_promoted());
4490        assert!(!path.with_extension("promote").exists());
4491    }
4492
4493    #[tokio::test]
4494    async fn close_waits_for_active_response_lease() {
4495        let directory = tempfile::tempdir().unwrap();
4496        let finalizer = finalizer(directory.path());
4497        let session = session(Workflow::Signed);
4498        let lease = crate::session::SessionLease::new(Arc::clone(&session));
4499        let close_session = Arc::clone(&session);
4500        let task = tokio::spawn(async move {
4501            finalizer
4502                .close_session(close_session, StopReason::SessionClosed)
4503                .await
4504        });
4505        tokio::task::yield_now().await;
4506        assert!(!task.is_finished(), "close overtook an active response");
4507        drop(lease);
4508        assert!(matches!(
4509            task.await.unwrap().unwrap(),
4510            FinalizeOutcome::Receipt { .. }
4511        ));
4512    }
4513
4514    /// Round-18 F6: FIFO eviction lets one legitimate recurring
4515    /// artifact re-warn ONCE after it's evicted, but does not cause
4516    /// every legitimate artifact to re-warn together on the same
4517    /// tick when a rotating-timestamp attacker fills the cap.
4518    /// Round-20 F6: `WarnedArtifacts::new(0)` used to degenerate
4519    /// into oscillate-at-size-1 rather than reject or clamp. Now
4520    /// clamps to cap.max(1) so a future config-wiring bug that
4521    /// passes 0 doesn't silently break "warn once per artifact".
4522    #[test]
4523    fn warned_artifacts_clamps_zero_cap_to_one() {
4524        let mut warned = WarnedArtifacts::new(0);
4525        // First distinct entry is accepted.
4526        assert!(warned.insert(PathBuf::from("a")));
4527        assert_eq!(warned.len(), 1);
4528        // Same entry is deduplicated (the whole point).
4529        assert!(!warned.insert(PathBuf::from("a")));
4530        // A distinct entry evicts the first — cap=1 (clamped).
4531        assert!(warned.insert(PathBuf::from("b")));
4532        assert_eq!(warned.len(), 1);
4533    }
4534
4535    #[test]
4536    fn warned_artifacts_evicts_one_at_a_time_not_all_at_once() {
4537        let mut warned = WarnedArtifacts::new(3);
4538        assert!(warned.insert(PathBuf::from("a")));
4539        assert!(warned.insert(PathBuf::from("b")));
4540        assert!(warned.insert(PathBuf::from("c")));
4541        assert_eq!(warned.len(), 3);
4542        // Reinserting an existing entry is a no-op — still in-set,
4543        // no warn.
4544        assert!(!warned.insert(PathBuf::from("b")));
4545        // Fourth distinct entry evicts the OLDEST ("a"), NOT all.
4546        // Other legitimate entries (b, c) still tracked.
4547        assert!(warned.insert(PathBuf::from("d")));
4548        assert_eq!(warned.len(), 3);
4549        assert!(!warned.insert(PathBuf::from("b")));
4550        assert!(!warned.insert(PathBuf::from("c")));
4551        assert!(!warned.insert(PathBuf::from("d")));
4552        // "a" was evicted, so a re-warn on "a" returns true.
4553        // This new insert evicts b (now the oldest) — but c and d
4554        // survive. That's the FIFO contract: ONE eviction per
4555        // insert, not a full flush.
4556        assert!(warned.insert(PathBuf::from("a")));
4557        assert_eq!(warned.len(), 3);
4558        assert!(!warned.insert(PathBuf::from("c")));
4559        assert!(!warned.insert(PathBuf::from("d")));
4560        assert!(!warned.insert(PathBuf::from("a")));
4561    }
4562
4563    #[tokio::test]
4564    async fn torn_journal_tail_is_truncated_without_losing_complete_records() {
4565        let directory = tempfile::tempdir().unwrap();
4566        let path = directory.path().join("journal.ndjson");
4567        std::fs::write(&path, b"{\"complete\":true}\n{\"torn\":").unwrap();
4568        let lines = read_complete_journal(&path).await.unwrap();
4569        assert_eq!(lines, vec![r#"{"complete":true}"#]);
4570        assert_eq!(std::fs::read(&path).unwrap(), b"{\"complete\":true}\n");
4571    }
4572
4573    /// Round-13 F2: a journal containing NO newline anywhere (all
4574    /// bytes are a single partial record) must NOT be silently
4575    /// truncated to 0 bytes — that would destroy the only evidence
4576    /// of the failure. Quarantine to `<name>.corrupt-<uid>` instead
4577    /// and return an error so the reconciler leaves the sealed
4578    /// metadata sidecar in place.
4579    #[tokio::test]
4580    async fn journal_with_no_complete_lines_is_quarantined_not_truncated() {
4581        let directory = tempfile::tempdir().unwrap();
4582        let path = directory.path().join("journal.ndjson");
4583        std::fs::write(&path, b"{\"torn_before_first_newline\":").unwrap();
4584        let outcome = read_complete_journal(&path).await;
4585        // Must be an error, not an Ok(vec![]).
4586        let err = outcome.unwrap_err();
4587        assert!(
4588            format!("{err:?}").contains("no complete lines"),
4589            "expected quarantine error, got {err:?}",
4590        );
4591        // Original file must NOT exist any more — it was moved out.
4592        assert!(!path.exists(), "journal was not moved out of the recovery path");
4593        // Quarantine file must exist with the original content and
4594        // carry `.corrupt-` in its name.
4595        let entries: Vec<_> = std::fs::read_dir(directory.path())
4596            .unwrap()
4597            .filter_map(Result::ok)
4598            .filter(|entry| entry.file_name().to_string_lossy().contains(".corrupt-"))
4599            .collect();
4600        assert_eq!(entries.len(), 1, "expected exactly one quarantine file");
4601        let bytes = std::fs::read(entries[0].path()).unwrap();
4602        assert_eq!(&bytes, b"{\"torn_before_first_newline\":");
4603        // Round-37 F1: `FinalizeError::Atif`'s Display flows to
4604        // `tracing::warn!(%error, "ATIF spool recovery failed")`
4605        // and thence to `tracing_opentelemetry` -> OTLP -> SIEM.
4606        // The message body must NOT embed the absolute spool
4607        // directory (round-36 F1 basenamed the tracing FIELDS but
4608        // this ERROR STRING was missed). Assert:
4609        //   (a) the containing tempdir path is not in the message,
4610        //   (b) neither is any parent directory component.
4611        let msg = format!("{err}");
4612        assert!(
4613            !msg.contains(directory.path().to_string_lossy().as_ref()),
4614            "FinalizeError::Atif leaked spool dir absolute path: {msg}"
4615        );
4616        assert!(
4617            !msg.contains(std::path::MAIN_SEPARATOR_STR),
4618            "FinalizeError::Atif still contains a path separator: {msg}"
4619        );
4620        // The basename must still be present so an operator can
4621        // correlate the message with the quarantined file on disk.
4622        assert!(
4623            msg.contains("journal.ndjson"),
4624            "FinalizeError::Atif should still mention the journal basename: {msg}"
4625        );
4626    }
4627
4628    #[tokio::test]
4629    async fn failed_persistence_reopens_session_for_retry() {
4630        let directory = tempfile::tempdir().unwrap();
4631        let blocking_file = directory.path().join("not-a-directory");
4632        std::fs::write(&blocking_file, b"file").unwrap();
4633        let finalizer = finalizer(&blocking_file);
4634        let session = session(Workflow::Signed);
4635        assert!(finalizer
4636            .close_session(Arc::clone(&session), StopReason::SessionClosed)
4637            .await
4638            .is_err());
4639        assert!(!session.is_closed());
4640        assert!(session.try_close(), "failed close claim was not reset");
4641    }
4642
4643    #[tokio::test]
4644    async fn failed_unsigned_persistence_keeps_steps_for_retry() {
4645        let directory = tempfile::tempdir().unwrap();
4646        let spool = directory.path().join("spool");
4647        std::fs::write(&spool, b"blocking file").unwrap();
4648        let finalizer = finalizer(&spool);
4649        let session = session(Workflow::Unsigned);
4650        session
4651            .atif
4652            .lock()
4653            .push_step(av_atif::Step {
4654                step_id: 0,
4655                timestamp: None,
4656                source: av_atif::Source::User,
4657                message: serde_json::json!("survive"),
4658                reasoning_effort: None,
4659                reasoning_content: None,
4660                model_name: None,
4661                tool_calls: None,
4662                observation: None,
4663                metrics: None,
4664                is_copied_context: None,
4665                llm_call_count: None,
4666                extra: None,
4667            })
4668            .unwrap();
4669        assert!(finalizer
4670            .close_session(Arc::clone(&session), StopReason::SessionClosed)
4671            .await
4672            .is_err());
4673        assert!(!session.is_closed());
4674        std::fs::remove_file(&spool).unwrap();
4675        std::fs::create_dir(&spool).unwrap();
4676        let FinalizeOutcome::Atif { path } = finalizer
4677            .close_session(session, StopReason::SessionClosed)
4678            .await
4679            .unwrap()
4680        else {
4681            panic!("expected ATIF")
4682        };
4683        let trajectory: av_atif::Trajectory = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap();
4684        assert_eq!(trajectory.steps.len(), 1);
4685        assert_eq!(trajectory.steps[0].message, serde_json::json!("survive"));
4686    }
4687
4688    // ------------------------------------------------------------------
4689    // Congestion & bottleneck stress tests.
4690    // ------------------------------------------------------------------
4691
4692    /// Per-session lifecycle locks serialize close_session and promote
4693    /// per session id to prevent concurrent lifecycle-outbox rewrites;
4694    /// N distinct sessions do NOT queue behind a shared mutex. This
4695    /// test locks that behavior:
4696    /// N distinct sessions closing at once must all complete within a
4697    /// generous time bound and none must deadlock.
4698    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4699    async fn concurrent_closes_across_many_sessions_never_deadlock() {
4700        const SESSIONS: usize = 32;
4701        let directory = tempfile::tempdir().unwrap();
4702        let finalizer = Arc::new(finalizer(directory.path()));
4703        let mut tasks = Vec::with_capacity(SESSIONS);
4704        for i in 0..SESSIONS {
4705            let f = Arc::clone(&finalizer);
4706            let s = Arc::new(Session::new(
4707                format!("lifecycle-{i}"),
4708                Workflow::Signed,
4709                AgentIdentity {
4710                    version: "1".to_owned(),
4711                    charter: "test".into(),
4712                    instance_uid: format!("instance-{i}"),
4713                    ttl_remaining_s: Some(600),
4714                },
4715                Default::default(),
4716            ));
4717            tasks.push(tokio::spawn(async move {
4718                f.close_session(s, StopReason::SessionClosed).await
4719            }));
4720        }
4721        let results = tokio::time::timeout(
4722            std::time::Duration::from_secs(15),
4723            futures::future::join_all(tasks),
4724        )
4725        .await
4726        .expect("close_session tasks deadlocked under lifecycle_lock contention");
4727        for result in results {
4728            let outcome = result.expect("task panicked");
4729            assert!(outcome.is_ok(), "close failed: {outcome:?}");
4730        }
4731    }
4732
4733    /// Per-session lock table must not accumulate entries indefinitely.
4734    /// Every guard drop attempts to prune the corresponding entry
4735    /// (only when the map holds the last strong ref) — steady-state
4736    /// resident set = concurrent lifecycle ops, not total distinct
4737    /// session_ids ever seen. Otherwise an attacker firing 100k
4738    /// distinct session_ids could OOM the process.
4739    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4740    async fn per_session_lock_table_prunes_entries_after_close() {
4741        const SESSIONS: usize = 200;
4742        let directory = tempfile::tempdir().unwrap();
4743        let finalizer = Arc::new(finalizer(directory.path()));
4744        let locks = finalizer.lifecycle_locks();
4745        let mut tasks = Vec::with_capacity(SESSIONS);
4746        for i in 0..SESSIONS {
4747            let f = Arc::clone(&finalizer);
4748            let s = Arc::new(Session::new(
4749                format!("prune-{i}"),
4750                Workflow::Signed,
4751                AgentIdentity {
4752                    version: "1".to_owned(),
4753                    charter: "test".into(),
4754                    instance_uid: format!("instance-{i}"),
4755                    ttl_remaining_s: Some(600),
4756                },
4757                Default::default(),
4758            ));
4759            tasks.push(tokio::spawn(async move {
4760                f.close_session(s, StopReason::SessionClosed).await
4761            }));
4762        }
4763        for task in tasks {
4764            let _ = task.await.expect("task panicked");
4765        }
4766        // With no active lifecycle ops, the table should end up empty.
4767        // Some entries may briefly linger if a guard's drop is racing
4768        // another `arc_for` call, but a bounded settle window is fine.
4769        for _ in 0..20 {
4770            if locks.len() == 0 {
4771                return;
4772            }
4773            tokio::time::sleep(std::time::Duration::from_millis(25)).await;
4774        }
4775        panic!(
4776            "SessionLockTable did not prune to empty after {SESSIONS} closes; \
4777             residual entries = {}",
4778            locks.len()
4779        );
4780    }
4781
4782    /// A recovery scan of a large ATIF spool must not head-of-line-block
4783    /// a client-driven close on an unrelated session. Under the old
4784    /// global `lifecycle_lock` the client close waited for the entire
4785    /// scan to finish. With per-session locks, close on session B
4786    /// proceeds while recovery is still scanning candidate A.
4787    ///
4788    /// Seeded with real spool candidates that force per-file I/O in
4789    /// the scan; a concurrent close on a distinct session must
4790    /// complete well below the scan's total duration. Without a
4791    /// real spool the scan returns in microseconds and the test
4792    /// cannot distinguish per-session locks from the old global lock.
4793    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4794    async fn recovery_scan_does_not_head_of_line_block_unrelated_close() {
4795        let directory = tempfile::tempdir().unwrap();
4796        let finalizer = Arc::new(finalizer(directory.path()));
4797        // Seed the spool with a pile of candidates that fail provenance
4798        // (unauthenticated / not `.session.json`) so recovery iterates
4799        // read_dir + sidecar stat + quarantine-rename on each (round-44
4800        // F1 refuses them before any read or hash), taking observable
4801        // wall-clock time (512 candidates so the scan reliably outlasts
4802        // one unrelated close even on fast disks).
4803        for i in 0..512 {
4804            let path = directory.path().join(format!("scan-probe-{i}.json"));
4805            let payload = serde_json::json!({
4806                "atif_version": "1.7",
4807                "session_id": format!("scan-probe-{i}"),
4808                "agent": {"version": "1", "charter": "test", "instance_uid": "x"},
4809                "steps": [],
4810                "provenance": {"scheme": "none"},
4811            });
4812            tokio::fs::write(&path, serde_json::to_vec(&payload).unwrap())
4813                .await
4814                .unwrap();
4815        }
4816        let f_scan = Arc::clone(&finalizer);
4817        let scan_task = tokio::spawn(async move {
4818            let _ = f_scan
4819                .recover_spooled_sessions(&crate::session::SessionRegistry::new(), &Default::default())
4820                .await;
4821            std::time::Instant::now()
4822        });
4823        // Small yield so the scan task grabs `recovery_lock` first —
4824        // this is the state where the old global `lifecycle_lock`
4825        // would have blocked the client close below.
4826        tokio::task::yield_now().await;
4827
4828        let session = Arc::new(Session::new(
4829            "unrelated-close".to_owned(),
4830            Workflow::Signed,
4831            AgentIdentity {
4832                version: "1".to_owned(),
4833                charter: "test".into(),
4834                instance_uid: "instance-close".to_owned(),
4835                ttl_remaining_s: Some(600),
4836            },
4837            Default::default(),
4838        ));
4839        // Ordering bound, not wall-clock: under the OLD global
4840        // `lifecycle_lock` the close can only complete AFTER the scan
4841        // releases the lock, so close_end >= scan_end by construction.
4842        // Under per-session locks the close overlaps the still-running
4843        // 512-file scan and ends first. The previous absolute 200 ms
4844        // budget flaked under full-suite load (CPU starvation and fsync
4845        // contention inflate the close even though nothing is blocked);
4846        // an end-ordering comparison inflates both sides together. The
4847        // generous outer timeout only catches genuine deadlocks.
4848        let _ = tokio::time::timeout(
4849            std::time::Duration::from_secs(60),
4850            finalizer.close_session(session, StopReason::SessionClosed),
4851        )
4852        .await
4853        .expect("unrelated close deadlocked behind the recovery scan");
4854        let close_end = std::time::Instant::now();
4855        let scan_end = scan_task.await.unwrap();
4856        assert!(
4857            close_end < scan_end,
4858            "unrelated close finished only after the recovery scan ended — the head-of-line \
4859             blocking signature of a global lifecycle lock (close_end {close_end:?} >= scan_end \
4860             {scan_end:?})"
4861        );
4862    }
4863
4864    /// A saturated worker-side finalizer must not hold a session's
4865    /// lifecycle lock
4866    /// across independent await points that could stall other closers. We
4867    /// verify this indirectly: the aggregate wall-clock for 16 concurrent
4868    /// closes must stay within `10 × N ×` the uncontended single-close
4869    /// baseline (floored at 60 s for CI noise). A regression that
4870    /// awaited a slow I/O with the lock held would blow this bound.
4871    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4872    async fn close_latency_scales_reasonably_under_lock_contention() {
4873        let directory = tempfile::tempdir().unwrap();
4874        let finalizer = Arc::new(finalizer(directory.path()));
4875        // Warm-up: measure a single uncontended close.
4876        let warm = Arc::new(Session::new(
4877            "warm".to_owned(),
4878            Workflow::Signed,
4879            AgentIdentity {
4880                version: "1".to_owned(),
4881                charter: "test".into(),
4882                instance_uid: "warm".to_owned(),
4883                ttl_remaining_s: Some(600),
4884            },
4885            Default::default(),
4886        ));
4887        let uncontended = std::time::Instant::now();
4888        finalizer
4889            .close_session(warm, StopReason::SessionClosed)
4890            .await
4891            .unwrap();
4892        let baseline = uncontended.elapsed();
4893
4894        // Contended: 16 closes at once. Measure their WALL-CLOCK total.
4895        const N: usize = 16;
4896        let mut tasks = Vec::with_capacity(N);
4897        let started = std::time::Instant::now();
4898        for i in 0..N {
4899            let f = Arc::clone(&finalizer);
4900            let s = Arc::new(Session::new(
4901                format!("contended-{i}"),
4902                Workflow::Signed,
4903                AgentIdentity {
4904                    version: "1".to_owned(),
4905                    charter: "test".into(),
4906                    instance_uid: format!("contended-{i}"),
4907                    ttl_remaining_s: Some(600),
4908                },
4909                Default::default(),
4910            ));
4911            tasks.push(tokio::spawn(async move {
4912                f.close_session(s, StopReason::SessionClosed).await
4913            }));
4914        }
4915        for t in tasks {
4916            t.await.unwrap().unwrap();
4917        }
4918        let total = started.elapsed();
4919        // A shared serialized lock would give ~N * baseline. Anything
4920        // > 10 * N * baseline
4921        // signals we're holding a lock across additional awaits.
4922        let multiplier = u32::try_from(N * 10).unwrap_or(u32::MAX);
4923        let budget = baseline
4924            .saturating_mul(multiplier)
4925            .max(std::time::Duration::from_secs(60));
4926        assert!(
4927            total < budget,
4928            "16 contended closes took {total:?}, budget {budget:?} (baseline {baseline:?})",
4929        );
4930    }
4931}