1use crate::session::{Session, SessionRegistry, Workflow};
4use av_bridge::EventBus;
5use av_core::metrics::Registry;
6use av_core::time::elapsed_us;
7use av_events::StopReason;
8use av_receipts::{Receipt, ReceiptSubject, Signer};
9use serde::Serialize;
10use std::path::PathBuf;
11use std::sync::Arc;
12use std::time::Instant;
13
14const MAX_ATIF_RECOVERY_BYTES: u64 = 256 * 1024 * 1024;
20
21#[derive(Debug, Clone, Serialize)]
23#[serde(tag = "kind", rename_all = "snake_case")]
24pub enum FinalizeOutcome {
25 Receipt {
27 receipt: Box<Receipt>,
29 },
30 Atif {
32 path: PathBuf,
34 },
35 AlreadyClosed,
37}
38
39#[derive(Debug, thiserror::Error)]
41#[non_exhaustive]
42pub enum FinalizeError {
43 #[error("finalization task failed: {0}")]
45 Task(String),
46 #[error("receipt issuance failed: {0}")]
48 Receipt(String),
49 #[error("ATIF finalization failed: {0}")]
51 Atif(String),
52 #[error("promotion refused: {0}")]
54 Promotion(String),
55 #[error("session capture is incomplete; refusing final artifact")]
57 CaptureIncomplete,
58 #[error("lifecycle event publication failed: {0}")]
60 Bridge(String),
61}
62
63#[derive(Clone)]
65pub struct Finalizer {
66 signer: Arc<dyn Signer>,
67 spool_dir: PathBuf,
68 metrics: Arc<Registry>,
69 bridge: Option<Arc<dyn EventBus>>,
70 state_store: Option<Arc<dyn av_state::StateStore>>,
74 recovery_lock: Arc<tokio::sync::Mutex<()>>,
75 lifecycle_locks: Arc<SessionLockTable>,
82 quarantined_sessions: Arc<parking_lot::Mutex<std::collections::HashSet<String>>>,
83 warned_artifacts: Arc<parking_lot::Mutex<WarnedArtifacts>>,
91 journal_key: [u8; 32],
92}
93
94pub(crate) struct WarnedArtifacts {
106 order: std::collections::VecDeque<PathBuf>,
107 set: std::collections::HashSet<PathBuf>,
108 cap: usize,
109}
110
111impl WarnedArtifacts {
112 fn new(cap: usize) -> Self {
113 let cap = cap.max(1);
122 Self {
123 order: std::collections::VecDeque::new(),
124 set: std::collections::HashSet::new(),
125 cap,
126 }
127 }
128
129 fn insert(&mut self, path: PathBuf) -> bool {
130 if self.set.contains(&path) {
131 return false;
132 }
133 if self.order.len() >= self.cap {
134 if let Some(evicted) = self.order.pop_front() {
135 self.set.remove(&evicted);
136 }
137 }
138 self.order.push_back(path.clone());
139 self.set.insert(path);
140 true
141 }
142
143 #[cfg(test)]
144 fn len(&self) -> usize {
145 self.set.len()
146 }
147}
148
149const WARNED_ARTIFACTS_CAP: usize = 4096;
155
156#[derive(Default)]
168pub struct SessionLockTable {
169 inner: dashmap::DashMap<String, Arc<tokio::sync::Mutex<()>>>,
170}
171
172impl SessionLockTable {
173 fn arc_for(&self, session_id: &str) -> Arc<tokio::sync::Mutex<()>> {
174 self.inner
176 .entry(session_id.to_owned())
177 .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
178 .value()
179 .clone()
180 }
181
182 fn try_gc(&self, session_id: &str) {
186 self.inner
187 .remove_if(session_id, |_, arc| Arc::strong_count(arc) == 1);
188 }
189
190 #[cfg(test)]
192 pub(crate) fn len(&self) -> usize {
193 self.inner.len()
194 }
195}
196
197pub struct SessionLifecycleGuard {
200 permit: Option<tokio::sync::OwnedMutexGuard<()>>,
201 table: Arc<SessionLockTable>,
202 session_id: String,
203}
204
205impl Drop for SessionLifecycleGuard {
206 fn drop(&mut self) {
207 drop(self.permit.take());
212 self.table.try_gc(&self.session_id);
213 }
214}
215
216struct CloseClaim<'a> {
217 session: &'a Session,
218 committed: bool,
219}
220
221#[derive(serde::Serialize, serde::Deserialize)]
222struct LifecycleOutbox {
223 session_id: String,
224 kind: String,
225 topic: String,
226 key: String,
227 value: serde_json::Value,
228 ack: Option<av_bridge::PublishAck>,
229}
230
231#[derive(serde::Serialize, serde::Deserialize)]
232struct AtifProvenance {
233 session_id: String,
234 digest: String,
235}
236
237#[derive(serde::Serialize, serde::Deserialize)]
238struct PromotionMarker {
239 session_id: String,
240 trajectory_digest: String,
241}
242
243impl Drop for CloseClaim<'_> {
244 fn drop(&mut self) {
245 if !self.committed {
246 self.session.reset_close();
247 }
248 }
249}
250
251impl Finalizer {
252 pub fn metrics(&self) -> &Arc<Registry> {
256 &self.metrics
257 }
258
259 pub fn new(signer: Arc<dyn Signer>, spool_dir: PathBuf, metrics: Arc<Registry>) -> Self {
261 let journal_key = crate::journal::key_from_signer(signer.as_ref());
262 Self {
263 signer,
264 spool_dir,
265 metrics,
266 bridge: None,
267 state_store: None,
268 recovery_lock: Arc::new(tokio::sync::Mutex::new(())),
269 lifecycle_locks: Arc::new(SessionLockTable::default()),
270 quarantined_sessions: Arc::new(parking_lot::Mutex::new(std::collections::HashSet::new())),
271 warned_artifacts: Arc::new(parking_lot::Mutex::new(WarnedArtifacts::new(
272 WARNED_ARTIFACTS_CAP,
273 ))),
274 journal_key,
275 }
276 }
277
278 pub fn with_bridge(
280 signer: Arc<dyn Signer>,
281 spool_dir: PathBuf,
282 metrics: Arc<Registry>,
283 bridge: Arc<dyn EventBus>,
284 ) -> Self {
285 let journal_key = crate::journal::key_from_signer(signer.as_ref());
286 Self {
287 signer,
288 spool_dir,
289 metrics,
290 bridge: Some(bridge),
291 state_store: None,
292 recovery_lock: Arc::new(tokio::sync::Mutex::new(())),
293 lifecycle_locks: Arc::new(SessionLockTable::default()),
294 quarantined_sessions: Arc::new(parking_lot::Mutex::new(std::collections::HashSet::new())),
295 warned_artifacts: Arc::new(parking_lot::Mutex::new(WarnedArtifacts::new(
296 WARNED_ARTIFACTS_CAP,
297 ))),
298 journal_key,
299 }
300 }
301
302 async fn acquire_lifecycle(&self, session_id: &str) -> SessionLifecycleGuard {
307 let arc = self.lifecycle_locks.arc_for(session_id);
308 let permit = arc.lock_owned().await;
309 SessionLifecycleGuard {
310 permit: Some(permit),
311 table: Arc::clone(&self.lifecycle_locks),
312 session_id: session_id.to_owned(),
313 }
314 }
315
316 #[cfg(test)]
317 pub(crate) fn lifecycle_locks(&self) -> Arc<SessionLockTable> {
318 Arc::clone(&self.lifecycle_locks)
319 }
320
321 fn warn_once(&self, path: PathBuf) -> bool {
332 self.warned_artifacts.lock().insert(path)
333 }
334
335 #[must_use]
338 pub fn with_state_store(mut self, store: Arc<dyn av_state::StateStore>) -> Self {
339 self.state_store = Some(store);
340 self
341 }
342
343 fn clear_budget_state(&self, session_id: &str) {
349 if let Some(store) = self.state_store.as_deref() {
350 store.remove_prefix(&av_state::ActionBudget::session_prefix(session_id));
351 }
352 }
353
354 #[tracing::instrument(
357 name = "agentvisor.session.close",
358 skip_all,
359 fields(session.id = %session.id, workflow = ?session.workflow)
360 )]
361 pub async fn close_session(
362 &self,
363 session: Arc<Session>,
364 stop_reason: StopReason,
365 ) -> Result<FinalizeOutcome, FinalizeError> {
366 let _lifecycle = self.acquire_lifecycle(&session.id).await;
367 self.close_session_locked(session, stop_reason).await
368 }
369
370 async fn close_session_locked(
371 &self,
372 session: Arc<Session>,
373 stop_reason: StopReason,
374 ) -> Result<FinalizeOutcome, FinalizeError> {
375 let close_guard = session.close_guard();
376 if !session.try_close() {
377 return Ok(FinalizeOutcome::AlreadyClosed);
378 }
379 drop(close_guard);
380 let mut claim = CloseClaim {
381 session: &session,
382 committed: false,
383 };
384 session.wait_for_streams().await;
385 session.wait_for_worker_jobs().await;
386 if session.capture_failed() {
387 self.metrics
388 .counter(
389 "av_incomplete_sessions_total",
390 "Sessions refused due to incomplete capture",
391 )
392 .inc();
393 session.mark_artifact_committed();
397 claim.committed = true;
398 self.clear_budget_state(&session.id);
399 return Err(FinalizeError::CaptureIncomplete);
400 }
401 let started = Instant::now();
402 let outcome = match session.workflow {
403 Workflow::Signed => {
404 let subject = {
405 let chain = session.chain.lock();
406 ReceiptSubject::EventChain {
407 chain_head: chain.head_hex(),
408 event_count: chain.count(),
409 }
410 };
411 let persisted_receipt = { session.receipt.lock().clone() };
412 let receipt = if let Some(receipt) = persisted_receipt {
413 self.verify_configured_receipt(&receipt)?;
414 if receipt.body.subject != subject {
415 return Err(FinalizeError::Receipt(
416 "persisted receipt subject does not match reconstructed chain".to_owned(),
417 ));
418 }
419 receipt
420 } else {
421 let body = session.receipt_body(subject, stop_reason);
422 let sign_started = Instant::now();
423 let receipt = Receipt::issue(body, self.signer.as_ref())
424 .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
425 self.metrics
426 .histogram("av_receipt_sign_duration_seconds", "Receipt signing latency")
427 .observe_us(elapsed_us(sign_started));
428 self.persist_receipt(&session.id, &receipt).await?;
429 *session.receipt.lock() = Some(receipt.clone());
430 receipt
431 };
432 session.mark_artifact_committed();
433 self.emit_receipt_event(&session, &receipt).await?;
434 FinalizeOutcome::Receipt {
435 receipt: Box::new(receipt),
436 }
437 }
438 Workflow::Unsigned => {
439 let existing_path = { session.atif_path.lock().clone() };
440 let path = if let Some(path) = existing_path {
441 path
442 } else {
443 let mut trajectory = session.snapshot_trajectory();
444 if trajectory.steps.is_empty() {
447 session.mark_artifact_committed();
448 claim.committed = true;
449 self.clear_budget_state(&session.id);
450 return Err(FinalizeError::Atif(
451 "cannot finalize an unsigned session with no captured steps".to_owned(),
452 ));
453 }
454 let identity = session.current_identity();
455 trajectory.agent.extra = Some(serde_json::json!({
456 "charter": identity.charter,
457 "instance_uid": identity.instance_uid,
458 "ttl_remaining_s": identity.ttl_remaining_s,
459 }));
460 if let Some(metrics) = trajectory.final_metrics.as_mut() {
461 metrics.total_prompt_tokens = Some(
462 session
463 .totals
464 .prompt_tokens
465 .load(std::sync::atomic::Ordering::Acquire),
466 );
467 metrics.total_completion_tokens = Some(
468 session
469 .totals
470 .completion_tokens
471 .load(std::sync::atomic::Ordering::Acquire),
472 );
473 metrics.total_cached_tokens = Some(
474 session
475 .totals
476 .cached_tokens
477 .load(std::sync::atomic::Ordering::Acquire),
478 );
479 metrics.total_cost_usd = Some(
480 session
481 .totals
482 .cost_usd_micros
483 .load(std::sync::atomic::Ordering::Acquire)
484 as f64
485 / av_core::units::USD_MICROS_PER_DOLLAR as f64,
486 );
487 metrics.extra = Some(serde_json::json!({
488 "tool_calls": session.totals.tool_calls.load(std::sync::atomic::Ordering::Acquire),
489 "tool_allowed": session.totals.tool_allowed.load(std::sync::atomic::Ordering::Acquire),
490 "tool_blocked": session.totals.tool_blocked.load(std::sync::atomic::Ordering::Acquire),
491 "cost_usd_micros": session.totals.cost_usd_micros.load(std::sync::atomic::Ordering::Acquire),
492 "stop_reason_id": session.recorded_stop_reason_id(),
493 }));
494 }
495 let name = format!(
496 "{}.json",
497 &av_core::digest::sha256_hex(session.id.as_bytes())[..32]
498 );
499 let path = self.spool_dir.join(name);
500 let write_path = path.clone();
501 tokio::task::spawn_blocking(move || av_atif::write_atomic(&trajectory, &write_path))
502 .await
503 .map_err(|error| FinalizeError::Task(error.to_string()))?
504 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
505 *session.atif_path.lock() = Some(path.clone());
506 path
507 };
508 self.ensure_atif_provenance(&path, &session.id).await?;
509 session.mark_artifact_committed();
510 FinalizeOutcome::Atif { path }
511 }
512 };
513 let workflow = session.workflow.as_str();
514 self.emit_bridge_event(
515 &session,
516 av_events::EventClass::Session,
517 serde_json::json!({"action": "closed", "workflow": workflow}),
518 crate::journal::SESSION_CLOSE_OUTBOX_KIND,
519 )
520 .await?;
521 self.remove_step_journal(&session.id).await?;
522 self.remove_lifecycle_outbox(&session.id, crate::journal::RECEIPT_OUTBOX_KIND)
523 .await?;
524 self.remove_lifecycle_outbox(&session.id, crate::journal::SESSION_CLOSE_OUTBOX_KIND)
525 .await?;
526 self.metrics
527 .histogram(
528 "av_session_finalize_duration_seconds",
529 "Session finalization latency",
530 )
531 .observe_us(elapsed_us(started));
532 self.metrics
533 .counter("av_sessions_finalized_total", "Sessions finalized")
534 .inc();
535 claim.committed = true;
536 session.mark_close_complete();
539 self.clear_budget_state(&session.id);
540 Ok(outcome)
541 }
542
543 #[tracing::instrument(
545 name = "agentvisor.session.promote",
546 skip_all,
547 fields(session.id = %session.id)
548 )]
549 pub async fn promote(&self, session: Arc<Session>) -> Result<Receipt, FinalizeError> {
550 let _lifecycle = self.acquire_lifecycle(&session.id).await;
551 if session.workflow != Workflow::Unsigned {
552 return session
553 .receipt
554 .lock()
555 .clone()
556 .ok_or_else(|| FinalizeError::Promotion("signed session has no issued receipt".to_owned()));
557 }
558 if !session.is_closed() {
559 self.close_session_locked(Arc::clone(&session), StopReason::SessionClosed)
560 .await?;
561 }
562 let persisted_receipt = { session.receipt.lock().clone() };
563 if session.is_promoted() {
564 let receipt = persisted_receipt.ok_or_else(|| {
565 FinalizeError::Promotion("promoted session has no persisted receipt".to_owned())
566 })?;
567 let atif_path_opt: Option<std::path::PathBuf> = { session.atif_path.lock().clone() };
584 if let Some(atif_path) = atif_path_opt {
585 let marker = atif_path.with_extension("promote");
586 if marker.exists() {
587 if let Err(error) = remove_outbox(&marker).await {
588 tracing::warn!(
589 %error,
590 path = %av_core::fsutil::basename(&marker),
591 "failed to clean up orphan promotion marker (promotion still succeeded)"
592 );
593 }
594 }
595 }
596 return Ok(receipt);
597 }
598 let path =
599 session.atif_path.lock().clone().ok_or_else(|| {
600 FinalizeError::Promotion("session has no persisted ATIF artifact".to_owned())
601 })?;
602 let marker = path.with_extension("promote");
603 if !path.with_extension("atif-auth").exists() {
604 return Err(FinalizeError::Atif(
605 "ATIF artifact has no authenticated provenance".to_owned(),
606 ));
607 }
608 self.ensure_atif_provenance(&path, &session.id).await?;
609 let bytes = read_capped_async(path.clone(), av_core::fsutil::MAX_ATIF_BYTES)
610 .await
611 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
612 let trajectory: av_atif::Trajectory =
613 serde_json::from_slice(&bytes).map_err(|error| FinalizeError::Atif(error.to_string()))?;
614 let issues = av_atif::validate_trajectory(&trajectory, av_atif::Mode::Strict);
615 if !issues.is_empty() {
616 const RENDER_ISSUE_HEAD: usize = 16;
623 let total = issues.len();
624 let head: Vec<_> = issues.iter().take(RENDER_ISSUE_HEAD).collect();
625 return Err(FinalizeError::Atif(format!(
626 "strict validation failed ({total} issues, showing first {}): {head:?}",
627 head.len()
628 )));
629 }
630 let trajectory_digest = av_core::digest::sha256_hex(&bytes);
631 let subject = ReceiptSubject::AtifTrajectory {
632 trajectory_digest: trajectory_digest.clone(),
633 step_count: trajectory.steps.len() as u64,
634 retroactive: true,
635 };
636 let marker_payload = PromotionMarker {
637 session_id: session.id.clone(),
638 trajectory_digest,
639 };
640 if marker.exists() {
641 let sealed = read_capped_async(marker.clone(), av_core::fsutil::MAX_CONTROL_BYTES)
642 .await
643 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
644 let actual: PromotionMarker =
645 crate::journal::open(&self.journal_key, "promotion-marker", 0, &sealed)
646 .map_err(FinalizeError::Atif)?;
647 if actual.session_id != marker_payload.session_id
648 || actual.trajectory_digest != marker_payload.trajectory_digest
649 {
650 return Err(FinalizeError::Atif(
651 "promotion marker does not match session and trajectory".to_owned(),
652 ));
653 }
654 } else {
655 let sealed = crate::journal::seal(&self.journal_key, "promotion-marker", 0, &marker_payload)
656 .map_err(FinalizeError::Atif)?;
657 persist_marker(&marker, &sealed).await?;
658 }
659 if !session.try_promote() {
660 return Err(FinalizeError::Promotion(
661 "promotion is already in progress".to_owned(),
662 ));
663 }
664 let receipt = if let Some(receipt) = persisted_receipt {
665 if let Err(error) = self.verify_configured_receipt(&receipt) {
666 session.reset_promotion();
667 return Err(error);
668 }
669 if receipt.body.subject != subject {
670 session.reset_promotion();
671 return Err(FinalizeError::Receipt(
672 "persisted promotion receipt does not match ATIF artifact".to_owned(),
673 ));
674 }
675 receipt
676 } else {
677 let body = session.receipt_body(subject, StopReason::SessionClosed);
678 let issued = Receipt::issue(body, self.signer.as_ref())
679 .map_err(|error| FinalizeError::Receipt(error.to_string()));
680 let receipt = match issued {
681 Ok(receipt) => receipt,
682 Err(error) => {
683 session.reset_promotion();
684 return Err(error);
685 }
686 };
687 if let Err(error) = self.persist_receipt(&session.id, &receipt).await {
688 session.reset_promotion();
689 return Err(error);
690 }
691 *session.receipt.lock() = Some(receipt.clone());
692 receipt
693 };
694 if let Err(error) = self.emit_receipt_event(&session, &receipt).await {
695 session.reset_promotion();
696 return Err(error);
697 }
698 session.finish_promotion();
699 remove_outbox(&marker).await?;
700 self.remove_lifecycle_outbox(&session.id, crate::journal::RECEIPT_OUTBOX_KIND)
701 .await?;
702 self.metrics
703 .counter("av_sessions_promoted_total", "Unsigned sessions promoted")
704 .inc();
705 Ok(receipt)
706 }
707
708 #[tracing::instrument(name = "agentvisor.recovery", skip_all)]
714 pub async fn recover_spooled_sessions(
715 &self,
716 sessions: &SessionRegistry,
717 breaker: &av_loopdetect::BreakerConfig,
718 ) -> Result<usize, FinalizeError> {
719 let _recovery = self.recovery_lock.lock().await;
720 let mut quarantined = crate::worker::inflight_response_sessions(&self.spool_dir, &self.journal_key)
726 .await
727 .map_err(FinalizeError::Atif)?;
728 quarantined.extend(
729 crate::routes::unresolved_tool_sessions(&self.spool_dir, &self.journal_key)
730 .await
731 .map_err(FinalizeError::Atif)?,
732 );
733 quarantined.retain(|id| sessions.get(id).is_none());
739 if !quarantined.is_empty() {
740 let mut known = self.quarantined_sessions.lock();
745 let new: Vec<&String> = quarantined.iter().filter(|id| !known.contains(*id)).collect();
746 if !new.is_empty() {
747 tracing::warn!(
748 sessions = new.len(),
749 "quarantining sessions with incomplete effects"
750 );
751 }
752 known.extend(quarantined.iter().cloned());
753 }
754 self.replay_lifecycle_outboxes().await?;
755 let signed_recovered = self.recover_signed_journals(sessions, breaker).await?;
756 self.consolidate_step_journals(sessions, breaker).await?;
757 let mut entries = match tokio::fs::read_dir(&self.spool_dir).await {
758 Ok(entries) => entries,
759 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
760 Err(error) => return Err(FinalizeError::Atif(error.to_string())),
761 };
762 let mut recovered = 0usize;
763 while let Some(entry) = entries
764 .next_entry()
765 .await
766 .map_err(|error| FinalizeError::Atif(error.to_string()))?
767 {
768 let path = entry.path();
769 if path.extension().and_then(std::ffi::OsStr::to_str) != Some("json") {
770 continue;
771 }
772 if path
777 .file_name()
778 .and_then(std::ffi::OsStr::to_str)
779 .is_some_and(|name| name.ends_with(".session.json"))
780 {
781 continue;
782 }
783 if !path.with_extension("atif-auth").exists() {
805 self.metrics
806 .counter(
807 "av_atif_recovery_skipped_total{reason=\"unauthenticated\"}",
808 "ATIF spool files skipped during recovery",
809 )
810 .inc();
811 let mut quarantine = path.clone();
812 let stem = quarantine
813 .file_name()
814 .and_then(std::ffi::OsStr::to_str)
815 .unwrap_or("orphan-atif")
816 .to_owned();
817 let new_name = format!("{stem}.corrupt-{}", av_core::new_event_uid());
818 quarantine.set_file_name(new_name);
819 match tokio::fs::rename(&path, &quarantine).await {
820 Ok(()) => {
821 if self.warn_once(quarantine.clone()) {
822 tracing::warn!(
823 original = %av_core::fsutil::basename(&path),
824 quarantine = %av_core::fsutil::basename(&quarantine),
825 "quarantined ATIF spool file with no authenticated provenance"
826 );
827 }
828 }
829 Err(error) => {
830 if self.warn_once(path.clone()) {
831 tracing::warn!(
832 %error,
833 path = %av_core::fsutil::basename(&path),
834 "failed to quarantine ATIF spool file with no authenticated provenance; will retry next tick"
835 );
836 }
837 }
838 }
839 continue;
840 }
841 let metadata = match tokio::fs::metadata(&path).await {
847 Ok(m) => m,
848 Err(error) => {
849 tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "skipping ATIF spool file whose metadata is unreadable");
850 continue;
851 }
852 };
853 if metadata.len() > MAX_ATIF_RECOVERY_BYTES {
854 self.metrics
855 .counter(
856 "av_atif_recovery_skipped_total{reason=\"too_large\"}",
857 "ATIF spool files skipped during recovery",
858 )
859 .inc();
860 tracing::warn!(
861 path = %av_core::fsutil::basename(&path),
862 size = metadata.len(),
863 max = MAX_ATIF_RECOVERY_BYTES,
864 "ignoring oversize ATIF spool file",
865 );
866 continue;
867 }
868 let bytes = match read_capped_async(path.clone(), av_core::fsutil::MAX_ATIF_BYTES).await {
876 Ok(bytes) => bytes,
877 Err(error) => {
878 self.metrics
879 .counter(
880 "av_atif_recovery_skipped_total{reason=\"read_error\"}",
881 "ATIF spool files skipped during recovery",
882 )
883 .inc();
884 tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "skipping unreadable ATIF spool file");
885 continue;
886 }
887 };
888 let trajectory: av_atif::Trajectory = match serde_json::from_slice(&bytes) {
889 Ok(trajectory) => trajectory,
890 Err(error) => {
891 self.metrics
892 .counter(
893 "av_atif_recovery_skipped_total{reason=\"invalid_json\"}",
894 "ATIF spool files skipped during recovery",
895 )
896 .inc();
897 tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "ignoring invalid ATIF spool file");
898 continue;
899 }
900 };
901 if !av_atif::validate_trajectory(&trajectory, av_atif::Mode::Strict).is_empty() {
902 self.metrics
903 .counter(
904 "av_atif_recovery_skipped_total{reason=\"nonconformant\"}",
905 "ATIF spool files skipped during recovery",
906 )
907 .inc();
908 tracing::warn!(path = %av_core::fsutil::basename(&path), "ignoring nonconformant ATIF spool file");
909 continue;
910 }
911 let Some(session_id) = trajectory.session_id.clone() else {
912 continue;
913 };
914 if let Err(error) = self.ensure_atif_provenance(&path, &session_id).await {
919 self.metrics
920 .counter(
921 "av_atif_recovery_skipped_total{reason=\"provenance\"}",
922 "ATIF spool files skipped during recovery",
923 )
924 .inc();
925 if self.warn_once(path.clone()) {
926 tracing::warn!(
927 %error,
928 path = %av_core::fsutil::basename(&path),
929 "ignoring ATIF spool file whose provenance does not verify"
930 );
931 }
932 continue;
933 }
934 if sessions.get(&session_id).is_some() {
935 continue;
936 }
937 enum AtifCandidateOutcome {
949 Recovered,
950 Skipped,
951 }
952 let outcome: Result<AtifCandidateOutcome, FinalizeError> = async {
953 let _lifecycle = self.acquire_lifecycle(&session_id).await;
957 let extra = trajectory.agent.extra.as_ref();
958 let instance_uid = extra
959 .and_then(|value| value.get("instance_uid"))
960 .and_then(serde_json::Value::as_str)
961 .unwrap_or("recovered")
962 .to_owned();
963 let charter = extra
964 .and_then(|value| value.get("charter"))
965 .and_then(|value| {
966 serde_json::from_value::<av_events::CharterFile>(value.clone())
967 .ok()
968 .or_else(|| value.as_str().map(Into::into))
969 })
970 .unwrap_or_else(|| "recovered".into());
971 let ttl_remaining_s = extra
972 .and_then(|value| value.get("ttl_remaining_s"))
973 .and_then(serde_json::Value::as_u64);
974 let recovered_session = match sessions.try_insert_recovered(
975 Session::recover_unsigned(
976 session_id.clone(),
977 av_events::AgentIdentity {
978 version: trajectory.agent.version.clone(),
979 charter,
980 instance_uid,
981 ttl_remaining_s,
982 },
983 breaker.clone(),
984 path.clone(),
985 trajectory.final_metrics.as_ref(),
986 )
987 .map_err(FinalizeError::Atif)?,
988 ) {
989 Ok(inserted) => inserted,
990 Err(_active) => {
991 tracing::info!(session = %av_core::fsutil::basename(&path), "unsigned recovery skipped: session already active");
992 return Ok(AtifCandidateOutcome::Skipped);
993 }
994 };
995 let receipt_path = self.receipt_path(&recovered_session.id);
996 match tokio::fs::metadata(&receipt_path).await {
1003 Ok(_) => {
1004 let bytes = read_capped_async(
1005 receipt_path.clone(),
1006 av_core::fsutil::MAX_RECEIPT_BYTES,
1007 )
1008 .await
1009 .map_err(|error| {
1010 FinalizeError::Receipt(format!("existing receipt unreadable: {error}"))
1011 })?;
1012 let receipt = Receipt::from_json_slice(&bytes)
1023 .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
1024 self.verify_configured_receipt(&receipt)?;
1025 if path.with_extension("promote").exists() {
1026 recovered_session.restore_pending_receipt(receipt);
1027 } else {
1028 recovered_session.restore_receipt(receipt);
1029 }
1030 }
1031 Err(error) if error.kind() == std::io::ErrorKind::NotFound
1032 || error.kind() == std::io::ErrorKind::NotADirectory =>
1033 {
1034 }
1036 Err(error) => {
1037 return Err(FinalizeError::Receipt(format!(
1038 "existing receipt stat failed: {error}"
1039 )));
1040 }
1041 }
1042 Ok(AtifCandidateOutcome::Recovered)
1043 }.await;
1044 match outcome {
1045 Ok(AtifCandidateOutcome::Recovered) => recovered += 1,
1046 Ok(AtifCandidateOutcome::Skipped) => {}
1047 Err(error) => {
1048 self.metrics
1049 .counter(
1050 "av_atif_trajectory_recovery_skipped_total",
1051 "ATIF trajectories skipped during recovery due to per-session errors (round-42 F1)",
1052 )
1053 .inc();
1054 if self.warn_once(path.clone()) {
1055 tracing::warn!(
1056 %error,
1057 path = %av_core::fsutil::basename(&path),
1058 "skipping ATIF trajectory recovery due to per-session error; other sessions continue"
1059 );
1060 }
1061 }
1062 }
1063 }
1064 self.remove_acked_lifecycle_outboxes().await?;
1065 Ok(recovered + signed_recovered)
1066 }
1067
1068 async fn recover_signed_journals(
1081 &self,
1082 sessions: &SessionRegistry,
1083 breaker: &av_loopdetect::BreakerConfig,
1084 ) -> Result<usize, FinalizeError> {
1085 enum SignedCandidateOutcome {
1089 Recovered,
1093 Skipped,
1097 }
1098 let mut entries = match tokio::fs::read_dir(&self.spool_dir).await {
1099 Ok(entries) => entries,
1100 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
1101 Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1102 };
1103 let mut recovered = 0usize;
1104 while let Some(entry) = entries
1105 .next_entry()
1106 .await
1107 .map_err(|error| FinalizeError::Atif(error.to_string()))?
1108 {
1109 let metadata_path = entry.path();
1110 let Some(name) = metadata_path.file_name().and_then(std::ffi::OsStr::to_str) else {
1111 continue;
1112 };
1113 let Some(stem) = name.strip_suffix(".session.json") else {
1114 continue;
1115 };
1116 let outcome: Result<SignedCandidateOutcome, FinalizeError> = async {
1124 let metadata = self.read_journal_metadata(&metadata_path).await?;
1125 if metadata
1126 .get("journal_version")
1127 .and_then(serde_json::Value::as_u64)
1128 != Some(2)
1129 {
1130 if self.warn_once(metadata_path.clone()) {
1145 tracing::warn!(
1146 path = %av_core::fsutil::basename(&metadata_path),
1147 version = ?metadata.get("journal_version"),
1148 "sidecar has unsupported journal_version; skipping this session so recovery can proceed for the rest"
1149 );
1150 }
1151 return Ok(SignedCandidateOutcome::Skipped);
1152 }
1153 if metadata.get("workflow").and_then(serde_json::Value::as_str) != Some(Workflow::Signed.as_str())
1154 {
1155 return Ok(SignedCandidateOutcome::Skipped);
1156 }
1157 let session_id = metadata
1158 .get("session_id")
1159 .and_then(serde_json::Value::as_str)
1160 .ok_or_else(|| FinalizeError::Atif("journal metadata has no session_id".into()))?;
1161 if sessions.get(session_id).is_some() {
1162 return Ok(SignedCandidateOutcome::Skipped);
1163 }
1164 let _lifecycle = self.acquire_lifecycle(session_id).await;
1169 let identity: av_events::AgentIdentity = serde_json::from_value(
1170 metadata
1171 .get("identity")
1172 .cloned()
1173 .ok_or_else(|| FinalizeError::Atif("journal metadata has no identity".into()))?,
1174 )
1175 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1176 let journal_path = self.spool_dir.join(format!("{stem}.events.ndjson"));
1177 let journal = if journal_path.exists() {
1178 read_complete_journal(&journal_path).await?
1179 } else {
1180 Vec::new()
1181 };
1182 if journal.is_empty() {
1183 if quarantine_sibling_exists(&self.spool_dir, stem).await? {
1190 let quarantine_metadata = self
1191 .spool_dir
1192 .join(format!("{stem}.session.json.corrupt-{}", av_core::new_event_uid()));
1193 match tokio::fs::rename(&metadata_path, &quarantine_metadata).await {
1194 Ok(()) => tracing::warn!(
1195 metadata = %av_core::fsutil::basename(&metadata_path),
1196 quarantine = %av_core::fsutil::basename(&quarantine_metadata),
1197 "sealed metadata sidecar quarantined alongside its torn signed journal (round-14 F5)"
1198 ),
1199 Err(error) => tracing::error!(
1200 metadata = %av_core::fsutil::basename(&metadata_path),
1201 %error,
1202 "failed to quarantine metadata sidecar; leaving in place so a future recovery can try again"
1203 ),
1204 }
1205 return Ok(SignedCandidateOutcome::Skipped);
1206 }
1207 tokio::fs::remove_file(metadata_path.clone())
1208 .await
1209 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1210 return Ok(SignedCandidateOutcome::Skipped);
1211 }
1212 let session = Arc::new(Session::new(
1213 session_id.to_owned(),
1214 Workflow::Signed,
1215 identity,
1216 breaker.clone(),
1217 ));
1218 let mut next_sequence = 0u64;
1219 let mut tool_calls = 0u64;
1220 let mut tool_allowed = 0u64;
1221 let mut tool_blocked = 0u64;
1222 let mut prompt_tokens = 0u64;
1223 let mut completion_tokens = 0u64;
1224 let mut cached_tokens = 0u64;
1225 let mut cost_usd_micros = 0u64;
1226 let mut pending_responses = std::collections::HashSet::new();
1227 let domain = format!("{}:active", session.id);
1228 for (index, line) in journal.into_iter().enumerate() {
1229 let index = u64::try_from(index)
1230 .map_err(|_| FinalizeError::Atif("active journal index overflow".to_owned()))?;
1231 let record: crate::worker::ActiveJournalRecord =
1232 crate::journal::open(&self.journal_key, &domain, index, line.as_bytes())
1233 .map_err(FinalizeError::Atif)?;
1234 let event: av_events::OcsfEvent = serde_json::from_value(record.event.clone())
1235 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1236 if event.session_uid != session.id {
1237 return Err(FinalizeError::Atif(format!(
1238 "signed journal event belongs to session {:?}, expected {:?}",
1239 event.session_uid, session.id
1240 )));
1241 }
1242 if record.atif_step.is_some() || record.identity != event.ai_agent {
1243 return Err(FinalizeError::Atif(
1244 "signed active record has inconsistent workflow or identity".to_owned(),
1245 ));
1246 }
1247 track_response_attempt(&mut pending_responses, record.response_attempt.as_ref())?;
1248 if event.metadata.sequence != index {
1249 return Err(FinalizeError::Atif(
1250 "signed event sequence does not match active journal index".to_owned(),
1251 ));
1252 }
1253 if record.identity.version != session.identity.version
1254 || record.identity.charter != session.identity.charter
1255 || record.identity.instance_uid != session.identity.instance_uid
1256 {
1257 return Err(FinalizeError::Atif(
1258 "active journal changed the session identity".to_owned(),
1259 ));
1260 }
1261 session.refresh_identity(&record.identity);
1262 next_sequence = index
1263 .checked_add(1)
1264 .ok_or_else(|| FinalizeError::Atif("event sequence overflow".to_owned()))?;
1265 session
1266 .chain
1267 .lock()
1268 .append(&record.event)
1269 .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
1270 tool_calls = checked_recovery_add(tool_calls, record.tool_calls, "tool calls")?;
1271 tool_allowed = checked_recovery_add(tool_allowed, record.tool_allowed, "allowed tools")?;
1272 tool_blocked = checked_recovery_add(tool_blocked, record.tool_blocked, "blocked tools")?;
1273 prompt_tokens = checked_recovery_add(prompt_tokens, record.prompt_tokens, "prompt tokens")?;
1274 completion_tokens =
1275 checked_recovery_add(completion_tokens, record.completion_tokens, "completion tokens")?;
1276 cached_tokens = checked_recovery_add(cached_tokens, record.cached_tokens, "cached tokens")?;
1277 cost_usd_micros = checked_recovery_add(cost_usd_micros, record.cost_usd_micros, "cost")?;
1278 if let Some(id) = record.stop_reason_id {
1279 let reason = av_events::StopReason::from_id(id);
1280 if reason != av_events::StopReason::Unknown {
1281 session.record_stop_reason(reason);
1282 }
1283 }
1284 self.ensure_active_event_published(&session.id, &event, &record.event)
1285 .await?;
1286 }
1287 if tool_allowed
1288 .checked_add(tool_blocked)
1289 .is_none_or(|classified| classified > tool_calls)
1290 {
1291 return Err(FinalizeError::Atif(
1292 "signed journal has inconsistent tool accounting".to_owned(),
1293 ));
1294 }
1295 session
1296 .totals
1297 .tool_calls
1298 .store(tool_calls, std::sync::atomic::Ordering::Release);
1299 session
1300 .totals
1301 .tool_allowed
1302 .store(tool_allowed, std::sync::atomic::Ordering::Release);
1303 session
1304 .totals
1305 .tool_blocked
1306 .store(tool_blocked, std::sync::atomic::Ordering::Release);
1307 session
1308 .totals
1309 .prompt_tokens
1310 .store(prompt_tokens, std::sync::atomic::Ordering::Release);
1311 session
1312 .totals
1313 .completion_tokens
1314 .store(completion_tokens, std::sync::atomic::Ordering::Release);
1315 session
1316 .totals
1317 .cached_tokens
1318 .store(cached_tokens, std::sync::atomic::Ordering::Release);
1319 session
1320 .totals
1321 .cost_usd_micros
1322 .store(cost_usd_micros, std::sync::atomic::Ordering::Release);
1323 let inconsistent_responses = !pending_responses.is_empty();
1324 session.restore_next_seq(next_sequence);
1325 let expected_subject = {
1326 let chain = session.chain.lock();
1327 ReceiptSubject::EventChain {
1328 chain_head: chain.head_hex(),
1329 event_count: chain.count(),
1330 }
1331 };
1332 let receipt_path = self.receipt_path(&session.id);
1333 match tokio::fs::metadata(&receipt_path).await {
1348 Ok(_) => {
1349 let bytes = read_capped_async(
1350 receipt_path.clone(),
1351 av_core::fsutil::MAX_RECEIPT_BYTES,
1352 )
1353 .await
1354 .map_err(|error| {
1355 FinalizeError::Receipt(format!(
1356 "existing receipt unreadable: {error}"
1357 ))
1358 })?;
1359 let receipt = Receipt::from_json_slice(&bytes)
1363 .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
1364 self.verify_configured_receipt(&receipt)?;
1365 if receipt.body.subject != expected_subject {
1366 return Err(FinalizeError::Receipt(
1367 "persisted receipt does not attest the recovered signed journal".to_owned(),
1368 ));
1369 }
1370 *session.receipt.lock() = Some(receipt);
1371 }
1372 Err(error) if error.kind() == std::io::ErrorKind::NotFound
1373 || error.kind() == std::io::ErrorKind::NotADirectory =>
1374 {
1375 }
1381 Err(error) => {
1382 return Err(FinalizeError::Receipt(format!(
1383 "existing receipt stat failed: {error}"
1384 )));
1385 }
1386 }
1387 let unwrapped = Arc::try_unwrap(session)
1388 .map_err(|_| FinalizeError::Task("signed recovery retained session".to_owned()))?;
1389 unwrapped.mark_artifact_committed();
1405 let session = match sessions.try_insert_recovered(unwrapped) {
1406 Ok(inserted) => inserted,
1407 Err(_active) => {
1408 tracing::info!(session = %session_id, "signed recovery skipped: session already active");
1409 return Ok(SignedCandidateOutcome::Skipped);
1410 }
1411 };
1412 if inconsistent_responses {
1413 self.quarantined_sessions.lock().insert(session.id.clone());
1416 session.mark_capture_failed();
1417 return Ok(SignedCandidateOutcome::Recovered);
1418 }
1419 if self.quarantined_sessions.lock().contains(&session.id) {
1420 session.mark_capture_failed();
1421 return Ok(SignedCandidateOutcome::Recovered);
1422 }
1423 match self
1440 .close_session_locked(Arc::clone(&session), StopReason::SessionClosed)
1441 .await
1442 {
1443 Ok(_) => Ok(SignedCandidateOutcome::Recovered),
1444 Err(FinalizeError::CaptureIncomplete) => {
1445 Err(FinalizeError::CaptureIncomplete)
1446 }
1447 Err(error) => {
1448 sessions.remove(&session.id);
1449 Err(error)
1450 }
1451 }
1452 }.await;
1453 match outcome {
1454 Ok(SignedCandidateOutcome::Recovered) => recovered += 1,
1455 Ok(SignedCandidateOutcome::Skipped) => {}
1456 Err(error) => {
1457 self.metrics
1458 .counter(
1459 "av_signed_recovery_skipped_total",
1460 "Signed sessions skipped during recovery due to per-session errors (round-41 F1)",
1461 )
1462 .inc();
1463 if self.warn_once(metadata_path.clone()) {
1464 tracing::warn!(
1465 %error,
1466 path = %av_core::fsutil::basename(&metadata_path),
1467 "skipping signed session recovery due to per-session error; other sessions continue"
1468 );
1469 }
1470 }
1471 }
1472 }
1473 Ok(recovered)
1474 }
1475
1476 async fn ensure_active_event_published(
1477 &self,
1478 session_id: &str,
1479 event: &av_events::OcsfEvent,
1480 value: &serde_json::Value,
1481 ) -> Result<(), FinalizeError> {
1482 let topic = event.class_name.topic();
1483 let event_uid = &event.metadata.uid;
1484 if let Some(ack) =
1485 crate::worker::read_broker_ack(&self.spool_dir, session_id, event_uid, &self.journal_key)
1486 .await
1487 .map_err(FinalizeError::Bridge)?
1488 {
1489 if ack.topic != topic {
1490 return Err(FinalizeError::Bridge(
1491 "broker acknowledgment topic does not match active event".to_owned(),
1492 ));
1493 }
1494 return Ok(());
1495 }
1496 let bridge = self.bridge.as_ref().map(Arc::clone).ok_or_else(|| {
1497 FinalizeError::Bridge("unacknowledged active event has no configured broker".to_owned())
1498 })?;
1499 let topic = topic.to_owned();
1500 let key = event.ai_agent.instance_uid.clone();
1501 let value = value.clone();
1502 let uid = event_uid.clone();
1503 let lookup_bridge = Arc::clone(&bridge);
1504 let lookup_topic = topic.clone();
1505 let lookup_key = key.clone();
1506 let lookup_uid = uid.clone();
1507 if let Some(ack) = tokio::task::spawn_blocking(move || {
1508 lookup_bridge.find_event_by_uid(&lookup_topic, &lookup_key, &lookup_uid)
1509 })
1510 .await
1511 .map_err(|error| FinalizeError::Task(error.to_string()))?
1512 .map_err(|error| FinalizeError::Bridge(error.to_string()))?
1513 {
1514 crate::worker::persist_broker_ack(
1515 &self.spool_dir,
1516 session_id,
1517 event_uid,
1518 &ack,
1519 &self.journal_key,
1520 )
1521 .await
1522 .map_err(FinalizeError::Bridge)?;
1523 return Ok(());
1524 }
1525 let ack = tokio::task::spawn_blocking(move || bridge.publish_idempotent(&topic, &key, &value, &uid))
1526 .await
1527 .map_err(|error| FinalizeError::Task(error.to_string()))?
1528 .map_err(|error| FinalizeError::Bridge(error.to_string()))?;
1529 crate::worker::persist_broker_ack(&self.spool_dir, session_id, event_uid, &ack, &self.journal_key)
1530 .await
1531 .map_err(FinalizeError::Bridge)
1532 }
1533
1534 async fn consolidate_step_journals(
1535 &self,
1536 sessions: &SessionRegistry,
1537 breaker: &av_loopdetect::BreakerConfig,
1538 ) -> Result<(), FinalizeError> {
1539 let mut entries = match tokio::fs::read_dir(&self.spool_dir).await {
1540 Ok(entries) => entries,
1541 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
1542 Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1543 };
1544 while let Some(entry) = entries
1545 .next_entry()
1546 .await
1547 .map_err(|error| FinalizeError::Atif(error.to_string()))?
1548 {
1549 let metadata_path = entry.path();
1550 let Some(name) = metadata_path.file_name().and_then(std::ffi::OsStr::to_str) else {
1551 continue;
1552 };
1553 let Some(stem) = name.strip_suffix(".session.json") else {
1554 continue;
1555 };
1556 let outcome: Result<(), FinalizeError> = async {
1563 let final_path = self.spool_dir.join(format!("{stem}.json"));
1564 let journal_path = self.spool_dir.join(format!("{stem}.events.ndjson"));
1565 let metadata = self.read_journal_metadata(&metadata_path).await?;
1566 if metadata
1567 .get("journal_version")
1568 .and_then(serde_json::Value::as_u64)
1569 != Some(2)
1570 {
1571 if self.warn_once(metadata_path.clone()) {
1576 tracing::warn!(
1577 path = %av_core::fsutil::basename(&metadata_path),
1578 version = ?metadata.get("journal_version"),
1579 "sidecar has unsupported journal_version; skipping this session so recovery can proceed for the rest"
1580 );
1581 }
1582 return Ok(());
1583 }
1584 if metadata.get("workflow").and_then(serde_json::Value::as_str) == Some(Workflow::Signed.as_str())
1585 {
1586 return Ok(());
1587 }
1588 let session_id = metadata
1589 .get("session_id")
1590 .and_then(serde_json::Value::as_str)
1591 .ok_or_else(|| FinalizeError::Atif("journal metadata has no session_id".into()))?;
1592 if sessions.get(session_id).is_some() {
1593 return Ok(());
1594 }
1595 let _lifecycle = self.acquire_lifecycle(session_id).await;
1600 let identity: av_events::AgentIdentity = serde_json::from_value(
1601 metadata
1602 .get("identity")
1603 .cloned()
1604 .ok_or_else(|| FinalizeError::Atif("journal metadata has no identity".into()))?,
1605 )
1606 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1607 let journal = if journal_path.exists() {
1608 read_complete_journal(&journal_path).await?
1609 } else {
1610 Vec::new()
1611 };
1612 if self.quarantined_sessions.lock().contains(session_id) {
1613 let quarantined = Session::new(
1614 session_id.to_owned(),
1615 Workflow::Unsigned,
1616 identity.clone(),
1617 breaker.clone(),
1618 );
1619 quarantined.restore_journal_index(
1620 u64::try_from(journal.len())
1621 .map_err(|_| FinalizeError::Atif("active journal length overflow".to_owned()))?,
1622 );
1623 quarantined.mark_capture_failed();
1624 quarantined.mark_artifact_committed();
1630 sessions.insert_recovered(quarantined);
1631 return Ok(());
1632 }
1633 if journal.is_empty() {
1634 if quarantine_sibling_exists(&self.spool_dir, stem).await? {
1639 let quarantine_metadata = self
1640 .spool_dir
1641 .join(format!("{stem}.session.json.corrupt-{}", av_core::new_event_uid()));
1642 match tokio::fs::rename(&metadata_path, &quarantine_metadata).await {
1643 Ok(()) => tracing::warn!(
1644 metadata = %av_core::fsutil::basename(&metadata_path),
1645 quarantine = %av_core::fsutil::basename(&quarantine_metadata),
1646 "sealed metadata sidecar quarantined alongside its torn unsigned journal (round-14 F5)"
1647 ),
1648 Err(error) => tracing::error!(
1649 metadata = %av_core::fsutil::basename(&metadata_path),
1650 %error,
1651 "failed to quarantine metadata sidecar; leaving in place so a future recovery can try again"
1652 ),
1653 }
1654 return Ok(());
1655 }
1656 tokio::fs::remove_file(metadata_path.clone())
1657 .await
1658 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1659 return Ok(());
1660 }
1661 let journal_len = u64::try_from(journal.len())
1662 .map_err(|_| FinalizeError::Atif("active journal length overflow".to_owned()))?;
1663 let agent = av_atif::Agent {
1664 name: "agentvisor-ai-harness".into(),
1665 version: identity.version.clone(),
1666 model_name: None,
1667 tool_definitions: None,
1668 extra: Some(serde_json::json!({
1669 "charter": identity.charter,
1670 "instance_uid": identity.instance_uid,
1671 })),
1672 };
1673 let mut builder = av_atif::TrajectoryBuilder::new(agent, Some(session_id.to_owned()));
1674 let domain = format!("{session_id}:active");
1675 let mut latest_identity = identity.clone();
1676 let mut prompt_tokens = 0u64;
1677 let mut completion_tokens = 0u64;
1678 let mut cached_tokens = 0u64;
1679 let mut cost_usd_micros = 0u64;
1680 let mut tool_calls = 0u64;
1681 let mut tool_allowed = 0u64;
1682 let mut tool_blocked = 0u64;
1683 let mut stop_reason_id = None;
1684 let mut pending_responses = std::collections::HashSet::new();
1685 for (index, line) in journal.into_iter().enumerate() {
1686 let index = u64::try_from(index)
1687 .map_err(|_| FinalizeError::Atif("active journal index overflow".to_owned()))?;
1688 let record: crate::worker::ActiveJournalRecord =
1689 crate::journal::open(&self.journal_key, &domain, index, line.as_bytes())
1690 .map_err(FinalizeError::Atif)?;
1691 let event: av_events::OcsfEvent = serde_json::from_value(record.event.clone())
1692 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1693 if event.session_uid != session_id || event.ai_agent != record.identity {
1694 return Err(FinalizeError::Atif(
1695 "unsigned active record has inconsistent session or identity".to_owned(),
1696 ));
1697 }
1698 track_response_attempt(&mut pending_responses, record.response_attempt.as_ref())?;
1699 if event.metadata.sequence != index {
1700 return Err(FinalizeError::Atif(
1701 "unsigned event sequence does not match active journal index".to_owned(),
1702 ));
1703 }
1704 if record.identity.version != identity.version
1705 || record.identity.charter != identity.charter
1706 || record.identity.instance_uid != identity.instance_uid
1707 {
1708 return Err(FinalizeError::Atif(
1709 "active journal changed the unsigned session identity".to_owned(),
1710 ));
1711 }
1712 self.ensure_active_event_published(session_id, &event, &record.event)
1713 .await?;
1714 let step = record.atif_step.ok_or_else(|| {
1715 FinalizeError::Atif("unsigned active record has no ATIF step".to_owned())
1716 })?;
1717 builder
1718 .push_step(step)
1719 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1720 latest_identity = record.identity;
1721 prompt_tokens = checked_recovery_add(prompt_tokens, record.prompt_tokens, "prompt tokens")?;
1722 completion_tokens =
1723 checked_recovery_add(completion_tokens, record.completion_tokens, "completion tokens")?;
1724 cached_tokens = checked_recovery_add(cached_tokens, record.cached_tokens, "cached tokens")?;
1725 cost_usd_micros = checked_recovery_add(cost_usd_micros, record.cost_usd_micros, "cost")?;
1726 tool_calls = checked_recovery_add(tool_calls, record.tool_calls, "tool calls")?;
1727 tool_allowed = checked_recovery_add(tool_allowed, record.tool_allowed, "allowed tools")?;
1728 tool_blocked = checked_recovery_add(tool_blocked, record.tool_blocked, "blocked tools")?;
1729 if record.stop_reason_id.is_some() {
1730 stop_reason_id = record.stop_reason_id;
1731 }
1732 }
1733 if tool_allowed
1734 .checked_add(tool_blocked)
1735 .is_none_or(|classified| classified > tool_calls)
1736 {
1737 return Err(FinalizeError::Atif(
1738 "unsigned journal has inconsistent tool accounting".to_owned(),
1739 ));
1740 }
1741 if !pending_responses.is_empty() {
1742 let quarantined = Session::new(
1743 session_id.to_owned(),
1744 Workflow::Unsigned,
1745 latest_identity,
1746 breaker.clone(),
1747 );
1748 quarantined.restore_journal_index(journal_len);
1749 quarantined.restore_next_seq(journal_len);
1750 quarantined
1751 .totals
1752 .tool_calls
1753 .store(tool_calls, std::sync::atomic::Ordering::Release);
1754 quarantined
1755 .totals
1756 .tool_allowed
1757 .store(tool_allowed, std::sync::atomic::Ordering::Release);
1758 quarantined
1759 .totals
1760 .tool_blocked
1761 .store(tool_blocked, std::sync::atomic::Ordering::Release);
1762 quarantined
1763 .totals
1764 .prompt_tokens
1765 .store(prompt_tokens, std::sync::atomic::Ordering::Release);
1766 quarantined
1767 .totals
1768 .completion_tokens
1769 .store(completion_tokens, std::sync::atomic::Ordering::Release);
1770 quarantined
1771 .totals
1772 .cached_tokens
1773 .store(cached_tokens, std::sync::atomic::Ordering::Release);
1774 quarantined
1775 .totals
1776 .cost_usd_micros
1777 .store(cost_usd_micros, std::sync::atomic::Ordering::Release);
1778 quarantined.mark_capture_failed();
1779 quarantined.mark_artifact_committed();
1783 match sessions.try_insert_recovered(quarantined) {
1786 Ok(_) => {
1787 self.quarantined_sessions.lock().insert(session_id.to_owned());
1788 }
1789 Err(_active) => {
1790 tracing::info!(
1791 session = %session_id,
1792 "unsigned quarantine skipped: session already active",
1793 );
1794 }
1795 }
1796 return Ok(());
1797 }
1798 let mut trajectory = builder.finish();
1799 trajectory.agent.extra = Some(serde_json::json!({
1800 "charter": latest_identity.charter,
1801 "instance_uid": latest_identity.instance_uid,
1802 "ttl_remaining_s": latest_identity.ttl_remaining_s,
1803 }));
1804 if let Some(metrics) = trajectory.final_metrics.as_mut() {
1805 metrics.total_prompt_tokens = Some(prompt_tokens);
1806 metrics.total_completion_tokens = Some(completion_tokens);
1807 metrics.total_cached_tokens = Some(cached_tokens);
1808 metrics.total_cost_usd =
1809 Some(cost_usd_micros as f64 / av_core::units::USD_MICROS_PER_DOLLAR as f64);
1810 metrics.extra = Some(serde_json::json!({
1811 "tool_calls": tool_calls,
1812 "tool_allowed": tool_allowed,
1813 "tool_blocked": tool_blocked,
1814 "cost_usd_micros": cost_usd_micros,
1815 "stop_reason_id": stop_reason_id.map_or(0u64, u64::from),
1829 }));
1830 }
1831 if final_path.exists() {
1832 let existing: av_atif::Trajectory = serde_json::from_slice(
1833 &read_capped_async(final_path.clone(), av_core::fsutil::MAX_ATIF_BYTES)
1834 .await
1835 .map_err(|error| FinalizeError::Atif(error.to_string()))?,
1836 )
1837 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1838 trajectory.trajectory_id.clone_from(&existing.trajectory_id);
1839 if trajectory != existing {
1840 return Err(FinalizeError::Atif(
1841 "persisted ATIF does not match authenticated active journal".to_owned(),
1842 ));
1843 }
1844 } else {
1845 let write_path = final_path.clone();
1846 tokio::task::spawn_blocking(move || av_atif::write_atomic(&trajectory, &write_path))
1847 .await
1848 .map_err(|error| FinalizeError::Task(error.to_string()))?
1849 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1850 }
1851 self.ensure_atif_provenance(&final_path, session_id).await?;
1852 self.remove_step_journal(session_id).await?;
1853 Ok(())
1854 }.await;
1855 if let Err(error) = outcome {
1856 self.metrics
1857 .counter(
1858 "av_unsigned_recovery_skipped_total",
1859 "Unsigned step-journal consolidations skipped during recovery due to per-session errors (round-41 F1)",
1862 )
1863 .inc();
1864 if self.warn_once(metadata_path.clone()) {
1865 tracing::warn!(
1866 %error,
1867 path = %av_core::fsutil::basename(&metadata_path),
1868 "skipping unsigned session consolidation due to per-session error; other sessions continue"
1869 );
1870 }
1871 }
1872 }
1873 Ok(())
1874 }
1875
1876 async fn remove_step_journal(&self, session_id: &str) -> Result<(), FinalizeError> {
1877 let digest = av_core::digest::sha256_hex(session_id.as_bytes());
1878 let stem = digest.get(..32).unwrap_or(&digest).to_owned();
1879 let spool_dir = self.spool_dir.clone();
1880 tokio::task::spawn_blocking(move || -> Result<(), FinalizeError> {
1881 let mut spool_changed = false;
1882 for suffix in ["session.json", "steps.ndjson", "events.ndjson"] {
1883 let path = spool_dir.join(format!("{stem}.{suffix}"));
1884 match std::fs::remove_file(&path) {
1885 Ok(()) => spool_changed = true,
1886 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1887 Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1888 }
1889 }
1890 if spool_changed {
1891 av_core::fsutil::sync_directory(&spool_dir)
1892 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1893 }
1894 let ack_parent = spool_dir.join("broker-acks");
1895 let ack_path = ack_parent.join(&stem);
1896 match std::fs::remove_dir_all(&ack_path) {
1897 Ok(()) => av_core::fsutil::sync_directory(&ack_parent)
1898 .map_err(|error| FinalizeError::Atif(error.to_string()))?,
1899 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1900 Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1901 }
1902 Ok(())
1903 })
1904 .await
1905 .map_err(|error| FinalizeError::Task(error.to_string()))?
1906 }
1907
1908 pub async fn retry_marked_promotions(&self, sessions: &SessionRegistry) -> Result<usize, FinalizeError> {
1910 let mut entries = match tokio::fs::read_dir(&self.spool_dir).await {
1911 Ok(entries) => entries,
1912 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
1913 Err(error) => return Err(FinalizeError::Atif(error.to_string())),
1914 };
1915 let mut promoted = 0usize;
1916 while let Some(entry) = entries
1917 .next_entry()
1918 .await
1919 .map_err(|error| FinalizeError::Atif(error.to_string()))?
1920 {
1921 let path = entry.path();
1922 if path.extension().and_then(std::ffi::OsStr::to_str) != Some("promote") {
1923 continue;
1924 }
1925 let sealed = match read_capped_async(path.clone(), av_core::fsutil::MAX_CONTROL_BYTES).await {
1933 Ok(bytes) => bytes,
1934 Err(error) => {
1935 tracing::warn!(
1936 %error,
1937 path = %av_core::fsutil::basename(&path),
1938 "skipping unreadable promotion marker"
1939 );
1940 continue;
1941 }
1942 };
1943 let marker: PromotionMarker =
1944 match crate::journal::open(&self.journal_key, "promotion-marker", 0, &sealed) {
1945 Ok(m) => m,
1946 Err(error) => {
1947 tracing::warn!(
1948 %error,
1949 path = %av_core::fsutil::basename(&path),
1950 "skipping unauthenticated promotion marker"
1951 );
1952 continue;
1953 }
1954 };
1955 let Some(session) = sessions.get(&marker.session_id) else {
1956 continue;
1957 };
1958 if !session.is_closed() {
1963 tracing::info!(
1964 session = %marker.session_id,
1965 "promotion retry skipped: session is currently active",
1966 );
1967 continue;
1968 }
1969 match self.promote(session).await {
1970 Ok(_) => promoted += 1,
1971 Err(error) => {
1972 tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "promotion retry failed");
1973 }
1974 }
1975 }
1976 Ok(promoted)
1977 }
1978
1979 fn receipt_path(&self, session_id: &str) -> PathBuf {
1980 self.spool_dir.join("receipts").join(format!(
1981 "{}.json",
1982 &av_core::digest::sha256_hex(session_id.as_bytes())[..32]
1983 ))
1984 }
1985
1986 async fn ensure_atif_provenance(
1987 &self,
1988 path: &std::path::Path,
1989 session_id: &str,
1990 ) -> Result<AtifProvenance, FinalizeError> {
1991 let bytes = read_capped_async(path.to_path_buf(), av_core::fsutil::MAX_ATIF_BYTES)
1994 .await
1995 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
1996 let expected = AtifProvenance {
1997 session_id: session_id.to_owned(),
1998 digest: av_core::digest::sha256_hex(&bytes),
1999 };
2000 let provenance_path = path.with_extension("atif-auth");
2001 if provenance_path.exists() {
2002 let sealed = read_capped_async(provenance_path.clone(), av_core::fsutil::MAX_CONTROL_BYTES)
2003 .await
2004 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2005 let actual: AtifProvenance =
2006 crate::journal::open(&self.journal_key, "atif-provenance", 0, &sealed)
2007 .map_err(FinalizeError::Atif)?;
2008 if actual.session_id != expected.session_id || actual.digest != expected.digest {
2009 return Err(FinalizeError::Atif(
2010 "ATIF provenance does not match artifact bytes and session".to_owned(),
2011 ));
2012 }
2013 return Ok(actual);
2014 }
2015 let sealed = crate::journal::seal(&self.journal_key, "atif-provenance", 0, &expected)
2016 .map_err(FinalizeError::Atif)?;
2017 persist_marker(&provenance_path, &sealed).await?;
2018 Ok(expected)
2019 }
2020
2021 async fn read_journal_metadata(
2022 &self,
2023 path: &std::path::Path,
2024 ) -> Result<serde_json::Value, FinalizeError> {
2025 let bytes = read_capped_async(path.to_path_buf(), av_core::fsutil::MAX_CONTROL_BYTES)
2029 .await
2030 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2031 crate::journal::open(&self.journal_key, "metadata", 0, &bytes).map_err(FinalizeError::Atif)
2032 }
2033
2034 fn verify_configured_receipt(&self, receipt: &Receipt) -> Result<(), FinalizeError> {
2035 if receipt.body.key_id != self.signer.key_id() {
2036 return Err(FinalizeError::Receipt(format!(
2037 "receipt key {:?} does not match configured key {:?}",
2038 receipt.body.key_id,
2039 self.signer.key_id()
2040 )));
2041 }
2042 let mut keyring = av_receipts::Keyring::new();
2043 keyring
2044 .add_signer(self.signer.as_ref())
2045 .map_err(|error| FinalizeError::Receipt(error.to_string()))?;
2046 receipt
2047 .verify(&keyring)
2048 .map_err(|error| FinalizeError::Receipt(error.to_string()))
2049 }
2050
2051 async fn persist_receipt(&self, session_id: &str, receipt: &Receipt) -> Result<(), FinalizeError> {
2052 let path = self.receipt_path(session_id);
2053 let bytes =
2054 serde_json::to_vec_pretty(receipt).map_err(|error| FinalizeError::Receipt(error.to_string()))?;
2055 tokio::task::spawn_blocking(move || av_core::fsutil::write_atomic(&path, &bytes))
2056 .await
2057 .map_err(|error| FinalizeError::Task(error.to_string()))?
2058 .map_err(|error| FinalizeError::Receipt(error.to_string()))
2059 }
2060
2061 async fn emit_receipt_event(&self, session: &Session, receipt: &Receipt) -> Result<(), FinalizeError> {
2062 self.emit_bridge_event(
2063 session,
2064 av_events::EventClass::Receipt,
2065 serde_json::json!({
2066 "receipt_id": receipt.body.receipt_id,
2067 "key_id": receipt.body.key_id,
2068 "subject": receipt.body.subject,
2069 "receipt": receipt,
2070 }),
2071 crate::journal::RECEIPT_OUTBOX_KIND,
2072 )
2073 .await
2074 }
2075
2076 async fn emit_bridge_event(
2077 &self,
2078 session: &Session,
2079 class: av_events::EventClass,
2080 payload: serde_json::Value,
2081 kind: &str,
2082 ) -> Result<(), FinalizeError> {
2083 let Some(bridge) = self.bridge.as_ref().map(Arc::clone) else {
2084 return Ok(());
2085 };
2086 let path = self.lifecycle_outbox_path(&session.id, kind);
2087 let mut outbox = if path.exists() {
2088 let sealed = read_capped_async(path.clone(), av_core::fsutil::MAX_CONTROL_BYTES)
2089 .await
2090 .map_err(|error| FinalizeError::Bridge(error.to_string()))?;
2091 let outbox: LifecycleOutbox = crate::journal::open(
2092 &self.journal_key,
2093 crate::journal::LIFECYCLE_OUTBOX_DOMAIN,
2094 0,
2095 &sealed,
2096 )
2097 .map_err(FinalizeError::Bridge)?;
2098 if outbox.session_id != session.id || outbox.kind != kind {
2099 return Err(FinalizeError::Bridge(
2100 "lifecycle outbox does not match its session and kind".to_owned(),
2101 ));
2102 }
2103 if let Some(persisted_seq) = outbox
2109 .value
2110 .get("metadata")
2111 .and_then(|metadata| metadata.get("sequence"))
2112 .and_then(serde_json::Value::as_u64)
2113 {
2114 if session.peek_seq() <= persisted_seq {
2115 session.advance_seq_past(persisted_seq);
2116 }
2117 }
2118 outbox
2119 } else {
2120 let event_seq = session.peek_seq();
2125 let event = av_events::OcsfEventBuilder::new(
2126 class,
2127 session.id.clone(),
2128 session.current_identity(),
2129 event_seq,
2130 )
2131 .payload(payload)
2132 .build()
2133 .map_err(|error| FinalizeError::Bridge(error.to_string()))?;
2134 let outbox = LifecycleOutbox {
2135 session_id: session.id.clone(),
2136 kind: kind.to_owned(),
2137 topic: class.topic().to_owned(),
2138 key: session.current_identity().instance_uid,
2139 value: serde_json::to_value(event)
2140 .map_err(|error| FinalizeError::Bridge(error.to_string()))?,
2141 ack: None,
2142 };
2143 persist_outbox(&path, &outbox, &self.journal_key).await?;
2144 session.advance_seq_past(event_seq);
2145 outbox
2146 };
2147 if outbox.ack.is_some() {
2148 return Ok(());
2149 }
2150 let topic = outbox.topic.clone();
2151 let key = outbox.key.clone();
2152 let value = outbox.value.clone();
2153 let event_uid = lifecycle_event_uid(&value)?;
2154 let ack = match resolve_lifecycle_ack(bridge, topic, key, value, event_uid).await {
2155 Ok(ack) => ack,
2156 Err(error) => {
2157 self.metrics
2158 .counter(
2159 "av_lifecycle_event_errors_total",
2160 "Lifecycle events not published",
2161 )
2162 .inc();
2163 return Err(error);
2164 }
2165 };
2166 outbox.ack = Some(ack);
2167 persist_outbox(&path, &outbox, &self.journal_key).await?;
2168 Ok(())
2169 }
2170
2171 async fn replay_lifecycle_outboxes(&self) -> Result<usize, FinalizeError> {
2172 let Some(bridge) = self.bridge.as_ref().map(Arc::clone) else {
2173 return Ok(0);
2174 };
2175 let directory = self.spool_dir.join(crate::spool::OUTBOX);
2176 let mut entries = match tokio::fs::read_dir(&directory).await {
2177 Ok(entries) => entries,
2178 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
2179 Err(error) => return Err(FinalizeError::Bridge(error.to_string())),
2180 };
2181 let mut replayed = 0usize;
2182 while let Some(entry) = entries
2183 .next_entry()
2184 .await
2185 .map_err(|error| FinalizeError::Bridge(error.to_string()))?
2186 {
2187 let path = entry.path();
2188 if path.extension().and_then(std::ffi::OsStr::to_str) != Some("json") {
2189 continue;
2190 }
2191 let sealed = match read_capped_async(path.clone(), av_core::fsutil::MAX_CONTROL_BYTES).await {
2192 Ok(bytes) => bytes,
2193 Err(error) => {
2194 tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "skipping unreadable outbox file");
2195 continue;
2196 }
2197 };
2198 let mut outbox: LifecycleOutbox = match crate::journal::open(
2199 &self.journal_key,
2200 crate::journal::LIFECYCLE_OUTBOX_DOMAIN,
2201 0,
2202 &sealed,
2203 ) {
2204 Ok(outbox) => outbox,
2205 Err(error) => {
2210 tracing::warn!(%error, path = %av_core::fsutil::basename(&path), "skipping malformed outbox");
2211 continue;
2212 }
2213 };
2214 if path != self.lifecycle_outbox_path(&outbox.session_id, &outbox.kind) {
2215 tracing::warn!(
2216 path = %av_core::fsutil::basename(&path),
2217 "skipping outbox whose filename does not match its authenticated session_id/kind"
2218 );
2219 continue;
2220 }
2221 if outbox.ack.is_none() {
2222 let topic = outbox.topic.clone();
2223 let key = outbox.key.clone();
2224 let value = outbox.value.clone();
2225 let event_uid = lifecycle_event_uid(&value)?;
2226 outbox.ack =
2227 Some(resolve_lifecycle_ack(Arc::clone(&bridge), topic, key, value, event_uid).await?);
2228 persist_outbox(&path, &outbox, &self.journal_key).await?;
2229 replayed = replayed.saturating_add(1);
2230 }
2231 }
2232 Ok(replayed)
2233 }
2234
2235 pub(crate) async fn complete_pending_closes(
2259 &self,
2260 sessions: &SessionRegistry,
2261 ) -> Result<usize, FinalizeError> {
2262 let pending = sessions.pending_close_sessions();
2263 let mut completed = 0usize;
2264 for session in pending {
2265 let _lifecycle = self.acquire_lifecycle(&session.id).await;
2286 if !session.artifact_committed_flag() || session.close_complete_flag() {
2291 continue;
2292 }
2293 let workflow = session.workflow.as_str();
2294 let outcome: Result<(), FinalizeError> = async {
2295 self.emit_bridge_event(
2296 &session,
2297 av_events::EventClass::Session,
2298 serde_json::json!({"action": "closed", "workflow": workflow}),
2299 crate::journal::SESSION_CLOSE_OUTBOX_KIND,
2300 )
2301 .await?;
2302 self.remove_step_journal(&session.id).await?;
2303 self.remove_lifecycle_outbox(&session.id, crate::journal::RECEIPT_OUTBOX_KIND)
2304 .await?;
2305 self.remove_lifecycle_outbox(&session.id, crate::journal::SESSION_CLOSE_OUTBOX_KIND)
2306 .await?;
2307 session.mark_close_complete();
2308 Ok(())
2309 }
2310 .await;
2311 match outcome {
2312 Ok(()) => completed = completed.saturating_add(1),
2313 Err(error) => {
2314 self.metrics
2315 .counter(
2316 "av_pending_close_completion_failed_total",
2317 "Pending-close completions that failed to finish their tail (round-43 F1)",
2318 )
2319 .inc();
2320 let key = self.spool_dir.join(format!("pending-close::{}", session.id));
2321 if self.warn_once(key) {
2322 tracing::warn!(
2323 session = %session.id,
2324 %error,
2325 "pending-close completion failed; will retry next tick",
2326 );
2327 }
2328 }
2329 }
2330 }
2331 Ok(completed)
2332 }
2333
2334 async fn remove_acked_lifecycle_outboxes(&self) -> Result<(), FinalizeError> {
2335 let directory = self.spool_dir.join(crate::spool::OUTBOX);
2336 let mut entries = match tokio::fs::read_dir(&directory).await {
2337 Ok(entries) => entries,
2338 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
2339 Err(error) => return Err(FinalizeError::Bridge(error.to_string())),
2340 };
2341 while let Some(entry) = entries
2342 .next_entry()
2343 .await
2344 .map_err(|error| FinalizeError::Bridge(error.to_string()))?
2345 {
2346 let path = entry.path();
2347 if path.extension().and_then(std::ffi::OsStr::to_str) != Some("json") {
2348 continue;
2349 }
2350 let sealed = read_capped_async(path.clone(), av_core::fsutil::MAX_CONTROL_BYTES)
2351 .await
2352 .map_err(|error| FinalizeError::Bridge(error.to_string()))?;
2353 let outbox: LifecycleOutbox = crate::journal::open(
2354 &self.journal_key,
2355 crate::journal::LIFECYCLE_OUTBOX_DOMAIN,
2356 0,
2357 &sealed,
2358 )
2359 .map_err(FinalizeError::Bridge)?;
2360 if path != self.lifecycle_outbox_path(&outbox.session_id, &outbox.kind) {
2361 return Err(FinalizeError::Bridge(
2362 "lifecycle outbox path does not match authenticated payload".to_owned(),
2363 ));
2364 }
2365 if outbox.ack.is_some() {
2366 remove_outbox(&path).await?;
2367 }
2368 }
2369 Ok(())
2370 }
2371
2372 fn lifecycle_outbox_path(&self, session_id: &str, kind: &str) -> PathBuf {
2373 let session_hash = &av_core::digest::sha256_hex(session_id.as_bytes())[..32];
2374 self.spool_dir
2375 .join(crate::spool::OUTBOX)
2376 .join(format!("{session_hash}.{kind}.json"))
2377 }
2378
2379 async fn remove_lifecycle_outbox(&self, session_id: &str, kind: &str) -> Result<(), FinalizeError> {
2380 remove_outbox(&self.lifecycle_outbox_path(session_id, kind)).await
2381 }
2382}
2383
2384async fn persist_outbox(
2385 path: &std::path::Path,
2386 outbox: &LifecycleOutbox,
2387 journal_key: &[u8; 32],
2388) -> Result<(), FinalizeError> {
2389 let path = path.to_path_buf();
2390 let bytes = crate::journal::seal(journal_key, crate::journal::LIFECYCLE_OUTBOX_DOMAIN, 0, outbox)
2391 .map_err(FinalizeError::Bridge)?;
2392 tokio::task::spawn_blocking(move || av_core::fsutil::write_atomic(&path, &bytes))
2393 .await
2394 .map_err(|error| FinalizeError::Task(error.to_string()))?
2395 .map_err(|error| FinalizeError::Bridge(error.to_string()))
2396}
2397
2398async fn persist_marker(path: &std::path::Path, bytes: &[u8]) -> Result<(), FinalizeError> {
2399 let path = path.to_path_buf();
2400 let bytes = bytes.to_vec();
2401 tokio::task::spawn_blocking(move || av_core::fsutil::write_atomic(&path, &bytes))
2402 .await
2403 .map_err(|error| FinalizeError::Task(error.to_string()))?
2404 .map_err(|error| FinalizeError::Atif(error.to_string()))
2405}
2406
2407async fn remove_outbox(path: &std::path::Path) -> Result<(), FinalizeError> {
2408 let path = path.to_path_buf();
2409 tokio::task::spawn_blocking(move || -> Result<(), FinalizeError> {
2410 let parent = path
2411 .parent()
2412 .ok_or_else(|| FinalizeError::Bridge("outbox has no parent".to_owned()))?;
2413 match std::fs::remove_file(&path) {
2414 Ok(()) => av_core::fsutil::sync_directory(parent)
2415 .map_err(|error| FinalizeError::Bridge(error.to_string())),
2416 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
2417 Err(error) => Err(FinalizeError::Bridge(error.to_string())),
2418 }
2419 })
2420 .await
2421 .map_err(|error| FinalizeError::Task(error.to_string()))?
2422}
2423
2424pub fn spawn_reconciler(
2428 sessions: Arc<SessionRegistry>,
2429 finalizer: Finalizer,
2430 idle_s: u64,
2431 tick_s: u64,
2432 breaker: av_loopdetect::BreakerConfig,
2433 metrics: Arc<Registry>,
2434) -> tokio::task::JoinHandle<()> {
2435 use futures::future::FutureExt as _;
2436 tokio::spawn(async move {
2437 let mut interval = tokio::time::interval(std::time::Duration::from_secs(tick_s.max(1)));
2438 interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2444 loop {
2445 interval.tick().await;
2446 let started = Instant::now();
2447 let outcome = std::panic::AssertUnwindSafe(async {
2454 if let Err(error) = finalizer.recover_spooled_sessions(&sessions, &breaker).await {
2455 tracing::warn!(%error, "ATIF spool recovery failed");
2456 metrics
2457 .counter("av_reconcile_errors_total", "Reconciliation errors")
2458 .inc();
2459 }
2460 if let Err(error) = finalizer.retry_marked_promotions(&sessions).await {
2461 tracing::warn!(%error, "durable promotion retry failed");
2462 metrics
2463 .counter("av_reconcile_errors_total", "Reconciliation errors")
2464 .inc();
2465 }
2466 if let Err(error) = finalizer.complete_pending_closes(&sessions).await {
2473 tracing::warn!(%error, "pending-close completion sweep failed");
2474 metrics
2475 .counter("av_reconcile_errors_total", "Reconciliation errors")
2476 .inc();
2477 }
2478 for session in sessions.idle_sessions(idle_s) {
2479 let session_id = session.id.clone();
2480 if let Err(error) = finalizer.close_session(session, StopReason::SessionClosed).await {
2481 tracing::warn!(session = %session_id, %error, "idle session finalization failed");
2482 metrics
2483 .counter("av_reconcile_errors_total", "Reconciliation errors")
2484 .inc();
2485 }
2486 }
2487 let evicted = sessions.evict_finalized(idle_s);
2488 if !evicted.is_empty() {
2489 tracing::debug!(count = evicted.len(), "evicted finalized signed sessions");
2490 }
2491 })
2492 .catch_unwind()
2493 .await;
2494 if let Err(panic) = outcome {
2495 let msg = panic
2496 .downcast_ref::<&'static str>()
2497 .copied()
2498 .or_else(|| panic.downcast_ref::<String>().map(String::as_str))
2499 .unwrap_or("panic payload was not a string");
2500 metrics
2501 .counter(
2502 "av_reconciler_panics_total",
2503 "Reconciler tick body panicked; loop supervised via catch_unwind",
2504 )
2505 .inc();
2506 tracing::error!(
2507 panic = %msg,
2508 "reconciler tick body panicked; continuing on the next tick"
2509 );
2510 }
2511 metrics
2512 .histogram("av_reconcile_duration_seconds", "Idle reconciliation duration")
2513 .observe_us(elapsed_us(started));
2514 }
2515 })
2516}
2517
2518fn checked_recovery_add(current: u64, value: u64, field: &str) -> Result<u64, FinalizeError> {
2519 current
2520 .checked_add(value)
2521 .filter(|total| *total <= av_core::error::JCS_SAFE_MAX)
2522 .ok_or_else(|| FinalizeError::Atif(format!("recovered {field} overflow")))
2523}
2524
2525fn track_response_attempt(
2526 pending: &mut std::collections::HashSet<String>,
2527 attempt: Option<&crate::worker::ResponseAttempt>,
2528) -> Result<(), FinalizeError> {
2529 let Some(attempt) = attempt else {
2530 return Ok(());
2531 };
2532 if attempt.terminal {
2533 if !pending.remove(&attempt.id) {
2534 pending.insert(format!("orphan-terminal:{}", attempt.id));
2535 }
2536 } else if !pending.insert(attempt.id.clone()) {
2537 return Err(FinalizeError::Atif(
2538 "active journal repeats a response attempt id".to_owned(),
2539 ));
2540 }
2541 Ok(())
2542}
2543
2544fn lifecycle_event_uid(value: &serde_json::Value) -> Result<String, FinalizeError> {
2545 value
2546 .get("metadata")
2547 .and_then(|metadata| metadata.get("uid"))
2548 .and_then(serde_json::Value::as_str)
2549 .map(str::to_owned)
2550 .ok_or_else(|| FinalizeError::Bridge("lifecycle event has no metadata UID".to_owned()))
2551}
2552
2553async fn resolve_lifecycle_ack(
2554 bridge: Arc<dyn EventBus>,
2555 topic: String,
2556 key: String,
2557 value: serde_json::Value,
2558 event_uid: String,
2559) -> Result<av_bridge::PublishAck, FinalizeError> {
2560 let lookup_bridge = Arc::clone(&bridge);
2561 let lookup_topic = topic.clone();
2562 let lookup_key = key.clone();
2563 let lookup_uid = event_uid.clone();
2564 if let Some(ack) = tokio::task::spawn_blocking(move || {
2565 lookup_bridge.find_event_by_uid(&lookup_topic, &lookup_key, &lookup_uid)
2566 })
2567 .await
2568 .map_err(|error| FinalizeError::Task(error.to_string()))?
2569 .map_err(|error| FinalizeError::Bridge(error.to_string()))?
2570 {
2571 return Ok(ack);
2572 }
2573 tokio::task::spawn_blocking(move || bridge.publish_idempotent(&topic, &key, &value, &event_uid))
2574 .await
2575 .map_err(|error| FinalizeError::Task(error.to_string()))?
2576 .map_err(|error| FinalizeError::Bridge(error.to_string()))
2577}
2578
2579async fn quarantine_sibling_exists(spool_dir: &std::path::Path, stem: &str) -> Result<bool, FinalizeError> {
2588 let prefix = format!("{stem}.events.ndjson.corrupt-");
2589 let spool_dir = spool_dir.to_path_buf();
2590 tokio::task::spawn_blocking(move || -> Result<bool, FinalizeError> {
2591 let entries = match std::fs::read_dir(&spool_dir) {
2592 Ok(entries) => entries,
2593 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false),
2594 Err(error) => return Err(FinalizeError::Atif(error.to_string())),
2595 };
2596 for entry in entries {
2597 let entry = entry.map_err(|error| FinalizeError::Atif(error.to_string()))?;
2598 let name = entry.file_name();
2599 if let Some(name) = name.to_str() {
2600 if name.starts_with(&prefix) {
2601 return Ok(true);
2602 }
2603 }
2604 }
2605 Ok(false)
2606 })
2607 .await
2608 .map_err(|error| FinalizeError::Task(error.to_string()))?
2609}
2610
2611async fn read_capped_async(path: std::path::PathBuf, max_bytes: u64) -> Result<Vec<u8>, std::io::Error> {
2619 tokio::task::spawn_blocking(move || av_core::fsutil::read_capped(&path, max_bytes))
2620 .await
2621 .map_err(|e| std::io::Error::other(e.to_string()))?
2622}
2623
2624async fn read_complete_journal(path: &std::path::Path) -> Result<Vec<String>, FinalizeError> {
2625 let path = path.to_path_buf();
2626 tokio::task::spawn_blocking(move || -> Result<Vec<String>, FinalizeError> {
2627 let bytes = av_core::fsutil::read_capped(&path, av_core::fsutil::MAX_ATIF_BYTES)
2631 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2632 if bytes.is_empty() {
2633 return Ok(Vec::new());
2634 }
2635 let complete_len = if bytes.last() == Some(&b'\n') {
2636 bytes.len()
2637 } else {
2638 bytes
2639 .iter()
2640 .rposition(|byte| *byte == b'\n')
2641 .map_or(0, |index| index + 1)
2642 };
2643 if complete_len == 0 && !bytes.is_empty() {
2653 let mut quarantine = path.clone();
2654 let stem = quarantine
2655 .file_name()
2656 .and_then(std::ffi::OsStr::to_str)
2657 .unwrap_or("journal");
2658 let new_name = format!("{stem}.corrupt-{}", av_core::new_event_uid());
2659 quarantine.set_file_name(new_name);
2660 let rename_error_message = match std::fs::rename(&path, &quarantine) {
2661 Ok(()) => {
2662 tracing::error!(
2663 original = %av_core::fsutil::basename(&path),
2664 quarantine = %av_core::fsutil::basename(&quarantine),
2665 bytes = bytes.len(),
2666 "journal has no complete lines; quarantined for post-mortem instead of silent 0-truncate"
2667 );
2668 None
2669 }
2670 Err(rename_error) => {
2671 tracing::error!(
2672 path = %av_core::fsutil::basename(&path),
2673 bytes = bytes.len(),
2674 error = %rename_error,
2675 "journal has no complete lines and quarantine rename failed; refusing to truncate"
2676 );
2677 Some(rename_error.to_string())
2678 }
2679 };
2680 let name = av_core::fsutil::basename(&path);
2696 let qname = av_core::fsutil::basename(&quarantine);
2697 return Err(FinalizeError::Atif(match rename_error_message {
2698 None => format!(
2699 "journal {name} contained no complete lines ({} bytes); quarantined at {qname}",
2700 bytes.len()
2701 ),
2702 Some(rename_error) => format!(
2703 "journal {name} contained no complete lines ({} bytes); quarantine rename to {qname} failed: {rename_error}; bytes remain at {name}",
2704 bytes.len()
2705 ),
2706 }));
2707 }
2708 if complete_len < bytes.len() {
2709 tracing::warn!(
2710 path = %av_core::fsutil::basename(&path),
2711 stored = bytes.len(),
2712 keeping = complete_len,
2713 dropping = bytes.len() - complete_len,
2714 "trimming partial trailing line from journal recovery"
2715 );
2716 let file = std::fs::OpenOptions::new()
2717 .write(true)
2718 .open(&path)
2719 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2720 file.set_len(complete_len as u64)
2721 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2722 file.sync_all()
2723 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2724 }
2725 let complete = String::from_utf8(bytes.get(..complete_len).unwrap_or_default().to_vec())
2726 .map_err(|error| FinalizeError::Atif(error.to_string()))?;
2727 Ok(complete
2728 .lines()
2729 .filter(|line| !line.trim().is_empty())
2730 .map(str::to_owned)
2731 .collect())
2732 })
2733 .await
2734 .map_err(|error| FinalizeError::Task(error.to_string()))?
2735}
2736
2737#[cfg(test)]
2738mod tests {
2739 #![allow(
2740 clippy::expect_used,
2741 clippy::indexing_slicing,
2742 clippy::panic,
2743 clippy::unwrap_used
2744 )]
2745
2746 use super::*;
2747 use av_bridge::{BusError, PublishAck, StoredEvent};
2748 use av_events::AgentIdentity;
2749 use av_receipts::Ed25519Signer;
2750
2751 struct FailFirstReceiptBus {
2752 fail: std::sync::atomic::AtomicBool,
2753 attempts: parking_lot::Mutex<Vec<(String, serde_json::Value)>>,
2754 }
2755
2756 impl EventBus for FailFirstReceiptBus {
2757 fn publish(
2758 &self,
2759 topic: &str,
2760 _key: &str,
2761 value: &serde_json::Value,
2762 ) -> Result<PublishAck, BusError> {
2763 self.attempts.lock().push((topic.to_owned(), value.clone()));
2764 if topic == "agent.receipt" && self.fail.swap(false, std::sync::atomic::Ordering::AcqRel) {
2765 return Err(BusError::Backend("injected receipt outage".to_owned()));
2766 }
2767 Ok(PublishAck {
2768 topic: topic.to_owned(),
2769 partition: 0,
2770 offset: self.attempts.lock().len() as u64,
2771 })
2772 }
2773
2774 fn fetch(
2775 &self,
2776 _topic: &str,
2777 _partition: u32,
2778 _offset: u64,
2779 _max: usize,
2780 ) -> Result<Vec<StoredEvent>, BusError> {
2781 Ok(Vec::new())
2782 }
2783
2784 fn partitions(&self, _topic: &str) -> Result<u32, BusError> {
2785 Ok(1)
2786 }
2787
2788 fn topics(&self) -> Vec<String> {
2789 av_events::EventClass::all()
2790 .iter()
2791 .map(|class| class.topic().to_owned())
2792 .collect()
2793 }
2794 }
2795
2796 fn session(workflow: Workflow) -> Arc<Session> {
2797 Arc::new(Session::new(
2798 "lifecycle-session".to_owned(),
2799 workflow,
2800 AgentIdentity {
2801 version: "1".to_owned(),
2802 charter: "test".into(),
2803 instance_uid: "instance-1".to_owned(),
2804 ttl_remaining_s: Some(600),
2805 },
2806 Default::default(),
2807 ))
2808 }
2809
2810 fn finalizer(directory: &std::path::Path) -> Finalizer {
2811 Finalizer::new(
2812 Arc::new(Ed25519Signer::from_seed(&[7; 32])),
2813 directory.to_path_buf(),
2814 Arc::new(Registry::new()),
2815 )
2816 }
2817
2818 #[tokio::test]
2825 async fn emit_bridge_event_persist_failure_does_not_burn_a_seq() {
2826 let directory = tempfile::tempdir().unwrap();
2827 std::fs::write(directory.path().join(crate::spool::OUTBOX), b"").unwrap();
2831 let bus = Arc::new(FailFirstReceiptBus {
2832 fail: std::sync::atomic::AtomicBool::new(false),
2833 attempts: parking_lot::Mutex::new(Vec::new()),
2834 });
2835 let finalizer = Finalizer::with_bridge(
2836 Arc::new(Ed25519Signer::from_seed(&[7; 32])),
2837 directory.path().to_path_buf(),
2838 Arc::new(Registry::new()),
2839 bus,
2840 );
2841 let session = session(Workflow::Signed);
2842 let seq_before = session.peek_seq();
2843 let result = finalizer
2846 .close_session(Arc::clone(&session), StopReason::SessionClosed)
2847 .await;
2848 assert!(
2849 result.is_err(),
2850 "close_session must fail when outbox persist is blocked, got {result:?}",
2851 );
2852 assert_eq!(
2853 session.peek_seq(),
2854 seq_before,
2855 "peek_seq must not advance past a failed persist_outbox — the burned \
2856 seq would misalign the journal on retry",
2857 );
2858 }
2859
2860 #[tokio::test]
2868 async fn emit_bridge_event_reading_persisted_outbox_advances_seq_past_it() {
2869 let directory = tempfile::tempdir().unwrap();
2870 let bus = Arc::new(FailFirstReceiptBus {
2871 fail: std::sync::atomic::AtomicBool::new(false),
2872 attempts: parking_lot::Mutex::new(Vec::new()),
2873 });
2874 let finalizer = Finalizer::with_bridge(
2875 Arc::new(Ed25519Signer::from_seed(&[7; 32])),
2876 directory.path().to_path_buf(),
2877 Arc::new(Registry::new()),
2878 bus.clone(),
2879 );
2880 let session = session(Workflow::Signed);
2881 let receipt_seq = 42u64;
2884 let receipt_event_uid = av_core::new_event_uid();
2885 let value = serde_json::json!({
2886 "metadata": { "sequence": receipt_seq, "uid": receipt_event_uid },
2887 "topic": av_events::EventClass::Receipt.topic(),
2888 });
2889 let outbox = LifecycleOutbox {
2890 session_id: session.id.clone(),
2891 kind: crate::journal::RECEIPT_OUTBOX_KIND.to_owned(),
2892 topic: av_events::EventClass::Receipt.topic().to_owned(),
2893 key: session.identity.instance_uid.clone(),
2894 value,
2895 ack: Some(av_bridge::PublishAck {
2896 topic: av_events::EventClass::Receipt.topic().to_owned(),
2897 partition: 0,
2898 offset: 1,
2899 }),
2900 };
2901 let outbox_path = finalizer.lifecycle_outbox_path(&session.id, crate::journal::RECEIPT_OUTBOX_KIND);
2902 std::fs::create_dir_all(outbox_path.parent().unwrap()).unwrap();
2903 let sealed = crate::journal::seal(
2904 &finalizer.journal_key,
2905 crate::journal::LIFECYCLE_OUTBOX_DOMAIN,
2906 0,
2907 &outbox,
2908 )
2909 .unwrap();
2910 std::fs::write(&outbox_path, sealed).unwrap();
2911 assert!(
2912 session.peek_seq() < receipt_seq,
2913 "precondition: in-memory seq must trail the persisted outbox seq",
2914 );
2915 finalizer
2916 .close_session(Arc::clone(&session), StopReason::SessionClosed)
2917 .await
2918 .unwrap();
2919 let attempts = bus.attempts.lock();
2920 let session_close = attempts
2921 .iter()
2922 .find(|(topic, _)| topic == av_events::EventClass::Session.topic())
2923 .expect("SESSION_CLOSE lifecycle event must reach the bridge");
2924 let session_close_seq = session_close
2925 .1
2926 .get("metadata")
2927 .and_then(|metadata| metadata.get("sequence"))
2928 .and_then(serde_json::Value::as_u64)
2929 .expect("published event must carry a numeric metadata.sequence");
2930 assert!(
2931 session_close_seq > receipt_seq,
2932 "SESSION_CLOSE seq ({session_close_seq}) must exceed the persisted RECEIPT_OUTBOX seq ({receipt_seq}) — otherwise consumers see duplicate metadata.sequence values within one session",
2933 );
2934 }
2935
2936 #[tokio::test]
2946 async fn recovered_signed_session_rejects_leases_even_when_finalize_errors() {
2947 use av_events::{EventClass, OcsfEventBuilder, StatusId};
2948 use std::io::Write as _;
2949
2950 let directory = tempfile::tempdir().unwrap();
2951 let signer: Arc<dyn Signer> = Arc::new(Ed25519Signer::from_seed(&[29; 32]));
2952 let journal_key = crate::journal::key_from_signer(signer.as_ref());
2953 let session_id = "signed-recovery-lease-guard";
2954 let identity = AgentIdentity {
2955 version: "1".into(),
2956 charter: "test".into(),
2957 instance_uid: "instance-lease-guard".into(),
2958 ttl_remaining_s: Some(600),
2959 };
2960
2961 let digest = av_core::digest::sha256_hex(session_id.as_bytes());
2963 let stem = &digest[..32];
2964 let metadata_payload = serde_json::json!({
2965 "journal_version": 2,
2966 "session_id": session_id,
2967 "identity": identity,
2968 "workflow": "signed",
2969 });
2970 let metadata_sealed = crate::journal::seal(&journal_key, "metadata", 0, &metadata_payload).unwrap();
2971 std::fs::write(
2972 directory.path().join(format!("{stem}.session.json")),
2973 &metadata_sealed,
2974 )
2975 .unwrap();
2976
2977 let event = OcsfEventBuilder::new(
2979 EventClass::Compression,
2980 session_id.to_owned(),
2981 identity.clone(),
2982 0,
2983 )
2984 .status(StatusId::Success)
2985 .payload(serde_json::json!({}))
2986 .build()
2987 .unwrap();
2988 let event_uid = event.metadata.uid.clone();
2989 let record = crate::worker::ActiveJournalRecord {
2990 event: serde_json::to_value(&event).unwrap(),
2991 identity: identity.clone(),
2992 atif_step: None,
2993 tool_calls: 0,
2994 tool_allowed: 0,
2995 tool_blocked: 0,
2996 prompt_tokens: 0,
2997 completion_tokens: 0,
2998 cached_tokens: 0,
2999 cost_usd_micros: 0,
3000 stop_reason_id: None,
3001 response_attempt: None,
3002 };
3003 let domain = format!("{session_id}:active");
3004 let sealed = crate::journal::seal(&journal_key, &domain, 0, &record).unwrap();
3005 let mut journal_file =
3006 std::fs::File::create(directory.path().join(format!("{stem}.events.ndjson"))).unwrap();
3007 journal_file.write_all(&sealed).unwrap();
3008 journal_file.write_all(b"\n").unwrap();
3009 drop(journal_file);
3010
3011 crate::worker::persist_broker_ack(
3013 directory.path(),
3014 session_id,
3015 &event_uid,
3016 &PublishAck {
3017 topic: EventClass::Compression.topic().to_owned(),
3018 partition: 0,
3019 offset: 1,
3020 },
3021 &journal_key,
3022 )
3023 .await
3024 .unwrap();
3025
3026 std::fs::write(directory.path().join("receipts"), b"").unwrap();
3034
3035 let registry = crate::session::SessionRegistry::new();
3036 let finalizer = Finalizer::with_bridge(
3037 signer,
3038 directory.path().to_path_buf(),
3039 Arc::new(Registry::new()),
3040 Arc::new(FailFirstReceiptBus {
3041 fail: std::sync::atomic::AtomicBool::new(false),
3042 attempts: parking_lot::Mutex::new(Vec::new()),
3043 }),
3044 );
3045 let result = finalizer
3046 .recover_spooled_sessions(®istry, &Default::default())
3047 .await;
3048 assert!(
3064 result.is_ok(),
3065 "round-41 F1: per-session errors warn+continue; got {result:?}",
3066 );
3067
3068 assert!(
3069 registry.get(session_id).is_none(),
3070 "round-42 F3: after a transient close error the session must be removed from the registry so the next reconciler tick can re-adopt via the still-present journal sidecar — leaving it in the registry with `is_closed()` = true would starve recovery forever",
3071 );
3072 }
3073
3074 #[tokio::test]
3091 async fn round_41_f1_corrupt_signed_sidecar_does_not_block_other_signed_recovery() {
3092 let directory = tempfile::tempdir().unwrap();
3093 let poison_stem = "poisonpoisonpoisonpoisonpoison32";
3099 std::fs::write(
3100 directory.path().join(format!("{poison_stem}.session.json")),
3101 b"{\"garbage\": true, \"not\": \"a valid sealed metadata\"}",
3102 )
3103 .unwrap();
3104 std::fs::write(
3105 directory.path().join(format!("{poison_stem}.events.ndjson")),
3106 b"{}\n",
3107 )
3108 .unwrap();
3109 let registry = crate::session::SessionRegistry::new();
3110 let finalizer = finalizer(directory.path());
3111 let outcome = finalizer
3112 .recover_spooled_sessions(®istry, &Default::default())
3113 .await;
3114 assert!(
3115 outcome.is_ok(),
3116 "poisoned sidecar must not fail the outer recover_spooled_sessions; got {outcome:?}"
3117 );
3118 assert!(
3119 registry.get(poison_stem).is_none(),
3120 "poisoned session id must NEVER be installed into the registry"
3121 );
3122 }
3123
3124 #[tokio::test]
3125 async fn signed_close_issues_exactly_one_offline_verifiable_receipt() {
3126 let directory = tempfile::tempdir().unwrap();
3127 let finalizer = finalizer(directory.path());
3128 let session = session(Workflow::Signed);
3129
3130 let first = finalizer
3131 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3132 .await
3133 .unwrap();
3134 let FinalizeOutcome::Receipt { receipt } = first else {
3135 panic!("expected receipt")
3136 };
3137 receipt.verify_embedded().unwrap();
3138 assert!(matches!(
3139 finalizer
3140 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3141 .await
3142 .unwrap(),
3143 FinalizeOutcome::AlreadyClosed
3144 ));
3145 assert_eq!(
3146 session.receipt.lock().as_ref().unwrap().body.receipt_id,
3147 receipt.body.receipt_id
3148 );
3149 }
3150
3151 #[tokio::test]
3152 async fn lifecycle_outbox_retries_the_same_receipt_event() {
3153 let directory = tempfile::tempdir().unwrap();
3154 let bus = Arc::new(FailFirstReceiptBus {
3155 fail: std::sync::atomic::AtomicBool::new(true),
3156 attempts: parking_lot::Mutex::new(Vec::new()),
3157 });
3158 let finalizer = Finalizer::with_bridge(
3159 Arc::new(Ed25519Signer::from_seed(&[17; 32])),
3160 directory.path().to_path_buf(),
3161 Arc::new(Registry::new()),
3162 bus.clone(),
3163 );
3164 let session = session(Workflow::Signed);
3165 assert!(matches!(
3166 finalizer
3167 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3168 .await,
3169 Err(FinalizeError::Bridge(_))
3170 ));
3171 assert!(session.is_closed());
3172 assert!(
3173 session.try_lease().is_none(),
3174 "artifact commit must keep admission closed"
3175 );
3176 let receipt_id = session.receipt.lock().as_ref().unwrap().body.receipt_id.clone();
3177 let outcome = finalizer
3178 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3179 .await
3180 .unwrap();
3181 let FinalizeOutcome::Receipt { receipt } = outcome else {
3182 panic!("expected receipt")
3183 };
3184 assert_eq!(receipt.body.receipt_id, receipt_id);
3185 let attempts = bus.attempts.lock();
3186 let receipt_events: Vec<_> = attempts
3187 .iter()
3188 .filter(|(topic, _)| topic == "agent.receipt")
3189 .collect();
3190 assert_eq!(receipt_events.len(), 2);
3191 assert_eq!(
3192 receipt_events[0].1["metadata"]["uid"],
3193 receipt_events[1].1["metadata"]["uid"]
3194 );
3195 assert!(!finalizer.lifecycle_outbox_path(&session.id, "receipt").exists());
3196 }
3197
3198 #[tokio::test]
3208 async fn round_43_f1_pending_close_completes_via_reconciler_sweep() {
3209 let directory = tempfile::tempdir().unwrap();
3210 let bus = Arc::new(FailFirstReceiptBus {
3211 fail: std::sync::atomic::AtomicBool::new(true),
3212 attempts: parking_lot::Mutex::new(Vec::new()),
3213 });
3214 let finalizer = Finalizer::with_bridge(
3215 Arc::new(Ed25519Signer::from_seed(&[43; 32])),
3216 directory.path().to_path_buf(),
3217 Arc::new(Registry::new()),
3218 bus.clone(),
3219 );
3220 let sessions = SessionRegistry::new();
3221 let session = session(Workflow::Signed);
3222 sessions.insert_recovered(Arc::try_unwrap(session.clone()).unwrap_or_else(|arc| {
3224 Session::new(
3225 arc.id.clone(),
3226 arc.workflow,
3227 arc.current_identity(),
3228 Default::default(),
3229 )
3230 }));
3231 let registered = sessions.get(&session.id).unwrap();
3234
3235 assert!(matches!(
3241 finalizer
3242 .close_session(Arc::clone(®istered), StopReason::SessionClosed)
3243 .await,
3244 Err(FinalizeError::Bridge(_))
3245 ));
3246 assert!(
3247 registered.artifact_committed_flag(),
3248 "signed close must have persisted the receipt and marked artifact_committed before the emit_receipt_event failure",
3249 );
3250 assert!(
3251 !registered.close_complete_flag(),
3252 "close_complete must NOT be set yet — the finalization tail did not run",
3253 );
3254 let pending = sessions.pending_close_sessions();
3256 assert_eq!(pending.len(), 1, "sweep must see the orphaned session");
3257 assert_eq!(pending[0].id, registered.id);
3258
3259 finalizer.replay_lifecycle_outboxes().await.unwrap();
3264 let completed = finalizer.complete_pending_closes(&sessions).await.unwrap();
3265 assert_eq!(completed, 1, "sweep must complete the orphan");
3266 assert!(
3267 registered.close_complete_flag(),
3268 "close_complete must be set after the sweep — round-43 F1 invariant",
3269 );
3270 assert!(
3271 sessions.pending_close_sessions().is_empty(),
3272 "no sessions remain in the pending-close set once completed",
3273 );
3274 let digest = av_core::digest::sha256_hex(registered.id.as_bytes());
3276 let stem = digest.get(..32).unwrap();
3277 assert!(
3278 !directory.path().join(format!("{stem}.session.json")).exists(),
3279 "step journal metadata sidecar must be removed by the completion sweep",
3280 );
3281 }
3282
3283 #[tokio::test]
3293 async fn round_44_f1_sidecar_less_atif_is_quarantined_without_reading_bytes() {
3294 let directory = tempfile::tempdir().unwrap();
3295 let finalizer = finalizer(directory.path());
3296 let orphan = directory.path().join("hostileplant0000000000000000.json");
3304 std::fs::write(&orphan, b"{not valid json at all - this MUST NOT be parsed").unwrap();
3305
3306 let registry = SessionRegistry::new();
3307 let outcome = finalizer
3308 .recover_spooled_sessions(®istry, &Default::default())
3309 .await;
3310 assert!(
3311 outcome.is_ok(),
3312 "orphan must not fail the outer scan; got {outcome:?}"
3313 );
3314
3315 assert!(
3320 !orphan.exists(),
3321 "sidecar-less ATIF must be quarantined-renamed after first sighting so subsequent ticks don't re-read it",
3322 );
3323 let mut found_quarantine = false;
3326 for entry in std::fs::read_dir(directory.path()).unwrap() {
3327 let entry = entry.unwrap();
3328 if let Some(name) = entry.file_name().to_str() {
3329 if name.contains(".json.corrupt-") {
3330 found_quarantine = true;
3331 }
3332 }
3333 }
3334 assert!(
3335 found_quarantine,
3336 "quarantined file must be preserved on disk under a `.corrupt-<uid>` name for forensic inspection",
3337 );
3338 }
3339
3340 #[tokio::test]
3352 async fn round_44_f2_empty_unsigned_quarantine_excluded_from_pending_close_sweep() {
3353 let directory = tempfile::tempdir().unwrap();
3354 let finalizer = finalizer(directory.path());
3355 let sessions = SessionRegistry::new();
3356 let empty = session(Workflow::Unsigned);
3357 let empty_id = empty.id.clone();
3359 sessions.insert_recovered(Arc::try_unwrap(empty).unwrap_or_else(|arc| {
3360 Session::new(
3361 arc.id.clone(),
3362 arc.workflow,
3363 arc.current_identity(),
3364 Default::default(),
3365 )
3366 }));
3367 let registered = sessions.get(&empty_id).unwrap();
3368
3369 let result = finalizer
3376 .close_session(Arc::clone(®istered), StopReason::SessionClosed)
3377 .await;
3378 assert!(
3379 matches!(result, Err(FinalizeError::Atif(_))),
3380 "empty unsigned close must reject; got {result:?}",
3381 );
3382 assert!(
3383 registered.artifact_committed_flag(),
3384 "empty-unsigned reject seals with artifact_committed to stop idle-sweep churn",
3385 );
3386 assert!(
3387 registered.is_empty_unsigned_quarantine(),
3388 "empty-unsigned reject must be recognizable as a quarantine",
3389 );
3390
3391 let pending = sessions.pending_close_sessions();
3397 assert!(
3398 pending.iter().all(|s| s.id != empty_id),
3399 "empty-unsigned quarantine must be excluded from pending_close_sessions()",
3400 );
3401 let completed = finalizer.complete_pending_closes(&sessions).await.unwrap();
3402 assert_eq!(
3403 completed, 0,
3404 "sweep must NOT complete the empty-unsigned quarantine — evidence would be lost",
3405 );
3406 assert!(
3407 !registered.close_complete_flag(),
3408 "close_complete must remain 0 for the empty-unsigned quarantine so it can NOT be reopened by get_or_open (reopen=true) — preserving the incident record",
3409 );
3410 }
3411
3412 #[tokio::test]
3413 async fn concurrent_close_waits_for_failed_lifecycle_attempt() {
3414 let directory = tempfile::tempdir().unwrap();
3415 let bus = Arc::new(FailFirstReceiptBus {
3416 fail: std::sync::atomic::AtomicBool::new(true),
3417 attempts: parking_lot::Mutex::new(Vec::new()),
3418 });
3419 let finalizer = Finalizer::with_bridge(
3420 Arc::new(Ed25519Signer::from_seed(&[23; 32])),
3421 directory.path().to_path_buf(),
3422 Arc::new(Registry::new()),
3423 bus,
3424 );
3425 let session = session(Workflow::Signed);
3426 let first_finalizer = finalizer.clone();
3427 let first_session = Arc::clone(&session);
3428 let first = tokio::spawn(async move {
3429 first_finalizer
3430 .close_session(first_session, StopReason::SessionClosed)
3431 .await
3432 });
3433 tokio::task::yield_now().await;
3434 let second = finalizer.close_session(session, StopReason::SessionClosed).await;
3435 let first = first.await.unwrap();
3436 assert!(matches!(first, Err(FinalizeError::Bridge(_))));
3437 assert!(matches!(second, Ok(FinalizeOutcome::Receipt { .. })));
3438 }
3439
3440 #[tokio::test]
3441 async fn startup_replays_lifecycle_outbox_without_session_journal() {
3442 let directory = tempfile::tempdir().unwrap();
3443 let bus = Arc::new(FailFirstReceiptBus {
3444 fail: std::sync::atomic::AtomicBool::new(true),
3445 attempts: parking_lot::Mutex::new(Vec::new()),
3446 });
3447 let finalizer = Finalizer::with_bridge(
3448 Arc::new(Ed25519Signer::from_seed(&[19; 32])),
3449 directory.path().to_path_buf(),
3450 Arc::new(Registry::new()),
3451 bus.clone(),
3452 );
3453 let session = session(Workflow::Signed);
3454 assert!(matches!(
3455 finalizer.close_session(session, StopReason::SessionClosed).await,
3456 Err(FinalizeError::Bridge(_))
3457 ));
3458 assert!(finalizer
3459 .lifecycle_outbox_path("lifecycle-session", "receipt")
3460 .exists());
3461
3462 let sessions = SessionRegistry::new();
3463 assert_eq!(
3464 finalizer
3465 .recover_spooled_sessions(&sessions, &Default::default())
3466 .await
3467 .unwrap(),
3468 0
3469 );
3470 assert!(sessions.get("lifecycle-session").is_none());
3471 assert!(!finalizer
3472 .lifecycle_outbox_path("lifecycle-session", "receipt")
3473 .exists());
3474 assert_eq!(
3475 bus.attempts
3476 .lock()
3477 .iter()
3478 .filter(|(topic, _)| topic == "agent.receipt")
3479 .count(),
3480 2
3481 );
3482 }
3483
3484 #[tokio::test]
3485 async fn unsigned_close_and_promotion_are_strict_and_idempotent() {
3486 let directory = tempfile::tempdir().unwrap();
3487 let finalizer = finalizer(directory.path());
3488 let session = session(Workflow::Unsigned);
3489 session
3490 .atif
3491 .lock()
3492 .push_step(av_atif::Step {
3493 step_id: 0,
3494 timestamp: Some(av_core::time::now_iso8601()),
3495 source: av_atif::Source::Agent,
3496 message: serde_json::json!("done"),
3497 reasoning_effort: None,
3498 reasoning_content: None,
3499 model_name: None,
3500 tool_calls: None,
3501 observation: None,
3502 metrics: Some(av_atif::Metrics {
3503 prompt_tokens: Some(10),
3504 completion_tokens: Some(2),
3505 cached_tokens: Some(4),
3506 cost_usd: Some(0.001),
3507 logprobs: None,
3508 completion_token_ids: None,
3509 prompt_token_ids: None,
3510 extra: None,
3511 }),
3512 is_copied_context: None,
3513 llm_call_count: Some(1),
3514 extra: None,
3515 })
3516 .unwrap();
3517
3518 let outcome = finalizer
3519 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3520 .await
3521 .unwrap();
3522 let FinalizeOutcome::Atif { path } = outcome else {
3523 panic!("expected ATIF artifact")
3524 };
3525 let value: serde_json::Value = serde_json::from_slice(&tokio::fs::read(path).await.unwrap()).unwrap();
3526 assert!(av_atif::validate_value(&value, av_atif::Mode::Strict).is_empty());
3527
3528 let first = finalizer.promote(Arc::clone(&session)).await.unwrap();
3529 let second = finalizer.promote(Arc::clone(&session)).await.unwrap();
3530 assert_eq!(first.body.receipt_id, second.body.receipt_id);
3531 first.verify_embedded().unwrap();
3532 assert!(matches!(
3533 first.body.subject,
3534 ReceiptSubject::AtifTrajectory {
3535 step_count: 1,
3536 retroactive: true,
3537 ..
3538 }
3539 ));
3540 }
3541
3542 #[tokio::test]
3543 async fn unsigned_restart_preserves_receipt_accounting_and_identity() {
3544 let directory = tempfile::tempdir().unwrap();
3545 let finalizer = finalizer(directory.path());
3546 let original = session(Workflow::Unsigned);
3547 original
3548 .totals
3549 .tool_calls
3550 .store(2, std::sync::atomic::Ordering::Release);
3551 original
3552 .totals
3553 .tool_allowed
3554 .store(1, std::sync::atomic::Ordering::Release);
3555 original
3556 .totals
3557 .tool_blocked
3558 .store(1, std::sync::atomic::Ordering::Release);
3559 original
3560 .totals
3561 .prompt_tokens
3562 .store(17, std::sync::atomic::Ordering::Release);
3563 original
3564 .totals
3565 .completion_tokens
3566 .store(9, std::sync::atomic::Ordering::Release);
3567 original
3568 .totals
3569 .cached_tokens
3570 .store(3, std::sync::atomic::Ordering::Release);
3571 original
3572 .totals
3573 .cost_usd_micros
3574 .store(1_234_567, std::sync::atomic::Ordering::Release);
3575 original.record_stop_reason(StopReason::PolicyBlocked);
3576 original
3577 .atif
3578 .lock()
3579 .push_step(av_atif::Step {
3580 step_id: 0,
3581 timestamp: None,
3582 source: av_atif::Source::User,
3583 message: serde_json::json!("test"),
3584 reasoning_effort: None,
3585 reasoning_content: None,
3586 model_name: None,
3587 tool_calls: None,
3588 observation: None,
3589 metrics: None,
3590 is_copied_context: None,
3591 llm_call_count: None,
3592 extra: None,
3593 })
3594 .unwrap();
3595 finalizer
3596 .close_session(original, StopReason::SessionClosed)
3597 .await
3598 .unwrap();
3599
3600 let registry = SessionRegistry::new();
3601 finalizer
3602 .recover_spooled_sessions(®istry, &Default::default())
3603 .await
3604 .unwrap();
3605 let recovered = registry.get("lifecycle-session").unwrap();
3606 assert_eq!(recovered.current_identity().ttl_remaining_s, Some(600));
3607 let receipt = finalizer.promote(recovered).await.unwrap();
3608 assert_eq!(receipt.body.tool_calls.total, 2);
3609 assert_eq!(receipt.body.tool_calls.allowed, 1);
3610 assert_eq!(receipt.body.tool_calls.blocked, 1);
3611 assert_eq!(receipt.body.cost.prompt_tokens, 17);
3612 assert_eq!(receipt.body.cost.completion_tokens, 9);
3613 assert_eq!(receipt.body.cost.cached_tokens, 3);
3614 assert_eq!(receipt.body.cost.cost_usd_micros, 1_234_567);
3615 assert_eq!(receipt.body.stop_reason_id, StopReason::PolicyBlocked.id());
3616 }
3617
3618 #[tokio::test]
3619 async fn incomplete_capture_never_produces_receipt_or_atif() {
3620 let directory = tempfile::tempdir().unwrap();
3621 let finalizer = finalizer(directory.path());
3622 for workflow in [Workflow::Signed, Workflow::Unsigned] {
3623 let session = session(workflow);
3624 session.mark_capture_failed();
3625 assert!(matches!(
3626 finalizer
3627 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3628 .await,
3629 Err(FinalizeError::CaptureIncomplete)
3630 ));
3631 assert!(session.receipt.lock().is_none());
3632 assert!(session.atif_path.lock().is_none());
3633 }
3634 }
3635
3636 #[tokio::test]
3650 async fn close_session_seals_capture_failed_session_so_idle_sweep_stops_churning() {
3651 let directory = tempfile::tempdir().unwrap();
3652 let finalizer = finalizer(directory.path());
3653 for workflow in [Workflow::Signed, Workflow::Unsigned] {
3654 let session = session(workflow);
3655 session.mark_capture_failed();
3656 let result = finalizer
3657 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3658 .await;
3659 assert!(matches!(result, Err(FinalizeError::CaptureIncomplete)));
3660 assert!(
3661 session.is_closed(),
3662 "close_session_locked must seal a capture_failed session (mark_artifact_committed + claim.committed = true) on the CaptureIncomplete return path so subsequent idle-sweep passes skip it via the `!is_closed()` filter — otherwise CloseClaim drops unarmed, reset_close puts `closed` back to 0, is_closed() stays false, and the idle sweeper churns forever on this session (workflow: {workflow:?})",
3663 );
3664 assert!(session.receipt.lock().is_none());
3665 assert!(session.atif_path.lock().is_none());
3666 }
3667 }
3668
3669 #[tokio::test]
3689 async fn close_session_seals_empty_unsigned_session_so_idle_sweep_stops_churning() {
3690 let directory = tempfile::tempdir().unwrap();
3691 let finalizer = finalizer(directory.path());
3692 let session = session(Workflow::Unsigned);
3693 assert_eq!(
3694 session.atif.lock().clone().finish().steps.len(),
3695 0,
3696 "precondition: session has no captured steps",
3697 );
3698 let result = finalizer
3699 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3700 .await;
3701 assert!(
3702 matches!(result, Err(FinalizeError::Atif(_))),
3703 "empty unsigned close must surface an ATIF error to the caller: {result:?}",
3704 );
3705 assert!(
3706 session.is_closed(),
3707 "close_session_locked must seal an empty unsigned session (mark_artifact_committed + claim.committed = true) when write_atomic's strict validation rejects the empty trajectory — otherwise CloseClaim drops unarmed, reset_close puts `closed` back to 0, is_closed() stays false, and the idle sweeper churns forever on this session (write_atomic → validate → \"must contain at least one step\" → Err → reset_close → picked up next tick → repeat)",
3708 );
3709 assert!(
3710 session.atif_path.lock().is_none(),
3711 "no ATIF file was ever produced for the empty session",
3712 );
3713 }
3714
3715 #[tokio::test]
3723 async fn recovery_tick_does_not_quarantine_live_sessions_with_inflight_markers() {
3724 let directory = tempfile::tempdir().unwrap();
3725 let finalizer = finalizer(directory.path());
3726 let live = session(Workflow::Unsigned);
3727 let registry = crate::session::SessionRegistry::new();
3728 let live = registry.insert_recovered(Arc::try_unwrap(live).map_err(|_| ()).unwrap());
3729 crate::worker::create_response_marker(
3730 directory.path(),
3731 &finalizer.journal_key,
3732 &live.id,
3733 "digest".to_owned(),
3734 )
3735 .await
3736 .unwrap();
3737 finalizer
3738 .recover_spooled_sessions(®istry, &Default::default())
3739 .await
3740 .unwrap();
3741 assert!(
3742 !finalizer.quarantined_sessions.lock().contains(&live.id),
3743 "a live session's in-flight marker must not put it in quarantine",
3744 );
3745 assert!(
3746 !live.capture_failed(),
3747 "a recovery tick must not poison a live session mid-request",
3748 );
3749 }
3750
3751 #[tokio::test]
3760 async fn live_session_with_id_in_quarantine_set_can_still_close() {
3761 let directory = tempfile::tempdir().unwrap();
3762 let finalizer = finalizer(directory.path());
3763 finalizer
3765 .quarantined_sessions
3766 .lock()
3767 .insert("lifecycle-session".to_owned());
3768 let live = session(Workflow::Unsigned);
3769 assert!(!live.capture_failed(), "precondition: live session is clean");
3770 live.atif
3772 .lock()
3773 .push_step(av_atif::Step {
3774 step_id: 0,
3775 timestamp: None,
3776 source: av_atif::Source::Agent,
3777 message: serde_json::json!("live response"),
3778 reasoning_effort: None,
3779 reasoning_content: None,
3780 model_name: Some("test-model".into()),
3781 tool_calls: None,
3782 observation: None,
3783 metrics: Some(av_atif::Metrics {
3784 prompt_tokens: Some(1),
3785 completion_tokens: Some(1),
3786 cached_tokens: Some(0),
3787 cost_usd: Some(0.0),
3788 logprobs: None,
3789 completion_token_ids: None,
3790 prompt_token_ids: None,
3791 extra: None,
3792 }),
3793 is_copied_context: None,
3794 llm_call_count: Some(1),
3795 extra: None,
3796 })
3797 .unwrap();
3798 let outcome = finalizer
3799 .close_session(Arc::clone(&live), StopReason::SessionClosed)
3800 .await
3801 .expect("live session must finalize despite id sharing space with a set entry");
3802 match outcome {
3803 FinalizeOutcome::Atif { .. } => {}
3804 other => panic!("expected FinalizeOutcome::Atif, got {other:?}"),
3805 }
3806 assert!(
3807 !live.capture_failed(),
3808 "close_session must not poison the live session's capture_failed flag",
3809 );
3810 }
3811
3812 #[tokio::test]
3819 async fn recovery_does_not_clobber_a_live_session_with_the_same_id() {
3820 let directory = tempfile::tempdir().unwrap();
3821 let finalizer = finalizer(directory.path());
3822 let closed_session = session(Workflow::Unsigned);
3824 closed_session
3825 .atif
3826 .lock()
3827 .push_step(av_atif::Step {
3828 step_id: 0,
3829 timestamp: None,
3830 source: av_atif::Source::Agent,
3831 message: serde_json::json!("archived response"),
3832 reasoning_effort: None,
3833 reasoning_content: None,
3834 model_name: Some("test-model".into()),
3835 tool_calls: None,
3836 observation: None,
3837 metrics: Some(av_atif::Metrics {
3838 prompt_tokens: Some(1),
3839 completion_tokens: Some(1),
3840 cached_tokens: Some(0),
3841 cost_usd: Some(0.0),
3842 logprobs: None,
3843 completion_token_ids: None,
3844 prompt_token_ids: None,
3845 extra: None,
3846 }),
3847 is_copied_context: None,
3848 llm_call_count: Some(1),
3849 extra: None,
3850 })
3851 .unwrap();
3852 finalizer
3853 .close_session(Arc::clone(&closed_session), StopReason::SessionClosed)
3854 .await
3855 .unwrap();
3856 let registry = SessionRegistry::new();
3858 let live = registry.get_or_open(
3859 "lifecycle-session",
3860 Workflow::Unsigned,
3861 &AgentIdentity {
3862 version: "1".to_owned(),
3863 charter: "test".into(),
3864 instance_uid: "instance-1".to_owned(),
3865 ttl_remaining_s: Some(600),
3866 },
3867 &Default::default(),
3868 );
3869 assert!(!live.is_closed(), "precondition: live session is open");
3870 assert!(live.receipt.lock().is_none(), "precondition: no receipt yet");
3871 assert!(live.atif_path.lock().is_none(), "precondition: no atif path yet");
3872 finalizer
3874 .recover_spooled_sessions(®istry, &Default::default())
3875 .await
3876 .unwrap();
3877 assert!(
3878 !live.is_closed(),
3879 "live session must not be force-closed by recovery"
3880 );
3881 assert!(
3882 live.receipt.lock().is_none(),
3883 "live session's receipt must not be overwritten from the stale artifact",
3884 );
3885 assert!(
3886 live.atif_path.lock().is_none(),
3887 "live session's atif_path must not be reassigned to the stale artifact",
3888 );
3889 assert_eq!(registry.len(), 1, "recovery must not add a duplicate entry");
3890 }
3891
3892 #[tokio::test]
3900 async fn recovery_skips_tampered_artifact_and_still_recovers_the_rest() {
3901 let directory = tempfile::tempdir().unwrap();
3902 let metrics = Arc::new(Registry::new());
3903 let finalizer = Finalizer::new(
3904 Arc::new(Ed25519Signer::from_seed(&[7; 32])),
3905 directory.path().to_path_buf(),
3906 Arc::clone(&metrics),
3907 );
3908 let step = av_atif::Step {
3909 step_id: 0,
3910 timestamp: None,
3911 source: av_atif::Source::Agent,
3912 message: serde_json::json!("archived response"),
3913 reasoning_effort: None,
3914 reasoning_content: None,
3915 model_name: Some("test-model".into()),
3916 tool_calls: None,
3917 observation: None,
3918 metrics: Some(av_atif::Metrics {
3919 prompt_tokens: Some(1),
3920 completion_tokens: Some(1),
3921 cached_tokens: Some(0),
3922 cost_usd: Some(0.0),
3923 logprobs: None,
3924 completion_token_ids: None,
3925 prompt_token_ids: None,
3926 extra: None,
3927 }),
3928 is_copied_context: None,
3929 llm_call_count: Some(1),
3930 extra: None,
3931 };
3932 let identity = AgentIdentity {
3933 version: "1".to_owned(),
3934 charter: "test".into(),
3935 instance_uid: "instance-1".to_owned(),
3936 ttl_remaining_s: Some(600),
3937 };
3938 let mut artifact_paths = Vec::new();
3939 for id in ["tampered-session", "healthy-session"] {
3940 let session = Arc::new(Session::new(
3941 id.to_owned(),
3942 Workflow::Unsigned,
3943 identity.clone(),
3944 Default::default(),
3945 ));
3946 session.atif.lock().push_step(step.clone()).unwrap();
3947 let outcome = finalizer
3948 .close_session(Arc::clone(&session), StopReason::SessionClosed)
3949 .await
3950 .unwrap();
3951 match outcome {
3952 FinalizeOutcome::Atif { path } => artifact_paths.push(path),
3953 other => panic!("expected FinalizeOutcome::Atif, got {other:?}"),
3954 }
3955 }
3956 let tampered = &artifact_paths[0];
3959 let mut bytes = std::fs::read(tampered).unwrap();
3960 let position = bytes
3961 .windows("archived".len())
3962 .position(|window| window == b"archived")
3963 .expect("artifact must contain the step message");
3964 bytes[position] = b'X';
3965 std::fs::write(tampered, &bytes).unwrap();
3966
3967 let registry = SessionRegistry::new();
3968 finalizer
3969 .recover_spooled_sessions(®istry, &Default::default())
3970 .await
3971 .expect("one tampered artifact must not abort the recovery scan");
3972 assert!(
3973 registry.get("healthy-session").is_some(),
3974 "the healthy artifact must still recover when a sibling is tampered",
3975 );
3976 assert!(
3977 registry.get("tampered-session").is_none(),
3978 "the tampered artifact must not recover a session",
3979 );
3980 assert!(
3981 tampered.exists(),
3982 "the tampered artifact must stay on disk as evidence",
3983 );
3984 let rendered = metrics.render();
3985 assert!(
3986 rendered.contains("av_atif_recovery_skipped_total{reason=\"provenance\"}"),
3987 "skips must be visible to operators via metrics; got: {rendered}",
3988 );
3989 }
3990
3991 #[tokio::test]
4004 async fn round_42_f1_corrupt_receipt_does_not_block_other_atif_recovery() {
4005 let directory = tempfile::tempdir().unwrap();
4006 let metrics = Arc::new(Registry::new());
4007 let finalizer = Finalizer::new(
4008 Arc::new(Ed25519Signer::from_seed(&[42; 32])),
4009 directory.path().to_path_buf(),
4010 Arc::clone(&metrics),
4011 );
4012 let step = av_atif::Step {
4013 step_id: 0,
4014 timestamp: None,
4015 source: av_atif::Source::Agent,
4016 message: serde_json::json!("archived response"),
4017 reasoning_effort: None,
4018 reasoning_content: None,
4019 model_name: Some("test-model".into()),
4020 tool_calls: None,
4021 observation: None,
4022 metrics: Some(av_atif::Metrics {
4023 prompt_tokens: Some(1),
4024 completion_tokens: Some(1),
4025 cached_tokens: Some(0),
4026 cost_usd: Some(0.0),
4027 logprobs: None,
4028 completion_token_ids: None,
4029 prompt_token_ids: None,
4030 extra: None,
4031 }),
4032 is_copied_context: None,
4033 llm_call_count: Some(1),
4034 extra: None,
4035 };
4036 let identity = AgentIdentity {
4037 version: "1".to_owned(),
4038 charter: "test".into(),
4039 instance_uid: "instance-1".to_owned(),
4040 ttl_remaining_s: Some(600),
4041 };
4042 for id in ["poisoned-receipt-session", "healthy-atif-session"] {
4047 let session = Arc::new(Session::new(
4048 id.to_owned(),
4049 Workflow::Unsigned,
4050 identity.clone(),
4051 Default::default(),
4052 ));
4053 session.atif.lock().push_step(step.clone()).unwrap();
4054 finalizer
4055 .close_session(Arc::clone(&session), StopReason::SessionClosed)
4056 .await
4057 .unwrap();
4058 }
4059
4060 let poisoned_receipt = directory.path().join("receipts").join(format!(
4067 "{}.json",
4068 &av_core::digest::sha256_hex(b"poisoned-receipt-session")[..32]
4069 ));
4070 std::fs::create_dir_all(poisoned_receipt.parent().unwrap()).unwrap();
4071 std::fs::write(&poisoned_receipt, b"{not valid receipt json").unwrap();
4072
4073 let registry = SessionRegistry::new();
4074 let outcome = finalizer
4075 .recover_spooled_sessions(®istry, &Default::default())
4076 .await;
4077 assert!(
4078 outcome.is_ok(),
4079 "round-42 F1: corrupt receipt must not abort the ATIF recovery scan; got {outcome:?}",
4080 );
4081 assert!(
4082 registry.get("healthy-atif-session").is_some(),
4083 "the healthy ATIF session MUST recover even when a sibling has a poisoned receipt — this is the HOL-block invariant",
4084 );
4085 let rendered = metrics.render();
4086 assert!(
4087 rendered.contains("av_atif_trajectory_recovery_skipped_total"),
4088 "per-session ATIF recovery skips must be visible to operators via a dedicated counter; got: {rendered}",
4089 );
4090 }
4091
4092 #[tokio::test]
4100 async fn promotion_retry_does_not_force_close_a_live_session() {
4101 let directory = tempfile::tempdir().unwrap();
4102 let finalizer = finalizer(directory.path());
4103 let closed_session = session(Workflow::Unsigned);
4105 closed_session
4106 .atif
4107 .lock()
4108 .push_step(av_atif::Step {
4109 step_id: 0,
4110 timestamp: None,
4111 source: av_atif::Source::Agent,
4112 message: serde_json::json!("archived response"),
4113 reasoning_effort: None,
4114 reasoning_content: None,
4115 model_name: Some("test-model".into()),
4116 tool_calls: None,
4117 observation: None,
4118 metrics: Some(av_atif::Metrics {
4119 prompt_tokens: Some(1),
4120 completion_tokens: Some(1),
4121 cached_tokens: Some(0),
4122 cost_usd: Some(0.0),
4123 logprobs: None,
4124 completion_token_ids: None,
4125 prompt_token_ids: None,
4126 extra: None,
4127 }),
4128 is_copied_context: None,
4129 llm_call_count: Some(1),
4130 extra: None,
4131 })
4132 .unwrap();
4133 let outcome = finalizer
4134 .close_session(Arc::clone(&closed_session), StopReason::SessionClosed)
4135 .await
4136 .unwrap();
4137 let FinalizeOutcome::Atif { path } = outcome else {
4138 panic!("expected ATIF artifact")
4139 };
4140 let trajectory_bytes = tokio::fs::read(&path).await.unwrap();
4141 let promotion_marker = crate::journal::seal(
4142 &finalizer.journal_key,
4143 "promotion-marker",
4144 0,
4145 &PromotionMarker {
4146 session_id: closed_session.id.clone(),
4147 trajectory_digest: av_core::digest::sha256_hex(&trajectory_bytes),
4148 },
4149 )
4150 .unwrap();
4151 tokio::fs::write(path.with_extension("promote"), &promotion_marker)
4152 .await
4153 .unwrap();
4154 let registry = SessionRegistry::new();
4156 let live = registry.get_or_open(
4157 &closed_session.id,
4158 Workflow::Unsigned,
4159 &AgentIdentity {
4160 version: "1".to_owned(),
4161 charter: "test".into(),
4162 instance_uid: "instance-1".to_owned(),
4163 ttl_remaining_s: Some(600),
4164 },
4165 &Default::default(),
4166 );
4167 live.atif
4171 .lock()
4172 .push_step(av_atif::Step {
4173 step_id: 0,
4174 timestamp: None,
4175 source: av_atif::Source::Agent,
4176 message: serde_json::json!("live response"),
4177 reasoning_effort: None,
4178 reasoning_content: None,
4179 model_name: Some("test-model".into()),
4180 tool_calls: None,
4181 observation: None,
4182 metrics: Some(av_atif::Metrics {
4183 prompt_tokens: Some(2),
4184 completion_tokens: Some(3),
4185 cached_tokens: Some(0),
4186 cost_usd: Some(0.0),
4187 logprobs: None,
4188 completion_token_ids: None,
4189 prompt_token_ids: None,
4190 extra: None,
4191 }),
4192 is_copied_context: None,
4193 llm_call_count: Some(1),
4194 extra: None,
4195 })
4196 .unwrap();
4197 assert!(!live.is_closed(), "precondition: live session is open");
4198 let promoted = finalizer.retry_marked_promotions(®istry).await.unwrap();
4199 assert_eq!(
4200 promoted, 0,
4201 "promotion retry must not count a skipped live session as promoted",
4202 );
4203 assert!(
4204 !live.is_closed(),
4205 "live session must not be force-closed by promotion retry",
4206 );
4207 assert!(
4208 live.receipt.lock().is_none(),
4209 "live session must not receive a receipt from a stale promotion marker",
4210 );
4211 assert!(
4212 live.atif_path.lock().is_none(),
4213 "live session's atif_path must not be set by a background promotion retry — that would mean its trajectory was snapshotted to disk out of band",
4214 );
4215 assert_eq!(
4216 tokio::fs::read(&path).await.unwrap(),
4217 trajectory_bytes,
4218 "the archived ATIF artifact must not be overwritten by a background retry that force-finalized the live session",
4219 );
4220 assert!(
4223 path.with_extension("promote").exists(),
4224 "the promotion marker must remain on disk for future retries",
4225 );
4226 }
4227
4228 #[tokio::test]
4247 async fn recovery_marks_quarantined_unsigned_session_finalized_to_stop_idle_sweep_churn() {
4248 let directory = tempfile::tempdir().unwrap();
4249 let finalizer = finalizer(directory.path());
4250 let session_id = "quarantined-unsigned";
4251 let identity = AgentIdentity {
4252 version: "1".into(),
4253 charter: "test".into(),
4254 instance_uid: "instance-1".into(),
4255 ttl_remaining_s: Some(600),
4256 };
4257
4258 let digest = av_core::digest::sha256_hex(session_id.as_bytes());
4259 let stem = &digest[..32];
4260 let metadata_payload = serde_json::json!({
4261 "journal_version": 2,
4262 "session_id": session_id,
4263 "identity": identity,
4264 "workflow": "unsigned",
4265 });
4266 let metadata_sealed =
4267 crate::journal::seal(&finalizer.journal_key, "metadata", 0, &metadata_payload).unwrap();
4268 std::fs::write(
4269 directory.path().join(format!("{stem}.session.json")),
4270 &metadata_sealed,
4271 )
4272 .unwrap();
4273
4274 finalizer
4278 .quarantined_sessions
4279 .lock()
4280 .insert(session_id.to_owned());
4281
4282 let registry = SessionRegistry::new();
4283 finalizer
4284 .recover_spooled_sessions(®istry, &Default::default())
4285 .await
4286 .unwrap();
4287
4288 let recovered = registry
4289 .get(session_id)
4290 .expect("quarantined session must be inserted");
4291 assert!(
4292 recovered.capture_failed(),
4293 "quarantined session must carry the capture-failed verdict",
4294 );
4295 assert!(
4296 recovered.is_closed(),
4297 "the quarantined-unsigned recovery branch (consolidate_step_journals) must also mark the session finalized (artifact_committed) so the idle sweeper's `!is_closed()` filter skips it — otherwise every idle tick calls close_session_locked which returns CaptureIncomplete, CloseClaim resets the close, and the session churns forever burning CPU, log noise, and metrics without ever leaving the registry",
4298 );
4299 }
4300
4301 #[tokio::test]
4309 async fn atif_scan_ignores_live_session_journal_metadata() {
4310 let directory = tempfile::tempdir().unwrap();
4311 let metrics = Arc::new(Registry::new());
4312 let finalizer = Finalizer::new(
4313 Arc::new(Ed25519Signer::from_seed(&[7; 32])),
4314 directory.path().to_path_buf(),
4315 Arc::clone(&metrics),
4316 );
4317 let session_id = "still-open-session";
4318 let identity = AgentIdentity {
4319 version: "1".into(),
4320 charter: "test".into(),
4321 instance_uid: "instance-1".into(),
4322 ttl_remaining_s: Some(600),
4323 };
4324 let digest = av_core::digest::sha256_hex(session_id.as_bytes());
4325 let stem = &digest[..32];
4326 let metadata_payload = serde_json::json!({
4327 "journal_version": 2,
4328 "session_id": session_id,
4329 "identity": identity,
4330 "workflow": "unsigned",
4331 });
4332 let metadata_sealed =
4333 crate::journal::seal(&finalizer.journal_key, "metadata", 0, &metadata_payload).unwrap();
4334 std::fs::write(
4335 directory.path().join(format!("{stem}.session.json")),
4336 &metadata_sealed,
4337 )
4338 .unwrap();
4339
4340 let registry = SessionRegistry::new();
4342 let live = registry.get_or_open(session_id, Workflow::Unsigned, &identity, &Default::default());
4343 assert!(!live.is_closed(), "precondition: session is open");
4344
4345 finalizer
4346 .recover_spooled_sessions(®istry, &Default::default())
4347 .await
4348 .unwrap();
4349
4350 assert!(
4351 !metrics.render().contains("av_atif_recovery_skipped_total"),
4352 "the ATIF scan must skip *.session.json instead of counting it as an invalid spool file",
4353 );
4354 assert!(
4355 directory.path().join(format!("{stem}.session.json")).exists(),
4356 "the live session's journal metadata must survive the pass",
4357 );
4358 }
4359
4360 #[tokio::test]
4361 async fn restart_quarantines_inflight_response_without_stopping_recovery() {
4362 let directory = tempfile::tempdir().unwrap();
4363 let finalizer = finalizer(directory.path());
4364 crate::worker::create_response_marker(
4365 directory.path(),
4366 &finalizer.journal_key,
4367 "uncertain-session",
4368 "request-digest".to_owned(),
4369 )
4370 .await
4371 .unwrap();
4372
4373 let registry = SessionRegistry::new();
4374 assert_eq!(
4375 finalizer
4376 .recover_spooled_sessions(®istry, &Default::default())
4377 .await
4378 .unwrap(),
4379 0
4380 );
4381 assert!(finalizer
4382 .quarantined_sessions
4383 .lock()
4384 .contains("uncertain-session"));
4385 assert!(registry.get("uncertain-session").is_none());
4386 }
4387
4388 #[tokio::test]
4389 async fn restart_recovers_atif_and_retries_marked_promotion() {
4390 let directory = tempfile::tempdir().unwrap();
4391 let first = finalizer(directory.path());
4392 let original = session(Workflow::Unsigned);
4393 original
4394 .atif
4395 .lock()
4396 .push_step(av_atif::Step {
4397 step_id: 0,
4398 timestamp: None,
4399 source: av_atif::Source::Agent,
4400 message: serde_json::json!("recovered response"),
4401 reasoning_effort: None,
4402 reasoning_content: None,
4403 model_name: Some("test-model".into()),
4404 tool_calls: None,
4405 observation: None,
4406 metrics: Some(av_atif::Metrics {
4407 prompt_tokens: Some(5),
4408 completion_tokens: Some(2),
4409 cached_tokens: Some(1),
4410 cost_usd: Some(0.001),
4411 logprobs: None,
4412 completion_token_ids: None,
4413 prompt_token_ids: None,
4414 extra: None,
4415 }),
4416 is_copied_context: None,
4417 llm_call_count: Some(1),
4418 extra: None,
4419 })
4420 .unwrap();
4421 let outcome = first
4422 .close_session(Arc::clone(&original), StopReason::SessionClosed)
4423 .await
4424 .unwrap();
4425 let FinalizeOutcome::Atif { path } = outcome else {
4426 panic!("expected ATIF artifact")
4427 };
4428 let trajectory_bytes = tokio::fs::read(&path).await.unwrap();
4429 let promotion_marker = crate::journal::seal(
4430 &first.journal_key,
4431 "promotion-marker",
4432 0,
4433 &PromotionMarker {
4434 session_id: original.id.clone(),
4435 trajectory_digest: av_core::digest::sha256_hex(&trajectory_bytes),
4436 },
4437 )
4438 .unwrap();
4439 tokio::fs::write(path.with_extension("promote"), &promotion_marker)
4440 .await
4441 .unwrap();
4442
4443 let recovered_registry = SessionRegistry::new();
4444 let after_restart = finalizer(directory.path());
4445 assert_eq!(
4446 after_restart
4447 .recover_spooled_sessions(&recovered_registry, &Default::default())
4448 .await
4449 .unwrap(),
4450 1
4451 );
4452 assert_eq!(
4453 after_restart
4454 .retry_marked_promotions(&recovered_registry)
4455 .await
4456 .unwrap(),
4457 1
4458 );
4459 let recovered = recovered_registry.get(&original.id).unwrap();
4460 let receipt = recovered.receipt.lock().clone().unwrap();
4461 receipt.verify_embedded().unwrap();
4462 assert!(recovered.is_promoted());
4463 assert!(!path.with_extension("promote").exists());
4464
4465 tokio::fs::write(path.with_extension("promote"), &promotion_marker)
4466 .await
4467 .unwrap();
4468 let second_registry = SessionRegistry::new();
4469 assert_eq!(
4470 after_restart
4471 .recover_spooled_sessions(&second_registry, &Default::default())
4472 .await
4473 .unwrap(),
4474 1
4475 );
4476 let restored = second_registry.get(&original.id).unwrap();
4477 assert_eq!(
4478 restored.receipt.lock().as_ref().unwrap().body.receipt_id,
4479 receipt.body.receipt_id,
4480 "restart must restore the persisted receipt, not issue a duplicate"
4481 );
4482 assert_eq!(
4483 after_restart
4484 .retry_marked_promotions(&second_registry)
4485 .await
4486 .unwrap(),
4487 1
4488 );
4489 assert!(restored.is_promoted());
4490 assert!(!path.with_extension("promote").exists());
4491 }
4492
4493 #[tokio::test]
4494 async fn close_waits_for_active_response_lease() {
4495 let directory = tempfile::tempdir().unwrap();
4496 let finalizer = finalizer(directory.path());
4497 let session = session(Workflow::Signed);
4498 let lease = crate::session::SessionLease::new(Arc::clone(&session));
4499 let close_session = Arc::clone(&session);
4500 let task = tokio::spawn(async move {
4501 finalizer
4502 .close_session(close_session, StopReason::SessionClosed)
4503 .await
4504 });
4505 tokio::task::yield_now().await;
4506 assert!(!task.is_finished(), "close overtook an active response");
4507 drop(lease);
4508 assert!(matches!(
4509 task.await.unwrap().unwrap(),
4510 FinalizeOutcome::Receipt { .. }
4511 ));
4512 }
4513
4514 #[test]
4523 fn warned_artifacts_clamps_zero_cap_to_one() {
4524 let mut warned = WarnedArtifacts::new(0);
4525 assert!(warned.insert(PathBuf::from("a")));
4527 assert_eq!(warned.len(), 1);
4528 assert!(!warned.insert(PathBuf::from("a")));
4530 assert!(warned.insert(PathBuf::from("b")));
4532 assert_eq!(warned.len(), 1);
4533 }
4534
4535 #[test]
4536 fn warned_artifacts_evicts_one_at_a_time_not_all_at_once() {
4537 let mut warned = WarnedArtifacts::new(3);
4538 assert!(warned.insert(PathBuf::from("a")));
4539 assert!(warned.insert(PathBuf::from("b")));
4540 assert!(warned.insert(PathBuf::from("c")));
4541 assert_eq!(warned.len(), 3);
4542 assert!(!warned.insert(PathBuf::from("b")));
4545 assert!(warned.insert(PathBuf::from("d")));
4548 assert_eq!(warned.len(), 3);
4549 assert!(!warned.insert(PathBuf::from("b")));
4550 assert!(!warned.insert(PathBuf::from("c")));
4551 assert!(!warned.insert(PathBuf::from("d")));
4552 assert!(warned.insert(PathBuf::from("a")));
4557 assert_eq!(warned.len(), 3);
4558 assert!(!warned.insert(PathBuf::from("c")));
4559 assert!(!warned.insert(PathBuf::from("d")));
4560 assert!(!warned.insert(PathBuf::from("a")));
4561 }
4562
4563 #[tokio::test]
4564 async fn torn_journal_tail_is_truncated_without_losing_complete_records() {
4565 let directory = tempfile::tempdir().unwrap();
4566 let path = directory.path().join("journal.ndjson");
4567 std::fs::write(&path, b"{\"complete\":true}\n{\"torn\":").unwrap();
4568 let lines = read_complete_journal(&path).await.unwrap();
4569 assert_eq!(lines, vec![r#"{"complete":true}"#]);
4570 assert_eq!(std::fs::read(&path).unwrap(), b"{\"complete\":true}\n");
4571 }
4572
4573 #[tokio::test]
4580 async fn journal_with_no_complete_lines_is_quarantined_not_truncated() {
4581 let directory = tempfile::tempdir().unwrap();
4582 let path = directory.path().join("journal.ndjson");
4583 std::fs::write(&path, b"{\"torn_before_first_newline\":").unwrap();
4584 let outcome = read_complete_journal(&path).await;
4585 let err = outcome.unwrap_err();
4587 assert!(
4588 format!("{err:?}").contains("no complete lines"),
4589 "expected quarantine error, got {err:?}",
4590 );
4591 assert!(!path.exists(), "journal was not moved out of the recovery path");
4593 let entries: Vec<_> = std::fs::read_dir(directory.path())
4596 .unwrap()
4597 .filter_map(Result::ok)
4598 .filter(|entry| entry.file_name().to_string_lossy().contains(".corrupt-"))
4599 .collect();
4600 assert_eq!(entries.len(), 1, "expected exactly one quarantine file");
4601 let bytes = std::fs::read(entries[0].path()).unwrap();
4602 assert_eq!(&bytes, b"{\"torn_before_first_newline\":");
4603 let msg = format!("{err}");
4612 assert!(
4613 !msg.contains(directory.path().to_string_lossy().as_ref()),
4614 "FinalizeError::Atif leaked spool dir absolute path: {msg}"
4615 );
4616 assert!(
4617 !msg.contains(std::path::MAIN_SEPARATOR_STR),
4618 "FinalizeError::Atif still contains a path separator: {msg}"
4619 );
4620 assert!(
4623 msg.contains("journal.ndjson"),
4624 "FinalizeError::Atif should still mention the journal basename: {msg}"
4625 );
4626 }
4627
4628 #[tokio::test]
4629 async fn failed_persistence_reopens_session_for_retry() {
4630 let directory = tempfile::tempdir().unwrap();
4631 let blocking_file = directory.path().join("not-a-directory");
4632 std::fs::write(&blocking_file, b"file").unwrap();
4633 let finalizer = finalizer(&blocking_file);
4634 let session = session(Workflow::Signed);
4635 assert!(finalizer
4636 .close_session(Arc::clone(&session), StopReason::SessionClosed)
4637 .await
4638 .is_err());
4639 assert!(!session.is_closed());
4640 assert!(session.try_close(), "failed close claim was not reset");
4641 }
4642
4643 #[tokio::test]
4644 async fn failed_unsigned_persistence_keeps_steps_for_retry() {
4645 let directory = tempfile::tempdir().unwrap();
4646 let spool = directory.path().join("spool");
4647 std::fs::write(&spool, b"blocking file").unwrap();
4648 let finalizer = finalizer(&spool);
4649 let session = session(Workflow::Unsigned);
4650 session
4651 .atif
4652 .lock()
4653 .push_step(av_atif::Step {
4654 step_id: 0,
4655 timestamp: None,
4656 source: av_atif::Source::User,
4657 message: serde_json::json!("survive"),
4658 reasoning_effort: None,
4659 reasoning_content: None,
4660 model_name: None,
4661 tool_calls: None,
4662 observation: None,
4663 metrics: None,
4664 is_copied_context: None,
4665 llm_call_count: None,
4666 extra: None,
4667 })
4668 .unwrap();
4669 assert!(finalizer
4670 .close_session(Arc::clone(&session), StopReason::SessionClosed)
4671 .await
4672 .is_err());
4673 assert!(!session.is_closed());
4674 std::fs::remove_file(&spool).unwrap();
4675 std::fs::create_dir(&spool).unwrap();
4676 let FinalizeOutcome::Atif { path } = finalizer
4677 .close_session(session, StopReason::SessionClosed)
4678 .await
4679 .unwrap()
4680 else {
4681 panic!("expected ATIF")
4682 };
4683 let trajectory: av_atif::Trajectory = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap();
4684 assert_eq!(trajectory.steps.len(), 1);
4685 assert_eq!(trajectory.steps[0].message, serde_json::json!("survive"));
4686 }
4687
4688 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4699 async fn concurrent_closes_across_many_sessions_never_deadlock() {
4700 const SESSIONS: usize = 32;
4701 let directory = tempfile::tempdir().unwrap();
4702 let finalizer = Arc::new(finalizer(directory.path()));
4703 let mut tasks = Vec::with_capacity(SESSIONS);
4704 for i in 0..SESSIONS {
4705 let f = Arc::clone(&finalizer);
4706 let s = Arc::new(Session::new(
4707 format!("lifecycle-{i}"),
4708 Workflow::Signed,
4709 AgentIdentity {
4710 version: "1".to_owned(),
4711 charter: "test".into(),
4712 instance_uid: format!("instance-{i}"),
4713 ttl_remaining_s: Some(600),
4714 },
4715 Default::default(),
4716 ));
4717 tasks.push(tokio::spawn(async move {
4718 f.close_session(s, StopReason::SessionClosed).await
4719 }));
4720 }
4721 let results = tokio::time::timeout(
4722 std::time::Duration::from_secs(15),
4723 futures::future::join_all(tasks),
4724 )
4725 .await
4726 .expect("close_session tasks deadlocked under lifecycle_lock contention");
4727 for result in results {
4728 let outcome = result.expect("task panicked");
4729 assert!(outcome.is_ok(), "close failed: {outcome:?}");
4730 }
4731 }
4732
4733 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4740 async fn per_session_lock_table_prunes_entries_after_close() {
4741 const SESSIONS: usize = 200;
4742 let directory = tempfile::tempdir().unwrap();
4743 let finalizer = Arc::new(finalizer(directory.path()));
4744 let locks = finalizer.lifecycle_locks();
4745 let mut tasks = Vec::with_capacity(SESSIONS);
4746 for i in 0..SESSIONS {
4747 let f = Arc::clone(&finalizer);
4748 let s = Arc::new(Session::new(
4749 format!("prune-{i}"),
4750 Workflow::Signed,
4751 AgentIdentity {
4752 version: "1".to_owned(),
4753 charter: "test".into(),
4754 instance_uid: format!("instance-{i}"),
4755 ttl_remaining_s: Some(600),
4756 },
4757 Default::default(),
4758 ));
4759 tasks.push(tokio::spawn(async move {
4760 f.close_session(s, StopReason::SessionClosed).await
4761 }));
4762 }
4763 for task in tasks {
4764 let _ = task.await.expect("task panicked");
4765 }
4766 for _ in 0..20 {
4770 if locks.len() == 0 {
4771 return;
4772 }
4773 tokio::time::sleep(std::time::Duration::from_millis(25)).await;
4774 }
4775 panic!(
4776 "SessionLockTable did not prune to empty after {SESSIONS} closes; \
4777 residual entries = {}",
4778 locks.len()
4779 );
4780 }
4781
4782 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4794 async fn recovery_scan_does_not_head_of_line_block_unrelated_close() {
4795 let directory = tempfile::tempdir().unwrap();
4796 let finalizer = Arc::new(finalizer(directory.path()));
4797 for i in 0..512 {
4804 let path = directory.path().join(format!("scan-probe-{i}.json"));
4805 let payload = serde_json::json!({
4806 "atif_version": "1.7",
4807 "session_id": format!("scan-probe-{i}"),
4808 "agent": {"version": "1", "charter": "test", "instance_uid": "x"},
4809 "steps": [],
4810 "provenance": {"scheme": "none"},
4811 });
4812 tokio::fs::write(&path, serde_json::to_vec(&payload).unwrap())
4813 .await
4814 .unwrap();
4815 }
4816 let f_scan = Arc::clone(&finalizer);
4817 let scan_task = tokio::spawn(async move {
4818 let _ = f_scan
4819 .recover_spooled_sessions(&crate::session::SessionRegistry::new(), &Default::default())
4820 .await;
4821 std::time::Instant::now()
4822 });
4823 tokio::task::yield_now().await;
4827
4828 let session = Arc::new(Session::new(
4829 "unrelated-close".to_owned(),
4830 Workflow::Signed,
4831 AgentIdentity {
4832 version: "1".to_owned(),
4833 charter: "test".into(),
4834 instance_uid: "instance-close".to_owned(),
4835 ttl_remaining_s: Some(600),
4836 },
4837 Default::default(),
4838 ));
4839 let _ = tokio::time::timeout(
4849 std::time::Duration::from_secs(60),
4850 finalizer.close_session(session, StopReason::SessionClosed),
4851 )
4852 .await
4853 .expect("unrelated close deadlocked behind the recovery scan");
4854 let close_end = std::time::Instant::now();
4855 let scan_end = scan_task.await.unwrap();
4856 assert!(
4857 close_end < scan_end,
4858 "unrelated close finished only after the recovery scan ended — the head-of-line \
4859 blocking signature of a global lifecycle lock (close_end {close_end:?} >= scan_end \
4860 {scan_end:?})"
4861 );
4862 }
4863
4864 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
4872 async fn close_latency_scales_reasonably_under_lock_contention() {
4873 let directory = tempfile::tempdir().unwrap();
4874 let finalizer = Arc::new(finalizer(directory.path()));
4875 let warm = Arc::new(Session::new(
4877 "warm".to_owned(),
4878 Workflow::Signed,
4879 AgentIdentity {
4880 version: "1".to_owned(),
4881 charter: "test".into(),
4882 instance_uid: "warm".to_owned(),
4883 ttl_remaining_s: Some(600),
4884 },
4885 Default::default(),
4886 ));
4887 let uncontended = std::time::Instant::now();
4888 finalizer
4889 .close_session(warm, StopReason::SessionClosed)
4890 .await
4891 .unwrap();
4892 let baseline = uncontended.elapsed();
4893
4894 const N: usize = 16;
4896 let mut tasks = Vec::with_capacity(N);
4897 let started = std::time::Instant::now();
4898 for i in 0..N {
4899 let f = Arc::clone(&finalizer);
4900 let s = Arc::new(Session::new(
4901 format!("contended-{i}"),
4902 Workflow::Signed,
4903 AgentIdentity {
4904 version: "1".to_owned(),
4905 charter: "test".into(),
4906 instance_uid: format!("contended-{i}"),
4907 ttl_remaining_s: Some(600),
4908 },
4909 Default::default(),
4910 ));
4911 tasks.push(tokio::spawn(async move {
4912 f.close_session(s, StopReason::SessionClosed).await
4913 }));
4914 }
4915 for t in tasks {
4916 t.await.unwrap().unwrap();
4917 }
4918 let total = started.elapsed();
4919 let multiplier = u32::try_from(N * 10).unwrap_or(u32::MAX);
4923 let budget = baseline
4924 .saturating_mul(multiplier)
4925 .max(std::time::Duration::from_secs(60));
4926 assert!(
4927 total < budget,
4928 "16 contended closes took {total:?}, budget {budget:?} (baseline {baseline:?})",
4929 );
4930 }
4931}