Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(claude): route interleaved results by provenance (#2152)

Call Whitef0eb25d7433b
diff --git a/core/handlers/session_handler.py b/core/handlers/session_handler.py
index 82c40aca0..0158513f6 100644
--- a/core/handlers/session_handler.py
+++ b/core/handlers/session_handler.py
@@ -103,6 +103,15 @@ class ClaudeSessionNotFoundError(RuntimeError):
)
+class ClaudeInputNotSentError(RuntimeError):
+ """A local admission failure with definitive evidence of no native write."""
+
+ def __init__(self, reason: str, message_key: str):
+ self.reason = reason
+ self.message_key = message_key
+ super().__init__(reason)
+
+
class _ClaudeReceiverCleanupRequired(RuntimeError):
"""Signal that a dead generation must be retried after receiver cleanup."""
@@ -2719,6 +2728,14 @@ class SessionHandler(BaseHandler):
error_msg = str(error)
# Check for specific error types
+ if isinstance(error, ClaudeInputNotSentError):
+ # This input never reached Claude. Keep existing background work,
+ # persisted mappings and any replacement client intact.
+ await self._get_im_client(context).send_message(
+ context,
+ self._get_formatter(context).format_error(self._t(error.message_key)),
+ )
+ return False
if isinstance(error, ClaudeSessionNotFoundError):
logger.warning(
"Claude session %s not found for current working directory %s; keeping persisted mapping unchanged",
diff --git a/core/session_activities.py b/core/session_activities.py
index 9fe7099a4..b2dad9a2c 100644
--- a/core/session_activities.py
+++ b/core/session_activities.py
@@ -274,6 +274,18 @@ class SessionActivityRegistry:
str, list[tuple[datetime, str]]
] = defaultdict(list)
self._recovered_terminals: deque[SessionActivity] = deque()
+ # Failed/stopped/killed Activities can become terminal before the
+ # Claude Result that classifies their owner arrives. Keep those
+ # snapshots addressable until the terminal owner is known.
+ self._terminal_snapshots: dict[
+ tuple[str, str], dict[str, SessionActivity]
+ ] = defaultdict(dict)
+ self._terminal_snapshot_identities: dict[
+ tuple[str, str, str], RuntimeActivationIdentity
+ ] = {}
+ self._provenance_persistence_recovery: dict[
+ tuple[str, str], dict[str, str]
+ ] = {}
self._restore()
def set_output_settled_callback(
@@ -393,27 +405,95 @@ class SessionActivityRegistry:
def _persist_activity(self, activity: SessionActivity, *, phase: str) -> None:
upsert = getattr(self._store, "upsert_activity", None)
- if not callable(upsert):
+ if callable(upsert):
+ try:
+ upsert(activity.to_dict(), phase=phase)
+ except Exception:
+ logger.warning("Failed to persist Activity %s", activity.id, exc_info=True)
+ raise
+ # Any successful later write supersedes an older failed provenance
+ # attempt, including a transition to output or terminal ownership.
+ self._clear_provenance_persistence_recovery(activity)
+
+ def _clear_provenance_persistence_recovery(self, activity: SessionActivity) -> None:
+ key = (activity.backend, activity.runtime_key)
+ pending = self._provenance_persistence_recovery.get(key)
+ if pending is not None:
+ pending.pop(activity.id, None)
+ if not pending:
+ self._provenance_persistence_recovery.pop(key, None)
+
+ def _record_provenance_persistence_recovery(
+ self,
+ activity: SessionActivity,
+ *,
+ phase: str,
+ error: BaseException,
+ ) -> None:
+ key = (activity.backend, activity.runtime_key)
+ self._provenance_persistence_recovery.setdefault(key, {})[
+ activity.id
+ ] = f"{phase}: {type(error).__name__}: {error}"
+
+ def provenance_persistence_recovery(
+ self,
+ backend: str,
+ runtime_key: str,
+ ) -> dict[str, str]:
+ """Return provenance writes that remain retryable after a store failure."""
+
+ with self._lock:
+ return dict(
+ self._provenance_persistence_recovery.get(
+ (str(backend), str(runtime_key)),
+ {},
+ )
+ )
+
+ def _retry_provenance_persistence_locked(
+ self,
+ key: tuple[str, str],
+ ) -> None:
+ pending = self._provenance_persistence_recovery.get(key)
+ if not pending:
return
- try:
- upsert(activity.to_dict(), phase=phase)
- except Exception:
- logger.warning("Failed to persist Activity %s", activity.id, exc_info=True)
- raise
+
+ # A diagnostic describes a past attempt, not the current durable phase.
+ # Read lifecycle authority from the same containers that own the object.
+ activities: dict[str, tuple[SessionActivity, str]] = {}
+ for activity_key, activity in self._active.items():
+ if activity_key[:2] == key and activity.id in pending:
+ activities[activity.id] = (activity, "active")
+ for entry in self._completed_outputs.get(key) or ():
+ if entry.activity.id in pending:
+ activities[entry.activity.id] = (entry.activity, "awaiting_output")
+ for claimed in self._claimed_completed_outputs.values():
+ activity = claimed.entry.activity
+ if (activity.backend, activity.runtime_key) == key and activity.id in pending:
+ activities[activity.id] = (activity, "awaiting_output")
+ for activity in self._terminal_snapshots.get(key, {}).values():
+ if activity.id in pending:
+ activities[activity.id] = (activity, TERMINAL_SNAPSHOT_PHASE)
+
+ for activity, phase in activities.values():
+ try:
+ self._persist_activity(activity, phase=phase)
+ except Exception:
+ continue
def _delete_activity(self, activity: SessionActivity) -> None:
delete = getattr(self._store, "delete_activity", None)
- if not callable(delete):
- return
- try:
- delete(
- backend=activity.backend,
- runtime_key=activity.runtime_key,
- activity_id=activity.id,
- )
- except Exception:
- logger.warning("Failed to delete Activity snapshot %s", activity.id, exc_info=True)
- raise
+ if callable(delete):
+ try:
+ delete(
+ backend=activity.backend,
+ runtime_key=activity.runtime_key,
+ activity_id=activity.id,
+ )
+ except Exception:
+ logger.warning("Failed to delete Activity snapshot %s", activity.id, exc_info=True)
+ raise
+ self._clear_provenance_persistence_recovery(activity)
def _persist_connection(
self,
@@ -506,7 +586,13 @@ class SessionActivityRegistry:
completed_at=now,
)
)
- self._recovered_terminals.append(recovered)
+ self._terminal_snapshots[connection_key][recovered.id] = recovered
+ # Process-local generation numbers can repeat after a restart.
+ # Recovered terminals have no live classifier, not the identity of
+ # a freshly attached client whose counter happens to match.
+ finalized = self._finalize_generation_ended_snapshot_locked(recovered)
+ if finalized is None:
+ self._recovered_terminals.append(recovered)
def set_connection(
self,
@@ -694,6 +780,9 @@ class SessionActivityRegistry:
) -> SessionActivity:
key = self._key(backend, runtime_key, activity_id)
now = _now_iso()
+ incoming_metadata = dict(metadata or {})
+ if activation_identity is not None:
+ incoming_metadata["_runtime_generation"] = activation_identity.generation
existing = self._active.get(key)
if existing is None:
activity = SessionActivity(
@@ -708,13 +797,13 @@ class SessionActivityRegistry:
parent_activity_id=parent_activity_id,
turn_id=turn_id,
run_id=run_id,
- metadata=dict(metadata or {}),
+ metadata=incoming_metadata,
started_at=now,
updated_at=now,
)
else:
merged = dict(existing.metadata)
- merged.update(metadata or {})
+ merged.update(incoming_metadata)
activity = replace(
existing,
session_id=session_id or existing.session_id,
@@ -769,9 +858,17 @@ class SessionActivityRegistry:
return self._complete_locked(
key=key,
status=normalized,
- metadata=metadata,
+ metadata={
+ **(metadata or {}),
+ **(
+ {"_runtime_generation": activation_identity.generation}
+ if activation_identity is not None
+ else {}
+ ),
+ },
expects_output=expects_output,
retain_terminal_snapshot=retain_terminal_snapshot,
+ activation_identity=activation_identity,
)
return self._commit_activation_write(
@@ -789,6 +886,7 @@ class SessionActivityRegistry:
metadata: dict[str, Any] | None,
expects_output: bool,
retain_terminal_snapshot: bool,
+ activation_identity: RuntimeActivationIdentity | None = None,
) -> SessionActivity | None:
backend, runtime_key, _activity_id = key
now = _now_iso()
@@ -813,14 +911,65 @@ class SessionActivityRegistry:
)
elif retain_terminal_snapshot:
self._persist_activity(completed, phase=TERMINAL_SNAPSHOT_PHASE)
+ identity = self._active_identities.get(key) or activation_identity
self._active.pop(key, None)
self._active_identities.pop(key, None)
+ self._terminal_snapshots[(backend, runtime_key)][completed.id] = completed
+ if identity is not None:
+ self._terminal_snapshot_identities[key] = identity
else:
self._delete_activity(completed)
self._active.pop(key, None)
self._active_identities.pop(key, None)
return completed
+ def _finalize_generation_ended_snapshot_locked(
+ self,
+ activity: SessionActivity,
+ ) -> SessionActivity | None:
+ """Close provenance waiting when its native generation cannot answer."""
+
+ key = (activity.backend, activity.runtime_key)
+ snapshots = self._terminal_snapshots.get(key)
+ current = snapshots.get(activity.id) if snapshots is not None else None
+ if current is None or not current.metadata.get("provenance_pending"):
+ return None
+
+ metadata = dict(current.metadata)
+ metadata.pop("provenance_pending", None)
+ metadata.pop("provenance_human", None)
+ metadata.pop("provenance_detached", None)
+ metadata.pop("run_ids", None)
+ metadata.pop("delivery_key_external", None)
+ metadata["provenance_generation_ended"] = True
+ metadata["provenance_unresolved"] = True
+ finalized = replace(
+ current,
+ foreground=False,
+ detached_from_run=True,
+ turn_id=None,
+ run_id=None,
+ metadata=metadata,
+ updated_at=_now_iso(),
+ )
+ snapshots[finalized.id] = finalized
+ try:
+ self._persist_activity(finalized, phase=TERMINAL_SNAPSHOT_PHASE)
+ except Exception as error:
+ self._record_provenance_persistence_recovery(
+ finalized,
+ phase=TERMINAL_SNAPSHOT_PHASE,
+ error=error,
+ )
+ if not any(
+ previous.backend == finalized.backend
+ and previous.runtime_key == finalized.runtime_key
+ and previous.id == finalized.id
+ for previous in self._recovered_terminals
+ ):
+ self._recovered_terminals.append(finalized)
+ return finalized
+
def active_for_runtime(self, backend: str, runtime_key: str) -> list[SessionActivity]:
prefix = (str(backend), str(runtime_key))
with self._lock:
@@ -907,12 +1056,19 @@ class SessionActivityRegistry:
runtime_key: str,
*,
turn_ids: set[str] | None = None,
+ metadata_match: dict[str, Any] | None = None,
+ bind_receipt: bool = True,
) -> list[SessionActivity]:
"""Atomically claim one causal batch without disturbing other output.
Explicit ``turn_ids`` select every matching completion in FIFO order,
even when unrelated output is interleaved. Without them, the queue head
defines the batch; turn-less legacy completions remain single-item.
+ ``metadata_match`` is an eligibility predicate for the same FIFO and
+ persisted-receipt selection rules; it never creates a second batching
+ algorithm. A recovery owner may defer binding to retain the exact claim
+ and payload before a fallible receipt write; it must then bind that
+ claim before delivery or explicitly requeue it.
"""
key = (str(backend), str(runtime_key))
@@ -944,6 +1100,11 @@ class SessionActivityRegistry:
for entry in queue
if identities is None
or str(entry.activity.turn_id or "").strip() in identities
+ if metadata_match is None
+ or not any(
+ entry.activity.metadata.get(name) != value
+ for name, value in metadata_match.items()
+ )
),
None,
)
@@ -963,21 +1124,231 @@ class SessionActivityRegistry:
if identities is None:
head_turn_id = str(candidate.turn_id or "").strip()
if not head_turn_id:
+ phase_id = str(
+ (metadata_match or {}).get("provenance_phase_id") or ""
+ ).strip()
claimed = self._claim_completed_outputs(
backend,
runtime_key,
unbound_only=True,
- limit=1,
+ metadata_match=metadata_match,
+ limit=None if phase_id else 1,
+ )
+ return (
+ self._bind_claimed_output_batch_or_requeue(claimed)
+ if bind_receipt else claimed
)
- return self._bind_claimed_output_batch_or_requeue(claimed)
identities = {head_turn_id}
claimed = self._claim_completed_outputs(
backend,
runtime_key,
turn_ids=identities,
unbound_only=True,
+ metadata_match=metadata_match,
+ )
+ return (
+ self._bind_claimed_output_batch_or_requeue(claimed)
+ if bind_receipt else claimed
+ )
+
+ def classify_provisional_provenance(
+ self,
+ backend: str,
+ runtime_key: str,
+ *,
+ activity_ids: set[str],
+ parent_activity_ids: set[str],
+ turn_id: str | None = None,
+ run_ids: list[str] | tuple[str, ...] = (),
+ delivery_key_external: str | None = None,
+ phase_id: str | None = None,
+ detached: bool,
+ ) -> list[SessionActivity]:
+ """Classify only the provisional Activity phase named by its producer.
+
+ The receiver may know that a buffered phase is human-owned or detached
+ before a completion receipt is emitted. Update active, queued, and
+ already-claimed entries in place without changing receipt membership,
+ sequence, or idempotency metadata.
+ """
+
+ normalized_ids = {str(value or "").strip() for value in activity_ids}
+ normalized_parents = {
+ str(value or "").strip() for value in parent_activity_ids
+ }
+ normalized_ids.discard("")
+ normalized_parents.discard("")
+ normalized_run_ids = tuple(
+ dict.fromkeys(
+ str(value or "").strip()
+ for value in run_ids
+ if str(value or "").strip()
+ )
+ )
+ normalized_turn_id = str(turn_id or "").strip() or None
+ normalized_delivery_key = str(delivery_key_external or "").strip() or None
+ normalized_phase_id = str(phase_id or "").strip() or None
+ key = (str(backend), str(runtime_key))
+ classified: list[SessionActivity] = []
+
+ def matches(activity: SessionActivity) -> bool:
+ if not activity.metadata.get("provenance_pending"):
+ return False
+ return (
+ activity.id in normalized_ids
+ or str(activity.parent_activity_id or "").strip()
+ in normalized_parents
+ )
+
+ def classify(activity: SessionActivity) -> SessionActivity:
+ metadata = dict(activity.metadata)
+ metadata.pop("provenance_pending", None)
+ metadata.pop("provenance_human", None)
+ metadata.pop("provenance_detached", None)
+ if normalized_phase_id:
+ metadata["provenance_phase_id"] = normalized_phase_id
+ if detached:
+ metadata["provenance_detached"] = True
+ metadata.pop("delivery_key_external", None)
+ metadata.pop("run_ids", None)
+ return replace(
+ activity,
+ foreground=False,
+ detached_from_run=True,
+ turn_id=None,
+ run_id=None,
+ metadata=metadata,
+ updated_at=_now_iso(),
+ )
+
+ metadata["provenance_human"] = True
+ if normalized_delivery_key:
+ metadata["delivery_key_external"] = normalized_delivery_key
+ if normalized_run_ids:
+ metadata["run_ids"] = list(normalized_run_ids)
+ else:
+ metadata.pop("run_ids", None)
+ return replace(
+ activity,
+ detached_from_run=False,
+ turn_id=normalized_turn_id,
+ run_id=normalized_run_ids[0] if normalized_run_ids else None,
+ metadata=metadata,
+ updated_at=_now_iso(),
+ )
+
+ with self._lock:
+ self._retry_provenance_persistence_locked(key)
+ for activity_key, activity in list(self._active.items()):
+ if activity_key[:2] != key or not matches(activity):
+ continue
+ updated = classify(activity)
+ self._active[activity_key] = updated
+ try:
+ self._persist_activity(updated, phase="active")
+ except Exception as error:
+ self._record_provenance_persistence_recovery(
+ updated,
+ phase="active",
+ error=error,
+ )
+ classified.append(updated)
+
+ queue = self._completed_outputs.get(key)
+ if queue:
+ updated_entries: deque[_CompletedOutputEntry] = deque()
+ for entry in queue:
+ if entry.activity and matches(entry.activity):
+ updated = classify(entry.activity)
+ entry = replace(entry, activity=updated)
+ try:
+ self._persist_activity(updated, phase="awaiting_output")
+ except Exception as error:
+ self._record_provenance_persistence_recovery(
+ updated,
+ phase="awaiting_output",
+ error=error,
+ )
+ classified.append(updated)
+ updated_entries.append(entry)
+ self._completed_outputs[key] = updated_entries
+
+ for activity_key, claimed in list(self._claimed_completed_outputs.items()):
+ activity = claimed.entry.activity
+ if (activity.backend, activity.runtime_key) != key or not matches(activity):
+ continue
+ updated = classify(activity)
+ self._claimed_completed_outputs[activity_key] = replace(
+ claimed,
+ entry=replace(claimed.entry, activity=updated),
)
- return self._bind_claimed_output_batch_or_requeue(claimed)
+ try:
+ self._persist_activity(updated, phase="awaiting_output")
+ except Exception as error:
+ self._record_provenance_persistence_recovery(
+ updated,
+ phase="awaiting_output",
+ error=error,
+ )
+ classified.append(updated)
+ snapshots = self._terminal_snapshots.get(key)
+ if snapshots:
+ for activity_id, activity in list(snapshots.items()):
+ if not matches(activity):
+ continue
+ updated = classify(activity)
+ snapshots[activity_id] = updated
+ try:
+ self._persist_activity(updated, phase=TERMINAL_SNAPSHOT_PHASE)
+ except Exception as error:
+ self._record_provenance_persistence_recovery(
+ updated,
+ phase=TERMINAL_SNAPSHOT_PHASE,
+ error=error,
+ )
+ classified.append(updated)
+ if not self._provenance_persistence_recovery.get(key):
+ self._provenance_persistence_recovery.pop(key, None)
+ return classified
+
+ def has_competing_output(
+ self,
+ backend: str,
+ runtime_key: str,
+ *,
+ current_turn_id: str | None = None,
+ ) -> bool:
+ """Whether output exists that is not already bound to this human Turn."""
+
+ key = (str(backend), str(runtime_key))
+ normalized_turn_id = str(current_turn_id or "").strip()
+
+ def competes(activity: SessionActivity) -> bool:
+ if (
+ activity.metadata.get("provenance_human")
+ and normalized_turn_id
+ and activity.turn_id == normalized_turn_id
+ ):
+ return False
+ return True
+
+ with self._lock:
+ for activity in self._active.values():
+ if (activity.backend, activity.runtime_key) == key and competes(activity):
+ return True
+ for entry in self._completed_outputs.get(key) or ():
+ if competes(entry.activity):
+ return True
+ if any(
+ (claimed.entry.activity.backend, claimed.entry.activity.runtime_key) == key
+ and competes(claimed.entry.activity)
+ for claimed in self._claimed_completed_outputs.values()
+ ):
+ return True
+ return any(
+ competes(activity)
+ for activity in self._terminal_snapshots.get(key, {}).values()
+ )
def _bind_claimed_output_batch_or_requeue(
self,
@@ -994,6 +1365,8 @@ class SessionActivityRegistry:
def bind_completed_output_batch(
self,
activities: list[SessionActivity],
+ *,
+ batch_id: str | None = None,
) -> list[SessionActivity]:
"""Persist one receipt identity onto every claimed batch member."""
@@ -1015,8 +1388,8 @@ class SessionActivityRegistry:
next(iter(assigned_ids))
if assigned_ids
else (
- f"{activities[0].backend}:{activities[0].runtime_key}:"
- f"batch:{uuid.uuid4().hex}"
+ str(batch_id or "").strip()
+ or f"{activities[0].backend}:{activities[0].runtime_key}:batch:{uuid.uuid4().hex}"
)
)
raw_member_lists: list[tuple[str, ...]] = []
@@ -1146,6 +1519,8 @@ class SessionActivityRegistry:
raise RuntimeError(
"Durable Activity store cannot atomically bind an output batch"
)
+ for _activity_key, updated in updates:
+ self._clear_provenance_persistence_recovery(updated)
published_updates = []
for activity_key, updated in updates:
@@ -1229,6 +1604,7 @@ class SessionActivityRegistry:
runtime_key: str,
*,
turn_ids: set[str] | None = None,
+ metadata_match: dict[str, Any] | None = None,
max_age_seconds: float = 0,
recovered_only: bool = False,
limit: int | None = None,
@@ -1247,6 +1623,11 @@ class SessionActivityRegistry:
return []
now = time.monotonic()
with self._lock:
+ # Provenance classification is published to memory before its
+ # durable snapshot. Any later receipt claim is a valid recovery
+ # boundary for retrying that snapshot; recovery must not depend on
+ # another Result or a receiver EOF.
+ self._retry_provenance_persistence_locked(key)
queue = self._completed_outputs.get(key)
if not queue:
return []
@@ -1268,6 +1649,12 @@ class SessionActivityRegistry:
if unbound_only and assigned_batch_id:
retained.append(entry)
continue
+ if metadata_match is not None and any(
+ activity.metadata.get(name) != value
+ for name, value in metadata_match.items()
+ ):
+ retained.append(entry)
+ continue
activity_key = self._activity_key(activity)
is_recovered = activity_key in self._recovered_output_ids
if recovered_only and not is_recovered:
@@ -1526,6 +1913,12 @@ class SessionActivityRegistry:
claimed,
entry=replace(claimed.entry, activity=terminal_activity),
)
+ # Receipt settlement owns this terminal evidence before
+ # invoking the Run-policy callback. Its acknowledgement
+ # deliberately cannot delete ordinary awaiting-output rows.
+ self._terminal_snapshots[
+ (terminal_activity.backend, terminal_activity.runtime_key)
+ ][terminal_activity.id] = terminal_activity
self._claimed_completed_outputs[activity_key] = claimed
claimed_by_key[activity_key] = claimed
@@ -1839,15 +2232,51 @@ class SessionActivityRegistry:
def drain_recovered_terminals(self) -> list[SessionActivity]:
with self._lock:
- values = list(self._recovered_terminals)
- self._recovered_terminals.clear()
+ values = []
+ remaining = deque()
+ for previous in self._recovered_terminals:
+ current = self._terminal_snapshots.get(
+ (previous.backend, previous.runtime_key), {}
+ ).get(previous.id)
+ if current is None:
+ continue
+ if current.metadata.get("provenance_pending"):
+ remaining.append(current)
+ else:
+ values.append(current)
+ self._recovered_terminals = remaining
return values
+ def terminal_snapshots_for_runtime(
+ self, backend: str, runtime_key: str,
+ ) -> list[SessionActivity]:
+ """Return retained terminals without consuming classification evidence."""
+
+ with self._lock:
+ self._retry_provenance_persistence_locked((backend, runtime_key))
+ return list(self._terminal_snapshots.get((backend, runtime_key), {}).values())
+
def ack_recovered_terminal(self, activity: SessionActivity) -> None:
"""Delete a recovered live snapshot only after its Run policy settles."""
with self._lock:
- self._delete_activity(activity)
+ key = (str(activity.backend), str(activity.runtime_key))
+ snapshots = self._terminal_snapshots.get(key)
+ current = snapshots.get(str(activity.id)) if snapshots is not None else None
+ # Awaiting-output and claimed receipts have their own settlement
+ # protocol. This acknowledgement is only for an actual terminal
+ # snapshot; otherwise deleting the store row would lose a delivery
+ # receipt that is still retryable after restart.
+ if current is None or current.metadata.get("provenance_pending"):
+ return
+ self._delete_activity(current)
+ snapshots.pop(str(activity.id), None)
+ self._terminal_snapshot_identities.pop(
+ self._activity_key(current),
+ None,
+ )
+ if not snapshots:
+ self._terminal_snapshots.pop(key, None)
def has_backend_work(self, backend: str) -> bool:
"""Whether a backend has live Activities or undelivered completions."""
@@ -1861,6 +2290,7 @@ class SessionActivityRegistry:
for key, queue in self._completed_outputs.items()
)
or any(key[0] == identity for key in self._claimed_completed_outputs)
+ or any(key[0] == identity for key in self._terminal_snapshots)
)
def end_backend(self, backend: str, *, status: str = "killed") -> list[SessionActivity]:
@@ -1888,6 +2318,11 @@ class SessionActivityRegistry:
for item_backend, runtime_key, _activity_id in self._claimed_completed_outputs
if item_backend == identity
)
+ runtime_keys.update(
+ runtime_key
+ for item_backend, runtime_key in self._terminal_snapshots
+ if item_backend == identity
+ )
completed: list[SessionActivity] = []
for runtime_key in runtime_keys:
completed.extend(
@@ -1925,14 +2360,39 @@ class SessionActivityRegistry:
]
for activity in terminated_pending:
self._persist_activity(activity, phase=TERMINAL_SNAPSHOT_PHASE)
+ self._terminal_snapshots[
+ (activity.backend, activity.runtime_key)
+ ][activity.id] = activity
for key in [key for key in self._completed_outputs if key[0] == identity]:
self._completed_outputs.pop(key, None)
for key in [key for key in self._claimed_completed_outputs if key[0] == identity]:
self._claimed_completed_outputs.pop(key, None)
for activity in terminated_pending:
self._discard_recovered_output_id(self._activity_key(activity))
- completed.extend(terminated_pending)
- return completed
+ # Force-ended terminal snapshots remain registered until the normal
+ # acknowledgement removes both the durable row and in-memory owner.
+ # Without a store there is no restart-visible receipt to protect,
+ # so the in-memory-only registry may retire them immediately.
+ if self._store is None:
+ for key, snapshots in list(self._terminal_snapshots.items()):
+ if key[0] != identity:
+ continue
+ for activity_id in list(snapshots):
+ self._terminal_snapshot_identities.pop(
+ (*key, activity_id),
+ None,
+ )
+ self._terminal_snapshots.pop(key, None)
+ else:
+ for key, snapshots in self._terminal_snapshots.items():
+ if key[0] != identity:
+ continue
+ for activity in snapshots.values():
+ completed.append(activity)
+ return list({
+ self._activity_key(activity): activity
+ for activity in [*completed, *terminated_pending]
+ }.values())
def end_runtime(
self,
@@ -2005,9 +2465,24 @@ class SessionActivityRegistry:
metadata=None,
expects_output=False,
retain_terminal_snapshot=retain_terminal_snapshots,
+ activation_identity=activation_identity,
)
if activity is not None:
- completed.append(activity)
+ finalized = self._finalize_generation_ended_snapshot_locked(activity)
+ completed.append(finalized or activity)
+ for activity in list(self._terminal_snapshots.get(key, {}).values()):
+ snapshot_key = self._activity_key(activity)
+ if activation_identity is not None:
+ snapshot_identity = self._terminal_snapshot_identities.get(
+ snapshot_key
+ )
+ if snapshot_identity is not activation_identity:
+ continue
+ finalized = self._finalize_generation_ended_snapshot_locked(activity)
+ if finalized is not None and not any(
+ existing.id == finalized.id for existing in completed
+ ):
+ completed.append(finalized)
return completed
def session_state(self, session_id: str) -> dict[str, Any]:
diff --git a/docs/plans/claude-result-provenance.md b/docs/plans/claude-result-provenance.md
new file mode 100644
index 000000000..ed05f5782
--- /dev/null
+++ b/docs/plans/claude-result-provenance.md
@@ -0,0 +1,252 @@
+# Claude result provenance and background input admission
+
+## Problem
+
+Claude Code keeps one streaming SDK connection per runtime. A detached background
+Activity can produce Assistant and Result frames while a newer Avibe human Turn is
+waiting to write. The SDK's terminal `ResultMessage.origin` is the only reliable
+owner signal in this interleaved stream; Assistant frames do not carry that field.
+
+## Contract
+
+- Stamp every Avibe-originated Claude user query with `origin: {kind: human}`.
+- Admit a new human query without waiting for detached Activity output.
+- Check the exact Claude client generation immediately before the native write.
+- Classify terminal results by `origin` before claiming Activity output or popping a
+ pending human request. Human results settle the pending human Turn. Task
+ notifications and other injected origins remain detached.
+- Buffer Assistant and tool frames while an Activity makes ownership ambiguous, then
+ replay them only after the terminal Result identifies the phase. Grace-period
+ Activity flushes defer while that phase is buffered, and replay failures do not
+ prevent terminal settlement.
+- Treat a missing or unknown origin as foreground only when no competing Activity
+ evidence exists; otherwise preserve it as detached output and leave the pending
+ human request untouched.
+- Keep a synthetic owner for detached output until the durable emit succeeds.
+ A failed emit retains the selected text, claimed Activity batch, and
+ idempotency identity for retry while the receiver is still open or after an
+ EOF/error. The recovery owner, rather than receiver cleanup, releases the
+ runtime gate after durable local settlement; a generic EOF failure must not
+ replace or release it early.
+- Preserve durable prewrite evidence so a write that definitely did not happen can
+ be explicitly retried without replaying an attempted or ambiguous native write.
+
+## Ownership transitions
+
+The receiver treats a Claude response phase as provisional until its terminal
+`ResultMessage.origin` is classified. A pending Activity, a background tool,
+and a pending human request are separate facts; none of them alone transfers
+Turn, Run, or delivery ownership.
+
+1. A `ToolUseBlock` and matching `TaskStartedMessage` record provisional phase
+ and tool/task identity. `run_in_background` describes execution mode only;
+ it is not provenance.
+2. While the phase is provisional, Activity flushes may retain completed
+ outputs but cannot pop or settle a pending human request. Completed,
+ failed, stopped, and killed provisional Activities remain addressable as
+ terminal snapshots until classification; uncorrelated task events remain
+ unattributed.
+3. A human Result resolves only the positively correlated provisional tool/task
+ facts to the pending request's captured Turn, Run, and delivery identity.
+ This resolution happens before terminal Run settlement. A human-created
+ background task can therefore retain its Run lineage without making every
+ Activity in the runtime human-owned.
+4. A task-notification or other explicitly detached Result resolves only its
+ correlated phase as detached, preserves the pending human request, and
+ delivers the Activity batch through the existing durable receipt path.
+5. Missing or unknown origin remains conservative while competing Activity
+ evidence exists. Failed, stopped, and killed task terminals keep the
+ provenance barrier until the phase is classified or its request/generation
+ is retired; Activity count alone cannot clear it.
+6. Stop/retirement wins terminal ownership before buffered replay, Activity
+ claim, or unsolicited routing. A late human Result is consumed silently,
+ while the existing attempted-versus-definitely-unsent evidence remains
+ available for recovery.
+7. Provenance classification updates the in-memory Activity ownership before its
+ durable snapshot. A persistence failure records retryable recovery evidence
+ and cannot cause the already-consumed terminal Result to be skipped.
+8. A detached terminal Result selects one existing Activity receipt batch. Its
+ exact output text and batch identity remain together until the dispatcher and
+ local Activity settlement both succeed. A later Assistant frame cannot
+ replace that selected text with a CLI summary, and a retry never claims a
+ second batch.
+9. A synthetic agent-initiated request is an explicit terminal owner even when
+ its output is detached. Receiver EOF/error and generation cleanup may close
+ native admission, but they retain the synthetic request and its gate while
+ durable recovery is pending. Only the successful recovery path retires that
+ owner and admits the next human Turn. Detached output that arrives while a
+ real human request is pending retains its own retry payload, but does not
+ become a synthetic owner or hold the human request's runtime gate.
+10. Retained detached output is keyed by its response phase and output identity.
+ A later Assistant frame is provisional until its own Result is classified; it
+ cannot overwrite the selected text or idempotency identity of an earlier
+ retained phase. Once the earlier payload settles, the later phase may create
+ its own durable output identity.
+11. `awaiting_output` receipts and terminal snapshots have separate acknowledgers.
+ Activity Run classification may notify the Run owner, but only completed
+ output settlement releases a claimed receipt, and only terminal-snapshot
+ acknowledgement deletes a terminal snapshot. This keeps queued output
+ recoverable across a crash or restart.
+12. Receiver EOF/error/replacement retires the exact dead Claude client
+ generation even when a detached recovery owner remains. Generation end also
+ closes any still-provisional terminal Activity conservatively as unresolved
+ detached state, so no future Result is required and backend work cannot stay
+ permanently blocked by an owner that cannot arrive.
+
+The registry uses one FIFO candidate-selection and receipt-binding algorithm
+for metadata eligibility, Turn constraints, retries, and persisted local-only
+output batches. A metadata predicate can select a candidate, but it cannot
+create a second batching path or absorb a previously bound receipt.
+
+## Validation
+
+Consumer tests cover both terminal result orders, notification-before-human-result,
+multiple Activity completion aggregation, Assistant buffering, flush-vs-Result
+races, retained Activity text during a long-lived receiver retry, receiver
+error recovery ownership, failed/stopped/killed/completed provisional terminal
+lineage, buffered replay failure, unknown origin, client replacement and Stop
+races, exactly-once output, and durable unsent-input recovery. A hermetic Claude
+Agent SDK 0.2.158 plus bundled CLI probe verifies the outgoing origin shape and
+real Result provenance against the local mock upstream.
+
+The current implementation scope is the Claude receiver and existing Activity,
+dispatcher, receipt, steering, and generation owners only. It does not add a
+cross-backend event abstraction or redesign the Claude SDK.
+
+## Bounded recovery-ledger decision (2026-09-25)
+
+The orchestrator authorized one ownership-model correction after ten
+findings-bearing reviewed heads, through `16c1ed184b12`. The circuit breaker
+remains cumulative; this decision does not reset it or authorize independent
+comment-by-comment pushes. Existing threads remain unchanged during this pass.
+
+The Claude adapter now has one phase/output recovery ledger instead of the
+parallel runtime-keyed detached text, Activity, MessageOutput, and provisional
+payload maps. A record captures its immutable identity, exact client activation,
+provenance, selected text, claimed batch, MessageOutput, context, and synthetic
+request owner. A runtime key locates records; it does not determine which request
+may be retired. Human request settlement still uses the existing pending Request
+and native-input receipts, not a second human terminal state machine.
+
+| Boundary | Owning transition |
+| --- | --- |
+| Assistant or claimed Activity | Create/update only the current generation's provisional record. |
+| Detached Result | Freeze selected text, batch, output identity, and context before any delivery await. A later failure Result gets another record. |
+| Delivery failure before acceptance | Keep the same record and retry through the existing managed Activity flush worker. |
+| External acceptance but failed local settlement without durable Message evidence | Keep the claim and original payload; refine only the record's retry policy to the dispatcher's existing local-settlement-only path. Never resend externally. |
+| Durable delivery and local settlement | Retire that record and only its captured synthetic Request/token. An unrelated current synthetic or human owner is untouched. |
+| EOF/error/Stop/replacement | Retire exactly the dead client. Frozen records survive; provisional detached records from that generation are conservatively frozen without borrowing replacement provenance. |
+| Activity classification | Update lineage and notify the Run owner without deleting awaiting/claimed output receipts. |
+| Terminal-snapshot acknowledgement | Delete only an indexed terminal snapshot, after Run-owner acceptance. Force-ended snapshots remain indexed until acknowledgement. |
+
+The existing admission fence is consulted before Result classification/replay,
+including while Stop is awaiting its native interrupt. A receiver revalidates its
+exact activation after waiting for a native frame; replacement cannot let the old
+receiver pop the new FIFO head or clear the new generation's phase state.
+Cancellation preserves a terminal recovery owner's pending request and gate.
+Recovery completion uses the captured owner/token rather than looking up whoever
+currently owns that runtime.
+An unclassified task that finishes after its pending human request has retired
+keeps its original Activity phase. Its late notification cannot become positive
+correlation for a newer human phase, and it remains classifiable without requiring
+another pending request to exist.
+
+Terminal snapshots retain their opaque activation identity in memory. A numeric
+generation is audit metadata only: a process restart can reuse the counter.
+Generation end finalizes only its own unresolved snapshots. Restart conservatively
+finalizes recovered provisional failed/stopped/killed/disconnected snapshots as
+generation-ended and unresolved, without assigning a new human Run. They then
+have a reachable drain/ack path.
+
+The managed worker rechecks the ledger after awaited delivery: a Result appended
+during a retry remains its responsibility even if the initial list was drained.
+There is no new generic queue, service, or per-output timer. Missing/unknown
+origin with competing Activity remains conservative; foreground execution mode
+and TaskStarted linkage are still not human-provenance evidence.
+
+Consumer coverage includes event-held native streams; old payload plus later
+failure and human phases; human and synthetic successor admission; Stop-first
+and Result-first ordering; EOF/error/replacement with persistent delivery failure;
+new Result during an in-flight retry; both generation-ending snapshot orders;
+SQLite classification/receipt restart; force-end service acknowledgement; and
+real-dispatcher external acceptance followed by failing local receipt storage.
+These are hermetic adapter/service/dispatcher consumers, not real Web/IM or SDK
+end-to-end tests. The ledger is process-local recovery across client generations;
+restart tests cover the existing durable Activity receipt/snapshot store, not a
+new durable outbox for never-accepted unsolicited text.
+
+## Bounded persistence/EOF correction and ancestry integration (2026-09-26)
+
+The orchestrator authorized one combined pass after the terminal review of
+`d0b3f86e78c6`: integrate master `6d464094b2a7`, retain its behavioral
+cross-backend Stop test with the Claude synthetic-owner fixture initialized,
+and correct two existing boundaries. This does not extend the recovery ledger
+or reset the cumulative breaker (eleven findings-bearing reviewed heads).
+
+- A provenance retry describes a failed write, not lifecycle authority. At
+ retry time the registry derives the phase from the current owner: active,
+ queued/claimed output, or terminal snapshot. A later successful write or
+ deletion supersedes that retry evidence, including atomic multi-Activity
+ batch binding. Failed writes retain evidence. Neither stale diagnostics nor
+ retries may downgrade an output receipt to active or resurrect a deleted row.
+- Buffered failure replay distinguishes a diagnostic replay exception from an
+ accepted terminal failure. At EOF, a request still owned after failed replay
+ follows the existing no-result settlement before releasing its service gate.
+ If replay already consumed the owner, EOF cannot emit another terminal.
+ Receiver errors use the same ownership rule; a retired receiver cannot
+ replay against a successor's client, request, or token.
+
+Consumer coverage uses a real SQLite Activity store with injected single/batch
+write failures, transitions from active to completed/failed/stopped/killed,
+queued and claimed receipts, terminal acknowledgement, and registry restart.
+The receiver/service matrix covers EOF/error, failure before emission and after
+owner consumption, successful authoritative replay, repeated cleanup, and
+Stop/replacement with an admitted successor. Gate acquisition/release is real
+AgentService behavior; outbound acceptance is recorded by a test double, not
+claimed as native SDK, durable Run, or Web/IM end-to-end verification. Existing
+dispatcher/durable-receipt consumer suites remain part of the focused gate.
+
+No review-thread operations, manual review trigger, PR merge, deployment,
+service restart, or Watch/cursor changes are part of this pass. Fresh exact-head
+automatic review and repository CI remain required after the single push;
+local green tests are not acceptance.
+
+## Bounded terminal ownership transfer correction (2026-09-26)
+
+The orchestrator authorized one coherent correction after `61751dee2389`.
+The cumulative breaker remains at twelve findings-bearing reviewed heads.
+This pass corrects the existing ownership-transfer boundaries, without a new
+queue, service, lock, provenance policy, or thread operation.
+
+- EOF/error replay carries the receiver's exact client and activation identity.
+ It validates that identity under the existing steering fence before consuming
+ buffered state or a pending request. The caller validates again after replay
+ and before fallback settlement or Activity flushing; a replacement during an
+ await cannot transfer a failure to the successor. The empty-buffer fast path
+ keeps the existing lock order and does not wait unnecessarily on steering.
+- A detached Result freezes selected text, phase identity, claim eligibility,
+ and output ownership before the first fallible receipt operation. The existing
+ FIFO selector can return its raw claim to that ledger owner. The owner then
+ constructs the MessageOutput with the existing receipt ID or its immutable
+ phase ID before binding. Binding and delivery retries retain exactly those
+ members, text, and idempotency identity; they neither reconstruct a CLI
+ summary nor absorb later completions. The existing managed worker retries
+ that record before claiming other completed outputs.
+- Completed-output settlement indexes its terminal snapshot before persistence
+ and callback. The terminal callback can therefore acknowledge the durable
+ evidence once, including when output-local settlement reports an error.
+ Classification alone still cannot acknowledge an awaiting/claimed receipt.
+
+Consumer tests cover EOF/error replacement while replay waits on the steering
+fence, replacement after replay returns, pre-emission/post-consumption replay
+failures, and Stop refusing a phase already owned by EOF. Event-held streams
+cover repeated receipt-storage failure through live delivery, EOF/error,
+Stop, and replacement, followed by recovery and successor admission. Real
+dispatcher/SQLite consumers cover pre-bound and new batches, delivery failure,
+later queued completions, stable output identity, and terminal acknowledgement
+with and without accepted Message evidence and local settlement errors.
+These are hermetic adapter/service/dispatcher consumers, not native SDK or
+Web/IM end-to-end tests. The ledger remains process-local; restart coverage
+concerns the existing durable Activity receipt store, not a new unsolicited
+output outbox. Fresh automatic exact-head review and CI remain required after
+the single push; local tests do not establish acceptance.
diff --git a/modules/agents/claude_agent.py b/modules/agents/claude_agent.py
index 7189fd327..bccaedb8d 100644
--- a/modules/agents/claude_agent.py
+++ b/modules/agents/claude_agent.py
@@ -2,12 +2,13 @@ import asyncio
import logging
import os
import uuid
-from dataclasses import dataclass
+from copy import copy
+from dataclasses import dataclass, field, replace
from typing import Any, Callable, Literal, Optional
from core.agent_auth_service import classify_auth_error
from core.backend_failure import backend_failure_notification_output, emit_backend_failure
-from core.handlers.session_handler import ClaudeSessionNotFoundError
+from core.handlers.session_handler import ClaudeInputNotSentError, ClaudeSessionNotFoundError
from core.message_dispatcher import ActivityOutputDeliveryError
from core.message_output import (
HARNESS_RUN_ID_TRIGGER_KINDS,
@@ -21,7 +22,7 @@ from core.native_dispatch_phase import mark_backend_dispatch_attempted, mark_pre
from core.processing_indicator import STOPPED_REACTION_EMOJI
from core.reply_enhancer import strip_silent_blocks
from core.resource_governance import pids_failure_labels
-from core.runtime_activation import RuntimeActivationIdentity
+from core.runtime_activation import RuntimeActivationIdentity, RuntimeActivationRegistry
from core.runtime_work import RuntimeWorkLane
from core.services.agent_steering import (
ActiveSteerTarget,
@@ -65,6 +66,36 @@ class _ClaudeInputReceipt:
state: Literal["writing", "accepted", "unknown"] = "writing"
+@dataclass(frozen=True)
+class _BufferedClaudeFailureReplay:
+ diagnostic: str
+ auth_handled: bool = False
+ replay_failed: bool = False
+
+
+@dataclass
+class _ClaudeOutputRecoveryRecord:
+ """One immutable phase/output owner retained across receiver generations."""
+
+ phase_id: str
+ composite_key: str
+ generation: object | None
+ owner: Literal["activity", "detached"]
+ lifecycle: Literal["provisional", "delivery_pending", "delivered"] = "provisional"
+ text: str = ""
+ activities: list[SessionActivity] = field(default_factory=list)
+ output: MessageOutput | None = None
+ request: AgentRequest | None = None
+ context: MessageContext | None = None
+ subtype: str = "success"
+ duration_ms: int = 0
+ delivering: bool = False
+ provenance: str | None = None
+ local_settlement_only: bool = False
+ claim_pending: bool = False
+ claim_metadata: dict[str, object] | None = None
+
+
class ClaudeAgent(BaseAgent):
"""Existing Claude Code integration extracted into an agent backend."""
@@ -97,22 +128,41 @@ class ClaudeAgent(BaseAgent):
self._pending_requests: dict[str, list[AgentRequest]] = {}
self._steering_locks: dict[str, asyncio.Lock] = {}
self._steering_generations: dict[str, int] = {}
+ # Monotonic admission epochs let a writer that was queued before Stop
+ # distinguish Stop-owned prewrite rejection from an unrelated runtime
+ # replacement after the lock has been released.
+ self._steering_stop_epochs: dict[str, int] = {}
self._native_input_receipts: dict[str, list[_ClaudeInputReceipt]] = {}
self._ambiguous_primary_results: dict[str, object] = {}
self._steering_input_shutdowns: set[str] = set()
self._ambiguous_interrupts: set[str] = set()
self._steering_closing: set[str] = set()
+ self._stop_owned_terminal_keys: set[str] = set()
+ self._stop_owned_terminal_epochs: dict[str, int] = {}
self._steering_writers: set[str] = set()
- self._detached_activity_outputs: dict[str, list[SessionActivity]] = {}
- self._detached_assistant_text: dict[str, str] = {}
- self._detached_unsolicited_outputs: set[str] = set()
- self._detached_unsolicited_text: dict[str, str] = {}
+ # Recovery state is keyed by immutable phase/output identity. The
+ # composite runtime key is only an index used to find records; it is
+ # never the owner of retained text, receipts, or idempotency.
+ self._output_recovery_ledger: dict[str, _ClaudeOutputRecoveryRecord] = {}
self._activity_flush_tasks: dict[str, asyncio.Task] = {}
self._activity_settle_events: dict[str, asyncio.Event] = {}
+ self._buffered_assistant_messages: dict[str, list[tuple[object, int | None]]] = {}
+ self._activity_provenance_barriers: set[str] = set()
+ self._activity_provenance_barrier_owners: dict[str, int] = {}
self._foreground_tool_use_ids: dict[str, set[str]] = {}
self._turns_with_foreground_tools: set[str] = set()
+ self._provisional_tool_use_ids: dict[str, set[str]] = {}
+ self._provisional_unbound_tool_use_ids: dict[str, set[str]] = {}
+ self._provisional_foreground_tool_use_ids: dict[str, set[str]] = {}
+ self._provisional_activity_ids: dict[str, set[str]] = {}
+ self._provisional_unbound_activity_ids: dict[str, set[str]] = {}
+ self._provisional_foreground_task_ids: dict[str, set[str]] = {}
+ self._provisional_phase_ids: dict[str, str] = {}
+ self._resolved_provisional_phase_ids: dict[str, str] = {}
self._detached_foreground_tool_use_ids: dict[str, set[str]] = {}
self._detached_foreground_task_ids: dict[str, set[str]] = {}
+ self._synthetic_pending_owners: dict[str, AgentRequest] = {}
+ self._provenance_recovery_evidence: dict[str, dict[str, object]] = {}
# Question handler for AskUserQuestion support (disabled)
# NOTE: Uncomment when SDK adds AskUserQuestion support
@@ -124,6 +174,217 @@ class ClaudeAgent(BaseAgent):
# )
self._question_handler = None
+ def _output_recovery_generation(self, composite_key: str) -> object | None:
+ client = self.claude_sessions.get(composite_key)
+ # Identity is opaque, not the reusable numeric generation counter. The
+ # client itself is the identity for adapters without an activation registry.
+ return self._client_activation_identity(client) or client
+
+ def _output_records_for_runtime(
+ self,
+ composite_key: str,
+ ) -> list[_ClaudeOutputRecoveryRecord]:
+ return [
+ record
+ for record in self._output_recovery_ledger.values()
+ if record.composite_key == composite_key
+ and record.lifecycle != "delivered"
+ ]
+
+ def _output_record(
+ self,
+ composite_key: str,
+ phase_id: str | None,
+ *,
+ owner: Literal["activity", "detached"],
+ ) -> _ClaudeOutputRecoveryRecord | None:
+ normalized = str(phase_id or "").strip()
+ if normalized:
+ record = self._output_recovery_ledger.get(normalized)
+ if (
+ record is not None and record.composite_key == composite_key
+ and record.owner == owner
+ ):
+ return record
+ return None
+ for record in self._output_records_for_runtime(composite_key):
+ if record.owner == owner:
+ return record
+ return None
+
+ def _upsert_output_record(
+ self,
+ composite_key: str,
+ *,
+ phase_id: str,
+ owner: Literal["activity", "detached"],
+ text: str = "",
+ activities: list[SessionActivity] | None = None,
+ output: MessageOutput | None = None,
+ lifecycle: Literal["provisional", "delivery_pending"] = "provisional",
+ ) -> _ClaudeOutputRecoveryRecord:
+ normalized_phase_id = str(phase_id or "").strip() or (
+ f"{composite_key}:output:{uuid.uuid4().hex}"
+ )
+ existing = self._output_recovery_ledger.get(normalized_phase_id)
+ if existing is not None:
+ if existing.composite_key != composite_key:
+ raise RuntimeError("Claude output phase identity collision")
+ # A DELIVERY_PENDING record is immutable. Later frames may create a
+ # different record, but they can never replace its selected payload,
+ # receipt batch, or idempotency identity.
+ if existing.lifecycle == "delivery_pending":
+ return existing
+ if text:
+ existing.text = str(text).strip()
+ if activities:
+ existing.activities = list(activities)
+ if output is not None:
+ existing.output = output
+ existing.lifecycle = lifecycle
+ return existing
+
+ record = _ClaudeOutputRecoveryRecord(
+ phase_id=normalized_phase_id,
+ composite_key=composite_key,
+ generation=self._output_recovery_generation(composite_key),
+ owner=owner,
+ lifecycle=lifecycle,
+ text=str(text or "").strip(),
+ activities=list(activities or ()),
+ output=output,
+ request=self._synthetic_pending_owners.get(composite_key),
+ )
+ self._output_recovery_ledger[normalized_phase_id] = record
+ return record
+
+ def _remove_output_record(
+ self,
+ record: _ClaudeOutputRecoveryRecord,
+ ) -> None:
+ record.lifecycle = "delivered"
+ self._output_recovery_ledger.pop(record.phase_id, None)
+
+ def _provisional_output_record(
+ self, composite_key: str, owner: Literal["activity", "detached"],
+ ) -> _ClaudeOutputRecoveryRecord | None:
+ return next((
+ record for record in self._output_records_for_runtime(composite_key)
+ if record.owner == owner and record.lifecycle == "provisional"
+ and record.generation is self._output_recovery_generation(composite_key)
+ ), None)
+
+ def _detached_phase(
+ self, composite_key: str, *, claim_activity: bool = False,
+ ) -> _ClaudeOutputRecoveryRecord:
+ """Find only the unclassified phase, never a retained terminal payload."""
+ record = self._provisional_output_record(composite_key, "detached")
+ if record is None:
+ record = self._upsert_output_record(
+ composite_key,
+ phase_id=f"{composite_key}:detached:{uuid.uuid4().hex}",
+ owner="detached",
+ )
+ if claim_activity:
+ registry = self._activity_registry()
+ record.claim_pending = bool(
+ registry is not None and registry.has_completed_output(self.name, composite_key)
+ )
+ phase_id = self._resolved_provisional_phase_ids.get(composite_key)
+ if phase_id is not None:
+ record.claim_metadata = {
+ "provenance_phase_id": phase_id,
+ "provenance_detached": True,
+ }
+ return record
+
+ @staticmethod
+ def _snapshot_output_context(context: MessageContext) -> MessageContext:
+ captured = copy(context)
+ captured.platform_specific = dict(
+ getattr(context, "platform_specific", None) or {}
+ )
+ return captured
+
+ def _classify_output_record(
+ self, record: _ClaudeOutputRecoveryRecord, context: MessageContext,
+ *, text: str | None, message=None,
+ ) -> None:
+ """Freeze payload and terminal owner synchronously before delivery awaits."""
+ if record.lifecycle != "provisional":
+ return
+ record.text = str(text or "").strip()
+ record.context = self._snapshot_output_context(context)
+ record.provenance = self._result_origin_kind(message)
+ record.subtype = str(getattr(message, "subtype", "success") or "")
+ record.duration_ms = getattr(message, "duration_ms", 0)
+ if record.output is None:
+ record.output = self._unsolicited_message_output(message, phase_id=record.phase_id)
+ record.lifecycle = "delivery_pending"
+ if record.request is not None and record.activities:
+ owned = self._request_activities(record.request)
+ if owned and [a.id for a in owned] == [a.id for a in record.activities]:
+ # Transfer the existing claim, rather than giving Request and
+ # recovery two independent copies of receipt ownership.
+ self._clear_request_activities(record.request)
+
+ def _retire_output_generation(self, composite_key: str, client, context=None) -> None:
+ """Retire native inference, never a frozen output receipt's recovery owner."""
+ generation = self._client_activation_identity(client) or client
+ for record in self._output_records_for_runtime(composite_key):
+ if record.generation is not generation or record.lifecycle != "provisional":
+ continue
+ captured = record.context or getattr(record.request, "context", None) or context
+ if captured is None:
+ continue
+ # A detached/claimed provisional payload has no future Result after
+ # retirement. Keep it detached and freeze its existing content;
+ # never let a replacement generation finish or overwrite this phase.
+ self._classify_output_record(
+ record, captured,
+ text=record.text or (
+ record.activities[-1].metadata.get("summary", "")
+ if record.activities else ""
+ ),
+ )
+ record.provenance = "unresolved_generation_end"
+ if any(record.lifecycle == "delivery_pending" for record in self._output_records_for_runtime(composite_key)):
+ recovery_context = next(
+ (record.context for record in self._output_records_for_runtime(composite_key) if record.context is not None),
+ context,
+ )
+ if recovery_context is not None:
+ self._schedule_completed_activity_flush(composite_key, recovery_context)
+
+ def _retain_activity_output_record(
+ self,
+ composite_key: str,
+ activities: list[SessionActivity],
+ *,
+ text: str = "",
+ ) -> _ClaudeOutputRecoveryRecord:
+ activity = activities[-1]
+ output = self._activity_message_output(
+ activity,
+ activities=activities,
+ detached=True,
+ completes_turn=False,
+ )
+ phase_id = str(
+ output.idempotency_key
+ or f"{composite_key}:activity:{activity.id}"
+ )
+ record = self._upsert_output_record(
+ composite_key,
+ phase_id=phase_id,
+ owner="activity",
+ text=text,
+ activities=activities,
+ output=output,
+ lifecycle="provisional",
+ )
+ return record
+
def _claude_error_diagnostic(
self,
composite_key: str,
@@ -165,6 +426,8 @@ class ClaudeAgent(BaseAgent):
client=None,
) -> str:
"""Return the durable notify text for Claude terminal errors."""
+ if isinstance(error, ClaudeInputNotSentError):
+ return f"❌ {self._translate_error(error.message_key)}"
if isinstance(error, ClaudeSessionNotFoundError):
detail = self._translate_error(
"error.claudeSessionNotFound",
@@ -197,12 +460,78 @@ class ClaudeAgent(BaseAgent):
return f"❌ {message}"
return f"❌ Claude error: {error}"
+ @staticmethod
+ async def _human_query_messages(message: str, session_id: str):
+ """Yield an SDK streaming user message with explicit human provenance."""
+
+ yield {
+ "type": "user",
+ "message": {"role": "user", "content": message},
+ "parent_tool_use_id": None,
+ "session_id": session_id,
+ "origin": {"kind": "human"},
+ }
+
+ def _assert_current_client_for_write(self, composite_key: str, client) -> None:
+ """Reject a retired Claude generation before marking a native write."""
+
+ registered = self.claude_sessions.get(composite_key)
+ if registered is not None and registered is not client:
+ raise ClaudeInputNotSentError(
+ "claude_runtime_changed_before_write",
+ "error.claudeInputRuntimeChanged",
+ )
+ if getattr(client, "_vibe_runtime_activation_retired", False) is True:
+ raise ClaudeInputNotSentError(
+ "claude_runtime_changed_before_write",
+ "error.claudeInputRuntimeChanged",
+ )
+ if composite_key in self._steering_closing_keys():
+ raise ClaudeInputNotSentError(
+ "claude_runtime_changed_before_write",
+ "error.claudeInputRuntimeChanged",
+ )
+ identity = self._client_activation_identity(client)
+ registry = getattr(getattr(self.controller, "agent_service", None), "activation_registry", None)
+ if identity is not None and registry is not None and not registry.is_current(identity):
+ raise ClaudeInputNotSentError(
+ "claude_runtime_changed_before_write",
+ "error.claudeInputRuntimeChanged",
+ )
+
+ async def _write_human_query(
+ self,
+ client,
+ composite_key: str,
+ message: str,
+ context: MessageContext,
+ ) -> None:
+ """Write one human query after the caller acquires the native-write fence.
+
+ ``_steering_lock`` is the sole admission fence for native writes and
+ Stop. Runtime-generation retirement may hold its own lock while it
+ marks a client retired, but it never waits for that lock from here:
+ eviction can then mark-and-wait without creating a reverse lock order.
+ """
+
+ self._assert_current_client_for_write(composite_key, client)
+ # The write is now admitted against the exact client generation. If
+ # transport.write fails after this point, delivery is attempted or
+ # ambiguous and must follow the normal recovery path.
+ mark_backend_dispatch_attempted(context)
+ await client.query(
+ self._human_query_messages(message, composite_key),
+ session_id=composite_key,
+ )
+
async def handle_message(self, request: AgentRequest) -> None:
context = request.context
runtime_base_session_id = request.base_session_id
runtime_session_key = request.composite_session_id
turn_registered = False
client = None
+ input_receipt = None
+ stop_owned = False
# Question callback handling (disabled - SDK doesn't support AskUserQuestion response)
# if self.ENABLE_ASK_USER_QUESTION and request.message.startswith("claude_question:"):
@@ -226,37 +555,57 @@ class ClaudeAgent(BaseAgent):
if callable(mark_session_active):
mark_session_active(runtime_session_key)
- # Claude does not expose a reliable query/result correlation id. If a
- # native background Activity can still speak, serialize this accepted
- # Inbox item until that output is delivered. The Session remains full
- # duplex while this backend avoids attributing a background Result to
- # the new user Turn.
- await self._wait_for_activity_output(runtime_session_key)
-
- # Queue reaction BEFORE sending query to avoid race condition where
- # a fast result arrives before the reaction is queued
- if request.ack_reaction_message_id and request.ack_reaction_emoji:
- if runtime_session_key not in self._pending_reactions:
- self._pending_reactions[runtime_session_key] = []
- self._pending_reactions[runtime_session_key].append(
- (request.ack_reaction_message_id, request.ack_reaction_emoji)
- )
- self._pending_requests.setdefault(runtime_session_key, []).append(request)
-
# Prepare message with file attachment info if present
message = self._prepare_message_with_files(request)
message = self.render_input(message, getattr(request, "input_metadata", None))
- input_receipt = self._register_native_input(
- runtime_session_key,
- message,
- kind="primary",
- )
+ admission_stop_epoch = self._steering_stop_epoch(runtime_session_key)
- mark_backend_dispatch_attempted(request.context)
try:
- await client.query(message, session_id=runtime_session_key)
+ async with self._steering_lock(runtime_session_key):
+ if (
+ self._steering_stop_epoch(runtime_session_key)
+ != admission_stop_epoch
+ ):
+ # Stop already won the only admission fence. The
+ # request was never registered and must retain its
+ # definitely-unsent recovery evidence without entering
+ # the generic runtime-error settlement path.
+ setattr(request, "_claude_stop_owned", True)
+ raise ClaudeInputNotSentError(
+ "claude_runtime_changed_before_write",
+ "error.claudeInputRuntimeChanged",
+ )
+
+ # Queue reaction and register the request while holding the
+ # same fence as Stop/native write. This makes registration
+ # itself part of the terminal ownership decision.
+ if request.ack_reaction_message_id and request.ack_reaction_emoji:
+ if runtime_session_key not in self._pending_reactions:
+ self._pending_reactions[runtime_session_key] = []
+ self._pending_reactions[runtime_session_key].append(
+ (request.ack_reaction_message_id, request.ack_reaction_emoji)
+ )
+ self._pending_requests.setdefault(runtime_session_key, []).append(request)
+ self._refresh_activity_provenance_barrier(runtime_session_key)
+ input_receipt = self._register_native_input(
+ runtime_session_key,
+ message,
+ kind="primary",
+ )
+ writers = self._steering_writer_keys()
+ writers.add(runtime_session_key)
+ try:
+ await self._write_human_query(
+ client,
+ runtime_session_key,
+ message,
+ request.context,
+ )
+ finally:
+ writers.discard(runtime_session_key)
except (Exception, asyncio.CancelledError):
- self._remove_native_input_receipt(runtime_session_key, input_receipt)
+ if input_receipt is not None:
+ self._remove_native_input_receipt(runtime_session_key, input_receipt)
raise
input_receipt.state = "accepted"
from core.skill_observability import accept_catalog
@@ -296,13 +645,18 @@ class ClaudeAgent(BaseAgent):
self._remove_pending_request(runtime_session_key, request)
self._mark_session_idle_if_no_pending_requests(runtime_session_key)
await self._delete_ack(context, request)
- self._release_service_runtime_turn(context)
+ if not bool(getattr(request, "_claude_stop_owned", False)):
+ self._release_service_runtime_turn(context)
raise
except Exception as e:
logger.error(f"Error processing Claude message: {e}", exc_info=True)
missing_session = isinstance(e, ClaudeSessionNotFoundError)
+ input_not_sent = isinstance(e, ClaudeInputNotSentError)
+ stop_owned = bool(getattr(request, "_claude_stop_owned", False))
if missing_session:
mark_prewrite_recovery_required(context, "native_session_not_found")
+ elif input_not_sent:
+ mark_prewrite_recovery_required(context, e.reason)
diagnostic = self._claude_error_diagnostic(
runtime_session_key,
e,
@@ -315,7 +669,7 @@ class ClaudeAgent(BaseAgent):
intentional_teardown = self._teardown_is_intentional(
runtime_session_key, e, client=client
)
- if not intentional_teardown:
+ if not intentional_teardown and not input_not_sent:
await self.record_model_hub_native_failure(context, diagnostic)
# Clean up the specific reaction for this request (not FIFO)
await self._remove_specific_pending_reaction(runtime_session_key, context, request)
@@ -328,10 +682,21 @@ class ClaudeAgent(BaseAgent):
client=client,
)
try:
+ if stop_owned:
+ # Stop owns the only terminal settlement. The prewrite
+ # failure above is durable recovery evidence only; emitting
+ # a runtime-changed error or another silent result here
+ # would settle the same user request twice.
+ self._requeue_request_activity(request)
+ logger.info(
+ "Claude request was rejected after Stop won admission for %s",
+ runtime_session_key,
+ )
+ return
# A typed local resume failure takes precedence over incidental
# auth words in the working path or captured process diagnostic.
handled = False
- if not missing_session:
+ if not missing_session and not input_not_sent:
handled = await self.controller.agent_auth_service.maybe_emit_auth_recovery_message(
context,
"claude",
@@ -405,7 +770,11 @@ class ClaudeAgent(BaseAgent):
contained=contained is True,
)
finally:
- self._release_service_runtime_turn(context)
+ # Stop owns a prewrite rejection's terminal settlement. Keep the
+ # service gate held until handle_stop finishes cleanup and emits
+ # its single silent terminal result.
+ if not stop_owned:
+ self._release_service_runtime_turn(context)
finally:
await self._delete_ack(context, request)
@@ -790,6 +1159,7 @@ class ClaudeAgent(BaseAgent):
composite_key: str,
*,
current_receiver_task: asyncio.Task | None = None,
+ expected_client=None,
preserve_pending_request_state: bool = False,
runtime_lock_held: bool = False,
activation_retired: bool = False,
@@ -797,38 +1167,68 @@ class ClaudeAgent(BaseAgent):
) -> None:
"""Drop Claude runtime state without canceling the current receiver task."""
+ current_client = self.claude_sessions.get(composite_key)
+ if expected_client is not None and current_client is not None and current_client is not expected_client:
+ return
+ cleanup_client = expected_client or current_client
+ cleanup_lock = self._steering_lock(composite_key)
await self._prepare_steering_cleanup(composite_key)
try:
await self._cleanup_runtime_session_state(
composite_key,
current_receiver_task=current_receiver_task,
+ expected_client=expected_client,
preserve_pending_request_state=preserve_pending_request_state,
runtime_lock_held=runtime_lock_held,
activation_retired=activation_retired,
reason=reason,
)
finally:
- self._retire_steering_state(composite_key)
+ current_client = self.claude_sessions.get(composite_key)
+ if current_client is None or current_client is cleanup_client:
+ self._retire_steering_state(composite_key, expected_lock=cleanup_lock)
async def _cleanup_runtime_session_state(
self,
composite_key: str,
*,
current_receiver_task: asyncio.Task | None = None,
+ expected_client=None,
preserve_pending_request_state: bool = False,
runtime_lock_held: bool = False,
activation_retired: bool = False,
reason: str = "claude_agent_cleanup",
) -> None:
+ current_client = self.claude_sessions.get(composite_key)
+ if expected_client is not None and current_client is not None and current_client is not expected_client:
+ return
+ self._retire_output_generation(composite_key, expected_client or current_client)
+ preserve_synthetic_delivery = self._has_synthetic_delivery_pending(composite_key)
+ preserve_pending_request_state = (
+ preserve_pending_request_state or preserve_synthetic_delivery
+ )
self._last_assistant_text.pop(composite_key, None)
self._pending_assistant_message.pop(composite_key, None)
+ self._buffered_assistant_messages.pop(composite_key, None)
+ self._activity_provenance_barriers.discard(composite_key)
+ self._activity_provenance_barrier_owners.pop(composite_key, None)
self._foreground_tool_use_ids.pop(composite_key, None)
self._turns_with_foreground_tools.discard(composite_key)
+ self._provisional_tool_use_ids.pop(composite_key, None)
+ self._provisional_unbound_tool_use_ids.pop(composite_key, None)
+ self._provisional_foreground_tool_use_ids.pop(composite_key, None)
+ self._provisional_activity_ids.pop(composite_key, None)
+ self._provisional_unbound_activity_ids.pop(composite_key, None)
+ self._provisional_foreground_task_ids.pop(composite_key, None)
+ self._provisional_phase_ids.pop(composite_key, None)
+ self._resolved_provisional_phase_ids.pop(composite_key, None)
self._clear_detached_foreground_tool_state(composite_key)
self._native_session_ids.pop(composite_key, None)
self._suppressed_synthetic_results.discard(composite_key)
self._suppressed_synthetic_error_text.pop(composite_key, None)
+ if not preserve_synthetic_delivery:
+ self._synthetic_pending_owners.pop(composite_key, None)
if not preserve_pending_request_state:
self._pending_reactions.pop(composite_key, None)
pending_requests = self._pending_requests.pop(composite_key, None) or []
@@ -837,12 +1237,29 @@ class ClaudeAgent(BaseAgent):
cleanup_name = "_cleanup_session_locked" if runtime_lock_held else "cleanup_session"
cleanup = getattr(self.session_handler, cleanup_name, None)
if callable(cleanup):
- await cleanup(
- composite_key,
- current_receiver_task=current_receiver_task,
- activation_retired=activation_retired,
- reason=reason,
- )
+ cleanup_kwargs = {
+ "current_receiver_task": current_receiver_task,
+ "activation_retired": activation_retired,
+ "reason": reason,
+ }
+ if expected_client is not None:
+ cleanup_kwargs["expected_client"] = expected_client
+ try:
+ await cleanup(composite_key, **cleanup_kwargs)
+ except TypeError as error:
+ # A few lightweight session-handler test doubles still expose
+ # the pre-generation-aware cleanup signature. Preserve the
+ # exact-client guard for the real handler while remaining
+ # compatible with those narrow adapters.
+ if "expected_client" not in str(error):
+ raise
+ cleanup_kwargs.pop("expected_client", None)
+ await cleanup(composite_key, **cleanup_kwargs)
+ return
+ if (
+ expected_client is not None
+ and self.claude_sessions.get(composite_key) is not expected_client
+ ):
return
receiver_task = self.receiver_tasks.pop(composite_key, None)
client = self.claude_sessions.pop(composite_key, None)
@@ -953,6 +1370,18 @@ class ClaudeAgent(BaseAgent):
self._steering_generations = generations
return generations.get(composite_key, 0)
+ def _steering_stop_epoch(self, composite_key: str) -> int:
+ epochs = getattr(self, "_steering_stop_epochs", None)
+ if epochs is None:
+ epochs = {}
+ self._steering_stop_epochs = epochs
+ return epochs.get(composite_key, 0)
+
+ def _advance_steering_stop_epoch(self, composite_key: str) -> int:
+ epoch = self._steering_stop_epoch(composite_key) + 1
+ self._steering_stop_epochs[composite_key] = epoch
+ return epoch
+
def _steering_closing_keys(self) -> set[str]:
closing = getattr(self, "_steering_closing", None)
if closing is None:
@@ -968,12 +1397,24 @@ class ClaudeAgent(BaseAgent):
return writers
async def _prepare_steering_cleanup(self, composite_key: str) -> None:
+ await self._close_steering_admission(composite_key)
+
+ def _mark_steering_closing(self, composite_key: str) -> None:
+ """Reject new native writes before an async receiver-end replay."""
+
self._steering_closing_keys().add(composite_key)
- locks = getattr(self, "_steering_locks", None) or {}
- lock = locks.get(composite_key)
- if lock is not None and composite_key in self._steering_writer_keys():
- async with lock:
- pass
+
+ async def _close_steering_admission(self, composite_key: str) -> None:
+ """Close native write admission and drain the current writer.
+
+ The closing marker prevents a waiter from entering the fence, while
+ acquiring the same lock makes an already-admitted write finish before
+ receiver replay or cleanup can settle the old generation.
+ """
+
+ self._mark_steering_closing(composite_key)
+ async with self._steering_lock(composite_key):
+ pass
def _retire_steering_state(
self,
@@ -1004,6 +1445,11 @@ class ClaudeAgent(BaseAgent):
ambiguous_interrupts = getattr(self, "_ambiguous_interrupts", None)
if ambiguous_interrupts is not None:
ambiguous_interrupts.discard(composite_key)
+ self._stop_owned_terminal_keys.discard(composite_key)
+ if not self._has_synthetic_delivery_pending(composite_key):
+ self._synthetic_pending_owners.pop(composite_key, None)
+ self._activity_provenance_barriers.discard(composite_key)
+ self._activity_provenance_barrier_owners.pop(composite_key, None)
self._steering_closing_keys().discard(composite_key)
self._steering_writer_keys().discard(composite_key)
@@ -1214,7 +1660,20 @@ class ClaudeAgent(BaseAgent):
)
try:
try:
- await client.query(message, session_id=composite_key)
+ await self._write_human_query(
+ client,
+ composite_key,
+ message,
+ target.agent_request.context,
+ )
+ except ClaudeInputNotSentError as exc:
+ self._remove_native_input_receipt(composite_key, input_receipt)
+ return steer_result(
+ SteerOutcome.REFUSED,
+ reason="runtime_generation_changed",
+ backend=self.name,
+ diagnostic=exc.reason,
+ )
except (asyncio.TimeoutError, TimeoutError) as exc:
input_receipt.state = "unknown"
self._advance_steering_generation(composite_key)
@@ -1326,6 +1785,11 @@ class ClaudeAgent(BaseAgent):
async def handle_stop(self, request: AgentRequest) -> bool:
composite_key = request.composite_session_id
if composite_key not in self.claude_sessions:
+ if self._has_synthetic_delivery_pending(composite_key):
+ # Terminal delivery already owns this turn. There is no native
+ # work left to interrupt and Stop must not steal its receipt.
+ self._schedule_completed_activity_flush(composite_key, request.context)
+ return True
request.stop_failure_reason = "not_active"
return False
@@ -1341,6 +1805,7 @@ class ClaudeAgent(BaseAgent):
try:
stopped_request = None
+ retained_terminal_owner = False
async with self._steering_lock(composite_key):
if (
self.claude_sessions.get(composite_key) is not client
@@ -1356,10 +1821,16 @@ class ClaudeAgent(BaseAgent):
self._steering_closing_keys().discard(composite_key)
self._ambiguous_interrupt_keys().add(composite_key)
raise
+ stop_epoch = self._advance_steering_stop_epoch(composite_key)
# Claim the pending Result owner before releasing the lock. A
# terminal frame queued behind interrupt must not settle this
# stopped Turn as a successful result.
- stopped_request = self._pop_pending_request(composite_key)
+ retained_terminal_owner = self._has_synthetic_delivery_pending(composite_key)
+ if not retained_terminal_owner:
+ stopped_request = self._pop_pending_request(composite_key)
+ if stopped_request is not None:
+ self._stop_owned_terminal_keys.add(composite_key)
+ self._stop_owned_terminal_epochs[composite_key] = stop_epoch
except Exception as err:
request.stop_failure_reason = "interrupt_failed"
logger.error(f"Failed to interrupt Claude session {composite_key}: {err}")
@@ -1404,6 +1875,10 @@ class ClaudeAgent(BaseAgent):
finally:
self._suppress_receiver_runtime_release.discard(composite_key)
+ if retained_terminal_owner:
+ self._schedule_completed_activity_flush(composite_key, request.context)
+ return True
+
try:
# A user-initiated stop is terminal but intentional, so it carries
# NO user-facing message: a single SILENT result settles the dot to
@@ -1450,6 +1925,19 @@ class ClaudeAgent(BaseAgent):
)
return True
+ def _receiver_generation_current(
+ self, composite_key: str, client,
+ identity: RuntimeActivationIdentity | None,
+ ) -> bool:
+ current = self.claude_sessions.get(composite_key)
+ if current is not None and current is not client:
+ return False
+ service = getattr(self.controller, "agent_service", None)
+ registry = getattr(service, "activation_registry", None)
+ if identity is not None and isinstance(registry, RuntimeActivationRegistry):
+ return registry.current(identity.backend, identity.resource_key) is identity
+ return True
+
async def _receive_messages(
self,
client,
@@ -1468,6 +1956,7 @@ class ClaudeAgent(BaseAgent):
if receiver_activation_identity is None:
receiver_activation_identity = self._client_activation_identity(client)
receiver_steering_lock = self._steering_lock(composite_key)
+ receiver_stop_epoch = self._steering_stop_epoch(composite_key)
self._set_activity_connection(
composite_key,
context,
@@ -1498,6 +1987,10 @@ class ClaudeAgent(BaseAgent):
try:
message = await anext(message_stream)
except StopAsyncIteration:
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
buffered = self._ambiguous_primary_results.pop(composite_key, None)
if buffered is None:
break
@@ -1508,6 +2001,10 @@ class ClaudeAgent(BaseAgent):
) = self._unpack_buffered_terminal(buffered)
settling_ambiguous_primary = True
except Exception:
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
buffered = self._ambiguous_primary_results.pop(composite_key, None)
if buffered is None:
raise
@@ -1522,6 +2019,10 @@ class ClaudeAgent(BaseAgent):
composite_key,
exc_info=True,
)
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
if settling_ambiguous_primary:
await self._cleanup_runtime_session_state(
composite_key,
@@ -1604,6 +2105,60 @@ class ClaudeAgent(BaseAgent):
# the one user-facing system exception: it belongs to the
# replacement run and must use that run's turn.
output_mode = None
+ result_owner = "agent"
+ if message_type == "result":
+ # Stop's interrupt may await before claiming its epoch.
+ # Observe its decision under the existing admission
+ # fence before classification, replay, or detached routing.
+ async with self._steering_lock(composite_key):
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
+ if self._stop_owned_result_for_receiver(
+ composite_key,
+ receiver_stop_epoch,
+ ):
+ self._clear_result_phase_state(composite_key)
+ self._stop_owned_terminal_keys.discard(composite_key)
+ logger.info(
+ "Suppressing Claude terminal result owned by Stop for %s",
+ composite_key,
+ )
+ continue
+ result_owner = self._result_owner(composite_key, message)
+ self._resolve_provisional_phase(
+ composite_key,
+ context,
+ owner=result_owner,
+ terminal_message=message,
+ )
+ if (
+ message_type == "assistant"
+ and self._should_buffer_assistant_message(composite_key)
+ ):
+ # TaskStarted can legally arrive before the terminal
+ # Result. Record foreground tool ownership now, while
+ # the visible assistant/toolcall output remains buffered
+ # until Result.origin classifies the phase.
+ for block in getattr(message, "content", []) or []:
+ if isinstance(block, ToolUseBlock):
+ self._track_tool_activity_mode(
+ composite_key,
+ block,
+ detached=False,
+ provisional=True,
+ )
+ self._buffered_assistant_messages.setdefault(
+ composite_key,
+ [],
+ ).append((message, terminal_steering_generation))
+ logger.debug(
+ "Buffered Claude Assistant frame until terminal provenance "
+ "is known for %s",
+ composite_key,
+ )
+ continue
if message_type in ("assistant", "result") or model_refusal_fallback_notice is not None:
output_mode = await self._maybe_begin_agent_initiated_turn(
context,
@@ -1612,8 +2167,13 @@ class ClaudeAgent(BaseAgent):
working_path,
session_key,
message_type=message_type,
+ result_owner=result_owner,
receiver_activation_identity=receiver_activation_identity,
)
+ if output_mode in {"activity", "detached"}:
+ record = self._provisional_output_record(composite_key, output_mode)
+ if record is not None and record.context is None:
+ record.context = self._snapshot_output_context(context)
if message_type == "assistant":
self.session_handler.touch_session_activity(composite_key)
@@ -1629,7 +2189,12 @@ class ClaudeAgent(BaseAgent):
block,
detached=(
output_mode == "detached"
- or composite_key in self._detached_activity_outputs
+ or output_mode == "activity"
+ ),
+ provisional=bool(
+ self._pending_requests.get(composite_key)
+ and output_mode
+ not in {"detached", "activity"}
),
)
# AskUserQuestion handling disabled - tool is disallowed via ClaudeAgentOptions
@@ -1668,46 +2233,32 @@ class ClaudeAgent(BaseAgent):
assistant_text = self._extract_text_blocks(message, context)
if output_mode == "detached":
if assistant_text:
- self._detached_unsolicited_text[composite_key] = assistant_text
- continue
- if composite_key in self._detached_activity_outputs:
- if assistant_text:
- self._detached_assistant_text[composite_key] = assistant_text
- continue
- async with self._steering_lock(composite_key):
- if composite_key in self._steering_closing_keys():
- logger.info(
- "Ignoring Claude assistant output during teardown for %s",
- composite_key,
- )
- continue
- diagnostic = self._terminal_backend_failure(message, assistant_text)
- if (
- diagnostic is not None
- and self._terminal_claim_superseded(
- composite_key,
- terminal_steering_generation,
- )
- ):
- self._discard_completed_primary_input_receipts(
- composite_key
- )
- self._suppressed_synthetic_results.add(composite_key)
- self._suppressed_synthetic_error_text[composite_key] = diagnostic
- logger.info(
- "Ignoring Claude terminal assistant failure superseded by steering for %s",
+ self._retain_detached_unsolicited_assistant_text(
composite_key,
+ assistant_text,
)
- continue
- if diagnostic is not None:
- self._clear_native_input_receipts(composite_key)
- failure_disposition = await self._handle_assistant_terminal_failure(
- context,
+ continue
+ if output_mode == "activity":
+ record = self._provisional_output_record(composite_key, "activity")
+ if assistant_text and record is not None:
+ record.text = assistant_text
+ continue
+ failure_disposition = await self._process_assistant_terminal_frame(
+ context,
+ composite_key,
+ message,
+ assistant_text,
+ terminal_steering_generation=terminal_steering_generation,
+ )
+ if failure_disposition in {"teardown", "superseded"}:
+ continue
+ if failure_disposition == "auth":
+ await self._cleanup_runtime_session(
composite_key,
- message,
- assistant_text,
+ current_receiver_task=asyncio.current_task(),
+ preserve_pending_request_state=True,
+ reason="assistant_auth_failure",
)
- if failure_disposition == "auth":
return
if failure_disposition:
continue
@@ -1799,6 +2350,7 @@ class ClaudeAgent(BaseAgent):
getattr(message, "subtype", "") or "",
formatted_message,
)
+ auth_failure_handled = False
async with self._steering_lock(composite_key):
if (
composite_key in self._steering_closing_keys()
@@ -1828,115 +2380,66 @@ class ClaudeAgent(BaseAgent):
if callable(mark_session_idle):
mark_session_idle(composite_key)
await self._clear_pending_reactions(composite_key, context)
- return
+ auth_failure_handled = True
+ if auth_failure_handled:
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ preserve_pending_request_state=True,
+ reason="result_auth_failure",
+ )
+ return
continue
if message_type == "result":
+ if await self._flush_buffered_assistant_messages(
+ composite_key,
+ context,
+ owner=result_owner,
+ terminal_steering_generation=terminal_steering_generation,
+ ):
+ self._clear_result_phase_state(composite_key)
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ preserve_pending_request_state=True,
+ reason="assistant_auth_failure",
+ )
+ return
raw_result_text = getattr(message, "result", None)
result_text = raw_result_text
- detached_activities = self._detached_activity_outputs.pop(
- composite_key,
- None,
- )
- if detached_activities:
- detached_activity = detached_activities[-1]
- detached_text = self._detached_assistant_text.get(
- composite_key
+ if output_mode in {"activity", "detached"}:
+ record = (
+ self._provisional_output_record(composite_key, "activity")
+ if output_mode == "activity"
+ else self._detached_phase(composite_key)
)
- result_text = self._select_detached_result_text(
- composite_key,
- message,
- raw_result_text,
- detached_text,
+ if record is None:
+ raise RuntimeError("Claude terminal phase has no output owner")
+ self._classify_output_record(
+ record, context,
+ text=self._select_detached_result_text(
+ composite_key, message, raw_result_text, record.text,
+ ),
+ message=message,
)
- self._detached_assistant_text.pop(composite_key, None)
- try:
- await self._emit_activity_result(
- context,
- detached_activity,
- result_text,
- detached=True,
- completes_turn=False,
- subtype=getattr(message, "subtype", "") or "",
- duration_ms=getattr(message, "duration_ms", 0),
- activities=detached_activities,
- )
- except ActivityOutputDeliveryError as err:
- if not err.delivered or err.durable:
- registry = self._activity_registry()
- if registry is not None:
- self._requeue_activities(
- registry,
- detached_activities,
- )
- if err.durable:
- self._schedule_completed_activity_flush(
- composite_key,
- context,
- )
- raise
- logger.error(
- "Delivered Claude Activity batch remains fail-closed "
- "after incomplete local settlement (runtime=%s error=%s)",
- composite_key,
- err,
- )
- registry = self._activity_registry()
- if (
- registry is not None
- and self._requeue_activities_for_local_settlement(
- registry,
- detached_activities,
- )
- ):
- self._schedule_completed_activity_flush(
- composite_key,
- context,
- )
- except Exception:
- registry = self._activity_registry()
- if registry is not None:
- self._requeue_activities(
- registry,
- detached_activities,
- )
- raise
- # The detached Activity output has no authority over
- # a newer pending request. Its Turn stays owned by its
- # own backend query and liveness/timeout path.
- self._mark_session_idle_if_runtime_free(composite_key)
- self._signal_activity_output_settled(composite_key)
+ # Freeze this phase before awaiting any earlier retry.
+ # Every later Result gets its own record, including a
+ # failure Result with no preceding Assistant frame.
+ await self._flush_output_recovery(composite_key, context)
self._clear_detached_foreground_tool_state(composite_key)
- continue
- if output_mode == "detached":
- detached_text = self._detached_unsolicited_text.get(
- composite_key
+ self._resolved_provisional_phase_ids.pop(
+ composite_key,
+ None,
)
- result_text = self._select_detached_result_text(
- composite_key,
- message,
- raw_result_text,
- detached_text,
- )
- self._detached_unsolicited_text.pop(composite_key, None)
- self._detached_unsolicited_outputs.discard(composite_key)
- if result_text:
- await self.emit_result_message(
- context,
- result_text,
- subtype=getattr(message, "subtype", "") or "",
- duration_ms=getattr(message, "duration_ms", 0),
- parse_mode="markdown",
- output=self._unsolicited_message_output(message),
- )
- self._clear_detached_foreground_tool_state(composite_key)
- continue
- terminal_superseded = False
- superseded_result_text = ""
- superseded_request = None
- async with self._steering_lock(composite_key):
- self._pending_assistant_message.pop(composite_key, None)
- if self._consume_suppressed_synthetic_result(
+ continue
+ terminal_superseded = False
+ superseded_result_text = ""
+ superseded_request = None
+ failure_disposition = None
+ async with self._steering_lock(composite_key):
+ self._pending_assistant_message.pop(composite_key, None)
+ if self._consume_suppressed_synthetic_result(
composite_key,
message,
raw_result_text,
@@ -2006,21 +2509,32 @@ class ClaudeAgent(BaseAgent):
if failure_disposition == "auth":
self._foreground_tool_use_ids.pop(composite_key, None)
self._turns_with_foreground_tools.discard(composite_key)
- return
- if failure_disposition:
+ if failure_disposition == "auth":
+ pass
+ elif failure_disposition:
self._foreground_tool_use_ids.pop(composite_key, None)
self._turns_with_foreground_tools.discard(composite_key)
continue
- # ResultMessage.result already contains the last
- # AssistantMessage, so only the terminal owner emits it.
- pending_request = self._pop_pending_request(composite_key)
- output_activities = self._request_activities(pending_request)
- output_activity = output_activities[-1] if output_activities else None
- result_text = self._select_terminal_text(
- composite_key,
- raw_result_text,
- )
+ if failure_disposition is None:
+ # ResultMessage.result already contains the last
+ # AssistantMessage, so only the terminal owner emits it.
+ pending_request = self._pop_pending_request(composite_key)
+ if (
+ self._synthetic_pending_owners.get(composite_key)
+ is pending_request
+ ):
+ self._synthetic_pending_owners.pop(
+ composite_key,
+ None,
+ )
+ self._refresh_activity_provenance_barrier(composite_key)
+ output_activities = self._request_activities(pending_request)
+ output_activity = output_activities[-1] if output_activities else None
+ result_text = self._select_terminal_text(
+ composite_key,
+ raw_result_text,
+ )
if terminal_superseded:
self._adopt_pending_turn_token(context, superseded_request)
@@ -2039,6 +2553,15 @@ class ClaudeAgent(BaseAgent):
)
continue
+ if failure_disposition == "auth":
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ preserve_pending_request_state=True,
+ reason="result_auth_failure",
+ )
+ return
+
# The receiver is long-lived and reused across a session's
# turns, so ``context`` still carries the FIRST turn's
# ``turn_token``. Adopt the token of the turn THIS result
@@ -2168,6 +2691,26 @@ class ClaudeAgent(BaseAgent):
backend_cleanup.set()
if close_after_payload.get("_close_after_backend_cleanup") is backend_cleanup:
close_after_payload.pop("_close_after_backend_cleanup", None)
+ if (
+ result_owner == "human"
+ and not settling_ambiguous_primary
+ ):
+ self._refresh_activity_provenance_barrier(composite_key)
+ registry = self._activity_registry()
+ if (
+ registry is not None
+ and registry.has_completed_output(
+ self.name,
+ composite_key,
+ )
+ ):
+ self._schedule_completed_activity_flush(
+ composite_key,
+ context,
+ expected_steering_generation=(
+ self._steering_generation(composite_key)
+ ),
+ )
if settling_ambiguous_primary:
return
continue
@@ -2179,47 +2722,118 @@ class ClaudeAgent(BaseAgent):
continue
# EOF owns terminal settlement. Announce it before awaiting any lock
# so an overlapping write cannot report ACCEPTED with no receiver.
- self._steering_closing_keys().add(composite_key)
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
+ self._mark_steering_closing(composite_key)
+ buffered_failure = await self._replay_buffered_terminal_failures(
+ composite_key,
+ context,
+ terminal_steering_generation=self._steering_generation(composite_key),
+ expected_client=client,
+ expected_activation_identity=receiver_activation_identity,
+ )
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
eof_steering_generation = self._steering_generation(composite_key)
+ pending_requests = self._pending_requests.get(composite_key) or []
+ synthetic_delivery_pending = self._has_synthetic_delivery_pending(
+ composite_key
+ )
+ # A failed detached emit still has a live synthetic owner. It is not
+ # an EOF-without-result turn: settle the retained output first so
+ # _handle_receiver_eof cannot replace it with a generic error or
+ # release the gate before durable delivery.
+ had_pending_requests = bool(pending_requests) and not synthetic_delivery_pending
+ if had_pending_requests:
+ if buffered_failure is None:
+ await self._handle_receiver_eof(
+ composite_key, context,
+ expected_client=client,
+ expected_activation_identity=receiver_activation_identity,
+ )
+ else:
+ await self._handle_receiver_eof(
+ composite_key,
+ context,
+ buffered_failure=buffered_failure,
+ expected_client=client,
+ expected_activation_identity=receiver_activation_identity,
+ )
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
await self._flush_completed_activity_outputs(
composite_key,
context,
expected_steering_generation=eof_steering_generation,
allow_closing=True,
+ expected_client=client,
+ expected_activation_identity=receiver_activation_identity,
)
await self._flush_detached_activity_output(composite_key, context)
- await self._flush_detached_unsolicited_output(composite_key, context)
- if not self._has_pending_requests(composite_key):
+ if synthetic_delivery_pending:
+ # The receiver generation is dead even when its detached
+ # recovery owner remains alive. Retire only this exact client;
+ # the retained owner and gate are generation-independent.
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ expected_client=client,
+ preserve_pending_request_state=True,
+ reason="receiver_eof_with_retained_detached_output",
+ )
+ return
+ if not had_pending_requests and not self._has_pending_requests(composite_key):
await self._cleanup_runtime_session(
composite_key,
current_receiver_task=asyncio.current_task(),
+ expected_client=client,
preserve_pending_request_state=True,
reason="receiver_eof",
)
return
- await self._handle_receiver_eof(
- composite_key,
- context,
- )
except asyncio.CancelledError:
# Receiver task was explicitly cancelled (e.g. /stop, /clear,
# or a new message replacing the session). Clean up reactions
# because this receiver will never process another result.
composite_key = composite_key or f"{base_session_id}:{working_path}"
- mark_session_idle = getattr(self.session_handler, "mark_session_idle", None)
- if callable(mark_session_idle):
- mark_session_idle(composite_key)
logger.info("Claude receiver cancelled for session %s", composite_key)
- await self._clear_pending_reactions(composite_key, context)
- if composite_key not in self._suppress_receiver_runtime_release:
- self._release_service_runtime_turn(context)
+ if self._receiver_generation_current(composite_key, client, receiver_activation_identity):
+ retained_owner = self._has_synthetic_delivery_pending(composite_key)
+ if not retained_owner:
+ self._mark_session_idle_if_no_pending_requests(composite_key)
+ await self._clear_pending_reactions(
+ composite_key, context, preserve_pending_requests=retained_owner,
+ )
+ if composite_key not in self._suppress_receiver_runtime_release and not retained_owner:
+ self._release_service_runtime_turn(context)
raise
except Exception as e:
composite_key = composite_key or f"{base_session_id}:{working_path}"
- # Claim teardown before waiting for an in-flight steering write. This
- # makes its post-write check return UNKNOWN instead of ACCEPTED while
- # the sole receiver is already exiting.
- self._steering_closing_keys().add(composite_key)
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
+ # Close admission before replaying the prior turn's failure. The
+ # replay is explicitly allowed to settle that already-buffered
+ # failure, while every new native write sees the closing fence.
+ self._mark_steering_closing(composite_key)
+ buffered_failure = await self._replay_buffered_terminal_failures(
+ composite_key,
+ context,
+ terminal_steering_generation=self._steering_generation(composite_key),
+ expected_client=client,
+ expected_activation_identity=receiver_activation_identity,
+ )
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
logger.error(
f"Error in Claude receiver for session {composite_key}: {e}",
exc_info=True,
@@ -2228,7 +2842,23 @@ class ClaudeAgent(BaseAgent):
composite_key,
context,
e,
+ buffered_failure=buffered_failure,
+ expected_client=client,
+ expected_activation_identity=receiver_activation_identity,
+ )
+ if not self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ ):
+ return
+ await self._flush_completed_activity_outputs(
+ composite_key,
+ context,
+ expected_steering_generation=self._steering_generation(composite_key),
+ allow_closing=True,
+ expected_client=client,
+ expected_activation_identity=receiver_activation_identity,
)
+ await self._flush_detached_activity_output(composite_key, context)
# NOTE: no `finally` cleanup of pending reactions here.
# When the receiver ends normally (stream exhausted after a result),
# new messages may have already queued their reactions via
@@ -2238,37 +2868,126 @@ class ClaudeAgent(BaseAgent):
# normal-result case is handled by _remove_pending_reaction()
# inside the loop.
finally:
- self._suppressed_synthetic_results.discard(composite_key)
- self._suppressed_synthetic_error_text.pop(composite_key, None)
- self._foreground_tool_use_ids.pop(composite_key, None)
- self._turns_with_foreground_tools.discard(composite_key)
- self._clear_detached_foreground_tool_state(composite_key)
- detached_activities = self._detached_activity_outputs.pop(composite_key, None)
- if detached_activities:
- registry = self._activity_registry()
- if registry is not None:
- self._requeue_activities(registry, detached_activities)
- self._schedule_completed_activity_flush(composite_key, context)
- self._detached_assistant_text.pop(composite_key, None)
- self._detached_unsolicited_outputs.discard(composite_key)
- self._detached_unsolicited_text.pop(composite_key, None)
+ owns_generation = self._receiver_generation_current(
+ composite_key, client, receiver_activation_identity,
+ )
+ if owns_generation:
+ self._suppressed_synthetic_results.discard(composite_key)
+ self._suppressed_synthetic_error_text.pop(composite_key, None)
+ self._foreground_tool_use_ids.pop(composite_key, None)
+ self._turns_with_foreground_tools.discard(composite_key)
+ self._clear_detached_foreground_tool_state(composite_key)
+ self._retire_output_generation(composite_key, client, context)
+ if any(
+ record.lifecycle == "delivery_pending"
+ for record in self._output_records_for_runtime(composite_key)
+ ):
+ self._schedule_completed_activity_flush(composite_key, context)
self._end_activity_runtime(
composite_key,
activation_identity=receiver_activation_identity,
)
- if composite_key is not None:
+ if composite_key is not None and owns_generation:
self._retire_steering_state(
composite_key,
expected_lock=receiver_steering_lock,
)
+ async def _replay_buffered_terminal_failures(
+ self,
+ composite_key: str,
+ context: MessageContext,
+ *,
+ terminal_steering_generation: int,
+ expected_client,
+ expected_activation_identity: RuntimeActivationIdentity | None,
+ ) -> _BufferedClaudeFailureReplay | None:
+ """Replay buffered structured failures before receiver-end settlement.
+
+ Origin-less Assistant text is intentionally not replayed here: without a
+ terminal Result it has no safe owner. Structured failure frames are
+ different because ``_process_assistant_terminal_frame`` already owns the
+ shared teardown, steering, receipt, and synthetic-result policy.
+ """
+
+ if not self._buffered_assistant_messages.get(composite_key):
+ return None
+ async with self._steering_lock(composite_key):
+ if not self._receiver_generation_current(
+ composite_key, expected_client, expected_activation_identity,
+ ):
+ return None
+ messages = self._buffered_assistant_messages.pop(composite_key, None)
+ if not messages:
+ return None
+
+ for message, frame_steering_generation in messages:
+ try:
+ text = self._extract_text_blocks(message, context)
+ except Exception:
+ text = str(getattr(message, "result", "") or "")
+ logger.warning(
+ "Failed to extract buffered Claude Assistant failure text for %s",
+ composite_key,
+ exc_info=True,
+ )
+ diagnostic = self._terminal_backend_failure(message, text)
+ if diagnostic is None:
+ continue
+ try:
+ disposition = await self._process_assistant_terminal_frame(
+ context,
+ composite_key,
+ message,
+ text,
+ terminal_steering_generation=(
+ frame_steering_generation
+ if frame_steering_generation is not None
+ else terminal_steering_generation
+ ),
+ allow_closing=True,
+ expected_client=expected_client,
+ expected_activation_identity=expected_activation_identity,
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ # The receiver-end path must still run even if recording or
+ # emitting the replayed failure fails. The pending request is
+ # deliberately left for that path to settle conservatively.
+ logger.warning(
+ "Failed to replay buffered Claude terminal failure for %s",
+ composite_key,
+ exc_info=True,
+ )
+ return _BufferedClaudeFailureReplay(
+ diagnostic=diagnostic,
+ replay_failed=True,
+ )
+ if disposition == "auth":
+ return _BufferedClaudeFailureReplay(
+ diagnostic=diagnostic,
+ auth_handled=True,
+ )
+ if disposition == "failure":
+ return _BufferedClaudeFailureReplay(diagnostic=diagnostic)
+ return None
+
async def _handle_receiver_eof(
self,
composite_key: str,
context: MessageContext,
+ *,
+ buffered_failure: _BufferedClaudeFailureReplay | None = None,
+ expected_client=None,
+ expected_activation_identity: RuntimeActivationIdentity | None = None,
) -> None:
"""Settle a Claude receiver that ended without a ResultMessage."""
async with self._steering_lock(composite_key):
+ if expected_client is not None and not self._receiver_generation_current(
+ composite_key, expected_client, expected_activation_identity,
+ ):
+ return
pending_request = self._pop_pending_request(composite_key)
if pending_request is None:
return
@@ -2290,8 +3009,12 @@ class ClaudeAgent(BaseAgent):
# The receiver context may belong to an earlier turn. Adopt the FIFO
# owner's identity before any visible or durable failure output.
self._adopt_pending_turn_token(context, pending_request)
- terminal_error = "Claude receiver ended without a terminal result"
- client = self.claude_sessions.get(composite_key)
+ terminal_error = (
+ buffered_failure.diagnostic
+ if buffered_failure is not None and buffered_failure.replay_failed
+ else "Claude receiver ended without a terminal result"
+ )
+ client = expected_client or self.claude_sessions.get(composite_key)
returncode = get_claude_client_returncode(client)
auth_handled = False
# Only the ``handle_session_error`` branch below can classify this EOF as a
@@ -2301,6 +3024,63 @@ class ClaudeAgent(BaseAgent):
# the default has to be "not contained": an unclassified EOF is a real
# failure and must keep saying so.
contained = False
+ # A failed replay is only a diagnostic. The still-owned request needs
+ # the ordinary no-result settlement below, before releasing its gate.
+ if buffered_failure is not None and not buffered_failure.replay_failed:
+ if buffered_failure.auth_handled:
+ self._retire_failed_auth_turn(
+ composite_key,
+ context,
+ failed_request=pending_request,
+ )
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ expected_client=client,
+ preserve_pending_request_state=True,
+ reason="receiver_auth_failure",
+ )
+ await self._remove_ack_reaction(pending_request)
+ self._discard_pending_reaction(composite_key)
+ await self._clear_pending_reactions(composite_key, context)
+ self._mark_session_idle_if_no_pending_requests(composite_key)
+ self._release_service_runtime_turn(context)
+ return
+
+ diagnostic = buffered_failure.diagnostic
+ self._requeue_request_activity(pending_request)
+ handle_session_error = getattr(self.session_handler, "handle_session_error", None)
+ if callable(handle_session_error):
+ contained = (
+ await handle_session_error(
+ composite_key,
+ context,
+ RuntimeError(diagnostic),
+ client=client,
+ )
+ is True
+ )
+ logger.warning(
+ "Claude receiver ended after buffered terminal failure for session %s",
+ composite_key,
+ )
+ await self._remove_specific_pending_reaction(composite_key, context, pending_request)
+ await self._remove_ack_reaction(pending_request)
+ self._last_assistant_text.pop(composite_key, None)
+ self._pending_assistant_message.pop(composite_key, None)
+ self._refresh_activity_provenance_barrier(composite_key)
+ self._mark_session_idle_if_no_pending_requests(composite_key)
+ if returncode is None or not callable(handle_session_error):
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ expected_client=client,
+ preserve_pending_request_state=True,
+ reason="receiver_eof_after_buffered_failure",
+ )
+ self._release_service_runtime_turn(context)
+ return
+
if returncode is not None:
eof_error = RuntimeError(terminal_error)
diagnostic = self._claude_error_diagnostic(
@@ -2335,6 +3115,7 @@ class ClaudeAgent(BaseAgent):
await self._cleanup_runtime_session(
composite_key,
current_receiver_task=asyncio.current_task(),
+ expected_client=client,
preserve_pending_request_state=True,
reason="receiver_auth_failure",
)
@@ -2397,6 +3178,7 @@ class ClaudeAgent(BaseAgent):
await self._cleanup_runtime_session(
composite_key,
current_receiver_task=asyncio.current_task(),
+ expected_client=client,
preserve_pending_request_state=True,
reason="receiver_eof_without_result",
)
@@ -2415,8 +3197,27 @@ class ClaudeAgent(BaseAgent):
composite_key: str,
context: MessageContext,
error: Exception,
+ *,
+ buffered_failure: _BufferedClaudeFailureReplay | None = None,
+ expected_client=None,
+ expected_activation_identity: RuntimeActivationIdentity | None = None,
) -> None:
+ retain_runtime_turn = False
+ # A receiver exception retires this client generation. Close native
+ # write admission before any replay or cleanup awaits so a concurrent
+ # primary/steer cannot enter the dying receiver. Keep the steering
+ # lock limited to the state snapshot; session cleanup may acquire the
+ # generation lock and must never do so while this lock is held.
+ if expected_client is not None and not self._receiver_generation_current(
+ composite_key, expected_client, expected_activation_identity,
+ ):
+ return
+ self._mark_steering_closing(composite_key)
async with self._steering_lock(composite_key):
+ if expected_client is not None and not self._receiver_generation_current(
+ composite_key, expected_client, expected_activation_identity,
+ ):
+ return
mark_session_idle = getattr(self.session_handler, "mark_session_idle", None)
if callable(mark_session_idle):
mark_session_idle(composite_key)
@@ -2426,11 +3227,109 @@ class ClaudeAgent(BaseAgent):
pending = self._pending_requests.get(composite_key) or []
pending_request = pending[0] if pending else None
self._adopt_pending_turn_token(context, pending_request)
- await self._clear_pending_reactions(composite_key, context)
+ # Clearing reactions can touch the IM adapter, so keep that await out of
+ # the admission fence as well. The fence is already closed and the
+ # state snapshot above identifies the failed turn.
+ synthetic_delivery_pending = self._has_synthetic_delivery_pending(composite_key)
+ try:
+ if synthetic_delivery_pending:
+ await self._clear_pending_reactions(
+ composite_key,
+ context,
+ preserve_pending_requests=True,
+ )
+ else:
+ await self._clear_pending_reactions(composite_key, context)
+
+ if (
+ buffered_failure is None
+ and synthetic_delivery_pending
+ ):
+ # The receiver is gone, but the detached output still has a
+ # durable recovery owner. Do not replace it with a generic
+ # receiver error or retire the gate before its retry settles.
+ retain_runtime_turn = True
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ expected_client=expected_client,
+ preserve_pending_request_state=True,
+ reason="receiver_error_with_retained_detached_output",
+ )
+ return
+
+ if buffered_failure is not None:
+ if buffered_failure.auth_handled:
+ if pending_request is not None:
+ current_pending = self._pending_requests.get(composite_key) or []
+ self._retire_failed_auth_turn(
+ composite_key,
+ context,
+ failed_request=(
+ None
+ if current_pending
+ and current_pending[0] is pending_request
+ else pending_request
+ ),
+ )
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ expected_client=expected_client,
+ preserve_pending_request_state=True,
+ reason="receiver_error_auth_failure",
+ )
+ if pending_request is not None:
+ await self._remove_ack_reaction(pending_request)
+ self._discard_pending_reaction(composite_key)
+ self._refresh_activity_provenance_barrier(composite_key)
+ self._mark_session_idle_if_no_pending_requests(composite_key)
+ return
+
+ if buffered_failure.replay_failed:
+ # A replay exception needs the ordinary receiver-error
+ # fallback, but if the shared terminal path already removed
+ # the request there is no remaining turn to settle again.
+ if pending_request is None:
+ logger.warning(
+ "Buffered Claude failure replay failed after terminal "
+ "ownership was already consumed for %s",
+ composite_key,
+ )
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ expected_client=expected_client,
+ preserve_pending_request_state=True,
+ reason="receiver_error_after_buffered_failure",
+ )
+ self._refresh_activity_provenance_barrier(composite_key)
+ return
+ buffered_failure = None
+ else:
+ # The structured failure already emitted the authoritative
+ # terminal failure. Receiver transport failure is cleanup
+ # context, not a second user-visible error.
+ logger.warning(
+ "Claude receiver error followed buffered terminal failure for %s: %s",
+ composite_key,
+ buffered_failure.diagnostic,
+ )
+ await self._cleanup_runtime_session(
+ composite_key,
+ current_receiver_task=asyncio.current_task(),
+ expected_client=expected_client,
+ preserve_pending_request_state=True,
+ reason="receiver_error_after_buffered_failure",
+ )
+ self._refresh_activity_provenance_barrier(composite_key)
+ self._mark_session_idle_if_no_pending_requests(composite_key)
+ return
+
# Read the client once and reuse it for both the health gate and the
# handler, so a replacement registering in between cannot make the
# two disagree about which generation actually failed.
- errored_client = self.claude_sessions.get(composite_key)
+ errored_client = expected_client or self.claude_sessions.get(composite_key)
diagnostic = self._claude_error_diagnostic(
composite_key,
error,
@@ -2455,17 +3354,27 @@ class ClaudeAgent(BaseAgent):
terminal_error=diagnostic,
)
if handled:
- self._retire_failed_auth_turn(composite_key, context)
+ if pending_request is not None:
+ current_pending = self._pending_requests.get(composite_key) or []
+ self._retire_failed_auth_turn(
+ composite_key,
+ context,
+ failed_request=(
+ None
+ if current_pending and current_pending[0] is pending_request
+ else pending_request
+ ),
+ )
await self._cleanup_runtime_session(
composite_key,
current_receiver_task=asyncio.current_task(),
+ expected_client=errored_client,
preserve_pending_request_state=True,
reason="receiver_error_auth_failure",
)
if pending_request is not None:
await self._remove_ack_reaction(pending_request)
self._discard_pending_reaction(composite_key)
- await self._clear_pending_reactions(composite_key, context)
self._mark_session_idle_if_no_pending_requests(composite_key)
else:
# Give a claimed Activity batch back to the Registry BEFORE the
@@ -2516,7 +3425,9 @@ class ClaudeAgent(BaseAgent):
diagnostic,
contained=contained is True,
)
- self._release_service_runtime_turn(context)
+ finally:
+ if not retain_runtime_turn:
+ self._release_service_runtime_turn(context)
async def _handle_assistant_terminal_failure(
self,
@@ -2542,6 +3453,57 @@ class ClaudeAgent(BaseAgent):
self._suppressed_synthetic_error_text[composite_key] = diagnostic
return "failure"
+ async def _process_assistant_terminal_frame(
+ self,
+ context: MessageContext,
+ composite_key: str,
+ message,
+ text: str,
+ *,
+ terminal_steering_generation: int,
+ allow_closing: bool = False,
+ expected_client=None,
+ expected_activation_identity: RuntimeActivationIdentity | None = None,
+ ) -> str | None:
+ """Apply one Assistant failure through the shared steering decision path."""
+
+ async with self._steering_lock(composite_key):
+ if expected_client is not None and not self._receiver_generation_current(
+ composite_key, expected_client, expected_activation_identity,
+ ):
+ return "superseded"
+ if composite_key in self._steering_closing_keys() and not allow_closing:
+ logger.info(
+ "Ignoring Claude assistant output during teardown for %s",
+ composite_key,
+ )
+ return "teardown"
+
+ diagnostic = self._terminal_backend_failure(message, text)
+ if (
+ diagnostic is not None
+ and self._terminal_claim_superseded(
+ composite_key,
+ terminal_steering_generation,
+ )
+ ):
+ self._discard_completed_primary_input_receipts(composite_key)
+ self._suppressed_synthetic_results.add(composite_key)
+ self._suppressed_synthetic_error_text[composite_key] = diagnostic
+ logger.info(
+ "Ignoring Claude terminal assistant failure superseded by steering for %s",
+ composite_key,
+ )
+ return "superseded"
+ if diagnostic is not None:
+ self._clear_native_input_receipts(composite_key)
+ return await self._handle_assistant_terminal_failure(
+ context,
+ composite_key,
+ message,
+ text,
+ )
+
async def _handle_terminal_failure_result(
self,
context: MessageContext,
@@ -2585,12 +3547,6 @@ class ClaudeAgent(BaseAgent):
)
if handled_auth:
self._retire_failed_auth_turn(composite_key, context)
- await self._cleanup_runtime_session(
- composite_key,
- current_receiver_task=asyncio.current_task(),
- preserve_pending_request_state=True,
- reason="transport_auth_failure",
- )
if pending_request is not None:
await self._remove_ack_reaction(pending_request)
self._discard_pending_reaction(composite_key)
@@ -2791,6 +3747,92 @@ class ClaudeAgent(BaseAgent):
return []
return list(getattr(request, "output_activities", None) or [])
+ def _retire_synthetic_pending_owner(
+ self,
+ composite_key: str,
+ context: MessageContext,
+ *,
+ owner: AgentRequest | None,
+ ) -> bool:
+ """Release this output's captured owner, never the current runtime owner."""
+
+ synthetic = owner
+ if (
+ synthetic is not None
+ and (
+ self._request_activities(synthetic)
+ or any(
+ record.request is synthetic
+ for record in self._output_records_for_runtime(composite_key)
+ )
+ )
+ ):
+ # Delivery has not crossed the durable boundary. Keep the same
+ # owner responsible for the retained payload and its gate.
+ return False
+ if synthetic is None:
+ return False
+ if self._synthetic_pending_owners.get(composite_key) is synthetic:
+ self._synthetic_pending_owners.pop(composite_key, None)
+ pending = self._pending_requests.get(composite_key) or []
+ if pending and pending[0] is synthetic:
+ self._pop_pending_request(composite_key)
+ self._clear_request_activities(synthetic)
+ self._mark_session_idle_if_no_pending_requests(composite_key)
+ self._release_service_runtime_turn(
+ getattr(synthetic, "context", None) or context
+ )
+ return True
+
+ def _has_synthetic_delivery_pending(self, composite_key: str) -> bool:
+ """Whether a synthetic turn still owns retained detached output."""
+
+ synthetic = self._synthetic_pending_owners.get(composite_key)
+ pending = self._pending_requests.get(composite_key) or []
+ if synthetic is None and pending:
+ candidate = pending[0]
+ if getattr(candidate, "_claude_synthetic_owner", False):
+ synthetic = candidate
+ self._synthetic_pending_owners[composite_key] = candidate
+ if synthetic is None:
+ # Detached output can coexist with a real pending human request.
+ # That output needs retry, but it does not own the human turn or
+ # its runtime gate. Only an explicit synthetic owner may hold the
+ # gate across receiver cleanup.
+ return False
+ return bool(
+ self._request_activities(synthetic)
+ or any(
+ record.request is synthetic
+ and record.lifecycle == "delivery_pending"
+ for record in self._output_records_for_runtime(composite_key)
+ )
+ )
+
+ def _notify_classified_terminal_activities(
+ self,
+ activities: list[SessionActivity],
+ ) -> None:
+ """Hand deferred terminal Activities to their existing Run owner."""
+
+ service = getattr(self.controller, "agent_service", None)
+ notify = getattr(service, "on_activity_terminal", None)
+ if not callable(notify):
+ return
+ for activity in activities:
+ status = str(getattr(activity, "status", "") or "").strip().lower()
+ if status not in {
+ "completed",
+ "failed",
+ "stopped",
+ "killed",
+ "disconnected",
+ }:
+ continue
+ if getattr(activity, "completed_at", None) is None:
+ continue
+ notify(activity)
+
def _attach_request_activities(
self,
request: AgentRequest,
@@ -2908,7 +3950,7 @@ class ClaudeAgent(BaseAgent):
def _activity_output_pending(self, composite_key: str) -> bool:
registry = self._activity_registry()
return bool(
- composite_key in self._detached_activity_outputs
+ self._output_record(composite_key, None, owner="activity") is not None
or (
registry is not None
and (
@@ -2998,6 +4040,7 @@ class ClaudeAgent(BaseAgent):
block: ToolUseBlock,
*,
detached: bool = False,
+ provisional: bool = False,
) -> None:
"""Remember which Claude task frames belong to foreground tool steps."""
@@ -3014,6 +4057,18 @@ class ClaudeAgent(BaseAgent):
runs_in_background = bool(
isinstance(tool_input, dict) and tool_input.get("run_in_background") is True
)
+ if provisional:
+ self._provisional_tool_use_ids.setdefault(
+ composite_key,
+ set(),
+ ).add(tool_use_id)
+ if runs_in_background:
+ return
+ self._provisional_foreground_tool_use_ids.setdefault(
+ composite_key,
+ set(),
+ ).add(tool_use_id)
+ return
if runs_in_background:
return
if detached:
@@ -3027,33 +4082,461 @@ class ClaudeAgent(BaseAgent):
)
self._turns_with_foreground_tools.add(composite_key)
- def _clear_detached_foreground_tool_state(self, composite_key: str) -> None:
- self._detached_foreground_tool_use_ids.pop(composite_key, None)
- self._detached_foreground_task_ids.pop(composite_key, None)
+ def _clear_detached_foreground_tool_state(self, composite_key: str) -> None:
+ self._detached_foreground_tool_use_ids.pop(composite_key, None)
+ self._detached_foreground_task_ids.pop(composite_key, None)
+
+ def _ensure_provisional_phase(self, composite_key: str) -> str:
+ phase_id = self._provisional_phase_ids.get(composite_key)
+ if phase_id is None:
+ phase_id = f"{composite_key}:provisional:{uuid.uuid4().hex}"
+ self._provisional_phase_ids[composite_key] = phase_id
+ return phase_id
+
+ def _provisional_phase_tool_ids(self, composite_key: str) -> set[str]:
+ tool_ids = set(
+ self._provisional_tool_use_ids.get(composite_key) or ()
+ )
+ for message, _generation in self._buffered_assistant_messages.get(
+ composite_key,
+ (),
+ ):
+ for block in getattr(message, "content", []) or []:
+ if not isinstance(block, ToolUseBlock):
+ continue
+ tool_id = str(getattr(block, "id", "") or "").strip()
+ if tool_id:
+ tool_ids.add(tool_id)
+ return tool_ids
+
+ def _resolve_provisional_phase(
+ self,
+ composite_key: str,
+ context: MessageContext,
+ *,
+ owner: str,
+ terminal_message=None,
+ ) -> None:
+ """Classify buffered tool/task facts before terminal output can await."""
+
+ provisional_activity_ids = set(
+ self._provisional_activity_ids.get(composite_key) or ()
+ )
+ unbound_activity_ids = set(
+ self._provisional_unbound_activity_ids.get(composite_key) or ()
+ )
+ provisional_tool_ids = self._provisional_phase_tool_ids(composite_key)
+ unbound_tool_ids = set(
+ self._provisional_unbound_tool_use_ids.get(composite_key) or ()
+ )
+ provisional_foreground_tool_ids = set(
+ self._provisional_foreground_tool_use_ids.get(composite_key) or ()
+ )
+ phase_id = self._provisional_phase_ids.get(composite_key)
+ if owner not in {"human", "detached"}:
+ return
+ if phase_id:
+ self._resolved_provisional_phase_ids[composite_key] = phase_id
+
+ registry = self._activity_registry()
+ pending = self._pending_requests.get(composite_key) or []
+ pending_request = pending[0] if pending else None
+ turn_id = self._current_turn_id(composite_key, context) if pending else None
+ run_ids = self._activity_run_ids(composite_key, context) if pending else []
+ source = (
+ getattr(pending_request, "context", None)
+ if pending_request is not None
+ else context
+ )
+ delivery_key = str(
+ (getattr(source, "platform_specific", None) or {}).get(
+ "delivery_key_external"
+ )
+ or ""
+ ).strip()
+
+ if owner == "human":
+ if provisional_foreground_tool_ids:
+ self._foreground_tool_use_ids.setdefault(
+ composite_key,
+ set(),
+ ).update(provisional_foreground_tool_ids)
+ self._turns_with_foreground_tools.add(composite_key)
+ activity_ids = provisional_activity_ids - unbound_activity_ids
+ detached = False
+ else:
+ detached_tool_ids = self._detached_foreground_tool_use_ids.setdefault(
+ composite_key,
+ set(),
+ )
+ detached_tool_ids.update(provisional_foreground_tool_ids)
+ activity_ids = provisional_activity_ids
+ detached = True
+
+ if registry is None or not activity_ids and not provisional_tool_ids:
+ return
+ classification_failed = False
+ try:
+ classified = registry.classify_provisional_provenance(
+ self.name,
+ composite_key,
+ activity_ids=activity_ids,
+ parent_activity_ids=(
+ provisional_tool_ids
+ if owner == "detached"
+ else provisional_tool_ids - unbound_tool_ids
+ ),
+ turn_id=turn_id,
+ run_ids=run_ids,
+ delivery_key_external=delivery_key,
+ phase_id=phase_id,
+ detached=detached,
+ )
+ except Exception as error:
+ # Terminal Result provenance is authoritative even when the
+ # registry's durable classification write is temporarily
+ # unavailable. Keep a bounded recovery record and let the shared
+ # Result path settle the request exactly once; the receiver must
+ # not skip this already-consumed terminal frame.
+ self._provenance_recovery_evidence[composite_key] = {
+ "owner": owner,
+ "phase_id": phase_id,
+ "activity_ids": tuple(sorted(activity_ids)),
+ "tool_use_ids": tuple(sorted(provisional_tool_ids)),
+ "error": f"{type(error).__name__}: {error}",
+ }
+ logger.warning(
+ "Claude provisional provenance classification needs recovery "
+ "for %s; continuing with terminal owner %s",
+ composite_key,
+ owner,
+ exc_info=True,
+ )
+ classification_failed = True
+ if not classification_failed and classified:
+ self._notify_classified_terminal_activities(classified)
+ if owner == "human":
+ # Keep uncorrelated task facts provisional for a later detached
+ # Result. Only a TaskStarted tied to a tool phase proves that the
+ # human terminal owns that Activity.
+ self._provisional_foreground_tool_use_ids.pop(composite_key, None)
+ self._provisional_foreground_task_ids.pop(composite_key, None)
+ if not unbound_activity_ids:
+ self._provisional_activity_ids.pop(composite_key, None)
+ self._provisional_unbound_activity_ids.pop(composite_key, None)
+ self._provisional_tool_use_ids.pop(composite_key, None)
+ self._provisional_unbound_tool_use_ids.pop(composite_key, None)
+ self._provisional_phase_ids.pop(composite_key, None)
+ self._resolved_provisional_phase_ids.pop(composite_key, None)
+ elif owner == "detached":
+ self._provisional_activity_ids.pop(composite_key, None)
+ self._provisional_unbound_activity_ids.pop(composite_key, None)
+ self._provisional_tool_use_ids.pop(composite_key, None)
+ self._provisional_unbound_tool_use_ids.pop(composite_key, None)
+ self._provisional_foreground_tool_use_ids.pop(composite_key, None)
+ self._provisional_foreground_task_ids.pop(composite_key, None)
+ self._provisional_phase_ids.pop(composite_key, None)
+ if classification_failed:
+ return
+ if classified and phase_id and owner == "human":
+ try:
+ claimed = registry.claim_completed_output_batch(
+ self.name,
+ composite_key,
+ metadata_match={
+ "provenance_phase_id": phase_id,
+ "provenance_human": True,
+ },
+ )
+ except Exception as error:
+ # The Result is already consumed and its origin is authoritative.
+ # Receipt binding is preparatory work: retain exact terminal
+ # evidence, leave the Activity claim retryable, and let the
+ # normal terminal path settle this Result exactly once.
+ self._provenance_recovery_evidence[composite_key] = {
+ "owner": owner,
+ "phase_id": phase_id,
+ "activity_ids": tuple(sorted(activity_ids)),
+ "tool_use_ids": tuple(sorted(provisional_tool_ids)),
+ "terminal": {
+ "subtype": str(
+ getattr(terminal_message, "subtype", "") or ""
+ ),
+ "result": str(
+ getattr(terminal_message, "result", "") or ""
+ ),
+ },
+ "error": f"{type(error).__name__}: {error}",
+ }
+ logger.warning(
+ "Claude provisional receipt binding needs recovery for %s; "
+ "continuing with terminal owner %s",
+ composite_key,
+ owner,
+ exc_info=True,
+ )
+ return
+ if pending_request is not None and claimed:
+ self._attach_request_activities(pending_request, claimed)
+ self._refresh_activity_provenance_barrier(composite_key)
+
+ def _select_terminal_text(
+ self,
+ composite_key: str,
+ sdk_result: str | None,
+ ) -> str:
+ """Select terminal content from assistant TextBlocks, never tool labels."""
+
+ assistant_text = str(
+ self._last_assistant_text.get(composite_key) or ""
+ ).strip()
+ if assistant_text:
+ return assistant_text
+ if composite_key in self._turns_with_foreground_tools:
+ return "<silent>Claude turn completed without assistant text.</silent>"
+ return str(sdk_result or "")
+
+ @staticmethod
+ def _result_origin_kind(message) -> str | None:
+ origin = getattr(message, "origin", None)
+ if isinstance(origin, dict):
+ kind = origin.get("kind")
+ else:
+ kind = getattr(origin, "kind", None)
+ normalized = str(kind or "").strip().lower()
+ return normalized or None
+
+ def _has_competing_activity(self, composite_key: str) -> bool:
+ if (
+ self._output_records_for_runtime(composite_key)
+ or composite_key in self._activity_provenance_barriers
+ ):
+ return True
+ registry = self._activity_registry()
+ pending = self._pending_requests.get(composite_key) or []
+ pending_context = getattr(pending[0], "context", None) if pending else None
+ current_turn_id = (
+ self._current_turn_id(composite_key, pending_context)
+ if pending
+ else None
+ )
+ has_competing_output = (
+ getattr(registry, "has_competing_output", None)
+ if registry is not None
+ else None
+ )
+ return bool(
+ callable(has_competing_output)
+ and has_competing_output(
+ self.name,
+ composite_key,
+ current_turn_id=current_turn_id,
+ )
+ )
+
+ def _refresh_activity_provenance_barrier(self, composite_key: str) -> None:
+ """Track a pending human turn whose terminal owner is not classified."""
+
+ pending = self._pending_requests.get(composite_key) or []
+ if not pending:
+ self._activity_provenance_barriers.discard(composite_key)
+ self._activity_provenance_barrier_owners.pop(composite_key, None)
+ return
+ owner = id(pending[0])
+ previous_owner = self._activity_provenance_barrier_owners.get(composite_key)
+ if previous_owner != owner:
+ self._activity_provenance_barriers.discard(composite_key)
+ self._activity_provenance_barrier_owners.pop(composite_key, None)
+ if composite_key in self._activity_provenance_barriers:
+ return
+ if (
+ self._output_records_for_runtime(composite_key)
+ or self._has_competing_activity(composite_key)
+ ):
+ self._activity_provenance_barriers.add(composite_key)
+ self._activity_provenance_barrier_owners[composite_key] = owner
+
+ def _should_buffer_assistant_message(self, composite_key: str) -> bool:
+ """Hold an origin-less Assistant frame while Activity ownership is open."""
+
+ records = self._output_records_for_runtime(composite_key)
+ if records:
+ pending = self._pending_requests.get(composite_key) or []
+ human_pending = bool(pending and not getattr(pending[0], "_claude_synthetic_owner", False))
+ return human_pending or any(record.lifecycle == "delivery_pending" for record in records)
+ return (
+ composite_key in self._activity_provenance_barriers
+ or self._has_competing_activity(composite_key)
+ )
+
+ def _result_owner(self, composite_key: str, message) -> str:
+ """Classify a terminal frame before it can claim shared state."""
+
+ origin_kind = self._result_origin_kind(message)
+ if origin_kind == "human":
+ return "human"
+ if origin_kind:
+ return "detached"
+ if self._pending_requests.get(composite_key):
+ return "detached" if self._has_competing_activity(composite_key) else "human"
+ return "agent"
+
+ def _stop_owned_result_for_receiver(
+ self,
+ composite_key: str,
+ receiver_stop_epoch: int,
+ ) -> bool:
+ """Whether this receiver predates a Stop that already settled its turn."""
- def _select_terminal_text(
+ stop_epoch = self._stop_owned_terminal_epochs.get(composite_key)
+ return (
+ (
+ stop_epoch is not None
+ and stop_epoch > receiver_stop_epoch
+ )
+ or (
+ composite_key in self._stop_owned_terminal_keys
+ and composite_key in self._steering_closing_keys()
+ )
+ )
+
+ async def _flush_buffered_assistant_messages(
self,
composite_key: str,
- sdk_result: str | None,
- ) -> str:
- """Select terminal content from assistant TextBlocks, never tool labels."""
+ context: MessageContext,
+ *,
+ owner: str,
+ terminal_steering_generation: int,
+ ) -> bool:
+ """Replay ambiguous Assistant frames after terminal provenance is known."""
- assistant_text = str(
- self._last_assistant_text.get(composite_key) or ""
- ).strip()
- if assistant_text:
- return assistant_text
- if composite_key in self._turns_with_foreground_tools:
- return "<silent>Claude turn completed without assistant text.</silent>"
- return str(sdk_result or "")
+ messages = self._buffered_assistant_messages.pop(composite_key, None)
+ if not messages:
+ return False
+
+ detached = owner == "detached"
+ try:
+ formatter = self._get_formatter(context)
+ except Exception: # noqa: BLE001 - terminal settlement must survive replay formatting
+ formatter = None
+ logger.warning(
+ "Failed to create Claude formatter while replaying buffered "
+ "Assistant frames for %s",
+ composite_key,
+ exc_info=True,
+ )
+ for message, frame_steering_generation in messages:
+ try:
+ assistant_text = self._extract_text_blocks(message, context)
+ context_tokens = self._extract_context_tokens(message)
+ if context_tokens:
+ self.controller.note_session_tokens(context, total=context_tokens)
+ toolcalls = []
+ for block in getattr(message, "content", []) or []:
+ if not isinstance(block, ToolUseBlock):
+ continue
+ self._track_tool_activity_mode(
+ composite_key,
+ block,
+ detached=detached,
+ )
+ if formatter is not None:
+ toolcalls.append(
+ (
+ formatter.format_toolcall(
+ block.name,
+ block.input,
+ get_relative_path=lambda path: self.get_relative_path(path, context),
+ ),
+ formatter.format_toolcall_label(
+ block.name,
+ block.input,
+ get_relative_path=lambda path: self.get_relative_path(path, context),
+ ),
+ )
+ )
+
+ if detached:
+ if assistant_text:
+ record = self._provisional_output_record(composite_key, "activity")
+ if record is None:
+ record = self._detached_phase(composite_key)
+ record.text = assistant_text
+ continue
+
+ failure_disposition = await self._process_assistant_terminal_frame(
+ context,
+ composite_key,
+ message,
+ assistant_text,
+ terminal_steering_generation=(
+ frame_steering_generation
+ if frame_steering_generation is not None
+ else terminal_steering_generation
+ ),
+ )
+ if failure_disposition in {"teardown", "superseded", "auth", "failure"}:
+ if failure_disposition == "auth":
+ return True
+ continue
+ if assistant_text:
+ self._last_assistant_text[composite_key] = assistant_text
+ pending = self._pending_requests.get(composite_key) or []
+ self._adopt_pending_turn_token(
+ context,
+ pending[0] if pending else None,
+ )
+ previous = self._pending_assistant_message.pop(composite_key, None)
+ if previous:
+ await self.controller.emit_agent_message(
+ context,
+ "assistant",
+ previous,
+ parse_mode="markdown",
+ )
+ for toolcall_text, toolcall_label in toolcalls:
+ await self.controller.emit_agent_message(
+ context,
+ "toolcall",
+ toolcall_text,
+ parse_mode="markdown",
+ status_label=toolcall_label,
+ )
+ if formatter is not None and assistant_text:
+ self._pending_assistant_message[composite_key] = (
+ formatter.format_assistant_message([assistant_text])
+ )
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ # Assistant replay is ancillary to the terminal Result. Keep
+ # the Result on the receiver so its provenance can still settle
+ # the owning human/Activity phase.
+ logger.warning(
+ "Failed to replay buffered Claude Assistant frame for %s; "
+ "continuing to terminal Result",
+ composite_key,
+ exc_info=True,
+ )
+ return False
def _clear_result_phase_state(self, composite_key: str) -> None:
"""Retire response-local evidence before Claude consumes another input."""
self._last_assistant_text.pop(composite_key, None)
self._pending_assistant_message.pop(composite_key, None)
+ self._buffered_assistant_messages.pop(composite_key, None)
+ self._provisional_tool_use_ids.pop(composite_key, None)
+ self._provisional_unbound_tool_use_ids.pop(composite_key, None)
+ self._provisional_foreground_tool_use_ids.pop(composite_key, None)
+ self._provisional_activity_ids.pop(composite_key, None)
+ self._provisional_unbound_activity_ids.pop(composite_key, None)
+ self._provisional_foreground_task_ids.pop(composite_key, None)
+ self._provisional_phase_ids.pop(composite_key, None)
+ self._resolved_provisional_phase_ids.pop(composite_key, None)
self._foreground_tool_use_ids.pop(composite_key, None)
self._turns_with_foreground_tools.discard(composite_key)
+ self._refresh_activity_provenance_barrier(composite_key)
async def _emit_primary_phase_output(
self,
@@ -3176,10 +4659,22 @@ class ClaudeAgent(BaseAgent):
return sdk_result or assistant_text
if assistant_text:
return assistant_text
- if composite_key in self._detached_foreground_tool_use_ids:
+ if self._detached_foreground_tool_use_ids.get(composite_key):
return "<silent>Claude turn completed without assistant text.</silent>"
return sdk_result
+ def _retain_detached_unsolicited_assistant_text(
+ self,
+ composite_key: str,
+ text: str,
+ ) -> None:
+ """Keep later detached phases separate from a retained payload."""
+
+ normalized = str(text or "").strip()
+ if not normalized:
+ return
+ self._detached_phase(composite_key).text = normalized
+
def _current_turn_id(
self,
composite_key: str,
@@ -3262,10 +4757,76 @@ class ClaudeAgent(BaseAgent):
)
return True
foreground_tool_ids = self._foreground_tool_use_ids.get(composite_key) or set()
+ provisional_foreground_tool_ids = (
+ getattr(self, "_provisional_foreground_tool_use_ids", {}).get(composite_key)
+ or set()
+ )
foreground = bool(
(existing_activity is not None and existing_activity.foreground)
or (tool_use_id and tool_use_id in foreground_tool_ids)
+ or (tool_use_id and tool_use_id in provisional_foreground_tool_ids)
+ )
+ pending = self._pending_requests.get(composite_key) or []
+ # A pending human request is not enough to prove that this Activity
+ # belongs to it. Keep all newly-created Activities provisional until
+ # Result.origin classifies the phase, even when a buffered foreground
+ # ToolUseBlock already supplied an operational foreground hint.
+ provenance_pending = bool(
+ (pending and existing_activity is None)
+ or (
+ existing_activity is not None
+ and existing_activity.metadata.get("provenance_pending")
+ )
)
+ phase_id = None
+ if provenance_pending:
+ self._activity_provenance_barriers.add(composite_key)
+ if pending:
+ self._activity_provenance_barrier_owners[composite_key] = id(pending[0])
+ prior_phase = (
+ str(existing_activity.metadata.get("provenance_phase_id") or "").strip()
+ if existing_activity is not None else ""
+ )
+ phase_id = prior_phase or self._ensure_provisional_phase(composite_key)
+ current_phase = self._provisional_phase_ids.get(composite_key) == phase_id
+ # A task may outlive the human request whose phase first observed
+ # it. Its terminal event keeps that immutable phase and must not
+ # become positive correlation for the current/newer human request.
+ if current_phase:
+ self._provisional_activity_ids.setdefault(
+ composite_key,
+ set(),
+ ).add(task_id)
+ # A background tool is still positive phase evidence. The
+ # ``run_in_background`` flag describes execution mode, not whether
+ # the resulting Activity belongs to the human Result or a detached
+ # task-notification Result. Keep foreground-only tracking separate
+ # for output selection, but use every provisionally observed tool
+ # ID when correlating TaskStarted to its Assistant phase.
+ known_tool_ids = (
+ foreground_tool_ids
+ | provisional_foreground_tool_ids
+ | (self._provisional_tool_use_ids.get(composite_key) or set())
+ )
+ if current_phase and (not tool_use_id or tool_use_id not in known_tool_ids):
+ self._provisional_unbound_activity_ids.setdefault(
+ composite_key,
+ set(),
+ ).add(task_id)
+ if tool_use_id:
+ self._provisional_unbound_tool_use_ids.setdefault(
+ composite_key,
+ set(),
+ ).add(tool_use_id)
+ if current_phase and foreground:
+ self._provisional_foreground_task_ids.setdefault(
+ composite_key,
+ set(),
+ ).add(task_id)
+ elif existing_activity is not None:
+ phase_id = str(
+ existing_activity.metadata.get("provenance_phase_id") or ""
+ ).strip() or None
# The SDK task ``summary`` is a CLI-generated receipt such as
# 'Background command "..." completed (exit code 0)'. It is not
# assistant output, so it never becomes the Activity's visible
@@ -3279,19 +4840,29 @@ class ClaudeAgent(BaseAgent):
}.items()
if value not in (None, "")
}
+ if provenance_pending:
+ metadata["provenance_pending"] = True
+ if phase_id:
+ metadata["provenance_phase_id"] = phase_id
if existing_activity is None:
- pending = self._pending_requests.get(composite_key) or []
- source = getattr(pending[0], "context", None) if pending else context
- delivery_key = str(
- (getattr(source, "platform_specific", None) or {}).get(
- "delivery_key_external"
- )
- or ""
- ).strip()
- if delivery_key:
- metadata["delivery_key_external"] = delivery_key
- run_ids = self._activity_run_ids(composite_key, context)
- turn_id = self._current_turn_id(composite_key, context)
+ if provenance_pending:
+ # Activity ownership is provisional until Result.origin
+ # classifies this phase. Never inherit a newer human request's
+ # Harness identity while the task's owner is ambiguous.
+ run_ids = []
+ turn_id = None
+ else:
+ source = getattr(pending[0], "context", None) if pending else context
+ delivery_key = str(
+ (getattr(source, "platform_specific", None) or {}).get(
+ "delivery_key_external"
+ )
+ or ""
+ ).strip()
+ if delivery_key:
+ metadata["delivery_key_external"] = delivery_key
+ run_ids = self._activity_run_ids(composite_key, context)
+ turn_id = self._current_turn_id(composite_key, context)
else:
run_ids = []
if existing_activity.run_id:
@@ -3351,7 +4922,12 @@ class ClaudeAgent(BaseAgent):
logger.warning("Ignoring Claude %s with non-terminal status %r", event, status)
return True
- if existing_activity is None:
+ if existing_activity is None or (
+ existing_activity is not None
+ and foreground
+ and not existing_activity.foreground
+ and bool(pending)
+ ):
registry.start(
backend=self.name,
runtime_key=composite_key,
@@ -3372,8 +4948,19 @@ class ClaudeAgent(BaseAgent):
activity_id=task_id,
status=status,
metadata=metadata,
- expects_output=status == "completed" and not foreground,
- retain_terminal_snapshot=status != "completed" and not foreground,
+ expects_output=(
+ status == "completed"
+ and not foreground
+ ),
+ retain_terminal_snapshot=(
+ status != "completed"
+ and (not foreground or bool(metadata.get("provenance_pending")))
+ )
+ or (
+ status == "completed"
+ and foreground
+ and bool(metadata.get("provenance_pending"))
+ ),
activation_identity=activation_identity,
)
if completed is not None and status == "completed" and not foreground:
@@ -3383,7 +4970,12 @@ class ClaudeAgent(BaseAgent):
notify(RuntimeWorkLane.ACTIVITY_OUTPUTS)
service = getattr(self.controller, "agent_service", None)
on_terminal = getattr(service, "on_activity_terminal", None)
- if completed is not None and not foreground and callable(on_terminal):
+ if (
+ completed is not None
+ and not foreground
+ and callable(on_terminal)
+ and not bool((completed.metadata or {}).get("provenance_pending"))
+ ):
on_terminal(completed)
if status != "completed" or foreground:
self._mark_session_idle_if_runtime_free(composite_key)
@@ -3448,20 +5040,22 @@ class ClaudeAgent(BaseAgent):
duration_ms: int = 0,
request: AgentRequest | None = None,
activities: list[SessionActivity] | None = None,
+ output: MessageOutput | None = None,
) -> None:
"""Apply the Activity's explicit visible-or-silent output policy."""
request_output = getattr(request, "output", None) if request is not None else None
- output = (
- request_output
- if isinstance(request_output, MessageOutput)
- else self._activity_message_output(
- activity,
- activities=activities,
- detached=detached,
- completes_turn=completes_turn,
+ if output is None:
+ output = (
+ request_output
+ if isinstance(request_output, MessageOutput)
+ else self._activity_message_output(
+ activity,
+ activities=activities,
+ detached=detached,
+ completes_turn=completes_turn,
+ )
)
- )
visible_text = str(text or "")
if not strip_silent_blocks(visible_text).strip():
await self.controller.emit_agent_message(
@@ -3484,7 +5078,11 @@ class ClaudeAgent(BaseAgent):
self._require_activity_delivery(activity, message_id)
@staticmethod
- def _unsolicited_message_output(message) -> MessageOutput:
+ def _unsolicited_message_output(
+ message,
+ *,
+ phase_id: str | None = None,
+ ) -> MessageOutput:
native_id = str(
getattr(message, "uuid", "")
or getattr(message, "message_id", "")
@@ -3509,94 +5107,167 @@ class ClaudeAgent(BaseAgent):
# Older SDK frames expose no stable identity. Allocate one once
# for this MessageOutput instead of deduplicating by visible text.
identity = uuid.uuid4().hex
+ if phase_id:
+ identity = f"{identity}:{phase_id}"
+ metadata = {"backend": "claude", "source": "agent_initiated"}
+ if phase_id:
+ metadata["provenance_phase_id"] = phase_id
return MessageOutput(
completes_turn=False,
completes_run=False,
detached=True,
idempotency_key=f"claude-unsolicited:{identity}",
causation_id=native_id or None,
- metadata={"backend": "claude", "source": "agent_initiated"},
+ metadata=metadata,
)
async def _flush_detached_unsolicited_output(
self,
composite_key: str,
context: MessageContext,
- ) -> None:
- if composite_key not in self._detached_unsolicited_outputs:
- return
- text = self._detached_unsolicited_text.pop(composite_key, "").strip()
- self._detached_unsolicited_outputs.discard(composite_key)
- if not text:
- return
- await self.emit_result_message(
- context,
- text,
- parse_mode="markdown",
- output=self._unsolicited_message_output(None),
- )
+ ) -> bool:
+ return await self._flush_output_recovery(composite_key, context)
async def _flush_detached_activity_output(
self,
composite_key: str,
context: MessageContext,
- ) -> None:
- activities = self._detached_activity_outputs.pop(composite_key, None)
- if not activities:
- return
- activity = activities[-1]
- text = self._detached_assistant_text.pop(composite_key, "").strip()
- if not text:
- text = str(activity.metadata.get("summary") or "").strip()
- try:
- await self._emit_activity_result(
- context,
- activity,
- text,
- detached=True,
- completes_turn=False,
- activities=activities,
+ ) -> bool:
+ # Grace/EOF may terminalize an already claimed Activity without a
+ # Result. This never consumes the pending human request.
+ record = self._provisional_output_record(composite_key, "activity")
+ if record is not None and record.activities:
+ self._classify_output_record(
+ record, context,
+ text=record.text or record.activities[-1].metadata.get("summary", ""),
)
- except ActivityOutputDeliveryError as err:
- if not err.delivered:
- registry = self._activity_registry()
- if registry is not None:
- self._requeue_activities(registry, activities)
+ return await self._flush_output_recovery(composite_key, context)
+
+ async def _flush_output_recovery(
+ self, composite_key: str, context: MessageContext,
+ ) -> bool:
+ """Deliver frozen ledger records, independent of live client generation."""
+ records = [
+ record for record in self._output_records_for_runtime(composite_key)
+ if record.lifecycle == "delivery_pending"
+ ]
+ if not records:
+ return False
+ for record in records:
+ if record.delivering:
+ # The receiver and managed grace worker share this owner. No
+ # second send may enter while the same receipt is in flight.
+ return True
+ record.delivering = True
+ try:
+ if record.claim_pending:
+ registry = self._activity_registry()
+ if registry is None:
+ raise RuntimeError("Claude Activity recovery lost its registry")
+ if not record.activities:
+ if registry.has_claimed_output(self.name, composite_key):
+ self._schedule_completed_activity_flush(composite_key, context)
+ return True
+ # Select once using the existing FIFO/receipt algorithm.
+ # Binding is a separate fallible step: this record owns
+ # the exact members and payload before attempting it.
+ record.activities = registry.claim_completed_output_batch(
+ self.name, composite_key,
+ metadata_match=record.claim_metadata or {"provenance_pending": True},
+ bind_receipt=False,
+ )
+ if not record.activities and record.claim_metadata is None:
+ record.activities = registry.claim_completed_output_batch(
+ self.name, composite_key, bind_receipt=False,
+ )
+ if record.activities:
+ if record.output.activity_batch_id is None:
+ receipt_id = str(
+ record.activities[0].metadata.get("output_batch_id")
+ or record.phase_id
+ )
+ receipt_members = [
+ replace(activity, metadata={
+ **activity.metadata, "output_batch_id": receipt_id,
+ })
+ for activity in record.activities
+ ]
+ record.output = self._activity_message_output(
+ receipt_members[-1], activities=receipt_members,
+ detached=True, completes_turn=False,
+ )
+ record.activities = registry.bind_completed_output_batch(
+ record.activities, batch_id=record.output.activity_batch_id,
+ )
+ record.output = self._activity_message_output(
+ record.activities[-1], activities=record.activities,
+ detached=True, completes_turn=False,
+ )
+ record.owner = "activity"
+ record.claim_pending = False
+ if record.activities:
+ output = record.output
+ if record.local_settlement_only:
+ output = replace(
+ output,
+ metadata={**output.metadata, "activity_local_settlement_only": True},
+ )
+ await self._emit_activity_result(
+ record.context or context,
+ record.activities[-1], record.text,
+ detached=True, completes_turn=False,
+ subtype=record.subtype, duration_ms=record.duration_ms,
+ activities=record.activities, output=output,
+ )
+ elif record.text:
+ message_id = await self.emit_result_message(
+ record.context or context, record.text,
+ subtype=record.subtype, duration_ms=record.duration_ms,
+ parse_mode="markdown", output=record.output,
+ )
+ if message_id is None:
+ raise ActivityOutputDeliveryError(
+ "Claude detached output was not accepted", delivered=False,
+ )
+ except asyncio.CancelledError:
+ self._schedule_completed_activity_flush(composite_key, context)
raise
- if err.durable:
- registry = self._activity_registry()
- if registry is not None:
- self._requeue_activities(registry, activities)
- self._schedule_completed_activity_flush(
- composite_key,
- context,
- )
- return
- logger.error(
- "Delivered Claude Activity batch remains fail-closed after incomplete "
- "local settlement (runtime=%s error=%s)",
- composite_key,
- err,
- )
- registry = self._activity_registry()
- if (
- registry is not None
- and self._requeue_activities_for_local_settlement(
- registry,
- activities,
+ except ActivityOutputDeliveryError as error:
+ if error.delivered and not error.durable:
+ # Positive transport evidence forbids another external send,
+ # even when every durable write failed. The existing
+ # dispatcher/Registry local-settlement path retains this
+ # exact claim; only the delivery policy changes.
+ record.local_settlement_only = True
+ self._schedule_completed_activity_flush(composite_key, context)
+ logger.warning(
+ "Claude Activity receipt recovery remains pending (phase=%s delivered=%s)",
+ record.phase_id, error.delivered, exc_info=True,
)
- ):
- self._schedule_completed_activity_flush(
- composite_key,
- context,
+ return True
+ except Exception:
+ self._schedule_completed_activity_flush(composite_key, context)
+ logger.warning(
+ "Claude output recovery remains pending (phase=%s)",
+ record.phase_id, exc_info=True,
)
- except Exception:
- registry = self._activity_registry()
- if registry is not None:
- self._requeue_activities(registry, activities)
- raise
+ return True
+ finally:
+ record.delivering = False
+ self._remove_output_record(record)
+ self._retire_synthetic_pending_owner(composite_key, context, owner=record.request)
self._mark_session_idle_if_runtime_free(composite_key)
self._signal_activity_output_settled(composite_key)
+ # Another Result can append a record while this batch is in flight.
+ # The active managed worker must retain responsibility for that record
+ # rather than exiting with a stale, already-drained list snapshot.
+ retry = any(
+ record.lifecycle == "delivery_pending"
+ for record in self._output_records_for_runtime(composite_key)
+ )
+ if retry:
+ self._schedule_completed_activity_flush(composite_key, context)
+ return retry
async def _flush_completed_activity_outputs(
self,
@@ -3605,6 +5276,8 @@ class ClaudeAgent(BaseAgent):
*,
expected_steering_generation: int | None = None,
allow_closing: bool = False,
+ expected_client=None,
+ expected_activation_identity: RuntimeActivationIdentity | None = None,
) -> bool:
"""Deliver task notifications that ended the receiver without a Result."""
@@ -3619,6 +5292,8 @@ class ClaudeAgent(BaseAgent):
context,
expected_steering_generation=expected_steering_generation,
allow_closing=allow_closing,
+ expected_client=expected_client,
+ expected_activation_identity=expected_activation_identity,
),
)
return await self._flush_completed_activity_outputs_owned(
@@ -3626,6 +5301,8 @@ class ClaudeAgent(BaseAgent):
context,
expected_steering_generation=expected_steering_generation,
allow_closing=allow_closing,
+ expected_client=expected_client,
+ expected_activation_identity=expected_activation_identity,
)
async def _flush_completed_activity_outputs_owned(
@@ -3635,25 +5312,92 @@ class ClaudeAgent(BaseAgent):
*,
expected_steering_generation: int | None = None,
allow_closing: bool = False,
+ expected_client=None,
+ expected_activation_identity: RuntimeActivationIdentity | None = None,
) -> bool:
"""Claim, emit, and settle while the exact runtime partition is held."""
registry = self._activity_registry()
if registry is None:
return False
+ if (
+ self._output_record(composite_key, None, owner="activity") is not None
+ or any(record.claim_pending for record in self._output_records_for_runtime(composite_key))
+ ):
+ # The detached Result already owns this exact claimed batch. Let
+ # _flush_detached_activity_output retry its retained payload instead
+ # of claiming a second batch or mixing receipt identities.
+ return False
if expected_steering_generation is None:
expected_steering_generation = self._steering_generation(composite_key)
while True:
async with self._steering_lock(composite_key):
+ if expected_client is not None and not self._receiver_generation_current(
+ composite_key, expected_client, expected_activation_identity,
+ ):
+ return False
if composite_key in self._steering_closing_keys() and not allow_closing:
logger.info(
"Ignoring Claude Activity terminal output superseded by steering for %s",
composite_key,
)
return False
+ if composite_key in self._ambiguous_interrupt_keys():
+ logger.info(
+ "Deferring Claude Activity terminal output while interrupt "
+ "ownership is ambiguous for %s",
+ composite_key,
+ )
+ return True
+ steering_state = self._pending_steering_input_state(composite_key)
+ if (
+ self._terminal_claim_superseded(
+ composite_key,
+ expected_steering_generation,
+ )
+ and steering_state != "unknown"
+ ):
+ logger.info(
+ "Ignoring Claude Activity terminal output superseded by steering for %s",
+ composite_key,
+ )
+ return False
pending = self._pending_requests.get(composite_key) or []
pending_request = pending[0] if pending else None
- if pending_request is not None:
+ synthetic_pending_owner = (
+ pending_request is not None
+ and self._synthetic_pending_owners.get(composite_key)
+ is pending_request
+ and bool(self._request_activities(pending_request))
+ )
+ if synthetic_pending_owner:
+ activities = self._request_activities(pending_request)
+ if not activities:
+ activities = registry.claim_completed_output_batch(
+ self.name,
+ composite_key,
+ )
+ if not activities:
+ return False
+ self._retain_activity_output_record(composite_key, activities)
+ elif pending_request is not None:
+ self._refresh_activity_provenance_barrier(composite_key)
+ if (
+ composite_key in self._activity_provenance_barriers
+ and not self._request_activities(pending_request)
+ ):
+ # A completed Activity is not evidence that the pending
+ # human request owns it. Keep the registry claim queued
+ # until a terminal Result classifies the phase; this
+ # barrier also covers the no-Assistant-frame case where
+ # the old grace timer could otherwise pop the request.
+ self._activity_provenance_barriers.add(composite_key)
+ logger.info(
+ "Deferring Claude Activity flush until terminal "
+ "provenance is classified for %s",
+ composite_key,
+ )
+ return True
if self._request_activities(pending_request):
# A bound claim set is one concrete outbound batch. Later
# completions, even from the same Turn, wait for the next
@@ -3683,6 +5427,11 @@ class ClaudeAgent(BaseAgent):
matched_request = self._pop_pending_request(composite_key)
else:
matched_request = None
+ if synthetic_pending_owner:
+ await self._flush_detached_activity_output(composite_key, context)
+ return bool(
+ registry.has_completed_output(self.name, composite_key)
+ )
if matched_request is not None:
self._adopt_pending_turn_token(context, matched_request)
retained = self._request_activities(matched_request)
@@ -3727,6 +5476,7 @@ class ClaudeAgent(BaseAgent):
)
self._last_assistant_text.pop(composite_key, None)
self._pending_assistant_message.pop(composite_key, None)
+ self._buffered_assistant_messages.pop(composite_key, None)
self._mark_session_idle_if_no_pending_requests(composite_key)
self._signal_activity_output_settled(composite_key)
continue
@@ -3734,7 +5484,13 @@ class ClaudeAgent(BaseAgent):
activities = registry.claim_completed_output_batch(
self.name,
composite_key,
+ metadata_match={"provenance_pending": True},
)
+ if not activities:
+ activities = registry.claim_completed_output_batch(
+ self.name,
+ composite_key,
+ )
if not activities:
return False
activity = activities[-1]
@@ -3766,8 +5522,8 @@ class ClaudeAgent(BaseAgent):
except Exception:
self._requeue_activities(registry, activities)
raise
- self._mark_session_idle_if_runtime_free(composite_key)
- self._signal_activity_output_settled(composite_key)
+ self._mark_session_idle_if_runtime_free(composite_key)
+ self._signal_activity_output_settled(composite_key)
def _schedule_completed_activity_flush(
self,
@@ -3786,10 +5542,14 @@ class ClaudeAgent(BaseAgent):
retry = False
try:
await asyncio.sleep(self.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS)
- retry = await self._flush_completed_activity_outputs(
- composite_key,
- context,
- expected_steering_generation=expected_steering_generation,
+ retry = await self._flush_detached_activity_output(composite_key, context)
+ retry = (
+ await self._flush_completed_activity_outputs(
+ composite_key,
+ context,
+ expected_steering_generation=expected_steering_generation,
+ )
+ or retry
)
except asyncio.CancelledError:
raise
@@ -3822,6 +5582,7 @@ class ClaudeAgent(BaseAgent):
session_key: str,
*,
message_type: str | None = None,
+ result_owner: str = "agent",
receiver_activation_identity: RuntimeActivationIdentity | None = None,
) -> str | None:
"""Open a turn for UNSOLICITED backend output (no Avibe query behind it).
@@ -3842,10 +5603,36 @@ class ClaudeAgent(BaseAgent):
existing result path retains its normal lifecycle behavior.
"""
registry = self._activity_registry()
- detached_activities = self._detached_activity_outputs.get(composite_key)
- if detached_activities:
+ pending = self._pending_requests.get(composite_key) or []
+ defer_activity_claim = message_type == "result" and (
+ result_owner == "detached" or result_owner == "human" and not pending
+ )
+ if message_type == "result" and result_owner == "human":
+ # Result.origin is the terminal owner. Preserve any detached state
+ # for a later task-notification Result instead of letting it
+ # intercept this human settlement.
+ if pending:
+ return None
+
+ if self._provisional_output_record(composite_key, "activity") is not None:
return "activity"
- if composite_key in self._detached_unsolicited_outputs:
+ if self._output_records_for_runtime(composite_key):
+ self._detached_phase(composite_key, claim_activity=defer_activity_claim)
+ return "detached"
+ synthetic_owner = self._synthetic_pending_owners.get(composite_key)
+ if (
+ message_type == "result"
+ and result_owner == "detached"
+ and synthetic_owner is not None
+ ):
+ owned_activities = self._request_activities(synthetic_owner)
+ if owned_activities:
+ # Reuse the receipt batch already claimed by the synthetic
+ # request. A second registry claim is intentionally prohibited
+ # and would strand the original Activity receipt.
+ self._retain_activity_output_record(composite_key, owned_activities)
+ return "activity"
+ self._detached_phase(composite_key, claim_activity=True)
return "detached"
# A structured AssistantMessage failure pops the turn's real pending
# request and arms ``_suppressed_synthetic_results`` for its paired
@@ -3854,7 +5641,15 @@ class ClaudeAgent(BaseAgent):
# when the paired result is consumed or the receiver exits.
if composite_key in self._suppressed_synthetic_results:
return None
- pending = self._pending_requests.get(composite_key) or []
+ if message_type == "result" and result_owner == "human":
+ # No pending human request remains, so this is an unsolicited
+ # frame despite the SDK's human origin marker. It still needs the
+ # same synthetic owner as any other unsolicited terminal frame so
+ # the runtime gate stays held until durable delivery.
+ result_owner = "detached"
+ if message_type == "result" and result_owner == "detached" and pending:
+ self._detached_phase(composite_key, claim_activity=True)
+ return "detached"
if pending:
pending_request = pending[0]
retained = self._request_activities(pending_request)
@@ -3884,7 +5679,7 @@ class ClaudeAgent(BaseAgent):
)
if not activities:
return None
- self._detached_activity_outputs[composite_key] = activities
+ self._retain_activity_output_record(composite_key, activities)
logger.info(
"Claude Activity batch %s output detached from the current user turn in %s",
",".join(activity.id for activity in activities),
@@ -3894,7 +5689,7 @@ class ClaudeAgent(BaseAgent):
completed_activities = (
registry.claim_completed_output_batch(self.name, composite_key)
- if registry is not None
+ if registry is not None and not defer_activity_claim
else []
)
service = getattr(self.controller, "agent_service", None)
@@ -3923,10 +5718,13 @@ class ClaudeAgent(BaseAgent):
composite_key,
)
if completed_activities:
- self._detached_activity_outputs[composite_key] = completed_activities
+ self._retain_activity_output_record(
+ composite_key,
+ completed_activities,
+ )
return "activity"
if message_type in {"assistant", "result"}:
- self._detached_unsolicited_outputs.add(composite_key)
+ self._detached_phase(composite_key, claim_activity=defer_activity_claim)
return "detached"
return None
# Mark the session ACTIVE so this in-flight agent-initiated turn gets the
@@ -3944,7 +5742,7 @@ class ClaudeAgent(BaseAgent):
mark_active(composite_key)
latest_activity = completed_activities[-1] if completed_activities else None
request = AgentRequest(
- context=context,
+ context=self._snapshot_output_context(context),
message="",
user_message="",
working_path=working_path,
@@ -3962,11 +5760,22 @@ class ClaudeAgent(BaseAgent):
),
output_activities=completed_activities,
)
+ request._claude_synthetic_owner = True
self._pending_requests.setdefault(composite_key, []).append(request)
+ self._synthetic_pending_owners[composite_key] = request
logger.info(
"Opened agent-initiated turn for session %s (unsolicited backend output)",
composite_key,
)
+ if message_type == "result" and result_owner == "detached":
+ if completed_activities:
+ self._retain_activity_output_record(
+ composite_key,
+ completed_activities,
+ )
+ return "activity"
+ self._detached_phase(composite_key, claim_activity=defer_activity_claim)
+ return "detached"
return None
def _mark_session_idle_if_no_pending_requests(self, composite_key: str) -> bool:
@@ -4038,10 +5847,20 @@ class ClaudeAgent(BaseAgent):
logger.debug(f"Failed to remove reaction ack: {err}")
return
- async def _clear_pending_reactions(self, composite_key: str, context: MessageContext) -> None:
+ async def _clear_pending_reactions(
+ self,
+ composite_key: str,
+ context: MessageContext,
+ *,
+ preserve_pending_requests: bool = False,
+ ) -> None:
"""Clear all pending reactions for a session (for error cleanup)."""
reactions = self._pending_reactions.pop(composite_key, None)
- requests = self._pending_requests.pop(composite_key, None)
+ requests = (
+ None
+ if preserve_pending_requests
+ else self._pending_requests.pop(composite_key, None)
+ )
if reactions:
for message_id, emoji in reactions:
try:
@@ -4232,13 +6051,6 @@ class ClaudeAgent(BaseAgent):
output=terminal_output_for(pending_request),
terminal_error=text,
)
- if handled:
- await self._cleanup_runtime_session(
- composite_key,
- current_receiver_task=asyncio.current_task(),
- preserve_pending_request_state=True,
- reason="result_auth_failure",
- )
return handled
@staticmethod
diff --git a/modules/agents/service.py b/modules/agents/service.py
index 4d5745646..46a45ce06 100644
--- a/modules/agents/service.py
+++ b/modules/agents/service.py
@@ -209,14 +209,21 @@ class AgentService:
def on_activity_terminal(self, activity: Any) -> bool:
"""Let the Run owner acknowledge one terminal Activity."""
+ if (getattr(activity, "metadata", None) or {}).get("provenance_pending"):
+ # The native task can finish before the Result classifies its Run.
+ # No Run IDs is not proof that the terminal has no owner.
+ return False
service = getattr(self.controller, "scheduled_task_service", None)
settle = getattr(service, "settle_activity_runs", None)
if not callable(settle):
return False
try:
settle(activity)
- if str(getattr(activity, "status", "") or "") != "completed":
- self.activities.ack_recovered_terminal(activity)
+ # ``ack_recovered_terminal`` is a no-op for ordinary active or
+ # output-queue entries. Calling it for every classified terminal
+ # also retires a completed foreground Activity that had to remain a
+ # provenance snapshot until its Result arrived.
+ self.activities.ack_recovered_terminal(activity)
return True
except Exception:
logger.warning(
diff --git a/tests/test_agent_service.py b/tests/test_agent_service.py
index 9c2c68fa2..832d5c8e1 100644
--- a/tests/test_agent_service.py
+++ b/tests/test_agent_service.py
@@ -464,6 +464,40 @@ def test_activity_runtime_disconnect_retains_snapshot_until_run_settles(
engine.dispose()
+def test_activity_runtime_end_closes_unresolved_provenance_before_run_settlement():
+ settled = []
+ controller = SimpleNamespace(
+ scheduled_task_service=SimpleNamespace(
+ settle_activity_runs=settled.append,
+ ),
+ )
+ service = AgentService(controller)
+ service.activities.start(
+ backend="claude",
+ runtime_key="runtime-1",
+ session_id="session-1",
+ activity_id="task-failed",
+ kind="background_task",
+ metadata={"provenance_pending": True, "provenance_phase_id": "phase-1"},
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key="runtime-1",
+ activity_id="task-failed",
+ status="failed",
+ retain_terminal_snapshot=True,
+ )
+
+ completed = service.end_activity_runtime("claude", "runtime-1")
+
+ assert [(activity.id, activity.status) for activity in completed] == [
+ ("task-failed", "failed"),
+ ]
+ assert settled[0].metadata["provenance_generation_ended"] is True
+ assert settled[0].metadata["provenance_unresolved"] is True
+ assert service.activities.has_backend_work("claude") is False
+
+
def test_agent_service_serializes_same_runtime_until_terminal_release() -> None:
async def _run():
controller = _Controller()
diff --git a/tests/test_agent_steering.py b/tests/test_agent_steering.py
index ffef8dd1a..33b87007b 100644
--- a/tests/test_agent_steering.py
+++ b/tests/test_agent_steering.py
@@ -112,7 +112,11 @@ class _ClaudeClient:
self.queries = [("primary", "runtime-key")]
self._transport = SimpleNamespace(end_input=AsyncMock())
- async def query(self, text: str, *, session_id: str) -> None:
+ async def query(self, text, *, session_id: str) -> None:
+ if hasattr(text, "__aiter__"):
+ frames = [frame async for frame in text]
+ assert len(frames) == 1
+ text = frames[0]["message"]["content"]
self.queries.append((text, session_id))
if self.error is not None:
raise self.error
@@ -936,7 +940,11 @@ async def test_shared_boundary_finishes_native_reconciliation_before_propagating
release_query = asyncio.Event()
class _BlockingClaudeClient(_ClaudeClient):
- async def query(self, text: str, *, session_id: str) -> None:
+ async def query(self, text, *, session_id: str) -> None:
+ if hasattr(text, "__aiter__"):
+ frames = [frame async for frame in text]
+ assert len(frames) == 1
+ text = frames[0]["message"]["content"]
self.queries.append((text, session_id))
query_started.set()
await release_query.wait()
diff --git a/tests/test_agent_stop_settlement.py b/tests/test_agent_stop_settlement.py
index be6cc5b70..9858ef88a 100644
--- a/tests/test_agent_stop_settlement.py
+++ b/tests/test_agent_stop_settlement.py
@@ -129,6 +129,7 @@ def _claude_stop_case():
agent = object.__new__(ClaudeAgent)
agent.claude_sessions = {composite_key: SimpleNamespace(interrupt=AsyncMock())}
agent._pending_requests = {}
+ agent._synthetic_pending_owners = {}
agent._suppress_receiver_runtime_release = set()
agent._cleanup_runtime_session = AsyncMock()
agent._mark_session_idle_if_no_pending_requests = lambda _key: True
diff --git a/tests/test_claude_agent_initiated_turn.py b/tests/test_claude_agent_initiated_turn.py
index 9ca58817c..ed642afcc 100644
--- a/tests/test_claude_agent_initiated_turn.py
+++ b/tests/test_claude_agent_initiated_turn.py
@@ -59,6 +59,7 @@ class ResultMessage:
duration_ms = 1
duration_api_ms = 1
session_id = "claude-native-session"
+ origin = {"kind": "human"}
def __init__(self, *, num_turns: int = 1):
self.num_turns = num_turns
@@ -135,7 +136,9 @@ def _task_notification_then_result_client():
def receive_messages(self):
async def _iterate():
yield TaskNotificationMessage()
- yield ResultMessage()
+ result = ResultMessage()
+ result.origin = {"kind": "task-notification"}
+ yield result
return _iterate()
@@ -167,6 +170,11 @@ def _completed_task_notification_then_wait_client(release: asyncio.Event):
return _Client()
+async def _wait_until(predicate, *, interval: float = 0.01):
+ while not predicate():
+ await asyncio.sleep(interval)
+
+
def _foreground_bash_then_delayed_result_client(
notification_seen: asyncio.Event,
release_result: asyncio.Event,
@@ -181,6 +189,7 @@ def _foreground_bash_then_delayed_result_client(
subtype = "success"
result = "Push branch, confirm repo"
duration_ms = 1
+ origin = {"kind": "human"}
class _Client:
def receive_messages(self):
@@ -807,370 +816,1418 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
self.assertFalse(service.activities.has_completed_output("claude", composite_key))
self.assertFalse(gate.lock.locked())
- async def test_pending_activity_batch_scans_past_older_queue_head(self):
+ async def test_empty_synthetic_owner_does_not_claim_later_activity(self):
agent, service = _build_agent()
- composite_key = "session-interleaved-batch:/tmp/work"
+ composite_key = "session-synthetic-empty-owner:/tmp/work"
context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
platform_specific={
- "agent_session_id": "sess-interleaved-batch",
- "turn_token": "current-turn",
+ "agent_runtime_turn_key": composite_key,
+ "agent_session_id": "sess-synthetic-empty-owner",
},
)
- pending_request = SimpleNamespace(
- context=SimpleNamespace(
- platform_specific={"turn_token": "current-turn"},
- ),
- output_activities=[],
+
+ mode = await agent._maybe_begin_agent_initiated_turn(
+ context,
+ composite_key,
+ "sess-synthetic-empty-owner",
+ "/tmp/work",
+ "session-key",
+ message_type="assistant",
)
- agent._pending_requests[composite_key] = [pending_request]
+ self.assertIsNone(mode)
+ synthetic = agent._synthetic_pending_owners[composite_key]
+ self.assertEqual(synthetic.output_activities, [])
- for activity_id, turn_id in (
- ("task-detached", "older-turn"),
- ("task-current-a", "current-turn"),
- ("task-current-b", "current-turn"),
+ service.activities.start(
+ backend="claude",
+ runtime_key=composite_key,
+ session_id="sess-synthetic-empty-owner",
+ activity_id="task-later",
+ kind="local_bash",
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=composite_key,
+ activity_id="task-later",
+ status="completed",
+ metadata={"summary": "Later task finished"},
+ expects_output=True,
+ )
+ agent.emit_result_message = AsyncMock()
+
+ retry = await agent._flush_completed_activity_outputs(
+ composite_key,
+ context,
+ )
+
+ self.assertTrue(retry)
+ self.assertEqual(synthetic.output_activities, [])
+ self.assertTrue(
+ service.activities.has_completed_output("claude", composite_key)
+ )
+ self.assertFalse(agent.emit_result_message.await_count)
+
+ agent._retire_synthetic_pending_owner(composite_key, context, owner=synthetic)
+ self.assertFalse(agent._has_pending_requests(composite_key))
+ self.assertFalse(service._get_turn_gate(composite_key).lock.locked())
+
+ async def test_synthetic_owner_transfers_claimed_activity_batch_to_detached_result(self):
+ agent, service = _build_agent()
+ composite_key = "session-synthetic-claimed-batch:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_session_id": "sess-synthetic-claimed-batch",
+ },
+ )
+ for activity_id, summary in (
+ ("task-one", "First task finished"),
):
service.activities.start(
backend="claude",
runtime_key=composite_key,
- session_id="sess-interleaved-batch",
+ session_id="sess-synthetic-claimed-batch",
activity_id=activity_id,
- kind="local_bash",
- turn_id=turn_id,
+ kind="local_agent",
)
service.activities.complete(
backend="claude",
runtime_key=composite_key,
activity_id=activity_id,
status="completed",
+ metadata={"summary": summary},
expects_output=True,
)
- await agent._maybe_begin_agent_initiated_turn(
- context,
- composite_key,
- "sess-interleaved-batch",
- "/tmp/work",
- "session-key",
- message_type="result",
- )
+ fallback_seen = asyncio.Event()
+ release_result = asyncio.Event()
+ release_receiver = asyncio.Event()
- self.assertEqual(
- [activity.id for activity in pending_request.output_activities],
- ["task-current-a", "task-current-b"],
- )
- self.assertTrue(
- service.activities.settle_completed_output_batch(
- pending_request.output,
- accepted_message_exists=True,
- )
- )
- agent._clear_request_activities(pending_request)
- detached = service.activities.claim_completed_output("claude", composite_key)
- self.assertIsNotNone(detached)
- self.assertEqual(detached.id, "task-detached")
- service.activities.ack_completed_output(detached)
+ class ModelRefusalFallbackMessage:
+ subtype = "model_refusal_fallback"
+ data = {
+ "originalModel": "claude-fable-5",
+ "fallbackModel": "claude-opus-4-8",
+ "content": "Provider fallback",
+ }
- async def test_terminal_only_task_event_keeps_current_turn_origin(self):
- agent, service = _build_agent()
- composite_key = "session-terminal-only:/tmp/work"
- pending_context = SimpleNamespace(
- platform_specific={"turn_token": "current-turn"},
- )
- agent._pending_requests[composite_key] = [SimpleNamespace(context=pending_context)]
- context = SimpleNamespace(
- platform_specific={"agent_session_id": "sess-terminal-only"},
- )
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ yield ModelRefusalFallbackMessage()
+ fallback_seen.set()
+ await release_result.wait()
+ result = ResultMessage()
+ result.origin = {"kind": "task-notification"}
+ yield result
+ await release_receiver.wait()
- self.assertTrue(
- agent._handle_activity_message(
- TaskNotificationMessage(),
- composite_key,
+ return _iterate()
+
+ result_emitted = asyncio.Event()
+ dispatcher_emit = _dispatcher_owned_emit(service)
+
+ async def emit_result(*args, **kwargs):
+ result_emitted.set()
+ return await dispatcher_emit(*args, **kwargs)
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _Client(),
+ "sess-synthetic-claimed-batch",
+ "/tmp/work",
context,
+ composite_key=composite_key,
)
)
-
- activity = service.activities.claim_completed_output("claude", composite_key)
- self.assertIsNotNone(activity)
- self.assertEqual(activity.turn_id, "current-turn")
-
- async def test_failed_activity_snapshot_waits_for_run_owner_ack(self):
- agent, service = _build_agent()
- composite_key = "session-failed-activity:/tmp/work"
- context = SimpleNamespace(
- platform_specific={"agent_session_id": "sess-failed-activity"},
+ await asyncio.wait_for(fallback_seen.wait(), timeout=1)
+ synthetic = agent._synthetic_pending_owners[composite_key]
+ self.assertEqual(
+ [activity.id for activity in synthetic.output_activities],
+ ["task-one"],
)
+
service.activities.start(
backend="claude",
runtime_key=composite_key,
- session_id="sess-failed-activity",
- activity_id="task-failed",
+ session_id="sess-synthetic-claimed-batch",
+ activity_id="task-two",
kind="local_agent",
- run_id="run-failed",
- )
- agent.controller.scheduled_task_service = SimpleNamespace(
- settle_activity_runs=Mock(side_effect=RuntimeError("store unavailable")),
)
- original_complete = service.activities.complete
- service.activities.complete = Mock(wraps=original_complete)
- original_ack = service.activities.ack_recovered_terminal
- service.activities.ack_recovered_terminal = Mock(wraps=original_ack)
- failed = SimpleNamespace(
- subtype="task_notification",
- task_id="task-failed",
- status="failed",
- summary="Background verification failed",
- data={},
+ service.activities.complete(
+ backend="claude",
+ runtime_key=composite_key,
+ activity_id="task-two",
+ status="completed",
+ metadata={"summary": "Second task finished"},
+ expects_output=True,
)
- self.assertTrue(
- agent._handle_activity_message(
- failed,
- composite_key,
- context,
- )
+ release_result.set()
+ await asyncio.wait_for(result_emitted.wait(), timeout=1)
+ self.assertEqual(
+ agent.emit_result_message.await_args.kwargs["output"].activity_ids,
+ ("task-one",),
+ )
+ next_activity = service.activities.claim_completed_output(
+ "claude",
+ composite_key,
)
+ self.assertIsNotNone(next_activity)
+ self.assertEqual(next_activity.id, "task-two")
+ service.activities.requeue_completed_output(next_activity)
- call = service.activities.complete.call_args
- self.assertTrue(call.kwargs["retain_terminal_snapshot"])
- completed = agent.controller.scheduled_task_service.settle_activity_runs.call_args.args[0]
- self.assertEqual(completed.status, "failed")
- service.activities.ack_recovered_terminal.assert_not_called()
+ release_receiver.set()
+ await asyncio.wait_for(receiver, timeout=1)
- async def test_activity_keeps_origin_delivery_target_when_a_newer_turn_arrives(self):
+ async def test_synthetic_activity_delivery_retries_while_receiver_stays_open(self):
agent, service = _build_agent()
- composite_key = "session-delivery-origin:/tmp/work"
- origin_context = SimpleNamespace(
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0
+ composite_key = "session-synthetic-activity-retry:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
platform_specific={
- "turn_token": "origin-turn",
- "delivery_key_external": "slack::channel::C-ORIGIN",
+ "agent_runtime_turn_key": composite_key,
+ "agent_session_id": "sess-synthetic-activity-retry",
},
)
- agent._pending_requests[composite_key] = [SimpleNamespace(context=origin_context)]
- receiver_context = SimpleNamespace(
- platform_specific={"agent_session_id": "sess-delivery-origin"},
+ service.activities.start(
+ backend="claude",
+ runtime_key=composite_key,
+ session_id="sess-synthetic-activity-retry",
+ activity_id="task-one",
+ kind="local_agent",
)
-
- self.assertTrue(
- agent._handle_activity_message(
- TaskStartedMessage(),
- composite_key,
- receiver_context,
- )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=composite_key,
+ activity_id="task-one",
+ status="completed",
+ metadata={"summary": "First task finished"},
+ expects_output=True,
)
- agent._pending_requests[composite_key] = [
- SimpleNamespace(
- context=SimpleNamespace(
- platform_specific={
- "turn_token": "newer-turn",
- "delivery_key_external": "slack::channel::C-NEWER",
- },
+
+ fallback_seen = asyncio.Event()
+ result_seen = asyncio.Event()
+ retry_succeeded = asyncio.Event()
+ release_receiver = asyncio.Event()
+ output_attempts = []
+
+ class ModelRefusalFallbackMessage:
+ subtype = "model_refusal_fallback"
+ data = {
+ "originalModel": "claude-fable-5",
+ "fallbackModel": "claude-opus-4-8",
+ "content": "Provider fallback",
+ }
+
+ agent.controller.emit_agent_message = AsyncMock(return_value="message-id")
+
+ async def emit_result(_context, _text, **kwargs):
+ output = kwargs["output"]
+ output_attempts.append(output)
+ if len(output_attempts) == 1:
+ raise RuntimeError("temporary Activity delivery failure")
+ self.assertTrue(
+ service.activities.settle_completed_output_batch(
+ output,
+ accepted_message_exists=True,
)
)
- ]
+ retry_succeeded.set()
+ return "message-id"
- self.assertTrue(
- agent._handle_activity_message(
- TaskNotificationMessage(),
- composite_key,
- receiver_context,
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ yield ModelRefusalFallbackMessage()
+ fallback_seen.set()
+ result = ResultMessage()
+ result.origin = {"kind": "task-notification"}
+ yield result
+ result_seen.set()
+ await release_receiver.wait()
+
+ return _iterate()
+
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _Client(),
+ "sess-synthetic-activity-retry",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
)
)
+ await asyncio.wait_for(fallback_seen.wait(), timeout=1)
+ await asyncio.wait_for(result_seen.wait(), timeout=1)
+ await asyncio.wait_for(retry_succeeded.wait(), timeout=1)
+ await asyncio.wait_for(
+ _wait_until(
+ lambda: composite_key not in agent._synthetic_pending_owners,
+ ),
+ timeout=1,
+ )
- activity = service.activities.claim_completed_output("claude", composite_key)
- self.assertIsNotNone(activity)
+ self.assertEqual(len(output_attempts), 2)
self.assertEqual(
- activity.metadata["delivery_key_external"],
- "slack::channel::C-ORIGIN",
+ output_attempts[0].idempotency_key,
+ output_attempts[1].idempotency_key,
)
+ self.assertEqual(output_attempts[0].activity_ids, ("task-one",))
+ self.assertNotIn(composite_key, agent._synthetic_pending_owners)
+ self.assertFalse(agent._has_pending_requests(composite_key))
+ self.assertFalse(service.activities.has_completed_output("claude", composite_key))
+ self.assertFalse(service.activities.has_claimed_output("claude", composite_key))
+ self.assertFalse(service._get_turn_gate(composite_key).lock.locked())
- async def test_completed_task_notification_at_eof_settles_without_sdk_summary(self):
- agent, service = _build_agent()
- composite_key = "session-eof:/tmp/work"
+ release_receiver.set()
+ await asyncio.wait_for(receiver, timeout=1)
+
+ async def test_retained_detached_text_is_not_overwritten_by_later_assistant_phase(
+ self,
+ ):
+ agent, _service = _build_agent()
+ composite_key = "session-retained-detached-phase:/tmp/work"
context = SimpleNamespace(
user_id="U1",
channel_id="C1",
platform="avibe",
platform_specific={
"agent_runtime_turn_key": composite_key,
- "agent_runtime_turn_token": "old-turn",
- "turn_token": "origin-turn",
- "agent_session_id": "sess-eof",
+ "agent_session_id": "sess-retained-detached-phase",
},
)
- agent.emit_result_message = AsyncMock()
+ retained = agent._detached_phase(composite_key)
+ agent._classify_output_record(retained, context, text="retained payload")
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ assistant_seen = asyncio.Event()
+ release_result = asyncio.Event()
+
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ assistant = AssistantMessage()
+ assistant.content = [TextBlock(text="later phase")]
+ yield assistant
+ assistant_seen.set()
+ await release_result.wait()
+ result = ResultMessage()
+ result.origin = {"kind": "task-notification"}
+ yield result
+
+ return _iterate()
+
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _Client(),
+ "sess-retained-detached-phase",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
+ )
+ )
+ await asyncio.wait_for(assistant_seen.wait(), timeout=1)
+ self.assertEqual(
+ retained.text,
+ "retained payload",
+ )
+ self.assertEqual(
+ agent._extract_text_blocks(
+ agent._buffered_assistant_messages[composite_key][0][0], context,
+ ),
+ "later phase",
+ )
+ self.assertEqual(
+ len(agent._buffered_assistant_messages[composite_key]),
+ 1,
+ )
+ release_result.set()
+ await asyncio.wait_for(receiver, timeout=1)
+
+ self.assertEqual(agent.emit_result_message.await_count, 2)
+ self.assertEqual(
+ agent.emit_result_message.await_args_list[0].args[1],
+ "retained payload",
+ )
+ self.assertEqual(
+ agent.emit_result_message.await_args_list[1].args[1],
+ "later phase",
+ )
+ first_output = agent.emit_result_message.await_args_list[0].kwargs["output"]
+ later_output = agent.emit_result_message.await_args_list[1].kwargs["output"]
+ self.assertNotEqual(
+ first_output.idempotency_key,
+ later_output.idempotency_key,
+ )
+ self.assertNotEqual(
+ first_output.metadata["provenance_phase_id"],
+ later_output.metadata["provenance_phase_id"],
+ )
+ self.assertFalse(agent._output_records_for_runtime(composite_key))
+
+ async def test_later_failure_result_cannot_rewrite_delivery_pending_payload(self):
+ agent, service = _build_agent()
+ key = "session-late-failure:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1", channel_id="C1", platform="avibe",
+ platform_specific={"agent_runtime_turn_key": key},
+ )
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+ agent._get_formatter = lambda _ctx: SimpleNamespace(
+ format_assistant_message=lambda parts: "\n".join(parts),
+ )
+ terminal_seen = asyncio.Event()
+ release = asyncio.Event()
+ accept = asyncio.Event()
+ attempts = []
+
+ async def emit(_context, text, **kwargs):
+ attempts.append((text, kwargs["output"]))
+ if not accept.is_set():
+ raise RuntimeError("transport temporarily unavailable")
+ return "accepted"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit)
+
+ class Client:
+ def receive_messages(self):
+ async def frames():
+ assistant = AssistantMessage()
+ assistant.content = [TextBlock(text="original selected payload")]
+ yield assistant
+ result = ResultMessage()
+ result.origin = {"kind": "task-notification"}
+ result.result = "original selected payload"
+ yield result
+ later = ResultMessage(num_turns=2)
+ later.origin = {"kind": "task-notification"}
+ later.subtype = "error_during_execution"
+ later.result = "later failure"
+ yield later
+ terminal_seen.set()
+ await release.wait()
+ return frames()
+
+ receiver = asyncio.create_task(agent._receive_messages(
+ Client(), "late-failure", "/tmp/work", context, composite_key=key,
+ ))
+ try:
+ await asyncio.wait_for(terminal_seen.wait(), timeout=1)
+ original_id = attempts[0][1].idempotency_key
+ self.assertTrue(all(
+ text == "original selected payload"
+ for text, output in attempts if output.idempotency_key == original_id
+ ))
+ accept.set()
+ await asyncio.wait_for(_wait_until(
+ lambda: not agent._has_pending_requests(key)
+ ), timeout=1)
+ identities = {output.idempotency_key for _, output in attempts}
+ self.assertEqual(len(identities), 2)
+ self.assertTrue(all(
+ text == "original selected payload"
+ for text, output in attempts if output.idempotency_key == original_id
+ ))
+ self.assertFalse(service.runtime_turn_active(key))
+ finally:
+ release.set()
+ await asyncio.wait_for(receiver, timeout=1)
+
+ async def test_eof_retires_dead_client_while_detached_recovery_owner_survives(self):
+ agent, service = _build_agent()
+ composite_key = "session-dead-client-recovery:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_session_id": "sess-dead-client-recovery",
+ },
+ )
+ agent.emit_result_message = AsyncMock(
+ side_effect=RuntimeError("delivery unavailable"),
+ )
+ receiver_started = asyncio.Event()
+
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ assistant = AssistantMessage()
+ assistant.content = [TextBlock(text="detached payload")]
+ yield assistant
+ receiver_started.set()
+ result = ResultMessage()
+ result.origin = {"kind": "task-notification"}
+ yield result
+
+ return _iterate()
+
+ client = _Client()
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ client,
+ "sess-dead-client-recovery",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
+ )
+ )
+ agent.claude_sessions[composite_key] = client
+ agent.receiver_tasks[composite_key] = receiver
+ await asyncio.wait_for(receiver_started.wait(), timeout=1)
+ await asyncio.wait_for(receiver, timeout=1)
+
+ self.assertNotIn(composite_key, agent.claude_sessions)
+ self.assertNotIn(composite_key, agent.receiver_tasks)
+ self.assertTrue(agent._has_synthetic_delivery_pending(composite_key))
+ self.assertTrue(service.runtime_turn_active(composite_key))
+
+ recovery_task = agent._activity_flush_tasks.pop(composite_key, None)
+ if recovery_task is not None:
+ recovery_task.cancel()
+ await asyncio.gather(recovery_task, return_exceptions=True)
+
+ async def test_pending_activity_batch_scans_past_older_queue_head(self):
+ agent, service = _build_agent()
+ composite_key = "session-interleaved-batch:/tmp/work"
+ context = SimpleNamespace(
+ platform_specific={
+ "agent_session_id": "sess-interleaved-batch",
+ "turn_token": "current-turn",
+ },
+ )
+ pending_request = SimpleNamespace(
+ context=SimpleNamespace(
+ platform_specific={"turn_token": "current-turn"},
+ ),
+ output_activities=[],
+ )
+ agent._pending_requests[composite_key] = [pending_request]
+
+ for activity_id, turn_id in (
+ ("task-detached", "older-turn"),
+ ("task-current-a", "current-turn"),
+ ("task-current-b", "current-turn"),
+ ):
+ service.activities.start(
+ backend="claude",
+ runtime_key=composite_key,
+ session_id="sess-interleaved-batch",
+ activity_id=activity_id,
+ kind="local_bash",
+ turn_id=turn_id,
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=composite_key,
+ activity_id=activity_id,
+ status="completed",
+ expects_output=True,
+ )
+
+ await agent._maybe_begin_agent_initiated_turn(
+ context,
+ composite_key,
+ "sess-interleaved-batch",
+ "/tmp/work",
+ "session-key",
+ message_type="result",
+ )
+
+ self.assertEqual(
+ [activity.id for activity in pending_request.output_activities],
+ ["task-current-a", "task-current-b"],
+ )
+ self.assertTrue(
+ service.activities.settle_completed_output_batch(
+ pending_request.output,
+ accepted_message_exists=True,
+ )
+ )
+ agent._clear_request_activities(pending_request)
+ detached = service.activities.claim_completed_output("claude", composite_key)
+ self.assertIsNotNone(detached)
+ self.assertEqual(detached.id, "task-detached")
+ service.activities.ack_completed_output(detached)
+
+ async def test_terminal_only_task_event_stays_provenance_pending(self):
+ agent, service = _build_agent()
+ composite_key = "session-terminal-only:/tmp/work"
+ pending_context = SimpleNamespace(
+ platform_specific={"turn_token": "current-turn"},
+ )
+ agent._pending_requests[composite_key] = [SimpleNamespace(context=pending_context)]
+ context = SimpleNamespace(
+ platform_specific={"agent_session_id": "sess-terminal-only"},
+ )
+
+ self.assertTrue(
+ agent._handle_activity_message(
+ TaskNotificationMessage(),
+ composite_key,
+ context,
+ )
+ )
+
+ activity = service.activities.claim_completed_output("claude", composite_key)
+ self.assertIsNotNone(activity)
+ self.assertIsNone(activity.turn_id)
+ self.assertTrue(activity.metadata["provenance_pending"])
+
+ async def test_failed_activity_snapshot_waits_for_run_owner_ack(self):
+ agent, service = _build_agent()
+ composite_key = "session-failed-activity:/tmp/work"
+ context = SimpleNamespace(
+ platform_specific={"agent_session_id": "sess-failed-activity"},
+ )
+ service.activities.start(
+ backend="claude",
+ runtime_key=composite_key,
+ session_id="sess-failed-activity",
+ activity_id="task-failed",
+ kind="local_agent",
+ run_id="run-failed",
+ )
+ agent.controller.scheduled_task_service = SimpleNamespace(
+ settle_activity_runs=Mock(side_effect=RuntimeError("store unavailable")),
+ )
+ original_complete = service.activities.complete
+ service.activities.complete = Mock(wraps=original_complete)
+ original_ack = service.activities.ack_recovered_terminal
+ service.activities.ack_recovered_terminal = Mock(wraps=original_ack)
+ failed = SimpleNamespace(
+ subtype="task_notification",
+ task_id="task-failed",
+ status="failed",
+ summary="Background verification failed",
+ data={},
+ )
+
+ self.assertTrue(
+ agent._handle_activity_message(
+ failed,
+ composite_key,
+ context,
+ )
+ )
+
+ call = service.activities.complete.call_args
+ self.assertTrue(call.kwargs["retain_terminal_snapshot"])
+ completed = agent.controller.scheduled_task_service.settle_activity_runs.call_args.args[0]
+ self.assertEqual(completed.status, "failed")
+ service.activities.ack_recovered_terminal.assert_not_called()
+
+ async def test_ambiguous_activity_does_not_copy_pending_delivery_target(self):
+ agent, service = _build_agent()
+ composite_key = "session-delivery-origin:/tmp/work"
+ origin_context = SimpleNamespace(
+ platform_specific={
+ "turn_token": "origin-turn",
+ "delivery_key_external": "slack::channel::C-ORIGIN",
+ },
+ )
+ agent._pending_requests[composite_key] = [SimpleNamespace(context=origin_context)]
+ receiver_context = SimpleNamespace(
+ platform_specific={"agent_session_id": "sess-delivery-origin"},
+ )
+
+ self.assertTrue(
+ agent._handle_activity_message(
+ TaskStartedMessage(),
+ composite_key,
+ receiver_context,
+ )
+ )
+ agent._pending_requests[composite_key] = [
+ SimpleNamespace(
+ context=SimpleNamespace(
+ platform_specific={
+ "turn_token": "newer-turn",
+ "delivery_key_external": "slack::channel::C-NEWER",
+ },
+ )
+ )
+ ]
+
+ self.assertTrue(
+ agent._handle_activity_message(
+ TaskNotificationMessage(),
+ composite_key,
+ receiver_context,
+ )
+ )
+
+ activity = service.activities.claim_completed_output("claude", composite_key)
+ self.assertIsNotNone(activity)
+ self.assertNotIn("delivery_key_external", activity.metadata)
+ self.assertIsNone(activity.turn_id)
+
+ async def test_completed_task_notification_at_eof_settles_without_sdk_summary(self):
+ agent, service = _build_agent()
+ composite_key = "session-eof:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_runtime_turn_token": "old-turn",
+ "turn_token": "origin-turn",
+ "agent_session_id": "sess-eof",
+ },
+ )
+ agent.emit_result_message = AsyncMock()
agent.controller.emit_agent_message = _dispatcher_owned_emit(service)
await agent._receive_messages(
- _completed_task_notification_client(),
- "session-eof",
+ _completed_task_notification_client(),
+ "session-eof",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
+ )
+
+ # The SDK summary is a CLI receipt, not assistant output: it stays
+ # invisible and the Activity settles through the silent result path.
+ agent.emit_result_message.assert_not_awaited()
+ silent_calls = [
+ call
+ for call in agent.controller.emit_agent_message.await_args_list
+ if call.args[1:3] == ("result", "")
+ ]
+ self.assertEqual(len(silent_calls), 1)
+ output = silent_calls[0].kwargs["output"]
+ self.assertTrue(output.detached)
+ self.assertFalse(output.completes_turn)
+ self.assertTrue(output.completes_run)
+ self.assertIsNone(service.activities.claim_completed_output("claude", composite_key))
+
+ async def test_completed_task_notification_flushes_while_receiver_stays_open(self):
+ agent, service = _build_agent()
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0
+ composite_key = "session-live-flush:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_runtime_turn_token": "old-turn",
+ "turn_token": "origin-turn",
+ "agent_session_id": "sess-live-flush",
+ },
+ )
+ emitted = asyncio.Event()
+ dispatcher_emit = _dispatcher_owned_emit(service)
+
+ async def _emit(*args, **kwargs):
+ if args[1:3] == ("result", ""):
+ emitted.set()
+ return await dispatcher_emit(*args, **kwargs)
+
+ agent.emit_result_message = AsyncMock()
+ agent.controller.emit_agent_message = AsyncMock(side_effect=_emit)
+ release = asyncio.Event()
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _completed_task_notification_then_wait_client(release),
+ "session-live-flush",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
+ )
+ )
+ try:
+ await asyncio.wait_for(emitted.wait(), timeout=1)
+ self.assertFalse(receiver.done())
+ agent.emit_result_message.assert_not_awaited()
+ output = next(
+ call.kwargs["output"]
+ for call in agent.controller.emit_agent_message.await_args_list
+ if call.args[1:3] == ("result", "")
+ )
+ self.assertTrue(output.detached)
+ self.assertFalse(output.completes_turn)
+ self.assertTrue(output.completes_run)
+ self.assertIsNone(
+ service.activities.claim_completed_output("claude", composite_key)
+ )
+ finally:
+ release.set()
+ await receiver
+
+ async def test_foreground_tool_activity_waits_for_result_and_uses_assistant_text(self):
+ agent, service = _build_agent()
+ agent._adopt_pending_turn_token = ClaudeAgent._adopt_pending_turn_token
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0
+ formatter = SimpleNamespace(
+ escape_special_chars=lambda text: text.replace("`", "\\`").replace(
+ "*", "\\*"
+ ),
+ format_assistant_message=lambda parts: "\n\n".join(parts),
+ format_toolcall=lambda *_args, **_kwargs: "Bash tool call",
+ format_toolcall_label=lambda *_args, **_kwargs: "Push branch, confirm repo",
+ )
+ agent._get_formatter = lambda _context: formatter
+ emitted_provenance = []
+
+ async def _emit_result(context, _text, *, request=None, output=None, **_kwargs):
+ semantics = output or terminal_output_for(request)
+ emitted_provenance.append(semantics.provenance(context))
+ return "message-id"
+
+ agent.emit_result_message = AsyncMock(side_effect=_emit_result)
+ composite_key = "session-foreground-bash:/tmp/work"
+ origin_context = SimpleNamespace(
+ platform_specific={
+ "task_trigger_kind": "agent_run",
+ "task_execution_id": "run-origin",
+ "turn_token": "turn-origin",
+ }
+ )
+ queued_context = SimpleNamespace(
+ platform_specific={
+ "task_trigger_kind": "agent_run",
+ "task_execution_id": "run-followup",
+ "turn_token": "turn-followup",
+ }
+ )
+ origin_request = SimpleNamespace(context=origin_context)
+ queued_request = SimpleNamespace(context=queued_context)
+ agent._pending_requests[composite_key] = [origin_request, queued_request]
+ receiver_context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={"agent_session_id": "sess-foreground-bash"},
+ )
+ notification_seen = asyncio.Event()
+ release_result = asyncio.Event()
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _foreground_bash_then_delayed_result_client(
+ notification_seen,
+ release_result,
+ ),
+ "session-foreground-bash",
+ "/tmp/work",
+ receiver_context,
+ composite_key=composite_key,
+ )
+ )
+
+ await asyncio.wait_for(notification_seen.wait(), timeout=1)
+ await asyncio.sleep(0.01)
+
+ agent.emit_result_message.assert_not_awaited()
+ self.assertEqual(
+ agent._pending_requests[composite_key],
+ [origin_request, queued_request],
+ )
+ self.assertFalse(
+ service.activities.has_completed_output("claude", composite_key)
+ )
+
+ release_result.set()
+ await asyncio.wait_for(receiver, timeout=1)
+
+ agent.emit_result_message.assert_awaited_once()
+ self.assertEqual(
+ agent.emit_result_message.await_args.args[1],
+ (
+ "Two clean commits. Let me run `git push` and **confirm** the "
+ "repo/remote for the PR."
+ ),
+ )
+ self.assertEqual(agent._pending_requests[composite_key], [queued_request])
+ self.assertEqual(emitted_provenance[0]["run_id"], "run-origin")
+
+ async def test_terminal_result_survives_receipt_binding_persistence_failure(self):
+ class _ToggleStore:
+ def __init__(self):
+ self.fail = False
+ self.records = {}
+
+ def list_connections(self):
+ return []
+
+ def list_activities(self):
+ return []
+
+ def upsert_activity(self, payload, *, phase):
+ if self.fail:
+ raise RuntimeError("activity store unavailable")
+ self.records[payload["id"]] = (dict(payload), phase)
+
+ agent, service = _build_agent()
+ store = _ToggleStore()
+ service.activities = SessionActivityRegistry(store)
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0
+ agent._get_formatter = lambda _context: SimpleNamespace(
+ format_toolcall=lambda *_args, **_kwargs: "fixture tool",
+ format_toolcall_label=lambda *_args, **_kwargs: "fixture tool",
+ )
+ composite_key = "session-provenance-store-failure:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_session_id": "sess-provenance-store-failure",
+ "turn_token": "human-turn",
+ },
+ )
+ request = SimpleNamespace(
+ context=SimpleNamespace(
+ platform_specific={
+ "turn_token": "human-turn",
+ "task_trigger_kind": "agent_run",
+ "task_execution_id": "human-run",
+ }
+ )
+ )
+ agent._pending_requests[composite_key] = [request]
+ emitted = []
+ result_processed = asyncio.Event()
+ release_receiver = asyncio.Event()
+
+ async def emit_result(_context, text, **kwargs):
+ emitted.append((text, kwargs))
+ output = kwargs.get("output")
+ if isinstance(output, MessageOutput):
+ service.activities.settle_completed_output_batch(
+ output,
+ accepted_message_exists=True,
+ )
+ return "message-id"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+
+ async def emit_agent_message(_context, _kind, _text, **kwargs):
+ output = kwargs.get("output")
+ if isinstance(output, MessageOutput):
+ service.activities.settle_completed_output_batch(
+ output,
+ accepted_message_exists=True,
+ )
+ return "message-id"
+
+ agent.controller.emit_agent_message = AsyncMock(
+ side_effect=emit_agent_message,
+ )
+
+ def _tool_block():
+ block = object.__new__(ToolUseBlock)
+ block.id = "tool-background"
+ block.name = "Bash"
+ block.input = {
+ "command": "sleep 1",
+ "run_in_background": True,
+ }
+ return block
+
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ assistant = AssistantMessage()
+ assistant.content = [_tool_block()]
+ yield assistant
+
+ started = TaskStartedMessage()
+ started.task_id = "task-background"
+ started.tool_use_id = "tool-background"
+ started.task_type = "local_bash"
+ yield started
+
+ completed = TaskNotificationMessage()
+ completed.task_id = "task-background"
+ completed.tool_use_id = "tool-background"
+ completed.status = "completed"
+ yield completed
+ store.fail = True
+ yield ResultMessage()
+ result_processed.set()
+ await release_receiver.wait()
+
+ return _iterate()
+
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _Client(),
+ "session-provenance-store-failure",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
+ )
+ )
+ await asyncio.wait_for(result_processed.wait(), timeout=1)
+
+ self.assertEqual(len(emitted), 1)
+ self.assertEqual(emitted[0][0], "✅ master 回归环境已就绪")
+ self.assertFalse(agent._has_pending_requests(composite_key))
+ self.assertIn(composite_key, agent._provenance_recovery_evidence)
+ self.assertTrue(service.activities.has_completed_output("claude", composite_key))
+
+ store.fail = False
+ should_retry = await agent._flush_completed_activity_outputs(
+ composite_key,
+ context,
+ )
+ self.assertFalse(should_retry)
+ self.assertFalse(service.activities.has_completed_output("claude", composite_key))
+ self.assertFalse(service.activities.has_claimed_output("claude", composite_key))
+ release_receiver.set()
+ await asyncio.wait_for(receiver, timeout=1)
+ self.assertFalse(service.activities.has_completed_output("claude", composite_key))
+
+ async def test_detached_delivery_retries_while_receiver_stays_open(self):
+ agent, service = _build_agent()
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0
+ composite_key = "session-detached-live-retry:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_session_id": "sess-detached-live-retry",
+ },
+ )
+ result_seen = asyncio.Event()
+ retry_succeeded = asyncio.Event()
+ release_receiver = asyncio.Event()
+ attempts = 0
+
+ async def emit_result(*_args, **_kwargs):
+ nonlocal attempts
+ attempts += 1
+ if attempts == 1:
+ raise RuntimeError("temporary delivery failure")
+ retry_succeeded.set()
+ return "message-id"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ assistant = AssistantMessage()
+ assistant.content = [
+ TextBlock(text="Detached background result"),
+ ]
+ yield assistant
+ detached_result = ResultMessage()
+ detached_result.origin = {"kind": "task-notification"}
+ yield detached_result
+ result_seen.set()
+ await release_receiver.wait()
+
+ return _iterate()
+
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _Client(),
+ "sess-detached-live-retry",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
+ )
+ )
+ await asyncio.wait_for(result_seen.wait(), timeout=1)
+ await asyncio.wait_for(retry_succeeded.wait(), timeout=1)
+
+ self.assertFalse(receiver.done())
+ self.assertEqual(attempts, 2)
+ self.assertFalse(agent._has_pending_requests(composite_key))
+ self.assertFalse(service.runtime_turn_active(composite_key))
+
+ release_receiver.set()
+ await asyncio.wait_for(receiver, timeout=1)
+
+ async def test_detached_activity_retry_keeps_terminal_text_and_owner_while_stream_is_open(
+ self,
+ ):
+ agent, service = _build_agent()
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+ composite_key = "session-detached-activity-live-retry:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_session_id": "sess-detached-activity-live-retry",
+ },
+ )
+ followup_processed = asyncio.Event()
+ release_receiver = asyncio.Event()
+ allow_success = asyncio.Event()
+ attempts: list[tuple[str, MessageOutput]] = []
+
+ async def emit_result(_context, text, **kwargs):
+ output = kwargs["output"]
+ attempts.append((text, output))
+ if not allow_success.is_set():
+ return None
+ service.activities.settle_completed_output_batch(
+ output,
+ accepted_message_exists=True,
+ )
+ return f"message-{len(attempts)}"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ started = TaskStartedMessage()
+ started.task_id = "task-live-retry"
+ started.tool_use_id = "tool-live-retry"
+ yield started
+
+ completed = TaskNotificationMessage()
+ completed.task_id = "task-live-retry"
+ completed.summary = "CLI summary"
+ completed.tool_use_id = "tool-live-retry"
+ yield completed
+
+ assistant = AssistantMessage()
+ assistant.content = [
+ TextBlock(text="ACTUAL ASSISTANT RESULT"),
+ ]
+ yield assistant
+
+ detached_result = ResultMessage()
+ detached_result.result = "CLI summary"
+ detached_result.origin = {"kind": "task-notification"}
+ yield detached_result
+
+ followup = AssistantMessage()
+ followup.content = [TextBlock(text="CLI summary")]
+ yield followup
+ followup_processed.set()
+ await release_receiver.wait()
+
+ return _iterate()
+
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _Client(),
+ "sess-detached-activity-live-retry",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
+ )
+ )
+ await asyncio.wait_for(followup_processed.wait(), timeout=1)
+ self.assertIsNotNone(agent._output_record(composite_key, None, owner="activity"))
+ self.assertTrue(agent._has_synthetic_delivery_pending(composite_key))
+ self.assertTrue(service.runtime_turn_active(composite_key))
+
+ await asyncio.sleep(0.03)
+ self.assertGreaterEqual(len(attempts), 2)
+ self.assertTrue(all(text == "ACTUAL ASSISTANT RESULT" for text, _ in attempts))
+ self.assertTrue(agent._has_synthetic_delivery_pending(composite_key))
+
+ allow_success.set()
+ await asyncio.wait_for(
+ asyncio.create_task(
+ _wait_until(
+ lambda: agent._output_record(composite_key, None, owner="activity") is None
+ )
+ ),
+ timeout=1,
+ )
+ self.assertFalse(agent._has_pending_requests(composite_key))
+ self.assertFalse(service.runtime_turn_active(composite_key))
+ self.assertFalse(service.activities.has_claimed_output("claude", composite_key))
+ self.assertEqual(
+ attempts[0][1].idempotency_key,
+ attempts[-1][1].idempotency_key,
+ )
+
+ release_receiver.set()
+ await asyncio.wait_for(receiver, timeout=1)
+
+ async def test_receiver_error_keeps_detached_recovery_owner_until_retry_succeeds(
+ self,
+ ):
+ agent, service = _build_agent()
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+ composite_key = "session-detached-activity-error-recovery:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_session_id": "sess-detached-activity-error-recovery",
+ },
+ )
+ allow_success = asyncio.Event()
+ attempts: list[str] = []
+
+ async def emit_result(_context, text, **kwargs):
+ attempts.append(text)
+ if not allow_success.is_set():
+ return None
+ service.activities.settle_completed_output_batch(
+ kwargs["output"],
+ accepted_message_exists=True,
+ )
+ return "message-id"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ started = TaskStartedMessage()
+ started.task_id = "task-error-recovery"
+ yield started
+
+ completed = TaskNotificationMessage()
+ completed.task_id = "task-error-recovery"
+ completed.summary = "CLI summary"
+ yield completed
+
+ assistant = AssistantMessage()
+ assistant.content = [
+ TextBlock(text="ACTUAL ASSISTANT RESULT"),
+ ]
+ yield assistant
+
+ detached_result = ResultMessage()
+ detached_result.result = "CLI summary"
+ detached_result.origin = {"kind": "task-notification"}
+ yield detached_result
+ raise RuntimeError("receiver transport failed")
+
+ return _iterate()
+
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _Client(),
+ "sess-detached-activity-error-recovery",
+ "/tmp/work",
+ context,
+ composite_key=composite_key,
+ )
+ )
+ await asyncio.wait_for(receiver, timeout=1)
+
+ self.assertTrue(attempts)
+ self.assertIsNotNone(agent._output_record(composite_key, None, owner="activity"))
+ self.assertTrue(agent._has_synthetic_delivery_pending(composite_key))
+ self.assertTrue(agent._has_pending_requests(composite_key))
+ self.assertTrue(service.runtime_turn_active(composite_key))
+ self.assertEqual(attempts[0], "ACTUAL ASSISTANT RESULT")
+ self.assertEqual(agent.controller.emit_agent_message.await_count, 0)
+
+ recovery_task = agent._activity_flush_tasks.pop(composite_key, None)
+ if recovery_task is not None:
+ recovery_task.cancel()
+ await asyncio.gather(recovery_task, return_exceptions=True)
+ allow_success.set()
+ self.assertFalse(
+ await agent._flush_detached_activity_output(composite_key, context)
+ )
+ self.assertFalse(agent._has_pending_requests(composite_key))
+ self.assertFalse(service.runtime_turn_active(composite_key))
+ self.assertFalse(service.activities.has_claimed_output("claude", composite_key))
+
+ async def test_eof_keeps_detached_recovery_owner_after_persistent_retry_failure(
+ self,
+ ):
+ agent, service = _build_agent()
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+ composite_key = "session-detached-activity-eof-recovery:/tmp/work"
+ context = SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_session_id": "sess-detached-activity-eof-recovery",
+ },
+ )
+ service.activities.start(
+ backend="claude",
+ runtime_key=composite_key,
+ session_id="sess-detached-activity-eof-recovery",
+ activity_id="task-eof-recovery",
+ kind="local_agent",
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=composite_key,
+ activity_id="task-eof-recovery",
+ status="completed",
+ metadata={"summary": "CLI summary"},
+ expects_output=True,
+ )
+ allow_success = asyncio.Event()
+ attempts: list[str] = []
+
+ async def emit_result(_context, text, **kwargs):
+ attempts.append(text)
+ if not allow_success.is_set():
+ raise RuntimeError("persistent delivery failure")
+ service.activities.settle_completed_output_batch(
+ kwargs["output"],
+ accepted_message_exists=True,
+ )
+ return "message-id"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ assistant = AssistantMessage()
+ assistant.content = [
+ TextBlock(text="ACTUAL ASSISTANT RESULT"),
+ ]
+ yield assistant
+
+ detached_result = ResultMessage()
+ detached_result.result = "CLI summary"
+ detached_result.origin = {"kind": "task-notification"}
+ yield detached_result
+
+ return _iterate()
+
+ await agent._receive_messages(
+ _Client(),
+ "sess-detached-activity-eof-recovery",
"/tmp/work",
context,
composite_key=composite_key,
)
- # The SDK summary is a CLI receipt, not assistant output: it stays
- # invisible and the Activity settles through the silent result path.
- agent.emit_result_message.assert_not_awaited()
- silent_calls = [
- call
- for call in agent.controller.emit_agent_message.await_args_list
- if call.args[1:3] == ("result", "")
- ]
- self.assertEqual(len(silent_calls), 1)
- output = silent_calls[0].kwargs["output"]
- self.assertTrue(output.detached)
- self.assertFalse(output.completes_turn)
- self.assertTrue(output.completes_run)
- self.assertIsNone(service.activities.claim_completed_output("claude", composite_key))
+ self.assertGreaterEqual(len(attempts), 2)
+ self.assertTrue(agent._has_synthetic_delivery_pending(composite_key))
+ self.assertTrue(agent._has_pending_requests(composite_key))
+ self.assertTrue(service.runtime_turn_active(composite_key))
+ self.assertEqual(attempts[0], "ACTUAL ASSISTANT RESULT")
+ self.assertTrue(service.activities.has_claimed_output("claude", composite_key))
- async def test_completed_task_notification_flushes_while_receiver_stays_open(self):
+ recovery_task = agent._activity_flush_tasks.pop(composite_key, None)
+ if recovery_task is not None:
+ recovery_task.cancel()
+ await asyncio.gather(recovery_task, return_exceptions=True)
+ allow_success.set()
+ self.assertFalse(
+ await agent._flush_detached_activity_output(composite_key, context)
+ )
+ self.assertFalse(agent._has_pending_requests(composite_key))
+ self.assertFalse(agent._has_synthetic_delivery_pending(composite_key))
+ self.assertFalse(service.runtime_turn_active(composite_key))
+ self.assertFalse(service.activities.has_claimed_output("claude", composite_key))
+
+ async def test_missing_origin_with_competing_activity_stays_detached(self):
agent, service = _build_agent()
- agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0
- composite_key = "session-live-flush:/tmp/work"
+ composite_key = "session-ambiguous-origin:/tmp/work"
context = SimpleNamespace(
user_id="U1",
channel_id="C1",
platform="avibe",
platform_specific={
- "agent_runtime_turn_key": composite_key,
- "agent_runtime_turn_token": "old-turn",
- "turn_token": "origin-turn",
- "agent_session_id": "sess-live-flush",
+ "agent_session_id": "sess-ambiguous-origin",
+ "turn_token": "human-turn",
},
)
- emitted = asyncio.Event()
- dispatcher_emit = _dispatcher_owned_emit(service)
-
- async def _emit(*args, **kwargs):
- if args[1:3] == ("result", ""):
- emitted.set()
- return await dispatcher_emit(*args, **kwargs)
-
- agent.emit_result_message = AsyncMock()
- agent.controller.emit_agent_message = AsyncMock(side_effect=_emit)
- release = asyncio.Event()
- receiver = asyncio.create_task(
- agent._receive_messages(
- _completed_task_notification_then_wait_client(release),
- "session-live-flush",
- "/tmp/work",
- context,
- composite_key=composite_key,
+ request = SimpleNamespace(
+ context=SimpleNamespace(
+ platform_specific={"turn_token": "human-turn"},
)
)
- try:
- await asyncio.wait_for(emitted.wait(), timeout=1)
- self.assertFalse(receiver.done())
- agent.emit_result_message.assert_not_awaited()
- output = next(
- call.kwargs["output"]
- for call in agent.controller.emit_agent_message.await_args_list
- if call.args[1:3] == ("result", "")
- )
- self.assertTrue(output.detached)
- self.assertFalse(output.completes_turn)
- self.assertTrue(output.completes_run)
- self.assertIsNone(
- service.activities.claim_completed_output("claude", composite_key)
- )
- finally:
- release.set()
- await receiver
-
- async def test_foreground_tool_activity_waits_for_result_and_uses_assistant_text(self):
- agent, service = _build_agent()
- agent._adopt_pending_turn_token = ClaudeAgent._adopt_pending_turn_token
- agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0
- formatter = SimpleNamespace(
- escape_special_chars=lambda text: text.replace("`", "\\`").replace(
- "*", "\\*"
- ),
- format_assistant_message=lambda parts: "\n\n".join(parts),
- format_toolcall=lambda *_args, **_kwargs: "Bash tool call",
- format_toolcall_label=lambda *_args, **_kwargs: "Push branch, confirm repo",
+ agent._pending_requests[composite_key] = [request]
+ service.activities.start(
+ backend="claude",
+ runtime_key=composite_key,
+ session_id="sess-ambiguous-origin",
+ activity_id="unrelated-task",
+ kind="local_agent",
)
- agent._get_formatter = lambda _context: formatter
- emitted_provenance = []
+ service.activities.complete(
+ backend="claude",
+ runtime_key=composite_key,
+ activity_id="unrelated-task",
+ status="completed",
+ metadata={"summary": "Unrelated task finished"},
+ expects_output=True,
+ )
+ result_seen = asyncio.Event()
+ release_receiver = asyncio.Event()
- async def _emit_result(context, _text, *, request=None, output=None, **_kwargs):
- semantics = output or terminal_output_for(request)
- emitted_provenance.append(semantics.provenance(context))
- return "message-id"
+ class ResultMessage:
+ subtype = "success"
+ result = "Legacy result without provenance"
+ duration_ms = 1
+ origin = None
- agent.emit_result_message = AsyncMock(side_effect=_emit_result)
- composite_key = "session-foreground-bash:/tmp/work"
- origin_context = SimpleNamespace(
- platform_specific={
- "task_trigger_kind": "agent_run",
- "task_execution_id": "run-origin",
- "turn_token": "turn-origin",
- }
- )
- queued_context = SimpleNamespace(
- platform_specific={
- "task_trigger_kind": "agent_run",
- "task_execution_id": "run-followup",
- "turn_token": "turn-followup",
- }
- )
- origin_request = SimpleNamespace(context=origin_context)
- queued_request = SimpleNamespace(context=queued_context)
- agent._pending_requests[composite_key] = [origin_request, queued_request]
- receiver_context = SimpleNamespace(
- user_id="U1",
- channel_id="C1",
- platform="avibe",
- platform_specific={"agent_session_id": "sess-foreground-bash"},
- )
- notification_seen = asyncio.Event()
- release_result = asyncio.Event()
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ assistant = AssistantMessage()
+ tool = object.__new__(ToolUseBlock)
+ tool.id = "tool-ambiguous"
+ tool.name = "Bash"
+ tool.input = {"command": "echo ambiguous"}
+ assistant.content = [tool]
+ yield assistant
+ yield ResultMessage()
+ result_seen.set()
+ await release_receiver.wait()
+
+ return _iterate()
+
+ agent.emit_result_message = AsyncMock(return_value="message-id")
receiver = asyncio.create_task(
agent._receive_messages(
- _foreground_bash_then_delayed_result_client(
- notification_seen,
- release_result,
- ),
- "session-foreground-bash",
+ _Client(),
+ "sess-ambiguous-origin",
"/tmp/work",
- receiver_context,
+ context,
composite_key=composite_key,
)
)
+ await asyncio.wait_for(result_seen.wait(), timeout=1)
- await asyncio.wait_for(notification_seen.wait(), timeout=1)
- await asyncio.sleep(0.01)
-
+ self.assertEqual(agent._result_owner(composite_key, ResultMessage()), "detached")
+ self.assertTrue(agent._has_pending_requests(composite_key))
agent.emit_result_message.assert_not_awaited()
- self.assertEqual(
- agent._pending_requests[composite_key],
- [origin_request, queued_request],
- )
- self.assertFalse(
- service.activities.has_completed_output("claude", composite_key)
- )
+ detached_calls = [
+ call
+ for call in agent.controller.emit_agent_message.await_args_list
+ if call.kwargs.get("output") is not None
+ ]
+ self.assertEqual(len(detached_calls), 1)
+ self.assertTrue(detached_calls[0].kwargs["output"].detached)
- release_result.set()
+ release_receiver.set()
await asyncio.wait_for(receiver, timeout=1)
- agent.emit_result_message.assert_awaited_once()
- self.assertEqual(
- agent.emit_result_message.await_args.args[1],
- (
- "Two clean commits. Let me run `git push` and **confirm** the "
- "repo/remote for the PR."
- ),
- )
- self.assertEqual(agent._pending_requests[composite_key], [queued_request])
- self.assertEqual(emitted_provenance[0]["run_id"], "run-origin")
-
async def test_detached_unsolicited_error_keeps_sdk_failure_text(self):
agent, service = _build_agent()
composite_key = "session-detached-unsolicited-error:/tmp/work"
context = SimpleNamespace(
platform_specific={"agent_session_id": "sess-detached-unsolicited-error"}
)
- agent._detached_unsolicited_outputs.add(composite_key)
- agent._detached_unsolicited_text[composite_key] = "Earlier assistant text"
+ agent._detached_phase(composite_key).text = "Earlier assistant text"
agent.emit_result_message = _dispatcher_owned_emit(service)
class ResultMessage:
@@ -1220,8 +2277,8 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
status="completed",
expects_output=True,
)
- agent._detached_activity_outputs[composite_key] = (
- service.activities.claim_completed_output_batch("claude", composite_key)
+ agent._retain_activity_output_record(
+ composite_key, service.activities.claim_completed_output_batch("claude", composite_key),
)
agent._foreground_tool_use_ids[composite_key] = {"toolu_new_turn"}
agent._turns_with_foreground_tools.add(composite_key)
@@ -1277,7 +2334,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
}
)
)
- agent._detached_unsolicited_outputs.add(composite_key)
+ agent._detached_phase(composite_key)
agent.emit_result_message = _dispatcher_owned_emit(service)
def _content_block(block_type, **values):
@@ -1373,8 +2430,9 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
"claude",
composite_key,
)
- agent._detached_activity_outputs[composite_key] = activities
- agent._detached_assistant_text[composite_key] = "Earlier assistant text"
+ agent._retain_activity_output_record(
+ composite_key, activities, text="Earlier assistant text",
+ )
agent._emit_activity_result = AsyncMock()
class ResultMessage:
@@ -2207,7 +3265,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(client.sent, ["Background verification finished"])
self.assertFalse(service.activities.has_completed_output("claude", composite_key))
- async def test_timed_flush_drains_prequeued_same_turn_batch(self):
+ async def test_timed_flush_defers_prequeued_same_turn_batch_until_result(self):
agent, service = _build_agent()
composite_key = "session-batched-flush:/tmp/work"
pending_request = SimpleNamespace(
@@ -2245,14 +3303,12 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
context,
)
- self.assertFalse(should_retry)
- agent.emit_result_message.assert_awaited_once()
- output = agent.emit_result_message.await_args.kwargs["output"]
- self.assertEqual(output.activity_id, "task-rebuild")
- self.assertFalse(service.activities.has_completed_output("claude", composite_key))
+ self.assertTrue(should_retry)
+ agent.emit_result_message.assert_not_awaited()
+ self.assertTrue(service.activities.has_completed_output("claude", composite_key))
self.assertEqual(pending_request.output_activities, [])
- async def test_detached_activity_output_requeues_when_delivery_returns_none(self):
+ async def test_detached_activity_output_retains_claim_when_delivery_returns_none(self):
agent, service = _build_agent()
composite_key = "session-delivery-failed:/tmp/work"
context = SimpleNamespace(
@@ -2276,18 +3332,22 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
)
activity = service.activities.claim_completed_output("claude", composite_key)
self.assertIsNotNone(activity)
- agent._detached_activity_outputs[composite_key] = [activity]
- agent._detached_assistant_text[composite_key] = "Full background result"
+ record = agent._retain_activity_output_record(
+ composite_key, [activity], text="Full background result",
+ )
agent.emit_result_message = AsyncMock(return_value=None)
- with self.assertRaisesRegex(RuntimeError, "was not persisted or delivered"):
- await agent._flush_detached_activity_output(composite_key, context)
+ should_retry = await agent._flush_detached_activity_output(composite_key, context)
- claimed = service.activities.claim_completed_output("claude", composite_key)
- self.assertIsNotNone(claimed)
- self.assertEqual(claimed.id, "task-690")
+ self.assertTrue(should_retry)
+ self.assertIsNotNone(agent._output_record(composite_key, None, owner="activity"))
+ self.assertEqual(
+ record.text,
+ "Full background result",
+ )
+ self.assertTrue(service.activities.has_claimed_output("claude", composite_key))
- async def test_detached_durable_settlement_error_stays_out_of_receiver_failure(self):
+ async def test_detached_durable_settlement_error_retries_same_claim(self):
agent, service = _build_agent()
composite_key = "session-detached-durable-retry:/tmp/work"
context = SimpleNamespace(
@@ -2314,8 +3374,9 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
composite_key,
)
self.assertIsNotNone(activity)
- agent._detached_activity_outputs[composite_key] = [activity]
- agent._detached_assistant_text[composite_key] = "Full background result"
+ agent._retain_activity_output_record(
+ composite_key, [activity], text="Full background result",
+ )
agent._emit_activity_result = AsyncMock(
side_effect=ActivityOutputDeliveryError(
"durable output needs local settlement",
@@ -2326,15 +3387,20 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
)
with patch.object(agent, "_schedule_completed_activity_flush") as schedule:
- await agent._flush_detached_activity_output(composite_key, context)
+ should_retry = await agent._flush_detached_activity_output(composite_key, context)
schedule.assert_called_once_with(composite_key, context)
- reclaimed = service.activities.claim_completed_output(
- "claude",
- composite_key,
+ self.assertTrue(should_retry)
+ self.assertIsNotNone(agent._output_record(composite_key, None, owner="activity"))
+ self.assertTrue(service.activities.has_claimed_output("claude", composite_key))
+
+ agent._emit_activity_result = ClaudeAgent._emit_activity_result.__get__(agent)
+ agent.emit_result_message = _dispatcher_owned_emit(service)
+ self.assertFalse(
+ await agent._flush_detached_activity_output(composite_key, context)
)
- self.assertIsNotNone(reclaimed)
- self.assertEqual(reclaimed.id, "task-detached-durable-retry")
+ self.assertIsNone(agent._output_record(composite_key, None, owner="activity"))
+ self.assertFalse(service.activities.has_claimed_output("claude", composite_key))
async def test_requeued_request_activity_restores_terminal_turn_policy(self):
agent, service = _build_agent()
@@ -2424,7 +3490,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(service.activities.requeue_completed_outputs(retried), 2)
- async def test_failed_pending_batch_restores_older_global_output_order(self):
+ async def test_pending_batch_defers_without_explicit_activity_binding(self):
agent, service = _build_agent()
composite_key = "session-global-requeue-order:/tmp/work"
pending_request = SimpleNamespace(
@@ -2460,26 +3526,15 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
)
agent.emit_result_message = AsyncMock(return_value=None)
- with self.assertRaisesRegex(RuntimeError, "was not persisted or delivered"):
- await agent._flush_completed_activity_outputs(composite_key, context)
-
- restored = []
- while activity := service.activities.claim_completed_output(
- "claude",
+ should_retry = await agent._flush_completed_activity_outputs(
composite_key,
- ):
- restored.append(activity)
- service.activities.ack_completed_output(activity)
- self.assertEqual(
- [activity.id for activity in restored],
- ["task-old", "task-current-b"],
- )
- self.assertEqual(
- restored[-1].metadata["output_batch_activity_ids"],
- ["task-current-a", "task-current-b"],
+ context,
)
- for activity in restored:
- service.activities.ack_completed_output(activity)
+
+ self.assertTrue(should_retry)
+ agent.emit_result_message.assert_not_awaited()
+ self.assertTrue(service.activities.has_completed_output("claude", composite_key))
+ self.assertEqual(pending_request.output_activities, [])
async def test_terminal_only_activity_waits_queued_until_pending_turn_finishes(self):
agent, service = _build_agent()
@@ -2528,7 +3583,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
self.assertFalse(service.activities.has_completed_output("claude", composite_key))
self.assertFalse(agent._activity_output_pending(composite_key))
- async def test_same_turn_summary_delivery_failure_retries_detached(self):
+ async def test_same_turn_summary_does_not_claim_pending_turn(self):
agent, service = _build_agent()
composite_key = "session-same-turn-retry:/tmp/work"
pending_request = SimpleNamespace(
@@ -2554,48 +3609,17 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
metadata={"summary": "Background verification finished"},
expects_output=True,
)
- emit_attempts = 0
-
- async def emit_result(*_args, **kwargs):
- nonlocal emit_attempts
- emit_attempts += 1
- if emit_attempts == 1:
- return None
- self.assertTrue(
- service.activities.settle_completed_output_batch(
- kwargs["output"],
- accepted_message_exists=True,
- )
- )
- return "delivered-message-id"
-
- agent.emit_result_message = AsyncMock(side_effect=emit_result)
- agent._remove_result_pending_reaction = AsyncMock()
-
- with self.assertRaisesRegex(RuntimeError, "was not persisted or delivered"):
- await agent._flush_completed_activity_outputs(composite_key, context)
+ agent.emit_result_message = AsyncMock(return_value=None)
- self.assertFalse(agent._has_pending_requests(composite_key))
- agent._remove_result_pending_reaction.assert_awaited_once_with(
+ should_retry = await agent._flush_completed_activity_outputs(
composite_key,
context,
- pending_request,
)
- self.assertTrue(service.activities.has_completed_output("claude", composite_key))
- tidy_output = agent.controller.emit_agent_message.await_args.kwargs["output"]
- self.assertTrue(tidy_output.completes_turn)
- self.assertFalse(tidy_output.settles_run)
-
- await agent._flush_completed_activity_outputs(composite_key, context)
- self.assertFalse(agent._has_pending_requests(composite_key))
- first_output = agent.emit_result_message.await_args_list[0].kwargs["output"]
- second_output = agent.emit_result_message.await_args_list[1].kwargs["output"]
- self.assertFalse(first_output.detached)
- self.assertTrue(first_output.completes_turn)
- self.assertTrue(second_output.detached)
- self.assertFalse(second_output.completes_turn)
- self.assertFalse(service.activities.has_completed_output("claude", composite_key))
+ self.assertTrue(should_retry)
+ self.assertTrue(agent._has_pending_requests(composite_key))
+ agent.emit_result_message.assert_not_awaited()
+ self.assertTrue(service.activities.has_completed_output("claude", composite_key))
async def test_result_frame_activity_delivery_failure_retries_detached(self):
agent, service = _build_agent()
@@ -2714,7 +3738,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
self.assertFalse(service.activities.has_active("claude", composite_key))
self.assertEqual(mark_idle_calls, [composite_key])
- async def test_task_start_uses_current_request_run_not_stale_receiver_run(self):
+ async def test_ambiguous_task_start_does_not_copy_current_request_run(self):
agent, service = _build_agent()
composite_key = "session-lineage:/tmp/work"
pending_context = SimpleNamespace(
@@ -2743,11 +3767,12 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
)
activity = service.activities.active_for_runtime("claude", composite_key)[0]
- self.assertEqual(activity.run_id, "run-current")
- self.assertEqual(activity.turn_id, "current-turn")
- self.assertEqual(activity.metadata["run_ids"], ["run-current"])
+ self.assertIsNone(activity.run_id)
+ self.assertIsNone(activity.turn_id)
+ self.assertNotIn("run_ids", activity.metadata)
+ self.assertTrue(activity.metadata["provenance_pending"])
- async def test_task_progress_keeps_original_run_lineage(self):
+ async def test_task_progress_keeps_ambiguous_activity_unattributed(self):
agent, service = _build_agent()
composite_key = "session-progress-lineage:/tmp/work"
origin_context = SimpleNamespace(
@@ -2792,9 +3817,10 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
)
activity = service.activities.active_for_runtime("claude", composite_key)[0]
- self.assertEqual(activity.run_id, "run-origin")
- self.assertEqual(activity.turn_id, "turn-origin")
- self.assertEqual(activity.metadata["run_ids"], ["run-origin"])
+ self.assertIsNone(activity.run_id)
+ self.assertIsNone(activity.turn_id)
+ self.assertNotIn("run_ids", activity.metadata)
+ self.assertTrue(activity.metadata["provenance_pending"])
self.assertTrue(
agent._handle_activity_message(
@@ -2805,9 +3831,10 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
)
completed = service.activities.claim_completed_output("claude", composite_key)
self.assertIsNotNone(completed)
- self.assertEqual(completed.run_id, "run-origin")
- self.assertEqual(completed.turn_id, "turn-origin")
- self.assertEqual(completed.metadata["run_ids"], ["run-origin"])
+ self.assertIsNone(completed.run_id)
+ self.assertIsNone(completed.turn_id)
+ self.assertNotIn("run_ids", completed.metadata)
+ self.assertTrue(completed.metadata["provenance_pending"])
async def test_failed_task_does_not_claim_a_missing_followup_output(self):
mark_idle_calls: list[str] = []
@@ -2935,7 +3962,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
composite_key=composite_key,
)
- agent.emit_result_message.assert_awaited_once()
+ self.assertEqual(agent.emit_result_message.await_count, 2)
agent._maybe_backfill_session_title.assert_called_once_with(
pending_request,
"claude-native-session",
@@ -2992,7 +4019,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(mode, "activity")
agent.emit_result_message.assert_not_awaited()
- self.assertIn(composite_key, agent._detached_activity_outputs)
+ self.assertIsNotNone(agent._output_record(composite_key, None, owner="activity"))
self.assertEqual(gate.token, "USER-TURN")
await agent._receive_messages(
@@ -3007,7 +4034,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
activity_output = first_call.kwargs["output"]
self.assertTrue(activity_output.detached)
self.assertFalse(activity_output.completes_turn)
- self.assertNotIn(composite_key, agent._detached_activity_outputs)
+ self.assertIsNone(agent._output_record(composite_key, None, owner="activity"))
self.assertEqual(gate.token, "USER-TURN")
user_request = SimpleNamespace(
@@ -3108,7 +4135,7 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
]
self.assertEqual(len(set(output_ids)), 2)
- async def test_task_completed_inside_its_origin_turn_remains_attached(self):
+ async def test_task_completion_is_detached_from_human_result(self):
agent, service = _build_agent()
composite_key = "session-current:/tmp/work"
gate = service._get_turn_gate(composite_key)
@@ -3122,7 +4149,8 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
"turn_token": "current-turn",
}
)
- agent._pending_requests[composite_key] = [SimpleNamespace(context=user_context)]
+ pending_request = SimpleNamespace(context=user_context)
+ agent._pending_requests[composite_key] = [pending_request]
receiver_context = SimpleNamespace(
user_id="U1",
channel_id="C1",
@@ -3144,11 +4172,19 @@ class ReceiverOpensAgentInitiatedTurnTests(unittest.IsolatedAsyncioTestCase):
composite_key=composite_key,
)
- output = agent.emit_result_message.await_args.kwargs["request"].output
- self.assertFalse(output.detached)
- self.assertTrue(output.completes_turn)
- self.assertTrue(output.completes_run)
- self.assertEqual(output.activity_id, "task-690")
+ # The current master contract deliberately treats the SDK Task summary
+ # as a non-visible receipt. With no Assistant text, the detached
+ # Activity settles through the controller's silent result path rather
+ # than ``emit_result_message``.
+ self.assertEqual(agent.emit_result_message.await_count, 1)
+ human_call = agent.emit_result_message.await_args
+ self.assertIs(human_call.kwargs["request"], pending_request)
+ self.assertNotIn("output", human_call.kwargs)
+ detached_call = agent.controller.emit_agent_message.await_args
+ detached_output = detached_call.kwargs["output"]
+ self.assertTrue(detached_output.detached)
+ self.assertFalse(detached_output.completes_turn)
+ self.assertEqual(detached_output.activity_id, "task-690")
self.assertFalse(agent._has_pending_requests(composite_key))
async def test_non_actionable_fallback_does_not_open_agent_initiated_turn(self):
diff --git a/tests/test_claude_agent_sessions.py b/tests/test_claude_agent_sessions.py
index dc783f394..8c6617330 100644
--- a/tests/test_claude_agent_sessions.py
+++ b/tests/test_claude_agent_sessions.py
@@ -24,6 +24,20 @@ from modules.agents.service import AgentService
from modules.claude_sdk_compat import TextBlock, UserMessage
+async def _query_text(query):
+ """Read the explicit-provenance streaming prompt passed to the Claude SDK."""
+
+ if hasattr(query, "__aiter__"):
+ frames = [frame async for frame in query]
+ assert len(frames) == 1
+ frame = frames[0]
+ return (
+ frame["message"]["content"],
+ frame["origin"],
+ )
+ return query, None
+
+
async def test_skill_catalog_is_offered_once_per_accepted_claude_client(monkeypatch):
controller = _StubController()
client = SimpleNamespace(query=AsyncMock(), _vibe_pending_skill_catalog={"entries": []})
@@ -275,17 +289,20 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
second = asyncio.create_task(service.handle_message("claude", _request("second")))
await asyncio.sleep(0.05)
- self.assertEqual(
- queries,
- [("first", runtime_key)],
- "the second prompt must not be written into the same Claude runtime before the first result",
- )
+ self.assertEqual(len(queries), 1)
+ self.assertEqual(await _query_text(queries[0][0]), ("first", {"kind": "human"}))
+ self.assertEqual(queries[0][1], runtime_key)
release_result.set()
await asyncio.wait_for(first, timeout=3)
await asyncio.wait_for(second, timeout=3)
- self.assertEqual(queries, [("first", runtime_key), ("second", runtime_key)])
+ self.assertEqual(len(queries), 2)
+ second_prompt, second_origin = await _query_text(queries[1][0])
+ self.assertEqual(
+ (second_prompt, second_origin, queries[1][1]),
+ ("second", {"kind": "human"}, runtime_key),
+ )
async def test_cancelled_waiter_does_not_leak_runtime_session_lock(self):
controller = _StubController()
@@ -370,7 +387,16 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
third = asyncio.create_task(service.handle_message("claude", _request("third")))
await asyncio.wait_for(third, timeout=3)
- self.assertEqual(queries, [("first", runtime_key), ("third", runtime_key)])
+ self.assertEqual(
+ [
+ (await _query_text(message), session_id)
+ for message, session_id in queries
+ ],
+ [
+ (("first", {"kind": "human"}), runtime_key),
+ (("third", {"kind": "human"}), runtime_key),
+ ],
+ )
self.assertNotIn(runtime_key, agent._pending_requests)
async def test_handle_message_receiver_eof_without_result_settles_current_turn(self):
@@ -446,7 +472,13 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
if receiver_task is not None:
await asyncio.wait_for(receiver_task, timeout=1)
- self.assertEqual(queries, [("first", runtime_key)])
+ self.assertEqual(
+ [
+ (await _query_text(message), session_id)
+ for message, session_id in queries
+ ],
+ [(("first", {"kind": "human"}), runtime_key)],
+ )
self.assertEqual(mark_active_calls, [runtime_key])
self.assertIn(runtime_key, mark_idle_calls)
controller.emit_agent_message.assert_awaited_once_with(
@@ -1957,9 +1989,11 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
controller.session_handler.get_or_create_claude_session.assert_awaited_once()
self.assertEqual(mark_active_calls, [runtime_key])
- client.query.assert_awaited_once_with(
- "[Now: 2026-09-06 11:10:00 UTC+00:00]\n[Sender<U1>]\nhello", session_id=runtime_key,
- )
+ self.assertEqual(client.query.await_count, 1)
+ prompt, origin = await _query_text(client.query.await_args.args[0])
+ self.assertEqual(prompt, "[Now: 2026-09-06 11:10:00 UTC+00:00]\n[Sender<U1>]\nhello")
+ self.assertEqual(origin, {"kind": "human"})
+ self.assertEqual(client.query.await_args.kwargs, {"session_id": runtime_key})
self.assertEqual(request.message, "hello")
self.assertIn(runtime_key, agent._pending_requests)
self.assertIn(runtime_key, agent._pending_reactions)
@@ -2468,7 +2502,9 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
return _iterate()
- await agent._receive_messages(_FailingClient(), "session-1", "/tmp/work", context)
+ client = controller.claude_sessions[composite_key]
+ client.receive_messages = _FailingClient().receive_messages
+ await agent._receive_messages(client, "session-1", "/tmp/work", context)
controller.agent_auth_service.maybe_emit_auth_recovery_message.assert_awaited_once()
agent.session_handler.handle_session_error.assert_not_awaited()
@@ -2477,6 +2513,7 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
current_receiver_task=asyncio.current_task(),
activation_retired=False,
reason="receiver_error_auth_failure",
+ expected_client=client,
)
self.assertFalse(agent._pending_requests.get(composite_key))
@@ -3028,9 +3065,10 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
"OAuth login failed; Claude process terminated: SIGABRT"
),
)
- controller.claude_sessions[composite_key] = SimpleNamespace(
+ client = SimpleNamespace(
_transport=SimpleNamespace(_process=SimpleNamespace(returncode=-6)),
)
+ controller.claude_sessions[composite_key] = client
controller.receiver_tasks[composite_key] = asyncio.current_task()
await agent._handle_receiver_eof(composite_key, stale_context)
@@ -3042,6 +3080,7 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
current_receiver_task=asyncio.current_task(),
activation_retired=False,
reason="receiver_auth_failure",
+ expected_client=client,
)
self.assertFalse(agent._pending_requests.get(composite_key))
@@ -3199,14 +3238,16 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
return _iterate()
- await agent._receive_messages(_Client(), "session-1", "/tmp/work", context)
+ client = controller.claude_sessions[composite_key]
+ client.receive_messages = _Client().receive_messages
+ await agent._receive_messages(client, "session-1", "/tmp/work", context)
controller.agent_auth_service.maybe_emit_auth_recovery_message.assert_awaited_once()
controller.session_handler.cleanup_session.assert_awaited_once_with(
composite_key,
current_receiver_task=asyncio.current_task(),
activation_retired=False,
- reason="transport_auth_failure",
+ reason="result_auth_failure",
)
self.assertNotIn(composite_key, controller.receiver_tasks)
self.assertNotIn(composite_key, controller.claude_sessions)
@@ -3821,14 +3862,16 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
return _iterate()
- await agent._receive_messages(_Client(), "session-1", "/tmp/work", context)
+ client = controller.claude_sessions[composite_key]
+ client.receive_messages = _Client().receive_messages
+ await agent._receive_messages(client, "session-1", "/tmp/work", context)
controller.agent_auth_service.maybe_emit_auth_recovery_message.assert_awaited_once()
controller.session_handler.cleanup_session.assert_awaited_once_with(
composite_key,
current_receiver_task=asyncio.current_task(),
activation_retired=False,
- reason="transport_auth_failure",
+ reason="assistant_auth_failure",
)
self.assertEqual(agent._remove_ack_reaction.await_count, 2)
self.assertEqual(agent._remove_ack_reaction.await_args_list[0].args, (pending_request_1,))
@@ -3871,7 +3914,9 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
return _iterate()
- await agent._receive_messages(_Client(), "session-1", "/tmp/work", context)
+ client = controller.claude_sessions[composite_key]
+ client.receive_messages = _Client().receive_messages
+ await agent._receive_messages(client, "session-1", "/tmp/work", context)
controller.agent_auth_service.maybe_emit_auth_recovery_message.assert_awaited_once_with(
context,
@@ -4471,7 +4516,9 @@ class ClaudeAgentSessionTests(unittest.IsolatedAsyncioTestCase):
return _iterate()
- await agent._receive_messages(_Client(), "session-1", "/tmp/work", context)
+ client = controller.claude_sessions[composite_key]
+ client.receive_messages = _Client().receive_messages
+ await agent._receive_messages(client, "session-1", "/tmp/work", context)
controller.agent_auth_service.maybe_emit_auth_recovery_message.assert_awaited_once()
self.assertNotIn(composite_key, controller.receiver_tasks)
diff --git a/tests/test_claude_output_recovery_ledger.py b/tests/test_claude_output_recovery_ledger.py
new file mode 100644
index 000000000..bcb0a3f1f
--- /dev/null
+++ b/tests/test_claude_output_recovery_ledger.py
@@ -0,0 +1,1571 @@
+"""Consumer boundaries for phase-owned recovery; no native service or SDK calls."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from core.runtime_activation import RuntimeActivationRegistry
+from core.session_activities import SessionActivityRegistry, activity_completion_output
+from modules.agents.base import AgentRequest
+from modules.agents.claude_agent import ClaudeAgent
+from modules.agents.service import AgentService
+from modules.claude_sdk_compat import TextBlock
+from modules.im import MessageContext
+from storage.db import create_sqlite_engine
+from storage.importer import ensure_sqlite_state
+from storage.session_activities import SQLiteSessionActivityStore
+from tests.test_claude_agent_initiated_turn import (
+ AssistantMessage,
+ ResultMessage,
+ _build_agent,
+ _ActivityDeliveryClient,
+ _install_activity_dispatcher,
+ _wait_until,
+)
+
+
+def context_for(key, token=""):
+ return SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": key,
+ "agent_runtime_turn_token": token,
+ "turn_token": token,
+ },
+ )
+
+
[email protected]("ending", ["stop", "eof", "error", "replacement"])
+async def test_generation_end_preserves_terminal_recovery_owner_and_gate(ending):
+ agent, service = _build_agent()
+ key = "recovery-stop:/tmp/work"
+ context = context_for(key)
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+ agent._get_formatter = lambda _ctx: SimpleNamespace(
+ format_assistant_message=lambda parts: "\n".join(parts),
+ )
+ terminal_seen, stream_wait, allow_delivery = (
+ asyncio.Event(),
+ asyncio.Event(),
+ asyncio.Event(),
+ )
+ attempts = []
+
+ async def emit(_context, text, **kwargs):
+ attempts.append((text, kwargs["output"]))
+ if not allow_delivery.is_set():
+ raise RuntimeError("outbound unavailable")
+ return "durably-accepted"
+
+ async def controller_emit(ctx, _kind, _text, **kwargs):
+ if kwargs["output"].completes_turn:
+ service.release_runtime_turn(ctx)
+ return "terminal"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit)
+ agent.controller.emit_agent_message = AsyncMock(side_effect=controller_emit)
+ agent._handle_receiver_exception = ClaudeAgent._handle_receiver_exception.__get__(agent)
+
+ class Client:
+ interrupt = AsyncMock()
+ disconnect = AsyncMock()
+
+ def receive_messages(self):
+ async def frames():
+ assistant = AssistantMessage()
+ assistant.content = [TextBlock(text="retained result")]
+ yield assistant
+ result = ResultMessage()
+ result.result = "retained result"
+ result.origin = {"kind": "task-notification"}
+ yield result
+ terminal_seen.set()
+ await stream_wait.wait()
+ if ending == "error":
+ raise RuntimeError("native connection lost")
+
+ return frames()
+
+ client = Client()
+ agent.claude_sessions[key] = client
+ receiver = asyncio.create_task(
+ agent._receive_messages(client, "recovery-stop", "/tmp/work", context, composite_key=key)
+ )
+ agent.receiver_tasks[key] = receiver
+ try:
+ await asyncio.wait_for(terminal_seen.wait(), timeout=1)
+ owner = agent._pending_requests[key][0]
+ original_token = service._get_turn_gate(key).token
+ stop = AgentRequest(
+ context=context_for(key),
+ message="",
+ user_message="",
+ working_path="/tmp/work",
+ base_session_id="recovery-stop",
+ composite_session_id=key,
+ session_key="session-key",
+ )
+ if ending == "stop":
+ assert await service.handle_stop("claude", stop)
+ elif ending == "replacement":
+ await agent._cleanup_runtime_session(key, expected_client=client)
+ else:
+ stream_wait.set()
+ await asyncio.wait_for(receiver, timeout=1)
+ assert key not in agent.claude_sessions
+ assert service.runtime_turn_active(key)
+ assert service._get_turn_gate(key).token == original_token
+ assert agent._pending_requests[key] == [owner]
+ agent.controller.emit_agent_message.assert_not_awaited()
+ allow_delivery.set()
+ await asyncio.wait_for(_wait_until(lambda: not agent._has_pending_requests(key)), timeout=1)
+ assert not service.runtime_turn_active(key)
+ assert not agent._output_records_for_runtime(key)
+ assert {text for text, _ in attempts} == {"retained result"}
+ assert len({output.idempotency_key for _, output in attempts}) == 1
+ successor = context_for(key)
+ assert await service.begin_agent_initiated_turn("claude", successor, key)
+ service.release_runtime_turn(successor)
+ finally:
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry is not None:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
+
+
[email protected]("synthetic_successor", [False, True])
+async def test_human_phase_and_successor_cannot_overwrite_or_be_released_by_old_retry(synthetic_successor):
+ agent, service = _build_agent()
+ key = "retained-with-human:/tmp/work"
+ context = context_for(key)
+ await service.begin_agent_initiated_turn("claude", context, key)
+ human = AgentRequest(
+ context=context,
+ message="human input",
+ user_message="human input",
+ working_path="/tmp/work",
+ base_session_id="retained-with-human",
+ composite_session_id=key,
+ session_key="session-key",
+ )
+ agent._pending_requests[key] = [human]
+ agent._adopt_pending_turn_token = ClaudeAgent._adopt_pending_turn_token
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+ agent._get_formatter = lambda _: SimpleNamespace(
+ format_assistant_message=lambda parts: "\n".join(parts),
+ )
+ frames_done, release, recover = asyncio.Event(), asyncio.Event(), asyncio.Event()
+ detached_attempts, human_emits = [], []
+
+ async def emit(ctx, text, **kwargs):
+ output = kwargs.get("output")
+ if output is not None and output.detached:
+ detached_attempts.append((text, output))
+ if not recover.is_set():
+ raise RuntimeError("detached delivery unavailable")
+ else:
+ human_emits.append(text)
+ service.release_runtime_turn(ctx)
+ return "accepted"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit)
+
+ class Client:
+ def receive_messages(self):
+ async def frames():
+ detached = ResultMessage()
+ detached.origin = {"kind": "task-notification"}
+ detached.result = "old detached payload"
+ yield detached
+ assistant = AssistantMessage()
+ assistant.content = [TextBlock(text="new human answer")]
+ yield assistant
+ human_result = ResultMessage()
+ human_result.result = "new human answer"
+ yield human_result
+ frames_done.set()
+ await release.wait()
+
+ return frames()
+
+ client = Client()
+ agent.claude_sessions[key] = client
+ receiver = asyncio.create_task(
+ agent._receive_messages(client, "retained-with-human", "/tmp/work", context, composite_key=key)
+ )
+ try:
+ await asyncio.wait_for(frames_done.wait(), timeout=1)
+ assert human_emits == ["new human answer"]
+ assert not agent._has_pending_requests(key)
+ successor_context = context_for(key)
+ token = await service.begin_agent_initiated_turn("claude", successor_context, key)
+ assert token
+ successor = SimpleNamespace(context=successor_context)
+ agent._pending_requests[key] = [successor]
+ if synthetic_successor:
+ successor._claude_synthetic_owner = True
+ agent._synthetic_pending_owners[key] = successor
+ recover.set()
+ await asyncio.wait_for(
+ _wait_until(lambda: not agent._output_records_for_runtime(key)),
+ timeout=1,
+ )
+ assert {text for text, _ in detached_attempts} == {"old detached payload"}
+ assert len({output.idempotency_key for _, output in detached_attempts}) == 1
+ assert agent._pending_requests[key] == [successor]
+ assert service._get_turn_gate(key).token == token
+ assert service.runtime_turn_active(key)
+ service.release_runtime_turn(successor_context)
+ finally:
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry is not None:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
+
+
+def activity_store(tmp_path):
+ db_path = tmp_path / "state" / "vibe.sqlite"
+ ensure_sqlite_state(db_path=db_path, primary_platform="avibe")
+ engine = create_sqlite_engine(db_path)
+ yield SQLiteSessionActivityStore(engine)
+ engine.dispose()
+
+
[email protected]("ending", ["eof", "error"])
[email protected]("replace_while_waiting", [False, True])
+async def test_failure_replay_revalidates_after_steering_wait(ending, replace_while_waiting):
+ """A preflight check cannot authorize replay after cleanup wins the lock."""
+ from tests.test_claude_result_provenance import TaskStartedMessage, _failure_assistant
+
+ agent, service = _build_agent()
+ key = "old-replay:/tmp/work"
+ activations = RuntimeActivationRegistry()
+ service.activation_registry = activations
+ old_identity = activations.attach("claude", key)
+ context = context_for(key)
+ assert await service.begin_agent_initiated_turn("claude", context, key, activation_identity=old_identity)
+ agent._pending_requests[key] = [SimpleNamespace(context=context)]
+ agent._adopt_pending_turn_token = ClaudeAgent._adopt_pending_turn_token
+ agent._handle_assistant_terminal_failure = ClaudeAgent._handle_assistant_terminal_failure.__get__(agent)
+ agent._handle_receiver_eof = ClaudeAgent._handle_receiver_eof.__get__(agent)
+ agent.record_model_hub_native_failure = AsyncMock()
+ agent._remove_ack_reaction = AsyncMock()
+ agent.controller.agent_auth_service = SimpleNamespace(maybe_emit_auth_recovery_message=AsyncMock(return_value=False))
+ agent.session_handler.handle_session_error = AsyncMock(return_value=True)
+ ready, end = asyncio.Event(), asyncio.Event()
+ emissions = []
+
+ async def emit(ctx, kind, text, **kwargs):
+ emissions.append((
+ kind, text, ctx.platform_specific["agent_runtime_turn_token"],
+ kwargs["output"].completes_turn,
+ ))
+ if kwargs["output"].completes_turn:
+ service.release_runtime_turn(ctx)
+ return "accepted"
+
+ agent.controller.emit_agent_message = AsyncMock(side_effect=emit)
+
+ class Client:
+ _vibe_runtime_activation_identity = old_identity
+ disconnect = AsyncMock()
+
+ def receive_messages(self):
+ async def stream():
+ yield TaskStartedMessage("old-background")
+ yield _failure_assistant("OLD PHASE FAILURE")
+ ready.set()
+ await end.wait()
+ if ending == "error":
+ raise RuntimeError("old transport disconnected")
+ return stream()
+
+ old = Client()
+ agent.claude_sessions[key] = old
+ receiver = asyncio.create_task(agent._receive_messages(old, "old-replay", "/tmp/work", context, composite_key=key))
+ lock = agent._steering_lock(key)
+ replacement = None
+ new_context = context_for(key)
+ successor = SimpleNamespace(context=new_context)
+
+ async def replace_receiver():
+ await agent._cleanup_runtime_session(key, expected_client=old)
+ service.release_runtime_turn(context)
+ identity = activations.attach("claude", key)
+ agent.claude_sessions[key] = SimpleNamespace(_vibe_runtime_activation_identity=identity)
+ assert await service.begin_agent_initiated_turn("claude", new_context, key, activation_identity=identity)
+ agent._pending_requests[key] = [successor]
+
+ try:
+ await asyncio.wait_for(ready.wait(), 1)
+ assert agent._buffered_assistant_messages[key]
+ if replace_while_waiting:
+ await lock.acquire()
+ replacement = asyncio.create_task(replace_receiver())
+ await asyncio.wait_for(_wait_until(lambda: len(lock._waiters or ()) == 1), 1)
+ end.set()
+ await asyncio.wait_for(_wait_until(lambda: len(lock._waiters or ()) == 2), 1)
+ # Cleanup precedes replay in the lock FIFO after replay preflight.
+ lock.release()
+ await asyncio.wait_for(replacement, 1)
+ else:
+ end.set()
+ await asyncio.wait_for(receiver, 1)
+ if replace_while_waiting:
+ assert agent._pending_requests.get(key) == [successor]
+ assert service.runtime_turn_active(key)
+ assert service._get_turn_gate(key).token == new_context.platform_specific["agent_runtime_turn_token"]
+ assert emissions == []
+ else:
+ assert len([item for item in emissions if item[3]]) == 1
+ assert len([item for item in emissions if item[1]]) == 1
+ assert "OLD PHASE FAILURE" in emissions[0][1]
+ assert not service.runtime_turn_active(key)
+ finally:
+ if lock.locked():
+ lock.release()
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+ if replacement is not None:
+ replacement.cancel()
+ await asyncio.gather(replacement, return_exceptions=True)
+ service.release_runtime_turn(new_context)
+ service.release_runtime_turn(context)
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
+
+
[email protected]("local_error", [False, True])
[email protected]("accepted_message", [False, True])
+def test_delivered_terminal_evidence_is_acknowledged_before_restart(
+ activity_store, local_error, accepted_message,
+):
+ """Successful Run-policy acknowledgement must remove its terminal evidence."""
+ agent, service = _build_agent()
+ registry = SessionActivityRegistry(activity_store)
+ service.activities = registry
+ settled = []
+ agent.controller.scheduled_task_service = SimpleNamespace(
+ settle_activity_runs=lambda activity: settled.append(activity.id),
+ )
+ key = "delivered-terminal"
+ registry.start(
+ backend="claude", runtime_key=key, session_id="ses-probe", activity_id="task-one",
+ kind="background_task", run_id="run-one",
+ )
+ registry.complete(
+ backend="claude", runtime_key=key, activity_id="task-one", status="completed", expects_output=True,
+ )
+ batch = registry.claim_completed_output_batch("claude", key)
+ output = activity_completion_output(batch[0], activities=batch, detached=True, completes_turn=False)
+ assert registry.settle_completed_output_batch(
+ output, accepted_message_exists=accepted_message,
+ settlement_error=RuntimeError("local finalization failed") if local_error else None,
+ settle_terminal=service.on_activity_terminal,
+ )
+ assert not registry.claimed_completed_output_batch_for_output(output)
+ assert activity_store.list_activities() == []
+ restarted = SessionActivityRegistry(activity_store)
+ assert restarted.drain_recovered_terminals() == []
+ assert not restarted.has_backend_work("claude")
+ assert settled == (["task-one"] if local_error else [])
+
+
[email protected]("fail_binding", [False, True])
[email protected]("ending", ["live", "eof", "error", "stop", "replacement"])
+async def test_unowned_terminal_text_survives_claim_failure(
+ activity_store, monkeypatch, fail_binding, ending,
+):
+ """The consumed Result, not a CLI summary, owns text across receipt failure."""
+ from tests.test_claude_result_provenance import ResultMessage, TaskNotificationMessage, TaskStartedMessage
+
+ agent, service = _build_agent()
+ registry = SessionActivityRegistry(activity_store)
+ service.activities = registry
+ key = "unowned-result:/tmp/work"
+ context = context_for(key)
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+ agent._adopt_pending_turn_token = ClaudeAgent._adopt_pending_turn_token
+ original = activity_store.upsert_activity
+ allow_binding = asyncio.Event()
+ binding_attempts = []
+
+ def persist(activity, *, phase):
+ if activity["metadata"].get("output_batch_id"):
+ binding_attempts.append((
+ activity["id"], activity["metadata"]["output_batch_id"],
+ tuple(activity["metadata"]["output_batch_activity_ids"]),
+ ))
+ if fail_binding and not allow_binding.is_set():
+ raise RuntimeError("receipt binding unavailable")
+ original(activity, phase=phase)
+
+ monkeypatch.setattr(activity_store, "upsert_activity", persist)
+ payloads = []
+
+ async def accepted(ctx, kind, text, **kwargs):
+ payloads.append((text, kwargs["output"]))
+ output = kwargs["output"]
+ assert registry.settle_completed_output_batch(output, accepted_message_exists=True)
+ if output.completes_turn:
+ service.release_runtime_turn(ctx)
+ return "accepted-output"
+
+ async def emit_result(ctx, text, **kwargs):
+ return await accepted(ctx, "result", text, **kwargs)
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+ agent.controller.emit_agent_message = AsyncMock(side_effect=accepted)
+ after_result, hold = asyncio.Event(), asyncio.Event()
+
+ class Client:
+ interrupt = AsyncMock()
+ disconnect = AsyncMock()
+
+ def receive_messages(self):
+ async def stream():
+ yield TaskStartedMessage("task-one")
+ yield TaskNotificationMessage("task-one", "CLI receipt, not answer")
+ yield ResultMessage("ACTUAL BACKGROUND ANSWER", origin={"kind": "task-notification"})
+ after_result.set()
+ await hold.wait()
+ if ending == "error":
+ raise RuntimeError("receiver disconnected during binding recovery")
+ return stream()
+
+ client = Client()
+ agent.claude_sessions[key] = client
+ receiver = asyncio.create_task(agent._receive_messages(client, "unowned-result", "/tmp/work", context, composite_key=key))
+ agent.receiver_tasks[key] = receiver
+ try:
+ await asyncio.wait_for(after_result.wait(), 1)
+ if fail_binding:
+ await asyncio.wait_for(_wait_until(lambda: len(binding_attempts) >= 2), 1)
+ records = agent._output_records_for_runtime(key)
+ assert len(records) == 1
+ record = records[0]
+ assert record.lifecycle == "delivery_pending"
+ assert record.text == "ACTUAL BACKGROUND ANSWER"
+ assert [activity.id for activity in record.activities] == ["task-one"]
+ assert record.output.activity_batch_id == binding_attempts[0][1]
+ retained_identity = record.output.idempotency_key
+ assert not payloads
+ assert service.runtime_turn_active(key)
+ assert agent._pending_requests[key] == [record.request]
+ if ending in {"eof", "error"}:
+ hold.set()
+ await asyncio.wait_for(receiver, 1)
+ elif ending == "replacement":
+ await agent._cleanup_runtime_session(key, expected_client=client)
+ elif ending == "stop":
+ stop = AgentRequest(
+ context=context_for(key), message="", user_message="", working_path="/tmp/work",
+ base_session_id="unowned-result", composite_session_id=key, session_key="session-key",
+ )
+ assert await service.handle_stop("claude", stop)
+ if ending != "live":
+ assert key not in agent.claude_sessions
+ # Repeated retirement cannot remove the receipt recovery owner.
+ await agent._cleanup_runtime_session(key, expected_client=client)
+ assert agent._pending_requests[key] == [record.request]
+ assert service.runtime_turn_active(key)
+ allow_binding.set()
+ await asyncio.wait_for(_wait_until(lambda: bool(payloads)), 1)
+ assert [text for text, _ in payloads] == ["ACTUAL BACKGROUND ANSWER"]
+ assert payloads[0][1].activity_ids == ("task-one",)
+ assert len(set(binding_attempts)) == 1
+ assert payloads[0][1].activity_batch_id == binding_attempts[0][1]
+ if fail_binding:
+ assert payloads[0][1].idempotency_key == retained_identity
+ if ending == "live" or not fail_binding:
+ assert not receiver.done()
+ assert not agent._has_pending_requests(key)
+ assert not service.runtime_turn_active(key)
+ assert not registry.has_claimed_output("claude", key)
+ assert activity_store.list_activities() == []
+ successor = context_for(key)
+ assert await service.begin_agent_initiated_turn("claude", successor, key)
+ service.release_runtime_turn(successor)
+ finally:
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
+
+
[email protected]("ending", ["eof", "error"])
[email protected]("fault", ["before_emit", "after_pop"])
[email protected]("retirement", ["stop", "replacement"])
+async def test_receiver_end_revalidates_after_replay_before_fallback(ending, fault, retirement):
+ """Replay's lock check does not authorize its caller after another await."""
+ from tests.test_claude_result_provenance import TaskStartedMessage, _failure_assistant
+
+ agent, service = _build_agent()
+ key = "replay-fallback:/tmp/work"
+ activations = RuntimeActivationRegistry()
+ service.activation_registry = activations
+ old_identity = activations.attach("claude", key)
+ context = context_for(key)
+ assert await service.begin_agent_initiated_turn("claude", context, key, activation_identity=old_identity)
+ request = AgentRequest(
+ context=context, message="human", user_message="human", working_path="/tmp/work",
+ base_session_id="replay-fallback", composite_session_id=key, session_key="session-key",
+ )
+ agent._pending_requests[key] = [request]
+ agent._adopt_pending_turn_token = ClaudeAgent._adopt_pending_turn_token
+ agent._handle_assistant_terminal_failure = ClaudeAgent._handle_assistant_terminal_failure.__get__(agent)
+ agent._handle_receiver_eof = ClaudeAgent._handle_receiver_eof.__get__(agent)
+ agent.record_model_hub_native_failure = AsyncMock(
+ side_effect=[RuntimeError("pre-consumption failure"), None] if fault == "before_emit" else None,
+ )
+ agent._remove_ack_reaction = AsyncMock(
+ side_effect=[RuntimeError("post-consumption failure"), None] if fault == "after_pop" else None,
+ )
+ agent.controller.agent_auth_service = SimpleNamespace(maybe_emit_auth_recovery_message=AsyncMock(return_value=False))
+ agent.session_handler.handle_session_error = AsyncMock(return_value=True)
+ replay_finished, return_replay = asyncio.Event(), asyncio.Event()
+ replay = agent._replay_buffered_terminal_failures
+
+ async def hold_replay(*args, **kwargs):
+ result = await replay(*args, **kwargs)
+ assert result.replay_failed
+ replay_finished.set()
+ await return_replay.wait()
+ return result
+
+ agent._replay_buffered_terminal_failures = hold_replay
+ terminals = []
+
+ async def accepted(ctx, kind, text, **kwargs):
+ if kwargs["output"].completes_turn:
+ terminals.append((text, ctx.platform_specific["agent_runtime_turn_token"]))
+ service.release_runtime_turn(ctx)
+ return "accepted"
+
+ agent.controller.emit_agent_message = AsyncMock(side_effect=accepted)
+
+ class Client:
+ _vibe_runtime_activation_identity = old_identity
+ interrupt = AsyncMock()
+ disconnect = AsyncMock()
+
+ def receive_messages(self):
+ async def frames():
+ yield TaskStartedMessage("task")
+ yield _failure_assistant("old failure")
+ if ending == "error":
+ raise RuntimeError("old receiver failed")
+ return frames()
+
+ old = Client()
+ agent.claude_sessions[key] = old
+ receiver = asyncio.create_task(agent._receive_messages(old, "replay-fallback", "/tmp/work", context, composite_key=key))
+ new_context = context_for(key)
+ try:
+ await asyncio.wait_for(replay_finished.wait(), 1)
+ if retirement == "stop":
+ # EOF/error already closed native admission. Stop cannot steal its
+ # terminal claim; replacement still may retire that exact client.
+ assert not await service.handle_stop("claude", request)
+ await agent._cleanup_runtime_session(key, expected_client=old)
+ service.release_runtime_turn(context)
+ previous_terminals = list(terminals)
+ identity = activations.attach("claude", key)
+ new_client = SimpleNamespace(_vibe_runtime_activation_identity=identity)
+ agent.claude_sessions[key] = new_client
+ assert await service.begin_agent_initiated_turn("claude", new_context, key, activation_identity=identity)
+ successor = SimpleNamespace(context=new_context)
+ agent._pending_requests[key] = [successor]
+ return_replay.set()
+ await asyncio.wait_for(receiver, 1)
+ assert agent.claude_sessions[key] is new_client
+ assert agent._pending_requests[key] == [successor]
+ assert service.runtime_turn_active(key)
+ assert service._get_turn_gate(key).token == new_context.platform_specific["agent_runtime_turn_token"]
+ assert terminals == previous_terminals
+ assert len(terminals) == (1 if fault == "after_pop" else 0)
+ finally:
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+ service.release_runtime_turn(new_context)
+ service.release_runtime_turn(context)
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
+
+
[email protected]("bound_receipt", [False, True])
+async def test_detached_claim_retry_keeps_batch_boundary_with_real_dispatcher(
+ activity_store, monkeypatch, bound_receipt,
+):
+ """Later completions cannot join a retained receipt or lose their own flush."""
+ from tests.test_claude_result_provenance import ResultMessage
+
+ agent, service = _build_agent()
+ registry = SessionActivityRegistry(activity_store)
+ service.activities = registry
+ delivery = _ActivityDeliveryClient(fail_delivery=True)
+ _install_activity_dispatcher(agent, delivery)
+ key = "batch-claim-retry:/tmp/work"
+ context = MessageContext(
+ user_id="U1", channel_id="C1", platform="discord",
+ platform_specific={"agent_runtime_turn_key": key, "agent_session_id": "ses-batch-retry"},
+ )
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+
+ def complete(activity_id, summary=""):
+ registry.start(
+ backend="claude", runtime_key=key, session_id="ses-batch-retry",
+ activity_id=activity_id, kind="background_task", turn_id="origin-turn",
+ )
+ registry.complete(
+ backend="claude", runtime_key=key, activity_id=activity_id,
+ status="completed", expects_output=True, metadata={"summary": summary},
+ )
+
+ complete("one")
+ complete("two")
+ prior_output = None
+ if bound_receipt:
+ batch = registry.claim_completed_output_batch("claude", key)
+ prior_output = activity_completion_output(batch[-1], activities=batch, detached=True, completes_turn=False)
+ assert registry.requeue_completed_outputs(batch) == 2
+ allow_binding = asyncio.Event()
+ binding_attempts = []
+ original = activity_store.upsert_activities
+
+ def persist_batch(activities, *, phase):
+ binding_attempts.append(tuple(
+ (activity["id"], activity["metadata"]["output_batch_id"]) for activity in activities
+ ))
+ if not allow_binding.is_set():
+ raise RuntimeError("atomic batch binding unavailable")
+ return original(activities, phase=phase)
+
+ monkeypatch.setattr(activity_store, "upsert_activities", persist_batch)
+ terminal_seen, hold = asyncio.Event(), asyncio.Event()
+
+ class Client:
+ def receive_messages(self):
+ async def frames():
+ yield ResultMessage("EXACT TERMINAL TEXT", origin={"kind": "task-notification"})
+ terminal_seen.set()
+ await hold.wait()
+ return frames()
+
+ client = Client()
+ agent.claude_sessions[key] = client
+ # The real dispatcher owns Activity settlement. Message mirroring is
+ # disabled here, so durable evidence comes from the real SQLite receipts.
+ with (
+ patch("core.message_dispatcher.persist_agent_message", return_value=None),
+ patch("core.message_dispatcher.agent_message_exists", return_value=False),
+ ):
+ receiver = asyncio.create_task(agent._receive_messages(client, "batch-claim-retry", "/tmp/work", context, composite_key=key))
+ try:
+ await asyncio.wait_for(terminal_seen.wait(), 1)
+ record = agent._output_records_for_runtime(key)[0]
+ output_identity = record.output.idempotency_key
+ if bound_receipt:
+ assert output_identity == prior_output.idempotency_key
+ else:
+ await asyncio.wait_for(_wait_until(lambda: len(binding_attempts) >= 2), 1)
+ assert [activity.id for activity in record.activities] == ["one", "two"]
+ complete("three", "later task output")
+ allow_binding.set()
+ await asyncio.wait_for(_wait_until(lambda: not record.claim_pending), 1)
+ assert record.output.idempotency_key == output_identity
+ assert record.output.activity_ids == ("one", "two")
+ assert registry.has_claimed_output("claude", key)
+ assert delivery.sent == []
+ delivery.fail_delivery = False
+ await asyncio.wait_for(_wait_until(lambda: len(delivery.sent) == 2), 1)
+ assert delivery.sent == ["EXACT TERMINAL TEXT", "later task output"]
+ if not bound_receipt:
+ assert len(set(binding_attempts)) == 1
+ assert record.output.idempotency_key == output_identity
+ assert not registry.has_claimed_output("claude", key)
+ assert not registry.has_completed_output("claude", key)
+ assert activity_store.list_activities() == []
+ assert SessionActivityRegistry(activity_store).drain_recovered_terminals() == []
+ assert not service.runtime_turn_active(key)
+ assert not agent._has_pending_requests(key)
+ assert not agent._output_records_for_runtime(key)
+ finally:
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
+
+
[email protected]("status", ["completed", "failed", "stopped", "killed"])
+def test_later_completion_retires_failed_active_provenance_write(
+ activity_store, monkeypatch, status,
+):
+ registry = SessionActivityRegistry(activity_store)
+ key = "provenance-lifecycle"
+ registry.start(
+ backend="claude", runtime_key=key, session_id="ses-1",
+ activity_id="task-one", kind="background_task",
+ metadata={"provenance_pending": True},
+ )
+ original = activity_store.upsert_activity
+ fault = "classification"
+
+ def persist(activity, *, phase):
+ if fault == "classification" and activity["metadata"].get("provenance_human"):
+ raise RuntimeError("classification unavailable")
+ if fault == "binding" and activity["metadata"].get("output_batch_id"):
+ raise RuntimeError("binding unavailable")
+ original(activity, phase=phase)
+
+ monkeypatch.setattr(activity_store, "upsert_activity", persist)
+ registry.classify_provisional_provenance(
+ "claude", key, activity_ids={"task-one"}, parent_activity_ids=set(),
+ turn_id="human-turn", run_ids=["human-run"], detached=False,
+ )
+ assert registry.provenance_persistence_recovery("claude", key)
+ fault = ""
+ registry.complete(
+ backend="claude", runtime_key=key, activity_id="task-one", status=status,
+ expects_output=status == "completed", retain_terminal_snapshot=True,
+ )
+ assert registry.provenance_persistence_recovery("claude", key) == {}
+ expected_phase = "awaiting_output" if status == "completed" else "terminal"
+ assert activity_store.list_activities()[0]["phase"] == expected_phase
+ if status == "completed":
+ fault = "binding"
+ with pytest.raises(RuntimeError, match="binding unavailable"):
+ registry.claim_completed_output_batch("claude", key)
+ assert activity_store.list_activities()[0]["phase"] == "awaiting_output"
+ fault = ""
+ restarted = SessionActivityRegistry(activity_store)
+ if status == "completed":
+ restored = restarted.claim_completed_output_batch("claude", key)
+ assert [item.id for item in restored] == ["task-one"]
+ output = activity_completion_output(
+ restored[0], activities=restored, detached=True, completes_turn=False,
+ )
+ assert restarted.settle_completed_output_batch(output, accepted_message_exists=True)
+ else:
+ restored = restarted.drain_recovered_terminals()
+ assert [item.status for item in restored] == [status]
+ restarted.ack_recovered_terminal(restored[0])
+ assert activity_store.list_activities() == []
+
+
[email protected]("owner", ["active", "queued", "claimed", "terminal"])
+def test_provenance_retry_phase_comes_from_current_owner_not_error_text(
+ activity_store, owner,
+):
+ registry = SessionActivityRegistry(activity_store)
+ key = "provenance-owner"
+ activity = registry.start(
+ backend="claude", runtime_key=key, session_id="ses-1",
+ activity_id="task-one", kind="background_task",
+ )
+ if owner != "active":
+ activity = registry.complete(
+ backend="claude", runtime_key=key, activity_id=activity.id,
+ status="failed" if owner == "terminal" else "completed",
+ expects_output=owner != "terminal", retain_terminal_snapshot=True,
+ )
+ if owner == "claimed":
+ activity = registry.claim_completed_output_batch("claude", key)[0]
+ # A recorded attempt is diagnostic evidence, never lifecycle authority.
+ registry._record_provenance_persistence_recovery(
+ activity, phase="active", error=RuntimeError("older attempt"),
+ )
+ registry.classify_provisional_provenance(
+ "claude", key, activity_ids=set(), parent_activity_ids=set(), detached=False,
+ )
+ expected = {
+ "active": "active", "queued": "awaiting_output",
+ "claimed": "awaiting_output", "terminal": "terminal",
+ }[owner]
+ assert activity_store.list_activities()[0]["phase"] == expected
+ assert registry.provenance_persistence_recovery("claude", key) == {}
+
+
[email protected]("owner", ["active", "claimed", "terminal"])
+def test_authoritative_deletion_clears_provenance_retry_without_resurrection(
+ activity_store, monkeypatch, owner,
+):
+ registry = SessionActivityRegistry(activity_store)
+ key = "provenance-delete"
+ registry.start(
+ backend="claude", runtime_key=key, session_id="ses-1",
+ activity_id="task-one", kind="background_task",
+ metadata={"provenance_pending": True},
+ )
+ if owner != "active":
+ registry.complete(
+ backend="claude", runtime_key=key, activity_id="task-one",
+ status="failed" if owner == "terminal" else "completed",
+ expects_output=owner == "claimed", retain_terminal_snapshot=True,
+ )
+ if owner == "claimed":
+ registry.claim_completed_output_batch("claude", key)
+ original = activity_store.upsert_activity
+
+ def fail_classification(activity, *, phase):
+ if activity["metadata"].get("provenance_human"):
+ raise RuntimeError("classification unavailable")
+ original(activity, phase=phase)
+
+ monkeypatch.setattr(activity_store, "upsert_activity", fail_classification)
+ classified = registry.classify_provisional_provenance(
+ "claude", key, activity_ids={"task-one"}, parent_activity_ids=set(),
+ turn_id="human-turn", run_ids=["human-run"], detached=False,
+ )[0]
+ assert registry.provenance_persistence_recovery("claude", key)
+ if owner == "active":
+ registry.complete(
+ backend="claude", runtime_key=key, activity_id=classified.id,
+ status="killed",
+ )
+ elif owner == "terminal":
+ registry.ack_recovered_terminal(classified)
+ else:
+ output = activity_completion_output(
+ classified, activities=[classified], detached=True, completes_turn=False,
+ )
+ assert registry.settle_completed_output_batch(output, accepted_message_exists=True)
+ assert registry.provenance_persistence_recovery("claude", key) == {}
+ monkeypatch.setattr(activity_store, "upsert_activity", original)
+ assert registry.terminal_snapshots_for_runtime("claude", key) == []
+ assert activity_store.list_activities() == []
+ assert not SessionActivityRegistry(activity_store).has_backend_work("claude")
+
+
+def test_atomic_batch_binding_supersedes_only_successful_provenance_retries(
+ activity_store, monkeypatch,
+):
+ registry = SessionActivityRegistry(activity_store)
+ key = "provenance-batch"
+ for activity_id in ("one", "two"):
+ registry.start(
+ backend="claude", runtime_key=key, session_id="ses-1",
+ activity_id=activity_id, kind="background_task",
+ metadata={"provenance_pending": True},
+ )
+ registry.complete(
+ backend="claude", runtime_key=key, activity_id=activity_id,
+ status="completed", expects_output=True,
+ )
+ original_one = activity_store.upsert_activity
+ original_batch = activity_store.upsert_activities
+
+ def failed_one(*_args, **_kwargs):
+ raise RuntimeError("single write failed")
+
+ monkeypatch.setattr(activity_store, "upsert_activity", failed_one)
+ registry.classify_provisional_provenance(
+ "claude", key, activity_ids={"one", "two"}, parent_activity_ids=set(),
+ turn_id="human-turn", run_ids=["human-run"], detached=False,
+ )
+ assert set(registry.provenance_persistence_recovery("claude", key)) == {"one", "two"}
+
+ def failed_batch(*_args, **_kwargs):
+ raise RuntimeError("batch failed")
+
+ monkeypatch.setattr(activity_store, "upsert_activities", failed_batch)
+ with pytest.raises(RuntimeError, match="batch failed"):
+ registry.claim_completed_output_batch("claude", key)
+ assert set(registry.provenance_persistence_recovery("claude", key)) == {"one", "two"}
+ monkeypatch.setattr(activity_store, "upsert_activities", original_batch)
+ batch = registry.claim_completed_output_batch("claude", key)
+ assert [item.id for item in batch] == ["one", "two"]
+ assert registry.provenance_persistence_recovery("claude", key) == {}
+ monkeypatch.setattr(activity_store, "upsert_activity", original_one)
+ restarted = SessionActivityRegistry(activity_store)
+ restored = restarted.claim_completed_output_batch("claude", key)
+ assert [item.id for item in restored] == ["one", "two"]
+ assert {item.run_id for item in restored} == {"human-run"}
+ assert {item.metadata["output_batch_id"] for item in restored} == {
+ batch[0].metadata["output_batch_id"]
+ }
+
+
[email protected]("ending", ["eof", "error"])
[email protected]("fault", ["before_emit", "after_pop", "none"])
+async def test_buffered_failure_replay_settles_owned_turn_once_before_release(ending, fault):
+ from tests.test_claude_result_provenance import TaskStartedMessage, _failure_assistant
+
+ agent, service = _build_agent()
+ key = f"failure-replay-{ending}-{fault}:/tmp/work"
+ context = context_for(key)
+ assert await service.begin_agent_initiated_turn("claude", context, key)
+ request = SimpleNamespace(context=context, output=None, output_activities=[])
+ agent._pending_requests[key] = [request]
+ agent._handle_receiver_eof = ClaudeAgent._handle_receiver_eof.__get__(agent)
+ agent._handle_assistant_terminal_failure = (
+ ClaudeAgent._handle_assistant_terminal_failure.__get__(agent)
+ )
+ agent.controller.agent_auth_service = SimpleNamespace(
+ maybe_emit_auth_recovery_message=AsyncMock(return_value=False),
+ )
+ agent.session_handler.handle_session_error = AsyncMock(return_value=True)
+ agent.session_handler.cleanup_session = AsyncMock()
+ agent.record_model_hub_native_failure = AsyncMock(
+ side_effect=[RuntimeError("record failed"), None]
+ if fault == "before_emit" else None,
+ )
+ agent._remove_ack_reaction = AsyncMock(
+ side_effect=[RuntimeError("reaction failed"), None]
+ if fault == "after_pop" else None,
+ )
+ terminal = []
+
+ async def accepted(ctx, kind, text, **kwargs):
+ if kind == "result" and kwargs["output"].completes_turn:
+ terminal.append((ctx.platform_specific.copy(), kwargs))
+ assert service.runtime_turn_active(key)
+ service.release_runtime_turn(ctx)
+ return "accepted-output"
+
+ agent.controller.emit_agent_message = AsyncMock(side_effect=accepted)
+
+ class Client:
+ def receive_messages(self):
+ async def frames():
+ yield TaskStartedMessage("competing-task")
+ yield _failure_assistant("backend exploded")
+ if ending == "error":
+ raise RuntimeError("receiver disconnected")
+ return frames()
+
+ client = Client()
+ agent.claude_sessions[key] = client
+ await agent._receive_messages(
+ client, "failure-replay", "/tmp/work", context, composite_key=key,
+ )
+ assert len(terminal) == 1
+ assert terminal[0][0]["agent_runtime_turn_token"] == context.platform_specific[
+ "agent_runtime_turn_token"
+ ]
+ assert terminal[0][1]["is_error"]
+ assert not agent._has_pending_requests(key)
+ assert not service.runtime_turn_active(key)
+ # Repeated EOF cleanup has no terminal owner left and cannot emit again.
+ await agent._handle_receiver_eof(key, context)
+ assert len(terminal) == 1
+ successor = context_for(key)
+ assert await service.begin_agent_initiated_turn("claude", successor, key)
+ service.release_runtime_turn(successor)
+
+
[email protected]("retirement", ["stop", "replacement"])
+async def test_retired_receiver_cannot_replay_buffered_failure_against_successor(retirement):
+ from tests.test_claude_result_provenance import TaskStartedMessage, _failure_assistant
+
+ agent, service = _build_agent()
+ key = f"replay-retired-{retirement}:/tmp/work"
+ context = context_for(key)
+ assert await service.begin_agent_initiated_turn("claude", context, key)
+ request = AgentRequest(
+ context=context, message="human", user_message="human", working_path="/tmp/work",
+ base_session_id="replay-retired", composite_session_id=key, session_key="session-key",
+ )
+ agent._pending_requests[key] = [request]
+ agent._handle_receiver_eof = ClaudeAgent._handle_receiver_eof.__get__(agent)
+ agent._process_assistant_terminal_frame = AsyncMock(
+ wraps=agent._process_assistant_terminal_frame,
+ )
+ waiting, finish = asyncio.Event(), asyncio.Event()
+ terminals = []
+
+ async def accepted(ctx, kind, text, **kwargs):
+ if kind == "result" and kwargs["output"].completes_turn:
+ terminals.append(kwargs["output"])
+ service.release_runtime_turn(ctx)
+ return "accepted-output"
+
+ agent.controller.emit_agent_message = AsyncMock(side_effect=accepted)
+
+ class Client:
+ interrupt = AsyncMock()
+ disconnect = AsyncMock()
+
+ def receive_messages(self):
+ async def frames():
+ yield TaskStartedMessage("competing-task")
+ yield _failure_assistant("obsolete backend failure")
+ waiting.set()
+ await finish.wait()
+ return frames()
+
+ old = Client()
+ agent.claude_sessions[key] = old
+ receiver = asyncio.create_task(
+ agent._receive_messages(old, "replay-retired", "/tmp/work", context, composite_key=key)
+ )
+ try:
+ await asyncio.wait_for(waiting.wait(), timeout=1)
+ assert agent._buffered_assistant_messages[key]
+ if retirement == "stop":
+ assert await service.handle_stop("claude", request)
+ assert len(terminals) == 1
+ else:
+ service.release_runtime_turn(context)
+ successor_context = context_for(key)
+ assert await service.begin_agent_initiated_turn("claude", successor_context, key)
+ successor = SimpleNamespace(context=successor_context)
+ new_client = SimpleNamespace()
+ agent.claude_sessions[key] = new_client
+ agent._pending_requests[key] = [successor]
+ token = service._get_turn_gate(key).token
+ finish.set()
+ await asyncio.wait_for(receiver, timeout=1)
+ assert agent.claude_sessions[key] is new_client
+ assert agent._pending_requests[key] == [successor]
+ assert service.runtime_turn_active(key)
+ assert service._get_turn_gate(key).token == token
+ assert len(terminals) == (1 if retirement == "stop" else 0)
+ agent._process_assistant_terminal_frame.assert_not_awaited()
+ service.release_runtime_turn(successor_context)
+ finally:
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+
+
+def test_classification_run_ack_does_not_remove_output_receipt_across_restart(activity_store):
+ settled = []
+ service = AgentService(
+ SimpleNamespace(scheduled_task_service=SimpleNamespace(settle_activity_runs=settled.append)),
+ activities=SessionActivityRegistry(activity_store),
+ )
+ registry = service.activities
+ registry.start(
+ backend="claude",
+ runtime_key="runtime",
+ session_id="ses-1",
+ activity_id="queued",
+ kind="background_task",
+ metadata={"provenance_pending": True},
+ )
+ registry.complete(
+ backend="claude",
+ runtime_key="runtime",
+ activity_id="queued",
+ status="completed",
+ expects_output=True,
+ )
+ classified = registry.classify_provisional_provenance(
+ "claude",
+ "runtime",
+ activity_ids={"queued"},
+ parent_activity_ids=set(),
+ turn_id="human-turn",
+ run_ids=["human-run"],
+ delivery_key_external="human-delivery",
+ phase_id="human-phase",
+ detached=False,
+ )
+ assert service.on_activity_terminal(classified[0])
+ assert settled[0].run_id == "human-run"
+ assert activity_store.list_activities()[0]["phase"] == "awaiting_output"
+ batch = registry.claim_completed_output_batch("claude", "runtime")
+ output = activity_completion_output(batch[-1], activities=batch, detached=True, completes_turn=False)
+ restarted = SessionActivityRegistry(activity_store)
+ restored = restarted.claim_completed_output_batch("claude", "runtime")
+ restored_output = activity_completion_output(
+ restored[-1],
+ activities=restored,
+ detached=True,
+ completes_turn=False,
+ )
+ assert restored_output.idempotency_key == output.idempotency_key
+ assert restored_output.activity_ids == output.activity_ids
+ assert restored[0].run_id == "human-run"
+ assert restarted.settle_completed_output_batch(restored_output, accepted_message_exists=True)
+ assert activity_store.list_activities() == []
+
+
+def test_force_end_service_ack_removes_durable_terminal_once(activity_store):
+ settled = []
+ service = AgentService(
+ SimpleNamespace(scheduled_task_service=SimpleNamespace(settle_activity_runs=settled.append)),
+ activities=SessionActivityRegistry(activity_store),
+ )
+ for runtime in ("first", "second"):
+ service.activities.start(
+ backend="claude",
+ runtime_key=runtime,
+ session_id="ses-1",
+ activity_id="same-native-id",
+ kind="background_task",
+ run_id=runtime,
+ )
+ service.force_end_backend_activities("claude")
+ assert sorted(activity.run_id for activity in settled) == ["first", "second"]
+ assert not service.activities.has_backend_work("claude")
+ assert activity_store.list_activities() == []
+ assert SessionActivityRegistry(activity_store).drain_recovered_terminals() == []
+
+
+async def test_ledger_retries_local_receipt_without_external_redelivery(activity_store):
+ agent, service = _build_agent()
+ service.activities = SessionActivityRegistry(activity_store)
+ delivery = _ActivityDeliveryClient()
+ _install_activity_dispatcher(agent, delivery)
+ context = MessageContext(
+ user_id="U1",
+ channel_id="C1",
+ platform="discord",
+ platform_specific={"agent_session_id": "ses-local-only"},
+ )
+ key = "receipt-local-only:/tmp/work"
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="ses-local-only",
+ activity_id="task",
+ kind="background_task",
+ run_id="run-origin",
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=key,
+ activity_id="task",
+ status="completed",
+ expects_output=True,
+ )
+ batch = service.activities.claim_completed_output_batch("claude", key)
+ record = agent._retain_activity_output_record(key, batch)
+ agent._classify_output_record(record, context, text="selected payload")
+ receipt = record.output.idempotency_key
+ agent.controller.scheduled_task_service = SimpleNamespace(settle_activity_runs=lambda _: None)
+
+ class RunStore:
+ def get_run(self, _run_id):
+ return {"status": "running"}
+
+ def record_run_output(self, *_args, **_kwargs):
+ raise RuntimeError("Run storage unavailable")
+
+ def close(self):
+ pass
+
+ upsert = activity_store.upsert_activity
+
+ def fail_terminal(payload, *, phase):
+ if phase == "terminal":
+ raise RuntimeError("terminal receipt storage unavailable")
+ return upsert(payload, phase=phase)
+
+ try:
+ with (
+ patch("core.message_dispatcher.persist_agent_message", return_value=None),
+ patch("core.message_dispatcher.agent_message_exists", return_value=False),
+ patch("core.message_dispatcher.SQLiteBackgroundTaskStore", return_value=RunStore()),
+ ):
+ with (
+ patch.object(activity_store, "upsert_activity", side_effect=fail_terminal),
+ patch.object(activity_store, "delete_activity", side_effect=RuntimeError("delete unavailable")),
+ ):
+ assert await agent._flush_output_recovery(key, context)
+ assert await agent._flush_output_recovery(key, context)
+ assert delivery.sent == ["selected payload"]
+ assert service.activities.has_claimed_output("claude", key)
+ assert not await agent._flush_output_recovery(key, context)
+ assert delivery.sent == ["selected payload"]
+ assert record.output.idempotency_key == receipt
+ assert not service.activities.has_claimed_output("claude", key)
+ assert not agent._output_records_for_runtime(key)
+ finally:
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry is not None:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
+
+
+async def test_result_arriving_during_managed_retry_keeps_worker_responsibility():
+ agent, service = _build_agent()
+ key = "retry-interleaving:/tmp/work"
+ context = context_for(key)
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 0.01
+ agent._get_formatter = lambda _: SimpleNamespace(format_assistant_message=lambda parts: "\n".join(parts))
+ retry_entered, release_retry, later_seen, stay_open = (
+ asyncio.Event(),
+ asyncio.Event(),
+ asyncio.Event(),
+ asyncio.Event(),
+ )
+ attempts, delivered = [], []
+
+ async def emit(_ctx, text, **kwargs):
+ attempts.append((text, kwargs["output"].idempotency_key))
+ if len(attempts) == 1:
+ raise RuntimeError("first delivery failed")
+ if len(attempts) == 2:
+ retry_entered.set()
+ await release_retry.wait()
+ delivered.append(text)
+ return "accepted"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit)
+
+ class Client:
+ def receive_messages(self):
+ async def frames():
+ assistant = AssistantMessage()
+ assistant.content = [TextBlock(text="first output")]
+ yield assistant
+ first = ResultMessage()
+ first.origin = {"kind": "task-notification"}
+ first.result = "first output"
+ yield first
+ await retry_entered.wait()
+ later = ResultMessage(num_turns=2)
+ later.origin = {"kind": "task-notification"}
+ later.result = "later output"
+ yield later
+ later_seen.set()
+ await stay_open.wait()
+
+ return frames()
+
+ client = Client()
+ agent.claude_sessions[key] = client
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ client,
+ "retry-interleaving",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+ )
+ try:
+ await asyncio.wait_for(later_seen.wait(), timeout=1)
+ assert service.runtime_turn_active(key)
+ release_retry.set()
+ await asyncio.wait_for(
+ _wait_until(lambda: not agent._has_pending_requests(key)),
+ timeout=1,
+ )
+ assert delivered == ["first output", "later output"]
+ assert attempts[0][1] == attempts[1][1] != attempts[-1][1]
+ assert not agent._output_records_for_runtime(key)
+ assert not service.runtime_turn_active(key)
+ assert not receiver.done()
+ finally:
+ release_retry.set()
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry is not None:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
+
+
[email protected]("status", ["failed", "stopped", "killed", "disconnected"])
+def test_restart_resolves_unclassified_terminal_without_borrowing_new_generation(activity_store, status):
+ activations = RuntimeActivationRegistry()
+ identity = activations.attach("claude", "runtime")
+ registry = SessionActivityRegistry(activity_store, activation_registry=activations)
+ registry.start(
+ backend="claude",
+ runtime_key="runtime",
+ session_id="ses-1",
+ activity_id="unresolved",
+ kind="background_task",
+ activation_identity=identity,
+ metadata={"provenance_pending": True, "provenance_phase_id": "old-phase"},
+ )
+ registry.complete(
+ backend="claude",
+ runtime_key="runtime",
+ activity_id="unresolved",
+ status=status,
+ retain_terminal_snapshot=True,
+ activation_identity=identity,
+ )
+ restarted_activations = RuntimeActivationRegistry()
+ restarted_activations.attach("claude", "runtime") # generation counter repeats
+ restarted = SessionActivityRegistry(activity_store, activation_registry=restarted_activations)
+ terminals = restarted.drain_recovered_terminals()
+ assert len(terminals) == 1
+ terminal = terminals[0]
+ assert terminal.status == status
+ assert terminal.metadata["provenance_generation_ended"]
+ assert terminal.metadata["provenance_unresolved"]
+ assert terminal.run_id is None and terminal.turn_id is None
+ restarted.ack_recovered_terminal(terminal)
+ assert not restarted.has_backend_work("claude")
+ assert not activity_store.list_activities()
+
+
[email protected]("status", ["completed", "failed", "stopped", "killed"])
[email protected]("new_human", [False, True])
+def test_provisional_task_can_finish_after_pending_human_has_retired(status, new_human):
+ agent, service = _build_agent()
+ key = "unclassified-after-human:/tmp/work"
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="ses-1",
+ activity_id="unbound-task",
+ kind="background_task",
+ metadata={"provenance_pending": True, "provenance_phase_id": "prior-phase"},
+ )
+ if new_human:
+ agent._pending_requests[key] = [SimpleNamespace(context=context_for(key, "new-human"))]
+ agent._ensure_provisional_phase(key)
+ terminal = SimpleNamespace(
+ subtype="task_notification",
+ task_id="unbound-task",
+ status=status,
+ data={},
+ )
+ assert agent._handle_activity_message(terminal, key, context_for(key))
+ assert not service.activities.active_for_runtime("claude", key)
+ assert agent._has_pending_requests(key) is new_human
+ assert "unbound-task" not in agent._provisional_activity_ids.get(key, set())
+ classified = service.activities.classify_provisional_provenance(
+ "claude",
+ key,
+ activity_ids={"unbound-task"},
+ parent_activity_ids=set(),
+ turn_id=None,
+ run_ids=[],
+ delivery_key_external=None,
+ phase_id="prior-phase",
+ detached=True,
+ )
+ assert len(classified) == 1
+ assert classified[0].status == status
+ assert classified[0].turn_id is None and classified[0].run_id is None
+
+
+async def test_replaced_receiver_late_result_and_eof_cannot_consume_new_request():
+ agent, service = _build_agent()
+ key = "recovery-replaced:/tmp/work"
+ activations = RuntimeActivationRegistry()
+ service.activation_registry = activations
+ old_identity = activations.attach("claude", key)
+ waiting, release = asyncio.Event(), asyncio.Event()
+
+ class OldClient:
+ _vibe_runtime_activation_identity = old_identity
+
+ def receive_messages(self):
+ async def frames():
+ waiting.set()
+ await release.wait()
+ result = ResultMessage()
+ result.result = "OLD human result"
+ yield result
+
+ return frames()
+
+ old = OldClient()
+ agent.claude_sessions[key] = old
+ agent.emit_result_message = AsyncMock(return_value="delivered")
+ receiver = asyncio.create_task(
+ agent._receive_messages(old, "recovery-replaced", "/tmp/work", context_for(key), composite_key=key)
+ )
+ await asyncio.wait_for(waiting.wait(), timeout=1)
+ new_identity = activations.attach("claude", key)
+ new_client = SimpleNamespace(_vibe_runtime_activation_identity=new_identity)
+ agent.claude_sessions[key] = new_client
+ new_context = context_for(key, "new-token")
+ successor = SimpleNamespace(context=new_context)
+ agent._pending_requests[key] = [successor]
+ agent._last_assistant_text[key] = "NEW human draft"
+ gate = service._get_turn_gate(key)
+ await gate.lock.acquire()
+ gate.token = "new-token"
+ gate.backend = "claude"
+ release.set()
+ await asyncio.wait_for(receiver, timeout=1)
+ assert agent.claude_sessions[key] is new_client
+ assert agent._pending_requests[key] == [successor]
+ assert agent._last_assistant_text[key] == "NEW human draft"
+ assert gate.token == "new-token" and gate.lock.locked()
+ agent.emit_result_message.assert_not_awaited()
+ agent._handle_receiver_eof.assert_not_awaited()
+ service.release_runtime_turn(new_context)
+
+
[email protected]("origin", ["human", "task-notification"])
+async def test_stop_interrupt_claim_precedes_every_late_result_route(origin):
+ agent, service = _build_agent()
+ key = "stop-interrupt:/tmp/work"
+ context = context_for(key)
+ await service.begin_agent_initiated_turn("claude", context, key)
+ pending = AgentRequest(
+ context=context,
+ message="input",
+ user_message="input",
+ working_path="/tmp/work",
+ base_session_id="stop-interrupt",
+ composite_session_id=key,
+ session_key="session-key",
+ )
+ agent._pending_requests[key] = [pending]
+ agent._adopt_pending_turn_token = ClaudeAgent._adopt_pending_turn_token
+ stream_waiting, result_ready, interrupt_entered, interrupt_release = (
+ asyncio.Event(),
+ asyncio.Event(),
+ asyncio.Event(),
+ asyncio.Event(),
+ )
+ stay_open = asyncio.Event()
+
+ class Client:
+ disconnect = AsyncMock()
+
+ async def interrupt(self):
+ interrupt_entered.set()
+ await interrupt_release.wait()
+
+ def receive_messages(self):
+ async def frames():
+ stream_waiting.set()
+ await result_ready.wait()
+ result = ResultMessage()
+ result.origin = {"kind": origin}
+ result.result = "late result must not become unsolicited"
+ yield result
+ await stay_open.wait()
+
+ return frames()
+
+ async def emit(ctx, _kind, _text, **kwargs):
+ if kwargs["output"].completes_turn:
+ service.release_runtime_turn(ctx)
+ return "settled"
+
+ agent.controller.emit_agent_message = AsyncMock(side_effect=emit)
+ agent.emit_result_message = AsyncMock(return_value="must-not-deliver")
+ client = Client()
+ agent.claude_sessions[key] = client
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ client,
+ "stop-interrupt",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+ )
+ agent.receiver_tasks[key] = receiver
+ await asyncio.wait_for(stream_waiting.wait(), timeout=1)
+ stop = AgentRequest(
+ context=context_for(key),
+ message="",
+ user_message="",
+ working_path="/tmp/work",
+ base_session_id="stop-interrupt",
+ composite_session_id=key,
+ session_key="session-key",
+ )
+ stopping = asyncio.create_task(service.handle_stop("claude", stop))
+ try:
+ await asyncio.wait_for(interrupt_entered.wait(), timeout=1)
+ result_ready.set()
+ await asyncio.sleep(0)
+ agent.emit_result_message.assert_not_awaited()
+ assert service.runtime_turn_active(key)
+ interrupt_release.set()
+ assert await asyncio.wait_for(stopping, timeout=1)
+ agent.emit_result_message.assert_not_awaited()
+ agent.controller.emit_agent_message.assert_awaited_once()
+ call = agent.controller.emit_agent_message.await_args
+ assert call.args[2] == "" and call.kwargs["level"] == "silent"
+ assert not service.runtime_turn_active(key)
+ assert not agent._has_pending_requests(key)
+ assert not agent._output_records_for_runtime(key)
+ finally:
+ interrupt_release.set()
+ receiver.cancel()
+ await asyncio.gather(receiver, stopping, return_exceptions=True)
+
+
+async def test_provisional_detached_payload_is_frozen_before_generation_replacement():
+ agent, service = _build_agent()
+ key = "provisional-replacement:/tmp/work"
+ context = context_for(key)
+ # A queued human owns the gate but has not registered with the adapter yet.
+ await service.begin_agent_initiated_turn("claude", context, key)
+ receiver_context = context_for(key)
+ assistant_seen, stay_open = asyncio.Event(), asyncio.Event()
+ agent._get_formatter = lambda _: SimpleNamespace(format_assistant_message=lambda parts: "\n".join(parts))
+
+ class Client:
+ disconnect = AsyncMock()
+
+ def receive_messages(self):
+ async def frames():
+ assistant = AssistantMessage()
+ assistant.content = [TextBlock(text="old provisional notification")]
+ yield assistant
+ assistant_seen.set()
+ await stay_open.wait()
+
+ return frames()
+
+ old = Client()
+ agent.claude_sessions[key] = old
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ old,
+ "provisional-replacement",
+ "/tmp/work",
+ receiver_context,
+ composite_key=key,
+ )
+ )
+ agent.receiver_tasks[key] = receiver
+ try:
+ await asyncio.wait_for(assistant_seen.wait(), timeout=1)
+ original = agent._output_records_for_runtime(key)[0]
+ await agent._cleanup_runtime_session(key, expected_client=old)
+ assert original.lifecycle == "delivery_pending"
+ assert original.text == "old provisional notification"
+ agent.claude_sessions[key] = SimpleNamespace()
+ assert agent._provisional_output_record(key, "detached") is None
+ replacement = agent._detached_phase(key)
+ replacement.text = "new generation text"
+ assert replacement.phase_id != original.phase_id
+ assert original.text == "old provisional notification"
+ assert service.runtime_turn_active(key)
+ finally:
+ service.release_runtime_turn(context)
+ receiver.cancel()
+ await asyncio.gather(receiver, return_exceptions=True)
+ retry = agent._activity_flush_tasks.pop(key, None)
+ if retry is not None:
+ retry.cancel()
+ await asyncio.gather(retry, return_exceptions=True)
diff --git a/tests/test_claude_receiver_result_settle.py b/tests/test_claude_receiver_result_settle.py
index d8271efac..f93b2c6f5 100644
--- a/tests/test_claude_receiver_result_settle.py
+++ b/tests/test_claude_receiver_result_settle.py
@@ -32,6 +32,7 @@ class _ResultMessage:
subtype = "success"
result = "done"
duration_ms = 1
+ origin = {"kind": "human"}
class _AssistantFailureMessage:
@@ -45,6 +46,15 @@ class SystemMessage:
data = {"subtype": "init"}
+async def _query_text(query):
+ if hasattr(query, "__aiter__"):
+ frames = [frame async for frame in query]
+ assert len(frames) == 1
+ frame = frames[0]
+ return frame["message"]["content"], frame["origin"]
+ return query, None
+
+
def _one_result_client():
class _Client:
def receive_messages(self):
@@ -337,7 +347,11 @@ class ResultSettlesTurnOnEmitFailureTests(unittest.IsolatedAsyncioTestCase):
receipt = await steer_task
await asyncio.sleep(0)
self.assertIs(receipt.outcome, SteerOutcome.ACCEPTED)
- self.assertEqual(client.query_call, ("补充:`exact`", composite_key))
+ prompt, origin = await _query_text(client.query_call[0])
+ self.assertEqual(
+ (prompt, origin, client.query_call[1]),
+ ("补充:`exact`", {"kind": "human"}, composite_key),
+ )
self.assertEqual(agent._pending_requests[composite_key], [primary_request])
agent.emit_result_message.assert_not_awaited()
@@ -629,8 +643,9 @@ class ResultSettlesTurnOnEmitFailureTests(unittest.IsolatedAsyncioTestCase):
return _iterate()
+ client = _ConsumedClient()
await agent._receive_messages(
- _ConsumedClient(),
+ client,
"session-ambiguous-consumed",
"/tmp/work",
context,
@@ -645,7 +660,11 @@ class ResultSettlesTurnOnEmitFailureTests(unittest.IsolatedAsyncioTestCase):
output=ANY,
)
agent.emit_result_message.assert_not_awaited()
- agent._handle_receiver_eof.assert_awaited_once_with(composite_key, context)
+ agent._handle_receiver_eof.assert_awaited_once_with(
+ composite_key, context,
+ expected_client=client,
+ expected_activation_identity=None,
+ )
self.assertNotIn(composite_key, agent._ambiguous_primary_results)
async def test_failed_ambiguous_half_close_preserves_work_until_later_result(self):
@@ -792,14 +811,14 @@ class ResultSettlesTurnOnEmitFailureTests(unittest.IsolatedAsyncioTestCase):
await flush_task
self.assertIs(receipt.outcome, SteerOutcome.ACCEPTED)
self.assertEqual(agent._pending_requests[composite_key], [primary_request])
- agent._claim_activity_batch_for_turns.assert_called_once()
- registry.requeue_completed_outputs.assert_called_once_with([activity])
+ agent._claim_activity_batch_for_turns.assert_not_called()
+ registry.requeue_completed_outputs.assert_not_called()
agent.emit_result_message.assert_not_awaited()
receiver_task.cancel()
await asyncio.gather(receiver_task, return_exceptions=True)
- async def test_activity_settlement_clears_ambiguous_interrupt(self):
+ async def test_activity_flush_does_not_consume_pending_human_turn(self):
mark_idle_calls: list[str] = []
agent = _build_agent(mark_idle_calls)
context = SimpleNamespace(user_id="U1", channel_id="C1", platform_specific={})
@@ -828,10 +847,10 @@ class ResultSettlesTurnOnEmitFailureTests(unittest.IsolatedAsyncioTestCase):
context,
)
- self.assertFalse(retry)
- self.assertFalse(agent._has_pending_requests(composite_key))
- self.assertNotIn(composite_key, agent._ambiguous_interrupt_keys())
- agent._emit_activity_result.assert_awaited_once()
+ self.assertTrue(retry)
+ self.assertEqual(agent._pending_requests[composite_key], [primary_request])
+ self.assertIn(composite_key, agent._ambiguous_interrupt_keys())
+ agent._emit_activity_result.assert_not_awaited()
self.assertFalse(receiver_task.done())
receiver_task.cancel()
@@ -952,6 +971,7 @@ class ResultSettlesTurnOnEmitFailureTests(unittest.IsolatedAsyncioTestCase):
agent._cleanup_runtime_session.assert_awaited_once_with(
composite_key,
current_receiver_task=receiver_task,
+ expected_client=client,
preserve_pending_request_state=True,
reason="receiver_eof_without_result",
)
diff --git a/tests/test_claude_result_provenance.py b/tests/test_claude_result_provenance.py
new file mode 100644
index 000000000..043de9611
--- /dev/null
+++ b/tests/test_claude_result_provenance.py
@@ -0,0 +1,1966 @@
+from __future__ import annotations
+
+import asyncio
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from core.handlers.session_handler import ClaudeInputNotSentError
+from core.native_dispatch_phase import (
+ DISPATCH_PHASE_PREWRITE,
+ backend_dispatch_attempted,
+ prewrite_failure_evidence,
+ set_dispatch_phase,
+)
+from modules.claude_sdk_compat import TextBlock, ToolUseBlock
+from modules.agents.claude_agent import ClaudeAgent
+
+from tests.test_claude_agent_initiated_turn import _build_agent, _dispatcher_owned_emit
+
+
+class TaskStartedMessage:
+ subtype = "task_started"
+
+ def __init__(self, task_id: str, *, tool_use_id: str | None = None):
+ self.task_id = task_id
+ self.description = f"Run {task_id}"
+ self.task_type = "local_agent"
+ self.tool_use_id = tool_use_id
+ self.data = {}
+
+
+class TaskNotificationMessage:
+ subtype = "task_notification"
+
+ def __init__(self, task_id: str, summary: str):
+ self.task_id = task_id
+ self.status = "completed"
+ self.summary = summary
+ self.output_file = f"/tmp/{task_id}.output"
+ self.data = {}
+
+
+_MISSING = object()
+
+
+class ResultMessage:
+ subtype = "success"
+ duration_ms = 1
+ session_id = "claude-native-session"
+
+ def __init__(self, text: str, *, origin=_MISSING):
+ self.result = text
+ if origin is not _MISSING:
+ self.origin = origin
+
+
+class AssistantMessage:
+ error = ""
+ usage = None
+
+ def __init__(self, *content):
+ self.content = list(content)
+
+
+def _block(block_type, **values):
+ block = object.__new__(block_type)
+ for name, value in values.items():
+ setattr(block, name, value)
+ return block
+
+
+def _failure_assistant(text: str = "transport failed"):
+ message = AssistantMessage(_block(TextBlock, text=text))
+ message.is_error = True
+ message.error = text
+ return message
+
+
+def _context(composite_key: str, *, turn_token: str = "human-turn"):
+ return SimpleNamespace(
+ user_id="U1",
+ channel_id="C1",
+ platform="avibe",
+ platform_specific={
+ "agent_runtime_turn_key": composite_key,
+ "agent_runtime_turn_token": "runtime-token",
+ "turn_token": turn_token,
+ "agent_session_id": "sess-provenance",
+ },
+ )
+
+
+def _pending_request(composite_key: str):
+ payload = dict(_context(composite_key).platform_specific)
+ return SimpleNamespace(
+ context=SimpleNamespace(platform_specific=payload),
+ output=None,
+ output_activities=[],
+ )
+
+
+def _client(messages):
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ for message in messages:
+ yield message
+
+ return _iterate()
+
+ return _Client()
+
+
+def _failing_client(messages, error):
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ for message in messages:
+ yield message
+ raise error
+
+ return _iterate()
+
+ return _Client()
+
+
+def _task_pair(task_id: str):
+ return [
+ TaskStartedMessage(task_id),
+ TaskNotificationMessage(task_id, f"{task_id} finished"),
+ ]
+
+
+class ClaudeResultProvenanceTests(unittest.IsolatedAsyncioTestCase):
+ async def _receive(self, messages, *, key="session-provenance:/tmp/work"):
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ await agent._receive_messages(
+ _client(messages),
+ "sess-provenance",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+ return agent, service, request
+
+ async def test_notification_before_human_result_does_not_claim_human_turn(self):
+ messages = [
+ *_task_pair("task-before-human"),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ResultMessage("background reply", origin={"kind": "task-notification"}),
+ ]
+
+ agent, _service, request = await self._receive(messages)
+
+ self.assertEqual(agent.emit_result_message.await_count, 2)
+ human_call, activity_call = agent.emit_result_message.await_args_list
+ self.assertIs(human_call.kwargs["request"], request)
+ self.assertTrue(activity_call.kwargs["output"].detached)
+ self.assertEqual(
+ activity_call.kwargs["output"].activity_ids,
+ ("task-before-human",),
+ )
+ self.assertFalse(agent._has_pending_requests("session-provenance:/tmp/work"))
+
+ async def test_task_result_before_human_result_is_detached_then_human_settles(self):
+ messages = [
+ *_task_pair("task-before-human"),
+ ResultMessage("background reply", origin={"kind": "task-notification"}),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+
+ agent, _service, request = await self._receive(messages)
+
+ self.assertEqual(agent.emit_result_message.await_count, 2)
+ activity_call, human_call = agent.emit_result_message.await_args_list
+ self.assertTrue(activity_call.kwargs["output"].detached)
+ self.assertIs(human_call.kwargs["request"], request)
+ self.assertFalse(agent._has_pending_requests("session-provenance:/tmp/work"))
+
+ async def test_human_result_preserves_detached_activity_until_followup_result(self):
+ key = "session-human-before-detached:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-human-before-detached",
+ activity_id="task-detached-first",
+ kind="local_agent",
+ turn_id="background-turn",
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=key,
+ activity_id="task-detached-first",
+ status="completed",
+ metadata={"summary": "background finished"},
+ expects_output=True,
+ )
+ detached = service.activities.claim_completed_output("claude", key)
+ self.assertIsNotNone(detached)
+ agent._retain_activity_output_record(key, [detached])
+ agent.emit_result_message = _dispatcher_owned_emit(service)
+
+ await agent._receive_messages(
+ _client(
+ [
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ResultMessage(
+ "background reply",
+ origin={"kind": "task-notification"},
+ ),
+ ]
+ ),
+ "sess-human-before-detached",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertEqual(agent.emit_result_message.await_count, 2)
+ human_call, activity_call = agent.emit_result_message.await_args_list
+ self.assertIs(human_call.kwargs["request"], request)
+ self.assertNotIn("output", human_call.kwargs)
+ self.assertIsNone(activity_call.kwargs.get("request"))
+ self.assertTrue(activity_call.kwargs["output"].detached)
+ self.assertEqual(
+ activity_call.kwargs["output"].activity_ids,
+ ("task-detached-first",),
+ )
+ self.assertFalse(agent._has_pending_requests(key))
+ self.assertFalse(service.activities.has_completed_output("claude", key))
+
+ async def test_buffered_assistant_frames_replay_after_human_result_classification(self):
+ key = "session-buffered-provenance:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+
+ class _Formatter:
+ @staticmethod
+ def format_assistant_message(parts):
+ return "\n".join(parts)
+
+ @staticmethod
+ def format_toolcall(name, input_data, **_kwargs):
+ return f"{name}({input_data['command']})"
+
+ @staticmethod
+ def format_toolcall_label(name, input_data, **_kwargs):
+ return f"{name}: {input_data['command']}"
+
+ agent._get_formatter = lambda _context: _Formatter()
+ messages = [
+ TaskStartedMessage("task-buffer"),
+ AssistantMessage(
+ _block(TextBlock, text="progress from the foreground phase"),
+ ),
+ AssistantMessage(
+ _block(
+ ToolUseBlock,
+ id="tool-buffer",
+ name="Bash",
+ input={"command": "pwd"},
+ ),
+ _block(TextBlock, text="final foreground phase"),
+ ),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+
+ await agent._receive_messages(
+ _client(messages),
+ "sess-buffered-provenance",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ assistant_calls = [
+ call
+ for call in agent.controller.emit_agent_message.await_args_list
+ if len(call.args) > 1 and call.args[1] in {"assistant", "toolcall"}
+ ]
+ self.assertEqual(
+ [(call.args[1], call.args[2]) for call in assistant_calls],
+ [
+ ("assistant", "progress from the foreground phase"),
+ ("toolcall", "Bash(pwd)"),
+ ],
+ )
+ self.assertEqual(agent.emit_result_message.await_count, 1)
+ self.assertIs(agent.emit_result_message.await_args.kwargs["request"], request)
+ self.assertFalse(service.activities.has_completed_output("claude", key))
+
+ async def test_buffered_foreground_tool_is_owned_before_task_started(self):
+ key = "session-buffered-foreground-tool:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-buffered-foreground-tool",
+ activity_id="background-task",
+ kind="local_agent",
+ turn_id="background-turn",
+ )
+
+ await agent._receive_messages(
+ _client(
+ [
+ AssistantMessage(
+ _block(
+ ToolUseBlock,
+ id="foreground-tool",
+ name="Bash",
+ input={"command": "pwd"},
+ )
+ ),
+ TaskStartedMessage(
+ "foreground-task",
+ tool_use_id="foreground-tool",
+ ),
+ TaskNotificationMessage(
+ "foreground-task",
+ "foreground finished",
+ ),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+ ),
+ "sess-buffered-foreground-tool",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertFalse(service.activities.has_completed_output("claude", key))
+ self.assertEqual(agent.emit_result_message.await_count, 1)
+ self.assertIs(agent.emit_result_message.await_args.kwargs["request"], request)
+
+ async def test_activity_flush_defers_until_buffered_phase_has_terminal_owner(self):
+ key = "session-provenance-flush-race:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = _dispatcher_owned_emit(service)
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-provenance-flush-race",
+ activity_id="task-flush-race",
+ kind="local_agent",
+ turn_id="task-turn",
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=key,
+ activity_id="task-flush-race",
+ status="completed",
+ metadata={"summary": "background finished"},
+ expects_output=True,
+ )
+ agent._buffered_assistant_messages[key] = [
+ (
+ AssistantMessage(
+ _block(TextBlock, text="ambiguous assistant phase"),
+ ),
+ agent._steering_generation(key),
+ )
+ ]
+
+ should_retry = await agent._flush_completed_activity_outputs(key, context)
+
+ self.assertTrue(should_retry)
+ self.assertIs(agent._pending_requests[key][0], request)
+ self.assertTrue(service.activities.has_completed_output("claude", key))
+ agent.emit_result_message.assert_not_awaited()
+
+ await agent._receive_messages(
+ _client(
+ [
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+ ),
+ "sess-provenance-flush-race",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertFalse(agent._has_pending_requests(key))
+ self.assertGreaterEqual(agent.emit_result_message.await_count, 1)
+ self.assertIs(
+ next(
+ call.kwargs["request"]
+ for call in agent.emit_result_message.await_args_list
+ if call.kwargs.get("request") is request
+ ),
+ request,
+ )
+ self.assertFalse(service.activities.has_completed_output("claude", key))
+
+ async def test_activity_flush_defers_without_assistant_frame_until_terminal_owner(self):
+ key = "session-provenance-flush-without-assistant:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = _dispatcher_owned_emit(service)
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-provenance-flush-without-assistant",
+ activity_id="task-flush-without-assistant",
+ kind="local_agent",
+ turn_id="task-turn",
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=key,
+ activity_id="task-flush-without-assistant",
+ status="completed",
+ metadata={"summary": "background finished"},
+ expects_output=True,
+ )
+
+ should_retry = await agent._flush_completed_activity_outputs(key, context)
+
+ self.assertTrue(should_retry)
+ self.assertIn(key, agent._activity_provenance_barriers)
+ self.assertIs(agent._pending_requests[key][0], request)
+ self.assertTrue(service.activities.has_completed_output("claude", key))
+ agent.emit_result_message.assert_not_awaited()
+
+ async def test_buffered_assistant_replay_failure_still_settles_terminal_result(self):
+ key = "session-buffered-replay-failure:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+
+ class _Formatter:
+ format_assistant_message = Mock(side_effect=RuntimeError("replay failed"))
+
+ agent._get_formatter = lambda _context: _Formatter()
+
+ await agent._receive_messages(
+ _client(
+ [
+ TaskStartedMessage("task-replay-failure"),
+ AssistantMessage(
+ _block(TextBlock, text="buffered before terminal"),
+ ),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+ ),
+ "sess-buffered-replay-failure",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertFalse(agent._has_pending_requests(key))
+ agent.emit_result_message.assert_awaited_once()
+ self.assertIs(agent.emit_result_message.await_args.kwargs["request"], request)
+ self.assertFalse(service.activities.has_completed_output("claude", key))
+
+ async def test_multiple_task_completions_are_one_detached_delivery_batch(self):
+ messages = [
+ TaskStartedMessage("task-a"),
+ TaskStartedMessage("task-b"),
+ TaskNotificationMessage("task-a", "A finished"),
+ TaskNotificationMessage("task-b", "B finished"),
+ ResultMessage("combined background reply", origin={"kind": "task-notification"}),
+ ]
+
+ agent, _service, request = await self._receive(messages)
+
+ self.assertEqual(agent.emit_result_message.await_count, 1)
+ call = agent.emit_result_message.await_args
+ self.assertIsNone(call.kwargs["request"])
+ self.assertEqual(
+ call.kwargs["output"].activity_ids,
+ ("task-a", "task-b"),
+ )
+ self.assertTrue(call.kwargs["output"].detached)
+ self.assertTrue(agent._has_pending_requests("session-provenance:/tmp/work"))
+ self.assertIs(agent._pending_requests["session-provenance:/tmp/work"][0], request)
+
+ async def test_assistant_after_detached_activity_flag_stays_detached(self):
+ key = "session-live-detached-flag:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-live-detached-flag",
+ activity_id="task-live-detached",
+ kind="local_agent",
+ turn_id="origin-turn",
+ )
+ service.activities.complete(
+ backend="claude",
+ runtime_key=key,
+ activity_id="task-live-detached",
+ status="completed",
+ metadata={"summary": "background task finished"},
+ expects_output=True,
+ )
+ activity = service.activities.claim_completed_output("claude", key)
+ self.assertIsNotNone(activity)
+ agent._retain_activity_output_record(key, [activity])
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+
+ assistant = AssistantMessage(
+ _block(TextBlock, text="detached progress"),
+ )
+ await agent._receive_messages(
+ _client(
+ [
+ assistant,
+ ResultMessage(
+ "detached final",
+ origin={"kind": "task-notification"},
+ ),
+ ]
+ ),
+ "sess-live-detached-flag",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertEqual(agent.emit_result_message.await_count, 1)
+ self.assertEqual(
+ agent.emit_result_message.await_args.args[1],
+ "detached progress",
+ )
+ self.assertTrue(agent.emit_result_message.await_args.kwargs["output"].detached)
+ self.assertNotIn("detached progress", agent._last_assistant_text.values())
+ self.assertIs(agent._pending_requests[key][0], request)
+
+ async def test_absent_and_unknown_origin_never_claim_a_human_turn_with_active_task(self):
+ messages = [
+ TaskStartedMessage("task-ambiguous"),
+ ResultMessage("absent origin"),
+ ResultMessage("unknown origin", origin={"kind": "mystery"}),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+
+ agent, _service, request = await self._receive(messages)
+
+ self.assertEqual(agent.emit_result_message.await_count, 3)
+ first, second, human = agent.emit_result_message.await_args_list
+ self.assertIsNone(first.kwargs.get("request"))
+ self.assertIsNone(second.kwargs.get("request"))
+ self.assertTrue(first.kwargs["output"].detached)
+ self.assertTrue(second.kwargs["output"].detached)
+ self.assertIs(human.kwargs["request"], request)
+ self.assertFalse(agent._has_pending_requests("session-provenance:/tmp/work"))
+
+ async def test_terminal_task_without_output_keeps_phase_barrier_until_human_result(
+ self,
+ ):
+ for status in ("failed", "stopped", "killed"):
+ with self.subTest(status=status):
+ key = f"session-no-output-{status}:/tmp/work"
+ agent, _service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ terminal = TaskNotificationMessage("task-no-output", "not user text")
+ terminal.status = status
+
+ await agent._receive_messages(
+ _client(
+ [
+ TaskStartedMessage("task-no-output"),
+ terminal,
+ AssistantMessage(
+ _block(TextBlock, text="detached failure detail")
+ ),
+ ResultMessage("unclassified result"),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+ ),
+ "sess-no-output",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ human_calls = [
+ call
+ for call in agent.emit_result_message.await_args_list
+ if call.kwargs.get("request") is request
+ ]
+ self.assertEqual(len(human_calls), 1)
+ self.assertEqual(human_calls[0].args[1], "human reply")
+
+ async def test_terminal_provisional_activity_statuses_rebind_before_run_settlement(
+ self,
+ ):
+ for status in ("failed", "stopped", "killed", "completed"):
+ with self.subTest(status=status):
+ key = f"session-terminal-lineage-{status}:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ request.context.platform_specific.update(
+ {
+ "task_trigger_kind": "agent_run",
+ "task_execution_id": "human-run",
+ "accepted_agent_run_ids": ["human-run"],
+ "delivery_key_external": "human-delivery",
+ }
+ )
+ agent._pending_requests[key] = [request]
+ agent._get_formatter = lambda _context: SimpleNamespace(
+ format_toolcall=lambda *_args, **_kwargs: "fixture tool",
+ format_toolcall_label=lambda *_args, **_kwargs: "fixture tool",
+ format_assistant_message=lambda parts: "\n".join(parts),
+ )
+ settle_activity = Mock()
+ service.controller.scheduled_task_service = SimpleNamespace(
+ settle_activity_runs=settle_activity,
+ )
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+
+ tool = _block(
+ ToolUseBlock,
+ id=f"tool-{status}",
+ name="Bash",
+ input={"command": "fixture"},
+ )
+ started = TaskStartedMessage(
+ f"task-{status}",
+ tool_use_id=f"tool-{status}",
+ )
+ terminal = TaskNotificationMessage(
+ f"task-{status}",
+ "terminal fixture",
+ )
+ terminal.status = status
+
+ await agent._receive_messages(
+ _client(
+ [
+ AssistantMessage(tool),
+ started,
+ terminal,
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+ ),
+ f"sess-terminal-lineage-{status}",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ settle_activity.assert_called_once()
+ settled = settle_activity.call_args.args[0]
+ self.assertEqual(settled.run_id, "human-run")
+ self.assertEqual(settled.turn_id, "human-turn")
+ self.assertEqual(
+ settled.metadata["delivery_key_external"],
+ "human-delivery",
+ )
+ self.assertFalse(
+ service.activities.terminal_snapshots_for_runtime(
+ "claude",
+ key,
+ )
+ )
+ self.assertFalse(agent._has_pending_requests(key))
+
+ async def test_detached_tool_output_cannot_replace_following_human_result(self):
+ key = "session-detached-tool-before-human:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent._get_formatter = lambda _context: SimpleNamespace(
+ format_assistant_message=lambda parts: "\n".join(parts),
+ format_toolcall=lambda *_args, **_kwargs: "fixture tool",
+ format_toolcall_label=lambda *_args, **_kwargs: "fixture tool",
+ )
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-detached-tool-before-human",
+ activity_id="background",
+ kind="background_task",
+ turn_id="old-turn",
+ )
+
+ await agent._receive_messages(
+ _client(
+ [
+ AssistantMessage(
+ _block(
+ ToolUseBlock,
+ id="detached-tool",
+ name="Bash",
+ input={"command": "fixture", "run_in_background": False},
+ )
+ ),
+ ResultMessage(
+ "background text",
+ origin={"kind": "task-notification"},
+ ),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+ ),
+ "sess-detached-tool-before-human",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ human_calls = [
+ call
+ for call in agent.emit_result_message.await_args_list
+ if call.kwargs.get("request") is request
+ ]
+ self.assertEqual(len(human_calls), 1)
+ self.assertEqual(human_calls[0].args[1], "human reply")
+
+ async def test_ambiguous_task_started_does_not_copy_human_harness_attribution(self):
+ key = "session-ambiguous-harness:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ request.context.platform_specific.update(
+ {
+ "task_trigger_kind": "agent_run",
+ "task_execution_id": "human-run",
+ "accepted_agent_run_ids": ["human-run", "accepted-run"],
+ "delivery_key_external": "human-delivery",
+ }
+ )
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = _dispatcher_owned_emit(service)
+
+ await agent._receive_messages(
+ _client(
+ [
+ TaskStartedMessage("task-ambiguous-harness"),
+ TaskNotificationMessage(
+ "task-ambiguous-harness",
+ "background finished",
+ ),
+ ResultMessage(
+ "background reply",
+ origin={"kind": "task-notification"},
+ ),
+ ]
+ ),
+ "sess-ambiguous-harness",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertEqual(agent.emit_result_message.await_count, 1)
+ output = agent.emit_result_message.await_args.kwargs["output"]
+ self.assertTrue(output.detached)
+ self.assertEqual(output.run_ids, ())
+ self.assertIsNone(output.metadata["turn_id"])
+ self.assertTrue(agent._has_pending_requests(key))
+ self.assertIs(agent._pending_requests[key][0], request)
+
+ async def test_buffered_background_task_stays_provisional_until_detached_result(self):
+ key = "session-buffered-background-provisional:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ request.context.platform_specific.update(
+ {
+ "task_trigger_kind": "agent_run",
+ "task_execution_id": "human-run",
+ "accepted_agent_run_ids": ["human-run"],
+ "delivery_key_external": "human-delivery",
+ }
+ )
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = _dispatcher_owned_emit(service)
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-buffered-background-provisional",
+ activity_id="older-background",
+ kind="local_agent",
+ turn_id="older-turn",
+ )
+
+ await agent._receive_messages(
+ _client(
+ [
+ AssistantMessage(
+ _block(
+ ToolUseBlock,
+ id="background-tool",
+ name="Agent",
+ input={
+ "description": "run detached work",
+ "run_in_background": True,
+ },
+ )
+ ),
+ TaskStartedMessage(
+ "task-buffered-background",
+ tool_use_id="background-tool",
+ ),
+ TaskNotificationMessage(
+ "task-buffered-background",
+ "background finished",
+ ),
+ ResultMessage(
+ "background reply",
+ origin={"kind": "task-notification"},
+ ),
+ ]
+ ),
+ "sess-buffered-background-provisional",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertEqual(agent.emit_result_message.await_count, 1)
+ output = agent.emit_result_message.await_args.kwargs["output"]
+ self.assertTrue(output.detached)
+ self.assertEqual(output.run_ids, ())
+ self.assertEqual(output.activity_ids, ("task-buffered-background",))
+ self.assertIsNone(output.metadata["turn_id"])
+ self.assertTrue(agent._has_pending_requests(key))
+
+ async def test_write_fence_rejects_replaced_or_stopping_client_before_native_write(self):
+ key = "session-write-fence:/tmp/work"
+ agent, _service = _build_agent()
+ context = SimpleNamespace(platform_specific={})
+ set_dispatch_phase(context, DISPATCH_PHASE_PREWRITE)
+ old_client = SimpleNamespace(query=AsyncMock())
+ replacement = SimpleNamespace()
+ agent.claude_sessions[key] = replacement
+
+ with self.assertRaises(ClaudeInputNotSentError):
+ await agent._write_human_query(old_client, key, "hello", context)
+
+ old_client.query.assert_not_awaited()
+ self.assertIs(backend_dispatch_attempted(context), False)
+
+ agent.claude_sessions[key] = old_client
+ agent._steering_closing_keys().add(key)
+ with self.assertRaises(ClaudeInputNotSentError):
+ await agent._write_human_query(old_client, key, "stop race", context)
+ old_client.query.assert_not_awaited()
+ self.assertIs(backend_dispatch_attempted(context), False)
+
+ async def test_native_transport_failure_is_attempted_not_definitely_unsent(self):
+ key = "session-write-attempt:/tmp/work"
+ agent, _service = _build_agent()
+ context = SimpleNamespace(platform_specific={})
+ set_dispatch_phase(context, DISPATCH_PHASE_PREWRITE)
+ client = SimpleNamespace(
+ query=AsyncMock(side_effect=RuntimeError("transport failed"))
+ )
+ agent.claude_sessions[key] = client
+
+ with self.assertRaisesRegex(RuntimeError, "transport failed"):
+ await agent._write_human_query(client, key, "hello", context)
+
+ self.assertIs(backend_dispatch_attempted(context), True)
+ self.assertEqual(prewrite_failure_evidence(context), {})
+
+ async def test_buffered_failure_after_steer_is_suppressed_and_does_not_pop_new_turn(self):
+ key = "session-buffered-failure-steer:/tmp/work"
+ agent, _service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent._get_formatter = lambda _context: None
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+
+ failure_seen = asyncio.Event()
+ release_result = asyncio.Event()
+
+ class _Client:
+ def receive_messages(self):
+ async def _iterate():
+ yield TaskStartedMessage("task-failure-steer")
+ yield _failure_assistant()
+ failure_seen.set()
+ await release_result.wait()
+ yield ResultMessage("steered result", origin={"kind": "human"})
+
+ return _iterate()
+
+ steer_receipt = SimpleNamespace(
+ kind="steer",
+ state="accepted",
+ text="continue",
+ )
+ agent._native_input_receipt_map()[key] = [steer_receipt]
+
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ _Client(),
+ "sess-buffered-failure-steer",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+ )
+ await asyncio.wait_for(failure_seen.wait(), timeout=1)
+ agent._advance_steering_generation(key)
+ release_result.set()
+ await receiver
+
+ self.assertFalse(agent.emit_result_message.await_args_list)
+ self.assertIs(agent._pending_requests[key][0], request)
+ # The paired human Result consumes the suppression marker without
+ # settling/popping the still-current request.
+ self.assertNotIn(key, agent._suppressed_synthetic_results)
+ self.assertNotIn(key, agent._suppressed_synthetic_error_text)
+ self.assertNotIn(key, agent._native_input_receipts)
+
+ async def test_buffered_failure_after_stop_is_ignored_without_failure_settlement(self):
+ key = "session-buffered-failure-stop:/tmp/work"
+ agent, _service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ agent._steering_closing_keys().add(key)
+
+ await agent._receive_messages(
+ _client(
+ [
+ TaskStartedMessage("task-failure-stop"),
+ _failure_assistant(),
+ ResultMessage("stopped", origin={"kind": "human"}),
+ ]
+ ),
+ "sess-buffered-failure-stop",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ agent.emit_result_message.assert_not_awaited()
+ self.assertIs(agent._pending_requests[key][0], request)
+
+ async def test_eof_replays_buffered_failure_before_human_fallback_and_detached_flush(self):
+ key = "session-buffered-failure-eof:/tmp/work"
+ agent, service = _build_agent()
+ agent._handle_receiver_eof = ClaudeAgent._handle_receiver_eof.__get__(agent)
+ agent._handle_assistant_terminal_failure = (
+ ClaudeAgent._handle_assistant_terminal_failure.__get__(agent)
+ )
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.controller.emit_agent_message = _dispatcher_owned_emit(service)
+ agent.record_model_hub_native_failure = AsyncMock()
+ agent.controller.agent_auth_service = SimpleNamespace(
+ maybe_emit_auth_recovery_message=AsyncMock(return_value=False),
+ )
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-buffered-failure-eof",
+ activity_id="task-buffered-failure-eof",
+ kind="local_agent",
+ turn_id="task-turn",
+ )
+ client = _client(
+ [
+ TaskStartedMessage("task-buffered-failure-eof"),
+ TaskNotificationMessage(
+ "task-buffered-failure-eof",
+ "background finished",
+ ),
+ _failure_assistant("backend exploded"),
+ ]
+ )
+ client._transport = SimpleNamespace(_process=SimpleNamespace(returncode=None))
+ agent.controller.claude_sessions[key] = client
+ await agent._receive_messages(
+ client,
+ "sess-buffered-failure-eof",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ event_types = [
+ call.args[1]
+ for call in agent.controller.emit_agent_message.await_args_list
+ ]
+ self.assertEqual(event_types, ["notify", "result", "result"])
+ self.assertIn(
+ "backend exploded",
+ agent.controller.emit_agent_message.await_args_list[1].kwargs[
+ "terminal_error"
+ ],
+ )
+ self.assertTrue(
+ agent.controller.emit_agent_message.await_args_list[2]
+ .kwargs["output"]
+ .detached
+ )
+ self.assertFalse(service.activities.has_completed_output("claude", key))
+
+ async def test_receiver_error_replays_buffered_failure_before_detached_flush(self):
+ key = "session-buffered-failure-error:/tmp/work"
+ agent, service = _build_agent()
+ agent._handle_receiver_eof = ClaudeAgent._handle_receiver_eof.__get__(agent)
+ agent._handle_receiver_exception = ClaudeAgent._handle_receiver_exception.__get__(
+ agent
+ )
+ agent._handle_assistant_terminal_failure = (
+ ClaudeAgent._handle_assistant_terminal_failure.__get__(agent)
+ )
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.controller.emit_agent_message = _dispatcher_owned_emit(service)
+ agent.record_model_hub_native_failure = AsyncMock()
+ agent.controller.agent_auth_service = SimpleNamespace(
+ maybe_emit_auth_recovery_message=AsyncMock(return_value=False),
+ )
+ agent.session_handler.handle_session_error = AsyncMock(return_value=False)
+ cleanup_lock_states = []
+
+ async def cleanup_session(*_args, **_kwargs):
+ cleanup_lock_states.append(agent._steering_lock(key).locked())
+
+ agent.session_handler.cleanup_session = AsyncMock(
+ side_effect=cleanup_session,
+ )
+ service.activities.start(
+ backend="claude",
+ runtime_key=key,
+ session_id="sess-buffered-failure-error",
+ activity_id="task-buffered-failure-error",
+ kind="local_agent",
+ turn_id="task-turn",
+ )
+
+ class _FailingClient:
+ def receive_messages(self):
+ async def _iterate():
+ yield TaskStartedMessage("task-buffered-failure-error")
+ yield TaskNotificationMessage(
+ "task-buffered-failure-error",
+ "background finished",
+ )
+ yield _failure_assistant("backend exploded")
+ raise RuntimeError("receiver disconnected")
+
+ return _iterate()
+
+ await agent._receive_messages(
+ _FailingClient(),
+ "sess-buffered-failure-error",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ event_types = [
+ call.args[1]
+ for call in agent.controller.emit_agent_message.await_args_list
+ ]
+ self.assertEqual(event_types, ["notify", "result", "result"])
+ self.assertIn(
+ "backend exploded",
+ agent.controller.emit_agent_message.await_args_list[1].kwargs[
+ "terminal_error"
+ ],
+ )
+ self.assertTrue(
+ agent.controller.emit_agent_message.await_args_list[2]
+ .kwargs["output"]
+ .detached
+ )
+ self.assertFalse(service.activities.has_completed_output("claude", key))
+ agent.session_handler.handle_session_error.assert_not_awaited()
+ agent.session_handler.cleanup_session.assert_awaited_once()
+ self.assertEqual(cleanup_lock_states, [False])
+
+ async def test_receiver_error_contains_buffered_failure_replay_exception(self):
+ key = "session-buffered-failure-replay-error:/tmp/work"
+ agent, _service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.controller.agent_auth_service = SimpleNamespace(
+ maybe_emit_auth_recovery_message=AsyncMock(return_value=False),
+ )
+ agent.session_handler.handle_session_error = AsyncMock(return_value=True)
+ agent.record_model_hub_native_failure = AsyncMock()
+ agent._emit_no_result_settlement = AsyncMock()
+ agent._release_service_runtime_turn = Mock()
+ agent._process_assistant_terminal_frame = AsyncMock(
+ side_effect=RuntimeError("replay emit failed"),
+ )
+
+ class _FailingClient:
+ def receive_messages(self):
+ async def _iterate():
+ yield TaskStartedMessage("task-replay-error")
+ yield _failure_assistant("backend exploded")
+ raise RuntimeError("receiver disconnected")
+
+ return _iterate()
+
+ await agent._receive_messages(
+ _FailingClient(),
+ "sess-buffered-failure-replay-error",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ agent.session_handler.handle_session_error.assert_awaited_once()
+ agent._emit_no_result_settlement.assert_awaited_once()
+ agent._release_service_runtime_turn.assert_called_once_with(context)
+
+ async def test_eof_closes_write_admission_before_buffered_failure_replay(self):
+ key = "session-eof-write-fence:/tmp/work"
+ agent, _service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ replay_started = asyncio.Event()
+ release_replay = asyncio.Event()
+ eof_entered = asyncio.Event()
+ release_eof = asyncio.Event()
+ client = SimpleNamespace(query=AsyncMock())
+ agent.claude_sessions[key] = client
+
+ async def hold_eof(*_args, **_kwargs):
+ eof_entered.set()
+ await release_eof.wait()
+
+ agent._handle_receiver_eof = AsyncMock(side_effect=hold_eof)
+
+ async def replay_failure(*_args, **_kwargs):
+ replay_started.set()
+ await release_replay.wait()
+ return "failure"
+
+ agent._process_assistant_terminal_frame = replay_failure
+
+ receiver_client = _client(
+ [
+ TaskStartedMessage("task-eof-write-fence"),
+ _failure_assistant("backend exploded"),
+ ]
+ )
+ client.receive_messages = receiver_client.receive_messages
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ client,
+ "sess-eof-write-fence",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+ )
+ await asyncio.wait_for(replay_started.wait(), timeout=1)
+ self.assertIn(key, agent._steering_closing_keys())
+
+ async def fenced_write():
+ async with agent._steering_lock(key):
+ await agent._write_human_query(
+ client,
+ key,
+ "new input",
+ context,
+ )
+
+ write = asyncio.create_task(fenced_write())
+ release_replay.set()
+ await asyncio.wait_for(eof_entered.wait(), timeout=1)
+ with self.assertRaises(ClaudeInputNotSentError):
+ await asyncio.wait_for(write, timeout=1)
+
+ release_eof.set()
+ await receiver
+
+ async def test_receiver_error_closes_write_admission_before_replay(self):
+ key = "session-error-write-fence:/tmp/work"
+ agent, _service = _build_agent()
+ context = _context(key)
+ replay_started = asyncio.Event()
+ release_replay = asyncio.Event()
+ client = SimpleNamespace(query=AsyncMock())
+ agent.claude_sessions[key] = client
+
+ async def replay_failure(*_args, **_kwargs):
+ replay_started.set()
+ await release_replay.wait()
+ return None
+
+ agent._replay_buffered_terminal_failures = replay_failure
+ agent._handle_receiver_exception = AsyncMock()
+ receiver_client = _failing_client(
+ [
+ TaskStartedMessage("task-error-write-fence"),
+ _failure_assistant("backend exploded"),
+ ],
+ RuntimeError("receiver disconnected"),
+ )
+ client.receive_messages = receiver_client.receive_messages
+ receiver = asyncio.create_task(
+ agent._receive_messages(
+ client,
+ "sess-error-write-fence",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+ )
+ await asyncio.wait_for(replay_started.wait(), timeout=1)
+ self.assertIn(key, agent._steering_closing_keys())
+
+ async def fenced_write():
+ async with agent._steering_lock(key):
+ await agent._write_human_query(
+ client,
+ key,
+ "new input",
+ context,
+ )
+
+ write = asyncio.create_task(fenced_write())
+ with self.assertRaises(ClaudeInputNotSentError):
+ await asyncio.wait_for(write, timeout=1)
+ self.assertFalse(client.query.await_count)
+ release_replay.set()
+ await receiver
+
+ async def test_primary_write_and_stop_share_the_native_write_fence(self):
+ key = "session-primary-stop-fence:/tmp/work"
+ agent, _service = _build_agent()
+ query_started = asyncio.Event()
+ release_query = asyncio.Event()
+ interrupt_called = asyncio.Event()
+
+ class _Client:
+ _vibe_runtime_base_session_id = "sess-primary-stop-fence"
+ _vibe_runtime_session_key = key
+
+ async def query(self, _messages, *, session_id):
+ self.assert_session_id = session_id
+ query_started.set()
+ await release_query.wait()
+
+ async def interrupt(self):
+ interrupt_called.set()
+
+ async def disconnect(self):
+ return None
+
+ def receive_messages(self):
+ async def _iterate():
+ await asyncio.Future()
+ yield None
+
+ return _iterate()
+
+ client = _Client()
+ agent.claude_sessions[key] = client
+ receiver_task = asyncio.create_task(asyncio.Event().wait())
+ agent.receiver_tasks[key] = receiver_task
+ agent.session_handler.get_or_create_claude_session = AsyncMock(
+ return_value=client,
+ )
+ agent.session_handler.mark_session_active = Mock()
+ agent._prepare_message_with_files = lambda request: request.message
+ agent._delete_ack = AsyncMock()
+ agent.mark_runtime_turn_started = Mock()
+ context = _context(key)
+ request = SimpleNamespace(
+ context=context,
+ message="primary input",
+ working_path="/tmp/work",
+ base_session_id="sess-primary-stop-fence",
+ composite_session_id=key,
+ session_key="session-key",
+ subagent_name=None,
+ subagent_model=None,
+ subagent_reasoning_effort=None,
+ vibe_agent_model=None,
+ vibe_agent_reasoning_effort=None,
+ vibe_agent_system_prompt=None,
+ input_metadata=None,
+ ack_message_id=None,
+ ack_reaction_message_id=None,
+ ack_reaction_emoji=None,
+ files=None,
+ )
+ stop_request = SimpleNamespace(
+ context=_context(key, turn_token="stop-turn"),
+ composite_session_id=key,
+ stop_failure_reason=None,
+ )
+
+ primary_task = asyncio.create_task(agent.handle_message(request))
+ await asyncio.wait_for(query_started.wait(), timeout=1)
+ stop_task = asyncio.create_task(agent.handle_stop(stop_request))
+ await asyncio.sleep(0)
+ self.assertFalse(interrupt_called.is_set())
+
+ release_query.set()
+ await primary_task
+ self.assertTrue(await asyncio.wait_for(stop_task, timeout=1))
+ self.assertTrue(interrupt_called.is_set())
+ receiver_task.cancel()
+ await asyncio.gather(receiver_task, return_exceptions=True)
+
+ async def test_stop_wins_before_write_without_duplicate_prewrite_settlement(self):
+ key = "session-stop-before-write:/tmp/work"
+ agent, _service = _build_agent()
+ agent.session_handler.handle_session_error = AsyncMock()
+ agent._remove_ack_reaction = AsyncMock()
+ agent._delete_ack = AsyncMock()
+ agent._prepare_message_with_files = lambda request: request.message
+ agent.mark_runtime_turn_started = Mock()
+
+ class _Client:
+ _vibe_runtime_base_session_id = "sess-stop-before-write"
+ _vibe_runtime_session_key = key
+
+ def __init__(self):
+ self.query = AsyncMock()
+ self.interrupt_called = False
+
+ async def interrupt(self):
+ self.interrupt_called = True
+
+ async def disconnect(self):
+ return None
+
+ client = _Client()
+ agent.claude_sessions[key] = client
+ receiver_task = asyncio.create_task(asyncio.Event().wait())
+ agent.receiver_tasks[key] = receiver_task
+ agent.session_handler.get_or_create_claude_session = AsyncMock(
+ return_value=client,
+ )
+ context = _context(key)
+ set_dispatch_phase(context, DISPATCH_PHASE_PREWRITE)
+ request = SimpleNamespace(
+ context=context,
+ message="input stopped before write",
+ working_path="/tmp/work",
+ base_session_id="sess-stop-before-write",
+ composite_session_id=key,
+ session_key="session-key",
+ subagent_name=None,
+ subagent_model=None,
+ subagent_reasoning_effort=None,
+ vibe_agent_model=None,
+ vibe_agent_reasoning_effort=None,
+ vibe_agent_system_prompt=None,
+ input_metadata=None,
+ ack_message_id=None,
+ ack_reaction_message_id=None,
+ ack_reaction_emoji=None,
+ files=None,
+ )
+ stop_request = SimpleNamespace(
+ context=_context(key, turn_token="stop-turn"),
+ composite_session_id=key,
+ stop_failure_reason=None,
+ )
+
+ admission = agent._steering_lock(key)
+ await admission.acquire()
+ stop_task = asyncio.create_task(agent.handle_stop(stop_request))
+ await asyncio.sleep(0)
+ primary_task = asyncio.create_task(agent.handle_message(request))
+ await asyncio.sleep(0)
+ admission.release()
+
+ self.assertTrue(await asyncio.wait_for(stop_task, timeout=1))
+ await asyncio.wait_for(primary_task, timeout=1)
+
+ client.query.assert_not_awaited()
+ self.assertTrue(client.interrupt_called)
+ self.assertIs(backend_dispatch_attempted(context), False)
+ self.assertEqual(
+ prewrite_failure_evidence(context),
+ {
+ "reason": "claude_runtime_changed_before_write",
+ "requires_explicit_retry": True,
+ },
+ )
+ agent.session_handler.handle_session_error.assert_not_awaited()
+ # The only terminal emit is Stop's authoritative silent settlement.
+ self.assertEqual(agent.controller.emit_agent_message.await_count, 1)
+ self.assertEqual(
+ agent.controller.emit_agent_message.await_args.kwargs["level"],
+ "silent",
+ )
+ receiver_task.cancel()
+ await asyncio.gather(receiver_task, return_exceptions=True)
+
+ async def test_generation_retirement_waits_for_write_without_lock_cycle(self):
+ key = "session-generation-eviction-watchdog:/tmp/work"
+ agent, _service = _build_agent()
+ query_started = asyncio.Event()
+ release_query = asyncio.Event()
+ generation_lock = asyncio.Lock()
+
+ class _Client:
+ async def query(self, _messages, *, session_id):
+ query_started.set()
+ await release_query.wait()
+
+ async def disconnect(self):
+ return None
+
+ client = _Client()
+ agent.claude_sessions[key] = client
+ agent._pending_requests[key] = [_pending_request(key)]
+ agent.session_handler._claude_runtime_generation_lock = (
+ lambda _key: generation_lock
+ )
+ cleanup_session = AsyncMock()
+ agent.session_handler._cleanup_session_locked = cleanup_session
+
+ async def _primary_write():
+ async with agent._steering_lock(key):
+ await agent._write_human_query(
+ client,
+ key,
+ "primary input",
+ _context(key),
+ )
+
+ await generation_lock.acquire()
+ writer = asyncio.create_task(_primary_write())
+ await asyncio.wait_for(query_started.wait(), timeout=1)
+ eviction = asyncio.create_task(
+ agent.force_cleanup_stuck_active_session(
+ key,
+ runtime_lock_held=True,
+ )
+ )
+ await asyncio.sleep(0)
+ self.assertFalse(eviction.done())
+ cleanup_session.assert_not_awaited()
+
+ release_query.set()
+ await writer
+ await asyncio.wait_for(eviction, timeout=1)
+ cleanup_session.assert_awaited_once()
+ self.assertTrue(generation_lock.locked())
+ generation_lock.release()
+
+ async def test_handle_message_persists_definitely_unsent_input_recovery_evidence(self):
+ key = "session-unsent-recovery:/tmp/work"
+ agent, _service = _build_agent()
+ context = _context(key)
+ set_dispatch_phase(context, DISPATCH_PHASE_PREWRITE)
+ old_client = SimpleNamespace(query=AsyncMock())
+ agent.claude_sessions[key] = SimpleNamespace()
+ agent.session_handler.get_or_create_claude_session = AsyncMock(
+ return_value=old_client
+ )
+ agent.session_handler.handle_session_error = AsyncMock(return_value=True)
+ agent._prepare_message_with_files = lambda request: request.message
+ agent._delete_ack = AsyncMock()
+ agent._remove_ack_reaction = AsyncMock()
+ request = SimpleNamespace(
+ context=context,
+ message="retry me",
+ working_path="/tmp/work",
+ base_session_id="sess-unsent-recovery",
+ composite_session_id=key,
+ session_key="session-key",
+ subagent_name=None,
+ subagent_model=None,
+ subagent_reasoning_effort=None,
+ vibe_agent_model="claude-fixture",
+ input_metadata=None,
+ ack_message_id=None,
+ ack_reaction_message_id=None,
+ ack_reaction_emoji=None,
+ files=None,
+ )
+
+ await agent.handle_message(request)
+
+ old_client.query.assert_not_awaited()
+ self.assertIs(backend_dispatch_attempted(context), False)
+ self.assertEqual(
+ prewrite_failure_evidence(context),
+ {
+ "reason": "claude_runtime_changed_before_write",
+ "requires_explicit_retry": True,
+ },
+ )
+ self.assertFalse(agent._has_pending_requests(key))
+ agent.controller.emit_agent_message.assert_awaited()
+
+ async def test_stop_owned_result_during_reaction_cleanup_cannot_emit(self):
+ key = "session-stop-reaction-cleanup:/tmp/work"
+ agent, _service = _build_agent()
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ cleanup_entered = asyncio.Event()
+ release_cleanup = asyncio.Event()
+
+ async def hold_reaction(*_args, **_kwargs):
+ cleanup_entered.set()
+ await release_cleanup.wait()
+
+ agent._remove_specific_pending_reaction = hold_reaction
+ agent._remove_ack_reaction = AsyncMock()
+ agent._cleanup_runtime_session = AsyncMock()
+ client = SimpleNamespace(interrupt=AsyncMock())
+ agent.claude_sessions[key] = client
+ stop_request = SimpleNamespace(
+ context=_context(key),
+ composite_session_id=key,
+ stop_failure_reason=None,
+ )
+
+ stop = asyncio.create_task(agent.handle_stop(stop_request))
+ try:
+ await asyncio.wait_for(cleanup_entered.wait(), timeout=1)
+ self.assertNotIn(key, agent._pending_requests)
+ await agent._receive_messages(
+ _client([ResultMessage("late human result", origin={"kind": "human"})]),
+ "sess-stop-reaction-cleanup",
+ "/tmp/work",
+ _context(key),
+ composite_key=key,
+ )
+ finally:
+ release_cleanup.set()
+ self.assertTrue(await asyncio.wait_for(stop, timeout=1))
+
+ agent.emit_result_message.assert_not_awaited()
+ terminal = [
+ call
+ for call in agent.controller.emit_agent_message.await_args_list
+ if len(call.args) > 1 and call.args[1] == "result"
+ ]
+ self.assertEqual(len(terminal), 1)
+ self.assertEqual(terminal[0].kwargs["level"], "silent")
+
+ async def test_stop_owned_prewrite_keeps_agent_service_gate_until_cleanup(self):
+ key = "session-stop-service-gate:/tmp/work"
+ agent, service = _build_agent()
+ agent._prepare_message_with_files = lambda request: request.message
+ agent._delete_ack = AsyncMock()
+ agent._remove_ack_reaction = AsyncMock()
+ agent.session_handler.get_or_create_claude_session = AsyncMock()
+ agent.session_handler.handle_session_error = AsyncMock(return_value=True)
+ cleanup_entered = asyncio.Event()
+ release_cleanup = asyncio.Event()
+ query_started = asyncio.Event()
+
+ class _Client:
+ _vibe_runtime_base_session_id = "sess-stop-service-gate"
+ _vibe_runtime_session_key = key
+
+ async def query(self, *_args, **_kwargs):
+ query_started.set()
+
+ async def interrupt(self):
+ return None
+
+ async def disconnect(self):
+ return None
+
+ def receive_messages(self):
+ async def _iterate():
+ if False:
+ yield None
+
+ return _iterate()
+
+ client = _Client()
+ agent.claude_sessions[key] = client
+
+ async def get_client(*_args, **_kwargs):
+ agent.claude_sessions[key] = client
+ return client
+
+ agent.session_handler.get_or_create_claude_session.side_effect = get_client
+
+ original_cleanup = agent._cleanup_runtime_session
+ async def hold_cleanup(*_args, **_kwargs):
+ cleanup_entered.set()
+ await release_cleanup.wait()
+ await original_cleanup(*_args, **_kwargs)
+
+ agent._cleanup_runtime_session = hold_cleanup
+
+ async def settle_stop(context, *_args, **_kwargs):
+ service.release_runtime_turn(context)
+
+ agent.controller.emit_agent_message = AsyncMock(side_effect=settle_stop)
+ admission = agent._steering_lock(key)
+ await admission.acquire()
+
+ def _request(text: str):
+ return SimpleNamespace(
+ context=_context(key, turn_token=text),
+ message=text,
+ working_path="/tmp/work",
+ base_session_id="sess-stop-service-gate",
+ composite_session_id=key,
+ session_key="session-key",
+ subagent_name=None,
+ subagent_model=None,
+ subagent_reasoning_effort=None,
+ vibe_agent_model="claude-fixture",
+ vibe_agent_reasoning_effort=None,
+ vibe_agent_system_prompt=None,
+ input_metadata=None,
+ ack_message_id=None,
+ ack_reaction_message_id=None,
+ ack_reaction_emoji=None,
+ files=None,
+ )
+
+ stop_request = SimpleNamespace(
+ context=_context(key, turn_token="stop"),
+ composite_session_id=key,
+ stop_failure_reason=None,
+ )
+ stop = asyncio.create_task(service.handle_stop("claude", stop_request))
+ await asyncio.sleep(0)
+ primary = asyncio.create_task(service.handle_message("claude", _request("primary")))
+ await asyncio.sleep(0)
+ gate = service._get_turn_gate(key)
+ service._stamp_runtime_turn(stop_request, key, gate.token)
+ admission.release()
+
+ await asyncio.wait_for(cleanup_entered.wait(), timeout=1)
+ successor_request = _request("successor")
+ successor = asyncio.create_task(
+ service.handle_message("claude", successor_request)
+ )
+ await asyncio.sleep(0)
+ self.assertFalse(query_started.is_set())
+ self.assertTrue(gate.lock.locked())
+
+ release_cleanup.set()
+ await asyncio.wait_for(stop, timeout=1)
+ await asyncio.wait_for(query_started.wait(), timeout=1)
+ primary_result = await asyncio.wait_for(primary, timeout=1)
+ self.assertIsNone(primary_result)
+ await asyncio.wait_for(successor, timeout=1)
+ self.assertTrue(gate.lock.locked())
+ service.release_runtime_turn(successor_request.context)
+ self.assertFalse(gate.lock.locked())
+
+ async def test_provenance_persistence_failure_still_settles_terminal_result(self):
+ key = "session-provenance-persistence-failure:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ service.activities.classify_provisional_provenance = Mock(
+ side_effect=RuntimeError("activity store unavailable")
+ )
+
+ await agent._receive_messages(
+ _client(
+ [
+ AssistantMessage(
+ _block(
+ ToolUseBlock,
+ id="tool-persistence-failure",
+ name="Bash",
+ input={"command": "fixture"},
+ )
+ ),
+ TaskStartedMessage(
+ "task-persistence-failure",
+ tool_use_id="tool-persistence-failure",
+ ),
+ ResultMessage("human reply", origin={"kind": "human"}),
+ ]
+ ),
+ "sess-provenance-persistence-failure",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ agent.emit_result_message.assert_awaited_once()
+ self.assertIs(agent.emit_result_message.await_args.kwargs["request"], request)
+ self.assertFalse(agent._has_pending_requests(key))
+ self.assertEqual(
+ agent._provenance_recovery_evidence[key]["owner"],
+ "human",
+ )
+ self.assertEqual(
+ agent._provenance_recovery_evidence[key]["activity_ids"],
+ ("task-persistence-failure",),
+ )
+
+ async def test_detached_synthetic_owner_releases_before_next_human_admission(self):
+ key = "session-synthetic-detached-release:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ agent.emit_result_message = _dispatcher_owned_emit(service)
+
+ await agent._receive_messages(
+ _client(
+ [
+ AssistantMessage(_block(TextBlock, text="detached progress")),
+ ResultMessage(
+ "detached completion",
+ origin={"kind": "task-notification"},
+ ),
+ ]
+ ),
+ "sess-synthetic-detached-release",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertFalse(agent._has_pending_requests(key))
+ self.assertNotIn(key, agent._synthetic_pending_owners)
+ gate = service._get_turn_gate(key)
+ self.assertFalse(gate.lock.locked())
+ self.assertTrue(agent.emit_result_message.await_args.kwargs["output"].detached)
+
+ next_context = _context(key, turn_token="next-human")
+ next_token = await service.begin_agent_initiated_turn(
+ "claude",
+ next_context,
+ key,
+ )
+ self.assertTrue(next_token)
+ service.release_runtime_turn(next_context)
+
+ async def test_detached_synthetic_delivery_retries_with_same_output_identity(self):
+ key = "session-synthetic-detached-retry:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ attempts = []
+
+ async def emit_with_one_transient_failure(*_args, **kwargs):
+ attempts.append(kwargs["output"])
+ if len(attempts) == 1:
+ raise RuntimeError("temporary delivery failure")
+ return "message-id"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_with_one_transient_failure)
+
+ await agent._receive_messages(
+ _client(
+ [
+ AssistantMessage(_block(TextBlock, text="background progress")),
+ ResultMessage(
+ "background completion",
+ origin={"kind": "task-notification"},
+ ),
+ ]
+ ),
+ "sess-synthetic-detached-retry",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertEqual(len(attempts), 2)
+ self.assertIs(attempts[0], attempts[1])
+ self.assertEqual(
+ attempts[0].idempotency_key,
+ attempts[1].idempotency_key,
+ )
+ self.assertFalse(agent._has_pending_requests(key))
+ self.assertNotIn(key, agent._synthetic_pending_owners)
+ self.assertFalse(agent._output_records_for_runtime(key))
+ self.assertFalse(service._get_turn_gate(key).lock.locked())
+
+ # The synthetic owner is retired only after the retry succeeds; a
+ # subsequent human turn can then acquire the same runtime gate.
+ next_context = _context(key, turn_token="next-human")
+ next_token = await service.begin_agent_initiated_turn(
+ "claude",
+ next_context,
+ key,
+ )
+ self.assertTrue(next_token)
+ service.release_runtime_turn(next_context)
+
+ async def test_failed_detached_output_does_not_hold_human_turn_after_receiver_error(
+ self,
+ ):
+ key = "session-detached-output-with-human-pending:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ agent._pending_requests[key] = [request]
+ agent.ACTIVITY_OUTPUT_FLUSH_GRACE_SECONDS = 60
+ agent._get_formatter = lambda _context: SimpleNamespace(
+ format_assistant_message=lambda parts: "\n".join(parts),
+ format_toolcall=lambda *_args, **_kwargs: "",
+ format_toolcall_label=lambda *_args, **_kwargs: "",
+ )
+ agent.controller.agent_auth_service = SimpleNamespace(
+ maybe_emit_auth_recovery_message=AsyncMock(return_value=False),
+ )
+ agent.controller.session_handler.handle_session_error = AsyncMock(
+ return_value=False,
+ )
+ allow_retry = asyncio.Event()
+ attempts: list[str] = []
+
+ async def emit_result(_context, text, **_kwargs):
+ attempts.append(text)
+ if not allow_retry.is_set():
+ raise RuntimeError("detached delivery unavailable")
+ return "message-id"
+
+ agent.emit_result_message = AsyncMock(side_effect=emit_result)
+
+ await agent._receive_messages(
+ _failing_client(
+ [
+ AssistantMessage(_block(TextBlock, text="detached output")),
+ ResultMessage(
+ "detached result",
+ origin={"kind": "task-notification"},
+ ),
+ ],
+ RuntimeError("receiver transport failed"),
+ ),
+ "sess-detached-output-with-human-pending",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertEqual(attempts, ["detached result", "detached result"])
+ self.assertFalse(agent._has_pending_requests(key))
+ self.assertFalse(service.runtime_turn_active(key))
+ retained = agent._output_record(key, None, owner="detached")
+ self.assertIsNotNone(retained)
+ self.assertEqual(retained.text, "detached result")
+
+ allow_retry.set()
+ self.assertFalse(
+ await agent._flush_detached_unsolicited_output(key, context)
+ )
+ self.assertFalse(agent._output_records_for_runtime(key))
+ self.assertEqual(
+ attempts,
+ ["detached result", "detached result", "detached result"],
+ )
+
+ async def test_proven_human_background_activity_keeps_run_lineage(self):
+ key = "session-positive-background-lineage:/tmp/work"
+ agent, service = _build_agent()
+ context = _context(key)
+ request = _pending_request(key)
+ request.context.platform_specific.update(
+ {
+ "task_trigger_kind": "agent_run",
+ "task_execution_id": "human-run",
+ "accepted_agent_run_ids": ["human-run"],
+ "delivery_key_external": "human-delivery",
+ }
+ )
+ agent._pending_requests[key] = [request]
+
+ class _Formatter:
+ @staticmethod
+ def format_toolcall(*_args, **_kwargs):
+ return "fixture tool"
+
+ @staticmethod
+ def format_toolcall_label(*_args, **_kwargs):
+ return "fixture tool"
+
+ agent._get_formatter = lambda _context: _Formatter()
+ agent.emit_result_message = AsyncMock(return_value="message-id")
+ snapshots = []
+
+ class _Client:
+ def receive_messages(self):
+ async def stream():
+ yield AssistantMessage(
+ _block(
+ ToolUseBlock,
+ id="human-background-tool",
+ name="Bash",
+ input={
+ "command": "fixture",
+ "run_in_background": True,
+ },
+ )
+ )
+ yield TaskStartedMessage(
+ "human-background-task",
+ tool_use_id="human-background-tool",
+ )
+ yield ResultMessage(
+ "background started",
+ origin={"kind": "human"},
+ )
+ snapshots.extend(
+ service.activities.active_for_runtime("claude", key)
+ )
+
+ return stream()
+
+ await agent._receive_messages(
+ _Client(),
+ "sess-positive-background-lineage",
+ "/tmp/work",
+ context,
+ composite_key=key,
+ )
+
+ self.assertEqual(len(snapshots), 1)
+ self.assertEqual(snapshots[0].run_id, "human-run")
+ self.assertEqual(snapshots[0].turn_id, "human-turn")
+ self.assertEqual(
+ snapshots[0].metadata.get("delivery_key_external"),
+ "human-delivery",
+ )
+ self.assertFalse(snapshots[0].metadata.get("provenance_pending"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_session_activities.py b/tests/test_session_activities.py
index 74f3ea164..ba9cb87c5 100644
--- a/tests/test_session_activities.py
+++ b/tests/test_session_activities.py
@@ -412,6 +412,74 @@ def test_activity_ack_replaces_undeletable_snapshot_with_terminal_evidence():
assert store.upsert_activity.call_args.kwargs["phase"] == "terminal"
+def test_ack_recovered_terminal_does_not_delete_awaiting_output_receipt():
+ delete_activity = mock.Mock()
+ store = SimpleNamespace(
+ upsert_activity=mock.Mock(),
+ delete_activity=delete_activity,
+ )
+ registry = SessionActivityRegistry(store)
+ registry.start(
+ backend="claude",
+ runtime_key="runtime-1",
+ session_id="ses-1",
+ activity_id="task-1",
+ kind="background_task",
+ )
+ registry.complete(
+ backend="claude",
+ runtime_key="runtime-1",
+ activity_id="task-1",
+ status="completed",
+ expects_output=True,
+ )
+ claimed = registry.claim_completed_output("claude", "runtime-1")
+ assert claimed is not None
+
+ registry.ack_recovered_terminal(claimed)
+
+ delete_activity.assert_not_called()
+ assert registry.has_claimed_output("claude", "runtime-1") is True
+
+
+def test_generation_end_finalizes_unresolved_terminal_snapshot_for_recovery():
+ delete_activity = mock.Mock()
+ store = SimpleNamespace(
+ upsert_activity=mock.Mock(),
+ delete_activity=delete_activity,
+ )
+ registry = SessionActivityRegistry(store)
+ registry.start(
+ backend="claude",
+ runtime_key="runtime-1",
+ session_id="ses-1",
+ activity_id="task-failed",
+ kind="background_task",
+ metadata={"provenance_pending": True, "provenance_phase_id": "phase-1"},
+ )
+ registry.complete(
+ backend="claude",
+ runtime_key="runtime-1",
+ activity_id="task-failed",
+ status="failed",
+ retain_terminal_snapshot=True,
+ )
+
+ ended = registry.end_runtime("claude", "runtime-1")
+
+ assert [(activity.id, activity.status) for activity in ended] == [
+ ("task-failed", "failed"),
+ ]
+ assert ended[0].metadata["provenance_generation_ended"] is True
+ assert ended[0].metadata["provenance_unresolved"] is True
+ assert "provenance_pending" not in ended[0].metadata
+ assert registry.has_backend_work("claude") is True
+
+ registry.ack_recovered_terminal(ended[0])
+ delete_activity.assert_called_once()
+ assert registry.has_backend_work("claude") is False
+
+
def test_delivered_output_without_durable_evidence_stays_claimed_when_terminal_write_fails():
callback = mock.Mock()
store = SimpleNamespace(
@@ -984,6 +1052,83 @@ def test_requeued_bound_batch_does_not_absorb_later_same_turn_completion():
assert later[0].metadata["output_batch_id"] != first_batch_id
+def test_provisional_classification_updates_memory_before_retryable_persistence():
+ class _Store:
+ def __init__(self):
+ self.records = {}
+ self.fail = True
+
+ def upsert_activity(self, activity, *, phase):
+ if self.fail and activity["metadata"].get("provenance_human"):
+ raise RuntimeError("provenance store unavailable")
+ self.records[activity["id"]] = {
+ "activity": dict(activity),
+ "phase": phase,
+ }
+
+ store = _Store()
+ registry = SessionActivityRegistry(store)
+ registry.start(
+ backend="claude",
+ runtime_key="runtime-provenance-retry",
+ session_id="ses-provenance-retry",
+ activity_id="task-provisional",
+ kind="background_task",
+ metadata={
+ "provenance_pending": True,
+ "provenance_phase_id": "phase-1",
+ },
+ )
+
+ classified = registry.classify_provisional_provenance(
+ "claude",
+ "runtime-provenance-retry",
+ activity_ids={"task-provisional"},
+ parent_activity_ids=set(),
+ turn_id="human-turn",
+ run_ids=["human-run"],
+ delivery_key_external="delivery-human",
+ phase_id="phase-1",
+ detached=False,
+ )
+
+ assert classified[0].turn_id == "human-turn"
+ assert classified[0].run_id == "human-run"
+ assert classified[0].metadata["provenance_human"] is True
+ assert registry.active_for_runtime("claude", "runtime-provenance-retry") == classified
+ assert registry.provenance_persistence_recovery(
+ "claude",
+ "runtime-provenance-retry",
+ ) == {"task-provisional": "active: RuntimeError: provenance store unavailable"}
+ assert store.records["task-provisional"]["activity"]["metadata"][
+ "provenance_pending"
+ ] is True
+
+ store.fail = False
+ assert registry.classify_provisional_provenance(
+ "claude",
+ "runtime-provenance-retry",
+ activity_ids={"task-provisional"},
+ parent_activity_ids=set(),
+ turn_id="human-turn",
+ run_ids=["human-run"],
+ delivery_key_external="delivery-human",
+ phase_id="phase-1",
+ detached=False,
+ ) == []
+
+ assert registry.provenance_persistence_recovery(
+ "claude",
+ "runtime-provenance-retry",
+ ) == {}
+ assert store.records["task-provisional"]["activity"]["metadata"][
+ "provenance_human"
+ ] is True
+ assert "provenance_pending" not in store.records["task-provisional"]["activity"][
+ "metadata"
+ ]
+
+
def test_batch_callbacks_run_after_all_claims_release_and_outside_registry_lock():
registry = SessionActivityRegistry()
for activity_id in ("task-a", "task-b"):
@@ -1467,6 +1612,65 @@ def test_force_end_backend_retains_terminal_snapshot_until_ack(tmp_path: Path):
engine.dispose()
+def test_force_ended_snapshots_ack_without_restart(tmp_path: Path):
+ db_path = tmp_path / "state" / "vibe.sqlite"
+ ensure_sqlite_state(db_path=db_path, primary_platform="avibe")
+ engine = create_sqlite_engine(db_path)
+ store = SQLiteSessionActivityStore(engine)
+ registry = SessionActivityRegistry(store)
+ for activity_id in ("active", "queued", "claimed"):
+ registry.start(
+ backend="claude", runtime_key=activity_id, session_id="ses-1",
+ activity_id=activity_id, kind="background_task",
+ )
+ if activity_id != "active":
+ registry.complete(
+ backend="claude", runtime_key=activity_id, activity_id=activity_id,
+ status="completed", expects_output=True,
+ )
+ registry.claim_completed_output_batch("claude", "claimed")
+ ended = registry.end_backend("claude", status="killed")
+ assert len(ended) == 3
+ for activity in ended:
+ registry.ack_recovered_terminal(activity)
+ assert store.list_activities() == []
+ assert SessionActivityRegistry(store).drain_recovered_terminals() == []
+ assert not registry.has_backend_work("claude")
+ engine.dispose()
+
+
[email protected]("ending_old", [False, True])
+def test_ending_generation_leaves_other_provisional_snapshot_classifiable(ending_old):
+ from core.runtime_activation import RuntimeActivationRegistry
+
+ activations = RuntimeActivationRegistry()
+ registry = SessionActivityRegistry(activation_registry=activations)
+ old = activations.attach("claude", "runtime")
+ new = activations.attach("claude", "runtime") if ending_old else None
+ owner = new or old
+ registry.start(
+ backend="claude", runtime_key="runtime", session_id="ses-1",
+ activity_id="old-task", kind="background_task",
+ metadata={"provenance_pending": True}, activation_identity=owner,
+ )
+ registry.complete(
+ backend="claude", runtime_key="runtime", activity_id="old-task",
+ status="failed", retain_terminal_snapshot=True, activation_identity=owner,
+ )
+ ending = old if ending_old else activations.attach("claude", "runtime")
+ assert registry.end_runtime(
+ "claude", "runtime", retain_terminal_snapshots=True, activation_identity=ending,
+ ) == []
+ classified = registry.classify_provisional_provenance(
+ "claude", "runtime", activity_ids={"old-task"}, parent_activity_ids=set(),
+ turn_id="old-turn", run_ids=["old-run"], delivery_key_external="old-delivery",
+ phase_id="old-phase", detached=False,
+ )
+ assert len(classified) == 1
+ assert classified[0].run_id == "old-run"
+ assert not classified[0].metadata.get("provenance_generation_ended")
+
+
def test_force_end_backend_claimed_output_wins_late_delivery_race(tmp_path: Path):
db_path = tmp_path / "state" / "vibe.sqlite"
ensure_sqlite_state(db_path=db_path, primary_platform="avibe")
@@ -1498,11 +1702,63 @@ def test_force_end_backend_claimed_output_wins_late_delivery_race(tmp_path: Path
assert [(item.id, item.status) for item in completed] == [
("task-claimed", "killed"),
]
- assert registry.has_backend_work("claude") is False
+ assert registry.has_backend_work("claude") is True
assert registry.requeue_completed_output(claimed) is False
assert registry.ack_completed_output(claimed) is False
records = store.list_activities()
assert len(records) == 1
assert records[0]["phase"] == "terminal"
assert records[0]["activity"]["status"] == "killed"
+ registry.ack_recovered_terminal(completed[0])
+ assert store.list_activities() == []
+ assert registry.has_backend_work("claude") is False
engine.dispose()
+
+
+def test_metadata_retry_preserves_persisted_receipt_batch_boundary():
+ registry = SessionActivityRegistry()
+
+ def complete(activity_id: str) -> None:
+ registry.start(
+ backend="claude",
+ runtime_key="runtime-1",
+ session_id="ses-1",
+ activity_id=activity_id,
+ kind="background_task",
+ metadata={
+ "provenance_pending": True,
+ "provenance_phase_id": "phase-1",
+ },
+ )
+ registry.complete(
+ backend="claude",
+ runtime_key="runtime-1",
+ activity_id=activity_id,
+ status="completed",
+ expects_output=True,
+ )
+
+ complete("first")
+ first = registry.claim_completed_output_batch(
+ "claude",
+ "runtime-1",
+ metadata_match={
+ "provenance_pending": True,
+ "provenance_phase_id": "phase-1",
+ },
+ )
+ receipt = first[0].metadata["output_batch_id"]
+ assert registry.requeue_completed_outputs(first) == 1
+
+ complete("later")
+ retry = registry.claim_completed_output_batch(
+ "claude",
+ "runtime-1",
+ metadata_match={
+ "provenance_pending": True,
+ "provenance_phase_id": "phase-1",
+ },
+ )
+
+ assert [item.id for item in retry] == ["first"]
+ assert retry[0].metadata["output_batch_id"] == receipt
diff --git a/vibe/i18n/en.json b/vibe/i18n/en.json
index b1fc9531e..d82a1aaa3 100644
--- a/vibe/i18n/en.json
+++ b/vibe/i18n/en.json
@@ -427,6 +427,7 @@
"agentMemoryLimit": "The shared Agent cgroup recorded a memory limit event; concurrent Agent processes may be affected.",
"claudeCredentialTypeInvalid": "Choose either API key or auth token for the Claude Code credential.",
"claudeSessionNotFound": "Claude Code could not find the historical session `{sessionId}` in the current working directory:\n`{path}`\n\nClaude Code sessions are scoped to a working directory. This usually means the conversation was resumed after switching workdirs, or the local Claude Code session data changed. Restore the original session in its original working directory, then explicitly retry the pending input; or start a new conversation from the current directory.",
+ "claudeInputRuntimeChanged": "Your message was not sent because the Claude connection changed before the native write. Your conversation and background work are preserved; retry the pending message when ready.",
"sessionGeneric": "An error occurred: {error}",
"codexPromptRefreshUnavailable": "Codex could not confirm that it can safely refresh Avibe's instructions for this existing session. Check or update Codex, then retry this turn.",
"codexForkBoundaryUnavailable": "Codex could not determine a safe history boundary for this fork. Wait for the source turn to finish, then retry. If the problem persists, check or update Codex.",
diff --git a/vibe/i18n/zh.json b/vibe/i18n/zh.json
index 29510d707..a7c52bec6 100644
--- a/vibe/i18n/zh.json
+++ b/vibe/i18n/zh.json
@@ -427,6 +427,7 @@
"agentMemoryLimit": "共享 Agent cgroup 记录到内存上限事件;并发 Agent 进程可能受到影响。",
"claudeCredentialTypeInvalid": "Claude Code 凭据类型只能选择 API Key 或 Auth Token。",
"claudeSessionNotFound": "Claude Code 在当前工作目录下找不到历史会话 `{sessionId}`:\n`{path}`\n\nClaude Code 的会话和工作目录相关。通常是恢复会话后切换了工作目录,或本地 Claude Code 会话数据发生变化。请在原工作目录恢复原会话,再手动重试待发送的输入;也可以在当前目录开始一个新会话。",
+ "claudeInputRuntimeChanged": "这条消息尚未发送,因为 Claude 连接在本地写入前发生了变化。会话和后台任务均已保留,请在准备好后重试待发送的消息。",
"sessionGeneric": "发生错误:{error}",
"codexPromptRefreshUnavailable": "Codex 无法确认能否安全刷新此现有会话的 Avibe 指令。请检查或升级 Codex 后重试本回合。",
"codexForkBoundaryUnavailable": "Codex 无法确定此次分叉的安全历史边界。请等待源会话当前回合结束后重试;如果问题持续,请检查或升级 Codex。",