CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.
Python agent-frameworkagent-harnessagent-runtimeagentic-aiai-agentsai-frameworkanthropicharnessharness-engineeringharness-frameworklanggraphlanggraph-alternativellmmcpopenaipi-agentpi-agent-corepythonpython-agent
fix(agent): isolate parallel tool-batch faults + typed checkpoint corruption
Claude Code4d9d1409dd8d
diff --git a/cubepi/agent/tools.py b/cubepi/agent/tools.pyindex 167cc92..96f09af 100644--- a/cubepi/agent/tools.py+++ b/cubepi/agent/tools.py@@ -298,9 +298,17 @@ async def _execute_prepared(# AgentToolResult(is_error=True) without raising must surface here# (otherwise the model sees a successful result).return result, bool(result.is_error)- except HitlControlException:+ except (+ HitlControlException,+ asyncio.CancelledError,+ KeyboardInterrupt,+ SystemExit,+ ):raise- except Exception as exc:+ except BaseException as exc:+ # BaseException, not Exception: a tool body raising a bare+ # BaseException subclass must degrade to an error result too —+ # anything that escapes here detonates the whole batch.return _error_result(str(exc)), Truefinally:if token is not None:@@ -350,7 +358,14 @@ async def _finalize(if after_result.is_error is not Noneelse is_error)- except Exception as exc:+ except (+ HitlControlException,+ asyncio.CancelledError,+ KeyboardInterrupt,+ SystemExit,+ ):+ raise+ except BaseException as exc:result = _error_result(str(exc))is_error = True@@ -628,24 +643,120 @@ async def _execute_parallel(return fin# Now that every prepare has succeeded, fan out the executions.+ # `scheduled` stays index-aligned with `entries` so a failed task's+ # _PreparedToolCall (tool_call id/name, hitl_trace) is recoverable when+ # synthesizing its error outcome.scheduled: list[_FinalizedOutcome | asyncio.Task] = [entryif isinstance(entry, _FinalizedOutcome)else asyncio.create_task(_run(entry))for entry in entries]+ tasks = [s for s in scheduled if isinstance(s, asyncio.Task)]++ # Settle EVERY task before processing any outcome: one failing tool must+ # never drop its siblings' results (they were computed, and their side+ # effects happened — losing the ToolResultMessage would leave dangling+ # tool_calls in the checkpoint and duplicate the side effects on resume).+ try:+ await asyncio.gather(*tasks, return_exceptions=True)+ except asyncio.CancelledError:+ # Outer cancel: gather already propagated it to the children.+ # Settle them, salvage the results of tools that completed before+ # the cancel landed, then re-raise. The Agent layer backfills the+ # genuinely-unanswered ids (_complete_cancelled_tool_calls); without+ # the salvage it would stamp "[cancelled]" over real, side-effecting+ # completions. Best-effort: never mask the CancelledError.+ try:+ await asyncio.gather(*tasks, return_exceptions=True)+ salvaged = [+ s if isinstance(s, _FinalizedOutcome) else s.result()+ for s in scheduled+ if isinstance(s, _FinalizedOutcome)+ or (s.done() and not s.cancelled() and s.exception() is None)+ ]+ await _emit_tool_result_messages(salvaged, emit_fn)+ except asyncio.CancelledError:+ raise+ except Exception:+ pass+ raise+finalized_list: list[_FinalizedOutcome] = []- for entry in scheduled:- if isinstance(entry, asyncio.Task):- finalized_list.append(await entry)+ control_exc: BaseException | None = None+ for entry, slot in zip(entries, scheduled):+ if isinstance(slot, _FinalizedOutcome):+ finalized_list.append(slot)+ continue+ if slot.cancelled():+ # Task.exception() would re-raise the CancelledError; outer+ # cancellation re-raised above, so this is a tool self-cancel.+ exc: BaseException | None = asyncio.CancelledError()else:- finalized_list.append(entry)+ exc = slot.exception()+ if exc is None:+ finalized_list.append(slot.result())+ continue+ assert isinstance(entry, _PreparedToolCall)+ if isinstance(exc, (HitlControlException, KeyboardInterrupt, SystemExit)):+ # Must propagate (suspend/interpreter contracts) — but only+ # after every sibling result below has been emitted. The call+ # itself deliberately stays unanswered, with no End event:+ # same shape as a sequential detach; the HITL resume/abort+ # paths backfill it. First raiser (batch order) wins.+ if control_exc is None:+ control_exc = exc+ continue+ # Per-task isolation: a stray CancelledError (tool self-cancel with+ # no outer cancel — a tool bug) or any exception that slipped past+ # _execute_prepared/_finalize degrades to an error result for THIS+ # call only.+ text = (+ "[Tool execution cancelled]"+ if isinstance(exc, asyncio.CancelledError)+ else str(exc)+ )+ synthesized = _FinalizedOutcome(+ tool_call=entry.tool_call,+ result=_error_result(text),+ is_error=True,+ hitl_trace=entry.hitl_trace,+ )+ # Pair the Start emitted inside _run — an unpaired Start leaves+ # state.pending_tool_calls and trace spans open.+ await emit_event(+ emit_fn,+ ToolExecutionEndEvent(+ tool_call_id=entry.tool_call.id,+ tool_name=entry.tool_call.name,+ result=synthesized.result,+ is_error=True,+ terminate=False,+ ),+ )+ finalized_list.append(synthesized)+ if control_exc is not None:+ # Persist the siblings (each MessageEndEvent checkpoints+ # immediately), best-effort, then let the control exception+ # propagate exactly as the suspend/abort machinery expects.+ try:+ await _emit_tool_result_messages(finalized_list, emit_fn)+ except Exception:+ pass+ raise control_exc++ messages = await _emit_tool_result_messages(finalized_list, emit_fn)+ return ToolCallBatch(messages=messages, terminate=_should_terminate(finalized_list))+++async def _emit_tool_result_messages(+ finalized_list: list[_FinalizedOutcome], emit_fn: Callable+) -> list[ToolResultMessage]:messages: list[ToolResultMessage] = []for finalized in finalized_list:tool_msg = _make_tool_result_message(finalized)await emit_event(emit_fn, MessageStartEvent(message=tool_msg))await emit_event(emit_fn, MessageEndEvent(message=tool_msg))messages.append(tool_msg)-- return ToolCallBatch(messages=messages, terminate=_should_terminate(finalized_list))+ return messagesdiff --git a/cubepi/checkpointer/exceptions.py b/cubepi/checkpointer/exceptions.pyindex f9bbf5d..f783579 100644--- a/cubepi/checkpointer/exceptions.py+++ b/cubepi/checkpointer/exceptions.py@@ -85,6 +85,34 @@ class CheckpointerLockTimeoutError(CheckpointerError):(SQLite busy_timeout, etc.)."""+class CheckpointCorruptionError(CheckpointerError):+ """A persisted message row failed to deserialize during load.++ Raised instead of the raw decoder/validation error so hosts can+ distinguish data corruption from operational failures and locate the+ bad row (``row_ref`` names the backend table + primary key). Covers+ bad payload encoding, schema-invalid message data, and unknown roles.+ """++ def __init__(+ self,+ *,+ thread_id: str,+ backend: str,+ row_ref: str,+ cause: BaseException,+ ) -> None:+ super().__init__(+ f"corrupt checkpoint row for thread {thread_id!r} "+ f"({backend}, {row_ref}): {cause}"+ )+ self.thread_id = thread_id+ self.backend = backend+ self.row_ref = row_ref+ self.__cause__ = cause+ self.__suppress_context__ = True++class CompletionMarkerFailedError(CheckpointerError):"""mark_run_complete() failed AFTER the run's final append succeeded.Carries `run_id` so callers using prompt(run_id=None) can recoverdiff --git a/cubepi/checkpointer/mysql/checkpointer.py b/cubepi/checkpointer/mysql/checkpointer.pyindex 5c3454b..a877a6f 100644--- a/cubepi/checkpointer/mysql/checkpointer.py+++ b/cubepi/checkpointer/mysql/checkpointer.py@@ -20,6 +20,7 @@ from pydantic import TypeAdapterfrom cubepi.checkpointer.base import CheckpointDatafrom cubepi.checkpointer.exceptions import (+ CheckpointCorruptionError,RunAlreadyClaimedError,RunAlreadyCompletedError,RunNotClaimedError,@@ -64,6 +65,26 @@ _ROLE_TO_CLS: dict[str, type[Message]] = {}+def _deserialize_row(+ thread_id: str, seq: int, role: str, metadata: Any, payload: Any+) -> Message:+ """Deserialize one cubepi_messages row; corruption raises typed."""+ try:+ cls = _ROLE_TO_CLS.get(role)+ if cls is None:+ raise ValueError(f"unknown role in DB: {role!r}")+ data = msgpack.unpackb(bytes(payload), raw=False)+ data["metadata"] = _decode_json(metadata)+ return cls.model_validate(data)+ except Exception as exc:+ raise CheckpointCorruptionError(+ thread_id=thread_id,+ backend="mysql",+ row_ref=f"cubepi_messages.seq={seq}",+ cause=exc,+ ) from exc++def _parse_dsn(dsn: str) -> dict[str, Any]:"""Parse a mysql:// URL into aiomysql.create_pool kwargs."""u = urlparse(dsn)@@ -209,14 +230,10 @@ class MySQLCheckpointer:if not msg_rows and extra_row is None:return None- messages: list[Message] = []- for _seq, role, metadata, payload in msg_rows:- cls = _ROLE_TO_CLS.get(role)- if cls is None:- raise ValueError(f"unknown role in DB: {role!r}")- data = msgpack.unpackb(bytes(payload), raw=False)- data["metadata"] = _decode_json(metadata)- messages.append(cls.model_validate(data))+ messages: list[Message] = [+ _deserialize_row(thread_id, seq, role, metadata, payload)+ for seq, role, metadata, payload in msg_rows+ ]parent_thread_id: str | None = Noneif extra_row is not None:@@ -472,15 +489,10 @@ class MySQLCheckpointer:(thread_id, thread_id, cutoff),)rows = await cur.fetchall()- messages: list[Message] = []- for _seq, role, metadata, payload in rows:- cls = _ROLE_TO_CLS.get(role)- if cls is None:- raise ValueError(f"unknown role in DB: {role!r}")- data = msgpack.unpackb(bytes(payload), raw=False)- data["metadata"] = _decode_json(metadata)- messages.append(cls.model_validate(data))- return messages+ return [+ _deserialize_row(thread_id, seq, role, metadata, payload)+ for seq, role, metadata, payload in rows+ ]async def fork(self,diff --git a/cubepi/checkpointer/postgres/checkpointer.py b/cubepi/checkpointer/postgres/checkpointer.pyindex c527942..7c6304b 100644--- a/cubepi/checkpointer/postgres/checkpointer.py+++ b/cubepi/checkpointer/postgres/checkpointer.py@@ -16,6 +16,7 @@ from pydantic import TypeAdapterfrom cubepi.checkpointer.base import CheckpointDatafrom cubepi.checkpointer.exceptions import (+ CheckpointCorruptionError,RunAlreadyClaimedError,RunAlreadyCompletedError,RunNotClaimedError,@@ -88,6 +89,29 @@ _ROLE_TO_CLS: dict[str, type[Message]] = {}+def _deserialize_row(thread_id: str, r: Any) -> Message:+ """Deserialize one cubepi_messages row; corruption raises typed."""+ try:+ cls = _ROLE_TO_CLS.get(r["role"])+ if cls is None:+ raise ValueError(f"unknown role in DB: {r['role']!r}")+ data = msgpack.unpackb(bytes(r["payload"]), raw=False)+ # The DB metadata column is the source of truth for Message.metadata.+ # (payload also contains it, but column is the canonical view for querying.)+ raw_meta = r["metadata"]+ data["metadata"] = (+ json.loads(raw_meta) if isinstance(raw_meta, str) else (raw_meta or {})+ )+ return cls.model_validate(data)+ except Exception as exc:+ raise CheckpointCorruptionError(+ thread_id=thread_id,+ backend="postgres",+ row_ref=f"cubepi_messages.seq={r['seq']}",+ cause=exc,+ ) from exc++class PostgresCheckpointer:"""Checkpointer backed by PostgreSQL.@@ -168,19 +192,7 @@ class PostgresCheckpointer:if not msg_rows and extra_row is None:return None- messages: list[Message] = []- for r in msg_rows:- cls = _ROLE_TO_CLS.get(r["role"])- if cls is None:- raise ValueError(f"unknown role in DB: {r['role']!r}")- data = msgpack.unpackb(bytes(r["payload"]), raw=False)- # The DB metadata column is the source of truth for Message.metadata.- # (payload also contains it, but column is the canonical view for querying.)- raw_meta = r["metadata"]- data["metadata"] = (- json.loads(raw_meta) if isinstance(raw_meta, str) else (raw_meta or {})- )- messages.append(cls.model_validate(data))+ messages: list[Message] = [_deserialize_row(thread_id, r) for r in msg_rows]parent_thread_id: str | None = Noneif extra_row is not None:@@ -388,18 +400,7 @@ class PostgresCheckpointer:thread_id,cutoff,)- messages: list[Message] = []- for r in rows:- cls = _ROLE_TO_CLS.get(r["role"])- if cls is None:- raise ValueError(f"unknown role in DB: {r['role']!r}")- data = msgpack.unpackb(bytes(r["payload"]), raw=False)- raw_meta = r["metadata"]- data["metadata"] = (- json.loads(raw_meta) if isinstance(raw_meta, str) else (raw_meta or {})- )- messages.append(cls.model_validate(data))- return messages+ return [_deserialize_row(thread_id, r) for r in rows]async def fork(self,diff --git a/cubepi/checkpointer/sqlite.py b/cubepi/checkpointer/sqlite.pyindex a5d7da8..8d54a3c 100644--- a/cubepi/checkpointer/sqlite.py+++ b/cubepi/checkpointer/sqlite.py@@ -11,6 +11,7 @@ from pydantic import TypeAdapterfrom cubepi.checkpointer.base import CheckpointDatafrom cubepi.checkpointer.exceptions import (+ CheckpointCorruptionError,CheckpointerLockTimeoutError,RunAlreadyClaimedError,RunAlreadyCompletedError,@@ -152,7 +153,7 @@ class SQLiteCheckpointer:assert self._db is not Noneasync with self._lock:cursor = await self._db.execute(- "SELECT message_json FROM messages WHERE thread_id = ? ORDER BY id",+ "SELECT id, message_json FROM messages WHERE thread_id = ? ORDER BY id",(thread_id,),)rows = await cursor.fetchall()@@ -169,8 +170,7 @@ class SQLiteCheckpointer:messages = []for row in rows:- msg_data = json.loads(row[0])- messages.append(_deserialize_message(msg_data))+ messages.append(_deserialize_row(thread_id, row[0], row[1]))extra = json.loads(extra_row[0]) if extra_row else {}parent = extra_row[1] if extra_row else None@@ -413,7 +413,7 @@ class SQLiteCheckpointer:)cutoff = row[0]cur = await self._db.execute(- "SELECT message_json FROM messages WHERE thread_id = ? "+ "SELECT id, message_json FROM messages WHERE thread_id = ? ""AND (run_id IS NULL OR run_id IN ("" SELECT run_id FROM runs WHERE thread_id = ? "" AND completion_seq IS NOT NULL "@@ -422,7 +422,7 @@ class SQLiteCheckpointer:(thread_id, thread_id, cutoff),)rows = await cur.fetchall()- return [_deserialize_message(json.loads(r[0])) for r in rows]+ return [_deserialize_row(thread_id, r[0], r[1]) for r in rows]async def fork(self,@@ -534,4 +534,18 @@ def _deserialize_message(data: dict[str, Any]) -> Message:return AssistantMessage.model_validate(data)elif role == "tool_result":return ToolResultMessage.model_validate(data)- return cast(Message, data)+ # Unknown role is corruption, same as the postgres/mysql backends —+ # passing the raw dict through would fail far from the cause.+ raise ValueError(f"unknown role in DB: {role!r}")+++def _deserialize_row(thread_id: str, row_id: int, message_json: str) -> Message:+ try:+ return _deserialize_message(json.loads(message_json))+ except Exception as exc:+ raise CheckpointCorruptionError(+ thread_id=thread_id,+ backend="sqlite",+ row_ref=f"messages.id={row_id}",+ cause=exc,+ ) from excdiff --git a/dev/plans/2026-07-05-tool-batch-fault-isolation.md b/dev/plans/2026-07-05-tool-batch-fault-isolation.mdnew file mode 100644index 0000000..66f0dc7--- /dev/null+++ b/dev/plans/2026-07-05-tool-batch-fault-isolation.md@@ -0,0 +1,95 @@+# Plan — Parallel Tool-Batch Fault Isolation & Checkpoint Load Hardening++- Date: 2026-07-05+- Spec: `dev/specs/2026-07-05-tool-batch-fault-isolation.md`+- Branch: `2026-07-05-tool-batch-fault-isolation`++## Step 1 — broaden the narrow catches (`cubepi/agent/tools.py`)++1. `_execute_prepared` (`:301-304`): replace the+ `except HitlControlException: raise / except Exception` pair with a+ control-flow re-raise (`HitlControlException`, `asyncio.CancelledError`,+ `KeyboardInterrupt`, `SystemExit`) followed by `except BaseException` →+ `_error_result(str(exc)), True`.+2. `_finalize`'s `after_tool_call` guard (`:353`): same shape. Sequential+ executor inherits both fixes (shared helpers).++## Step 2 — rewrite `_execute_parallel` collect stage (`tools.py:630-651`)++Keep the two-phase prepare and `_run` untouched. Replace the bare-await loop:++1. Build `scheduled` as today (aligned index-wise with `entries`, so a task+ slot's `_PreparedToolCall` — hence `tool_call` id/name and `hitl_trace` —+ is recoverable for synthesis).+2. `await asyncio.gather(*tasks, return_exceptions=True)`.+ - `except asyncio.CancelledError` (outer cancel): re-gather to settle the+ now-cancelled children; best-effort (`except Exception: pass`) emit the+ `ToolResultMessage`s of slots that completed before the cancel (immediate+ outcomes included); re-raise. Agent-layer+ `_complete_cancelled_tool_calls` backfills only the truly-unanswered ids.+3. Classify settled slots in batch order:+ - result present → normal `_FinalizedOutcome`.+ - `HitlControlException` / `KeyboardInterrupt` / `SystemExit` → record+ first as `control_exc`; slot stays unanswered, **no** synthesized End+ event (matches sequential detach shape).+ - `CancelledError` without outer cancel (tool self-cancel) → synthesized+ error outcome, text `[Tool execution cancelled]`.+ - any other exception → synthesized error outcome, text `str(exc)`.+ - Synthesized outcomes emit a `ToolExecutionEndEvent` to pair with the+ Start emitted inside `_run` (keeps `pending_tool_calls`/trace balanced).+4. If `control_exc` is set: best-effort emit the `ToolResultMessage`s for all+ finalized/synthesized slots, then `raise control_exc`.+5. Otherwise: existing tail (build messages, emit, return `ToolCallBatch`).++## Step 3 — `CheckpointCorruptionError` (`cubepi/checkpointer/`)++1. `exceptions.py`: add `CheckpointCorruptionError(CheckpointerError)` with+ `thread_id`, `backend`, `row_ref`, `cause` (spec Part 3 shape).+2. `sqlite.py`:+ - `load()` (`:154-173`): `SELECT id, message_json`; per-row+ `except Exception` → `CheckpointCorruptionError(backend="sqlite",+ row_ref=f"messages.id={id}")`.+ - completed-prefix read (`:415-425`): same (`SELECT id, message_json`).+ - `_deserialize_message` (`:529-537`): unknown role now raises+ `ValueError` instead of returning the raw dict (wrapped by the callers'+ guards; only the two call sites above exist).+3. `postgres/checkpointer.py`: `load()` row loop (`:172-183`) and+ completed-prefix loop (`:392-403`): per-row guard,+ `row_ref=f"seq={r['seq']}"`; the existing unknown-role `ValueError` moves+ inside the guarded block so it wraps uniformly.+4. `mysql/checkpointer.py`: same at `:212-219` and `:475-483`,+ `row_ref=f"seq={seq}"`.+5. Guards catch `Exception` only (never wrap `CancelledError`).++## Step 4 — tests++1. `tests/agent/test_tools.py`, new `TestParallelFaultIsolation`:+ - HITL raise mid-batch → siblings' results emitted (ordered), exception+ re-raised, HITL slot unanswered w/o End event, no still-pending tasks.+ - `after_tool_call` raising a bare `BaseException` subclass → error result+ for that call only (parallel *and* sequential variants).+ - tool-body `CancelledError` (no outer cancel) → `[Tool execution+ cancelled]` error result, siblings unaffected.+ - outer cancellation mid-batch → completed slots' messages emitted before+ `CancelledError` re-raises; unfinished slots unanswered.+ - Start/End pairing for synthesized outcomes.+2. Checkpointer corruption tests (sqlite in `test_sqlite.py`; postgres/mysql+ in their suites behind the existing skip-if-no-DB fixtures): corrupt+ payload row → `CheckpointCorruptionError` with thread_id/backend/row_ref;+ unknown role → same (including sqlite's former silent fallthrough); both+ `load()` and the fork-source read.++## Step 5 — docs (required by workflow)++- `website/docs/guides/agents/tool-use.md`: short "Fault isolation in+ parallel batches" note — one failing tool never drops sibling results;+ HITL/cancel semantics preserved.+- `website/docs/guides/checkpointing/{sqlite,postgres,mysql}.md`:+ troubleshooting entry for `CheckpointCorruptionError` (what it means, that+ `row_ref` locates the bad row).++## Step 6 — gates & PR++`uv run pytest tests/` + `ruff check` + `ruff format --check` + `mypy cubepi`;+commit, push, `gh pr create`; then drive the PR codex review loop+(poll ~2 min → fix → `@codex`) until clean.diff --git a/dev/specs/2026-07-05-tool-batch-fault-isolation.md b/dev/specs/2026-07-05-tool-batch-fault-isolation.mdnew file mode 100644index 0000000..503d64a--- /dev/null+++ b/dev/specs/2026-07-05-tool-batch-fault-isolation.md@@ -0,0 +1,268 @@+# Parallel Tool-Batch Fault Isolation & Checkpoint Load Hardening++- Date: 2026-07-05+- Status: Draft for review+- Related issue: downstream report — one escaping exception in a parallel tool+ batch drops every `ToolResultMessage` in the batch, persisting dangling+ `tool_calls`; combined with no row-level fault handling in checkpointer+ `load()`, the thread 400s on every subsequent turn (permanently bricked).+- Related spec: `2026-06-27-hitl-answer-ledger.md` (two-phase parallel prepare;+ this spec covers the *execute* phase the two-phase split does not protect)+- Companion plan: `dev/plans/2026-07-05-tool-batch-fault-isolation.md`++## Problem++### 1. One escaping exception destroys the whole parallel batch++`_execute_parallel` collects its fan-out tasks with a bare `await` loop+(`cubepi/agent/tools.py:638-642`). Every `ToolResultMessage` is built only+*after* the loop completes (`:644-649`). If any single task raises an exception+that escapes `_execute_prepared`, the loop re-raises at the first failing+`await` and the message-construction block never runs — **zero** tool results+are emitted, including those of sibling tools that already succeeded.++Three exception classes escape today:++- **`HitlControlException`** — deliberately re-raised by the selective handler+ in `_execute_prepared` (`tools.py:301-302`). It subclasses `BaseException`+ (`cubepi/hitl/exceptions.py:4`) precisely so generic `except Exception`+ guards don't swallow a suspend. A tool body that prompts mid-execution+ (`HitlDetached`, `HitlAborted`, …) therefore detonates the batch.+- **`asyncio.CancelledError`** — `BaseException` since Python 3.8; not caught+ by `_execute_prepared`'s `except Exception` (`:303`).+- **Any `BaseException` raised by an `after_tool_call` hook** — `_finalize`+ guards the hook with `except Exception` only (`tools.py:353`), so a+ `BaseException`-shaped hook failure escapes with no handler anywhere in the+ stack.++Reproduced (3-tool parallel batch, one raising `HitlDetached`):++```+execute_tool_calls raised: HitlDetached+ToolResultMessages built: 0 # including the already-succeeded sibling+leaked still-running tasks: 1+side effects after grace: ['fast_ok done',+ 'slow_ok side effect COMMITTED after batch already failed']+```++The repro shows a second defect beyond the dropped results: the collection+loop never cancels or drains sibling tasks, so **still-running siblings leak**+— they keep executing detached, commit their side effects, and their results+are discarded. On resume the batch replays and duplicates those side effects.+This is exactly the hazard the two-phase prepare comment documents and+prevents for the *prepare* stage (`tools.py:539-547`); the *execute* stage has+no equivalent protection.++Downstream consequence: the assistant message carrying the `tool_calls` was+already checkpointed via `MessageEndEvent` (`agent.py:1194-1209`), but no+`tool_result` ever follows it. Every provider rejects such a transcript, so+the next turn — and every turn after — fails with a 400.++#### Existing partial defenses (and their gaps)++- **Two-phase prepare** (`tools.py:539-547`): prevents prepare-stage HITL from+ leaking started tasks. Does not cover a raise from inside a tool body or+ from `after_tool_call`.+- **Agent-layer cancel backfill** (`agent.py:1094-1101`,+ `_complete_cancelled_tool_calls`): on `CancelledError` synthesizes+ tool_results for every unanswered tool_call and re-raises. Covers the cancel+ path only, only for the stateful `Agent` (not `run_loop` driven directly),+ and does nothing about leaked sibling tasks or their lost results (the+ synthetic text claims cancellation even for tools that actually completed).+- **Sequential executor**: emits each `ToolResultMessage` as it goes+ (`tools.py:521-523`), so a mid-sequence raise persists earlier siblings.+ Later calls stay unanswered but the existing HITL resume/abort backfill+ paths answer them. Parallel has no such incremental persistence.++### 2. Checkpointer `load()` has no row-level fault handling++All three SQL backends deserialize message rows in a loop with no per-row+guard:++- sqlite: `cubepi/checkpointer/sqlite.py:171-173` (and the fork-source read at+ `:425`)+- postgres: `cubepi/checkpointer/postgres/checkpointer.py:172-183` (and+ `:396`)+- mysql: `cubepi/checkpointer/mysql/checkpointer.py:214-219` (and `:477-482`)++One corrupt payload (bad msgpack/JSON, failed `model_validate`, unknown role)+raises a raw `msgpack`/`ValueError`/`ValidationError` out of `load()` and the+entire thread becomes unloadable — the caller can't tell corruption from an+operational error, and can't tell *which* row is bad.++A related inconsistency: on an unknown role, postgres and mysql raise+`ValueError`, while sqlite's `_deserialize_message` silently returns the raw+dict (`sqlite.py:537`) — corruption flows straight into the message list as a+non-`Message` object and fails later, far from the cause.++## Goals++1. A failure in one tool call of a parallel batch must never destroy the+ results of its siblings, leak running sibling tasks, or persist dangling+ `tool_calls` — while preserving the HITL suspend/abort and cancellation+ contracts, which *require* certain exceptions to propagate.+2. Checkpoint corruption must surface as a typed, row-locatable error — never+ as a raw decoder exception, and never as silently-wrong data.++Non-goals: send-time transcript repair at the provider seam (hosts may keep+their own belt-and-suspenders); a skip-corrupt-rows load mode (see+Alternatives); multi-slot HITL within one batch (single-slot channel semantics+are owned by the answer-ledger spec).++## Design++### Part 1 — per-task fault isolation in `_execute_parallel`++The two-phase prepare stays exactly as is. Only the fan-out/collect stage+changes.++**Collect with `asyncio.gather(*tasks, return_exceptions=True)`** instead of+the bare `await` loop. Every task settles before any outcome is processed;+results stay in tool_call order. Then classify each slot:++1. **`_FinalizedOutcome`** — success path, unchanged.+2. **Ordinary failure** (any exception that is not `HitlControlException`,+ `CancelledError`, `KeyboardInterrupt`, or `SystemExit`) — synthesize an+ error outcome for that tool_call id:+ `AgentToolResult(content=[TextContent(text=str(exc))], is_error=True)`,+ plus a paired `ToolExecutionEndEvent` (the Start was emitted inside+ `_run`; an unpaired Start would leave `state.pending_tool_calls` and trace+ spans open — same invariant the prepare-phase comment protects).+ With Part 2 in place this is defense-in-depth: tool-body and hook failures+ are already converted inside the task.+3. **`HitlControlException`** — the suspend contract requires propagation, so+ it cannot become an error result. Order of operations: first emit the+ `ToolResultMessage`s for every *other* settled slot (each `MessageEndEvent`+ checkpoints immediately via `Agent._process_event`, `agent.py:1207-1209`),+ then re-raise the HITL exception. The HITL call's own tool_call+ deliberately stays unanswered and gets no End event — identical to today's+ sequential detach shape — and the existing detach/resume/abort paths+ answer it (synthetic-deny backfill scans unanswered ids of the last+ assistant message). If several slots raise HITL exceptions, the first in+ batch order wins; the rest also stay unanswered and are covered by the+ same backfill.+4. **`CancelledError` in a slot, without outer cancellation** — a tool body+ raised it spuriously (tool bug). Treat as an ordinary failure (case 2)+ with text `[Tool execution cancelled]`. Escalating one buggy tool's+ self-cancel into run-level cancellation would kill sibling work for no+ reason. *(Flagged as a decision point — the conservative alternative is to+ re-raise after draining, like case 3.)* Note the inherent+ sequential/parallel asymmetry: in sequential mode tools run in the+ executor's own task, so a self-raised `CancelledError` is+ indistinguishable from a genuine external cancel and must keep+ propagating (the Agent-layer backfill covers it); only the parallel+ executor can tell the two apart (`task.cancelled()` vs an exception with+ no outer cancel).+5. **Outer cancellation** (the executor itself is cancelled while awaiting+ the gather): `gather` propagates the cancel to all children. Catch the+ `CancelledError`, await the tasks once more with+ `return_exceptions=True` so they fully settle, emit the+ `ToolResultMessage`s of slots that completed *before* the cancel landed,+ and re-raise. The Agent-layer `_complete_cancelled_tool_calls` backfill+ then covers only the genuinely-unanswered ids — fixing today's secondary+ bug where completed tools get a synthetic "[cancelled]" result and their+ real results (and side effects) are lost, causing duplicate execution on+ resume.+6. **`KeyboardInterrupt` / `SystemExit`** — settle children, re-raise. Never+ converted to results.++The `terminate` aggregation (`_should_terminate`) runs over real plus+synthesized outcomes, unchanged in shape.++### Part 2 — broaden the narrow catches at the source++- `_execute_prepared` (`tools.py:303`): `except Exception` becomes a+ `BaseException` guard that re-raises `HitlControlException`,+ `CancelledError`, `KeyboardInterrupt`, `SystemExit` and converts everything+ else to an error result.+- `_finalize`'s `after_tool_call` guard (`tools.py:353`): same treatment. A+ hook raising a `BaseException` subclass currently escapes *both* executors+ and every Agent-layer handler (it is neither `CancelledError` nor+ `Exception`, `agent.py:1094/1102`) — after this change it degrades to an+ error result on that one call.++The sequential executor gets the same per-call robustness for free, since it+shares both helpers.++### Part 3 — typed, row-locatable checkpoint corruption++New exception in `cubepi/checkpointer/exceptions.py`, following the+`CompletionMarkerFailedError` shape:++```python+class CheckpointCorruptionError(CheckpointerError):+ """A persisted message row failed to deserialize during load."""++ def __init__(self, *, thread_id: str, backend: str, row_ref: str,+ cause: BaseException) -> None:+ super().__init__(+ f"corrupt checkpoint row for thread {thread_id!r} "+ f"({backend}, {row_ref}): {cause}")+ self.thread_id = thread_id+ self.backend = backend # "sqlite" | "postgres" | "mysql"+ self.row_ref = row_ref # e.g. "messages.id=42" — locates the row+ self.__cause__ = cause+ self.__suppress_context__ = True+```++Every row-deserialization site (the six listed in Problem §2) wraps its+per-row work — decode, role lookup, `model_validate` — in a guard that raises+`CheckpointCorruptionError` with the row's primary key. The SELECTs gain the+pk column where they don't already fetch it.++Unknown role becomes corruption too, in all three backends — including+sqlite, whose current silent raw-dict fallthrough (`sqlite.py:537`) is+replaced. **This is a deliberate behavior change**: any host that depended on+unknown-role dicts passing through sqlite load was already broken downstream;+failing loudly at the source with the row pk is strictly more debuggable.++`load()` still fails the whole thread on a corrupt row — but now with a typed+error that names the row, so hosts can catch `CheckpointCorruptionError`+distinctly from operational `CheckpointerError`s and repair or quarantine the+row surgically.++## Alternatives considered++- **`asyncio.TaskGroup` instead of `gather`** — TaskGroup cancels all siblings+ on first failure. That is the opposite of the requirement (siblings must+ finish and their results must persist), and ExceptionGroup unwrapping adds+ noise. `gather(return_exceptions=True)` has exactly the settle-everything+ semantics needed.+- **Convert HITL exceptions to error results** — breaks the suspend contract:+ detach/abort rely on propagation to reach `run_loop`'s outer handlers+ (`loop.py:445-458`) and the channel's pending-state cleanup.+- **Skip corrupt rows on load** (per-row tolerance in the recovery sense) —+ rejected for v1: silently dropping an assistant message that carries+ `tool_calls`, or a `tool_result`, manufactures the very dangling-tool_call+ transcript this spec exists to prevent. If a lossy mode is ever wanted it+ must synthesize structural repairs, not just skip — deferred until a+ concrete need appears; the typed error with `row_ref` is the enabling step+ either way.+- **Emit each parallel result as its task completes** (as-completed+ streaming, matching the sequential executor) — would also fix the loss but+ changes observable event ordering for hosts and interleaves checkpointer+ appends with running tools; settle-then-emit preserves today's ordering+ with the same durability outcome.++## Testing++- `tests/agent/test_tools.py` (parallel executor, using the existing+ `AgentTool` fixtures):+ - HITL raise mid-batch: siblings' `ToolResultMessage`s emitted and in+ order, `HitlDetached` re-raised, HITL call has no result and no End+ event, no still-running tasks after the raise.+ - `after_tool_call` raising a bare `BaseException`: converted to an error+ result for that call only; batch completes.+ - Tool body raising `CancelledError` without outer cancel: error result,+ siblings unaffected.+ - Outer cancellation mid-batch: completed slots' results emitted before the+ re-raise; unfinished slots have none (Agent backfill covers them —+ asserted at the Agent layer with the memory checkpointer).+ - Start/End event pairing holds for every synthesized outcome.+- Checkpointer suites (sqlite, postgres, mysql — reusing each backend's+ existing test harness): corrupt payload row → `CheckpointCorruptionError`+ carrying thread_id/backend/row_ref; unknown role → same, including the+ sqlite fallthrough replacement; fork-source read paths covered as well as+ `load()`.+- Regression: full existing suite (the repro script's scenario becomes a+ permanent test).diff --git a/tests/agent/test_checkpointer_integration.py b/tests/agent/test_checkpointer_integration.pyindex 7ed0cf7..55f120b 100644--- a/tests/agent/test_checkpointer_integration.py+++ b/tests/agent/test_checkpointer_integration.py@@ -197,14 +197,19 @@ class TestCheckpointerIntegration:class BlockParams(BaseModel):pass- async def cancel_execute(tool_call_id, params, *, signal=None, on_update=None):- raise asyncio.CancelledError()+ started = asyncio.Event()++ async def blocking_execute(+ tool_call_id, params, *, signal=None, on_update=None+ ):+ started.set()+ await asyncio.Event().wait() # blocks until the run is cancelledblock_tool = AgentTool(name="block",- description="Never returns; simulates a cancelled tool call",+ description="Never returns; cancelled mid-execution",parameters=BlockParams,- execute=cancel_execute,+ execute=blocking_execute,)checkpointer = MemoryCheckpointer()@@ -224,8 +229,11 @@ class TestCheckpointerIntegration:thread_id="thread-cancel",)+ prompt_task = asyncio.create_task(agent.prompt("Run the blocking tool"))+ await started.wait()+ prompt_task.cancel()with pytest.raises(asyncio.CancelledError):- await agent.prompt("Run the blocking tool")+ await prompt_taskdata = await checkpointer.load("thread-cancel")assert data is not None@@ -247,14 +255,19 @@ class TestCheckpointerIntegration:class BlockParams(BaseModel):pass- async def cancel_execute(tool_call_id, params, *, signal=None, on_update=None):- raise asyncio.CancelledError()+ started = asyncio.Event()++ async def blocking_execute(+ tool_call_id, params, *, signal=None, on_update=None+ ):+ started.set()+ await asyncio.Event().wait() # blocks until the run is cancelledblock_tool = AgentTool(name="block",- description="Cancels on execution",+ description="Never returns; cancelled mid-execution",parameters=BlockParams,- execute=cancel_execute,+ execute=blocking_execute,)checkpointer = MemoryCheckpointer()@@ -291,8 +304,11 @@ class TestCheckpointerIntegration:thread_id="thread-reuse",)+ prompt_task = asyncio.create_task(agent.prompt("second"))+ await started.wait()+ prompt_task.cancel()with pytest.raises(asyncio.CancelledError):- await agent.prompt("second")+ await prompt_taskdata = await checkpointer.load("thread-reuse")assert data is not Nonediff --git a/tests/agent/test_tools.py b/tests/agent/test_tools.pyindex ec671cb..cfa01b6 100644--- a/tests/agent/test_tools.py+++ b/tests/agent/test_tools.py@@ -1,5 +1,6 @@import asyncio+import pytestfrom pydantic import BaseModelfrom cubepi.agent.tools import execute_tool_calls@@ -644,3 +645,205 @@ class TestToolResultDetails:assert len(batch.messages) == 1assert batch.messages[0].details == {"execution_time": 42}+++class TestParallelFaultIsolation:+ """One failing tool in a parallel batch must never drop sibling results,+ leak running sibling tasks, or leave dangling tool_calls."""++ @staticmethod+ def _three_call_msg():+ return make_assistant_msg(+ [+ ToolCall(id="t1", name="a", arguments={"value": "x"}),+ ToolCall(id="t2", name="b", arguments={"value": "x"}),+ ToolCall(id="t3", name="c", arguments={"value": "x"}),+ ]+ )++ async def test_hitl_raise_preserves_and_persists_sibling_results(self):+ from cubepi.hitl.exceptions import HitlDetached++ side_effects = []++ async def fast_ok(tool_call_id, params, *, signal=None, on_update=None):+ side_effects.append("fast")+ return AgentToolResult(content=[TextContent(text="fast-ok")])++ async def hitl_raiser(tool_call_id, params, *, signal=None, on_update=None):+ await asyncio.sleep(0.01)+ raise HitlDetached()++ async def slow_ok(tool_call_id, params, *, signal=None, on_update=None):+ # Still running when the HITL exception fires — the batch must+ # wait for it (no detached task, no lost side effect).+ await asyncio.sleep(0.05)+ side_effects.append("slow")+ return AgentToolResult(content=[TextContent(text="slow-ok")])++ ctx = make_context(+ [+ make_echo_tool(name="a", execute_fn=fast_ok),+ make_echo_tool(name="b", execute_fn=hitl_raiser),+ make_echo_tool(name="c", execute_fn=slow_ok),+ ]+ )+ events = []+ with pytest.raises(HitlDetached):+ await execute_tool_calls(+ ctx,+ self._three_call_msg(),+ tool_execution="parallel",+ emit=lambda e: events.append(e),+ )++ # Every sibling settled before the re-raise; nothing leaked.+ assert side_effects == ["fast", "slow"]+ pending = [+ t+ for t in asyncio.all_tasks()+ if t is not asyncio.current_task() and not t.done()+ ]+ assert pending == []++ # Sibling ToolResultMessages were emitted (message_end checkpoints+ # them at the Agent layer); the HITL call stays unanswered.+ result_ids = [e.message.tool_call_id for e in events if e.type == "message_end"]+ assert result_ids == ["t1", "t3"]+ end_ids = [e.tool_call_id for e in events if e.type == "tool_execution_end"]+ assert "t2" not in end_ids # detach shape: Start without End++ async def test_tool_body_base_exception_becomes_error_result(self):+ class Weird(BaseException):+ pass++ async def raiser(tool_call_id, params, *, signal=None, on_update=None):+ raise Weird("weird failure")++ async def ok(tool_call_id, params, *, signal=None, on_update=None):+ return AgentToolResult(content=[TextContent(text="ok")])++ ctx = make_context(+ [+ make_echo_tool(name="a", execute_fn=ok),+ make_echo_tool(name="b", execute_fn=raiser),+ make_echo_tool(name="c", execute_fn=ok),+ ]+ )+ batch = await execute_tool_calls(+ ctx, self._three_call_msg(), tool_execution="parallel", emit=lambda e: None+ )+ assert [m.tool_call_id for m in batch.messages] == ["t1", "t2", "t3"]+ by_id = {m.tool_call_id: m for m in batch.messages}+ assert by_id["t2"].is_error+ assert "weird failure" in by_id["t2"].content[0].text+ assert not by_id["t1"].is_error+ assert not by_id["t3"].is_error++ async def test_after_tool_call_base_exception_isolated(self):+ class Weird(BaseException):+ pass++ async def after(after_ctx, *, signal=None):+ if after_ctx.tool_call.id == "t2":+ raise Weird("hook blew up")+ return None++ ctx = make_context(+ [+ make_echo_tool(name="a"),+ make_echo_tool(name="b"),+ make_echo_tool(name="c"),+ ]+ )+ for mode in ("parallel", "sequential"):+ batch = await execute_tool_calls(+ ctx,+ self._three_call_msg(),+ tool_execution=mode,+ after_tool_call=after,+ emit=lambda e: None,+ )+ by_id = {m.tool_call_id: m for m in batch.messages}+ assert len(by_id) == 3, mode+ assert by_id["t2"].is_error, mode+ assert "hook blew up" in by_id["t2"].content[0].text+ assert not by_id["t1"].is_error+ assert not by_id["t3"].is_error++ async def test_tool_self_cancel_becomes_error_result(self):+ async def self_cancel(tool_call_id, params, *, signal=None, on_update=None):+ raise asyncio.CancelledError()++ async def ok(tool_call_id, params, *, signal=None, on_update=None):+ return AgentToolResult(content=[TextContent(text="ok")])++ ctx = make_context(+ [+ make_echo_tool(name="a", execute_fn=ok),+ make_echo_tool(name="b", execute_fn=self_cancel),+ make_echo_tool(name="c", execute_fn=ok),+ ]+ )+ events = []+ batch = await execute_tool_calls(+ ctx,+ self._three_call_msg(),+ tool_execution="parallel",+ emit=lambda e: events.append(e),+ )+ by_id = {m.tool_call_id: m for m in batch.messages}+ assert len(by_id) == 3+ assert by_id["t2"].is_error+ assert "[Tool execution cancelled]" in by_id["t2"].content[0].text+ # The synthesized outcome still pairs the Start emitted inside the+ # task, so pending_tool_calls / trace spans don't leak.+ starts = [e.tool_call_id for e in events if e.type == "tool_execution_start"]+ ends = [e.tool_call_id for e in events if e.type == "tool_execution_end"]+ assert sorted(starts) == sorted(ends) == ["t1", "t2", "t3"]++ async def test_outer_cancel_salvages_completed_results(self):+ started = asyncio.Event()+ cancelled_flags = []++ async def fast_ok(tool_call_id, params, *, signal=None, on_update=None):+ return AgentToolResult(content=[TextContent(text="fast-ok")])++ async def hang(tool_call_id, params, *, signal=None, on_update=None):+ started.set()+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ cancelled_flags.append(tool_call_id)+ raise+ return AgentToolResult(content=[TextContent(text="unreachable")])++ ctx = make_context(+ [+ make_echo_tool(name="a", execute_fn=fast_ok),+ make_echo_tool(name="b", execute_fn=hang),+ make_echo_tool(name="c", execute_fn=hang),+ ]+ )+ events = []+ runner = asyncio.create_task(+ execute_tool_calls(+ ctx,+ self._three_call_msg(),+ tool_execution="parallel",+ emit=lambda e: events.append(e),+ )+ )+ await started.wait()+ await asyncio.sleep(0.02) # let fast_ok finish+ runner.cancel()+ with pytest.raises(asyncio.CancelledError):+ await runner++ # Hanging siblings were cancelled, not leaked.+ assert sorted(cancelled_flags) == ["t2", "t3"]+ # The completed tool's real result was salvaged and emitted before+ # the re-raise; the cancelled ones stay unanswered for the Agent+ # layer's backfill.+ result_ids = [e.message.tool_call_id for e in events if e.type == "message_end"]+ assert result_ids == ["t1"]diff --git a/tests/checkpointer/test_mysql.py b/tests/checkpointer/test_mysql.pyindex ee6ae6d..8354d81 100644--- a/tests/checkpointer/test_mysql.py+++ b/tests/checkpointer/test_mysql.py@@ -511,9 +511,12 @@ async def test_empty_version_table_raises_uninitialized(clean_mysql_db) -> None:@pytest.mark.asyncioasync def test_load_unknown_role_raises(clean_mysql_db) -> None:- """A message row with an unrecognized role string → ValueError on load."""+ """A message row with an unrecognized role string → typed corruption+ error on load, naming the row."""import msgpack+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError+from cubepi.checkpointer.mysql import MySQLCheckpointerfrom cubepi.checkpointer.mysql.checkpointer import _parse_dsn@@ -533,8 +536,10 @@ async def test_load_unknown_role_raises(clean_mysql_db) -> None:await conn.ensure_closed()async with MySQLCheckpointer(clean_mysql_db) as cp:- with pytest.raises(ValueError, match="unknown role in DB"):+ with pytest.raises(CheckpointCorruptionError, match="unknown role in DB") as ei:await cp.load("t-bad")+ assert isinstance(ei.value.__cause__, ValueError)+ assert ei.value.row_ref == "cubepi_messages.seq=1"@pytest.mark.asyncio@@ -559,3 +564,44 @@ async def test_missing_version_column_raises_uninitialized(clean_mysql_db) -> Nowith pytest.raises(CubepiSchemaUninitialized):async with MySQLCheckpointer(clean_mysql_db):pass+++async def test_mysql_load_corrupt_row_raises_typed(clean_mysql_db) -> None:+ """One bad payload row surfaces as CheckpointCorruptionError naming the+ row — not a raw msgpack error that hides which row is bad."""+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError+ from cubepi.checkpointer.mysql import MySQLCheckpointer+ from cubepi.checkpointer.mysql.checkpointer import _parse_dsn+ from cubepi.providers.base import TextContent, UserMessage++ await _setup_schema(clean_mysql_db)+ async with MySQLCheckpointer(clean_mysql_db) as cp:+ await cp.append(+ "t-corrupt",+ [+ UserMessage(content=[TextContent(text="ok")]),+ UserMessage(content=[TextContent(text="will corrupt")]),+ ],+ )+ conn = await aiomysql.connect(autocommit=True, **_parse_dsn(clean_mysql_db))+ try:+ async with conn.cursor() as cur:+ await cur.execute(+ "UPDATE cubepi_messages SET payload = %s "+ "WHERE thread_id = 't-corrupt' AND seq = ("+ " SELECT max_seq FROM (SELECT max(seq) AS max_seq "+ " FROM cubepi_messages WHERE thread_id = 't-corrupt') AS sub)",+ (b"\xc1 not msgpack",),+ )+ finally:+ await conn.ensure_closed()++ with pytest.raises(CheckpointCorruptionError) as excinfo:+ await cp.load("t-corrupt")++ err = excinfo.value+ assert err.thread_id == "t-corrupt"+ assert err.backend == "mysql"+ assert err.row_ref.startswith("cubepi_messages.seq=")+ assert err.__cause__ is not Nonediff --git a/tests/checkpointer/test_postgres.py b/tests/checkpointer/test_postgres.pyindex c5cfc95..b24b854 100644--- a/tests/checkpointer/test_postgres.py+++ b/tests/checkpointer/test_postgres.py@@ -348,3 +348,66 @@ async def test_empty_thread_load_returns_none(clean_db) -> None:async with PostgresCheckpointer(clean_db) as cp:data = await cp.load("nonexistent-thread")assert data is None+++async def test_postgres_load_corrupt_row_raises_typed(clean_db) -> None:+ """One bad payload row surfaces as CheckpointCorruptionError naming the+ row — not a raw msgpack error that hides which row is bad."""+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError+ from cubepi.checkpointer.postgres import PostgresCheckpointer+ from cubepi.providers.base import TextContent, UserMessage++ await _setup_schema(clean_db)+ async with PostgresCheckpointer(clean_db) as cp:+ await cp.append(+ "t-corrupt",+ [+ UserMessage(content=[TextContent(text="ok")]),+ UserMessage(content=[TextContent(text="will corrupt")]),+ ],+ )+ conn = await asyncpg.connect(clean_db)+ try:+ await conn.execute(+ "UPDATE cubepi_messages SET payload = $1 "+ "WHERE thread_id = 't-corrupt' AND seq = ("+ " SELECT max(seq) FROM cubepi_messages "+ " WHERE thread_id = 't-corrupt')",+ b"\xc1 not msgpack",+ )+ finally:+ await conn.close()++ with pytest.raises(CheckpointCorruptionError) as excinfo:+ await cp.load("t-corrupt")++ err = excinfo.value+ assert err.thread_id == "t-corrupt"+ assert err.backend == "postgres"+ assert err.row_ref.startswith("cubepi_messages.seq=")+ assert err.__cause__ is not None+++async def test_postgres_load_unknown_role_raises_typed(clean_db) -> None:+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError+ from cubepi.checkpointer.postgres import PostgresCheckpointer+ from cubepi.providers.base import TextContent, UserMessage++ await _setup_schema(clean_db)+ async with PostgresCheckpointer(clean_db) as cp:+ await cp.append("t-role", [UserMessage(content=[TextContent(text="ok")])])+ conn = await asyncpg.connect(clean_db)+ try:+ await conn.execute(+ "UPDATE cubepi_messages SET role = 'alien' WHERE thread_id = 't-role'"+ )+ finally:+ await conn.close()++ with pytest.raises(CheckpointCorruptionError) as excinfo:+ await cp.load("t-role")++ assert isinstance(excinfo.value.__cause__, ValueError)+ assert "alien" in str(excinfo.value)diff --git a/tests/checkpointer/test_sqlite.py b/tests/checkpointer/test_sqlite.pyindex ffe1d37..a271480 100644--- a/tests/checkpointer/test_sqlite.py+++ b/tests/checkpointer/test_sqlite.py@@ -101,12 +101,61 @@ class TestSQLiteCheckpointer:assert loaded.content[0].text == "result"async def test_deserialize_unknown_role(self, db_path):- """A message with an unknown role should be returned as a plain dict."""+ """Unknown roles are corruption, matching the postgres/mysql+ backends — the old silent raw-dict passthrough let bad data flow+ into the message list and fail far from the cause."""+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError+async with SQLiteCheckpointer(db_path) as cp:raw_msg = {"role": "custom", "data": "test"}await cp.append("thread-1", [raw_msg])- data = await cp.load("thread-1")- assert data is not None- assert len(data.messages) == 1- assert data.messages[0] == {"role": "custom", "data": "test"}+ with pytest.raises(CheckpointCorruptionError):+ await cp.load("thread-1")+++class TestCheckpointCorruption:+ async def test_corrupt_json_row_raises_typed(self, db_path):+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError++ async with SQLiteCheckpointer(db_path) as cp:+ await cp.append(+ "thread-1",+ [+ UserMessage(content=[TextContent(text="ok")]),+ UserMessage(content=[TextContent(text="will corrupt")]),+ ],+ )+ await cp._db.execute(+ 'UPDATE messages SET message_json = \'{"role": "user"\' '+ "WHERE id = (SELECT max(id) FROM messages WHERE thread_id = ?)",+ ("thread-1",),+ )+ await cp._db.commit()++ with pytest.raises(CheckpointCorruptionError) as excinfo:+ await cp.load("thread-1")++ err = excinfo.value+ assert err.thread_id == "thread-1"+ assert err.backend == "sqlite"+ assert err.row_ref.startswith("messages.id=")+ assert err.__cause__ is not None++ async def test_unknown_role_raises_typed(self, db_path):+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError++ async with SQLiteCheckpointer(db_path) as cp:+ await cp.append("thread-1", [UserMessage(content=[TextContent(text="ok")])])+ await cp._db.execute(+ "UPDATE messages SET message_json = "+ '\'{"role": "alien", "content": []}\' WHERE thread_id = ?',+ ("thread-1",),+ )+ await cp._db.commit()++ with pytest.raises(CheckpointCorruptionError) as excinfo:+ await cp.load("thread-1")++ assert isinstance(excinfo.value.__cause__, ValueError)+ assert "alien" in str(excinfo.value)diff --git a/website/docs/guides/agents/tool-use.md b/website/docs/guides/agents/tool-use.mdindex b28bce0..201c1c7 100644--- a/website/docs/guides/agents/tool-use.md+++ b/website/docs/guides/agents/tool-use.md@@ -123,6 +123,27 @@ The event stream emits `tool_execution_start` for all of them upfront, interleaved `tool_execution_update` events as each tool reportsprogress, then `tool_execution_end` per tool in completion order.+### Fault isolation++Failures are isolated per call: every tool in the batch runs to+completion regardless of what its siblings do, and each call gets+exactly one `tool_result` — a real one, or a synthesized error result+if the tool (or an `after_tool_call` hook) raised something+unexpected. One misbehaving tool never discards a sibling's completed+work, and a persisted assistant message is never left with unanswered+`tool_calls` (which providers reject on the next turn).++Two exception classes still propagate, by design:++- **HITL control exceptions** (a tool detaching for human input) — the+ batch first waits for every sibling to finish and persists their+ results, then the suspend propagates. Only the detaching call stays+ unanswered, and the HITL resume/abort flow answers it later.+- **Cancellation** — siblings are cancelled and awaited (no leaked+ tasks); results of tools that completed before the cancel are+ persisted so a resumed thread doesn't re-run them and duplicate+ their side effects.+## Forcing sequential executionThere are two ways to opt out of parallel mode:diff --git a/website/docs/guides/checkpointing/mysql.md b/website/docs/guides/checkpointing/mysql.mdindex 4fcb9e2..4163731 100644--- a/website/docs/guides/checkpointing/mysql.md+++ b/website/docs/guides/checkpointing/mysql.md@@ -270,6 +270,11 @@ def upgrade():- **Case-insensitive thread-ID collisions** — Thread IDs use the`utf8mb4_bin` collation so `UserA` and `usera` stay distinct. If youhand-roll the DDL, keep that collation.+- **`CheckpointCorruptionError` on `load()`** — A persisted message row+ failed to deserialize (bad msgpack payload, schema-invalid data, or an+ unknown role). The error's `row_ref` (e.g. `cubepi_messages.seq=42`)+ locates the bad row for inspection or surgical repair; `thread_id` and+ `__cause__` carry the rest. CubePi never skips corrupt rows silently.## See alsodiff --git a/website/docs/guides/checkpointing/postgres.md b/website/docs/guides/checkpointing/postgres.mdindex cea4061..e091003 100644--- a/website/docs/guides/checkpointing/postgres.md+++ b/website/docs/guides/checkpointing/postgres.md@@ -269,6 +269,13 @@ def upgrade():`MetaData` instance precisely so it can coexist with your app'smodels without colliding. Don't merge them into your global metadata— pass both to Alembic separately.+- **`CheckpointCorruptionError` on `load()`** — A persisted message row+ failed to deserialize (bad msgpack payload, schema-invalid data, or an+ unknown role). The error's `row_ref` (e.g. `cubepi_messages.seq=42`)+ locates the bad row for inspection or surgical repair; `thread_id` and+ `__cause__` carry the rest. CubePi never skips corrupt rows silently —+ dropping a message that carries `tool_calls` would leave the+ transcript in a state every provider rejects.## See alsodiff --git a/website/docs/guides/checkpointing/sqlite.md b/website/docs/guides/checkpointing/sqlite.mdindex 9271b73..99fa18c 100644--- a/website/docs/guides/checkpointing/sqlite.md+++ b/website/docs/guides/checkpointing/sqlite.md@@ -212,6 +212,13 @@ sqlite3 agent.db "VACUUM"- **Forgetting to set `thread_id`** — Without it, the agent has nopersistence binding. The checkpointer is silently ignored. Alwayspass both.+- **`CheckpointCorruptionError` on `load()`** — A persisted message row+ failed to deserialize (bad JSON, schema-invalid data, or an unknown+ role). The error's `row_ref` (e.g. `messages.id=42`) locates the bad+ row so you can inspect or repair it with plain SQL; `thread_id` and+ `__cause__` carry the rest of the context. CubePi never skips corrupt+ rows silently — dropping a message that carries `tool_calls` would+ leave the transcript in a state every provider rejects.## See also