Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

fix(agent): isolate parallel tool-batch faults + typed checkpoint corruption

Claude Code4d9d1409dd8d
diff --git a/cubepi/agent/tools.py b/cubepi/agent/tools.py
index 167cc92..96f09af 100644
--- a/cubepi/agent/tools.py
+++ b/cubepi/agent/tools.py
@@ -298,9 +298,17 @@ async def _execute_prepared(
# AgentToolResult(is_error=True) without raising must surface here
# (otherwise the model sees a successful result).
return result, bool(result.is_error)
- except HitlControlException:
+ except (
+ HitlControlException,
+ asyncio.CancelledError,
+ KeyboardInterrupt,
+ SystemExit,
+ ):
raise
- except Exception as exc:
+ except BaseException as exc:
+ # BaseException, not Exception: a tool body raising a bare
+ # BaseException subclass must degrade to an error result too —
+ # anything that escapes here detonates the whole batch.
return _error_result(str(exc)), True
finally:
if token is not None:
@@ -350,7 +358,14 @@ async def _finalize(
if after_result.is_error is not None
else is_error
)
- except Exception as exc:
+ except (
+ HitlControlException,
+ asyncio.CancelledError,
+ KeyboardInterrupt,
+ SystemExit,
+ ):
+ raise
+ except BaseException as exc:
result = _error_result(str(exc))
is_error = True
@@ -628,24 +643,120 @@ async def _execute_parallel(
return fin
# Now that every prepare has succeeded, fan out the executions.
+ # `scheduled` stays index-aligned with `entries` so a failed task's
+ # _PreparedToolCall (tool_call id/name, hitl_trace) is recoverable when
+ # synthesizing its error outcome.
scheduled: list[_FinalizedOutcome | asyncio.Task] = [
entry
if isinstance(entry, _FinalizedOutcome)
else asyncio.create_task(_run(entry))
for entry in entries
]
+ tasks = [s for s in scheduled if isinstance(s, asyncio.Task)]
+
+ # Settle EVERY task before processing any outcome: one failing tool must
+ # never drop its siblings' results (they were computed, and their side
+ # effects happened — losing the ToolResultMessage would leave dangling
+ # tool_calls in the checkpoint and duplicate the side effects on resume).
+ try:
+ await asyncio.gather(*tasks, return_exceptions=True)
+ except asyncio.CancelledError:
+ # Outer cancel: gather already propagated it to the children.
+ # Settle them, salvage the results of tools that completed before
+ # the cancel landed, then re-raise. The Agent layer backfills the
+ # genuinely-unanswered ids (_complete_cancelled_tool_calls); without
+ # the salvage it would stamp "[cancelled]" over real, side-effecting
+ # completions. Best-effort: never mask the CancelledError.
+ try:
+ await asyncio.gather(*tasks, return_exceptions=True)
+ salvaged = [
+ s if isinstance(s, _FinalizedOutcome) else s.result()
+ for s in scheduled
+ if isinstance(s, _FinalizedOutcome)
+ or (s.done() and not s.cancelled() and s.exception() is None)
+ ]
+ await _emit_tool_result_messages(salvaged, emit_fn)
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ pass
+ raise
+
finalized_list: list[_FinalizedOutcome] = []
- for entry in scheduled:
- if isinstance(entry, asyncio.Task):
- finalized_list.append(await entry)
+ control_exc: BaseException | None = None
+ for entry, slot in zip(entries, scheduled):
+ if isinstance(slot, _FinalizedOutcome):
+ finalized_list.append(slot)
+ continue
+ if slot.cancelled():
+ # Task.exception() would re-raise the CancelledError; outer
+ # cancellation re-raised above, so this is a tool self-cancel.
+ exc: BaseException | None = asyncio.CancelledError()
else:
- finalized_list.append(entry)
+ exc = slot.exception()
+ if exc is None:
+ finalized_list.append(slot.result())
+ continue
+ assert isinstance(entry, _PreparedToolCall)
+ if isinstance(exc, (HitlControlException, KeyboardInterrupt, SystemExit)):
+ # Must propagate (suspend/interpreter contracts) — but only
+ # after every sibling result below has been emitted. The call
+ # itself deliberately stays unanswered, with no End event:
+ # same shape as a sequential detach; the HITL resume/abort
+ # paths backfill it. First raiser (batch order) wins.
+ if control_exc is None:
+ control_exc = exc
+ continue
+ # Per-task isolation: a stray CancelledError (tool self-cancel with
+ # no outer cancel — a tool bug) or any exception that slipped past
+ # _execute_prepared/_finalize degrades to an error result for THIS
+ # call only.
+ text = (
+ "[Tool execution cancelled]"
+ if isinstance(exc, asyncio.CancelledError)
+ else str(exc)
+ )
+ synthesized = _FinalizedOutcome(
+ tool_call=entry.tool_call,
+ result=_error_result(text),
+ is_error=True,
+ hitl_trace=entry.hitl_trace,
+ )
+ # Pair the Start emitted inside _run — an unpaired Start leaves
+ # state.pending_tool_calls and trace spans open.
+ await emit_event(
+ emit_fn,
+ ToolExecutionEndEvent(
+ tool_call_id=entry.tool_call.id,
+ tool_name=entry.tool_call.name,
+ result=synthesized.result,
+ is_error=True,
+ terminate=False,
+ ),
+ )
+ finalized_list.append(synthesized)
+ if control_exc is not None:
+ # Persist the siblings (each MessageEndEvent checkpoints
+ # immediately), best-effort, then let the control exception
+ # propagate exactly as the suspend/abort machinery expects.
+ try:
+ await _emit_tool_result_messages(finalized_list, emit_fn)
+ except Exception:
+ pass
+ raise control_exc
+
+ messages = await _emit_tool_result_messages(finalized_list, emit_fn)
+ return ToolCallBatch(messages=messages, terminate=_should_terminate(finalized_list))
+
+
+async def _emit_tool_result_messages(
+ finalized_list: list[_FinalizedOutcome], emit_fn: Callable
+) -> list[ToolResultMessage]:
messages: list[ToolResultMessage] = []
for finalized in finalized_list:
tool_msg = _make_tool_result_message(finalized)
await emit_event(emit_fn, MessageStartEvent(message=tool_msg))
await emit_event(emit_fn, MessageEndEvent(message=tool_msg))
messages.append(tool_msg)
-
- return ToolCallBatch(messages=messages, terminate=_should_terminate(finalized_list))
+ return messages
diff --git a/cubepi/checkpointer/exceptions.py b/cubepi/checkpointer/exceptions.py
index f9bbf5d..f783579 100644
--- a/cubepi/checkpointer/exceptions.py
+++ b/cubepi/checkpointer/exceptions.py
@@ -85,6 +85,34 @@ class CheckpointerLockTimeoutError(CheckpointerError):
(SQLite busy_timeout, etc.)."""
+class CheckpointCorruptionError(CheckpointerError):
+ """A persisted message row failed to deserialize during load.
+
+ Raised instead of the raw decoder/validation error so hosts can
+ distinguish data corruption from operational failures and locate the
+ bad row (``row_ref`` names the backend table + primary key). Covers
+ bad payload encoding, schema-invalid message data, and unknown roles.
+ """
+
+ def __init__(
+ self,
+ *,
+ thread_id: str,
+ backend: str,
+ row_ref: str,
+ cause: BaseException,
+ ) -> None:
+ super().__init__(
+ f"corrupt checkpoint row for thread {thread_id!r} "
+ f"({backend}, {row_ref}): {cause}"
+ )
+ self.thread_id = thread_id
+ self.backend = backend
+ self.row_ref = row_ref
+ self.__cause__ = cause
+ self.__suppress_context__ = True
+
+
class CompletionMarkerFailedError(CheckpointerError):
"""mark_run_complete() failed AFTER the run's final append succeeded.
Carries `run_id` so callers using prompt(run_id=None) can recover
diff --git a/cubepi/checkpointer/mysql/checkpointer.py b/cubepi/checkpointer/mysql/checkpointer.py
index 5c3454b..a877a6f 100644
--- a/cubepi/checkpointer/mysql/checkpointer.py
+++ b/cubepi/checkpointer/mysql/checkpointer.py
@@ -20,6 +20,7 @@ from pydantic import TypeAdapter
from cubepi.checkpointer.base import CheckpointData
from cubepi.checkpointer.exceptions import (
+ CheckpointCorruptionError,
RunAlreadyClaimedError,
RunAlreadyCompletedError,
RunNotClaimedError,
@@ -64,6 +65,26 @@ _ROLE_TO_CLS: dict[str, type[Message]] = {
}
+def _deserialize_row(
+ thread_id: str, seq: int, role: str, metadata: Any, payload: Any
+) -> Message:
+ """Deserialize one cubepi_messages row; corruption raises typed."""
+ try:
+ cls = _ROLE_TO_CLS.get(role)
+ if cls is None:
+ raise ValueError(f"unknown role in DB: {role!r}")
+ data = msgpack.unpackb(bytes(payload), raw=False)
+ data["metadata"] = _decode_json(metadata)
+ return cls.model_validate(data)
+ except Exception as exc:
+ raise CheckpointCorruptionError(
+ thread_id=thread_id,
+ backend="mysql",
+ row_ref=f"cubepi_messages.seq={seq}",
+ cause=exc,
+ ) from exc
+
+
def _parse_dsn(dsn: str) -> dict[str, Any]:
"""Parse a mysql:// URL into aiomysql.create_pool kwargs."""
u = urlparse(dsn)
@@ -209,14 +230,10 @@ class MySQLCheckpointer:
if not msg_rows and extra_row is None:
return None
- messages: list[Message] = []
- for _seq, role, metadata, payload in msg_rows:
- cls = _ROLE_TO_CLS.get(role)
- if cls is None:
- raise ValueError(f"unknown role in DB: {role!r}")
- data = msgpack.unpackb(bytes(payload), raw=False)
- data["metadata"] = _decode_json(metadata)
- messages.append(cls.model_validate(data))
+ messages: list[Message] = [
+ _deserialize_row(thread_id, seq, role, metadata, payload)
+ for seq, role, metadata, payload in msg_rows
+ ]
parent_thread_id: str | None = None
if extra_row is not None:
@@ -472,15 +489,10 @@ class MySQLCheckpointer:
(thread_id, thread_id, cutoff),
)
rows = await cur.fetchall()
- messages: list[Message] = []
- for _seq, role, metadata, payload in rows:
- cls = _ROLE_TO_CLS.get(role)
- if cls is None:
- raise ValueError(f"unknown role in DB: {role!r}")
- data = msgpack.unpackb(bytes(payload), raw=False)
- data["metadata"] = _decode_json(metadata)
- messages.append(cls.model_validate(data))
- return messages
+ return [
+ _deserialize_row(thread_id, seq, role, metadata, payload)
+ for seq, role, metadata, payload in rows
+ ]
async def fork(
self,
diff --git a/cubepi/checkpointer/postgres/checkpointer.py b/cubepi/checkpointer/postgres/checkpointer.py
index c527942..7c6304b 100644
--- a/cubepi/checkpointer/postgres/checkpointer.py
+++ b/cubepi/checkpointer/postgres/checkpointer.py
@@ -16,6 +16,7 @@ from pydantic import TypeAdapter
from cubepi.checkpointer.base import CheckpointData
from cubepi.checkpointer.exceptions import (
+ CheckpointCorruptionError,
RunAlreadyClaimedError,
RunAlreadyCompletedError,
RunNotClaimedError,
@@ -88,6 +89,29 @@ _ROLE_TO_CLS: dict[str, type[Message]] = {
}
+def _deserialize_row(thread_id: str, r: Any) -> Message:
+ """Deserialize one cubepi_messages row; corruption raises typed."""
+ try:
+ cls = _ROLE_TO_CLS.get(r["role"])
+ if cls is None:
+ raise ValueError(f"unknown role in DB: {r['role']!r}")
+ data = msgpack.unpackb(bytes(r["payload"]), raw=False)
+ # The DB metadata column is the source of truth for Message.metadata.
+ # (payload also contains it, but column is the canonical view for querying.)
+ raw_meta = r["metadata"]
+ data["metadata"] = (
+ json.loads(raw_meta) if isinstance(raw_meta, str) else (raw_meta or {})
+ )
+ return cls.model_validate(data)
+ except Exception as exc:
+ raise CheckpointCorruptionError(
+ thread_id=thread_id,
+ backend="postgres",
+ row_ref=f"cubepi_messages.seq={r['seq']}",
+ cause=exc,
+ ) from exc
+
+
class PostgresCheckpointer:
"""Checkpointer backed by PostgreSQL.
@@ -168,19 +192,7 @@ class PostgresCheckpointer:
if not msg_rows and extra_row is None:
return None
- messages: list[Message] = []
- for r in msg_rows:
- cls = _ROLE_TO_CLS.get(r["role"])
- if cls is None:
- raise ValueError(f"unknown role in DB: {r['role']!r}")
- data = msgpack.unpackb(bytes(r["payload"]), raw=False)
- # The DB metadata column is the source of truth for Message.metadata.
- # (payload also contains it, but column is the canonical view for querying.)
- raw_meta = r["metadata"]
- data["metadata"] = (
- json.loads(raw_meta) if isinstance(raw_meta, str) else (raw_meta or {})
- )
- messages.append(cls.model_validate(data))
+ messages: list[Message] = [_deserialize_row(thread_id, r) for r in msg_rows]
parent_thread_id: str | None = None
if extra_row is not None:
@@ -388,18 +400,7 @@ class PostgresCheckpointer:
thread_id,
cutoff,
)
- messages: list[Message] = []
- for r in rows:
- cls = _ROLE_TO_CLS.get(r["role"])
- if cls is None:
- raise ValueError(f"unknown role in DB: {r['role']!r}")
- data = msgpack.unpackb(bytes(r["payload"]), raw=False)
- raw_meta = r["metadata"]
- data["metadata"] = (
- json.loads(raw_meta) if isinstance(raw_meta, str) else (raw_meta or {})
- )
- messages.append(cls.model_validate(data))
- return messages
+ return [_deserialize_row(thread_id, r) for r in rows]
async def fork(
self,
diff --git a/cubepi/checkpointer/sqlite.py b/cubepi/checkpointer/sqlite.py
index a5d7da8..8d54a3c 100644
--- a/cubepi/checkpointer/sqlite.py
+++ b/cubepi/checkpointer/sqlite.py
@@ -11,6 +11,7 @@ from pydantic import TypeAdapter
from cubepi.checkpointer.base import CheckpointData
from cubepi.checkpointer.exceptions import (
+ CheckpointCorruptionError,
CheckpointerLockTimeoutError,
RunAlreadyClaimedError,
RunAlreadyCompletedError,
@@ -152,7 +153,7 @@ class SQLiteCheckpointer:
assert self._db is not None
async with self._lock:
cursor = await self._db.execute(
- "SELECT message_json FROM messages WHERE thread_id = ? ORDER BY id",
+ "SELECT id, message_json FROM messages WHERE thread_id = ? ORDER BY id",
(thread_id,),
)
rows = await cursor.fetchall()
@@ -169,8 +170,7 @@ class SQLiteCheckpointer:
messages = []
for row in rows:
- msg_data = json.loads(row[0])
- messages.append(_deserialize_message(msg_data))
+ messages.append(_deserialize_row(thread_id, row[0], row[1]))
extra = json.loads(extra_row[0]) if extra_row else {}
parent = extra_row[1] if extra_row else None
@@ -413,7 +413,7 @@ class SQLiteCheckpointer:
)
cutoff = row[0]
cur = await self._db.execute(
- "SELECT message_json FROM messages WHERE thread_id = ? "
+ "SELECT id, message_json FROM messages WHERE thread_id = ? "
"AND (run_id IS NULL OR run_id IN ("
" SELECT run_id FROM runs WHERE thread_id = ? "
" AND completion_seq IS NOT NULL "
@@ -422,7 +422,7 @@ class SQLiteCheckpointer:
(thread_id, thread_id, cutoff),
)
rows = await cur.fetchall()
- return [_deserialize_message(json.loads(r[0])) for r in rows]
+ return [_deserialize_row(thread_id, r[0], r[1]) for r in rows]
async def fork(
self,
@@ -534,4 +534,18 @@ def _deserialize_message(data: dict[str, Any]) -> Message:
return AssistantMessage.model_validate(data)
elif role == "tool_result":
return ToolResultMessage.model_validate(data)
- return cast(Message, data)
+ # Unknown role is corruption, same as the postgres/mysql backends —
+ # passing the raw dict through would fail far from the cause.
+ raise ValueError(f"unknown role in DB: {role!r}")
+
+
+def _deserialize_row(thread_id: str, row_id: int, message_json: str) -> Message:
+ try:
+ return _deserialize_message(json.loads(message_json))
+ except Exception as exc:
+ raise CheckpointCorruptionError(
+ thread_id=thread_id,
+ backend="sqlite",
+ row_ref=f"messages.id={row_id}",
+ cause=exc,
+ ) from exc
diff --git a/dev/plans/2026-07-05-tool-batch-fault-isolation.md b/dev/plans/2026-07-05-tool-batch-fault-isolation.md
new file mode 100644
index 0000000..66f0dc7
--- /dev/null
+++ b/dev/plans/2026-07-05-tool-batch-fault-isolation.md
@@ -0,0 +1,95 @@
+# Plan — Parallel Tool-Batch Fault Isolation & Checkpoint Load Hardening
+
+- Date: 2026-07-05
+- Spec: `dev/specs/2026-07-05-tool-batch-fault-isolation.md`
+- Branch: `2026-07-05-tool-batch-fault-isolation`
+
+## Step 1 — broaden the narrow catches (`cubepi/agent/tools.py`)
+
+1. `_execute_prepared` (`:301-304`): replace the
+ `except HitlControlException: raise / except Exception` pair with a
+ control-flow re-raise (`HitlControlException`, `asyncio.CancelledError`,
+ `KeyboardInterrupt`, `SystemExit`) followed by `except BaseException` →
+ `_error_result(str(exc)), True`.
+2. `_finalize`'s `after_tool_call` guard (`:353`): same shape. Sequential
+ executor inherits both fixes (shared helpers).
+
+## Step 2 — rewrite `_execute_parallel` collect stage (`tools.py:630-651`)
+
+Keep the two-phase prepare and `_run` untouched. Replace the bare-await loop:
+
+1. Build `scheduled` as today (aligned index-wise with `entries`, so a task
+ slot's `_PreparedToolCall` — hence `tool_call` id/name and `hitl_trace` —
+ is recoverable for synthesis).
+2. `await asyncio.gather(*tasks, return_exceptions=True)`.
+ - `except asyncio.CancelledError` (outer cancel): re-gather to settle the
+ now-cancelled children; best-effort (`except Exception: pass`) emit the
+ `ToolResultMessage`s of slots that completed before the cancel (immediate
+ outcomes included); re-raise. Agent-layer
+ `_complete_cancelled_tool_calls` backfills only the truly-unanswered ids.
+3. Classify settled slots in batch order:
+ - result present → normal `_FinalizedOutcome`.
+ - `HitlControlException` / `KeyboardInterrupt` / `SystemExit` → record
+ first as `control_exc`; slot stays unanswered, **no** synthesized End
+ event (matches sequential detach shape).
+ - `CancelledError` without outer cancel (tool self-cancel) → synthesized
+ error outcome, text `[Tool execution cancelled]`.
+ - any other exception → synthesized error outcome, text `str(exc)`.
+ - Synthesized outcomes emit a `ToolExecutionEndEvent` to pair with the
+ Start emitted inside `_run` (keeps `pending_tool_calls`/trace balanced).
+4. If `control_exc` is set: best-effort emit the `ToolResultMessage`s for all
+ finalized/synthesized slots, then `raise control_exc`.
+5. Otherwise: existing tail (build messages, emit, return `ToolCallBatch`).
+
+## Step 3 — `CheckpointCorruptionError` (`cubepi/checkpointer/`)
+
+1. `exceptions.py`: add `CheckpointCorruptionError(CheckpointerError)` with
+ `thread_id`, `backend`, `row_ref`, `cause` (spec Part 3 shape).
+2. `sqlite.py`:
+ - `load()` (`:154-173`): `SELECT id, message_json`; per-row
+ `except Exception` → `CheckpointCorruptionError(backend="sqlite",
+ row_ref=f"messages.id={id}")`.
+ - completed-prefix read (`:415-425`): same (`SELECT id, message_json`).
+ - `_deserialize_message` (`:529-537`): unknown role now raises
+ `ValueError` instead of returning the raw dict (wrapped by the callers'
+ guards; only the two call sites above exist).
+3. `postgres/checkpointer.py`: `load()` row loop (`:172-183`) and
+ completed-prefix loop (`:392-403`): per-row guard,
+ `row_ref=f"seq={r['seq']}"`; the existing unknown-role `ValueError` moves
+ inside the guarded block so it wraps uniformly.
+4. `mysql/checkpointer.py`: same at `:212-219` and `:475-483`,
+ `row_ref=f"seq={seq}"`.
+5. Guards catch `Exception` only (never wrap `CancelledError`).
+
+## Step 4 — tests
+
+1. `tests/agent/test_tools.py`, new `TestParallelFaultIsolation`:
+ - HITL raise mid-batch → siblings' results emitted (ordered), exception
+ re-raised, HITL slot unanswered w/o End event, no still-pending tasks.
+ - `after_tool_call` raising a bare `BaseException` subclass → error result
+ for that call only (parallel *and* sequential variants).
+ - tool-body `CancelledError` (no outer cancel) → `[Tool execution
+ cancelled]` error result, siblings unaffected.
+ - outer cancellation mid-batch → completed slots' messages emitted before
+ `CancelledError` re-raises; unfinished slots unanswered.
+ - Start/End pairing for synthesized outcomes.
+2. Checkpointer corruption tests (sqlite in `test_sqlite.py`; postgres/mysql
+ in their suites behind the existing skip-if-no-DB fixtures): corrupt
+ payload row → `CheckpointCorruptionError` with thread_id/backend/row_ref;
+ unknown role → same (including sqlite's former silent fallthrough); both
+ `load()` and the fork-source read.
+
+## Step 5 — docs (required by workflow)
+
+- `website/docs/guides/agents/tool-use.md`: short "Fault isolation in
+ parallel batches" note — one failing tool never drops sibling results;
+ HITL/cancel semantics preserved.
+- `website/docs/guides/checkpointing/{sqlite,postgres,mysql}.md`:
+ troubleshooting entry for `CheckpointCorruptionError` (what it means, that
+ `row_ref` locates the bad row).
+
+## Step 6 — gates & PR
+
+`uv run pytest tests/` + `ruff check` + `ruff format --check` + `mypy cubepi`;
+commit, push, `gh pr create`; then drive the PR codex review loop
+(poll ~2 min → fix → `@codex`) until clean.
diff --git a/dev/specs/2026-07-05-tool-batch-fault-isolation.md b/dev/specs/2026-07-05-tool-batch-fault-isolation.md
new file mode 100644
index 0000000..503d64a
--- /dev/null
+++ b/dev/specs/2026-07-05-tool-batch-fault-isolation.md
@@ -0,0 +1,268 @@
+# Parallel Tool-Batch Fault Isolation & Checkpoint Load Hardening
+
+- Date: 2026-07-05
+- Status: Draft for review
+- Related issue: downstream report — one escaping exception in a parallel tool
+ batch drops every `ToolResultMessage` in the batch, persisting dangling
+ `tool_calls`; combined with no row-level fault handling in checkpointer
+ `load()`, the thread 400s on every subsequent turn (permanently bricked).
+- Related spec: `2026-06-27-hitl-answer-ledger.md` (two-phase parallel prepare;
+ this spec covers the *execute* phase the two-phase split does not protect)
+- Companion plan: `dev/plans/2026-07-05-tool-batch-fault-isolation.md`
+
+## Problem
+
+### 1. One escaping exception destroys the whole parallel batch
+
+`_execute_parallel` collects its fan-out tasks with a bare `await` loop
+(`cubepi/agent/tools.py:638-642`). Every `ToolResultMessage` is built only
+*after* the loop completes (`:644-649`). If any single task raises an exception
+that escapes `_execute_prepared`, the loop re-raises at the first failing
+`await` and the message-construction block never runs — **zero** tool results
+are emitted, including those of sibling tools that already succeeded.
+
+Three exception classes escape today:
+
+- **`HitlControlException`** — deliberately re-raised by the selective handler
+ in `_execute_prepared` (`tools.py:301-302`). It subclasses `BaseException`
+ (`cubepi/hitl/exceptions.py:4`) precisely so generic `except Exception`
+ guards don't swallow a suspend. A tool body that prompts mid-execution
+ (`HitlDetached`, `HitlAborted`, …) therefore detonates the batch.
+- **`asyncio.CancelledError`** — `BaseException` since Python 3.8; not caught
+ by `_execute_prepared`'s `except Exception` (`:303`).
+- **Any `BaseException` raised by an `after_tool_call` hook** — `_finalize`
+ guards the hook with `except Exception` only (`tools.py:353`), so a
+ `BaseException`-shaped hook failure escapes with no handler anywhere in the
+ stack.
+
+Reproduced (3-tool parallel batch, one raising `HitlDetached`):
+
+```
+execute_tool_calls raised: HitlDetached
+ToolResultMessages built: 0 # including the already-succeeded sibling
+leaked still-running tasks: 1
+side effects after grace: ['fast_ok done',
+ 'slow_ok side effect COMMITTED after batch already failed']
+```
+
+The repro shows a second defect beyond the dropped results: the collection
+loop never cancels or drains sibling tasks, so **still-running siblings leak**
+— they keep executing detached, commit their side effects, and their results
+are discarded. On resume the batch replays and duplicates those side effects.
+This is exactly the hazard the two-phase prepare comment documents and
+prevents for the *prepare* stage (`tools.py:539-547`); the *execute* stage has
+no equivalent protection.
+
+Downstream consequence: the assistant message carrying the `tool_calls` was
+already checkpointed via `MessageEndEvent` (`agent.py:1194-1209`), but no
+`tool_result` ever follows it. Every provider rejects such a transcript, so
+the next turn — and every turn after — fails with a 400.
+
+#### Existing partial defenses (and their gaps)
+
+- **Two-phase prepare** (`tools.py:539-547`): prevents prepare-stage HITL from
+ leaking started tasks. Does not cover a raise from inside a tool body or
+ from `after_tool_call`.
+- **Agent-layer cancel backfill** (`agent.py:1094-1101`,
+ `_complete_cancelled_tool_calls`): on `CancelledError` synthesizes
+ tool_results for every unanswered tool_call and re-raises. Covers the cancel
+ path only, only for the stateful `Agent` (not `run_loop` driven directly),
+ and does nothing about leaked sibling tasks or their lost results (the
+ synthetic text claims cancellation even for tools that actually completed).
+- **Sequential executor**: emits each `ToolResultMessage` as it goes
+ (`tools.py:521-523`), so a mid-sequence raise persists earlier siblings.
+ Later calls stay unanswered but the existing HITL resume/abort backfill
+ paths answer them. Parallel has no such incremental persistence.
+
+### 2. Checkpointer `load()` has no row-level fault handling
+
+All three SQL backends deserialize message rows in a loop with no per-row
+guard:
+
+- sqlite: `cubepi/checkpointer/sqlite.py:171-173` (and the fork-source read at
+ `:425`)
+- postgres: `cubepi/checkpointer/postgres/checkpointer.py:172-183` (and
+ `:396`)
+- mysql: `cubepi/checkpointer/mysql/checkpointer.py:214-219` (and `:477-482`)
+
+One corrupt payload (bad msgpack/JSON, failed `model_validate`, unknown role)
+raises a raw `msgpack`/`ValueError`/`ValidationError` out of `load()` and the
+entire thread becomes unloadable — the caller can't tell corruption from an
+operational error, and can't tell *which* row is bad.
+
+A related inconsistency: on an unknown role, postgres and mysql raise
+`ValueError`, while sqlite's `_deserialize_message` silently returns the raw
+dict (`sqlite.py:537`) — corruption flows straight into the message list as a
+non-`Message` object and fails later, far from the cause.
+
+## Goals
+
+1. A failure in one tool call of a parallel batch must never destroy the
+ results of its siblings, leak running sibling tasks, or persist dangling
+ `tool_calls` — while preserving the HITL suspend/abort and cancellation
+ contracts, which *require* certain exceptions to propagate.
+2. Checkpoint corruption must surface as a typed, row-locatable error — never
+ as a raw decoder exception, and never as silently-wrong data.
+
+Non-goals: send-time transcript repair at the provider seam (hosts may keep
+their own belt-and-suspenders); a skip-corrupt-rows load mode (see
+Alternatives); multi-slot HITL within one batch (single-slot channel semantics
+are owned by the answer-ledger spec).
+
+## Design
+
+### Part 1 — per-task fault isolation in `_execute_parallel`
+
+The two-phase prepare stays exactly as is. Only the fan-out/collect stage
+changes.
+
+**Collect with `asyncio.gather(*tasks, return_exceptions=True)`** instead of
+the bare `await` loop. Every task settles before any outcome is processed;
+results stay in tool_call order. Then classify each slot:
+
+1. **`_FinalizedOutcome`** — success path, unchanged.
+2. **Ordinary failure** (any exception that is not `HitlControlException`,
+ `CancelledError`, `KeyboardInterrupt`, or `SystemExit`) — synthesize an
+ error outcome for that tool_call id:
+ `AgentToolResult(content=[TextContent(text=str(exc))], is_error=True)`,
+ plus a paired `ToolExecutionEndEvent` (the Start was emitted inside
+ `_run`; an unpaired Start would leave `state.pending_tool_calls` and trace
+ spans open — same invariant the prepare-phase comment protects).
+ With Part 2 in place this is defense-in-depth: tool-body and hook failures
+ are already converted inside the task.
+3. **`HitlControlException`** — the suspend contract requires propagation, so
+ it cannot become an error result. Order of operations: first emit the
+ `ToolResultMessage`s for every *other* settled slot (each `MessageEndEvent`
+ checkpoints immediately via `Agent._process_event`, `agent.py:1207-1209`),
+ then re-raise the HITL exception. The HITL call's own tool_call
+ deliberately stays unanswered and gets no End event — identical to today's
+ sequential detach shape — and the existing detach/resume/abort paths
+ answer it (synthetic-deny backfill scans unanswered ids of the last
+ assistant message). If several slots raise HITL exceptions, the first in
+ batch order wins; the rest also stay unanswered and are covered by the
+ same backfill.
+4. **`CancelledError` in a slot, without outer cancellation** — a tool body
+ raised it spuriously (tool bug). Treat as an ordinary failure (case 2)
+ with text `[Tool execution cancelled]`. Escalating one buggy tool's
+ self-cancel into run-level cancellation would kill sibling work for no
+ reason. *(Flagged as a decision point — the conservative alternative is to
+ re-raise after draining, like case 3.)* Note the inherent
+ sequential/parallel asymmetry: in sequential mode tools run in the
+ executor's own task, so a self-raised `CancelledError` is
+ indistinguishable from a genuine external cancel and must keep
+ propagating (the Agent-layer backfill covers it); only the parallel
+ executor can tell the two apart (`task.cancelled()` vs an exception with
+ no outer cancel).
+5. **Outer cancellation** (the executor itself is cancelled while awaiting
+ the gather): `gather` propagates the cancel to all children. Catch the
+ `CancelledError`, await the tasks once more with
+ `return_exceptions=True` so they fully settle, emit the
+ `ToolResultMessage`s of slots that completed *before* the cancel landed,
+ and re-raise. The Agent-layer `_complete_cancelled_tool_calls` backfill
+ then covers only the genuinely-unanswered ids — fixing today's secondary
+ bug where completed tools get a synthetic "[cancelled]" result and their
+ real results (and side effects) are lost, causing duplicate execution on
+ resume.
+6. **`KeyboardInterrupt` / `SystemExit`** — settle children, re-raise. Never
+ converted to results.
+
+The `terminate` aggregation (`_should_terminate`) runs over real plus
+synthesized outcomes, unchanged in shape.
+
+### Part 2 — broaden the narrow catches at the source
+
+- `_execute_prepared` (`tools.py:303`): `except Exception` becomes a
+ `BaseException` guard that re-raises `HitlControlException`,
+ `CancelledError`, `KeyboardInterrupt`, `SystemExit` and converts everything
+ else to an error result.
+- `_finalize`'s `after_tool_call` guard (`tools.py:353`): same treatment. A
+ hook raising a `BaseException` subclass currently escapes *both* executors
+ and every Agent-layer handler (it is neither `CancelledError` nor
+ `Exception`, `agent.py:1094/1102`) — after this change it degrades to an
+ error result on that one call.
+
+The sequential executor gets the same per-call robustness for free, since it
+shares both helpers.
+
+### Part 3 — typed, row-locatable checkpoint corruption
+
+New exception in `cubepi/checkpointer/exceptions.py`, following the
+`CompletionMarkerFailedError` shape:
+
+```python
+class CheckpointCorruptionError(CheckpointerError):
+ """A persisted message row failed to deserialize during load."""
+
+ def __init__(self, *, thread_id: str, backend: str, row_ref: str,
+ cause: BaseException) -> None:
+ super().__init__(
+ f"corrupt checkpoint row for thread {thread_id!r} "
+ f"({backend}, {row_ref}): {cause}")
+ self.thread_id = thread_id
+ self.backend = backend # "sqlite" | "postgres" | "mysql"
+ self.row_ref = row_ref # e.g. "messages.id=42" — locates the row
+ self.__cause__ = cause
+ self.__suppress_context__ = True
+```
+
+Every row-deserialization site (the six listed in Problem §2) wraps its
+per-row work — decode, role lookup, `model_validate` — in a guard that raises
+`CheckpointCorruptionError` with the row's primary key. The SELECTs gain the
+pk column where they don't already fetch it.
+
+Unknown role becomes corruption too, in all three backends — including
+sqlite, whose current silent raw-dict fallthrough (`sqlite.py:537`) is
+replaced. **This is a deliberate behavior change**: any host that depended on
+unknown-role dicts passing through sqlite load was already broken downstream;
+failing loudly at the source with the row pk is strictly more debuggable.
+
+`load()` still fails the whole thread on a corrupt row — but now with a typed
+error that names the row, so hosts can catch `CheckpointCorruptionError`
+distinctly from operational `CheckpointerError`s and repair or quarantine the
+row surgically.
+
+## Alternatives considered
+
+- **`asyncio.TaskGroup` instead of `gather`** — TaskGroup cancels all siblings
+ on first failure. That is the opposite of the requirement (siblings must
+ finish and their results must persist), and ExceptionGroup unwrapping adds
+ noise. `gather(return_exceptions=True)` has exactly the settle-everything
+ semantics needed.
+- **Convert HITL exceptions to error results** — breaks the suspend contract:
+ detach/abort rely on propagation to reach `run_loop`'s outer handlers
+ (`loop.py:445-458`) and the channel's pending-state cleanup.
+- **Skip corrupt rows on load** (per-row tolerance in the recovery sense) —
+ rejected for v1: silently dropping an assistant message that carries
+ `tool_calls`, or a `tool_result`, manufactures the very dangling-tool_call
+ transcript this spec exists to prevent. If a lossy mode is ever wanted it
+ must synthesize structural repairs, not just skip — deferred until a
+ concrete need appears; the typed error with `row_ref` is the enabling step
+ either way.
+- **Emit each parallel result as its task completes** (as-completed
+ streaming, matching the sequential executor) — would also fix the loss but
+ changes observable event ordering for hosts and interleaves checkpointer
+ appends with running tools; settle-then-emit preserves today's ordering
+ with the same durability outcome.
+
+## Testing
+
+- `tests/agent/test_tools.py` (parallel executor, using the existing
+ `AgentTool` fixtures):
+ - HITL raise mid-batch: siblings' `ToolResultMessage`s emitted and in
+ order, `HitlDetached` re-raised, HITL call has no result and no End
+ event, no still-running tasks after the raise.
+ - `after_tool_call` raising a bare `BaseException`: converted to an error
+ result for that call only; batch completes.
+ - Tool body raising `CancelledError` without outer cancel: error result,
+ siblings unaffected.
+ - Outer cancellation mid-batch: completed slots' results emitted before the
+ re-raise; unfinished slots have none (Agent backfill covers them —
+ asserted at the Agent layer with the memory checkpointer).
+ - Start/End event pairing holds for every synthesized outcome.
+- Checkpointer suites (sqlite, postgres, mysql — reusing each backend's
+ existing test harness): corrupt payload row → `CheckpointCorruptionError`
+ carrying thread_id/backend/row_ref; unknown role → same, including the
+ sqlite fallthrough replacement; fork-source read paths covered as well as
+ `load()`.
+- Regression: full existing suite (the repro script's scenario becomes a
+ permanent test).
diff --git a/tests/agent/test_checkpointer_integration.py b/tests/agent/test_checkpointer_integration.py
index 7ed0cf7..55f120b 100644
--- a/tests/agent/test_checkpointer_integration.py
+++ b/tests/agent/test_checkpointer_integration.py
@@ -197,14 +197,19 @@ class TestCheckpointerIntegration:
class BlockParams(BaseModel):
pass
- async def cancel_execute(tool_call_id, params, *, signal=None, on_update=None):
- raise asyncio.CancelledError()
+ started = asyncio.Event()
+
+ async def blocking_execute(
+ tool_call_id, params, *, signal=None, on_update=None
+ ):
+ started.set()
+ await asyncio.Event().wait() # blocks until the run is cancelled
block_tool = AgentTool(
name="block",
- description="Never returns; simulates a cancelled tool call",
+ description="Never returns; cancelled mid-execution",
parameters=BlockParams,
- execute=cancel_execute,
+ execute=blocking_execute,
)
checkpointer = MemoryCheckpointer()
@@ -224,8 +229,11 @@ class TestCheckpointerIntegration:
thread_id="thread-cancel",
)
+ prompt_task = asyncio.create_task(agent.prompt("Run the blocking tool"))
+ await started.wait()
+ prompt_task.cancel()
with pytest.raises(asyncio.CancelledError):
- await agent.prompt("Run the blocking tool")
+ await prompt_task
data = await checkpointer.load("thread-cancel")
assert data is not None
@@ -247,14 +255,19 @@ class TestCheckpointerIntegration:
class BlockParams(BaseModel):
pass
- async def cancel_execute(tool_call_id, params, *, signal=None, on_update=None):
- raise asyncio.CancelledError()
+ started = asyncio.Event()
+
+ async def blocking_execute(
+ tool_call_id, params, *, signal=None, on_update=None
+ ):
+ started.set()
+ await asyncio.Event().wait() # blocks until the run is cancelled
block_tool = AgentTool(
name="block",
- description="Cancels on execution",
+ description="Never returns; cancelled mid-execution",
parameters=BlockParams,
- execute=cancel_execute,
+ execute=blocking_execute,
)
checkpointer = MemoryCheckpointer()
@@ -291,8 +304,11 @@ class TestCheckpointerIntegration:
thread_id="thread-reuse",
)
+ prompt_task = asyncio.create_task(agent.prompt("second"))
+ await started.wait()
+ prompt_task.cancel()
with pytest.raises(asyncio.CancelledError):
- await agent.prompt("second")
+ await prompt_task
data = await checkpointer.load("thread-reuse")
assert data is not None
diff --git a/tests/agent/test_tools.py b/tests/agent/test_tools.py
index ec671cb..cfa01b6 100644
--- a/tests/agent/test_tools.py
+++ b/tests/agent/test_tools.py
@@ -1,5 +1,6 @@
import asyncio
+import pytest
from pydantic import BaseModel
from cubepi.agent.tools import execute_tool_calls
@@ -644,3 +645,205 @@ class TestToolResultDetails:
assert len(batch.messages) == 1
assert batch.messages[0].details == {"execution_time": 42}
+
+
+class TestParallelFaultIsolation:
+ """One failing tool in a parallel batch must never drop sibling results,
+ leak running sibling tasks, or leave dangling tool_calls."""
+
+ @staticmethod
+ def _three_call_msg():
+ return make_assistant_msg(
+ [
+ ToolCall(id="t1", name="a", arguments={"value": "x"}),
+ ToolCall(id="t2", name="b", arguments={"value": "x"}),
+ ToolCall(id="t3", name="c", arguments={"value": "x"}),
+ ]
+ )
+
+ async def test_hitl_raise_preserves_and_persists_sibling_results(self):
+ from cubepi.hitl.exceptions import HitlDetached
+
+ side_effects = []
+
+ async def fast_ok(tool_call_id, params, *, signal=None, on_update=None):
+ side_effects.append("fast")
+ return AgentToolResult(content=[TextContent(text="fast-ok")])
+
+ async def hitl_raiser(tool_call_id, params, *, signal=None, on_update=None):
+ await asyncio.sleep(0.01)
+ raise HitlDetached()
+
+ async def slow_ok(tool_call_id, params, *, signal=None, on_update=None):
+ # Still running when the HITL exception fires — the batch must
+ # wait for it (no detached task, no lost side effect).
+ await asyncio.sleep(0.05)
+ side_effects.append("slow")
+ return AgentToolResult(content=[TextContent(text="slow-ok")])
+
+ ctx = make_context(
+ [
+ make_echo_tool(name="a", execute_fn=fast_ok),
+ make_echo_tool(name="b", execute_fn=hitl_raiser),
+ make_echo_tool(name="c", execute_fn=slow_ok),
+ ]
+ )
+ events = []
+ with pytest.raises(HitlDetached):
+ await execute_tool_calls(
+ ctx,
+ self._three_call_msg(),
+ tool_execution="parallel",
+ emit=lambda e: events.append(e),
+ )
+
+ # Every sibling settled before the re-raise; nothing leaked.
+ assert side_effects == ["fast", "slow"]
+ pending = [
+ t
+ for t in asyncio.all_tasks()
+ if t is not asyncio.current_task() and not t.done()
+ ]
+ assert pending == []
+
+ # Sibling ToolResultMessages were emitted (message_end checkpoints
+ # them at the Agent layer); the HITL call stays unanswered.
+ result_ids = [e.message.tool_call_id for e in events if e.type == "message_end"]
+ assert result_ids == ["t1", "t3"]
+ end_ids = [e.tool_call_id for e in events if e.type == "tool_execution_end"]
+ assert "t2" not in end_ids # detach shape: Start without End
+
+ async def test_tool_body_base_exception_becomes_error_result(self):
+ class Weird(BaseException):
+ pass
+
+ async def raiser(tool_call_id, params, *, signal=None, on_update=None):
+ raise Weird("weird failure")
+
+ async def ok(tool_call_id, params, *, signal=None, on_update=None):
+ return AgentToolResult(content=[TextContent(text="ok")])
+
+ ctx = make_context(
+ [
+ make_echo_tool(name="a", execute_fn=ok),
+ make_echo_tool(name="b", execute_fn=raiser),
+ make_echo_tool(name="c", execute_fn=ok),
+ ]
+ )
+ batch = await execute_tool_calls(
+ ctx, self._three_call_msg(), tool_execution="parallel", emit=lambda e: None
+ )
+ assert [m.tool_call_id for m in batch.messages] == ["t1", "t2", "t3"]
+ by_id = {m.tool_call_id: m for m in batch.messages}
+ assert by_id["t2"].is_error
+ assert "weird failure" in by_id["t2"].content[0].text
+ assert not by_id["t1"].is_error
+ assert not by_id["t3"].is_error
+
+ async def test_after_tool_call_base_exception_isolated(self):
+ class Weird(BaseException):
+ pass
+
+ async def after(after_ctx, *, signal=None):
+ if after_ctx.tool_call.id == "t2":
+ raise Weird("hook blew up")
+ return None
+
+ ctx = make_context(
+ [
+ make_echo_tool(name="a"),
+ make_echo_tool(name="b"),
+ make_echo_tool(name="c"),
+ ]
+ )
+ for mode in ("parallel", "sequential"):
+ batch = await execute_tool_calls(
+ ctx,
+ self._three_call_msg(),
+ tool_execution=mode,
+ after_tool_call=after,
+ emit=lambda e: None,
+ )
+ by_id = {m.tool_call_id: m for m in batch.messages}
+ assert len(by_id) == 3, mode
+ assert by_id["t2"].is_error, mode
+ assert "hook blew up" in by_id["t2"].content[0].text
+ assert not by_id["t1"].is_error
+ assert not by_id["t3"].is_error
+
+ async def test_tool_self_cancel_becomes_error_result(self):
+ async def self_cancel(tool_call_id, params, *, signal=None, on_update=None):
+ raise asyncio.CancelledError()
+
+ async def ok(tool_call_id, params, *, signal=None, on_update=None):
+ return AgentToolResult(content=[TextContent(text="ok")])
+
+ ctx = make_context(
+ [
+ make_echo_tool(name="a", execute_fn=ok),
+ make_echo_tool(name="b", execute_fn=self_cancel),
+ make_echo_tool(name="c", execute_fn=ok),
+ ]
+ )
+ events = []
+ batch = await execute_tool_calls(
+ ctx,
+ self._three_call_msg(),
+ tool_execution="parallel",
+ emit=lambda e: events.append(e),
+ )
+ by_id = {m.tool_call_id: m for m in batch.messages}
+ assert len(by_id) == 3
+ assert by_id["t2"].is_error
+ assert "[Tool execution cancelled]" in by_id["t2"].content[0].text
+ # The synthesized outcome still pairs the Start emitted inside the
+ # task, so pending_tool_calls / trace spans don't leak.
+ starts = [e.tool_call_id for e in events if e.type == "tool_execution_start"]
+ ends = [e.tool_call_id for e in events if e.type == "tool_execution_end"]
+ assert sorted(starts) == sorted(ends) == ["t1", "t2", "t3"]
+
+ async def test_outer_cancel_salvages_completed_results(self):
+ started = asyncio.Event()
+ cancelled_flags = []
+
+ async def fast_ok(tool_call_id, params, *, signal=None, on_update=None):
+ return AgentToolResult(content=[TextContent(text="fast-ok")])
+
+ async def hang(tool_call_id, params, *, signal=None, on_update=None):
+ started.set()
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ cancelled_flags.append(tool_call_id)
+ raise
+ return AgentToolResult(content=[TextContent(text="unreachable")])
+
+ ctx = make_context(
+ [
+ make_echo_tool(name="a", execute_fn=fast_ok),
+ make_echo_tool(name="b", execute_fn=hang),
+ make_echo_tool(name="c", execute_fn=hang),
+ ]
+ )
+ events = []
+ runner = asyncio.create_task(
+ execute_tool_calls(
+ ctx,
+ self._three_call_msg(),
+ tool_execution="parallel",
+ emit=lambda e: events.append(e),
+ )
+ )
+ await started.wait()
+ await asyncio.sleep(0.02) # let fast_ok finish
+ runner.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await runner
+
+ # Hanging siblings were cancelled, not leaked.
+ assert sorted(cancelled_flags) == ["t2", "t3"]
+ # The completed tool's real result was salvaged and emitted before
+ # the re-raise; the cancelled ones stay unanswered for the Agent
+ # layer's backfill.
+ result_ids = [e.message.tool_call_id for e in events if e.type == "message_end"]
+ assert result_ids == ["t1"]
diff --git a/tests/checkpointer/test_mysql.py b/tests/checkpointer/test_mysql.py
index ee6ae6d..8354d81 100644
--- a/tests/checkpointer/test_mysql.py
+++ b/tests/checkpointer/test_mysql.py
@@ -511,9 +511,12 @@ async def test_empty_version_table_raises_uninitialized(clean_mysql_db) -> None:
@pytest.mark.asyncio
async def test_load_unknown_role_raises(clean_mysql_db) -> None:
- """A message row with an unrecognized role string → ValueError on load."""
+ """A message row with an unrecognized role string → typed corruption
+ error on load, naming the row."""
import msgpack
+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError
+
from cubepi.checkpointer.mysql import MySQLCheckpointer
from cubepi.checkpointer.mysql.checkpointer import _parse_dsn
@@ -533,8 +536,10 @@ async def test_load_unknown_role_raises(clean_mysql_db) -> None:
await conn.ensure_closed()
async with MySQLCheckpointer(clean_mysql_db) as cp:
- with pytest.raises(ValueError, match="unknown role in DB"):
+ with pytest.raises(CheckpointCorruptionError, match="unknown role in DB") as ei:
await cp.load("t-bad")
+ assert isinstance(ei.value.__cause__, ValueError)
+ assert ei.value.row_ref == "cubepi_messages.seq=1"
@pytest.mark.asyncio
@@ -559,3 +564,44 @@ async def test_missing_version_column_raises_uninitialized(clean_mysql_db) -> No
with pytest.raises(CubepiSchemaUninitialized):
async with MySQLCheckpointer(clean_mysql_db):
pass
+
+
+async def test_mysql_load_corrupt_row_raises_typed(clean_mysql_db) -> None:
+ """One bad payload row surfaces as CheckpointCorruptionError naming the
+ row — not a raw msgpack error that hides which row is bad."""
+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError
+ from cubepi.checkpointer.mysql import MySQLCheckpointer
+ from cubepi.checkpointer.mysql.checkpointer import _parse_dsn
+ from cubepi.providers.base import TextContent, UserMessage
+
+ await _setup_schema(clean_mysql_db)
+ async with MySQLCheckpointer(clean_mysql_db) as cp:
+ await cp.append(
+ "t-corrupt",
+ [
+ UserMessage(content=[TextContent(text="ok")]),
+ UserMessage(content=[TextContent(text="will corrupt")]),
+ ],
+ )
+ conn = await aiomysql.connect(autocommit=True, **_parse_dsn(clean_mysql_db))
+ try:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "UPDATE cubepi_messages SET payload = %s "
+ "WHERE thread_id = 't-corrupt' AND seq = ("
+ " SELECT max_seq FROM (SELECT max(seq) AS max_seq "
+ " FROM cubepi_messages WHERE thread_id = 't-corrupt') AS sub)",
+ (b"\xc1 not msgpack",),
+ )
+ finally:
+ await conn.ensure_closed()
+
+ with pytest.raises(CheckpointCorruptionError) as excinfo:
+ await cp.load("t-corrupt")
+
+ err = excinfo.value
+ assert err.thread_id == "t-corrupt"
+ assert err.backend == "mysql"
+ assert err.row_ref.startswith("cubepi_messages.seq=")
+ assert err.__cause__ is not None
diff --git a/tests/checkpointer/test_postgres.py b/tests/checkpointer/test_postgres.py
index c5cfc95..b24b854 100644
--- a/tests/checkpointer/test_postgres.py
+++ b/tests/checkpointer/test_postgres.py
@@ -348,3 +348,66 @@ async def test_empty_thread_load_returns_none(clean_db) -> None:
async with PostgresCheckpointer(clean_db) as cp:
data = await cp.load("nonexistent-thread")
assert data is None
+
+
+async def test_postgres_load_corrupt_row_raises_typed(clean_db) -> None:
+ """One bad payload row surfaces as CheckpointCorruptionError naming the
+ row — not a raw msgpack error that hides which row is bad."""
+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError
+ from cubepi.checkpointer.postgres import PostgresCheckpointer
+ from cubepi.providers.base import TextContent, UserMessage
+
+ await _setup_schema(clean_db)
+ async with PostgresCheckpointer(clean_db) as cp:
+ await cp.append(
+ "t-corrupt",
+ [
+ UserMessage(content=[TextContent(text="ok")]),
+ UserMessage(content=[TextContent(text="will corrupt")]),
+ ],
+ )
+ conn = await asyncpg.connect(clean_db)
+ try:
+ await conn.execute(
+ "UPDATE cubepi_messages SET payload = $1 "
+ "WHERE thread_id = 't-corrupt' AND seq = ("
+ " SELECT max(seq) FROM cubepi_messages "
+ " WHERE thread_id = 't-corrupt')",
+ b"\xc1 not msgpack",
+ )
+ finally:
+ await conn.close()
+
+ with pytest.raises(CheckpointCorruptionError) as excinfo:
+ await cp.load("t-corrupt")
+
+ err = excinfo.value
+ assert err.thread_id == "t-corrupt"
+ assert err.backend == "postgres"
+ assert err.row_ref.startswith("cubepi_messages.seq=")
+ assert err.__cause__ is not None
+
+
+async def test_postgres_load_unknown_role_raises_typed(clean_db) -> None:
+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError
+ from cubepi.checkpointer.postgres import PostgresCheckpointer
+ from cubepi.providers.base import TextContent, UserMessage
+
+ await _setup_schema(clean_db)
+ async with PostgresCheckpointer(clean_db) as cp:
+ await cp.append("t-role", [UserMessage(content=[TextContent(text="ok")])])
+ conn = await asyncpg.connect(clean_db)
+ try:
+ await conn.execute(
+ "UPDATE cubepi_messages SET role = 'alien' WHERE thread_id = 't-role'"
+ )
+ finally:
+ await conn.close()
+
+ with pytest.raises(CheckpointCorruptionError) as excinfo:
+ await cp.load("t-role")
+
+ assert isinstance(excinfo.value.__cause__, ValueError)
+ assert "alien" in str(excinfo.value)
diff --git a/tests/checkpointer/test_sqlite.py b/tests/checkpointer/test_sqlite.py
index ffe1d37..a271480 100644
--- a/tests/checkpointer/test_sqlite.py
+++ b/tests/checkpointer/test_sqlite.py
@@ -101,12 +101,61 @@ class TestSQLiteCheckpointer:
assert loaded.content[0].text == "result"
async def test_deserialize_unknown_role(self, db_path):
- """A message with an unknown role should be returned as a plain dict."""
+ """Unknown roles are corruption, matching the postgres/mysql
+ backends — the old silent raw-dict passthrough let bad data flow
+ into the message list and fail far from the cause."""
+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError
+
async with SQLiteCheckpointer(db_path) as cp:
raw_msg = {"role": "custom", "data": "test"}
await cp.append("thread-1", [raw_msg])
- data = await cp.load("thread-1")
- assert data is not None
- assert len(data.messages) == 1
- assert data.messages[0] == {"role": "custom", "data": "test"}
+ with pytest.raises(CheckpointCorruptionError):
+ await cp.load("thread-1")
+
+
+class TestCheckpointCorruption:
+ async def test_corrupt_json_row_raises_typed(self, db_path):
+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError
+
+ async with SQLiteCheckpointer(db_path) as cp:
+ await cp.append(
+ "thread-1",
+ [
+ UserMessage(content=[TextContent(text="ok")]),
+ UserMessage(content=[TextContent(text="will corrupt")]),
+ ],
+ )
+ await cp._db.execute(
+ 'UPDATE messages SET message_json = \'{"role": "user"\' '
+ "WHERE id = (SELECT max(id) FROM messages WHERE thread_id = ?)",
+ ("thread-1",),
+ )
+ await cp._db.commit()
+
+ with pytest.raises(CheckpointCorruptionError) as excinfo:
+ await cp.load("thread-1")
+
+ err = excinfo.value
+ assert err.thread_id == "thread-1"
+ assert err.backend == "sqlite"
+ assert err.row_ref.startswith("messages.id=")
+ assert err.__cause__ is not None
+
+ async def test_unknown_role_raises_typed(self, db_path):
+ from cubepi.checkpointer.exceptions import CheckpointCorruptionError
+
+ async with SQLiteCheckpointer(db_path) as cp:
+ await cp.append("thread-1", [UserMessage(content=[TextContent(text="ok")])])
+ await cp._db.execute(
+ "UPDATE messages SET message_json = "
+ '\'{"role": "alien", "content": []}\' WHERE thread_id = ?',
+ ("thread-1",),
+ )
+ await cp._db.commit()
+
+ with pytest.raises(CheckpointCorruptionError) as excinfo:
+ await cp.load("thread-1")
+
+ assert isinstance(excinfo.value.__cause__, ValueError)
+ assert "alien" in str(excinfo.value)
diff --git a/website/docs/guides/agents/tool-use.md b/website/docs/guides/agents/tool-use.md
index b28bce0..201c1c7 100644
--- a/website/docs/guides/agents/tool-use.md
+++ b/website/docs/guides/agents/tool-use.md
@@ -123,6 +123,27 @@ The event stream emits `tool_execution_start` for all of them up
front, interleaved `tool_execution_update` events as each tool reports
progress, then `tool_execution_end` per tool in completion order.
+### Fault isolation
+
+Failures are isolated per call: every tool in the batch runs to
+completion regardless of what its siblings do, and each call gets
+exactly one `tool_result` — a real one, or a synthesized error result
+if the tool (or an `after_tool_call` hook) raised something
+unexpected. One misbehaving tool never discards a sibling's completed
+work, and a persisted assistant message is never left with unanswered
+`tool_calls` (which providers reject on the next turn).
+
+Two exception classes still propagate, by design:
+
+- **HITL control exceptions** (a tool detaching for human input) — the
+ batch first waits for every sibling to finish and persists their
+ results, then the suspend propagates. Only the detaching call stays
+ unanswered, and the HITL resume/abort flow answers it later.
+- **Cancellation** — siblings are cancelled and awaited (no leaked
+ tasks); results of tools that completed before the cancel are
+ persisted so a resumed thread doesn't re-run them and duplicate
+ their side effects.
+
## Forcing sequential execution
There are two ways to opt out of parallel mode:
diff --git a/website/docs/guides/checkpointing/mysql.md b/website/docs/guides/checkpointing/mysql.md
index 4fcb9e2..4163731 100644
--- a/website/docs/guides/checkpointing/mysql.md
+++ b/website/docs/guides/checkpointing/mysql.md
@@ -270,6 +270,11 @@ def upgrade():
- **Case-insensitive thread-ID collisions** — Thread IDs use the
`utf8mb4_bin` collation so `UserA` and `usera` stay distinct. If you
hand-roll the DDL, keep that collation.
+- **`CheckpointCorruptionError` on `load()`** — A persisted message row
+ failed to deserialize (bad msgpack payload, schema-invalid data, or an
+ unknown role). The error's `row_ref` (e.g. `cubepi_messages.seq=42`)
+ locates the bad row for inspection or surgical repair; `thread_id` and
+ `__cause__` carry the rest. CubePi never skips corrupt rows silently.
## See also
diff --git a/website/docs/guides/checkpointing/postgres.md b/website/docs/guides/checkpointing/postgres.md
index cea4061..e091003 100644
--- a/website/docs/guides/checkpointing/postgres.md
+++ b/website/docs/guides/checkpointing/postgres.md
@@ -269,6 +269,13 @@ def upgrade():
`MetaData` instance precisely so it can coexist with your app's
models without colliding. Don't merge them into your global metadata
— pass both to Alembic separately.
+- **`CheckpointCorruptionError` on `load()`** — A persisted message row
+ failed to deserialize (bad msgpack payload, schema-invalid data, or an
+ unknown role). The error's `row_ref` (e.g. `cubepi_messages.seq=42`)
+ locates the bad row for inspection or surgical repair; `thread_id` and
+ `__cause__` carry the rest. CubePi never skips corrupt rows silently —
+ dropping a message that carries `tool_calls` would leave the
+ transcript in a state every provider rejects.
## See also
diff --git a/website/docs/guides/checkpointing/sqlite.md b/website/docs/guides/checkpointing/sqlite.md
index 9271b73..99fa18c 100644
--- a/website/docs/guides/checkpointing/sqlite.md
+++ b/website/docs/guides/checkpointing/sqlite.md
@@ -212,6 +212,13 @@ sqlite3 agent.db "VACUUM"
- **Forgetting to set `thread_id`** — Without it, the agent has no
persistence binding. The checkpointer is silently ignored. Always
pass both.
+- **`CheckpointCorruptionError` on `load()`** — A persisted message row
+ failed to deserialize (bad JSON, schema-invalid data, or an unknown
+ role). The error's `row_ref` (e.g. `messages.id=42`) locates the bad
+ row so you can inspect or repair it with plain SQL; `thread_id` and
+ `__cause__` carry the rest of the context. CubePi never skips corrupt
+ rows silently — dropping a message that carries `tool_calls` would
+ leave the transcript in a state every provider rejects.
## See also