Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

Fix parallel HITL approval replay

xf gong6fb793ecb9cd
diff --git a/cubepi/agent/agent.py b/cubepi/agent/agent.py
index 65c6f5b..f6d24f3 100644
--- a/cubepi/agent/agent.py
+++ b/cubepi/agent/agent.py
@@ -847,6 +847,11 @@ class Agent(Generic[TMessage]):
"respond() requires a checkpointer that implements "
"load_pending (added in checkpointer v4)"
)
+ save_hitl_answer = getattr(self.checkpointer, "save_hitl_answer", None)
+ if save_hitl_answer is None:
+ raise HitlError(
+ "respond() requires a checkpointer that implements save_hitl_answer"
+ )
async with self._run_lock:
if not self._state._messages:
@@ -882,6 +887,12 @@ class Agent(Generic[TMessage]):
self._state.active_run_id = recovered_run_id
self._state.last_outcome = None
+ await save_hitl_answer(
+ self.thread_id,
+ question_id,
+ answer,
+ run_id=recovered_run_id,
+ )
self._channel.attach_resume_answer(question_id, answer)
try:
await self._run_hitl_resume()
@@ -980,6 +991,11 @@ class Agent(Generic[TMessage]):
# No unresolved assistant turn — conversation already closed
# by some other path. Still clear pending + emit for observability.
await save_pending(self.thread_id, None)
+ clear_answers = getattr(self.checkpointer, "clear_hitl_answers", None)
+ if clear_answers is not None:
+ await clear_answers(
+ self.thread_id, run_id=self._state.active_run_id
+ )
await self._process_event(AgentAbortedEvent(reason=reason))
return
@@ -1033,6 +1049,9 @@ class Agent(Generic[TMessage]):
# Defensive clear (Phase 1's _on_pending_cleared usually did this,
# but cross-process abort_pending may bypass Phase 1 entirely).
await save_pending(self.thread_id, None)
+ clear_answers = getattr(self.checkpointer, "clear_hitl_answers", None)
+ if clear_answers is not None:
+ await clear_answers(self.thread_id, run_id=owning_run_id)
await self._process_event(AgentAbortedEvent(reason=reason))
async def _run_hitl_resume(self) -> None:
diff --git a/cubepi/agent/loop.py b/cubepi/agent/loop.py
index b9f03b4..ff70f9d 100644
--- a/cubepi/agent/loop.py
+++ b/cubepi/agent/loop.py
@@ -335,6 +335,9 @@ async def _run_agent_loop_resume_body( # pragma: no cover — E2E tested
save_pending = getattr(checkpointer, "save_pending_request", None)
if save_pending is not None:
await save_pending(thread_id, None)
+ clear_answers = getattr(checkpointer, "clear_hitl_answers", None)
+ if clear_answers is not None:
+ await clear_answers(thread_id, run_id=last.run_id)
# Emit TurnEndEvent with the ACTUAL tool_results so listeners get the
# right payload (codex BLOCKING: previous draft emitted []).
diff --git a/cubepi/checkpointer/base.py b/cubepi/checkpointer/base.py
index db78a30..1c8aca5 100644
--- a/cubepi/checkpointer/base.py
+++ b/cubepi/checkpointer/base.py
@@ -1,11 +1,12 @@
from __future__ import annotations
+from collections.abc import Iterable
from dataclasses import dataclass, field
from typing import Protocol, runtime_checkable
from cubepi.hitl.types import HitlRequest
from cubepi.providers.base import Message
-from cubepi.types import JsonObject
+from cubepi.types import JsonObject, StructuredValue
@dataclass
@@ -90,3 +91,34 @@ class Checkpointer(Protocol):
"""Read (HitlRequest, run_id) atomically from the pending row,
or None when no pending request exists."""
...
+
+ async def save_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ answer: StructuredValue,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ """Persist an answered HITL request for replay during resume."""
+ ...
+
+ async def load_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ *,
+ run_id: str | None = None,
+ ) -> StructuredValue | None:
+ """Load a persisted HITL answer for replay, or None."""
+ ...
+
+ async def clear_hitl_answers(
+ self,
+ thread_id: str,
+ question_ids: Iterable[str] | None = None,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ """Clear persisted HITL answers for a run or selected question ids."""
+ ...
diff --git a/cubepi/checkpointer/memory.py b/cubepi/checkpointer/memory.py
index 61d6569..d6ae3f5 100644
--- a/cubepi/checkpointer/memory.py
+++ b/cubepi/checkpointer/memory.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import copy
import time
+from collections.abc import Iterable
from dataclasses import dataclass
from cubepi.checkpointer.base import CheckpointData
@@ -16,7 +17,7 @@ from cubepi.checkpointer.exceptions import (
)
from cubepi.hitl.types import HitlRequest
from cubepi.providers.base import Message
-from cubepi.types import JsonObject
+from cubepi.types import JsonObject, StructuredValue
@dataclass
@@ -30,11 +31,16 @@ def _legible_message_copy(m: Message) -> Message:
return m.model_copy(deep=True)
+def _run_key(run_id: str | None) -> str:
+ return run_id or ""
+
+
class MemoryCheckpointer:
def __init__(self) -> None:
self._store: dict[str, CheckpointData] = {}
self._pending: dict[str, HitlRequest] = {}
self._pending_run_id: dict[str, str | None] = {}
+ self._hitl_answers: dict[tuple[str, str, str], StructuredValue] = {}
self._runs: dict[str, dict[str, _RunState]] = {}
self._lock = asyncio.Lock()
@@ -90,6 +96,53 @@ class MemoryCheckpointer:
return None
return req, self._pending_run_id.get(thread_id)
+ async def save_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ answer: StructuredValue,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ key = (thread_id, _run_key(run_id), question_id)
+ async with self._lock:
+ self._hitl_answers[key] = copy.deepcopy(answer)
+
+ async def load_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ *,
+ run_id: str | None = None,
+ ) -> StructuredValue | None:
+ key = (thread_id, _run_key(run_id), question_id)
+ async with self._lock:
+ answer = self._hitl_answers.get(key)
+ return copy.deepcopy(answer)
+
+ async def clear_hitl_answers(
+ self,
+ thread_id: str,
+ question_ids: Iterable[str] | None = None,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ run_key = _run_key(run_id)
+ async with self._lock:
+ if question_ids is None:
+ keys = [
+ key
+ for key in self._hitl_answers
+ if key[0] == thread_id and key[1] == run_key
+ ]
+ else:
+ keys = [
+ (thread_id, run_key, question_id)
+ for question_id in set(question_ids)
+ ]
+ for key in keys:
+ self._hitl_answers.pop(key, None)
+
async def claim_run(self, thread_id: str, run_id: str) -> None:
async with self._lock:
runs = self._runs.setdefault(thread_id, {})
diff --git a/cubepi/checkpointer/mysql/alembic_helpers.py b/cubepi/checkpointer/mysql/alembic_helpers.py
index 15c241d..4c5efb4 100644
--- a/cubepi/checkpointer/mysql/alembic_helpers.py
+++ b/cubepi/checkpointer/mysql/alembic_helpers.py
@@ -129,6 +129,30 @@ def upgrade_v3_to_v4_op() -> str:
return f"{add_messages_run_id_column_op()} {create_runs_table_op()};"
+def create_hitl_answers_table_op() -> str:
+ """Return SQL creating the durable HITL answer ledger table."""
+ return (
+ "CREATE TABLE cubepi_hitl_answers ("
+ " thread_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL,"
+ " run_id VARCHAR(255) NOT NULL,"
+ " question_id VARCHAR(255) NOT NULL,"
+ " answer JSON NOT NULL,"
+ " answered_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,"
+ " PRIMARY KEY (thread_id, run_id, question_id)"
+ ") ENGINE=InnoDB"
+ )
+
+
+def upgrade_v4_to_v5_op() -> str:
+ """Return SQL applying the v4->v5 schema changes.
+
+ Creates the durable HITL answer ledger table. Hosts must also bump
+ ``cubepi_schema_version`` via ``write_schema_version_op()``
+ (EXPECTED_SCHEMA_VERSION is now 5).
+ """
+ return create_hitl_answers_table_op() + ";"
+
+
def write_schema_version_op() -> str:
"""Return SQL setting cubepi_schema_version to the current version.
diff --git a/cubepi/checkpointer/mysql/checkpointer.py b/cubepi/checkpointer/mysql/checkpointer.py
index fba605c..5c3454b 100644
--- a/cubepi/checkpointer/mysql/checkpointer.py
+++ b/cubepi/checkpointer/mysql/checkpointer.py
@@ -9,12 +9,14 @@ list of deliberate MySQL divergences.
from __future__ import annotations
import json
+from collections.abc import Iterable
from typing import Any
from urllib.parse import unquote, urlparse
import aiomysql
import msgpack
import pymysql
+from pydantic import TypeAdapter
from cubepi.checkpointer.base import CheckpointData
from cubepi.checkpointer.exceptions import (
@@ -37,7 +39,9 @@ from cubepi.providers.base import (
ToolResultMessage,
UserMessage,
)
-from cubepi.types import JsonObject
+from cubepi.types import JsonObject, StructuredValue
+
+_STRUCTURED_VALUE_ADAPTER: TypeAdapter[Any] = TypeAdapter(StructuredValue)
_ER_NO_SUCH_TABLE = 1146
_ER_BAD_FIELD_ERROR = 1054
@@ -82,6 +86,20 @@ def _decode_json(value: Any) -> dict[str, Any]:
return value
+def _run_key(run_id: str | None) -> str:
+ return run_id or ""
+
+
+def _serialize_structured_value(value: StructuredValue) -> str:
+ return _STRUCTURED_VALUE_ADAPTER.dump_json(value).decode("utf-8")
+
+
+def _decode_json_value(value: Any) -> StructuredValue:
+ if isinstance(value, str): # pragma: no cover - codec-dependent
+ return json.loads(value)
+ return value
+
+
class MySQLCheckpointer:
"""Checkpointer backed by MySQL (8.0.13+, InnoDB).
@@ -693,3 +711,90 @@ class MySQLCheckpointer:
)
row = await cur.fetchone()
return row[0] if row is not None else None
+
+ async def save_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ answer: StructuredValue,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ assert self._pool is not None
+ payload = _serialize_structured_value(answer)
+ async with self._pool.acquire() as conn:
+ await conn.begin()
+ try:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "INSERT INTO cubepi_threads (thread_id) VALUES (%s) "
+ "ON DUPLICATE KEY UPDATE thread_id = thread_id",
+ (thread_id,),
+ )
+ await cur.execute(
+ "INSERT INTO cubepi_hitl_answers "
+ "(thread_id, run_id, question_id, answer) "
+ "VALUES (%s, %s, %s, %s) "
+ "ON DUPLICATE KEY UPDATE "
+ "answer = VALUES(answer), "
+ "answered_at = CURRENT_TIMESTAMP",
+ (thread_id, _run_key(run_id), question_id, payload),
+ )
+ await conn.commit()
+ except BaseException: # pragma: no cover - defensive txn rollback
+ await conn.rollback()
+ raise
+
+ async def load_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ *,
+ run_id: str | None = None,
+ ) -> StructuredValue | None:
+ assert self._pool is not None
+ async with self._pool.acquire() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT answer FROM cubepi_hitl_answers "
+ "WHERE thread_id = %s AND run_id = %s AND question_id = %s",
+ (thread_id, _run_key(run_id), question_id),
+ )
+ row = await cur.fetchone()
+ if row is None:
+ return None
+ return _decode_json_value(row[0])
+
+ async def clear_hitl_answers(
+ self,
+ thread_id: str,
+ question_ids: Iterable[str] | None = None,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ assert self._pool is not None
+ run_key = _run_key(run_id)
+ async with self._pool.acquire() as conn:
+ await conn.begin()
+ try:
+ async with conn.cursor() as cur:
+ if question_ids is None:
+ await cur.execute(
+ "DELETE FROM cubepi_hitl_answers "
+ "WHERE thread_id = %s AND run_id = %s",
+ (thread_id, run_key),
+ )
+ else:
+ qids = list(dict.fromkeys(question_ids))
+ if qids:
+ placeholders = ",".join("%s" for _ in qids)
+ await cur.execute(
+ "DELETE FROM cubepi_hitl_answers "
+ "WHERE thread_id = %s AND run_id = %s "
+ f"AND question_id IN ({placeholders})",
+ (thread_id, run_key, *qids),
+ )
+ await conn.commit()
+ except BaseException: # pragma: no cover - defensive txn rollback
+ await conn.rollback()
+ raise
diff --git a/cubepi/checkpointer/mysql/models.py b/cubepi/checkpointer/mysql/models.py
index f666d53..1988b01 100644
--- a/cubepi/checkpointer/mysql/models.py
+++ b/cubepi/checkpointer/mysql/models.py
@@ -16,7 +16,7 @@ import sqlalchemy as sa
from sqlalchemy.dialects.mysql import JSON, LONGBLOB, VARCHAR
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
-EXPECTED_SCHEMA_VERSION = 4
+EXPECTED_SCHEMA_VERSION = 5
PARTITION_COUNT = 64
cubepi_metadata = sa.MetaData()
@@ -135,6 +135,21 @@ class CubepiRun(CubepiBase):
completion_seq: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True)
+class CubepiHitlAnswer(CubepiBase):
+ __tablename__ = "cubepi_hitl_answers"
+ __table_args__ = {"mysql_engine": "InnoDB"}
+
+ thread_id: Mapped[str] = mapped_column(_TID, primary_key=True)
+ run_id: Mapped[str] = mapped_column(VARCHAR(255), primary_key=True)
+ question_id: Mapped[str] = mapped_column(VARCHAR(255), primary_key=True)
+ answer: Mapped[Any] = mapped_column(JSON, nullable=False)
+ answered_at: Mapped[_dt.datetime] = mapped_column(
+ sa.TIMESTAMP,
+ nullable=False,
+ server_default=sa.text("CURRENT_TIMESTAMP"),
+ )
+
+
class CubepiSchemaVersion(CubepiBase):
__tablename__ = "cubepi_schema_version"
__table_args__ = {"mysql_engine": "InnoDB"}
diff --git a/cubepi/checkpointer/postgres/alembic_helpers.py b/cubepi/checkpointer/postgres/alembic_helpers.py
index 2d1234c..1a013f9 100644
--- a/cubepi/checkpointer/postgres/alembic_helpers.py
+++ b/cubepi/checkpointer/postgres/alembic_helpers.py
@@ -93,6 +93,26 @@ def upgrade_v3_to_v4_op() -> str:
return "\n".join(parts)
+def upgrade_v4_to_v5_op() -> str:
+ """Return SQL applying the v4->v5 schema changes.
+
+ Creates the durable HITL answer ledger table. Hosts must also bump
+ `cubepi_schema_version` via write_schema_version_op()
+ (EXPECTED_SCHEMA_VERSION is now 5).
+ """
+ return (
+ "CREATE TABLE IF NOT EXISTS cubepi_hitl_answers ("
+ " thread_id TEXT NOT NULL REFERENCES cubepi_threads(thread_id) "
+ "ON DELETE CASCADE,"
+ " run_id TEXT NOT NULL,"
+ " question_id TEXT NOT NULL,"
+ " answer JSONB NOT NULL,"
+ " answered_at TIMESTAMPTZ NOT NULL DEFAULT now(),"
+ " PRIMARY KEY (thread_id, run_id, question_id)"
+ ");"
+ )
+
+
def write_schema_version_op() -> str:
"""Return SQL setting cubepi_schema_version to the current version.
diff --git a/cubepi/checkpointer/postgres/checkpointer.py b/cubepi/checkpointer/postgres/checkpointer.py
index 3065c98..2bdbf21 100644
--- a/cubepi/checkpointer/postgres/checkpointer.py
+++ b/cubepi/checkpointer/postgres/checkpointer.py
@@ -7,10 +7,12 @@ msgpack payload encoding. Schema version verified on context entry.
from __future__ import annotations
import json
+from collections.abc import Iterable
from typing import Any
import asyncpg
import msgpack
+from pydantic import TypeAdapter
from cubepi.checkpointer.base import CheckpointData
from cubepi.checkpointer.exceptions import (
@@ -33,7 +35,23 @@ from cubepi.providers.base import (
ToolResultMessage,
UserMessage,
)
-from cubepi.types import JsonObject
+from cubepi.types import JsonObject, StructuredValue
+
+_STRUCTURED_VALUE_ADAPTER: TypeAdapter[Any] = TypeAdapter(StructuredValue)
+
+
+def _run_key(run_id: str | None) -> str:
+ return run_id or ""
+
+
+def _serialize_structured_value(value: StructuredValue) -> str:
+ return _STRUCTURED_VALUE_ADAPTER.dump_json(value).decode("utf-8")
+
+
+def _deserialize_structured_value(value: Any) -> StructuredValue:
+ if isinstance(value, str): # pragma: no cover - codec-dependent
+ return json.loads(value)
+ return value
def _role_of(msg: Message) -> str:
@@ -561,3 +579,82 @@ class PostgresCheckpointer:
thread_id,
)
return row["run_id"] if row is not None else None
+
+ async def save_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ answer: StructuredValue,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ assert self._pool is not None
+ payload = _serialize_structured_value(answer)
+ async with self._pool.acquire() as conn:
+ async with conn.transaction():
+ await conn.execute(
+ "INSERT INTO cubepi_threads (thread_id) "
+ "VALUES ($1) ON CONFLICT DO NOTHING",
+ thread_id,
+ )
+ await conn.execute(
+ "INSERT INTO cubepi_hitl_answers "
+ "(thread_id, run_id, question_id, answer) "
+ "VALUES ($1, $2, $3, $4::jsonb) "
+ "ON CONFLICT (thread_id, run_id, question_id) DO UPDATE "
+ "SET answer = EXCLUDED.answer, answered_at = now()",
+ thread_id,
+ _run_key(run_id),
+ question_id,
+ payload,
+ )
+
+ async def load_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ *,
+ run_id: str | None = None,
+ ) -> StructuredValue | None:
+ assert self._pool is not None
+ async with self._pool.acquire() as conn:
+ row = await conn.fetchrow(
+ "SELECT answer FROM cubepi_hitl_answers "
+ "WHERE thread_id = $1 AND run_id = $2 AND question_id = $3",
+ thread_id,
+ _run_key(run_id),
+ question_id,
+ )
+ if row is None:
+ return None
+ return _deserialize_structured_value(row["answer"])
+
+ async def clear_hitl_answers(
+ self,
+ thread_id: str,
+ question_ids: Iterable[str] | None = None,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ assert self._pool is not None
+ run_key = _run_key(run_id)
+ async with self._pool.acquire() as conn:
+ if question_ids is None:
+ await conn.execute(
+ "DELETE FROM cubepi_hitl_answers "
+ "WHERE thread_id = $1 AND run_id = $2",
+ thread_id,
+ run_key,
+ )
+ return
+ qids = list(dict.fromkeys(question_ids))
+ if not qids:
+ return
+ await conn.execute(
+ "DELETE FROM cubepi_hitl_answers "
+ "WHERE thread_id = $1 AND run_id = $2 "
+ "AND question_id = ANY($3::text[])",
+ thread_id,
+ run_key,
+ qids,
+ )
diff --git a/cubepi/checkpointer/postgres/models.py b/cubepi/checkpointer/postgres/models.py
index 4d45fac..ed8a516 100644
--- a/cubepi/checkpointer/postgres/models.py
+++ b/cubepi/checkpointer/postgres/models.py
@@ -15,7 +15,7 @@ import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
-EXPECTED_SCHEMA_VERSION = 4
+EXPECTED_SCHEMA_VERSION = 5
PARTITION_COUNT = 64
cubepi_metadata = sa.MetaData()
@@ -129,6 +129,24 @@ class CubepiRun(CubepiBase):
completion_seq: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True)
+class CubepiHitlAnswer(CubepiBase):
+ __tablename__ = "cubepi_hitl_answers"
+
+ thread_id: Mapped[str] = mapped_column(
+ sa.Text,
+ sa.ForeignKey("cubepi_threads.thread_id", ondelete="CASCADE"),
+ primary_key=True,
+ )
+ run_id: Mapped[str] = mapped_column(sa.Text, primary_key=True)
+ question_id: Mapped[str] = mapped_column(sa.Text, primary_key=True)
+ answer: Mapped[Any] = mapped_column(JSONB, nullable=False)
+ answered_at: Mapped[_dt.datetime] = mapped_column(
+ sa.TIMESTAMP(timezone=True),
+ nullable=False,
+ server_default=sa.text("now()"),
+ )
+
+
class CubepiSchemaVersion(CubepiBase):
__tablename__ = "cubepi_schema_version"
diff --git a/cubepi/checkpointer/sqlite.py b/cubepi/checkpointer/sqlite.py
index 0f59952..a5d7da8 100644
--- a/cubepi/checkpointer/sqlite.py
+++ b/cubepi/checkpointer/sqlite.py
@@ -2,10 +2,12 @@ from __future__ import annotations
import asyncio
import json
+from collections.abc import Iterable
from contextlib import asynccontextmanager
from typing import Any, AsyncIterator, cast
import aiosqlite
+from pydantic import TypeAdapter
from cubepi.checkpointer.base import CheckpointData
from cubepi.checkpointer.exceptions import (
@@ -24,7 +26,21 @@ from cubepi.providers.base import (
ToolResultMessage,
UserMessage,
)
-from cubepi.types import JsonObject
+from cubepi.types import JsonObject, StructuredValue
+
+_STRUCTURED_VALUE_ADAPTER: TypeAdapter[Any] = TypeAdapter(StructuredValue)
+
+
+def _run_key(run_id: str | None) -> str:
+ return run_id or ""
+
+
+def _serialize_structured_value(value: StructuredValue) -> str:
+ return _STRUCTURED_VALUE_ADAPTER.dump_json(value).decode("utf-8")
+
+
+def _deserialize_structured_value(value: str) -> StructuredValue:
+ return cast(StructuredValue, json.loads(value))
@asynccontextmanager
@@ -79,6 +95,16 @@ class SQLiteCheckpointer:
" created_at REAL NOT NULL DEFAULT (julianday('now'))"
")"
)
+ await self._db.execute(
+ "CREATE TABLE IF NOT EXISTS thread_hitl_answers ("
+ " thread_id TEXT NOT NULL,"
+ " run_id TEXT NOT NULL,"
+ " question_id TEXT NOT NULL,"
+ " answer_json TEXT NOT NULL,"
+ " answered_at REAL NOT NULL DEFAULT (julianday('now')),"
+ " PRIMARY KEY (thread_id, run_id, question_id)"
+ ")"
+ )
await self._db.execute(
"CREATE TABLE IF NOT EXISTS runs ("
" thread_id TEXT NOT NULL,"
@@ -310,6 +336,69 @@ class SQLiteCheckpointer:
row = await cursor.fetchone()
return row[0] if row else None
+ async def save_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ answer: StructuredValue,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ assert self._db is not None
+ payload = _serialize_structured_value(answer)
+ async with self._lock, _writer_txn(self._db):
+ await self._db.execute(
+ "INSERT OR REPLACE INTO thread_hitl_answers "
+ "(thread_id, run_id, question_id, answer_json) "
+ "VALUES (?, ?, ?, ?)",
+ (thread_id, _run_key(run_id), question_id, payload),
+ )
+
+ async def load_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ *,
+ run_id: str | None = None,
+ ) -> StructuredValue | None:
+ assert self._db is not None
+ async with self._lock:
+ cursor = await self._db.execute(
+ "SELECT answer_json FROM thread_hitl_answers "
+ "WHERE thread_id = ? AND run_id = ? AND question_id = ?",
+ (thread_id, _run_key(run_id), question_id),
+ )
+ row = await cursor.fetchone()
+ return _deserialize_structured_value(row[0]) if row else None
+
+ async def clear_hitl_answers(
+ self,
+ thread_id: str,
+ question_ids: Iterable[str] | None = None,
+ *,
+ run_id: str | None = None,
+ ) -> None:
+ assert self._db is not None
+ run_key = _run_key(run_id)
+ async with self._lock, _writer_txn(self._db):
+ if question_ids is None:
+ await self._db.execute(
+ "DELETE FROM thread_hitl_answers "
+ "WHERE thread_id = ? AND run_id = ?",
+ (thread_id, run_key),
+ )
+ return
+ qids = list(dict.fromkeys(question_ids))
+ if not qids:
+ return
+ placeholders = ",".join("?" for _ in qids)
+ await self._db.execute(
+ "DELETE FROM thread_hitl_answers "
+ f"WHERE thread_id = ? AND run_id = ? "
+ f"AND question_id IN ({placeholders})",
+ (thread_id, run_key, *qids),
+ )
+
async def snapshot(self, thread_id: str, *, after_run_id: str) -> list[Message]:
assert self._db is not None
async with self._lock:
diff --git a/cubepi/hitl/_trace.py b/cubepi/hitl/_trace.py
index db43213..74da111 100644
--- a/cubepi/hitl/_trace.py
+++ b/cubepi/hitl/_trace.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import contextlib
+from typing import Any, cast
class _NullSpan: # pragma: no cover - only exercised when opentelemetry is not installed
@@ -20,10 +21,11 @@ class _NullSpan: # pragma: no cover - only exercised when opentelemetry is not
@contextlib.contextmanager
def hitl_span(kind: str, **attrs):
try:
- from opentelemetry import trace
+ from opentelemetry import trace as otel_trace
except ImportError: # pragma: no cover - tracing extra not installed
yield _NullSpan()
return
+ trace = cast(Any, otel_trace)
tracer = trace.get_tracer("cubepi.hitl")
with tracer.start_as_current_span(f"hitl.{kind}") as span:
for k, v in attrs.items():
diff --git a/cubepi/hitl/channel.py b/cubepi/hitl/channel.py
index 5284b49..962d0de 100644
--- a/cubepi/hitl/channel.py
+++ b/cubepi/hitl/channel.py
@@ -151,6 +151,12 @@ class _BaseChannel:
def _bind_emit(self, emit) -> None:
self._emit = emit
+ async def _load_answer(self, question_id: str) -> StructuredValue | None:
+ return None
+
+ async def _save_answer(self, question_id: str, answer: StructuredValue) -> None:
+ return None
+
def _next_qid(self, kind: str, payload_repr: StructuredValue) -> str:
"""Derive the next question_id for an ask/confirm payload.
@@ -235,6 +241,19 @@ class _BaseChannel:
)
return ans
+ persisted_answer = await self._load_answer(question_id)
+ if persisted_answer is not None:
+ ans = _normalize_answer(payload, persisted_answer)
+ from_resume = True
+ outcome = _outcome_from_answer(kind, ans)
+ if self._emit is not None:
+ from cubepi.agent.types import HitlAnswerEvent
+
+ await self._emit_event(
+ HitlAnswerEvent(question_id=question_id, answer=ans)
+ )
+ return ans
+
if self._pending is not None:
raise HitlConcurrencyError(
f"channel busy: already pending {self._pending.question_id}"
@@ -362,6 +381,7 @@ class _BaseChannel:
f"answer for {question_id}; pending is "
f"{self._pending.question_id if self._pending else 'None'}"
)
+ await self._save_answer(question_id, answer)
if self._future is not None and not self._future.done():
self._future.set_result(answer)
if self._emit is not None: # pragma: no cover — integration tested
@@ -493,6 +513,12 @@ def _outcome_from_answer(kind: str, ans: StructuredValue) -> str:
return "answered"
+def _normalize_answer(payload: HitlPayload, ans: StructuredValue) -> StructuredValue:
+ if isinstance(payload, ApproveRequest) and not isinstance(ans, ApproveAnswer):
+ return ApproveAnswer.model_validate(ans)
+ return ans
+
+
def _outcome_from_exception(exc: BaseException) -> str:
from cubepi.hitl.exceptions import (
HitlAborted,
@@ -515,6 +541,21 @@ def _outcome_from_exception(exc: BaseException) -> str:
class InMemoryChannel(_BaseChannel):
"""In-process HITL channel; no persistence."""
+ def __init__(
+ self,
+ *,
+ default_timeout: float | None = None,
+ thread_id: str | None = None,
+ ) -> None:
+ super().__init__(default_timeout=default_timeout, thread_id=thread_id)
+ self._answer_ledger: dict[str, StructuredValue] = {}
+
+ async def _load_answer(self, question_id: str) -> StructuredValue | None:
+ return self._answer_ledger.get(question_id)
+
+ async def _save_answer(self, question_id: str, answer: StructuredValue) -> None:
+ self._answer_ledger[question_id] = answer
+
class CheckpointedChannel(_BaseChannel):
"""Cross-process HITL channel — persists the pending request via a
@@ -540,12 +581,17 @@ class CheckpointedChannel(_BaseChannel):
if not (
hasattr(checkpointer, "save_pending_request")
and hasattr(checkpointer, "load_pending_request")
+ and hasattr(checkpointer, "save_hitl_answer")
+ and hasattr(checkpointer, "load_hitl_answer")
+ and hasattr(checkpointer, "clear_hitl_answers")
):
from cubepi.hitl.exceptions import HitlError
raise HitlError(
"CheckpointedChannel requires a checkpointer with "
- "save_pending_request and load_pending_request methods. "
+ "save_pending_request, load_pending_request, "
+ "save_hitl_answer, load_hitl_answer, and "
+ "clear_hitl_answers methods. "
"First-party checkpointers (Memory/SQLite/Postgres/MySQL) "
"implement these; third-party Protocol-only impls may not."
)
@@ -559,6 +605,18 @@ class CheckpointedChannel(_BaseChannel):
self._run_id = run_id
self._allow_inside_custom_tool = allow_inside_custom_tool
+ async def _load_answer(self, question_id: str) -> StructuredValue | None:
+ assert self._thread_id is not None
+ return await self._checkpointer.load_hitl_answer(
+ self._thread_id, question_id, run_id=self._run_id
+ )
+
+ async def _save_answer(self, question_id: str, answer: StructuredValue) -> None:
+ assert self._thread_id is not None
+ await self._checkpointer.save_hitl_answer(
+ self._thread_id, question_id, answer, run_id=self._run_id
+ )
+
async def _on_pending_set(self, req: HitlRequest) -> None:
if _in_custom_tool_var.get() and not self._allow_inside_custom_tool:
from cubepi.hitl.exceptions import HitlDurabilityNotGuaranteed
diff --git a/cubepi/mcp/_tracing.py b/cubepi/mcp/_tracing.py
index 2ff1b02..57f75cd 100644
--- a/cubepi/mcp/_tracing.py
+++ b/cubepi/mcp/_tracing.py
@@ -36,6 +36,11 @@ from typing import TYPE_CHECKING, Any, AsyncIterator
if TYPE_CHECKING:
pass
+_otel_trace: Any
+SpanKind: Any
+Status: Any
+StatusCode: Any
+
try:
from opentelemetry import trace as _otel_trace
@@ -44,10 +49,10 @@ try:
_OTEL_AVAILABLE = True
except ImportError: # pragma: no cover — exercised only without the extra.
_OTEL_AVAILABLE = False
- _otel_trace = None # type: ignore[assignment]
- SpanKind = None # type: ignore[assignment,misc]
- Status = None # type: ignore[assignment,misc]
- StatusCode = None # type: ignore[assignment,misc]
+ _otel_trace = None
+ SpanKind = None
+ Status = None
+ StatusCode = None
# When :class:`cubepi.tracing.Tracer` is attached to an agent it
diff --git a/dev/plans/2026-06-27-hitl-answer-ledger.md b/dev/plans/2026-06-27-hitl-answer-ledger.md
new file mode 100644
index 0000000..7964417
--- /dev/null
+++ b/dev/plans/2026-06-27-hitl-answer-ledger.md
@@ -0,0 +1,258 @@
+# HITL Answer Ledger Implementation Plan
+
+- Date: 2026-06-27
+- Spec: `dev/specs/2026-06-27-hitl-answer-ledger.md`
+- Status: Draft for review
+
+## Objective
+
+Fix the ROOT-A deadlock where parallel two-phase tool preparation and the
+single-slot HITL resume channel can ping-pong between two approval-gated tool
+calls forever. The fix is a framework-level durable answer ledger keyed by
+thread, run, and question id.
+
+## Constraints
+
+- Do not force approval-gated tools into `execution_mode="sequential"`.
+- Do not weaken the two-phase parallel executor's side-effect safety.
+- Do not require hosts to classify HITL-gated tools.
+- Keep the public visible HITL model as one pending request at a time.
+- Run all repo commands through `uv`.
+
+## Phase 1: Add the Checkpointer Ledger Contract
+
+Files:
+
+- `cubepi/checkpointer/base.py`
+- `cubepi/checkpointer/memory.py`
+- `tests/checkpointer/test_memory_checkpointer.py` or the nearest existing
+ memory checkpointer test module
+
+Steps:
+
+1. Import `Iterable` and `StructuredValue` where needed.
+2. Add `save_hitl_answer`, `load_hitl_answer`, and `clear_hitl_answers` to the
+ `Checkpointer` protocol.
+3. Implement the methods in `MemoryCheckpointer` with a private dict keyed by
+ `(thread_id, run_id or "", question_id)`.
+4. Copy answers on save/load if the existing message/pending code uses deep
+ copies for comparable mutable state.
+5. Add focused unit tests:
+ - saving then loading returns the answer
+ - run id scopes answers
+ - clearing a single question id preserves other answers
+ - clearing all answers for a run removes only that run
+
+Acceptance:
+
+- Memory checkpointer tests pass.
+- The protocol names match the spec exactly.
+
+## Phase 2: Add Durable Storage to SQL Checkpointers
+
+Files:
+
+- `cubepi/checkpointer/sqlite.py`
+- `cubepi/checkpointer/postgres/checkpointer.py`
+- `cubepi/checkpointer/postgres/models.py`
+- `cubepi/checkpointer/mysql/checkpointer.py`
+- `cubepi/checkpointer/mysql/models.py`
+- existing SQL checkpointer tests under `tests/checkpointer/`
+
+Steps:
+
+1. Add a `cubepi_hitl_answers` table to each first-party SQL backend.
+2. Use columns:
+ - `thread_id`
+ - `run_id`, stored as non-null text with `""` representing `None`
+ - `question_id`
+ - `answer_json`
+ - `answered_at`
+3. Make `(thread_id, run_id, question_id)` the primary key.
+4. Implement upsert semantics for `save_hitl_answer`.
+5. Implement exact-key lookup for `load_hitl_answer`.
+6. Implement `clear_hitl_answers` with two modes:
+ - `question_ids is None`: delete all answers for `(thread_id, run_id)`
+ - `question_ids` supplied: delete only those ids for `(thread_id, run_id)`
+7. Add CRUD tests for SQLite. Extend Postgres/MySQL tests where the repo already
+ runs backend-specific checkpointer HITL tests.
+
+Acceptance:
+
+- SQLite tests pass locally.
+- Backend-specific tests are updated without adding hard dependencies to core.
+
+## Phase 3: Teach HITL Channels to Replay Ledger Answers
+
+Files:
+
+- `cubepi/hitl/channel.py`
+- `tests/hitl/test_in_memory_channel.py`
+- `tests/hitl/test_channel.py` or nearest existing checkpointed channel tests
+
+Steps:
+
+1. Extend `_BaseChannel` with optional ledger hooks:
+ - `_load_answer(question_id)`
+ - `_save_answer(question_id, answer)`
+ - `_clear_answers(...)` only if channel-level abort cleanup is already
+ centralized there
+2. Implement default no-op behavior for plain in-process channels, then override
+ as needed in `InMemoryChannel` and `CheckpointedChannel`.
+3. Add an in-memory answer dict to `InMemoryChannel` so same-process behavior
+ follows the same replay semantics.
+4. Update `CheckpointedChannel.__init__` validation to require the new ledger
+ methods when used for detached HITL.
+5. Change `_await_answer()` resolution order:
+ - load durable/in-memory ledger answer for `question_id`
+ - fall back to matching `_resume_slot`
+ - otherwise publish pending and wait/detach
+6. When `answer()` resolves the currently pending request in process, save the
+ answer before waking the waiter.
+7. Do not clear a ledger answer when `_await_answer()` returns it.
+8. Add tests proving:
+ - replayed ledger answers do not create a new pending request
+ - stale process-local resume slots still do not satisfy the wrong question id
+ - in-process `answer()` also records a replayable answer
+
+Acceptance:
+
+- Existing HITL channel tests still pass.
+- New replay tests fail without the ledger lookup and pass with it.
+
+## Phase 4: Persist Answers in Agent.respond and Clear Them Safely
+
+Files:
+
+- `cubepi/agent/agent.py`
+- `cubepi/agent/loop.py`
+- `cubepi/hitl/channel.py` if abort cleanup belongs there
+- `tests/hitl/test_loop_hitl_passthrough.py`
+- new or existing agent HITL resume tests
+
+Steps:
+
+1. In `Agent.respond()`, after validating pending `question_id` and recovered
+ run id, call `save_hitl_answer(thread_id, question_id, answer, run_id=...)`
+ before `_run_hitl_resume()`.
+2. Keep `attach_resume_answer()` as a same-process fast path, but make the
+ ledger the durable source of truth.
+3. Ensure resume detaching on the next pending request does not clear saved
+ answers.
+4. Clear ledger answers for `(thread_id, active_run_id)` only after the suspended
+ tool cycle has completed and tool results have been appended.
+5. Clear ledger answers when pending HITL is aborted/cancelled or when the run
+ reaches a terminal cleanup path.
+6. Prefer clearing all answers for the run over clearing only tool call ids, so
+ `ask_user` question ids are covered.
+7. Add tests for cleanup:
+ - after successful completion
+ - after abort
+ - no cleanup after detach on another request
+
+Acceptance:
+
+- A saved answer survives a detach on a later tool call.
+- Saved answers are not left behind after successful completion or abort.
+
+## Phase 5: Add the ROOT-A Regression Test
+
+Files:
+
+- `tests/hitl/test_loop_hitl_passthrough.py` or a new
+ `tests/hitl/test_parallel_approval_ledger.py`
+
+Test shape:
+
+1. Define two tools that require approval and record executions.
+2. Use a provider response that requests both tools in the same assistant turn
+ with parallel execution enabled.
+3. Start `Agent.prompt()` and observe pending `A`.
+4. Call `Agent.respond(question_id=A, answer=approve)`.
+5. Assert:
+ - no tool body has executed yet
+ - no tool results have been appended yet
+ - pending is now `B`
+6. Call `Agent.respond(question_id=B, answer=approve)`.
+7. Assert:
+ - both tool bodies executed exactly once
+ - two tool results were appended
+ - pending is cleared
+ - ledger answers for the run are cleared
+8. Repeat the same flow with a fresh `Agent` instance between the two approvals
+ to prove durable replay works after process restart.
+
+Acceptance:
+
+- The test reproduces the current deadlock before the implementation.
+- The test passes after the answer ledger implementation.
+
+## Phase 6: Cover Decision Variants
+
+Files:
+
+- `tests/hitl/test_loop_hitl_passthrough.py`
+- approval middleware tests under `tests/hitl/`
+
+Steps:
+
+1. Add a deny/reject variant where `A` is denied, resume detaches on `B`, then
+ `B` is approved. Verify the replayed denial emits the expected tool result
+ and no denied tool body executes.
+2. Add an edit variant if the existing approval middleware supports edited tool
+ arguments in tests. Verify the edited decision survives the second detach and
+ the tool receives edited args.
+3. Add an `ask_user` replay case if the existing `ask_user` tests can express a
+ multi-question resume flow without building a large fixture.
+
+Acceptance:
+
+- The ledger stores and replays structured answers, not just boolean approvals.
+
+## Phase 7: Update Documentation
+
+Files:
+
+- `website/docs/` HITL guide or recipe page
+- Chinese mirror if this docs tree keeps one for the changed page
+
+Steps:
+
+1. Document the new guarantee: approval-gated tools can remain parallel.
+2. Explain that approving one request in a parallel batch records the decision,
+ but execution waits until the remaining gates are answered.
+3. Remove or avoid guidance that tells users to mark gated tools sequential only
+ to avoid HITL deadlocks.
+4. Add a short example flow with two parallel approval-gated tool calls.
+
+Acceptance:
+
+- User-facing docs match the runtime behavior.
+- The feature is not considered complete until docs ship in the same PR.
+
+## Verification Commands
+
+Run focused tests first:
+
+```bash
+uv run pytest tests/hitl/test_in_memory_channel.py -v
+uv run pytest tests/hitl/test_loop_hitl_passthrough.py -v
+uv run pytest tests/checkpointer/ -k "memory or sqlite or hitl" -v
+```
+
+Then run broader checks:
+
+```bash
+uv run pytest tests/
+uv run ruff check cubepi/ tests/
+uv run ruff format --check cubepi/ tests/
+uv run mypy cubepi
+```
+
+## Review Checkpoints
+
+1. Review the spec and this plan before coding.
+2. After code and docs are ready, ask before entering the local codex review
+ loop required by `AGENTS.md`.
+3. Open a PR from the worktree branch after implementation and documentation.
+4. Drive the PR codex review loop until review and CI are clean.
diff --git a/dev/specs/2026-06-27-hitl-answer-ledger.md b/dev/specs/2026-06-27-hitl-answer-ledger.md
new file mode 100644
index 0000000..9e49188
--- /dev/null
+++ b/dev/specs/2026-06-27-hitl-answer-ledger.md
@@ -0,0 +1,278 @@
+# HITL Answer Ledger Design Spec
+
+- Date: 2026-06-27
+- Status: Draft for review
+- Related issue: ROOT-A, parallel two-phase tool preparation plus single-slot HITL deadlock
+- Companion plan: `dev/plans/2026-06-27-hitl-answer-ledger.md`
+
+## Problem
+
+cubepi's parallel tool executor intentionally uses two phases:
+
+1. Prepare every requested tool call. Preparation includes middleware gates such
+ as approval checks.
+2. Only after every tool call is prepared, execute the prepared calls in
+ parallel and append the resulting tool results.
+
+That split prevents a bad replay failure mode. If tool A executed before tool B
+finished preparation, and tool B then detached for human input, tool A could have
+performed side effects without a durable `ToolResult`. Resuming the same turn
+would then risk running tool A again.
+
+The split is correct for side-effect safety, but it conflicts with the current
+HITL resume channel. The channel has one visible pending request and one
+single-use resume answer slot. With two parallel approval-gated tool calls,
+`A` and `B`, the system can deadlock:
+
+1. User approves `A`.
+2. Resume prepares `A`, consumes the answer, then prepares `B`.
+3. `B` has no answer, so preparation detaches before any tool body executes.
+4. User approves `B`.
+5. Resume starts over from the same assistant message. `A` now has no answer,
+ so preparation detaches before `B` can be prepared.
+
+The pending request flips between `A` and `B`. No tool results or idempotency
+sentinels are written because execution never starts.
+
+Marking approval-gated tools as sequential avoids the deadlock, but it leaks a
+framework invariant into application policy. It also disables useful parallelism
+for batches that are safe to run together after their gates have all been
+answered.
+
+## Goals
+
+- Preserve the parallel executor's side-effect guarantee: no tool body in a
+ parallel batch starts until every tool call in that batch has prepared
+ successfully.
+- Preserve one visible pending HITL request at a time, so existing hosts do not
+ need to build a multi-prompt UI.
+- Make answered HITL prompts durable and replayable across resume attempts.
+- Keep application code out of the routing decision. Hosts should not need to
+ mark approval-gated tools as sequential to avoid this deadlock.
+- Support the existing approval middleware and `ask_user` flow without changing
+ public tool definitions.
+- Keep the design small and async-native, consistent with cubepi's current
+ checkpointer and channel abstractions.
+
+## Non-goals
+
+- Showing multiple simultaneous pending prompts in the public HITL API.
+- Executing an approved prefix of a parallel batch before the rest of the batch
+ has been approved.
+- Replacing cubepi's message-loop execution model with a graph runtime.
+- Making arbitrary custom tool bodies crash-safe when they call HITL after
+ performing side effects. `CheckpointedChannel` should continue to reject that
+ case by default.
+- Adding a new hard runtime dependency.
+
+## Proposed Design
+
+Add a durable HITL answer ledger. The ledger stores answered HITL requests by
+thread, run, and question id. Preparing a gated call first checks the ledger. If
+an answer already exists for that question id, preparation replays that answer
+without publishing a new pending request and without deleting the answer.
+
+This changes the deadlocked sequence into a progressive collection of answers:
+
+1. User approves `A`.
+2. Resume prepares `A` from the ledger, then detaches on `B`. No tool body runs.
+3. User approves `B`.
+4. Resume prepares `A` from the ledger and prepares `B` from the ledger.
+5. All preparation succeeds, so the executor runs `A` and `B` in parallel.
+6. Tool results are appended, pending state is cleared, and ledger entries for
+ the completed run are deleted.
+
+The answer ledger preserves two-phase execution. It does not execute `A` early;
+it only remembers that `A`'s gate has already been satisfied.
+
+### Ledger Key
+
+The logical key is:
+
+```text
+(thread_id, run_id, question_id)
+```
+
+`run_id` is nullable for legacy in-memory or non-run hosts. First-party durable
+checkpointers should normalize `None` to an empty string or equivalent storage
+key so the database primary key remains stable.
+
+`question_id` is the existing HITL question id. For approval middleware this is
+the tool call id. For `ask_user`, it is the id already produced by the channel.
+
+### Ledger Record
+
+The record only needs enough data to replay the answer:
+
+```python
+@dataclass
+class HitlAnswerRecord:
+ question_id: str
+ answer: StructuredValue
+ run_id: str | None = None
+ answered_at: float = field(default_factory=time.time)
+```
+
+Implementations may store a request snapshot for audit/debugging, but replay
+must not depend on it. The pending row remains the source of truth for the
+currently visible request.
+
+### Checkpointer API
+
+Extend the checkpointer protocol with answer ledger methods:
+
+```python
+async def save_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ answer: StructuredValue,
+ *,
+ run_id: str | None = None,
+) -> None: ...
+
+async def load_hitl_answer(
+ self,
+ thread_id: str,
+ question_id: str,
+ *,
+ run_id: str | None = None,
+) -> StructuredValue | None: ...
+
+async def clear_hitl_answers(
+ self,
+ thread_id: str,
+ question_ids: Iterable[str] | None = None,
+ *,
+ run_id: str | None = None,
+) -> None: ...
+```
+
+First-party checkpointers must implement these methods:
+
+- `MemoryCheckpointer`: an in-memory dict keyed by `(thread_id, run_key,
+ question_id)`.
+- `SQLiteCheckpointer`: a new table with `thread_id`, `run_id`,
+ `question_id`, `answer_json`, and `answered_at`.
+- `PostgresCheckpointer`: equivalent table with JSONB answer storage.
+- `MySQLCheckpointer`: equivalent table with JSON answer storage.
+
+`CheckpointedChannel` should validate these methods when it is used for
+detached HITL. A missing method should fail early with an actionable `HitlError`,
+the same way missing pending APIs are handled today.
+
+### Channel Semantics
+
+`_BaseChannel._await_answer()` should resolve answers in this order:
+
+1. If a durable ledger lookup is available and contains `question_id`, return
+ that answer immediately.
+2. Otherwise, if the process-local resume slot matches `question_id`, return
+ that answer. This keeps same-process behavior and legacy tests simple.
+3. Otherwise, publish the request as the single pending request and detach or
+ wait according to the current channel mode.
+
+Returning a ledger answer must not delete it. Deletion before the whole batch is
+durably resolved would recreate the same bug after a later detachment.
+
+When an answer is supplied through `channel.answer()` in-process, the channel
+should also save it to the ledger before waking the waiter. That keeps behavior
+consistent between direct in-process HITL and `Agent.respond()`.
+
+### Agent.respond Semantics
+
+`Agent.respond()` should:
+
+1. Load the pending request and recovered run id atomically through
+ `load_pending()`.
+2. Validate that the supplied `question_id` matches the visible pending request.
+3. Validate the agent/channel run binding against the recovered run id.
+4. Save the answer into the ledger before starting resume.
+5. Optionally attach the process-local resume answer for fast in-process replay.
+6. Resume the original tool cycle.
+
+If resume detaches on another request, the just-saved answer remains in the
+ledger. A later resume can replay it.
+
+### Cleanup Semantics
+
+Ledger answers are cleared only when they can no longer be needed for replay:
+
+- After the suspended tool cycle completes successfully and tool results have
+ been appended.
+- When the active HITL request is aborted or cancelled by the host.
+- When the agent clears pending state because the run has reached a terminal
+ outcome.
+
+Do not clear a ledger entry immediately after a single preparation step consumes
+it. For parallel batches, preparation may still detach on a later tool call.
+
+For simplicity and safety, successful completion can clear all ledger answers
+for `(thread_id, run_id)` rather than trying to clear only the exact tool call
+ids. This also covers `ask_user`, where the question id may not equal a tool
+call id.
+
+### User-visible Behavior
+
+After approving one tool in a multi-gated parallel batch, the tool may still not
+have executed. The approval means "this gate is answered"; execution begins only
+when all gates in the current parallel preparation batch are answered.
+
+Hosts can present this as:
+
+```text
+Approved. Waiting for remaining approvals before execution.
+```
+
+No API change is required for hosts that already call `Agent.respond()` against
+the currently pending request.
+
+## Compatibility
+
+- Existing pending request APIs remain unchanged.
+- Existing `execution_mode="sequential"` remains useful for genuinely ordered
+ or non-concurrency-safe tools, but it is not required for approval-gated tools.
+- Third-party checkpointers that use `CheckpointedChannel` for detached HITL
+ need to implement the ledger methods. The failure mode should be explicit at
+ channel construction or first detached response, not a silent fallback to the
+ deadlocking behavior.
+- In-process `InMemoryChannel` remains usable without a durable checkpointer,
+ but should still keep an in-memory ledger so the same replay rules apply.
+
+## Testing Requirements
+
+Add tests covering:
+
+- `MemoryCheckpointer`, `SQLiteCheckpointer`, `PostgresCheckpointer`, and
+ `MySQLCheckpointer` answer ledger CRUD where those backends already have HITL
+ tests.
+- `CheckpointedChannel` replays a saved answer for the same `question_id`
+ without publishing a new pending request.
+- A parallel batch with two approval-gated tools no longer deadlocks:
+ approving `A` records the answer and leaves pending `B`; approving `B`
+ executes both tools exactly once and appends two tool results.
+- Restart behavior: use a fresh `Agent` instance between approvals and verify
+ replay still succeeds.
+- Rejection/deny and edited approval decisions replay correctly.
+- Existing regression coverage that a later preparation detach prevents earlier
+ tool bodies from starting still passes.
+- Ledger entries are cleared after successful completion and after abort.
+
+## Documentation Requirements
+
+Update the HITL user-facing docs under `website/docs/` to explain:
+
+- Parallel approval-gated tool calls are safe.
+- Approving one item in a parallel batch records that decision, but execution
+ waits until all gates in the batch are satisfied.
+- Application authors do not need to mark approval-gated tools as sequential
+ solely for HITL safety.
+
+## Risks
+
+- Stale answers are dangerous if keyed too broadly. Use `run_id` when available
+ and keep question ids unique within a turn.
+- Clearing too early recreates the deadlock. Cleanup must happen after durable
+ tool results are appended or after terminal abort/cancel.
+- Third-party checkpointers need a clear migration path. The error should name
+ the new ledger methods and explain that detached HITL requires them.
diff --git a/tests/checkpointer/test_mysql.py b/tests/checkpointer/test_mysql.py
index f2c5fee..ee6ae6d 100644
--- a/tests/checkpointer/test_mysql.py
+++ b/tests/checkpointer/test_mysql.py
@@ -29,6 +29,7 @@ def test_exceptions_are_postgres_aliases() -> None:
def test_models_import() -> None:
from cubepi.checkpointer.mysql.models import (
EXPECTED_SCHEMA_VERSION,
+ CubepiHitlAnswer,
PARTITION_COUNT,
CubepiMessage,
CubepiSchemaVersion,
@@ -36,13 +37,15 @@ def test_models_import() -> None:
cubepi_metadata,
)
- assert EXPECTED_SCHEMA_VERSION == 4
+ assert EXPECTED_SCHEMA_VERSION == 5
assert PARTITION_COUNT == 64
assert CubepiThread.__tablename__ == "cubepi_threads"
assert CubepiMessage.__tablename__ == "cubepi_messages"
+ assert CubepiHitlAnswer.__tablename__ == "cubepi_hitl_answers"
assert CubepiSchemaVersion.__tablename__ == "cubepi_schema_version"
assert "cubepi_threads" in cubepi_metadata.tables
assert "cubepi_messages" in cubepi_metadata.tables
+ assert "cubepi_hitl_answers" in cubepi_metadata.tables
assert "cubepi_schema_version" in cubepi_metadata.tables
@@ -232,6 +235,7 @@ async def _setup_schema(dsn: str) -> None:
add_pending_request_column_op,
add_run_id_column_op,
upgrade_v3_to_v4_op,
+ upgrade_v4_to_v5_op,
)
await cur.execute(add_pending_request_column_op())
@@ -258,6 +262,9 @@ async def _setup_schema(dsn: str) -> None:
for stmt in upgrade_v3_to_v4_op().split(";"):
if stmt.strip():
await cur.execute(stmt)
+ for stmt in upgrade_v4_to_v5_op().split(";"):
+ if stmt.strip():
+ await cur.execute(stmt)
for stmt in write_schema_version_op().split(";"):
if stmt.strip():
await cur.execute(stmt)
@@ -389,7 +396,9 @@ async def test_version_mismatch_raises(clean_mysql_db) -> None:
with pytest.raises(CubepiSchemaMismatch) as exc_info:
async with MySQLCheckpointer(clean_mysql_db):
pass
- assert exc_info.value.expected == 4
+ from cubepi.checkpointer.mysql.models import EXPECTED_SCHEMA_VERSION
+
+ assert exc_info.value.expected == EXPECTED_SCHEMA_VERSION
assert exc_info.value.actual == 999
diff --git a/tests/checkpointer/test_mysql_schema.py b/tests/checkpointer/test_mysql_schema.py
index aad1892..7b1a6a1 100644
--- a/tests/checkpointer/test_mysql_schema.py
+++ b/tests/checkpointer/test_mysql_schema.py
@@ -6,8 +6,8 @@ import pytest
from cubepi.checkpointer.mysql.models import EXPECTED_SCHEMA_VERSION
-def test_expected_schema_version_is_4() -> None:
- assert EXPECTED_SCHEMA_VERSION == 4
+def test_expected_schema_version_is_5() -> None:
+ assert EXPECTED_SCHEMA_VERSION == 5
@pytest.mark.asyncio
@@ -54,3 +54,30 @@ async def test_cubepi_messages_has_run_id_column(mysql_v4_dsn) -> None:
finally:
await conn.ensure_closed()
assert row is not None
+
+
+async def test_cubepi_hitl_answers_table_present(mysql_v4_dsn) -> None:
+ from cubepi.checkpointer.mysql.checkpointer import _parse_dsn
+
+ cfg = _parse_dsn(mysql_v4_dsn)
+ conn = await aiomysql.connect(autocommit=True, **cfg)
+ try:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "SELECT column_name FROM information_schema.columns "
+ "WHERE table_schema = %s "
+ "AND table_name = 'cubepi_hitl_answers'",
+ (cfg["db"],),
+ )
+ rows = await cur.fetchall()
+ finally:
+ await conn.ensure_closed()
+ cols = {r[0].lower() for r in rows}
+ assert {
+ "thread_id",
+ "run_id",
+ "question_id",
+ "answer",
+ "answered_at",
+ } <= cols
diff --git a/tests/checkpointer/test_postgres.py b/tests/checkpointer/test_postgres.py
index 157a075..a5a80e7 100644
--- a/tests/checkpointer/test_postgres.py
+++ b/tests/checkpointer/test_postgres.py
@@ -10,6 +10,7 @@ import pytest
def test_models_import() -> None:
from cubepi.checkpointer.postgres.models import (
EXPECTED_SCHEMA_VERSION,
+ CubepiHitlAnswer,
PARTITION_COUNT,
CubepiMessage,
CubepiSchemaVersion,
@@ -17,15 +18,15 @@ def test_models_import() -> None:
cubepi_metadata,
)
- assert EXPECTED_SCHEMA_VERSION == 4
+ assert EXPECTED_SCHEMA_VERSION == 5
assert PARTITION_COUNT == 64
- # All three model classes are reachable via the public model module
assert CubepiThread.__tablename__ == "cubepi_threads"
assert CubepiMessage.__tablename__ == "cubepi_messages"
+ assert CubepiHitlAnswer.__tablename__ == "cubepi_hitl_answers"
assert CubepiSchemaVersion.__tablename__ == "cubepi_schema_version"
- # All three tables registered on cubepi_metadata
assert "cubepi_threads" in cubepi_metadata.tables
assert "cubepi_messages" in cubepi_metadata.tables
+ assert "cubepi_hitl_answers" in cubepi_metadata.tables
assert "cubepi_schema_version" in cubepi_metadata.tables
@@ -204,6 +205,7 @@ async def _setup_schema(dsn: str) -> None:
add_run_id_column_op,
create_message_partitions_op,
upgrade_v3_to_v4_op,
+ upgrade_v4_to_v5_op,
write_schema_version_op,
)
@@ -222,6 +224,7 @@ async def _setup_schema(dsn: str) -> None:
""")
# Apply v3→v4 (run_id on messages + cubepi_runs partitioned table).
await conn.execute(upgrade_v3_to_v4_op())
+ await conn.execute(upgrade_v4_to_v5_op())
await conn.execute(write_schema_version_op())
finally:
await conn.close()
@@ -330,7 +333,9 @@ async def test_version_mismatch_raises(clean_db) -> None:
with pytest.raises(CubepiSchemaMismatch) as exc_info:
async with PostgresCheckpointer(clean_db):
pass
- assert exc_info.value.expected == 4
+ from cubepi.checkpointer.postgres.models import EXPECTED_SCHEMA_VERSION
+
+ assert exc_info.value.expected == EXPECTED_SCHEMA_VERSION
assert exc_info.value.actual == 999
diff --git a/tests/checkpointer/test_postgres_schema.py b/tests/checkpointer/test_postgres_schema.py
index 672d5e1..ff62eb8 100644
--- a/tests/checkpointer/test_postgres_schema.py
+++ b/tests/checkpointer/test_postgres_schema.py
@@ -4,8 +4,8 @@ import pytest
from cubepi.checkpointer.postgres.models import EXPECTED_SCHEMA_VERSION
-def test_expected_schema_version_is_4():
- assert EXPECTED_SCHEMA_VERSION == 4
+def test_expected_schema_version_is_5():
+ assert EXPECTED_SCHEMA_VERSION == 5
@pytest.mark.asyncio
@@ -39,3 +39,23 @@ async def test_cubepi_messages_has_run_id_column(pg_v4_dsn):
assert row is not None
finally:
await conn.close()
+
+
+async def test_cubepi_hitl_answers_table_present(pg_v4_dsn):
+ conn = await asyncpg.connect(pg_v4_dsn)
+ try:
+ rows = await conn.fetch(
+ "SELECT column_name FROM information_schema.columns "
+ "WHERE table_name = 'cubepi_hitl_answers'"
+ )
+ cols = {r["column_name"] for r in rows}
+ assert {
+ "thread_id",
+ "run_id",
+ "question_id",
+ "answer",
+ "answered_at",
+ } <= cols
+ finally:
+ await conn.close()
diff --git a/tests/checkpointer/test_postgres_schema_version_v3.py b/tests/checkpointer/test_postgres_schema_version_v3.py
index c003841..30a9d1a 100644
--- a/tests/checkpointer/test_postgres_schema_version_v3.py
+++ b/tests/checkpointer/test_postgres_schema_version_v3.py
@@ -27,7 +27,7 @@ async def test_postgres_v2_database_refused_with_actionable_error(clean_db) -> N
pass
msg = str(ei.value)
- assert "expected 4" in msg or "expected=4" in msg
+ assert "expected 5" in msg or "expected=5" in msg
assert "actual 2" in msg or "actual=2" in msg
# Hint mentions the host alembic migration path so operators know what to do.
assert "alembic" in msg.lower()
@@ -59,6 +59,6 @@ async def test_mysql_v2_database_refused_with_actionable_error(clean_mysql_db) -
pass
msg = str(ei.value)
- assert "expected 4" in msg or "expected=4" in msg
+ assert "expected 5" in msg or "expected=5" in msg
assert "actual 2" in msg or "actual=2" in msg
assert "alembic" in msg.lower()
diff --git a/tests/hitl/test_checkpointed_channel_run_id.py b/tests/hitl/test_checkpointed_channel_run_id.py
index dc8fa9a..28f5a92 100644
--- a/tests/hitl/test_checkpointed_channel_run_id.py
+++ b/tests/hitl/test_checkpointed_channel_run_id.py
@@ -99,6 +99,7 @@ async def test_legacy_checkpointer_works_without_run_id_kwarg():
def __init__(self) -> None:
self._pending: dict[str, HitlRequest | None] = {}
+ self._answers: dict[str, object] = {}
async def save_pending_request(
self, thread_id: str, request
@@ -108,6 +109,25 @@ async def test_legacy_checkpointer_works_without_run_id_kwarg():
async def load_pending_request(self, thread_id: str):
return self._pending.get(thread_id)
+ async def save_hitl_answer(
+ self, thread_id: str, question_id: str, answer, *, run_id=None
+ ):
+ self._answers[question_id] = answer
+
+ async def load_hitl_answer(
+ self, thread_id: str, question_id: str, *, run_id=None
+ ):
+ return self._answers.get(question_id)
+
+ async def clear_hitl_answers(
+ self, thread_id: str, question_ids=None, *, run_id=None
+ ):
+ if question_ids is None:
+ self._answers.clear()
+ else:
+ for question_id in question_ids:
+ self._answers.pop(question_id, None)
+
cp = LegacyCheckpointer()
# No run_id at construction → channel must call legacy two-arg signature.
ch = CheckpointedChannel(checkpointer=cp, thread_id="t-legacy")
diff --git a/tests/hitl/test_checkpointer_pending_request.py b/tests/hitl/test_checkpointer_pending_request.py
index 0482e73..27e29d8 100644
--- a/tests/hitl/test_checkpointer_pending_request.py
+++ b/tests/hitl/test_checkpointer_pending_request.py
@@ -63,3 +63,54 @@ async def test_sqlite_create_table_idempotent(sqlite_cp):
"thread_id TEXT PRIMARY KEY, request_json TEXT NOT NULL, "
"created_at REAL NOT NULL DEFAULT (julianday('now')))"
)
+
+
+async def test_memory_hitl_answer_ledger_crud():
+ cp = MemoryCheckpointer()
+ await cp.save_hitl_answer("t-1", "q-1", {"decision": "approve"}, run_id="r-1")
+ await cp.save_hitl_answer("t-1", "q-2", {"decision": "deny"}, run_id="r-1")
+ await cp.save_hitl_answer("t-1", "q-1", {"decision": "edit"}, run_id="r-2")
+
+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-1") == {
+ "decision": "approve"
+ }
+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {"decision": "edit"}
+
+ await cp.clear_hitl_answers("t-1", ["q-1"], run_id="r-1")
+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-1") is None
+ assert await cp.load_hitl_answer("t-1", "q-2", run_id="r-1") == {"decision": "deny"}
+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {"decision": "edit"}
+
+ await cp.clear_hitl_answers("t-1", run_id="r-1")
+ assert await cp.load_hitl_answer("t-1", "q-2", run_id="r-1") is None
+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {"decision": "edit"}
+
+
+async def test_sqlite_hitl_answer_ledger_crud(sqlite_cp):
+ await sqlite_cp.save_hitl_answer(
+ "t-1", "q-1", {"decision": "approve"}, run_id="r-1"
+ )
+ await sqlite_cp.save_hitl_answer("t-1", "q-2", {"decision": "deny"}, run_id="r-1")
+ await sqlite_cp.save_hitl_answer("t-1", "q-1", {"decision": "edit"}, run_id="r-2")
+
+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-1") == {
+ "decision": "approve"
+ }
+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {
+ "decision": "edit"
+ }
+
+ await sqlite_cp.clear_hitl_answers("t-1", ["q-1"], run_id="r-1")
+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-1") is None
+ assert await sqlite_cp.load_hitl_answer("t-1", "q-2", run_id="r-1") == {
+ "decision": "deny"
+ }
+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {
+ "decision": "edit"
+ }
+
+ await sqlite_cp.clear_hitl_answers("t-1", run_id="r-1")
+ assert await sqlite_cp.load_hitl_answer("t-1", "q-2", run_id="r-1") is None
+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {
+ "decision": "edit"
+ }
diff --git a/tests/hitl/test_in_memory_channel.py b/tests/hitl/test_in_memory_channel.py
index 23c37a1..eda0085 100644
--- a/tests/hitl/test_in_memory_channel.py
+++ b/tests/hitl/test_in_memory_channel.py
@@ -209,6 +209,26 @@ async def test_attach_resume_answer_short_circuits_next_call():
assert ch.pending is None
+async def test_answer_ledger_short_circuits_replayed_approve():
+ ch = InMemoryChannel()
+
+ async def host():
+ while ch.pending is None:
+ await asyncio.sleep(0)
+ await ch.answer(ch.pending.question_id, ApproveAnswer(decision="approve"))
+
+ asyncio.create_task(host())
+ first = await ch.approve(tool_name="bash", tool_call_id="tc-7", args={})
+ assert first.decision == "approve"
+ assert ch.pending is None
+
+ replayed = await ch.approve(
+ tool_name="bash", tool_call_id="tc-7", args={}, timeout=0.01
+ )
+ assert replayed.decision == "approve"
+ assert ch.pending is None
+
+
async def test_resume_short_circuit_emits_hitl_answer_event():
"""Resume short-circuit must emit HitlAnswerEvent so subscribers
(e.g. IM outbound tailers) learn the question was answered."""
diff --git a/tests/hitl/test_parallel_approval_ledger.py b/tests/hitl/test_parallel_approval_ledger.py
new file mode 100644
index 0000000..c496c5b
--- /dev/null
+++ b/tests/hitl/test_parallel_approval_ledger.py
@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+from pydantic import BaseModel
+
+from cubepi.agent.agent import Agent
+from cubepi.agent.types import AgentTool, AgentToolResult
+from cubepi.checkpointer.memory import MemoryCheckpointer
+from cubepi.hitl import ApproveAnswer, AskUser
+from cubepi.hitl.channel import CheckpointedChannel
+from cubepi.hitl.middleware import ApprovalPolicyMiddleware
+from cubepi.providers.base import TextContent, ToolResultMessage
+from cubepi.providers.faux import FauxProvider, faux_assistant_message, faux_tool_call
+
+
+class _NoParams(BaseModel):
+ pass
+
+
+def _make_parallel_tool(name: str, executions: list[str]) -> AgentTool:
+ async def execute(call_id, args, *, signal=None, on_update=None):
+ executions.append(call_id)
+ return AgentToolResult(content=[TextContent(text=f"{name} ran")])
+
+ return AgentTool(
+ name=name,
+ description=name,
+ parameters=_NoParams,
+ execute=execute,
+ execution_mode="parallel",
+ )
+
+
+def _provider() -> FauxProvider:
+ provider = FauxProvider(provider_id="faux")
+ provider.set_responses(
+ [
+ faux_assistant_message(
+ [
+ faux_tool_call("solar", {}, id="tc-a"),
+ faux_tool_call("semiconductor", {}, id="tc-b"),
+ ],
+ stop_reason="tool_use",
+ ),
+ faux_assistant_message("done"),
+ ]
+ )
+ return provider
+
+
+def _agent(
+ *,
+ cp: MemoryCheckpointer,
+ provider: FauxProvider,
+ executions: list[str],
+) -> Agent:
+ ch = CheckpointedChannel(checkpointer=cp, thread_id="t-root-a", run_id="R1")
+ return Agent(
+ model=provider.model("faux"),
+ tools=[
+ _make_parallel_tool("solar", executions),
+ _make_parallel_tool("semiconductor", executions),
+ ],
+ middleware=[ApprovalPolicyMiddleware(ch, policy=lambda c: AskUser())],
+ channel=ch,
+ checkpointer=cp,
+ thread_id="t-root-a",
+ )
+
+
+async def _wait_for_pending(cp: MemoryCheckpointer, expected_qid: str) -> None:
+ for _ in range(200):
+ loaded = await cp.load_pending("t-root-a")
+ if loaded is not None and loaded[0].question_id == expected_qid:
+ return
+ await asyncio.sleep(0.01)
+ pytest.fail(f"pending request did not become {expected_qid!r}")
+
+
+async def test_parallel_approval_answers_are_replayed_until_batch_can_execute():
+ cp = MemoryCheckpointer()
+ provider = _provider()
+ executions: list[str] = []
+
+ initial_agent = _agent(cp=cp, provider=provider, executions=executions)
+ prompt_task = asyncio.create_task(initial_agent.prompt("run both", run_id="R1"))
+ await _wait_for_pending(cp, "tc-a")
+ await initial_agent.detach()
+ await prompt_task
+
+ first_resume = _agent(cp=cp, provider=provider, executions=executions)
+ first_resume_task = asyncio.create_task(
+ first_resume.respond(
+ question_id="tc-a", answer=ApproveAnswer(decision="approve")
+ )
+ )
+ await _wait_for_pending(cp, "tc-b")
+ await first_resume.detach()
+ await first_resume_task
+
+ assert executions == []
+ loaded = await cp.load_pending("t-root-a")
+ assert loaded is not None
+ assert loaded[0].question_id == "tc-b"
+
+ second_resume = _agent(cp=cp, provider=provider, executions=executions)
+ await second_resume.respond(
+ question_id="tc-b", answer=ApproveAnswer(decision="approve")
+ )
+
+ assert sorted(executions) == ["tc-a", "tc-b"]
+ assert await cp.load_pending("t-root-a") is None
+ assert await cp.load_hitl_answer("t-root-a", "tc-a", run_id="R1") is None
+ assert await cp.load_hitl_answer("t-root-a", "tc-b", run_id="R1") is None
+ tool_results = [
+ msg
+ for msg in second_resume.state.messages
+ if isinstance(msg, ToolResultMessage)
+ ]
+ assert [msg.tool_call_id for msg in tool_results] == ["tc-a", "tc-b"]
diff --git a/website/docs/guides/checkpointing/mysql.md b/website/docs/guides/checkpointing/mysql.md
index 071db2b..4fcb9e2 100644
--- a/website/docs/guides/checkpointing/mysql.md
+++ b/website/docs/guides/checkpointing/mysql.md
@@ -61,10 +61,11 @@ async with MySQLCheckpointer(
## Schema
-The checkpointer expects three tables: `cubepi_threads`,
-`cubepi_messages`, and `cubepi_schema_version`. Like Postgres (and
-unlike SQLite), CubePi **does not create these for you** — it verifies
-on `__aenter__` that they exist with the expected `schema_version`.
+The checkpointer expects `cubepi_threads`, `cubepi_messages`,
+`cubepi_runs`, `cubepi_hitl_answers`, and `cubepi_schema_version`. Like
+Postgres (and unlike SQLite), CubePi **does not create these for you** —
+it verifies on `__aenter__` that they exist with the expected
+`schema_version`.
If they're missing, you get `CubepiSchemaUninitialized`. If the version
doesn't match this CubePi release, you get `CubepiSchemaMismatch`. A
@@ -141,6 +142,16 @@ cubepi_messages
payload -- LONGBLOB (msgpack)
created_at
+cubepi_runs
+ thread_id, run_id -- composite PK
+ claimed_at / completed_at
+ completion_seq
+
+cubepi_hitl_answers
+ thread_id, run_id, question_id -- composite PK
+ answer -- JSON
+ answered_at
+
cubepi_schema_version
version (PK)
```
@@ -224,6 +235,29 @@ Pre-feature messages keep `run_id = NULL` and remain readable; see
[Legacy data behaviour](../agents/forking#legacy-data-behaviour) for the
fork-eligibility rules on mixed threads.
+## Schema v4 -> v5 migration
+
+Durable HITL answer replay bumps `EXPECTED_SCHEMA_VERSION` from 4 to 5.
+The upgrade creates `cubepi_hitl_answers`, which stores answered HITL
+requests by `(thread_id, run_id, question_id)` until the suspended tool
+cycle completes.
+
+```python
+# In a migration's upgrade():
+from cubepi.checkpointer.mysql.alembic_helpers import (
+ upgrade_v4_to_v5_op,
+ write_schema_version_op,
+)
+
+def upgrade():
+ for stmt in upgrade_v4_to_v5_op().split(";"):
+ if stmt.strip():
+ op.execute(stmt)
+ for stmt in write_schema_version_op().split(";"):
+ if stmt.strip():
+ op.execute(stmt)
+```
+
## Common pitfalls
- **`CubepiSchemaUninitialized`** — Your DB is empty, your migrations
diff --git a/website/docs/guides/checkpointing/postgres.md b/website/docs/guides/checkpointing/postgres.md
index 8d04d40..cea4061 100644
--- a/website/docs/guides/checkpointing/postgres.md
+++ b/website/docs/guides/checkpointing/postgres.md
@@ -54,8 +54,9 @@ async with PostgresCheckpointer(
## Schema
-The checkpointer expects three tables: `cubepi_threads`,
-`cubepi_messages`, and `cubepi_schema_version`. Unlike SQLite, CubePi
+The checkpointer expects `cubepi_threads`, `cubepi_messages`,
+`cubepi_runs`, `cubepi_hitl_answers`, and `cubepi_schema_version`.
+Unlike SQLite, CubePi
**does not create these for you** — it verifies on `__aenter__` that
they exist with the expected `schema_version`.
@@ -130,6 +131,16 @@ cubepi_messages
payload -- bytea (msgpack)
created_at
+cubepi_runs
+ thread_id, run_id -- composite PK
+ claimed_at / completed_at
+ completion_seq
+
+cubepi_hitl_answers
+ thread_id, run_id, question_id -- composite PK
+ answer -- JSONB
+ answered_at
+
cubepi_schema_version
version (PK)
```
@@ -213,6 +224,25 @@ Pre-feature messages keep `run_id = NULL` and remain readable; see
[Legacy data behaviour](../agents/forking#legacy-data-behaviour) for the
fork-eligibility rules on mixed threads.
+## Schema v4 -> v5 migration
+
+Durable HITL answer replay bumps `EXPECTED_SCHEMA_VERSION` from 4 to 5.
+The upgrade creates `cubepi_hitl_answers`, which stores answered HITL
+requests by `(thread_id, run_id, question_id)` until the suspended tool
+cycle completes.
+
+```python
+# In a migration's upgrade():
+from cubepi.checkpointer.postgres.alembic_helpers import (
+ upgrade_v4_to_v5_op,
+ write_schema_version_op,
+)
+
+def upgrade():
+ op.execute(upgrade_v4_to_v5_op())
+ op.execute(write_schema_version_op()) # bumps cubepi_schema_version to 5
+```
+
## Common pitfalls
- **`CubepiSchemaUninitialized`** — Your DB is empty or your
diff --git a/website/docs/guides/checkpointing/sqlite.md b/website/docs/guides/checkpointing/sqlite.md
index ddf4a44..9271b73 100644
--- a/website/docs/guides/checkpointing/sqlite.md
+++ b/website/docs/guides/checkpointing/sqlite.md
@@ -82,19 +82,34 @@ CREATE TABLE thread_extra (
This schema is append-only. CubePi never updates or deletes rows.
-## HITL pending table
+## HITL tables
-When the [HITL](../hitl/overview) module is in use, an additional table is
+When the [HITL](../hitl/overview) module is in use, additional tables are
created automatically on `__aenter__`:
```sql
CREATE TABLE IF NOT EXISTS thread_pending_request (
thread_id TEXT PRIMARY KEY,
request_json TEXT NOT NULL,
+ run_id TEXT,
created_at REAL NOT NULL DEFAULT (julianday('now'))
);
+
+CREATE TABLE IF NOT EXISTS thread_hitl_answers (
+ thread_id TEXT NOT NULL,
+ run_id TEXT NOT NULL,
+ question_id TEXT NOT NULL,
+ answer_json TEXT NOT NULL,
+ answered_at REAL NOT NULL DEFAULT (julianday('now')),
+ PRIMARY KEY (thread_id, run_id, question_id)
+);
```
+`thread_pending_request` stores the single visible HITL request for a
+thread. `thread_hitl_answers` stores already-answered prompts so a
+parallel approval batch can collect approvals one by one and then
+execute the batch only after every gate is satisfied.
+
No manual migration is needed — `CREATE TABLE IF NOT EXISTS` is
idempotent.
diff --git a/website/docs/guides/hitl/reference.md b/website/docs/guides/hitl/reference.md
index 345a0fb..9a1fd2b 100644
--- a/website/docs/guides/hitl/reference.md
+++ b/website/docs/guides/hitl/reference.md
@@ -120,6 +120,11 @@ assert await ch.ask([Question(key="k", prompt="p")]) == {"k": ""}
- **Single pending per thread.** The agent loop is sequential — at most one
HITL request is outstanding per `thread_id`. Concurrent `confirm/approve/ask`
raises `HitlConcurrencyError`.
+- **Parallel approval batches collect answers before execution.** If one
+ assistant turn contains multiple parallel tool calls that require approval,
+ CubePi still exposes one pending request at a time. Each approved answer is
+ persisted by `question_id` and replayed on the next resume attempt. Tool
+ bodies start only after every gate in the parallel batch has been answered.
- **Prompt-cache prefix invariant.** Between pause and resume, the messages
list changes only by appending tool-result messages and the next assistant
turn at the tail. No inserting, reordering, or mutating prior messages —
@@ -127,6 +132,7 @@ assert await ch.ask([Question(key="k", prompt="p")]) == {"k": ""}
- **`question_id == tool_call_id` for approve requests.** No aliasing or
mapping needed — hosts that already track `call_id` from the tool stream
pass it directly.
-- **Resume does not replay.** It re-enters the loop with the answer pre-loaded
- into the channel. The last assistant message's unresolved tool calls dictate
- what executes next. No node-based replay semantics.
+- **Resume re-enters the unresolved tool cycle.** The last assistant message's
+ unresolved tool calls dictate what prepares and executes next. Persisted HITL
+ answers can be replayed by `question_id`, but CubePi does not use node-based
+ graph replay semantics.