CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.
Python agent-frameworkagent-harnessagent-runtimeagentic-aiai-agentsai-frameworkanthropicharnessharness-engineeringharness-frameworklanggraphlanggraph-alternativellmmcpopenaipi-agentpi-agent-corepythonpython-agent
Fix parallel HITL approval replay
xf gong6fb793ecb9cd
diff --git a/cubepi/agent/agent.py b/cubepi/agent/agent.pyindex 65c6f5b..f6d24f3 100644--- a/cubepi/agent/agent.py+++ b/cubepi/agent/agent.py@@ -847,6 +847,11 @@ class Agent(Generic[TMessage]):"respond() requires a checkpointer that implements ""load_pending (added in checkpointer v4)")+ save_hitl_answer = getattr(self.checkpointer, "save_hitl_answer", None)+ if save_hitl_answer is None:+ raise HitlError(+ "respond() requires a checkpointer that implements save_hitl_answer"+ )async with self._run_lock:if not self._state._messages:@@ -882,6 +887,12 @@ class Agent(Generic[TMessage]):self._state.active_run_id = recovered_run_idself._state.last_outcome = None+ await save_hitl_answer(+ self.thread_id,+ question_id,+ answer,+ run_id=recovered_run_id,+ )self._channel.attach_resume_answer(question_id, answer)try:await self._run_hitl_resume()@@ -980,6 +991,11 @@ class Agent(Generic[TMessage]):# No unresolved assistant turn — conversation already closed# by some other path. Still clear pending + emit for observability.await save_pending(self.thread_id, None)+ clear_answers = getattr(self.checkpointer, "clear_hitl_answers", None)+ if clear_answers is not None:+ await clear_answers(+ self.thread_id, run_id=self._state.active_run_id+ )await self._process_event(AgentAbortedEvent(reason=reason))return@@ -1033,6 +1049,9 @@ class Agent(Generic[TMessage]):# Defensive clear (Phase 1's _on_pending_cleared usually did this,# but cross-process abort_pending may bypass Phase 1 entirely).await save_pending(self.thread_id, None)+ clear_answers = getattr(self.checkpointer, "clear_hitl_answers", None)+ if clear_answers is not None:+ await clear_answers(self.thread_id, run_id=owning_run_id)await self._process_event(AgentAbortedEvent(reason=reason))async def _run_hitl_resume(self) -> None:diff --git a/cubepi/agent/loop.py b/cubepi/agent/loop.pyindex b9f03b4..ff70f9d 100644--- a/cubepi/agent/loop.py+++ b/cubepi/agent/loop.py@@ -335,6 +335,9 @@ async def _run_agent_loop_resume_body( # pragma: no cover — E2E testedsave_pending = getattr(checkpointer, "save_pending_request", None)if save_pending is not None:await save_pending(thread_id, None)+ clear_answers = getattr(checkpointer, "clear_hitl_answers", None)+ if clear_answers is not None:+ await clear_answers(thread_id, run_id=last.run_id)# Emit TurnEndEvent with the ACTUAL tool_results so listeners get the# right payload (codex BLOCKING: previous draft emitted []).diff --git a/cubepi/checkpointer/base.py b/cubepi/checkpointer/base.pyindex db78a30..1c8aca5 100644--- a/cubepi/checkpointer/base.py+++ b/cubepi/checkpointer/base.py@@ -1,11 +1,12 @@from __future__ import annotations+from collections.abc import Iterablefrom dataclasses import dataclass, fieldfrom typing import Protocol, runtime_checkablefrom cubepi.hitl.types import HitlRequestfrom cubepi.providers.base import Message-from cubepi.types import JsonObject+from cubepi.types import JsonObject, StructuredValue@dataclass@@ -90,3 +91,34 @@ class Checkpointer(Protocol):"""Read (HitlRequest, run_id) atomically from the pending row,or None when no pending request exists."""...++ async def save_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ answer: StructuredValue,+ *,+ run_id: str | None = None,+ ) -> None:+ """Persist an answered HITL request for replay during resume."""+ ...++ async def load_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ *,+ run_id: str | None = None,+ ) -> StructuredValue | None:+ """Load a persisted HITL answer for replay, or None."""+ ...++ async def clear_hitl_answers(+ self,+ thread_id: str,+ question_ids: Iterable[str] | None = None,+ *,+ run_id: str | None = None,+ ) -> None:+ """Clear persisted HITL answers for a run or selected question ids."""+ ...diff --git a/cubepi/checkpointer/memory.py b/cubepi/checkpointer/memory.pyindex 61d6569..d6ae3f5 100644--- a/cubepi/checkpointer/memory.py+++ b/cubepi/checkpointer/memory.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport asyncioimport copyimport time+from collections.abc import Iterablefrom dataclasses import dataclassfrom cubepi.checkpointer.base import CheckpointData@@ -16,7 +17,7 @@ from cubepi.checkpointer.exceptions import ()from cubepi.hitl.types import HitlRequestfrom cubepi.providers.base import Message-from cubepi.types import JsonObject+from cubepi.types import JsonObject, StructuredValue@dataclass@@ -30,11 +31,16 @@ def _legible_message_copy(m: Message) -> Message:return m.model_copy(deep=True)+def _run_key(run_id: str | None) -> str:+ return run_id or ""++class MemoryCheckpointer:def __init__(self) -> None:self._store: dict[str, CheckpointData] = {}self._pending: dict[str, HitlRequest] = {}self._pending_run_id: dict[str, str | None] = {}+ self._hitl_answers: dict[tuple[str, str, str], StructuredValue] = {}self._runs: dict[str, dict[str, _RunState]] = {}self._lock = asyncio.Lock()@@ -90,6 +96,53 @@ class MemoryCheckpointer:return Nonereturn req, self._pending_run_id.get(thread_id)+ async def save_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ answer: StructuredValue,+ *,+ run_id: str | None = None,+ ) -> None:+ key = (thread_id, _run_key(run_id), question_id)+ async with self._lock:+ self._hitl_answers[key] = copy.deepcopy(answer)++ async def load_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ *,+ run_id: str | None = None,+ ) -> StructuredValue | None:+ key = (thread_id, _run_key(run_id), question_id)+ async with self._lock:+ answer = self._hitl_answers.get(key)+ return copy.deepcopy(answer)++ async def clear_hitl_answers(+ self,+ thread_id: str,+ question_ids: Iterable[str] | None = None,+ *,+ run_id: str | None = None,+ ) -> None:+ run_key = _run_key(run_id)+ async with self._lock:+ if question_ids is None:+ keys = [+ key+ for key in self._hitl_answers+ if key[0] == thread_id and key[1] == run_key+ ]+ else:+ keys = [+ (thread_id, run_key, question_id)+ for question_id in set(question_ids)+ ]+ for key in keys:+ self._hitl_answers.pop(key, None)+async def claim_run(self, thread_id: str, run_id: str) -> None:async with self._lock:runs = self._runs.setdefault(thread_id, {})diff --git a/cubepi/checkpointer/mysql/alembic_helpers.py b/cubepi/checkpointer/mysql/alembic_helpers.pyindex 15c241d..4c5efb4 100644--- a/cubepi/checkpointer/mysql/alembic_helpers.py+++ b/cubepi/checkpointer/mysql/alembic_helpers.py@@ -129,6 +129,30 @@ def upgrade_v3_to_v4_op() -> str:return f"{add_messages_run_id_column_op()} {create_runs_table_op()};"+def create_hitl_answers_table_op() -> str:+ """Return SQL creating the durable HITL answer ledger table."""+ return (+ "CREATE TABLE cubepi_hitl_answers ("+ " thread_id VARCHAR(255) COLLATE utf8mb4_bin NOT NULL,"+ " run_id VARCHAR(255) NOT NULL,"+ " question_id VARCHAR(255) NOT NULL,"+ " answer JSON NOT NULL,"+ " answered_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,"+ " PRIMARY KEY (thread_id, run_id, question_id)"+ ") ENGINE=InnoDB"+ )+++def upgrade_v4_to_v5_op() -> str:+ """Return SQL applying the v4->v5 schema changes.++ Creates the durable HITL answer ledger table. Hosts must also bump+ ``cubepi_schema_version`` via ``write_schema_version_op()``+ (EXPECTED_SCHEMA_VERSION is now 5).+ """+ return create_hitl_answers_table_op() + ";"++def write_schema_version_op() -> str:"""Return SQL setting cubepi_schema_version to the current version.diff --git a/cubepi/checkpointer/mysql/checkpointer.py b/cubepi/checkpointer/mysql/checkpointer.pyindex fba605c..5c3454b 100644--- a/cubepi/checkpointer/mysql/checkpointer.py+++ b/cubepi/checkpointer/mysql/checkpointer.py@@ -9,12 +9,14 @@ list of deliberate MySQL divergences.from __future__ import annotationsimport json+from collections.abc import Iterablefrom typing import Anyfrom urllib.parse import unquote, urlparseimport aiomysqlimport msgpackimport pymysql+from pydantic import TypeAdapterfrom cubepi.checkpointer.base import CheckpointDatafrom cubepi.checkpointer.exceptions import (@@ -37,7 +39,9 @@ from cubepi.providers.base import (ToolResultMessage,UserMessage,)-from cubepi.types import JsonObject+from cubepi.types import JsonObject, StructuredValue++_STRUCTURED_VALUE_ADAPTER: TypeAdapter[Any] = TypeAdapter(StructuredValue)_ER_NO_SUCH_TABLE = 1146_ER_BAD_FIELD_ERROR = 1054@@ -82,6 +86,20 @@ def _decode_json(value: Any) -> dict[str, Any]:return value+def _run_key(run_id: str | None) -> str:+ return run_id or ""+++def _serialize_structured_value(value: StructuredValue) -> str:+ return _STRUCTURED_VALUE_ADAPTER.dump_json(value).decode("utf-8")+++def _decode_json_value(value: Any) -> StructuredValue:+ if isinstance(value, str): # pragma: no cover - codec-dependent+ return json.loads(value)+ return value++class MySQLCheckpointer:"""Checkpointer backed by MySQL (8.0.13+, InnoDB).@@ -693,3 +711,90 @@ class MySQLCheckpointer:)row = await cur.fetchone()return row[0] if row is not None else None++ async def save_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ answer: StructuredValue,+ *,+ run_id: str | None = None,+ ) -> None:+ assert self._pool is not None+ payload = _serialize_structured_value(answer)+ async with self._pool.acquire() as conn:+ await conn.begin()+ try:+ async with conn.cursor() as cur:+ await cur.execute(+ "INSERT INTO cubepi_threads (thread_id) VALUES (%s) "+ "ON DUPLICATE KEY UPDATE thread_id = thread_id",+ (thread_id,),+ )+ await cur.execute(+ "INSERT INTO cubepi_hitl_answers "+ "(thread_id, run_id, question_id, answer) "+ "VALUES (%s, %s, %s, %s) "+ "ON DUPLICATE KEY UPDATE "+ "answer = VALUES(answer), "+ "answered_at = CURRENT_TIMESTAMP",+ (thread_id, _run_key(run_id), question_id, payload),+ )+ await conn.commit()+ except BaseException: # pragma: no cover - defensive txn rollback+ await conn.rollback()+ raise++ async def load_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ *,+ run_id: str | None = None,+ ) -> StructuredValue | None:+ assert self._pool is not None+ async with self._pool.acquire() as conn:+ async with conn.cursor() as cur:+ await cur.execute(+ "SELECT answer FROM cubepi_hitl_answers "+ "WHERE thread_id = %s AND run_id = %s AND question_id = %s",+ (thread_id, _run_key(run_id), question_id),+ )+ row = await cur.fetchone()+ if row is None:+ return None+ return _decode_json_value(row[0])++ async def clear_hitl_answers(+ self,+ thread_id: str,+ question_ids: Iterable[str] | None = None,+ *,+ run_id: str | None = None,+ ) -> None:+ assert self._pool is not None+ run_key = _run_key(run_id)+ async with self._pool.acquire() as conn:+ await conn.begin()+ try:+ async with conn.cursor() as cur:+ if question_ids is None:+ await cur.execute(+ "DELETE FROM cubepi_hitl_answers "+ "WHERE thread_id = %s AND run_id = %s",+ (thread_id, run_key),+ )+ else:+ qids = list(dict.fromkeys(question_ids))+ if qids:+ placeholders = ",".join("%s" for _ in qids)+ await cur.execute(+ "DELETE FROM cubepi_hitl_answers "+ "WHERE thread_id = %s AND run_id = %s "+ f"AND question_id IN ({placeholders})",+ (thread_id, run_key, *qids),+ )+ await conn.commit()+ except BaseException: # pragma: no cover - defensive txn rollback+ await conn.rollback()+ raisediff --git a/cubepi/checkpointer/mysql/models.py b/cubepi/checkpointer/mysql/models.pyindex f666d53..1988b01 100644--- a/cubepi/checkpointer/mysql/models.py+++ b/cubepi/checkpointer/mysql/models.py@@ -16,7 +16,7 @@ import sqlalchemy as safrom sqlalchemy.dialects.mysql import JSON, LONGBLOB, VARCHARfrom sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column-EXPECTED_SCHEMA_VERSION = 4+EXPECTED_SCHEMA_VERSION = 5PARTITION_COUNT = 64cubepi_metadata = sa.MetaData()@@ -135,6 +135,21 @@ class CubepiRun(CubepiBase):completion_seq: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True)+class CubepiHitlAnswer(CubepiBase):+ __tablename__ = "cubepi_hitl_answers"+ __table_args__ = {"mysql_engine": "InnoDB"}++ thread_id: Mapped[str] = mapped_column(_TID, primary_key=True)+ run_id: Mapped[str] = mapped_column(VARCHAR(255), primary_key=True)+ question_id: Mapped[str] = mapped_column(VARCHAR(255), primary_key=True)+ answer: Mapped[Any] = mapped_column(JSON, nullable=False)+ answered_at: Mapped[_dt.datetime] = mapped_column(+ sa.TIMESTAMP,+ nullable=False,+ server_default=sa.text("CURRENT_TIMESTAMP"),+ )++class CubepiSchemaVersion(CubepiBase):__tablename__ = "cubepi_schema_version"__table_args__ = {"mysql_engine": "InnoDB"}diff --git a/cubepi/checkpointer/postgres/alembic_helpers.py b/cubepi/checkpointer/postgres/alembic_helpers.pyindex 2d1234c..1a013f9 100644--- a/cubepi/checkpointer/postgres/alembic_helpers.py+++ b/cubepi/checkpointer/postgres/alembic_helpers.py@@ -93,6 +93,26 @@ def upgrade_v3_to_v4_op() -> str:return "\n".join(parts)+def upgrade_v4_to_v5_op() -> str:+ """Return SQL applying the v4->v5 schema changes.++ Creates the durable HITL answer ledger table. Hosts must also bump+ `cubepi_schema_version` via write_schema_version_op()+ (EXPECTED_SCHEMA_VERSION is now 5).+ """+ return (+ "CREATE TABLE IF NOT EXISTS cubepi_hitl_answers ("+ " thread_id TEXT NOT NULL REFERENCES cubepi_threads(thread_id) "+ "ON DELETE CASCADE,"+ " run_id TEXT NOT NULL,"+ " question_id TEXT NOT NULL,"+ " answer JSONB NOT NULL,"+ " answered_at TIMESTAMPTZ NOT NULL DEFAULT now(),"+ " PRIMARY KEY (thread_id, run_id, question_id)"+ ");"+ )++def write_schema_version_op() -> str:"""Return SQL setting cubepi_schema_version to the current version.diff --git a/cubepi/checkpointer/postgres/checkpointer.py b/cubepi/checkpointer/postgres/checkpointer.pyindex 3065c98..2bdbf21 100644--- a/cubepi/checkpointer/postgres/checkpointer.py+++ b/cubepi/checkpointer/postgres/checkpointer.py@@ -7,10 +7,12 @@ msgpack payload encoding. Schema version verified on context entry.from __future__ import annotationsimport json+from collections.abc import Iterablefrom typing import Anyimport asyncpgimport msgpack+from pydantic import TypeAdapterfrom cubepi.checkpointer.base import CheckpointDatafrom cubepi.checkpointer.exceptions import (@@ -33,7 +35,23 @@ from cubepi.providers.base import (ToolResultMessage,UserMessage,)-from cubepi.types import JsonObject+from cubepi.types import JsonObject, StructuredValue++_STRUCTURED_VALUE_ADAPTER: TypeAdapter[Any] = TypeAdapter(StructuredValue)+++def _run_key(run_id: str | None) -> str:+ return run_id or ""+++def _serialize_structured_value(value: StructuredValue) -> str:+ return _STRUCTURED_VALUE_ADAPTER.dump_json(value).decode("utf-8")+++def _deserialize_structured_value(value: Any) -> StructuredValue:+ if isinstance(value, str): # pragma: no cover - codec-dependent+ return json.loads(value)+ return valuedef _role_of(msg: Message) -> str:@@ -561,3 +579,82 @@ class PostgresCheckpointer:thread_id,)return row["run_id"] if row is not None else None++ async def save_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ answer: StructuredValue,+ *,+ run_id: str | None = None,+ ) -> None:+ assert self._pool is not None+ payload = _serialize_structured_value(answer)+ async with self._pool.acquire() as conn:+ async with conn.transaction():+ await conn.execute(+ "INSERT INTO cubepi_threads (thread_id) "+ "VALUES ($1) ON CONFLICT DO NOTHING",+ thread_id,+ )+ await conn.execute(+ "INSERT INTO cubepi_hitl_answers "+ "(thread_id, run_id, question_id, answer) "+ "VALUES ($1, $2, $3, $4::jsonb) "+ "ON CONFLICT (thread_id, run_id, question_id) DO UPDATE "+ "SET answer = EXCLUDED.answer, answered_at = now()",+ thread_id,+ _run_key(run_id),+ question_id,+ payload,+ )++ async def load_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ *,+ run_id: str | None = None,+ ) -> StructuredValue | None:+ assert self._pool is not None+ async with self._pool.acquire() as conn:+ row = await conn.fetchrow(+ "SELECT answer FROM cubepi_hitl_answers "+ "WHERE thread_id = $1 AND run_id = $2 AND question_id = $3",+ thread_id,+ _run_key(run_id),+ question_id,+ )+ if row is None:+ return None+ return _deserialize_structured_value(row["answer"])++ async def clear_hitl_answers(+ self,+ thread_id: str,+ question_ids: Iterable[str] | None = None,+ *,+ run_id: str | None = None,+ ) -> None:+ assert self._pool is not None+ run_key = _run_key(run_id)+ async with self._pool.acquire() as conn:+ if question_ids is None:+ await conn.execute(+ "DELETE FROM cubepi_hitl_answers "+ "WHERE thread_id = $1 AND run_id = $2",+ thread_id,+ run_key,+ )+ return+ qids = list(dict.fromkeys(question_ids))+ if not qids:+ return+ await conn.execute(+ "DELETE FROM cubepi_hitl_answers "+ "WHERE thread_id = $1 AND run_id = $2 "+ "AND question_id = ANY($3::text[])",+ thread_id,+ run_key,+ qids,+ )diff --git a/cubepi/checkpointer/postgres/models.py b/cubepi/checkpointer/postgres/models.pyindex 4d45fac..ed8a516 100644--- a/cubepi/checkpointer/postgres/models.py+++ b/cubepi/checkpointer/postgres/models.py@@ -15,7 +15,7 @@ import sqlalchemy as safrom sqlalchemy.dialects.postgresql import JSONBfrom sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column-EXPECTED_SCHEMA_VERSION = 4+EXPECTED_SCHEMA_VERSION = 5PARTITION_COUNT = 64cubepi_metadata = sa.MetaData()@@ -129,6 +129,24 @@ class CubepiRun(CubepiBase):completion_seq: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True)+class CubepiHitlAnswer(CubepiBase):+ __tablename__ = "cubepi_hitl_answers"++ thread_id: Mapped[str] = mapped_column(+ sa.Text,+ sa.ForeignKey("cubepi_threads.thread_id", ondelete="CASCADE"),+ primary_key=True,+ )+ run_id: Mapped[str] = mapped_column(sa.Text, primary_key=True)+ question_id: Mapped[str] = mapped_column(sa.Text, primary_key=True)+ answer: Mapped[Any] = mapped_column(JSONB, nullable=False)+ answered_at: Mapped[_dt.datetime] = mapped_column(+ sa.TIMESTAMP(timezone=True),+ nullable=False,+ server_default=sa.text("now()"),+ )++class CubepiSchemaVersion(CubepiBase):__tablename__ = "cubepi_schema_version"diff --git a/cubepi/checkpointer/sqlite.py b/cubepi/checkpointer/sqlite.pyindex 0f59952..a5d7da8 100644--- a/cubepi/checkpointer/sqlite.py+++ b/cubepi/checkpointer/sqlite.py@@ -2,10 +2,12 @@ from __future__ import annotationsimport asyncioimport json+from collections.abc import Iterablefrom contextlib import asynccontextmanagerfrom typing import Any, AsyncIterator, castimport aiosqlite+from pydantic import TypeAdapterfrom cubepi.checkpointer.base import CheckpointDatafrom cubepi.checkpointer.exceptions import (@@ -24,7 +26,21 @@ from cubepi.providers.base import (ToolResultMessage,UserMessage,)-from cubepi.types import JsonObject+from cubepi.types import JsonObject, StructuredValue++_STRUCTURED_VALUE_ADAPTER: TypeAdapter[Any] = TypeAdapter(StructuredValue)+++def _run_key(run_id: str | None) -> str:+ return run_id or ""+++def _serialize_structured_value(value: StructuredValue) -> str:+ return _STRUCTURED_VALUE_ADAPTER.dump_json(value).decode("utf-8")+++def _deserialize_structured_value(value: str) -> StructuredValue:+ return cast(StructuredValue, json.loads(value))@asynccontextmanager@@ -79,6 +95,16 @@ class SQLiteCheckpointer:" created_at REAL NOT NULL DEFAULT (julianday('now'))"")")+ await self._db.execute(+ "CREATE TABLE IF NOT EXISTS thread_hitl_answers ("+ " thread_id TEXT NOT NULL,"+ " run_id TEXT NOT NULL,"+ " question_id TEXT NOT NULL,"+ " answer_json TEXT NOT NULL,"+ " answered_at REAL NOT NULL DEFAULT (julianday('now')),"+ " PRIMARY KEY (thread_id, run_id, question_id)"+ ")"+ )await self._db.execute("CREATE TABLE IF NOT EXISTS runs ("" thread_id TEXT NOT NULL,"@@ -310,6 +336,69 @@ class SQLiteCheckpointer:row = await cursor.fetchone()return row[0] if row else None+ async def save_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ answer: StructuredValue,+ *,+ run_id: str | None = None,+ ) -> None:+ assert self._db is not None+ payload = _serialize_structured_value(answer)+ async with self._lock, _writer_txn(self._db):+ await self._db.execute(+ "INSERT OR REPLACE INTO thread_hitl_answers "+ "(thread_id, run_id, question_id, answer_json) "+ "VALUES (?, ?, ?, ?)",+ (thread_id, _run_key(run_id), question_id, payload),+ )++ async def load_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ *,+ run_id: str | None = None,+ ) -> StructuredValue | None:+ assert self._db is not None+ async with self._lock:+ cursor = await self._db.execute(+ "SELECT answer_json FROM thread_hitl_answers "+ "WHERE thread_id = ? AND run_id = ? AND question_id = ?",+ (thread_id, _run_key(run_id), question_id),+ )+ row = await cursor.fetchone()+ return _deserialize_structured_value(row[0]) if row else None++ async def clear_hitl_answers(+ self,+ thread_id: str,+ question_ids: Iterable[str] | None = None,+ *,+ run_id: str | None = None,+ ) -> None:+ assert self._db is not None+ run_key = _run_key(run_id)+ async with self._lock, _writer_txn(self._db):+ if question_ids is None:+ await self._db.execute(+ "DELETE FROM thread_hitl_answers "+ "WHERE thread_id = ? AND run_id = ?",+ (thread_id, run_key),+ )+ return+ qids = list(dict.fromkeys(question_ids))+ if not qids:+ return+ placeholders = ",".join("?" for _ in qids)+ await self._db.execute(+ "DELETE FROM thread_hitl_answers "+ f"WHERE thread_id = ? AND run_id = ? "+ f"AND question_id IN ({placeholders})",+ (thread_id, run_key, *qids),+ )+async def snapshot(self, thread_id: str, *, after_run_id: str) -> list[Message]:assert self._db is not Noneasync with self._lock:diff --git a/cubepi/hitl/_trace.py b/cubepi/hitl/_trace.pyindex db43213..74da111 100644--- a/cubepi/hitl/_trace.py+++ b/cubepi/hitl/_trace.py@@ -1,6 +1,7 @@from __future__ import annotationsimport contextlib+from typing import Any, castclass _NullSpan: # pragma: no cover - only exercised when opentelemetry is not installed@@ -20,10 +21,11 @@ class _NullSpan: # pragma: no cover - only exercised when opentelemetry is not@contextlib.contextmanagerdef hitl_span(kind: str, **attrs):try:- from opentelemetry import trace+ from opentelemetry import trace as otel_traceexcept ImportError: # pragma: no cover - tracing extra not installedyield _NullSpan()return+ trace = cast(Any, otel_trace)tracer = trace.get_tracer("cubepi.hitl")with tracer.start_as_current_span(f"hitl.{kind}") as span:for k, v in attrs.items():diff --git a/cubepi/hitl/channel.py b/cubepi/hitl/channel.pyindex 5284b49..962d0de 100644--- a/cubepi/hitl/channel.py+++ b/cubepi/hitl/channel.py@@ -151,6 +151,12 @@ class _BaseChannel:def _bind_emit(self, emit) -> None:self._emit = emit+ async def _load_answer(self, question_id: str) -> StructuredValue | None:+ return None++ async def _save_answer(self, question_id: str, answer: StructuredValue) -> None:+ return None+def _next_qid(self, kind: str, payload_repr: StructuredValue) -> str:"""Derive the next question_id for an ask/confirm payload.@@ -235,6 +241,19 @@ class _BaseChannel:)return ans+ persisted_answer = await self._load_answer(question_id)+ if persisted_answer is not None:+ ans = _normalize_answer(payload, persisted_answer)+ from_resume = True+ outcome = _outcome_from_answer(kind, ans)+ if self._emit is not None:+ from cubepi.agent.types import HitlAnswerEvent++ await self._emit_event(+ HitlAnswerEvent(question_id=question_id, answer=ans)+ )+ return ans+if self._pending is not None:raise HitlConcurrencyError(f"channel busy: already pending {self._pending.question_id}"@@ -362,6 +381,7 @@ class _BaseChannel:f"answer for {question_id}; pending is "f"{self._pending.question_id if self._pending else 'None'}")+ await self._save_answer(question_id, answer)if self._future is not None and not self._future.done():self._future.set_result(answer)if self._emit is not None: # pragma: no cover — integration tested@@ -493,6 +513,12 @@ def _outcome_from_answer(kind: str, ans: StructuredValue) -> str:return "answered"+def _normalize_answer(payload: HitlPayload, ans: StructuredValue) -> StructuredValue:+ if isinstance(payload, ApproveRequest) and not isinstance(ans, ApproveAnswer):+ return ApproveAnswer.model_validate(ans)+ return ans++def _outcome_from_exception(exc: BaseException) -> str:from cubepi.hitl.exceptions import (HitlAborted,@@ -515,6 +541,21 @@ def _outcome_from_exception(exc: BaseException) -> str:class InMemoryChannel(_BaseChannel):"""In-process HITL channel; no persistence."""+ def __init__(+ self,+ *,+ default_timeout: float | None = None,+ thread_id: str | None = None,+ ) -> None:+ super().__init__(default_timeout=default_timeout, thread_id=thread_id)+ self._answer_ledger: dict[str, StructuredValue] = {}++ async def _load_answer(self, question_id: str) -> StructuredValue | None:+ return self._answer_ledger.get(question_id)++ async def _save_answer(self, question_id: str, answer: StructuredValue) -> None:+ self._answer_ledger[question_id] = answer+class CheckpointedChannel(_BaseChannel):"""Cross-process HITL channel — persists the pending request via a@@ -540,12 +581,17 @@ class CheckpointedChannel(_BaseChannel):if not (hasattr(checkpointer, "save_pending_request")and hasattr(checkpointer, "load_pending_request")+ and hasattr(checkpointer, "save_hitl_answer")+ and hasattr(checkpointer, "load_hitl_answer")+ and hasattr(checkpointer, "clear_hitl_answers")):from cubepi.hitl.exceptions import HitlErrorraise HitlError("CheckpointedChannel requires a checkpointer with "- "save_pending_request and load_pending_request methods. "+ "save_pending_request, load_pending_request, "+ "save_hitl_answer, load_hitl_answer, and "+ "clear_hitl_answers methods. ""First-party checkpointers (Memory/SQLite/Postgres/MySQL) ""implement these; third-party Protocol-only impls may not.")@@ -559,6 +605,18 @@ class CheckpointedChannel(_BaseChannel):self._run_id = run_idself._allow_inside_custom_tool = allow_inside_custom_tool+ async def _load_answer(self, question_id: str) -> StructuredValue | None:+ assert self._thread_id is not None+ return await self._checkpointer.load_hitl_answer(+ self._thread_id, question_id, run_id=self._run_id+ )++ async def _save_answer(self, question_id: str, answer: StructuredValue) -> None:+ assert self._thread_id is not None+ await self._checkpointer.save_hitl_answer(+ self._thread_id, question_id, answer, run_id=self._run_id+ )+async def _on_pending_set(self, req: HitlRequest) -> None:if _in_custom_tool_var.get() and not self._allow_inside_custom_tool:from cubepi.hitl.exceptions import HitlDurabilityNotGuaranteeddiff --git a/cubepi/mcp/_tracing.py b/cubepi/mcp/_tracing.pyindex 2ff1b02..57f75cd 100644--- a/cubepi/mcp/_tracing.py+++ b/cubepi/mcp/_tracing.py@@ -36,6 +36,11 @@ from typing import TYPE_CHECKING, Any, AsyncIteratorif TYPE_CHECKING:pass+_otel_trace: Any+SpanKind: Any+Status: Any+StatusCode: Any+try:from opentelemetry import trace as _otel_trace@@ -44,10 +49,10 @@ try:_OTEL_AVAILABLE = Trueexcept ImportError: # pragma: no cover — exercised only without the extra._OTEL_AVAILABLE = False- _otel_trace = None # type: ignore[assignment]- SpanKind = None # type: ignore[assignment,misc]- Status = None # type: ignore[assignment,misc]- StatusCode = None # type: ignore[assignment,misc]+ _otel_trace = None+ SpanKind = None+ Status = None+ StatusCode = None# When :class:`cubepi.tracing.Tracer` is attached to an agent itdiff --git a/dev/plans/2026-06-27-hitl-answer-ledger.md b/dev/plans/2026-06-27-hitl-answer-ledger.mdnew file mode 100644index 0000000..7964417--- /dev/null+++ b/dev/plans/2026-06-27-hitl-answer-ledger.md@@ -0,0 +1,258 @@+# HITL Answer Ledger Implementation Plan++- Date: 2026-06-27+- Spec: `dev/specs/2026-06-27-hitl-answer-ledger.md`+- Status: Draft for review++## Objective++Fix the ROOT-A deadlock where parallel two-phase tool preparation and the+single-slot HITL resume channel can ping-pong between two approval-gated tool+calls forever. The fix is a framework-level durable answer ledger keyed by+thread, run, and question id.++## Constraints++- Do not force approval-gated tools into `execution_mode="sequential"`.+- Do not weaken the two-phase parallel executor's side-effect safety.+- Do not require hosts to classify HITL-gated tools.+- Keep the public visible HITL model as one pending request at a time.+- Run all repo commands through `uv`.++## Phase 1: Add the Checkpointer Ledger Contract++Files:++- `cubepi/checkpointer/base.py`+- `cubepi/checkpointer/memory.py`+- `tests/checkpointer/test_memory_checkpointer.py` or the nearest existing+ memory checkpointer test module++Steps:++1. Import `Iterable` and `StructuredValue` where needed.+2. Add `save_hitl_answer`, `load_hitl_answer`, and `clear_hitl_answers` to the+ `Checkpointer` protocol.+3. Implement the methods in `MemoryCheckpointer` with a private dict keyed by+ `(thread_id, run_id or "", question_id)`.+4. Copy answers on save/load if the existing message/pending code uses deep+ copies for comparable mutable state.+5. Add focused unit tests:+ - saving then loading returns the answer+ - run id scopes answers+ - clearing a single question id preserves other answers+ - clearing all answers for a run removes only that run++Acceptance:++- Memory checkpointer tests pass.+- The protocol names match the spec exactly.++## Phase 2: Add Durable Storage to SQL Checkpointers++Files:++- `cubepi/checkpointer/sqlite.py`+- `cubepi/checkpointer/postgres/checkpointer.py`+- `cubepi/checkpointer/postgres/models.py`+- `cubepi/checkpointer/mysql/checkpointer.py`+- `cubepi/checkpointer/mysql/models.py`+- existing SQL checkpointer tests under `tests/checkpointer/`++Steps:++1. Add a `cubepi_hitl_answers` table to each first-party SQL backend.+2. Use columns:+ - `thread_id`+ - `run_id`, stored as non-null text with `""` representing `None`+ - `question_id`+ - `answer_json`+ - `answered_at`+3. Make `(thread_id, run_id, question_id)` the primary key.+4. Implement upsert semantics for `save_hitl_answer`.+5. Implement exact-key lookup for `load_hitl_answer`.+6. Implement `clear_hitl_answers` with two modes:+ - `question_ids is None`: delete all answers for `(thread_id, run_id)`+ - `question_ids` supplied: delete only those ids for `(thread_id, run_id)`+7. Add CRUD tests for SQLite. Extend Postgres/MySQL tests where the repo already+ runs backend-specific checkpointer HITL tests.++Acceptance:++- SQLite tests pass locally.+- Backend-specific tests are updated without adding hard dependencies to core.++## Phase 3: Teach HITL Channels to Replay Ledger Answers++Files:++- `cubepi/hitl/channel.py`+- `tests/hitl/test_in_memory_channel.py`+- `tests/hitl/test_channel.py` or nearest existing checkpointed channel tests++Steps:++1. Extend `_BaseChannel` with optional ledger hooks:+ - `_load_answer(question_id)`+ - `_save_answer(question_id, answer)`+ - `_clear_answers(...)` only if channel-level abort cleanup is already+ centralized there+2. Implement default no-op behavior for plain in-process channels, then override+ as needed in `InMemoryChannel` and `CheckpointedChannel`.+3. Add an in-memory answer dict to `InMemoryChannel` so same-process behavior+ follows the same replay semantics.+4. Update `CheckpointedChannel.__init__` validation to require the new ledger+ methods when used for detached HITL.+5. Change `_await_answer()` resolution order:+ - load durable/in-memory ledger answer for `question_id`+ - fall back to matching `_resume_slot`+ - otherwise publish pending and wait/detach+6. When `answer()` resolves the currently pending request in process, save the+ answer before waking the waiter.+7. Do not clear a ledger answer when `_await_answer()` returns it.+8. Add tests proving:+ - replayed ledger answers do not create a new pending request+ - stale process-local resume slots still do not satisfy the wrong question id+ - in-process `answer()` also records a replayable answer++Acceptance:++- Existing HITL channel tests still pass.+- New replay tests fail without the ledger lookup and pass with it.++## Phase 4: Persist Answers in Agent.respond and Clear Them Safely++Files:++- `cubepi/agent/agent.py`+- `cubepi/agent/loop.py`+- `cubepi/hitl/channel.py` if abort cleanup belongs there+- `tests/hitl/test_loop_hitl_passthrough.py`+- new or existing agent HITL resume tests++Steps:++1. In `Agent.respond()`, after validating pending `question_id` and recovered+ run id, call `save_hitl_answer(thread_id, question_id, answer, run_id=...)`+ before `_run_hitl_resume()`.+2. Keep `attach_resume_answer()` as a same-process fast path, but make the+ ledger the durable source of truth.+3. Ensure resume detaching on the next pending request does not clear saved+ answers.+4. Clear ledger answers for `(thread_id, active_run_id)` only after the suspended+ tool cycle has completed and tool results have been appended.+5. Clear ledger answers when pending HITL is aborted/cancelled or when the run+ reaches a terminal cleanup path.+6. Prefer clearing all answers for the run over clearing only tool call ids, so+ `ask_user` question ids are covered.+7. Add tests for cleanup:+ - after successful completion+ - after abort+ - no cleanup after detach on another request++Acceptance:++- A saved answer survives a detach on a later tool call.+- Saved answers are not left behind after successful completion or abort.++## Phase 5: Add the ROOT-A Regression Test++Files:++- `tests/hitl/test_loop_hitl_passthrough.py` or a new+ `tests/hitl/test_parallel_approval_ledger.py`++Test shape:++1. Define two tools that require approval and record executions.+2. Use a provider response that requests both tools in the same assistant turn+ with parallel execution enabled.+3. Start `Agent.prompt()` and observe pending `A`.+4. Call `Agent.respond(question_id=A, answer=approve)`.+5. Assert:+ - no tool body has executed yet+ - no tool results have been appended yet+ - pending is now `B`+6. Call `Agent.respond(question_id=B, answer=approve)`.+7. Assert:+ - both tool bodies executed exactly once+ - two tool results were appended+ - pending is cleared+ - ledger answers for the run are cleared+8. Repeat the same flow with a fresh `Agent` instance between the two approvals+ to prove durable replay works after process restart.++Acceptance:++- The test reproduces the current deadlock before the implementation.+- The test passes after the answer ledger implementation.++## Phase 6: Cover Decision Variants++Files:++- `tests/hitl/test_loop_hitl_passthrough.py`+- approval middleware tests under `tests/hitl/`++Steps:++1. Add a deny/reject variant where `A` is denied, resume detaches on `B`, then+ `B` is approved. Verify the replayed denial emits the expected tool result+ and no denied tool body executes.+2. Add an edit variant if the existing approval middleware supports edited tool+ arguments in tests. Verify the edited decision survives the second detach and+ the tool receives edited args.+3. Add an `ask_user` replay case if the existing `ask_user` tests can express a+ multi-question resume flow without building a large fixture.++Acceptance:++- The ledger stores and replays structured answers, not just boolean approvals.++## Phase 7: Update Documentation++Files:++- `website/docs/` HITL guide or recipe page+- Chinese mirror if this docs tree keeps one for the changed page++Steps:++1. Document the new guarantee: approval-gated tools can remain parallel.+2. Explain that approving one request in a parallel batch records the decision,+ but execution waits until the remaining gates are answered.+3. Remove or avoid guidance that tells users to mark gated tools sequential only+ to avoid HITL deadlocks.+4. Add a short example flow with two parallel approval-gated tool calls.++Acceptance:++- User-facing docs match the runtime behavior.+- The feature is not considered complete until docs ship in the same PR.++## Verification Commands++Run focused tests first:++```bash+uv run pytest tests/hitl/test_in_memory_channel.py -v+uv run pytest tests/hitl/test_loop_hitl_passthrough.py -v+uv run pytest tests/checkpointer/ -k "memory or sqlite or hitl" -v+```++Then run broader checks:++```bash+uv run pytest tests/+uv run ruff check cubepi/ tests/+uv run ruff format --check cubepi/ tests/+uv run mypy cubepi+```++## Review Checkpoints++1. Review the spec and this plan before coding.+2. After code and docs are ready, ask before entering the local codex review+ loop required by `AGENTS.md`.+3. Open a PR from the worktree branch after implementation and documentation.+4. Drive the PR codex review loop until review and CI are clean.diff --git a/dev/specs/2026-06-27-hitl-answer-ledger.md b/dev/specs/2026-06-27-hitl-answer-ledger.mdnew file mode 100644index 0000000..9e49188--- /dev/null+++ b/dev/specs/2026-06-27-hitl-answer-ledger.md@@ -0,0 +1,278 @@+# HITL Answer Ledger Design Spec++- Date: 2026-06-27+- Status: Draft for review+- Related issue: ROOT-A, parallel two-phase tool preparation plus single-slot HITL deadlock+- Companion plan: `dev/plans/2026-06-27-hitl-answer-ledger.md`++## Problem++cubepi's parallel tool executor intentionally uses two phases:++1. Prepare every requested tool call. Preparation includes middleware gates such+ as approval checks.+2. Only after every tool call is prepared, execute the prepared calls in+ parallel and append the resulting tool results.++That split prevents a bad replay failure mode. If tool A executed before tool B+finished preparation, and tool B then detached for human input, tool A could have+performed side effects without a durable `ToolResult`. Resuming the same turn+would then risk running tool A again.++The split is correct for side-effect safety, but it conflicts with the current+HITL resume channel. The channel has one visible pending request and one+single-use resume answer slot. With two parallel approval-gated tool calls,+`A` and `B`, the system can deadlock:++1. User approves `A`.+2. Resume prepares `A`, consumes the answer, then prepares `B`.+3. `B` has no answer, so preparation detaches before any tool body executes.+4. User approves `B`.+5. Resume starts over from the same assistant message. `A` now has no answer,+ so preparation detaches before `B` can be prepared.++The pending request flips between `A` and `B`. No tool results or idempotency+sentinels are written because execution never starts.++Marking approval-gated tools as sequential avoids the deadlock, but it leaks a+framework invariant into application policy. It also disables useful parallelism+for batches that are safe to run together after their gates have all been+answered.++## Goals++- Preserve the parallel executor's side-effect guarantee: no tool body in a+ parallel batch starts until every tool call in that batch has prepared+ successfully.+- Preserve one visible pending HITL request at a time, so existing hosts do not+ need to build a multi-prompt UI.+- Make answered HITL prompts durable and replayable across resume attempts.+- Keep application code out of the routing decision. Hosts should not need to+ mark approval-gated tools as sequential to avoid this deadlock.+- Support the existing approval middleware and `ask_user` flow without changing+ public tool definitions.+- Keep the design small and async-native, consistent with cubepi's current+ checkpointer and channel abstractions.++## Non-goals++- Showing multiple simultaneous pending prompts in the public HITL API.+- Executing an approved prefix of a parallel batch before the rest of the batch+ has been approved.+- Replacing cubepi's message-loop execution model with a graph runtime.+- Making arbitrary custom tool bodies crash-safe when they call HITL after+ performing side effects. `CheckpointedChannel` should continue to reject that+ case by default.+- Adding a new hard runtime dependency.++## Proposed Design++Add a durable HITL answer ledger. The ledger stores answered HITL requests by+thread, run, and question id. Preparing a gated call first checks the ledger. If+an answer already exists for that question id, preparation replays that answer+without publishing a new pending request and without deleting the answer.++This changes the deadlocked sequence into a progressive collection of answers:++1. User approves `A`.+2. Resume prepares `A` from the ledger, then detaches on `B`. No tool body runs.+3. User approves `B`.+4. Resume prepares `A` from the ledger and prepares `B` from the ledger.+5. All preparation succeeds, so the executor runs `A` and `B` in parallel.+6. Tool results are appended, pending state is cleared, and ledger entries for+ the completed run are deleted.++The answer ledger preserves two-phase execution. It does not execute `A` early;+it only remembers that `A`'s gate has already been satisfied.++### Ledger Key++The logical key is:++```text+(thread_id, run_id, question_id)+```++`run_id` is nullable for legacy in-memory or non-run hosts. First-party durable+checkpointers should normalize `None` to an empty string or equivalent storage+key so the database primary key remains stable.++`question_id` is the existing HITL question id. For approval middleware this is+the tool call id. For `ask_user`, it is the id already produced by the channel.++### Ledger Record++The record only needs enough data to replay the answer:++```python+@dataclass+class HitlAnswerRecord:+ question_id: str+ answer: StructuredValue+ run_id: str | None = None+ answered_at: float = field(default_factory=time.time)+```++Implementations may store a request snapshot for audit/debugging, but replay+must not depend on it. The pending row remains the source of truth for the+currently visible request.++### Checkpointer API++Extend the checkpointer protocol with answer ledger methods:++```python+async def save_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ answer: StructuredValue,+ *,+ run_id: str | None = None,+) -> None: ...++async def load_hitl_answer(+ self,+ thread_id: str,+ question_id: str,+ *,+ run_id: str | None = None,+) -> StructuredValue | None: ...++async def clear_hitl_answers(+ self,+ thread_id: str,+ question_ids: Iterable[str] | None = None,+ *,+ run_id: str | None = None,+) -> None: ...+```++First-party checkpointers must implement these methods:++- `MemoryCheckpointer`: an in-memory dict keyed by `(thread_id, run_key,+ question_id)`.+- `SQLiteCheckpointer`: a new table with `thread_id`, `run_id`,+ `question_id`, `answer_json`, and `answered_at`.+- `PostgresCheckpointer`: equivalent table with JSONB answer storage.+- `MySQLCheckpointer`: equivalent table with JSON answer storage.++`CheckpointedChannel` should validate these methods when it is used for+detached HITL. A missing method should fail early with an actionable `HitlError`,+the same way missing pending APIs are handled today.++### Channel Semantics++`_BaseChannel._await_answer()` should resolve answers in this order:++1. If a durable ledger lookup is available and contains `question_id`, return+ that answer immediately.+2. Otherwise, if the process-local resume slot matches `question_id`, return+ that answer. This keeps same-process behavior and legacy tests simple.+3. Otherwise, publish the request as the single pending request and detach or+ wait according to the current channel mode.++Returning a ledger answer must not delete it. Deletion before the whole batch is+durably resolved would recreate the same bug after a later detachment.++When an answer is supplied through `channel.answer()` in-process, the channel+should also save it to the ledger before waking the waiter. That keeps behavior+consistent between direct in-process HITL and `Agent.respond()`.++### Agent.respond Semantics++`Agent.respond()` should:++1. Load the pending request and recovered run id atomically through+ `load_pending()`.+2. Validate that the supplied `question_id` matches the visible pending request.+3. Validate the agent/channel run binding against the recovered run id.+4. Save the answer into the ledger before starting resume.+5. Optionally attach the process-local resume answer for fast in-process replay.+6. Resume the original tool cycle.++If resume detaches on another request, the just-saved answer remains in the+ledger. A later resume can replay it.++### Cleanup Semantics++Ledger answers are cleared only when they can no longer be needed for replay:++- After the suspended tool cycle completes successfully and tool results have+ been appended.+- When the active HITL request is aborted or cancelled by the host.+- When the agent clears pending state because the run has reached a terminal+ outcome.++Do not clear a ledger entry immediately after a single preparation step consumes+it. For parallel batches, preparation may still detach on a later tool call.++For simplicity and safety, successful completion can clear all ledger answers+for `(thread_id, run_id)` rather than trying to clear only the exact tool call+ids. This also covers `ask_user`, where the question id may not equal a tool+call id.++### User-visible Behavior++After approving one tool in a multi-gated parallel batch, the tool may still not+have executed. The approval means "this gate is answered"; execution begins only+when all gates in the current parallel preparation batch are answered.++Hosts can present this as:++```text+Approved. Waiting for remaining approvals before execution.+```++No API change is required for hosts that already call `Agent.respond()` against+the currently pending request.++## Compatibility++- Existing pending request APIs remain unchanged.+- Existing `execution_mode="sequential"` remains useful for genuinely ordered+ or non-concurrency-safe tools, but it is not required for approval-gated tools.+- Third-party checkpointers that use `CheckpointedChannel` for detached HITL+ need to implement the ledger methods. The failure mode should be explicit at+ channel construction or first detached response, not a silent fallback to the+ deadlocking behavior.+- In-process `InMemoryChannel` remains usable without a durable checkpointer,+ but should still keep an in-memory ledger so the same replay rules apply.++## Testing Requirements++Add tests covering:++- `MemoryCheckpointer`, `SQLiteCheckpointer`, `PostgresCheckpointer`, and+ `MySQLCheckpointer` answer ledger CRUD where those backends already have HITL+ tests.+- `CheckpointedChannel` replays a saved answer for the same `question_id`+ without publishing a new pending request.+- A parallel batch with two approval-gated tools no longer deadlocks:+ approving `A` records the answer and leaves pending `B`; approving `B`+ executes both tools exactly once and appends two tool results.+- Restart behavior: use a fresh `Agent` instance between approvals and verify+ replay still succeeds.+- Rejection/deny and edited approval decisions replay correctly.+- Existing regression coverage that a later preparation detach prevents earlier+ tool bodies from starting still passes.+- Ledger entries are cleared after successful completion and after abort.++## Documentation Requirements++Update the HITL user-facing docs under `website/docs/` to explain:++- Parallel approval-gated tool calls are safe.+- Approving one item in a parallel batch records that decision, but execution+ waits until all gates in the batch are satisfied.+- Application authors do not need to mark approval-gated tools as sequential+ solely for HITL safety.++## Risks++- Stale answers are dangerous if keyed too broadly. Use `run_id` when available+ and keep question ids unique within a turn.+- Clearing too early recreates the deadlock. Cleanup must happen after durable+ tool results are appended or after terminal abort/cancel.+- Third-party checkpointers need a clear migration path. The error should name+ the new ledger methods and explain that detached HITL requires them.diff --git a/tests/checkpointer/test_mysql.py b/tests/checkpointer/test_mysql.pyindex f2c5fee..ee6ae6d 100644--- a/tests/checkpointer/test_mysql.py+++ b/tests/checkpointer/test_mysql.py@@ -29,6 +29,7 @@ def test_exceptions_are_postgres_aliases() -> None:def test_models_import() -> None:from cubepi.checkpointer.mysql.models import (EXPECTED_SCHEMA_VERSION,+ CubepiHitlAnswer,PARTITION_COUNT,CubepiMessage,CubepiSchemaVersion,@@ -36,13 +37,15 @@ def test_models_import() -> None:cubepi_metadata,)- assert EXPECTED_SCHEMA_VERSION == 4+ assert EXPECTED_SCHEMA_VERSION == 5assert PARTITION_COUNT == 64assert CubepiThread.__tablename__ == "cubepi_threads"assert CubepiMessage.__tablename__ == "cubepi_messages"+ assert CubepiHitlAnswer.__tablename__ == "cubepi_hitl_answers"assert CubepiSchemaVersion.__tablename__ == "cubepi_schema_version"assert "cubepi_threads" in cubepi_metadata.tablesassert "cubepi_messages" in cubepi_metadata.tables+ assert "cubepi_hitl_answers" in cubepi_metadata.tablesassert "cubepi_schema_version" in cubepi_metadata.tables@@ -232,6 +235,7 @@ async def _setup_schema(dsn: str) -> None:add_pending_request_column_op,add_run_id_column_op,upgrade_v3_to_v4_op,+ upgrade_v4_to_v5_op,)await cur.execute(add_pending_request_column_op())@@ -258,6 +262,9 @@ async def _setup_schema(dsn: str) -> None:for stmt in upgrade_v3_to_v4_op().split(";"):if stmt.strip():await cur.execute(stmt)+ for stmt in upgrade_v4_to_v5_op().split(";"):+ if stmt.strip():+ await cur.execute(stmt)for stmt in write_schema_version_op().split(";"):if stmt.strip():await cur.execute(stmt)@@ -389,7 +396,9 @@ async def test_version_mismatch_raises(clean_mysql_db) -> None:with pytest.raises(CubepiSchemaMismatch) as exc_info:async with MySQLCheckpointer(clean_mysql_db):pass- assert exc_info.value.expected == 4+ from cubepi.checkpointer.mysql.models import EXPECTED_SCHEMA_VERSION++ assert exc_info.value.expected == EXPECTED_SCHEMA_VERSIONassert exc_info.value.actual == 999diff --git a/tests/checkpointer/test_mysql_schema.py b/tests/checkpointer/test_mysql_schema.pyindex aad1892..7b1a6a1 100644--- a/tests/checkpointer/test_mysql_schema.py+++ b/tests/checkpointer/test_mysql_schema.py@@ -6,8 +6,8 @@ import pytestfrom cubepi.checkpointer.mysql.models import EXPECTED_SCHEMA_VERSION-def test_expected_schema_version_is_4() -> None:- assert EXPECTED_SCHEMA_VERSION == 4+def test_expected_schema_version_is_5() -> None:+ assert EXPECTED_SCHEMA_VERSION == 5@pytest.mark.asyncio@@ -54,3 +54,30 @@ async def test_cubepi_messages_has_run_id_column(mysql_v4_dsn) -> None:finally:await conn.ensure_closed()assert row is not None+++async def test_cubepi_hitl_answers_table_present(mysql_v4_dsn) -> None:+ from cubepi.checkpointer.mysql.checkpointer import _parse_dsn++ cfg = _parse_dsn(mysql_v4_dsn)+ conn = await aiomysql.connect(autocommit=True, **cfg)+ try:+ async with conn.cursor() as cur:+ await cur.execute(+ "SELECT column_name FROM information_schema.columns "+ "WHERE table_schema = %s "+ "AND table_name = 'cubepi_hitl_answers'",+ (cfg["db"],),+ )+ rows = await cur.fetchall()+ finally:+ await conn.ensure_closed()+ cols = {r[0].lower() for r in rows}+ assert {+ "thread_id",+ "run_id",+ "question_id",+ "answer",+ "answered_at",+ } <= colsdiff --git a/tests/checkpointer/test_postgres.py b/tests/checkpointer/test_postgres.pyindex 157a075..a5a80e7 100644--- a/tests/checkpointer/test_postgres.py+++ b/tests/checkpointer/test_postgres.py@@ -10,6 +10,7 @@ import pytestdef test_models_import() -> None:from cubepi.checkpointer.postgres.models import (EXPECTED_SCHEMA_VERSION,+ CubepiHitlAnswer,PARTITION_COUNT,CubepiMessage,CubepiSchemaVersion,@@ -17,15 +18,15 @@ def test_models_import() -> None:cubepi_metadata,)- assert EXPECTED_SCHEMA_VERSION == 4+ assert EXPECTED_SCHEMA_VERSION == 5assert PARTITION_COUNT == 64- # All three model classes are reachable via the public model moduleassert CubepiThread.__tablename__ == "cubepi_threads"assert CubepiMessage.__tablename__ == "cubepi_messages"+ assert CubepiHitlAnswer.__tablename__ == "cubepi_hitl_answers"assert CubepiSchemaVersion.__tablename__ == "cubepi_schema_version"- # All three tables registered on cubepi_metadataassert "cubepi_threads" in cubepi_metadata.tablesassert "cubepi_messages" in cubepi_metadata.tables+ assert "cubepi_hitl_answers" in cubepi_metadata.tablesassert "cubepi_schema_version" in cubepi_metadata.tables@@ -204,6 +205,7 @@ async def _setup_schema(dsn: str) -> None:add_run_id_column_op,create_message_partitions_op,upgrade_v3_to_v4_op,+ upgrade_v4_to_v5_op,write_schema_version_op,)@@ -222,6 +224,7 @@ async def _setup_schema(dsn: str) -> None:""")# Apply v3→v4 (run_id on messages + cubepi_runs partitioned table).await conn.execute(upgrade_v3_to_v4_op())+ await conn.execute(upgrade_v4_to_v5_op())await conn.execute(write_schema_version_op())finally:await conn.close()@@ -330,7 +333,9 @@ async def test_version_mismatch_raises(clean_db) -> None:with pytest.raises(CubepiSchemaMismatch) as exc_info:async with PostgresCheckpointer(clean_db):pass- assert exc_info.value.expected == 4+ from cubepi.checkpointer.postgres.models import EXPECTED_SCHEMA_VERSION++ assert exc_info.value.expected == EXPECTED_SCHEMA_VERSIONassert exc_info.value.actual == 999diff --git a/tests/checkpointer/test_postgres_schema.py b/tests/checkpointer/test_postgres_schema.pyindex 672d5e1..ff62eb8 100644--- a/tests/checkpointer/test_postgres_schema.py+++ b/tests/checkpointer/test_postgres_schema.py@@ -4,8 +4,8 @@ import pytestfrom cubepi.checkpointer.postgres.models import EXPECTED_SCHEMA_VERSION-def test_expected_schema_version_is_4():- assert EXPECTED_SCHEMA_VERSION == 4+def test_expected_schema_version_is_5():+ assert EXPECTED_SCHEMA_VERSION == 5@pytest.mark.asyncio@@ -39,3 +39,23 @@ async def test_cubepi_messages_has_run_id_column(pg_v4_dsn):assert row is not Nonefinally:await conn.close()+++async def test_cubepi_hitl_answers_table_present(pg_v4_dsn):+ conn = await asyncpg.connect(pg_v4_dsn)+ try:+ rows = await conn.fetch(+ "SELECT column_name FROM information_schema.columns "+ "WHERE table_name = 'cubepi_hitl_answers'"+ )+ cols = {r["column_name"] for r in rows}+ assert {+ "thread_id",+ "run_id",+ "question_id",+ "answer",+ "answered_at",+ } <= cols+ finally:+ await conn.close()diff --git a/tests/checkpointer/test_postgres_schema_version_v3.py b/tests/checkpointer/test_postgres_schema_version_v3.pyindex c003841..30a9d1a 100644--- a/tests/checkpointer/test_postgres_schema_version_v3.py+++ b/tests/checkpointer/test_postgres_schema_version_v3.py@@ -27,7 +27,7 @@ async def test_postgres_v2_database_refused_with_actionable_error(clean_db) -> Npassmsg = str(ei.value)- assert "expected 4" in msg or "expected=4" in msg+ assert "expected 5" in msg or "expected=5" in msgassert "actual 2" in msg or "actual=2" in msg# Hint mentions the host alembic migration path so operators know what to do.assert "alembic" in msg.lower()@@ -59,6 +59,6 @@ async def test_mysql_v2_database_refused_with_actionable_error(clean_mysql_db) -passmsg = str(ei.value)- assert "expected 4" in msg or "expected=4" in msg+ assert "expected 5" in msg or "expected=5" in msgassert "actual 2" in msg or "actual=2" in msgassert "alembic" in msg.lower()diff --git a/tests/hitl/test_checkpointed_channel_run_id.py b/tests/hitl/test_checkpointed_channel_run_id.pyindex dc8fa9a..28f5a92 100644--- a/tests/hitl/test_checkpointed_channel_run_id.py+++ b/tests/hitl/test_checkpointed_channel_run_id.py@@ -99,6 +99,7 @@ async def test_legacy_checkpointer_works_without_run_id_kwarg():def __init__(self) -> None:self._pending: dict[str, HitlRequest | None] = {}+ self._answers: dict[str, object] = {}async def save_pending_request(self, thread_id: str, request@@ -108,6 +109,25 @@ async def test_legacy_checkpointer_works_without_run_id_kwarg():async def load_pending_request(self, thread_id: str):return self._pending.get(thread_id)+ async def save_hitl_answer(+ self, thread_id: str, question_id: str, answer, *, run_id=None+ ):+ self._answers[question_id] = answer++ async def load_hitl_answer(+ self, thread_id: str, question_id: str, *, run_id=None+ ):+ return self._answers.get(question_id)++ async def clear_hitl_answers(+ self, thread_id: str, question_ids=None, *, run_id=None+ ):+ if question_ids is None:+ self._answers.clear()+ else:+ for question_id in question_ids:+ self._answers.pop(question_id, None)+cp = LegacyCheckpointer()# No run_id at construction → channel must call legacy two-arg signature.ch = CheckpointedChannel(checkpointer=cp, thread_id="t-legacy")diff --git a/tests/hitl/test_checkpointer_pending_request.py b/tests/hitl/test_checkpointer_pending_request.pyindex 0482e73..27e29d8 100644--- a/tests/hitl/test_checkpointer_pending_request.py+++ b/tests/hitl/test_checkpointer_pending_request.py@@ -63,3 +63,54 @@ async def test_sqlite_create_table_idempotent(sqlite_cp):"thread_id TEXT PRIMARY KEY, request_json TEXT NOT NULL, ""created_at REAL NOT NULL DEFAULT (julianday('now')))")+++async def test_memory_hitl_answer_ledger_crud():+ cp = MemoryCheckpointer()+ await cp.save_hitl_answer("t-1", "q-1", {"decision": "approve"}, run_id="r-1")+ await cp.save_hitl_answer("t-1", "q-2", {"decision": "deny"}, run_id="r-1")+ await cp.save_hitl_answer("t-1", "q-1", {"decision": "edit"}, run_id="r-2")++ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-1") == {+ "decision": "approve"+ }+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {"decision": "edit"}++ await cp.clear_hitl_answers("t-1", ["q-1"], run_id="r-1")+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-1") is None+ assert await cp.load_hitl_answer("t-1", "q-2", run_id="r-1") == {"decision": "deny"}+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {"decision": "edit"}++ await cp.clear_hitl_answers("t-1", run_id="r-1")+ assert await cp.load_hitl_answer("t-1", "q-2", run_id="r-1") is None+ assert await cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {"decision": "edit"}+++async def test_sqlite_hitl_answer_ledger_crud(sqlite_cp):+ await sqlite_cp.save_hitl_answer(+ "t-1", "q-1", {"decision": "approve"}, run_id="r-1"+ )+ await sqlite_cp.save_hitl_answer("t-1", "q-2", {"decision": "deny"}, run_id="r-1")+ await sqlite_cp.save_hitl_answer("t-1", "q-1", {"decision": "edit"}, run_id="r-2")++ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-1") == {+ "decision": "approve"+ }+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {+ "decision": "edit"+ }++ await sqlite_cp.clear_hitl_answers("t-1", ["q-1"], run_id="r-1")+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-1") is None+ assert await sqlite_cp.load_hitl_answer("t-1", "q-2", run_id="r-1") == {+ "decision": "deny"+ }+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {+ "decision": "edit"+ }++ await sqlite_cp.clear_hitl_answers("t-1", run_id="r-1")+ assert await sqlite_cp.load_hitl_answer("t-1", "q-2", run_id="r-1") is None+ assert await sqlite_cp.load_hitl_answer("t-1", "q-1", run_id="r-2") == {+ "decision": "edit"+ }diff --git a/tests/hitl/test_in_memory_channel.py b/tests/hitl/test_in_memory_channel.pyindex 23c37a1..eda0085 100644--- a/tests/hitl/test_in_memory_channel.py+++ b/tests/hitl/test_in_memory_channel.py@@ -209,6 +209,26 @@ async def test_attach_resume_answer_short_circuits_next_call():assert ch.pending is None+async def test_answer_ledger_short_circuits_replayed_approve():+ ch = InMemoryChannel()++ async def host():+ while ch.pending is None:+ await asyncio.sleep(0)+ await ch.answer(ch.pending.question_id, ApproveAnswer(decision="approve"))++ asyncio.create_task(host())+ first = await ch.approve(tool_name="bash", tool_call_id="tc-7", args={})+ assert first.decision == "approve"+ assert ch.pending is None++ replayed = await ch.approve(+ tool_name="bash", tool_call_id="tc-7", args={}, timeout=0.01+ )+ assert replayed.decision == "approve"+ assert ch.pending is None++async def test_resume_short_circuit_emits_hitl_answer_event():"""Resume short-circuit must emit HitlAnswerEvent so subscribers(e.g. IM outbound tailers) learn the question was answered."""diff --git a/tests/hitl/test_parallel_approval_ledger.py b/tests/hitl/test_parallel_approval_ledger.pynew file mode 100644index 0000000..c496c5b--- /dev/null+++ b/tests/hitl/test_parallel_approval_ledger.py@@ -0,0 +1,122 @@+from __future__ import annotations++import asyncio++import pytest+from pydantic import BaseModel++from cubepi.agent.agent import Agent+from cubepi.agent.types import AgentTool, AgentToolResult+from cubepi.checkpointer.memory import MemoryCheckpointer+from cubepi.hitl import ApproveAnswer, AskUser+from cubepi.hitl.channel import CheckpointedChannel+from cubepi.hitl.middleware import ApprovalPolicyMiddleware+from cubepi.providers.base import TextContent, ToolResultMessage+from cubepi.providers.faux import FauxProvider, faux_assistant_message, faux_tool_call+++class _NoParams(BaseModel):+ pass+++def _make_parallel_tool(name: str, executions: list[str]) -> AgentTool:+ async def execute(call_id, args, *, signal=None, on_update=None):+ executions.append(call_id)+ return AgentToolResult(content=[TextContent(text=f"{name} ran")])++ return AgentTool(+ name=name,+ description=name,+ parameters=_NoParams,+ execute=execute,+ execution_mode="parallel",+ )+++def _provider() -> FauxProvider:+ provider = FauxProvider(provider_id="faux")+ provider.set_responses(+ [+ faux_assistant_message(+ [+ faux_tool_call("solar", {}, id="tc-a"),+ faux_tool_call("semiconductor", {}, id="tc-b"),+ ],+ stop_reason="tool_use",+ ),+ faux_assistant_message("done"),+ ]+ )+ return provider+++def _agent(+ *,+ cp: MemoryCheckpointer,+ provider: FauxProvider,+ executions: list[str],+) -> Agent:+ ch = CheckpointedChannel(checkpointer=cp, thread_id="t-root-a", run_id="R1")+ return Agent(+ model=provider.model("faux"),+ tools=[+ _make_parallel_tool("solar", executions),+ _make_parallel_tool("semiconductor", executions),+ ],+ middleware=[ApprovalPolicyMiddleware(ch, policy=lambda c: AskUser())],+ channel=ch,+ checkpointer=cp,+ thread_id="t-root-a",+ )+++async def _wait_for_pending(cp: MemoryCheckpointer, expected_qid: str) -> None:+ for _ in range(200):+ loaded = await cp.load_pending("t-root-a")+ if loaded is not None and loaded[0].question_id == expected_qid:+ return+ await asyncio.sleep(0.01)+ pytest.fail(f"pending request did not become {expected_qid!r}")+++async def test_parallel_approval_answers_are_replayed_until_batch_can_execute():+ cp = MemoryCheckpointer()+ provider = _provider()+ executions: list[str] = []++ initial_agent = _agent(cp=cp, provider=provider, executions=executions)+ prompt_task = asyncio.create_task(initial_agent.prompt("run both", run_id="R1"))+ await _wait_for_pending(cp, "tc-a")+ await initial_agent.detach()+ await prompt_task++ first_resume = _agent(cp=cp, provider=provider, executions=executions)+ first_resume_task = asyncio.create_task(+ first_resume.respond(+ question_id="tc-a", answer=ApproveAnswer(decision="approve")+ )+ )+ await _wait_for_pending(cp, "tc-b")+ await first_resume.detach()+ await first_resume_task++ assert executions == []+ loaded = await cp.load_pending("t-root-a")+ assert loaded is not None+ assert loaded[0].question_id == "tc-b"++ second_resume = _agent(cp=cp, provider=provider, executions=executions)+ await second_resume.respond(+ question_id="tc-b", answer=ApproveAnswer(decision="approve")+ )++ assert sorted(executions) == ["tc-a", "tc-b"]+ assert await cp.load_pending("t-root-a") is None+ assert await cp.load_hitl_answer("t-root-a", "tc-a", run_id="R1") is None+ assert await cp.load_hitl_answer("t-root-a", "tc-b", run_id="R1") is None+ tool_results = [+ msg+ for msg in second_resume.state.messages+ if isinstance(msg, ToolResultMessage)+ ]+ assert [msg.tool_call_id for msg in tool_results] == ["tc-a", "tc-b"]diff --git a/website/docs/guides/checkpointing/mysql.md b/website/docs/guides/checkpointing/mysql.mdindex 071db2b..4fcb9e2 100644--- a/website/docs/guides/checkpointing/mysql.md+++ b/website/docs/guides/checkpointing/mysql.md@@ -61,10 +61,11 @@ async with MySQLCheckpointer(## Schema-The checkpointer expects three tables: `cubepi_threads`,-`cubepi_messages`, and `cubepi_schema_version`. Like Postgres (and-unlike SQLite), CubePi **does not create these for you** — it verifies-on `__aenter__` that they exist with the expected `schema_version`.+The checkpointer expects `cubepi_threads`, `cubepi_messages`,+`cubepi_runs`, `cubepi_hitl_answers`, and `cubepi_schema_version`. Like+Postgres (and unlike SQLite), CubePi **does not create these for you** —+it verifies on `__aenter__` that they exist with the expected+`schema_version`.If they're missing, you get `CubepiSchemaUninitialized`. If the versiondoesn't match this CubePi release, you get `CubepiSchemaMismatch`. A@@ -141,6 +142,16 @@ cubepi_messagespayload -- LONGBLOB (msgpack)created_at+cubepi_runs+ thread_id, run_id -- composite PK+ claimed_at / completed_at+ completion_seq++cubepi_hitl_answers+ thread_id, run_id, question_id -- composite PK+ answer -- JSON+ answered_at+cubepi_schema_versionversion (PK)```@@ -224,6 +235,29 @@ Pre-feature messages keep `run_id = NULL` and remain readable; see[Legacy data behaviour](../agents/forking#legacy-data-behaviour) for thefork-eligibility rules on mixed threads.+## Schema v4 -> v5 migration++Durable HITL answer replay bumps `EXPECTED_SCHEMA_VERSION` from 4 to 5.+The upgrade creates `cubepi_hitl_answers`, which stores answered HITL+requests by `(thread_id, run_id, question_id)` until the suspended tool+cycle completes.++```python+# In a migration's upgrade():+from cubepi.checkpointer.mysql.alembic_helpers import (+ upgrade_v4_to_v5_op,+ write_schema_version_op,+)++def upgrade():+ for stmt in upgrade_v4_to_v5_op().split(";"):+ if stmt.strip():+ op.execute(stmt)+ for stmt in write_schema_version_op().split(";"):+ if stmt.strip():+ op.execute(stmt)+```+## Common pitfalls- **`CubepiSchemaUninitialized`** — Your DB is empty, your migrationsdiff --git a/website/docs/guides/checkpointing/postgres.md b/website/docs/guides/checkpointing/postgres.mdindex 8d04d40..cea4061 100644--- a/website/docs/guides/checkpointing/postgres.md+++ b/website/docs/guides/checkpointing/postgres.md@@ -54,8 +54,9 @@ async with PostgresCheckpointer(## Schema-The checkpointer expects three tables: `cubepi_threads`,-`cubepi_messages`, and `cubepi_schema_version`. Unlike SQLite, CubePi+The checkpointer expects `cubepi_threads`, `cubepi_messages`,+`cubepi_runs`, `cubepi_hitl_answers`, and `cubepi_schema_version`.+Unlike SQLite, CubePi**does not create these for you** — it verifies on `__aenter__` thatthey exist with the expected `schema_version`.@@ -130,6 +131,16 @@ cubepi_messagespayload -- bytea (msgpack)created_at+cubepi_runs+ thread_id, run_id -- composite PK+ claimed_at / completed_at+ completion_seq++cubepi_hitl_answers+ thread_id, run_id, question_id -- composite PK+ answer -- JSONB+ answered_at+cubepi_schema_versionversion (PK)```@@ -213,6 +224,25 @@ Pre-feature messages keep `run_id = NULL` and remain readable; see[Legacy data behaviour](../agents/forking#legacy-data-behaviour) for thefork-eligibility rules on mixed threads.+## Schema v4 -> v5 migration++Durable HITL answer replay bumps `EXPECTED_SCHEMA_VERSION` from 4 to 5.+The upgrade creates `cubepi_hitl_answers`, which stores answered HITL+requests by `(thread_id, run_id, question_id)` until the suspended tool+cycle completes.++```python+# In a migration's upgrade():+from cubepi.checkpointer.postgres.alembic_helpers import (+ upgrade_v4_to_v5_op,+ write_schema_version_op,+)++def upgrade():+ op.execute(upgrade_v4_to_v5_op())+ op.execute(write_schema_version_op()) # bumps cubepi_schema_version to 5+```+## Common pitfalls- **`CubepiSchemaUninitialized`** — Your DB is empty or yourdiff --git a/website/docs/guides/checkpointing/sqlite.md b/website/docs/guides/checkpointing/sqlite.mdindex ddf4a44..9271b73 100644--- a/website/docs/guides/checkpointing/sqlite.md+++ b/website/docs/guides/checkpointing/sqlite.md@@ -82,19 +82,34 @@ CREATE TABLE thread_extra (This schema is append-only. CubePi never updates or deletes rows.-## HITL pending table+## HITL tables-When the [HITL](../hitl/overview) module is in use, an additional table is+When the [HITL](../hitl/overview) module is in use, additional tables arecreated automatically on `__aenter__`:```sqlCREATE TABLE IF NOT EXISTS thread_pending_request (thread_id TEXT PRIMARY KEY,request_json TEXT NOT NULL,+ run_id TEXT,created_at REAL NOT NULL DEFAULT (julianday('now')));++CREATE TABLE IF NOT EXISTS thread_hitl_answers (+ thread_id TEXT NOT NULL,+ run_id TEXT NOT NULL,+ question_id TEXT NOT NULL,+ answer_json TEXT NOT NULL,+ answered_at REAL NOT NULL DEFAULT (julianday('now')),+ PRIMARY KEY (thread_id, run_id, question_id)+);```+`thread_pending_request` stores the single visible HITL request for a+thread. `thread_hitl_answers` stores already-answered prompts so a+parallel approval batch can collect approvals one by one and then+execute the batch only after every gate is satisfied.+No manual migration is needed — `CREATE TABLE IF NOT EXISTS` isidempotent.diff --git a/website/docs/guides/hitl/reference.md b/website/docs/guides/hitl/reference.mdindex 345a0fb..9a1fd2b 100644--- a/website/docs/guides/hitl/reference.md+++ b/website/docs/guides/hitl/reference.md@@ -120,6 +120,11 @@ assert await ch.ask([Question(key="k", prompt="p")]) == {"k": ""}- **Single pending per thread.** The agent loop is sequential — at most oneHITL request is outstanding per `thread_id`. Concurrent `confirm/approve/ask`raises `HitlConcurrencyError`.+- **Parallel approval batches collect answers before execution.** If one+ assistant turn contains multiple parallel tool calls that require approval,+ CubePi still exposes one pending request at a time. Each approved answer is+ persisted by `question_id` and replayed on the next resume attempt. Tool+ bodies start only after every gate in the parallel batch has been answered.- **Prompt-cache prefix invariant.** Between pause and resume, the messageslist changes only by appending tool-result messages and the next assistantturn at the tail. No inserting, reordering, or mutating prior messages —@@ -127,6 +132,7 @@ assert await ch.ask([Question(key="k", prompt="p")]) == {"k": ""}- **`question_id == tool_call_id` for approve requests.** No aliasing ormapping needed — hosts that already track `call_id` from the tool streampass it directly.-- **Resume does not replay.** It re-enters the loop with the answer pre-loaded- into the channel. The last assistant message's unresolved tool calls dictate- what executes next. No node-based replay semantics.+- **Resume re-enters the unresolved tool cycle.** The last assistant message's+ unresolved tool calls dictate what prepares and executes next. Persisted HITL+ answers can be replayed by `question_id`, but CubePi does not use node-based+ graph replay semantics.