Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

fix(agent): address codex review — carry salvaged results, strict suspend persistence, propagate emitter failures

Claude Codeed26371c2804
diff --git a/cubepi/agent/loop.py b/cubepi/agent/loop.py
index ff70f9d..4263f6a 100644
--- a/cubepi/agent/loop.py
+++ b/cubepi/agent/loop.py
@@ -203,17 +203,25 @@ async def run_agent_loop_resume(
new_messages=new_messages,
set_outcome=set_outcome,
)
- except HitlDetached: # pragma: no cover — E2E tested
+ except HitlDetached as exc: # pragma: no cover — E2E tested
# The Agent caller (Agent.detach) emitted AgentSuspendedEvent already.
# Loop exits silently — assistant message and pending state remain
- # intact for the next respond() call.
+ # intact for the next respond() call. Sibling tool_results that
+ # completed before the detach were emitted by the executor; append
+ # them so callers persisting the returned messages keep them.
+ for msg in exc.partial_tool_results:
+ context.messages.append(msg)
+ new_messages.append(msg)
if set_outcome is not None:
set_outcome("suspended")
return new_messages
- except HitlAborted: # pragma: no cover — E2E tested
+ except HitlAborted as exc: # pragma: no cover — E2E tested
# The Agent caller (Agent.abort_pending) emitted AgentAbortedEvent
# already. Loop exits silently — synthetic deny + terminal aborted
# assistant message already appended by abort_pending.
+ for msg in exc.partial_tool_results:
+ context.messages.append(msg)
+ new_messages.append(msg)
if set_outcome is not None:
set_outcome("abandoned")
return new_messages
@@ -442,17 +450,26 @@ async def _run_loop(
emit=emit,
set_outcome=set_outcome,
)
- except HitlDetached:
+ except HitlDetached as exc:
# AgentSuspendedEvent already emitted by Agent.detach() — exit silently
# so AgentEndEvent doesn't double-signal termination. Assistant message
# and pending state remain intact for the next respond() call.
+ # Sibling tool_results that completed before the detach were emitted
+ # (and checkpointed) by the executor; append them so callers reading
+ # current_context/new_messages keep them too.
+ for msg in exc.partial_tool_results:
+ current_context.messages.append(msg)
+ new_messages.append(msg)
if set_outcome is not None:
set_outcome("suspended")
return
- except HitlAborted:
+ except HitlAborted as exc:
# AgentAbortedEvent already emitted by Agent.abort_pending() — exit
# silently. Synthetic deny + terminal aborted assistant message are
# already appended; conversation is closed.
+ for msg in exc.partial_tool_results:
+ current_context.messages.append(msg)
+ new_messages.append(msg)
if set_outcome is not None:
set_outcome("abandoned")
return
diff --git a/cubepi/agent/tools.py b/cubepi/agent/tools.py
index cb2a30b..9d7ff1b 100644
--- a/cubepi/agent/tools.py
+++ b/cubepi/agent/tools.py
@@ -474,70 +474,79 @@ async def _execute_sequential(
finalized_list: list[_FinalizedOutcome] = []
messages: list[ToolResultMessage] = []
- for idx, tc in enumerate(tool_calls):
- if preresolved is not None:
- rtc, was_resolved, resolver_error = preresolved[idx]
- else:
- # Lazy: resolve only when this call's turn comes, after every
- # earlier call in the batch has fully executed.
- rtc, was_resolved, resolver_error = await _resolve_tool_call(
- tc, context, resolve_tool_call, signal
- )
-
- await emit_event(
- emit_fn,
- ToolExecutionStartEvent(
- tool_call_id=rtc.id, tool_name=rtc.name, args=rtc.arguments
- ),
- )
-
- preparation = resolver_error or await _prepare_tool_call(
- context,
- assistant_message,
- rtc,
- before_tool_call,
- signal,
- resolved=was_resolved,
- )
+ try:
+ for idx, tc in enumerate(tool_calls):
+ if preresolved is not None:
+ rtc, was_resolved, resolver_error = preresolved[idx]
+ else:
+ # Lazy: resolve only when this call's turn comes, after every
+ # earlier call in the batch has fully executed.
+ rtc, was_resolved, resolver_error = await _resolve_tool_call(
+ tc, context, resolve_tool_call, signal
+ )
- if isinstance(preparation, _ImmediateOutcome):
- finalized = _FinalizedOutcome(
- tool_call=rtc,
- result=preparation.result,
- is_error=preparation.is_error,
- blocked_by_hook=preparation.blocked_by_hook,
- block_reason=preparation.block_reason,
- hitl_trace=preparation.hitl_trace,
+ await emit_event(
+ emit_fn,
+ ToolExecutionStartEvent(
+ tool_call_id=rtc.id, tool_name=rtc.name, args=rtc.arguments
+ ),
)
- else:
- result, is_error = await _execute_prepared(preparation, signal, emit_fn)
- finalized = await _finalize(
+
+ preparation = resolver_error or await _prepare_tool_call(
context,
assistant_message,
- preparation,
- result,
- is_error,
- after_tool_call,
+ rtc,
+ before_tool_call,
signal,
+ resolved=was_resolved,
)
- await emit_event(
- emit_fn,
- ToolExecutionEndEvent(
- tool_call_id=rtc.id,
- tool_name=rtc.name,
- result=finalized.result,
- is_error=finalized.is_error,
- terminate=bool(finalized.result.terminate),
- blocked_by_hook=finalized.blocked_by_hook,
- block_reason=finalized.block_reason,
- ),
- )
- tool_msg = _make_tool_result_message(finalized)
- await emit_event(emit_fn, MessageStartEvent(message=tool_msg))
- await emit_event(emit_fn, MessageEndEvent(message=tool_msg))
- finalized_list.append(finalized)
- messages.append(tool_msg)
+ if isinstance(preparation, _ImmediateOutcome):
+ finalized = _FinalizedOutcome(
+ tool_call=rtc,
+ result=preparation.result,
+ is_error=preparation.is_error,
+ blocked_by_hook=preparation.blocked_by_hook,
+ block_reason=preparation.block_reason,
+ hitl_trace=preparation.hitl_trace,
+ )
+ else:
+ result, is_error = await _execute_prepared(preparation, signal, emit_fn)
+ finalized = await _finalize(
+ context,
+ assistant_message,
+ preparation,
+ result,
+ is_error,
+ after_tool_call,
+ signal,
+ )
+
+ await emit_event(
+ emit_fn,
+ ToolExecutionEndEvent(
+ tool_call_id=rtc.id,
+ tool_name=rtc.name,
+ result=finalized.result,
+ is_error=finalized.is_error,
+ terminate=bool(finalized.result.terminate),
+ blocked_by_hook=finalized.blocked_by_hook,
+ block_reason=finalized.block_reason,
+ ),
+ )
+ tool_msg = _make_tool_result_message(finalized)
+ await emit_event(emit_fn, MessageStartEvent(message=tool_msg))
+ await emit_event(emit_fn, MessageEndEvent(message=tool_msg))
+ finalized_list.append(finalized)
+ messages.append(tool_msg)
+
+ except HitlControlException as exc:
+ # Completed calls' results were already emitted (and
+ # checkpointed) per-iteration; carry them on the exception so
+ # the stateless loop entry points can return them to callers
+ # that persist the loop's return value across the suspend.
+ exc.partial_tool_results = tuple(messages)
+ raise
return ToolCallBatch(messages=messages, terminate=_should_terminate(finalized_list))
@@ -687,6 +696,7 @@ async def _execute_parallel(
finalized_list: list[_FinalizedOutcome] = []
control_exc: BaseException | None = None
+ framework_exc: BaseException | None = None
for entry, slot in zip(entries, scheduled):
if isinstance(slot, _FinalizedOutcome):
finalized_list.append(slot)
@@ -710,18 +720,23 @@ async def _execute_parallel(
if control_exc is None:
control_exc = exc
continue
+ if not isinstance(exc, asyncio.CancelledError):
+ # Tool and hook failures are converted to error results at the
+ # source (_execute_prepared/_finalize), so anything else landing
+ # here is framework-side — typically emit_fn raising while
+ # processing a Start/End event. Emitter failures propagate at
+ # every other emit_event call site; synthesizing a bogus
+ # tool_result here would let the run continue with event
+ # processing broken. Deferred until every task has settled.
+ if framework_exc is None:
+ framework_exc = exc
+ continue
# Per-task isolation: a stray CancelledError (tool self-cancel with
- # no outer cancel — a tool bug) or any exception that slipped past
- # _execute_prepared/_finalize degrades to an error result for THIS
- # call only.
- text = (
- "[Tool execution cancelled]"
- if isinstance(exc, asyncio.CancelledError)
- else str(exc)
- )
+ # no outer cancel — a tool bug) degrades to an error result for
+ # THIS call only.
synthesized = _FinalizedOutcome(
tool_call=entry.tool_call,
- result=_error_result(text),
+ result=_error_result("[Tool execution cancelled]"),
is_error=True,
hitl_trace=entry.hitl_trace,
)
@@ -739,17 +754,27 @@ async def _execute_parallel(
)
finalized_list.append(synthesized)
+ if framework_exc is not None:
+ # Event processing is broken, so emitting sibling results below
+ # would fail too. Every task has already settled (no leaks) —
+ # propagate, taking precedence over a suspend: durably suspending
+ # a run whose event pipeline is failing is not safe.
+ raise framework_exc
+
if control_exc is not None:
- # Persist the siblings (each MessageEndEvent checkpoints
- # immediately), best-effort, then let the control exception
- # propagate exactly as the suspend/abort machinery expects.
- try:
- await _emit_tool_result_messages(finalized_list, emit_fn)
- except Exception:
- # Best-effort: sibling persistence must never swallow the
- # control exception — the suspend/abort machinery depends on
- # it propagating; unanswered ids are backfilled on resume.
- pass
+ # Persist the siblings BEFORE propagating the suspend (each
+ # MessageEndEvent checkpoints immediately). Deliberately NOT
+ # best-effort: suspending after a sibling's persistence failed
+ # would durably record a batch whose completed work is missing —
+ # resume would re-run tools whose side effects already happened.
+ # A persistence failure propagates instead (consistent with every
+ # other MessageEndEvent site): the run fails rather than suspends.
+ emitted = await _emit_tool_result_messages(finalized_list, emit_fn)
+ if isinstance(control_exc, HitlControlException):
+ # Stateless loop entry points append these to the message
+ # lists they return, so callers persisting the return value
+ # keep the completed siblings' results across the suspend.
+ control_exc.partial_tool_results = tuple(emitted)
raise control_exc
messages = await _emit_tool_result_messages(finalized_list, emit_fn)
diff --git a/cubepi/hitl/exceptions.py b/cubepi/hitl/exceptions.py
index d815f88..d1dc2c8 100644
--- a/cubepi/hitl/exceptions.py
+++ b/cubepi/hitl/exceptions.py
@@ -1,5 +1,10 @@
from __future__ import annotations
+from typing import TYPE_CHECKING, Sequence
+
+if TYPE_CHECKING: # pragma: no cover
+ from cubepi.providers.base import ToolResultMessage
+
class HitlControlException(BaseException):
"""Base for HITL control-flow exceptions.
@@ -7,8 +12,18 @@ class HitlControlException(BaseException):
Inherits BaseException so existing `except Exception:` handlers in
cubepi.agent.tools._prepare_tool_call and _execute_prepared do NOT
swallow these — mirrors asyncio.CancelledError.
+
+ ``partial_tool_results``: ToolResultMessages of sibling tool calls in
+ the same batch that completed (and were emitted/checkpointed) before
+ this control exception suspended the run. The tool executors set it so
+ the stateless loop entry points can append those results to the
+ message lists they return — a caller persisting the loop's return
+ value must not lose a completed sibling's result just because another
+ call in the batch detached.
"""
+ partial_tool_results: Sequence[ToolResultMessage] = ()
+
class HitlCancelled(HitlControlException):
def __init__(self, reason: str):
diff --git a/dev/specs/2026-07-05-tool-batch-fault-isolation.md b/dev/specs/2026-07-05-tool-batch-fault-isolation.md
index 503d64a..7d4ff06 100644
--- a/dev/specs/2026-07-05-tool-batch-fault-isolation.md
+++ b/dev/specs/2026-07-05-tool-batch-fault-isolation.md
@@ -121,20 +121,34 @@ the bare `await` loop. Every task settles before any outcome is processed;
results stay in tool_call order. Then classify each slot:
1. **`_FinalizedOutcome`** — success path, unchanged.
-2. **Ordinary failure** (any exception that is not `HitlControlException`,
- `CancelledError`, `KeyboardInterrupt`, or `SystemExit`) — synthesize an
- error outcome for that tool_call id:
- `AgentToolResult(content=[TextContent(text=str(exc))], is_error=True)`,
- plus a paired `ToolExecutionEndEvent` (the Start was emitted inside
- `_run`; an unpaired Start would leave `state.pending_tool_calls` and trace
- spans open — same invariant the prepare-phase comment protects).
- With Part 2 in place this is defense-in-depth: tool-body and hook failures
- are already converted inside the task.
+2. **Framework failure** (any exception that is not `HitlControlException`,
+ `CancelledError`, `KeyboardInterrupt`, or `SystemExit`) — with Part 2 in
+ place, tool-body and hook failures are converted to error results *at the
+ source*, inside the task. So an exception landing in the classification
+ stage is framework-side — in practice `emit_fn` raising while processing
+ a Start/End event. Emitter failures propagate at every other
+ `emit_event` call site, and synthesizing a bogus `tool_result` for one
+ would let the run continue with event processing broken; so it is
+ recorded and **re-raised after every task has settled** (no leaks, no
+ conversion). A framework failure takes precedence over a pending
+ suspend: durably suspending a run whose event pipeline is failing is not
+ safe.
3. **`HitlControlException`** — the suspend contract requires propagation, so
it cannot become an error result. Order of operations: first emit the
`ToolResultMessage`s for every *other* settled slot (each `MessageEndEvent`
checkpoints immediately via `Agent._process_event`, `agent.py:1207-1209`),
- then re-raise the HITL exception. The HITL call's own tool_call
+ then re-raise the HITL exception. This emission is deliberately **not**
+ best-effort: if a sibling's persistence fails, suspending anyway would
+ durably record a batch whose completed work is missing (resume would
+ re-run side-effecting tools), so the persistence failure propagates and
+ the run fails rather than suspends. The emitted sibling results are also
+ attached to the exception as
+ `HitlControlException.partial_tool_results`; the stateless loop entry
+ points (`_run_loop` / `run_agent_loop_resume` HITL handlers) append them
+ to the message lists they return, so callers persisting the loop's
+ return value — who never see the raised-through batch — keep the
+ completed siblings' results. (The sequential executor attaches the same
+ attribute for its already-emitted prefix.) The HITL call's own tool_call
deliberately stays unanswered and gets no End event — identical to today's
sequential detach shape — and the existing detach/resume/abort paths
answer it (synthetic-deny backfill scans unanswered ids of the last
diff --git a/tests/agent/test_loop.py b/tests/agent/test_loop.py
index 4763f67..f0ac49c 100644
--- a/tests/agent/test_loop.py
+++ b/tests/agent/test_loop.py
@@ -752,3 +752,51 @@ class TestStreamFallback:
]
assert len(message_starts) == 1
assert len(message_ends) == 1
+
+
+class TestLoopHitlPartialResults:
+ async def test_detach_returns_salvaged_sibling_results(self):
+ """A parallel batch that detaches mid-flight must not drop completed
+ siblings' tool_results from the messages the stateless loop returns —
+ callers persisting the return value would otherwise resume with
+ dangling tool_calls and re-run side-effecting work."""
+ from cubepi.hitl.exceptions import HitlDetached
+
+ async def detaching_execute(
+ tool_call_id, params, *, signal=None, on_update=None
+ ):
+ raise HitlDetached()
+
+ detach_tool = AgentTool(
+ name="detach",
+ description="Detaches for human input",
+ parameters=EchoParams,
+ execute=detaching_execute,
+ )
+ provider = FauxProvider(provider_id="faux")
+ provider.set_responses(
+ [
+ faux_assistant_message(
+ [
+ faux_tool_call("echo", {"value": "hi"}, id="t-ok"),
+ faux_tool_call("detach", {"value": "x"}, id="t-detach"),
+ ],
+ stop_reason="tool_use",
+ ),
+ ]
+ )
+ context = AgentContext(
+ system_prompt="", messages=[], tools=[make_echo_tool(), detach_tool]
+ )
+ messages = await run_agent_loop(
+ prompts=[make_user_message("go")],
+ context=context,
+ model=provider.model("faux-1"),
+ convert_to_llm=identity_converter,
+ emit=lambda e: None,
+ )
+
+ roles = [m.role for m in messages]
+ assert roles == ["user", "assistant", "tool_result"]
+ assert messages[2].tool_call_id == "t-ok"
+ assert "echoed: hi" in messages[2].content[0].text
diff --git a/tests/agent/test_tools.py b/tests/agent/test_tools.py
index 39fc8ca..1bf18f2 100644
--- a/tests/agent/test_tools.py
+++ b/tests/agent/test_tools.py
@@ -689,7 +689,7 @@ class TestParallelFaultIsolation:
]
)
events = []
- with pytest.raises(HitlDetached):
+ with pytest.raises(HitlDetached) as excinfo:
await execute_tool_calls(
ctx,
self._three_call_msg(),
@@ -697,6 +697,13 @@ class TestParallelFaultIsolation:
emit=lambda e: events.append(e),
)
+ # Persisted sibling results ride on the exception so the stateless
+ # loop entry points can return them to their callers.
+ assert [m.tool_call_id for m in excinfo.value.partial_tool_results] == [
+ "t1",
+ "t3",
+ ]
+
# Every sibling settled before the re-raise; nothing leaked.
assert side_effects == ["fast", "slow"]
pending = [
@@ -884,9 +891,10 @@ class TestFaultIsolationEdgeCases:
result_ids = [e.message.tool_call_id for e in events if e.type == "message_end"]
assert result_ids == ["t1"]
- async def test_hitl_reraise_survives_emit_failure(self):
- """Persisting sibling results is best-effort: an emit failure must
- not swallow the control exception the suspend machinery needs."""
+ async def test_hitl_sibling_persistence_failure_propagates(self):
+ """Suspending after a sibling result failed to persist would durably
+ record a batch missing completed work — the persistence failure must
+ win over the suspend."""
from cubepi.hitl.exceptions import HitlDetached
async def hitl_raiser(tool_call_id, params, *, signal=None, on_update=None):
@@ -894,7 +902,7 @@ class TestFaultIsolationEdgeCases:
def flaky_emit(event):
if event.type == "message_start":
- raise RuntimeError("listener blew up")
+ raise RuntimeError("checkpointer rejected the append")
ctx = make_context(
[
@@ -902,13 +910,66 @@ class TestFaultIsolationEdgeCases:
make_echo_tool(name="b", execute_fn=hitl_raiser),
]
)
- with pytest.raises(HitlDetached):
+ with pytest.raises(RuntimeError, match="checkpointer rejected"):
+ await execute_tool_calls(
+ ctx,
+ self._two_call_msg(),
+ tool_execution="parallel",
+ emit=flaky_emit,
+ )
+
+ async def test_worker_emit_failure_propagates_after_settle(self):
+ """emit_fn failures inside a worker are framework-side, not tool
+ failures: no bogus tool_result is synthesized, the exception
+ propagates — but only after every sibling has settled."""
+ side_effects = []
+
+ async def slow_ok(tool_call_id, params, *, signal=None, on_update=None):
+ await asyncio.sleep(0.03)
+ side_effects.append("slow settled")
+ return AgentToolResult(content=[TextContent(text="ok")])
+
+ def flaky_emit(event):
+ if event.type == "tool_execution_end" and event.tool_call_id == "t1":
+ raise RuntimeError("listener blew up")
+
+ ctx = make_context(
+ [
+ make_echo_tool(name="a"),
+ make_echo_tool(name="b", execute_fn=slow_ok),
+ ]
+ )
+ with pytest.raises(RuntimeError, match="listener blew up"):
await execute_tool_calls(
ctx,
self._two_call_msg(),
tool_execution="parallel",
emit=flaky_emit,
)
+ assert side_effects == ["slow settled"]
+
+ async def test_sequential_hitl_carries_partial_results(self):
+ """The sequential executor attaches already-emitted results to an
+ escaping HITL control exception too."""
+ from cubepi.hitl.exceptions import HitlDetached
+
+ async def hitl_raiser(tool_call_id, params, *, signal=None, on_update=None):
+ raise HitlDetached()
+
+ ctx = make_context(
+ [
+ make_echo_tool(name="a"),
+ make_echo_tool(name="b", execute_fn=hitl_raiser),
+ ]
+ )
+ with pytest.raises(HitlDetached) as excinfo:
+ await execute_tool_calls(
+ ctx,
+ self._two_call_msg(),
+ tool_execution="sequential",
+ emit=lambda e: None,
+ )
+ assert [m.tool_call_id for m in excinfo.value.partial_tool_results] == ["t1"]
async def test_cancel_reraise_survives_emit_failure(self):
"""Same best-effort contract on the outer-cancel salvage path."""