Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

fix(session): close remaining lifecycle races

xfgongf6fd4c5fa4ff
diff --git a/cubeloop/agent/agent.py b/cubeloop/agent/agent.py
index 33f27b8..3314349 100644
--- a/cubeloop/agent/agent.py
+++ b/cubeloop/agent/agent.py
@@ -887,6 +887,7 @@ class Agent(Generic[TMessage]):
run_id=self._state.active_run_id,
attempt_id=self.session.active_attempt_id,
on_turn_context=self.session._capture_turn_context,
+ set_input_admission=self.session._set_input_admission,
)
async def detach(self) -> None:
@@ -921,6 +922,7 @@ class Agent(Generic[TMessage]):
) -> None:
from cubeloop.session import RespondExecutionRequest
+ self.session._assert_idle()
recovered_run_id: str | None = None
if self.checkpointer is not None and self.thread_id is not None:
load_pending = getattr(self.checkpointer, "load_pending", None)
diff --git a/cubeloop/agent/loop.py b/cubeloop/agent/loop.py
index 1af939a..76f21c0 100644
--- a/cubeloop/agent/loop.py
+++ b/cubeloop/agent/loop.py
@@ -67,6 +67,7 @@ async def run_agent_loop(
run_id=context.run_id,
attempt_id=context.attempt_id,
on_turn_context=context.on_turn_context,
+ set_input_admission=context.set_input_admission,
)
await emit_event(emit, AgentStartEvent())
@@ -133,6 +134,7 @@ async def run_agent_loop_continue(
run_id=context.run_id,
attempt_id=context.attempt_id,
on_turn_context=context.on_turn_context,
+ set_input_admission=context.set_input_admission,
)
await emit_event(emit, AgentStartEvent())
@@ -379,6 +381,8 @@ async def _run_agent_loop_resume_body( # pragma: no cover — E2E tested
new_messages=new_messages,
)
if await should_stop_after_turn(stop_ctx):
+ if current_context.set_input_admission is not None:
+ current_context.set_input_admission(False)
await emit_event(emit, AgentEndEvent(messages=new_messages))
if set_outcome is not None:
set_outcome("complete")
@@ -386,6 +390,8 @@ async def _run_agent_loop_resume_body( # pragma: no cover — E2E tested
# Terminate-by-tool semantics (codex BLOCKING: previous draft ignored).
if terminated_by_tool:
+ if current_context.set_input_admission is not None:
+ current_context.set_input_admission(False)
await emit_event(emit, AgentEndEvent(messages=new_messages))
if set_outcome is not None:
set_outcome("complete")
@@ -542,6 +548,8 @@ async def _run_loop_inner(
# Emit message_end before turn/agent end events.
await emit_event(emit, MessageEndEvent(message=message))
await emit_event(emit, TurnEndEvent(message=message, tool_results=[]))
+ if current_context.set_input_admission is not None:
+ current_context.set_input_admission(False)
await emit_event(emit, AgentEndEvent(messages=new_messages))
if set_outcome is not None:
set_outcome("abandoned")
@@ -705,11 +713,15 @@ async def _run_loop_inner(
continue
if on_run_end:
+ if current_context.set_input_admission is not None:
+ current_context.set_input_admission(False)
inject = await on_run_end(current_context, signal=opts.signal)
if inject:
on_run_end_cycles += 1
if on_run_end_cycles > _MAX_ON_RUN_END_CYCLES:
break
+ if current_context.set_input_admission is not None:
+ current_context.set_input_admission(True)
for msg in inject:
await emit_event(emit, MessageStartEvent(message=msg))
await emit_event(emit, MessageEndEvent(message=msg))
@@ -718,8 +730,14 @@ async def _run_loop_inner(
first_turn = False
continue
+ else:
+ if current_context.set_input_admission is not None:
+ current_context.set_input_admission(False)
+
break
+ if current_context.set_input_admission is not None:
+ current_context.set_input_admission(False)
await emit_event(emit, AgentEndEvent(messages=new_messages))
if set_outcome is not None:
set_outcome("complete")
diff --git a/cubeloop/agent/tools.py b/cubeloop/agent/tools.py
index b1d1880..7b1708c 100644
--- a/cubeloop/agent/tools.py
+++ b/cubeloop/agent/tools.py
@@ -11,6 +11,7 @@ from pydantic import BaseModel
from pydantic import ValidationError
from cubeloop.hitl.exceptions import HitlControlException
+from cubeloop.session.turn_execution_context import ToolExecutionBinding
from cubeloop.agent.types import (
AfterToolCallContext,
@@ -43,7 +44,7 @@ class ToolCallBatch:
@dataclass
class _PreparedToolCall:
tool_call: ToolCall
- tool: AgentTool
+ tool: AgentTool | ToolExecutionBinding
args: BaseModel | JsonObject
hitl_trace: StructuredObject | None = None
@@ -207,7 +208,7 @@ async def _prepare_tool_call(
binding = (
turn_context.binding_for(tool_call.name) if turn_context is not None else None
)
- tool = binding.tool if binding is not None else None
+ tool: AgentTool | ToolExecutionBinding | None = binding
if tool is None and context.tools:
for t in context.tools:
if t.name == tool_call.name:
diff --git a/cubeloop/agent/types.py b/cubeloop/agent/types.py
index b5990d1..7896478 100644
--- a/cubeloop/agent/types.py
+++ b/cubeloop/agent/types.py
@@ -64,6 +64,7 @@ class AgentContext:
turn_id: str | None = None
turn_execution_context: Any = None
on_turn_context: Callable[[Any], Any] | None = None
+ set_input_admission: Callable[[bool], None] | None = None
# --- Hook context types ---
diff --git a/cubeloop/session/session.py b/cubeloop/session/session.py
index d354a10..b6eb4ec 100644
--- a/cubeloop/session/session.py
+++ b/cubeloop/session/session.py
@@ -186,8 +186,23 @@ class ExecutionSession:
)
if not self._accepting_input:
return InputReceipt(input_id=envelope.input_id, status="closed")
- if len(self._input_status) >= _INPUT_DEDUP_CAPACITY:
+ if (
+ envelope.message.run_id is not None
+ and envelope.message.run_id != self._active_run_id
+ ):
return InputReceipt(input_id=envelope.input_id, status="closed")
+ if len(self._input_status) >= _INPUT_DEDUP_CAPACITY:
+ terminal_id = next(
+ (
+ input_id
+ for input_id, status in self._input_status.items()
+ if status != "queued"
+ ),
+ None,
+ )
+ if terminal_id is None:
+ return InputReceipt(input_id=envelope.input_id, status="closed")
+ del self._input_status[terminal_id]
metadata = dict(envelope.message.metadata)
metadata["input_id"] = envelope.input_id
@@ -226,8 +241,7 @@ class ExecutionSession:
async def execute(self, request: ExecutionRequest) -> ExecutionResult:
"""Execute one attempt and return settled lifecycle facts."""
- if self._active_attempt_id is not None or self._agent._run_lock.locked():
- raise ExecutionBusy("execution session is already processing an attempt")
+ self._assert_idle()
self._active_attempt_id = request.attempt_id
self._active_run_id = request.run_id
@@ -236,7 +250,11 @@ class ExecutionSession:
self._seq = 0
self._delivery_errors = []
self._required_delivery_failure = None
- self._input_status.clear()
+ self._input_status = {
+ input_id: status
+ for input_id, status in self._input_status.items()
+ if status != "queued"
+ }
self._turn_contexts.clear()
caught: BaseException | None = None
try:
@@ -264,6 +282,8 @@ class ExecutionSession:
try:
try:
result = await self._settle(request, caught)
+ except asyncio.CancelledError as exc:
+ result = self._cancelled_result(request, exc)
except Exception as exc:
result = ExecutionResult(
run_id=request.run_id,
@@ -278,9 +298,17 @@ class ExecutionSession:
history_consistent=False,
delivery_errors=tuple(self._delivery_errors),
)
- final_errors = await self._publish(
- ExecutionFinished(result=result), terminal=True
+ publish_task = asyncio.create_task(
+ self._publish(ExecutionFinished(result=result), terminal=True)
)
+ while True:
+ try:
+ final_errors = await asyncio.shield(publish_task)
+ break
+ except asyncio.CancelledError:
+ if publish_task.done():
+ final_errors = publish_task.result()
+ break
if final_errors:
result = replace(
result,
@@ -291,6 +319,32 @@ class ExecutionSession:
self._active_attempt_id = None
self._active_run_id = None
+ def _assert_idle(self) -> None:
+ if self._active_attempt_id is not None or self._agent._run_lock.locked():
+ raise ExecutionBusy("execution session is already processing an attempt")
+
+ def _set_input_admission(self, accepting: bool) -> None:
+ self._accepting_input = accepting
+
+ def _cancelled_result(
+ self,
+ request: ExecutionRequest,
+ cause: asyncio.CancelledError,
+ ) -> ExecutionResult:
+ return ExecutionResult(
+ run_id=request.run_id,
+ attempt_id=request.attempt_id,
+ outcome="cancelled",
+ error=ExecutionError(
+ kind="cancelled",
+ message="execution cancelled",
+ cause=cause,
+ ),
+ checkpoint_committed=False,
+ history_consistent=False,
+ delivery_errors=tuple(self._delivery_errors),
+ )
+
async def _publish_agent_event(self, event: SessionEvent) -> None:
if self._active_attempt_id is None:
return
diff --git a/cubeloop/session/turn_execution_context.py b/cubeloop/session/turn_execution_context.py
index c6d0b3d..40fbc4e 100644
--- a/cubeloop/session/turn_execution_context.py
+++ b/cubeloop/session/turn_execution_context.py
@@ -1,10 +1,13 @@
from __future__ import annotations
-from dataclasses import dataclass, replace
+from dataclasses import dataclass, field, replace
from types import MappingProxyType
+from collections.abc import Awaitable, Callable
from typing import Any, Mapping
-from cubeloop.agent.types import AgentTool
+from pydantic import BaseModel
+
+from cubeloop.agent.types import AgentTool, AgentToolResult
from cubeloop.providers.base import Message, Model, ReasoningControl
@@ -64,7 +67,21 @@ class MessageView:
class ToolExecutionBinding:
name: str
definition: FrozenObject
- tool: AgentTool
+ parameters: type[BaseModel]
+ execute: Callable[..., Awaitable[AgentToolResult]]
+ hitl_builtin: bool
+ _source_id: int = field(repr=False)
+
+ @classmethod
+ def capture(cls, tool: AgentTool) -> ToolExecutionBinding:
+ return cls(
+ name=tool.name,
+ definition=_freeze(tool.to_definition()),
+ parameters=tool.parameters,
+ execute=tool.execute,
+ hitl_builtin=tool.hitl_builtin,
+ _source_id=id(tool),
+ )
@dataclass(frozen=True)
@@ -101,14 +118,7 @@ class TurnExecutionContext:
reasoning=_freeze(reasoning),
system_prompt=system_prompt,
messages=tuple(MessageView.capture(message) for message in messages),
- tools=tuple(
- ToolExecutionBinding(
- name=tool.name,
- definition=_freeze(tool.to_definition()),
- tool=tool,
- )
- for tool in tools or []
- ),
+ tools=tuple(ToolExecutionBinding.capture(tool) for tool in tools or []),
policy_revision=policy_revision,
)
@@ -118,14 +128,10 @@ class TurnExecutionContext:
def extend(self, tool: AgentTool) -> TurnExecutionContext:
existing = self.binding_for(tool.name)
if existing is not None:
- if existing.tool is not tool:
+ if existing._source_id != id(tool):
raise ValueError(
f"turn execution context already binds tool {tool.name!r}"
)
return self
- binding = ToolExecutionBinding(
- name=tool.name,
- definition=_freeze(tool.to_definition()),
- tool=tool,
- )
+ binding = ToolExecutionBinding.capture(tool)
return replace(self, tools=(*self.tools, binding))
diff --git a/tests/agent/test_execution_session.py b/tests/agent/test_execution_session.py
index 4a41069..db3475b 100644
--- a/tests/agent/test_execution_session.py
+++ b/tests/agent/test_execution_session.py
@@ -92,6 +92,71 @@ async def test_settlement_failure_returns_result_and_terminal_event() -> None:
assert finished[0].event.result == result
+async def test_cancellation_during_settlement_still_publishes_terminal_event() -> None:
+ entered = asyncio.Event()
+
+ class SlowPendingCheckpointer(MemoryCheckpointer):
+ async def load_pending(self, thread_id: str):
+ del thread_id
+ entered.set()
+ await asyncio.Future()
+
+ agent = Agent(
+ model=_provider(_answer()).model("faux-model"),
+ checkpointer=SlowPendingCheckpointer(),
+ thread_id="thread-1",
+ )
+ events = []
+ agent.session.subscribe(lambda event: events.append(event))
+ task = asyncio.create_task(
+ agent.session.execute(
+ PromptExecutionRequest(run_id="run-1", attempt_id="attempt-1", message="hi")
+ )
+ )
+ await asyncio.wait_for(entered.wait(), timeout=1)
+
+ task.cancel()
+ result = await task
+
+ assert result.outcome == "cancelled"
+ finished = [event for event in events if event.event.type == "execution_finished"]
+ assert len(finished) == 1
+ assert finished[0].event.result == result
+
+
+async def test_cancellation_during_terminal_delivery_does_not_drop_terminal() -> None:
+ terminal_entered = asyncio.Event()
+ release_terminal = asyncio.Event()
+ events = []
+
+ async def listener(envelope):
+ if envelope.event.type == "execution_finished":
+ terminal_entered.set()
+ await release_terminal.wait()
+ events.append(envelope)
+
+ agent = Agent(model=_provider(_answer()).model("faux-model"))
+ agent.session.subscribe(listener)
+ task = asyncio.create_task(
+ agent.session.execute(
+ PromptExecutionRequest(run_id="run-1", attempt_id="attempt-1", message="hi")
+ )
+ )
+ await asyncio.wait_for(terminal_entered.wait(), timeout=1)
+
+ task.cancel()
+ release_terminal.set()
+ result = await task
+
+ assert result.outcome == "completed"
+ assert (
+ len([event for event in events if event.event.type == "execution_finished"])
+ == 1
+ )
+
+
@pytest.mark.asyncio
async def test_respond_rejects_request_run_mismatch_before_resume() -> None:
checkpointer = MemoryCheckpointer()
@@ -160,6 +225,44 @@ async def test_agent_prompt_and_session_share_one_admission_gate() -> None:
assert [m.run_id for m in agent.state.messages] == ["run-1", "run-1"]
+async def test_agent_respond_checks_admission_before_loading_pending() -> None:
+ entered = asyncio.Event()
+ release = asyncio.Event()
+
+ class CountingCheckpointer(MemoryCheckpointer):
+ load_pending_calls = 0
+
+ async def load_pending(self, thread_id: str):
+ self.load_pending_calls += 1
+ return await super().load_pending(thread_id)
+
+ async def slow_response(messages, model):
+ del messages, model
+ entered.set()
+ await release.wait()
+ return _answer()
+
+ provider = _provider()
+ provider.set_responses([slow_response])
+ checkpointer = CountingCheckpointer()
+ agent = Agent(
+ model=provider.model("faux-model"),
+ checkpointer=checkpointer,
+ thread_id="thread-1",
+ )
+ active = asyncio.create_task(agent.prompt("hi", run_id="run-1"))
+ await asyncio.wait_for(entered.wait(), timeout=1)
+ calls_before = checkpointer.load_pending_calls
+
+ with pytest.raises(ExecutionBusy):
+ await agent.respond(answer=True)
+
+ assert checkpointer.load_pending_calls == calls_before
+ release.set()
+ await active
+
+
@pytest.mark.asyncio
async def test_load_checkpoint_restores_extra_in_place_even_without_messages() -> None:
checkpointer = MemoryCheckpointer()
diff --git a/tests/agent/test_session_input.py b/tests/agent/test_session_input.py
index d3dae19..6f6f2dc 100644
--- a/tests/agent/test_session_input.py
+++ b/tests/agent/test_session_input.py
@@ -234,3 +234,130 @@ async def test_session_input_id_overrides_caller_steering_key() -> None:
message.metadata.get("input_id") == "owned-id"
for message in agent.state.messages
)
+
+
+async def test_input_admission_closes_before_final_on_run_end_hook() -> None:
+ entered = asyncio.Event()
+ release = asyncio.Event()
+
+ async def on_run_end(context, signal=None):
+ del context, signal
+ entered.set()
+ await release.wait()
+
+ provider = FauxProvider(provider_id="faux")
+ provider.set_responses(
+ [AssistantMessage(content=[TextContent(text="done")], stop_reason="end_turn")]
+ )
+ agent = Agent(model=provider.model("faux-model"), on_run_end=on_run_end)
+ task = asyncio.create_task(agent.prompt("hi", run_id="run-1"))
+ await asyncio.wait_for(entered.wait(), timeout=1)
+
+ receipt = agent.session.submit_input(
+ InputEnvelope(
+ input_id="too-late",
+ message=UserMessage(content=[TextContent(text="late")]),
+ mode="follow_up",
+ )
+ )
+
+ assert receipt.status == "closed"
+ release.set()
+ await task
+
+
+async def test_committed_input_id_remains_deduplicated_across_attempts() -> None:
+ first_entered = asyncio.Event()
+ release_first = asyncio.Event()
+ second_entered = asyncio.Event()
+ release_second = asyncio.Event()
+
+ async def first_response(messages, model):
+ del messages, model
+ first_entered.set()
+ await release_first.wait()
+ return AssistantMessage(
+ content=[TextContent(text="first")], stop_reason="end_turn"
+ )
+
+ async def second_attempt(messages, model):
+ del messages, model
+ second_entered.set()
+ await release_second.wait()
+ return AssistantMessage(
+ content=[TextContent(text="second")], stop_reason="end_turn"
+ )
+
+ provider = FauxProvider(provider_id="faux")
+ provider.set_responses(
+ [
+ first_response,
+ AssistantMessage(
+ content=[TextContent(text="after input")], stop_reason="end_turn"
+ ),
+ second_attempt,
+ ]
+ )
+ agent = Agent(model=provider.model("faux-model"))
+ first = asyncio.create_task(agent.prompt("first", run_id="run-1"))
+ await asyncio.wait_for(first_entered.wait(), timeout=1)
+ envelope = InputEnvelope(
+ input_id="stable-id",
+ message=UserMessage(content=[TextContent(text="once")]),
+ mode="follow_up",
+ )
+ assert agent.session.submit_input(envelope).status == "queued"
+ release_first.set()
+ await first
+
+ second = asyncio.create_task(agent.prompt("second", run_id="run-2"))
+ await asyncio.wait_for(second_entered.wait(), timeout=1)
+ duplicate = agent.session.submit_input(envelope)
+
+ assert duplicate.status == "committed"
+ assert duplicate.durability == "memory"
+ release_second.set()
+ await second
+ assert (
+ sum(
+ message.metadata.get("input_id") == "stable-id"
+ for message in agent.state.messages
+ )
+ == 1
+ )
+
+
+async def test_input_with_mismatched_run_id_is_rejected_before_queueing() -> None:
+ entered = asyncio.Event()
+ release = asyncio.Event()
+
+ async def response(messages, model):
+ del messages, model
+ entered.set()
+ await release.wait()
+ return AssistantMessage(
+ content=[TextContent(text="done")], stop_reason="end_turn"
+ )
+
+ provider = FauxProvider(provider_id="faux")
+ provider.set_responses([response])
+ agent = Agent(model=provider.model("faux-model"))
+ task = asyncio.create_task(agent.prompt("hi", run_id="run-1"))
+ await asyncio.wait_for(entered.wait(), timeout=1)
+
+ receipt = agent.session.submit_input(
+ InputEnvelope(
+ input_id="wrong-run",
+ message=UserMessage(
+ content=[TextContent(text="wrong")], run_id="run-other"
+ ),
+ mode="steer",
+ )
+ )
+
+ assert receipt.status == "closed"
+ release.set()
+ await task
diff --git a/tests/agent/test_turn_execution_context.py b/tests/agent/test_turn_execution_context.py
index 7304dc4..0c1e4c2 100644
--- a/tests/agent/test_turn_execution_context.py
+++ b/tests/agent/test_turn_execution_context.py
@@ -25,6 +25,10 @@ class _Args(BaseModel):
value: str
+class _ReplacementArgs(BaseModel):
+ replacement: str
+
+
@pytest.mark.asyncio
async def test_context_captures_transformed_request_as_immutable_view() -> None:
provider = FauxProvider(provider_id="faux")
@@ -102,6 +106,8 @@ async def test_tool_call_uses_binding_captured_for_the_model_request() -> None:
async def replace_after_response(message, context, signal=None):
del message, signal
+ old_tool.parameters = _ReplacementArgs
+ old_tool.execute = new_execute
context.tools = [new_tool]
agent = Agent(