Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

fix(session): retain durable divergence state

xfgongc358358c1867
diff --git a/cubeloop/agent/agent.py b/cubeloop/agent/agent.py
index 692671d..cd4050e 100644
--- a/cubeloop/agent/agent.py
+++ b/cubeloop/agent/agent.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import copy
import time
import uuid
from collections.abc import Sequence
@@ -563,6 +564,7 @@ class Agent(Generic[TMessage]):
self._extra.clear()
self._extra.update(data.extra)
self._checkpoint_loaded = True
+ self.session._checkpoint_write_failed = False
await self._run_prompt(messages)
except BaseException:
@@ -925,6 +927,7 @@ class Agent(Generic[TMessage]):
from cubeloop.session import RespondExecutionRequest
self.session._assert_idle()
+ answer = copy.deepcopy(answer)
recovered_run_id: str | None = None
if self.checkpointer is not None and self.thread_id is not None:
load_pending = getattr(self.checkpointer, "load_pending", None)
@@ -980,6 +983,7 @@ class Agent(Generic[TMessage]):
self._extra.clear()
self._extra.update(data.extra or {})
self._checkpoint_loaded = True
+ self.session._checkpoint_write_failed = False
loaded = await load_pending(self.thread_id)
if loaded is None:
diff --git a/cubeloop/providers/base.py b/cubeloop/providers/base.py
index f40a354..bcee296 100644
--- a/cubeloop/providers/base.py
+++ b/cubeloop/providers/base.py
@@ -914,6 +914,10 @@ class BoundModel:
reasoning: ReasoningControl | None = None,
) -> AssistantMessage:
# Same positional-forwarding rationale as ``stream`` above.
+ if options is not None and options.on_model_attempt is not None:
+ result = options.on_model_attempt(self.spec)
+ if inspect.isawaitable(result):
+ await result
return await self.provider.generate(
self.spec,
messages,
diff --git a/cubeloop/session/session.py b/cubeloop/session/session.py
index e296b37..bb2daf4 100644
--- a/cubeloop/session/session.py
+++ b/cubeloop/session/session.py
@@ -141,6 +141,7 @@ class ExecutionSession:
self._agent._extra.update(extra)
self._agent._checkpoint_loaded = True
self._drop_memory_input_receipts()
+ self._checkpoint_write_failed = False
return data
def request_cancel(self) -> None:
@@ -306,7 +307,6 @@ class ExecutionSession:
self._accepting_input = True
self._seq = 0
self._delivery_errors = []
- self._checkpoint_write_failed = False
self._reconcile_consumed_inputs()
for input_id, status in tuple(self._input_status.items()):
if status != "queued":
@@ -687,7 +687,8 @@ class ExecutionSession:
error = ExecutionError(kind="execution", message=message)
checkpoint_committed = self._has_durable_checkpoint() and (
- outcome == "completed" or (outcome == "suspended" and pending_is_durable)
+ (outcome == "completed" and self._agent._run_aware)
+ or (outcome == "suspended" and pending_is_durable)
)
return ExecutionResult(
run_id=request.run_id,
diff --git a/tests/agent/test_execution_session.py b/tests/agent/test_execution_session.py
index c4fdbc9..428917b 100644
--- a/tests/agent/test_execution_session.py
+++ b/tests/agent/test_execution_session.py
@@ -309,26 +309,16 @@ async def test_respond_snapshots_mutable_answer_before_checkpoint_await() -> Non
assert channel.pending is not None
question_id = channel.pending.question_id
await agent.session.request_detach()
- await first
+ _ = await first
checkpointer.block_pending = True
answer = {"answer": "original"}
- resumed = asyncio.create_task(
- agent.session.execute(
- RespondExecutionRequest(
- run_id="run-1",
- attempt_id="attempt-2",
- question_id=question_id,
- answer=answer,
- )
- )
- )
+ resumed = asyncio.create_task(agent.respond(question_id=question_id, answer=answer))
await asyncio.wait_for(entered.wait(), timeout=1)
answer["answer"] = "mutated"
release.set()
- result = await resumed
+ _ = await resumed
- assert result.outcome == "completed"
assert len(observed) == 1
assert "original" in observed[0]
assert "mutated" not in observed[0]
@@ -526,7 +516,8 @@ async def test_checkpoint_message_write_failure_marks_history_inconsistent() ->
AssistantMessage(
content=[ToolCall(id="call-1", name="work", arguments={"value": "x"})],
stop_reason="tool_use",
- )
+ ),
+ _answer("next attempt"),
)
agent = Agent(
model=provider.model("faux-model"),
@@ -542,6 +533,39 @@ async def test_checkpoint_message_write_failure_marks_history_inconsistent() ->
assert result.outcome == "failed"
assert result.history_consistent is False
+ retry = await agent.session.execute(
+ PromptExecutionRequest(run_id="run-2", attempt_id="attempt-2", message="retry")
+ )
+ assert retry.outcome == "failed"
+ assert retry.history_consistent is False
+ assert retry.checkpoint_committed is False
+
+
+async def test_degraded_checkpointer_does_not_report_completed_commitment() -> None:
+ class DegradedCheckpointer:
+ async def load(self, thread_id):
+ return None
+
+ async def append(self, thread_id, messages):
+ return None
+
+ async def save_extra(self, thread_id, extra):
+ return None
+
+ agent = Agent(
+ model=_provider(_answer()).model("faux-model"),
+ checkpointer=DegradedCheckpointer(),
+ thread_id="thread-1",
+ )
+
+ result = await agent.session.execute(
+ PromptExecutionRequest(run_id="run-1", attempt_id="attempt-1", message="hi")
+ )
+
+ assert result.outcome == "completed"
+ assert result.checkpoint_committed is False
+
@pytest.mark.asyncio
async def test_agent_prompt_and_session_share_one_admission_gate() -> None:
@@ -1039,6 +1063,6 @@ async def test_reconciled_input_process_control_releases_session_ownership() ->
release.set()
with pytest.raises(ProcessExit, match="stop host"):
- await task
+ _ = await task
assert agent.session.active_attempt_id is None
await asyncio.wait_for(agent.wait_for_idle(), timeout=1)
diff --git a/tests/providers/test_bound_model_calls.py b/tests/providers/test_bound_model_calls.py
index a5f0d2a..c941174 100644
--- a/tests/providers/test_bound_model_calls.py
+++ b/tests/providers/test_bound_model_calls.py
@@ -83,6 +83,23 @@ async def test_bound_model_generate_forwards_to_provider() -> None:
assert call["reasoning"] == ReasoningControl(mode="off", effort="minimal")
+async def test_bound_model_generate_invokes_model_attempt_callback() -> None:
+ provider = _RecordingProvider()
+ bound = provider.model("model-x")
+ attempts: list[Model] = []
+
+ async def capture(model: Model) -> None:
+ attempts.append(model)
+
+ await bound.generate(
+ messages=[UserMessage(content=[TextContent(text="hi")])],
+ options=StreamOptions(on_model_attempt=capture),
+ )
+
+ assert attempts == [bound.spec]
+
+
@pytest.mark.asyncio
async def test_bound_model_stream_forwards_to_provider() -> None:
provider = FauxProvider(provider_id="faux")
diff --git a/tests/providers/test_fallback.py b/tests/providers/test_fallback.py
index 55f0c60..217bb9f 100644
--- a/tests/providers/test_fallback.py
+++ b/tests/providers/test_fallback.py
@@ -647,6 +647,34 @@ async def test_same_model_retry_recovers_without_secondary() -> None:
assert retries == [1, 2]
+async def test_generate_reports_every_retry_and_fallback_model_attempt() -> None:
+ error = ProviderUnavailable("blip", provider="primary", model="m")
+ primary_provider = _CountingRaiseProvider(
+ error, fail_times=99, provider_id="primary"
+ )
+ primary = BoundModel(
+ provider=primary_provider,
+ spec=Model(id="m", provider_id="primary"),
+ )
+ fallback = _faux("fallback", "ok")
+ attempts: list[str] = []
+ model = FallbackBoundModel(
+ chain=(primary, fallback),
+ max_retries_per_model=1,
+ retry_backoff=0.0,
+ )
+
+ await model.generate(
+ _messages(),
+ options=StreamOptions(
+ on_model_attempt=lambda spec: attempts.append(spec.provider_id)
+ ),
+ )
+
+ assert attempts == ["primary", "primary", "fallback"]
+
+
@pytest.mark.asyncio
async def test_retries_then_failover() -> None:
err = RateLimited("429", provider="primary", model="m")