Skip to content

Self-improving AI Employees for everyone

fix(temporal): drain Pi receipt checkpoints before nested Stop

trohitgbd0d7acc9ebd
diff --git a/server/services/temporal/pi_workflow.py b/server/services/temporal/pi_workflow.py
index 51c55eda..e72129f4 100644
--- a/server/services/temporal/pi_workflow.py
+++ b/server/services/temporal/pi_workflow.py
@@ -7,13 +7,11 @@ SDK maintenance never occupies a business action or a child concurrency permit.
from __future__ import annotations
-import asyncio
from datetime import timedelta
from typing import Any
from temporalio import workflow
from temporalio.common import RetryPolicy
-from temporalio.exceptions import ApplicationError
from temporalio.workflow import ActivityCancellationType
from .agent_workflow import AgentWorkflow, LLM_STEP_TIMEOUT, LLM_STEP_HEARTBEAT_TIMEOUT, LLM_STEP_RETRY
@@ -43,6 +41,7 @@ class PiCodingAgentWorkflow(AgentWorkflow):
self._pi_detached = False
self._pi_checkpoint = (context or {}).get("_pi_checkpoint")
self._pi_checkpointing = False
+ self._pi_receipt_checkpointing = False
self._pi_ready = False
self._pi_validations: dict[str, dict] = {}
self._pi_stop_pending = False
@@ -53,10 +52,10 @@ class PiCodingAgentWorkflow(AgentWorkflow):
if status["state"] == "paused" or status["producers_held"]:
self._pi_stop_pending = True
terminal = await self._stop_driver()
- if (terminal or {}).get("status") == "completed" and self._pi_checkpointing:
- # A DB snapshot write can finish concurrently with this Update.
- # Let the main task accept that already completed round before
- # fencing its epoch; no paid operation is admitted here.
+ if self._pi_checkpointing and ((terminal or {}).get("status") == "completed" or self._pi_receipt_checkpointing):
+ # An older driver can park concurrently with round completion.
+ # Drain its receipt-only reconstruction before fencing the epoch;
+ # no paid operation is admitted by that reconstruction.
await workflow.wait_condition(lambda: not self._pi_checkpointing)
self._pi.epoch += 1
return {**self._execution_control.status(), "pi_driver_checkpoint": self._pi_driver is None,
@@ -150,6 +149,7 @@ class PiCodingAgentWorkflow(AgentWorkflow):
# occupying private capacity while native cleanup drains children.
await self._stop_driver(cancel=True)
self._pi_checkpointing = False
+ self._pi_receipt_checkpointing = False
async def _clear_failed_skills(self, context):
await workflow.execute_activity("agent.skill.clear", {"workflow_id": context.get("workflow_id"),
@@ -167,7 +167,7 @@ class PiCodingAgentWorkflow(AgentWorkflow):
return fields
async def _start_driver(self) -> None:
- if self._pi_stop_pending:
+ if self._pi_stop_pending and not self._pi_receipt_checkpointing:
await self._wait_until_resumed()
self._pi_stop_pending = False
if self._pi_driver is not None:
@@ -320,7 +320,12 @@ class PiCodingAgentWorkflow(AgentWorkflow):
wires = await self._collect_tool_receipts(messages)
if len(wires) != len(self._pi_calls):
raise conflict("Pi complete turn is missing a tool receipt")
+ expected = set(self._pi.expected)
+ receipt_checkpoint = workflow.patched("pi-paused-receipt-checkpoint-v1")
+ if receipt_checkpoint and any((self._pi.operations.get(identity) or {}).get("status") != "completed" for identity in expected):
+ raise conflict("Pi checkpoint did not consume every recorded operation")
self._pi_checkpointing = True
+ self._pi_receipt_checkpointing = receipt_checkpoint
# Completion is bookkeeping, not a new paid action. It must also run
# while paused so admitted results are saved before a checkpoint parks.
while True:
@@ -337,10 +342,14 @@ class PiCodingAgentWorkflow(AgentWorkflow):
self._pi_driver = None
if result.get("status") == "completed":
break
- self._pi_checkpointing = False
- await self._wait_until_resumed()
- self._pi_checkpointing = True
- expected = set(self._pi.expected)
+ # Stop may already have told this driver to park before completion
+ # began. All effects now have receipts, so reconstruct the checkpoint
+ # while admission is closed instead of holding an action until Resume.
+ if not receipt_checkpoint:
+ # Histories recorded before the fix waited for Resume here.
+ self._pi_checkpointing = False
+ await self._wait_until_resumed()
+ self._pi_checkpointing = True
if result.get("epoch") != self._pi.epoch or result.get("scope") != self._pi_request["scope"] or result.get("turn") != self._pi.turn or result.get("config_digest") != self._pi_request["config_digest"] or result.get("binding_digest") != self._pi_request["binding_digest"]:
raise conflict("Pi checkpoint belongs to a stale driver or different configuration")
consumed = result.get("consumed_operations")
@@ -366,6 +375,7 @@ class PiCodingAgentWorkflow(AgentWorkflow):
self._pi_request = {}
self._pi.parked = True
self._pi_checkpointing = False
+ self._pi_receipt_checkpointing = False
async def _before_runtime_rollover(self):
if self._pi_driver is not None or self._pi_request or self._pi.operations or self._pi.expected or self._pi_checkpointing:
diff --git a/server/tests/temporal/test_pi_checkpoint_control.py b/server/tests/temporal/test_pi_checkpoint_control.py
new file mode 100644
index 00000000..0e15fb97
--- /dev/null
+++ b/server/tests/temporal/test_pi_checkpoint_control.py
@@ -0,0 +1,173 @@
+"""Deterministic driver-park ordering during an admitted Pi round's completion."""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from services.temporal import pi_workflow
+from services.temporal.pi_protocol import canonical_digest
+
+
+async def until(predicate):
+ async with asyncio.timeout(2):
+ while not predicate():
+ await asyncio.sleep(0)
+
+
[email protected]("inside_model_action", [True, False])
+async def test_parked_driver_reconstructs_receipts_before_stop_without_resume(monkeypatch, inside_model_action):
+ """The old driver parks after completion starts, with only one SDK slot."""
+ value = pi_workflow.PiCodingAgentWorkflow({"execution_control_revision": 0})
+ control = value._execution_control
+ scope = {"workflow_id": "agent", "generation_id": "0", "node_id": "coding"}
+ value._pi_request = {"scope": scope, "turn": 0, "messages": [],
+ "context_digest": canonical_digest([]), "config_digest": "config", "binding_digest": "bindings"}
+ identity = "turn:0:model"
+ payload = value._operation_payload(purpose="assistant")
+ value._pi.begin(0, {identity: {"kind": "model", "payload": payload}})
+ value._pi.register(0)
+ value._pi.submit({"id": identity, "kind": "model", "payload": {**payload, "epoch": 1}})
+ value._pi.complete(identity, {"kind": "final", "content": "done"})
+ messages = [{"role": "assistant", "content": "done"}]
+ old_driver = asyncio.get_running_loop().create_future()
+ value._pi_driver = old_driver
+ receipt_started = asyncio.Event()
+ release_receipt = asyncio.Event()
+ checkpoint_waiting = asyncio.Event()
+ driver_counts = {"active": 1, "max": 1, "starts": 0}
+ driver_tasks = []
+
+ async def wait_condition(predicate):
+ checkpoint_waiting.set()
+ await until(predicate)
+
+ async def receipt_driver():
+ try:
+ epoch = (await value.register_driver({"turn": 0}))["epoch"]
+ # Reconstruction attaches to a durable result; it cannot request a
+ # second model effect or a previously unadmitted tool position.
+ admitted = await value.submit_operation({"id": identity, "kind": "model",
+ "payload": {**payload, "epoch": epoch}})
+ assert admitted["status"] == "completed"
+ assert value.operation_status({"id": identity, "epoch": epoch})["result"] == {
+ "kind": "final", "content": "done"}
+ assert control.state == "paused" and value._pi_receipt_checkpointing
+ receipt_started.set()
+ await release_receipt.wait()
+ return {"status": "completed", "epoch": epoch, "scope": scope, "turn": 0,
+ "config_digest": "config", "binding_digest": "bindings", "consumed_operations": [identity],
+ "projection_digest": canonical_digest(messages), "snapshot": {"pi_blob": "saved-round"}}
+ finally:
+ driver_counts["active"] -= 1
+
+ def start_activity(name, request, **options):
+ assert name == "coding.pi_drive_turn" and request is value._pi_request
+ assert old_driver.done() and driver_counts["active"] == 0
+ assert control.state == "paused"
+ driver_counts["active"] += 1
+ driver_counts["max"] = max(driver_counts["max"], driver_counts["active"])
+ driver_counts["starts"] += 1
+ task = asyncio.create_task(receipt_driver())
+ driver_tasks.append(task)
+ return task
+
+ monkeypatch.setattr(pi_workflow.workflow, "patched", lambda _: True)
+ monkeypatch.setattr(pi_workflow.workflow, "wait_condition", wait_condition)
+ monkeypatch.setattr(pi_workflow.workflow, "start_activity", start_activity)
+ if inside_model_action:
+ await control.begin_action()
+ await control.set_control_state({"state": "paused", "revision": 1})
+
+ async def complete_round():
+ try:
+ await value._complete_runtime_turn(messages)
+ finally:
+ if inside_model_action:
+ control.end_action()
+
+ completion = asyncio.create_task(complete_round())
+ stop = next_action = None
+ try:
+ await until(lambda: value._pi_checkpointing)
+ assert not old_driver.done(), "Force the parked result after completion entered its await"
+ stop = asyncio.create_task(value.wait_for_checkpoint({"state": "paused", "revision": 1}))
+ if inside_model_action:
+ await asyncio.wait_for(checkpoint_waiting.wait(), 2)
+ else:
+ await until(lambda: value._pi_stop_pending and value._pi.parked)
+ assert not stop.done()
+ driver_counts["active"] -= 1
+ old_driver.set_result({"status": "parked"})
+ await asyncio.wait_for(receipt_started.wait(), 2)
+ assert value._pi_checkpointing and not stop.done()
+ next_action = asyncio.create_task(control.begin_action())
+ await asyncio.sleep(0)
+ assert not next_action.done(), "Receipt reconstruction must keep business admission closed"
+ release_receipt.set()
+ await asyncio.wait_for(completion, 2)
+ stopped = await asyncio.wait_for(stop, 2)
+ assert stopped["checkpoint"] and stopped["pi_driver_checkpoint"] and stopped["state"] == "paused"
+ assert stopped["pi_driver_epoch"] == 3
+ assert value.operation_status({"id": identity, "epoch": 2})["status"] == "park"
+ assert value._pi_context["_pi_checkpoint"]["reference"] == {"pi_blob": "saved-round"}
+ assert value._pi_stop_pending and not value._pi_checkpointing
+ assert driver_counts == {"active": 0, "max": 1, "starts": 1}
+ assert not next_action.done()
+ await control.set_control_state({"state": "running", "revision": 2, "producers_held": False})
+ await asyncio.wait_for(next_action, 2)
+ control.end_action()
+ assert control.status()["checkpoint"]
+ finally:
+ tasks = [completion, stop, next_action, *driver_tasks]
+ for task in tasks:
+ if task is not None and not task.done():
+ task.cancel()
+ await asyncio.gather(*(task for task in tasks if task is not None), return_exceptions=True)
+
+
+async def test_unpatched_history_keeps_parked_driver_resume_boundary(monkeypatch):
+ value = pi_workflow.PiCodingAgentWorkflow({"execution_control_revision": 0})
+ value._pi_request = {"scope": {"workflow_id": "agent"}, "messages": [],
+ "config_digest": "config", "binding_digest": "bindings"}
+ value._pi.begin(0, {})
+ value._pi.register(0)
+ old_driver = asyncio.get_running_loop().create_future()
+ value._pi_driver = old_driver
+ starts = []
+ resumed = asyncio.Event()
+
+ async def wait_condition(predicate):
+ await until(predicate)
+
+ def start_activity(name, request, **options):
+ assert resumed.is_set() and value._execution_control.state == "running"
+ starts.append(name)
+ result = asyncio.get_running_loop().create_future()
+ result.set_result({"status": "completed", "epoch": 1, "scope": request["scope"], "turn": 0,
+ "config_digest": "config", "binding_digest": "bindings", "consumed_operations": [],
+ "projection_digest": canonical_digest([]), "snapshot": {"pi_blob": "legacy-round"}})
+ return result
+
+ monkeypatch.setattr(pi_workflow.workflow, "patched", lambda _: False)
+ monkeypatch.setattr(pi_workflow.workflow, "wait_condition", wait_condition)
+ monkeypatch.setattr(pi_workflow.workflow, "start_activity", start_activity)
+ await value._execution_control.set_control_state({"state": "paused", "revision": 1})
+ completion = asyncio.create_task(value._complete_runtime_turn([]))
+ try:
+ await until(lambda: value._pi_checkpointing)
+ old_driver.set_result({"status": "parked"})
+ await until(lambda: not value._pi_checkpointing)
+ assert not completion.done() and not starts
+ resumed.set()
+ await value._execution_control.set_control_state({"state": "running", "revision": 2, "producers_held": False})
+ await asyncio.wait_for(completion, 2)
+ assert starts == ["coding.pi_drive_turn"]
+ assert value._pi_checkpoint["reference"] == {"pi_blob": "legacy-round"}
+ finally:
+ if not completion.done():
+ completion.cancel()
+ await asyncio.gather(completion, return_exceptions=True)
diff --git a/server/tests/temporal/test_pi_protocol.py b/server/tests/temporal/test_pi_protocol.py
index df00da04..7411e0b6 100644
--- a/server/tests/temporal/test_pi_protocol.py
+++ b/server/tests/temporal/test_pi_protocol.py
@@ -6,6 +6,12 @@ from temporalio.exceptions import ApplicationError
from services.temporal.pi_protocol import OperationLedger, canonical_digest, canonical_json
[email protected](autouse=True)
+def receipt_checkpoint_patch(monkeypatch):
+ # Direct coordinator unit calls do not run inside a Temporal history.
+ monkeypatch.setattr("services.temporal.pi_workflow.workflow.patched", lambda _: True)
+
+
def _request(epoch=1, **changes):
payload = {"scope": {"workflow_id": "w"}, "turn": 0, "purpose": "assistant", "epoch": epoch, **changes}
return {"id": "turn:0:model", "kind": "model", "payload": payload}
diff --git a/server/tests/temporal/test_pi_workflow_integration.py b/server/tests/temporal/test_pi_workflow_integration.py
index 1d9d2f90..1eadf012 100644
--- a/server/tests/temporal/test_pi_workflow_integration.py
+++ b/server/tests/temporal/test_pi_workflow_integration.py
@@ -7,7 +7,6 @@ from __future__ import annotations
import asyncio
from collections import Counter
from contextlib import nullcontext
-from datetime import timedelta
from pathlib import Path
import subprocess
import sys
@@ -15,7 +14,7 @@ import os
import tempfile
from uuid import uuid4
-from temporalio import activity
+from temporalio import activity, workflow
from temporalio.api.enums.v1 import EventType
from temporalio.client import WorkflowHistory
from temporalio.exceptions import ApplicationError
@@ -35,6 +34,46 @@ DRIVERS = Counter()
FAULTS = Counter()
CHECKPOINT_STARTED = asyncio.Event()
CHECKPOINT_RELEASE = asyncio.Event()
+PARK_RETIRE_STARTED = asyncio.Event()
+PARK_RETIRE_RELEASE = asyncio.Event()
+PARKED_CHILD = {}
+
+
+async def _hold_parked_child(payload, result):
+ if (payload["scope"]["node_id"] == "child" and
+ "delegate-stop" in activity.info().workflow_id and result.get("status") == "parked"):
+ # Hold the Activity result, and therefore the only driver slot, until
+ # the child's admitted model receipt is durable. The coordinator must
+ # encounter the still-live parked driver during turn completion.
+ info = activity.info()
+ PARKED_CHILD.update(workflow_id=info.workflow_id, run_id=info.workflow_run_id)
+ PARK_RETIRE_STARTED.set()
+ await PARK_RETIRE_RELEASE.wait()
+ return result
+
+
[email protected](sandboxed=False, name="PiCodingAgentWorkflow")
+class ObservedPiCodingAgentWorkflow(PiCodingAgentWorkflow):
+ @workflow.run
+ async def run(self, context: dict | None = None) -> dict:
+ return await super().run(context)
+
+ @workflow.query
+ def receipt_checkpoint_status(self):
+ # Queries add no history commands: replay below uses the production
+ # class. Observe the ledger without changing its parked epoch fence.
+ return {"checkpointing": self._pi_checkpointing,
+ "model_receipt_completed": (self._pi.operations.get("turn:0:model") or {}).get("status") == "completed",
+ "driver_active": self._pi_driver is not None and not self._pi_driver.done()}
+
+
+async def _wait_for_child_receipt_checkpoint(client):
+ child = client.get_workflow_handle(PARKED_CHILD["workflow_id"], run_id=PARKED_CHILD["run_id"])
+ while True:
+ status = await child.query("receipt_checkpoint_status")
+ if status["checkpointing"] and status["model_receipt_completed"] and status["driver_active"]:
+ return
+ await asyncio.sleep(0.01)
def _binding(name, node_type="pi_test_tool", node_id=None):
@@ -170,7 +209,7 @@ class ProtocolPeer:
model_id = f"turn:{payload['turn']}:model"
step = await operation(model_id, "model", {"purpose": "assistant"})
if step is None:
- return {"status": "parked"}
+ return await _hold_parked_child(payload, {"status": "parked"})
await handle.execute_update("assistant_ready", {"turn": payload["turn"], "epoch": epoch,
"context_digest": payload["context_digest"], "binding_digest": payload["binding_digest"],
"calls": [{"call_index": index, "call_id": call["id"],
@@ -182,7 +221,7 @@ class ProtocolPeer:
identity = tool_operation_id(payload["turn"], index, call)
result = await operation(identity, "tool", {"call_index": index, "call_id": call["id"]})
if result is None:
- return {"status": "parked"}
+ return await _hold_parked_child(payload, {"status": "parked"})
operations.append(identity)
results.append(result)
projection = [*payload["messages"], step["assistant_message"], *results]
@@ -234,11 +273,18 @@ async def _scenario(environment, mode, *, actual=False):
RELEASE.clear()
CHECKPOINT_STARTED.clear()
CHECKPOINT_RELEASE.clear()
- workflows = [PiCodingAgentWorkflow, AgentWorkflow]
+ PARK_RETIRE_STARTED.clear()
+ PARK_RETIRE_RELEASE.clear()
+ PARKED_CHILD.clear()
+ workflows = [ObservedPiCodingAgentWorkflow, AgentWorkflow]
if actual:
from services.temporal.pi_activities import PiDriverActivities
+ class RetiringPiDriverActivities(PiDriverActivities):
+ @activity.defn(name="coding.pi_drive_turn")
+ async def drive_turn(self, payload):
+ return await _hold_parked_child(payload, await super().drive_turn(payload))
client = LostResponseClient(environment.client, mode) if mode.startswith("fault-") else environment.client
- peer = PiDriverActivities(client).drive_turn
+ peer = RetiringPiDriverActivities(client).drive_turn
else:
peer = ProtocolPeer(environment.client).drive
from services.temporal.pi_activities import PiControlActivities, pi_put_blob
@@ -252,28 +298,54 @@ async def _scenario(environment, mode, *, actual=False):
"pi_local_control_version": 1, "pi_local_control_revision": 0},
id=f"pi-{mode}-{uuid4().hex}", task_queue=QUEUE)
if mode in {"stop", "model-stop", "delegate-stop", "task-wait"}:
+ checkpoint = None
try:
- await asyncio.wait_for(STARTED.wait(), 30)
- except TimeoutError:
- print(f"Pi start timeout: {(await handle.query('execution_control_status'))}", flush=True)
- print(f"Pi workflow status: {(await handle.describe()).status}", flush=True)
- raise
- stop = {"state": "paused", "revision": 1}
- if mode == "task-wait":
- description = await handle.describe()
- assert all(item.activity_type.name != "coding.pi_drive_turn"
- for item in description.raw_description.pending_activities), "Durable wait must release the driver slot"
- await handle.execute_update("set_control_state", stop)
- checkpoint = asyncio.create_task(handle.execute_update("wait_for_checkpoint", stop))
- RELEASE.set()
- stopped = await asyncio.wait_for(checkpoint, 15)
- assert stopped["checkpoint"] and stopped["pi_driver_checkpoint"]
- assert (await handle.query("operation_status", {"epoch": stopped["pi_driver_epoch"] - 1}))["status"] == "park"
- if not actual:
- assert DRIVERS["active"] == 0
- await asyncio.sleep(0.1)
- assert COUNTS["A"] == (1 if mode == "stop" else 0) and COUNTS["B"] == 0
- await handle.execute_update("set_control_state", {"state": "running", "revision": 2})
+ try:
+ await asyncio.wait_for(STARTED.wait(), 30)
+ except TimeoutError:
+ print(f"Pi start timeout: {(await handle.query('execution_control_status'))}", flush=True)
+ print(f"Pi workflow status: {(await handle.describe()).status}", flush=True)
+ raise
+ stop = {"state": "paused", "revision": 1}
+ if mode == "task-wait":
+ description = await handle.describe()
+ assert all(item.activity_type.name != "coding.pi_drive_turn"
+ for item in description.raw_description.pending_activities), "Durable wait must release the driver slot"
+ await handle.execute_update("set_control_state", stop)
+ checkpoint = asyncio.create_task(handle.execute_update("wait_for_checkpoint", stop))
+ if mode == "delegate-stop":
+ await asyncio.wait_for(PARK_RETIRE_STARTED.wait(), 15)
+ assert COUNTS["llm:child"] == 1
+ if not actual:
+ assert DRIVERS["active"] == 1, "The parked child must still occupy the sole driver slot"
+ RELEASE.set()
+ await asyncio.wait_for(_wait_for_child_receipt_checkpoint(environment.client), 15)
+ assert not checkpoint.done(), "Stop must await the still-live parked driver"
+ paused_effects = {name: COUNTS[name] for name in ("llm:agent", "llm:child", "A", "B", "T")}
+ assert paused_effects == {"llm:agent": 1, "llm:child": 1, "A": 0, "B": 0, "T": 0}
+ PARK_RETIRE_RELEASE.set()
+ else:
+ RELEASE.set()
+ stopped = await asyncio.wait_for(checkpoint, 15)
+ assert stopped["checkpoint"] and stopped["pi_driver_checkpoint"]
+ assert (await handle.query("operation_status", {"epoch": stopped["pi_driver_epoch"] - 1}))["status"] == "park"
+ if not actual:
+ assert DRIVERS["active"] == 0
+ await asyncio.sleep(0.1)
+ assert COUNTS["A"] == (1 if mode == "stop" else 0) and COUNTS["B"] == 0
+ if mode == "delegate-stop":
+ stopped_effects = {name: COUNTS[name] for name in paused_effects}
+ assert stopped_effects == paused_effects, (
+ "Receipt reconstruction must admit no new model or tool effects while paused",
+ {"before": paused_effects, "after": stopped_effects, "all_counters": dict(COUNTS)})
+ assert COUNTS["llm:child"] == 1
+ await handle.execute_update("set_control_state", {"state": "running", "revision": 2})
+ finally:
+ RELEASE.set()
+ PARK_RETIRE_RELEASE.set()
+ if checkpoint is not None and not checkpoint.done():
+ checkpoint.cancel()
+ await asyncio.gather(checkpoint, return_exceptions=True)
elif mode == "checkpoint-stop":
await asyncio.wait_for(CHECKPOINT_STARTED.wait(), 15)
stop = {"state": "paused", "revision": 1}
@@ -288,12 +360,21 @@ async def _scenario(environment, mode, *, actual=False):
await handle.execute_update("set_control_state", {"state": "running", "revision": 2})
result = await asyncio.wait_for(handle.result(), 25)
histories = []
+ child_histories = []
run_id = handle.first_execution_run_id
while run_id:
history = await environment.client.get_workflow_handle(handle.id, run_id=run_id).fetch_history()
histories.append(history)
run_id = next((event.workflow_execution_continued_as_new_event_attributes.new_execution_run_id
for event in history.events if event.event_type == EventType.EVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW), None)
+ if mode.startswith("delegate"):
+ for history in histories:
+ for event in history.events:
+ if event.event_type == EventType.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED:
+ child = event.child_workflow_execution_started_event_attributes.workflow_execution
+ child_histories.append(await environment.client.get_workflow_handle(
+ child.workflow_id, run_id=child.run_id).fetch_history())
+ assert child_histories, "Replay must include the delegated child's checkpoint history"
if mode == "large-overflow":
assert result["success"] is False and result["error_type"] == "AgentContinuationTooLarge", result
assert COUNTS["clear"] == COUNTS["failed_phase"] == COUNTS["llm:agent"] == 1
@@ -337,8 +418,9 @@ async def _scenario(environment, mode, *, actual=False):
assert carried["_pi_checkpoint"]["config_digest"]
assert carried["_agent_resume"]["prepared_payload"]["pi_harness_version"] == 1
assert carried["pi_local_control_revision"] == 0
- for history in histories:
- replay = await Replayer(workflows=workflows).replay_workflow(WorkflowHistory.from_json(history.workflow_id, history.to_json()))
+ for history in [*histories, *child_histories]:
+ replay = await Replayer(workflows=[PiCodingAgentWorkflow, AgentWorkflow]).replay_workflow(
+ WorkflowHistory.from_json(history.workflow_id, history.to_json()))
assert replay.replay_failure is None
@@ -363,7 +445,12 @@ async def _main():
return value
pi_blobs.put_blob = put
pi_blobs.get_blob = get
- async with await WorkflowEnvironment.start_time_skipping() as environment:
+ # The time-skipping server is unavailable on Windows. An explicitly
+ # supplied cached CLI exercises the same workflow and replay assertions.
+ cli = os.environ.get("TEMPORAL_TEST_CLI")
+ environment = (await WorkflowEnvironment.start_local(dev_server_existing_path=cli)
+ if cli else await WorkflowEnvironment.start_time_skipping())
+ async with environment:
with environment.auto_time_skipping_disabled():
focused = os.environ.get("PI_TEST_ACTUAL_SCENARIO")
if focused: