Python
run: the retry line gives the cause, and a stop signal ends the wait between retries
Rohit C Prasad5c957d2185b1
diff --git a/coworker/headless/runner.py b/coworker/headless/runner.pyindex 7db9ad22..cc7fa287 100644--- a/coworker/headless/runner.py+++ b/coworker/headless/runner.py@@ -365,7 +365,12 @@ def scripted_provider(path: Path):# A scripted provider failure: {"error": "Service unavailable",# "error_type": "APIError"} raises an exception of that name, so the# transient-error retry can be tested offline.+ # "cause" / "cause_type" add the exception underneath, as a client library+ # wraps a proxy or socket failure.exc_cls = type(str(t.get("error_type") or "APIError"), (Exception,), {})+ if t.get("cause"):+ cause_cls = type(str(t.get("cause_type") or "OSError"), (Exception,), {})+ raise exc_cls(str(t["error"])) from cause_cls(str(t["cause"]))raise exc_cls(str(t["error"]))calls = [ToolCall(@@ -484,6 +489,25 @@ def sanitize_key_env() -> list[str]:return cleaned+def root_cause_text(exc: BaseException) -> str:+ """The exception at the bottom of a failed model call's chain, in a few words: what a+ person needs to see beside "Connection error." Empty when there is nothing underneath."""+ cause = exc.__cause__ or exc.__context__+ last: Optional[BaseException] = None+ depth = 0+ while cause is not None and depth < 8:+ last = cause+ cause = cause.__cause__ or cause.__context__+ depth += 1+ if last is None:+ return ""+ name = type(last).__name__+ detail = " ".join(str(last).split())[:200]+ if name == "ProxyError":+ return f"the proxy answered {detail}" if detail else "the proxy refused the connection"+ return f"{name}: {detail}" if detail else name++def recording_provider(inner, log_path: Path):"""Wrap a ProviderClient so every model call leaves one line in model_calls.jsonl(stop reason, usage, ceiling, effort, serving host) and every exception is written to@@ -495,8 +519,14 @@ def recording_provider(inner, log_path: Path):class Recording(ProviderClient):def __init__(self) -> None:self.inner = inner+ # The root cause of the last failed call, for the line the person sees.+ self.last_cause = ""def _record(self, exc: BaseException, method: str) -> None:+ try:+ self.last_cause = redact(root_cause_text(exc))+ except Exception: # noqa: BLE001+ self.last_cause = ""try:lines = [f"=== {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} {method} ==="]lines.append("".join(traceback.format_exception(exc)))@@ -928,12 +958,38 @@ def _run(args: argparse.Namespace) -> int:import signal+ # A stop signal (what `timeout`, a CI job or `docker stop` sends) ends the run: the+ # engine is interrupted, and a wait between retries is cut short. Without the second+ # part a run waiting out a provider error ignored the signal until a hard kill.+ stop: dict[str, Any] = {"asked": False, "loop": None, "event": None}+def _on_term(signum, frame): # noqa: ARG001+ stop["asked"] = Truetry:engine.request_interrupt()+ loop, event = stop["loop"], stop["event"]+ if loop is not None and event is not None:+ loop.call_soon_threadsafe(event.set)finally:flush_provisional("killed")+ async def wait_or_stop(seconds: float) -> bool:+ """Wait before a retry. True when a stop signal arrived instead."""+ if stop["asked"]:+ return True+ try:+ await asyncio.wait_for(stop["event"].wait(), timeout=seconds)+ except asyncio.TimeoutError:+ return False+ return True++ def error_with_cause(data: dict[str, Any]) -> str:+ """`Connection error.` says nothing; the cause underneath does (a proxy's 403, a+ refused socket). Shown on the retry line so nobody has to open the record."""+ text = f"{data.get('error_type')}: {str(data.get('error'))[:120]}"+ cause = str(getattr(provider, "last_cause", "") or "")+ return f"{text} Cause: {cause}" if cause and cause not in text else text+for sig in (signal.SIGTERM, getattr(signal, "SIGHUP", None)):if sig is not None:try:@@ -942,6 +998,7 @@ def _run(args: argparse.Namespace) -> int:passasync def consume() -> None:+ stop["loop"], stop["event"] = asyncio.get_running_loop(), asyncio.Event()delays = provider_retry_delays()retries = 0message: Any = prompt@@ -978,20 +1035,21 @@ def _run(args: argparse.Namespace) -> int:"delay_seconds": delay,"error": last_data.get("error"),"error_type": last_data.get("error_type"),+ "cause": str(getattr(provider, "last_cause", "") or ""),},}events.append(rec)events_fh.write(json.dumps(rec, default=str) + "\n")events_fh.flush()print(- f"openworker run: provider error ({last_data.get('error_type')}: "- f"{str(last_data.get('error'))[:120]}); retry {retries}/{len(delays)} "- f"in {delay}s",+ f"openworker run: provider error ({error_with_cause(last_data)}); "+ f"retry {retries}/{len(delays)} in {delay}s",file=sys.stderr,flush=True,)flush_provisional("in_progress")- await asyncio.sleep(delay)+ if await wait_or_stop(delay):+ breakmessage = PROVIDER_RETRY_NUDGEcontinue# An empty reply ended the turn as "completed": same wait-and-nudge, same budget.@@ -1023,7 +1081,8 @@ def _run(args: argparse.Namespace) -> int:flush=True,)flush_provisional("in_progress")- await asyncio.sleep(delay)+ if await wait_or_stop(delay):+ breakmessage = EMPTY_REPLY_NUDGEcontinuebreak@@ -1073,6 +1132,8 @@ def _run(args: argparse.Namespace) -> int:except Exception: # noqa: BLE001pass+ if stop["asked"] and outcome in ("unknown", "timeout"):+ outcome = "interrupted" # a stop signal ended it, whatever the last event was# Outcome from the events when the loop ended on its own.if outcome == "unknown":outcome = "ended_without_turn_end"@@ -1122,7 +1183,11 @@ def _run(args: argparse.Namespace) -> int:return EXIT_INTERRUPTEDif outcome != "completed" and error_text:# Why it stopped, in a line: the full text is in the record.- print(f"openworker run: {redact(error_text.strip().splitlines()[-1])[:600]}", file=sys.stderr, flush=True)+ why = redact(error_text.strip().splitlines()[-1])[:600]+ cause = str(getattr(provider, "last_cause", "") or "")+ if outcome == "model_error" and cause and cause not in why:+ why = f"{why} Cause: {cause}"+ print(f"openworker run: {why}", file=sys.stderr, flush=True)return EXIT_FINISHED if outcome == "completed" else EXIT_STOPPEDdiff --git a/docs/headless.md b/docs/headless.mdindex 2fafe980..5ebddd1f 100644--- a/docs/headless.md+++ b/docs/headless.md@@ -156,7 +156,7 @@ record states it.| 1 | the run crashed before it could write a record || 2 | the command line could not be run as given || 3 | the run stopped early (time limit, model error, iteration limit, cut-off reply); the record is written |-| 130 | stopped with Ctrl-C; the record is written |+| 130 | stopped with Ctrl-C or a stop signal (what `timeout`, a CI job or `docker stop` sends); the record is written |### The record@@ -176,3 +176,8 @@ The trajectory and the exit code are the stable contract.A transient provider failure or an empty reply does not end the run: the runner waits(30 s, 60 s, 120 s, 240 s, 300 s, 300 s) and re-enters the conversation with a nudge.Permanent errors (bad key, unknown model, context overflow) are not retried.++Each retry prints a line with the error and its cause, for example+`provider error (APIConnectionError: Connection error. Cause: the proxy answered 403+Forbidden); retry 1/6 in 30s`. A stop signal or Ctrl-C ends the run at once, also during+the wait between retries, and the record is still written.diff --git a/tests/test_run_cli.py b/tests/test_run_cli.pyindex 4cf2291e..78a43cd3 100644--- a/tests/test_run_cli.py+++ b/tests/test_run_cli.py@@ -535,3 +535,83 @@ def test_with_auto_answer_a_folder_request_is_declined_by_rule(tmp_path: Path) -(folder,) = _session_folders(machine, ws)assert json.loads((folder / "summary.json").read_text(encoding="utf-8"))["answers"]["directory_requests_declined"] == 1++# -- a failing model call ---------------------------------------------------------------+++def test_the_root_cause_is_the_bottom_of_the_chain() -> None:+ from coworker.headless.runner import root_cause_text++ ProxyError = type("ProxyError", (Exception,), {})+ try:+ try:+ try:+ raise ProxyError("403 Forbidden")+ except Exception as low:+ raise RuntimeError("httpx failed") from low+ except Exception as mid:+ raise ConnectionError("Connection error.") from mid+ except Exception as top:+ assert root_cause_text(top) == "the proxy answered 403 Forbidden"++ try:+ try:+ raise OSError("[Errno 111] Connection refused")+ except Exception as low:+ raise ConnectionError("Connection error.") from low+ except Exception as top:+ assert root_cause_text(top) == "OSError: [Errno 111] Connection refused"++ assert root_cause_text(ValueError("nothing underneath")) == ""+++REFUSED_BY_A_PROXY = {"error": "Connection error.", "error_type": "APIConnectionError", "cause": "403 Forbidden", "cause_type": "ProxyError"}+++def test_the_retry_line_says_why_the_model_call_failed(tmp_path: Path) -> None:+ machine = tmp_path / "machine"+ machine.mkdir()+ cmd, ws = _command(tmp_path, [REFUSED_BY_A_PROXY, {"text": "Recovered."}], "--model", "anthropic/claude-sonnet-5")+ env = {**_env(machine), "OPENWORKER_RUN_RETRY_DELAYS": "0"}+ proc = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=300)+ assert proc.returncode == 0, proc.stderr+ assert "provider error (APIConnectionError: Connection error. Cause: the proxy answered 403 Forbidden); retry 1/1 in 0s" in proc.stderr+ (folder,) = _session_folders(machine, ws)+ events = [json.loads(line) for line in (folder / "events.jsonl").read_text(encoding="utf-8").splitlines()]+ (retry,) = [e for e in events if e["type"] == "provider_retry"]+ assert retry["data"]["cause"] == "the proxy answered 403 Forbidden"++ # When the retries run out, the closing line says why as well.+ second = tmp_path / "second"+ second.mkdir()+ cmd, _ws = _command(second, [REFUSED_BY_A_PROXY, REFUSED_BY_A_PROXY], "--model", "anthropic/claude-sonnet-5")+ proc = subprocess.run(cmd, env={**_env(machine), "OPENWORKER_RUN_RETRY_DELAYS": "0"}, capture_output=True, text=True, timeout=300)+ assert proc.returncode == 3+ assert proc.stderr.rstrip().splitlines()[-1].endswith("Cause: the proxy answered 403 Forbidden")++[email protected](sys.platform == "win32", reason="sends a stop signal")+def test_a_stop_signal_ends_the_run_during_the_wait_between_retries(tmp_path: Path) -> None:+ import signal++ machine = tmp_path / "machine"+ machine.mkdir()+ cmd, ws = _command(tmp_path, [REFUSED_BY_A_PROXY, {"text": "never reached"}], "--model", "anthropic/claude-sonnet-5")+ env = {**_env(machine), "OPENWORKER_RUN_RETRY_DELAYS": "600"}+ proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)+ try:+ shown = ""+ deadline = time.time() + 60+ while "retry 1/1 in 600s" not in shown:+ assert time.time() < deadline, shown+ if select.select([proc.stderr], [], [], 0.2)[0]:+ shown += os.read(proc.stderr.fileno(), 4096).decode("utf-8", errors="replace")+ time.sleep(0.5) # it is now waiting out the 600 seconds+ proc.send_signal(signal.SIGTERM)+ assert proc.wait(timeout=30) == 130 # it stopped; it did not wait+ finally:+ if proc.poll() is None:+ proc.kill()+ (folder,) = _session_folders(machine, ws)+ assert json.loads((folder / "summary.json").read_text(encoding="utf-8"))["outcome"] == "interrupted"+