Skip to content
Python

run: the retry line gives the cause, and a stop signal ends the wait between retries

Rohit C Prasad5c957d2185b1
diff --git a/coworker/headless/runner.py b/coworker/headless/runner.py
index 7db9ad22..cc7fa287 100644
--- a/coworker/headless/runner.py
+++ b/coworker/headless/runner.py
@@ -365,7 +365,12 @@ def scripted_provider(path: Path):
# A scripted provider failure: {"error": "Service unavailable",
# "error_type": "APIError"} raises an exception of that name, so the
# transient-error retry can be tested offline.
+ # "cause" / "cause_type" add the exception underneath, as a client library
+ # wraps a proxy or socket failure.
exc_cls = type(str(t.get("error_type") or "APIError"), (Exception,), {})
+ if t.get("cause"):
+ cause_cls = type(str(t.get("cause_type") or "OSError"), (Exception,), {})
+ raise exc_cls(str(t["error"])) from cause_cls(str(t["cause"]))
raise exc_cls(str(t["error"]))
calls = [
ToolCall(
@@ -484,6 +489,25 @@ def sanitize_key_env() -> list[str]:
return cleaned
+def root_cause_text(exc: BaseException) -> str:
+ """The exception at the bottom of a failed model call's chain, in a few words: what a
+ person needs to see beside "Connection error." Empty when there is nothing underneath."""
+ cause = exc.__cause__ or exc.__context__
+ last: Optional[BaseException] = None
+ depth = 0
+ while cause is not None and depth < 8:
+ last = cause
+ cause = cause.__cause__ or cause.__context__
+ depth += 1
+ if last is None:
+ return ""
+ name = type(last).__name__
+ detail = " ".join(str(last).split())[:200]
+ if name == "ProxyError":
+ return f"the proxy answered {detail}" if detail else "the proxy refused the connection"
+ return f"{name}: {detail}" if detail else name
+
+
def recording_provider(inner, log_path: Path):
"""Wrap a ProviderClient so every model call leaves one line in model_calls.jsonl
(stop reason, usage, ceiling, effort, serving host) and every exception is written to
@@ -495,8 +519,14 @@ def recording_provider(inner, log_path: Path):
class Recording(ProviderClient):
def __init__(self) -> None:
self.inner = inner
+ # The root cause of the last failed call, for the line the person sees.
+ self.last_cause = ""
def _record(self, exc: BaseException, method: str) -> None:
+ try:
+ self.last_cause = redact(root_cause_text(exc))
+ except Exception: # noqa: BLE001
+ self.last_cause = ""
try:
lines = [f"=== {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} {method} ==="]
lines.append("".join(traceback.format_exception(exc)))
@@ -928,12 +958,38 @@ def _run(args: argparse.Namespace) -> int:
import signal
+ # A stop signal (what `timeout`, a CI job or `docker stop` sends) ends the run: the
+ # engine is interrupted, and a wait between retries is cut short. Without the second
+ # part a run waiting out a provider error ignored the signal until a hard kill.
+ stop: dict[str, Any] = {"asked": False, "loop": None, "event": None}
+
def _on_term(signum, frame): # noqa: ARG001
+ stop["asked"] = True
try:
engine.request_interrupt()
+ loop, event = stop["loop"], stop["event"]
+ if loop is not None and event is not None:
+ loop.call_soon_threadsafe(event.set)
finally:
flush_provisional("killed")
+ async def wait_or_stop(seconds: float) -> bool:
+ """Wait before a retry. True when a stop signal arrived instead."""
+ if stop["asked"]:
+ return True
+ try:
+ await asyncio.wait_for(stop["event"].wait(), timeout=seconds)
+ except asyncio.TimeoutError:
+ return False
+ return True
+
+ def error_with_cause(data: dict[str, Any]) -> str:
+ """`Connection error.` says nothing; the cause underneath does (a proxy's 403, a
+ refused socket). Shown on the retry line so nobody has to open the record."""
+ text = f"{data.get('error_type')}: {str(data.get('error'))[:120]}"
+ cause = str(getattr(provider, "last_cause", "") or "")
+ return f"{text} Cause: {cause}" if cause and cause not in text else text
+
for sig in (signal.SIGTERM, getattr(signal, "SIGHUP", None)):
if sig is not None:
try:
@@ -942,6 +998,7 @@ def _run(args: argparse.Namespace) -> int:
pass
async def consume() -> None:
+ stop["loop"], stop["event"] = asyncio.get_running_loop(), asyncio.Event()
delays = provider_retry_delays()
retries = 0
message: Any = prompt
@@ -978,20 +1035,21 @@ def _run(args: argparse.Namespace) -> int:
"delay_seconds": delay,
"error": last_data.get("error"),
"error_type": last_data.get("error_type"),
+ "cause": str(getattr(provider, "last_cause", "") or ""),
},
}
events.append(rec)
events_fh.write(json.dumps(rec, default=str) + "\n")
events_fh.flush()
print(
- f"openworker run: provider error ({last_data.get('error_type')}: "
- f"{str(last_data.get('error'))[:120]}); retry {retries}/{len(delays)} "
- f"in {delay}s",
+ f"openworker run: provider error ({error_with_cause(last_data)}); "
+ f"retry {retries}/{len(delays)} in {delay}s",
file=sys.stderr,
flush=True,
)
flush_provisional("in_progress")
- await asyncio.sleep(delay)
+ if await wait_or_stop(delay):
+ break
message = PROVIDER_RETRY_NUDGE
continue
# An empty reply ended the turn as "completed": same wait-and-nudge, same budget.
@@ -1023,7 +1081,8 @@ def _run(args: argparse.Namespace) -> int:
flush=True,
)
flush_provisional("in_progress")
- await asyncio.sleep(delay)
+ if await wait_or_stop(delay):
+ break
message = EMPTY_REPLY_NUDGE
continue
break
@@ -1073,6 +1132,8 @@ def _run(args: argparse.Namespace) -> int:
except Exception: # noqa: BLE001
pass
+ if stop["asked"] and outcome in ("unknown", "timeout"):
+ outcome = "interrupted" # a stop signal ended it, whatever the last event was
# Outcome from the events when the loop ended on its own.
if outcome == "unknown":
outcome = "ended_without_turn_end"
@@ -1122,7 +1183,11 @@ def _run(args: argparse.Namespace) -> int:
return EXIT_INTERRUPTED
if outcome != "completed" and error_text:
# Why it stopped, in a line: the full text is in the record.
- print(f"openworker run: {redact(error_text.strip().splitlines()[-1])[:600]}", file=sys.stderr, flush=True)
+ why = redact(error_text.strip().splitlines()[-1])[:600]
+ cause = str(getattr(provider, "last_cause", "") or "")
+ if outcome == "model_error" and cause and cause not in why:
+ why = f"{why} Cause: {cause}"
+ print(f"openworker run: {why}", file=sys.stderr, flush=True)
return EXIT_FINISHED if outcome == "completed" else EXIT_STOPPED
diff --git a/docs/headless.md b/docs/headless.md
index 2fafe980..5ebddd1f 100644
--- a/docs/headless.md
+++ b/docs/headless.md
@@ -156,7 +156,7 @@ record states it.
| 1 | the run crashed before it could write a record |
| 2 | the command line could not be run as given |
| 3 | the run stopped early (time limit, model error, iteration limit, cut-off reply); the record is written |
-| 130 | stopped with Ctrl-C; the record is written |
+| 130 | stopped with Ctrl-C or a stop signal (what `timeout`, a CI job or `docker stop` sends); the record is written |
### The record
@@ -176,3 +176,8 @@ The trajectory and the exit code are the stable contract.
A transient provider failure or an empty reply does not end the run: the runner waits
(30 s, 60 s, 120 s, 240 s, 300 s, 300 s) and re-enters the conversation with a nudge.
Permanent errors (bad key, unknown model, context overflow) are not retried.
+
+Each retry prints a line with the error and its cause, for example
+`provider error (APIConnectionError: Connection error. Cause: the proxy answered 403
+Forbidden); retry 1/6 in 30s`. A stop signal or Ctrl-C ends the run at once, also during
+the wait between retries, and the record is still written.
diff --git a/tests/test_run_cli.py b/tests/test_run_cli.py
index 4cf2291e..78a43cd3 100644
--- a/tests/test_run_cli.py
+++ b/tests/test_run_cli.py
@@ -535,3 +535,83 @@ def test_with_auto_answer_a_folder_request_is_declined_by_rule(tmp_path: Path) -
(folder,) = _session_folders(machine, ws)
assert json.loads((folder / "summary.json").read_text(encoding="utf-8"))["answers"]["directory_requests_declined"] == 1
+
+# -- a failing model call ---------------------------------------------------------------
+
+
+def test_the_root_cause_is_the_bottom_of_the_chain() -> None:
+ from coworker.headless.runner import root_cause_text
+
+ ProxyError = type("ProxyError", (Exception,), {})
+ try:
+ try:
+ try:
+ raise ProxyError("403 Forbidden")
+ except Exception as low:
+ raise RuntimeError("httpx failed") from low
+ except Exception as mid:
+ raise ConnectionError("Connection error.") from mid
+ except Exception as top:
+ assert root_cause_text(top) == "the proxy answered 403 Forbidden"
+
+ try:
+ try:
+ raise OSError("[Errno 111] Connection refused")
+ except Exception as low:
+ raise ConnectionError("Connection error.") from low
+ except Exception as top:
+ assert root_cause_text(top) == "OSError: [Errno 111] Connection refused"
+
+ assert root_cause_text(ValueError("nothing underneath")) == ""
+
+
+REFUSED_BY_A_PROXY = {"error": "Connection error.", "error_type": "APIConnectionError", "cause": "403 Forbidden", "cause_type": "ProxyError"}
+
+
+def test_the_retry_line_says_why_the_model_call_failed(tmp_path: Path) -> None:
+ machine = tmp_path / "machine"
+ machine.mkdir()
+ cmd, ws = _command(tmp_path, [REFUSED_BY_A_PROXY, {"text": "Recovered."}], "--model", "anthropic/claude-sonnet-5")
+ env = {**_env(machine), "OPENWORKER_RUN_RETRY_DELAYS": "0"}
+ proc = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=300)
+ assert proc.returncode == 0, proc.stderr
+ assert "provider error (APIConnectionError: Connection error. Cause: the proxy answered 403 Forbidden); retry 1/1 in 0s" in proc.stderr
+ (folder,) = _session_folders(machine, ws)
+ events = [json.loads(line) for line in (folder / "events.jsonl").read_text(encoding="utf-8").splitlines()]
+ (retry,) = [e for e in events if e["type"] == "provider_retry"]
+ assert retry["data"]["cause"] == "the proxy answered 403 Forbidden"
+
+ # When the retries run out, the closing line says why as well.
+ second = tmp_path / "second"
+ second.mkdir()
+ cmd, _ws = _command(second, [REFUSED_BY_A_PROXY, REFUSED_BY_A_PROXY], "--model", "anthropic/claude-sonnet-5")
+ proc = subprocess.run(cmd, env={**_env(machine), "OPENWORKER_RUN_RETRY_DELAYS": "0"}, capture_output=True, text=True, timeout=300)
+ assert proc.returncode == 3
+ assert proc.stderr.rstrip().splitlines()[-1].endswith("Cause: the proxy answered 403 Forbidden")
+
+
[email protected](sys.platform == "win32", reason="sends a stop signal")
+def test_a_stop_signal_ends_the_run_during_the_wait_between_retries(tmp_path: Path) -> None:
+ import signal
+
+ machine = tmp_path / "machine"
+ machine.mkdir()
+ cmd, ws = _command(tmp_path, [REFUSED_BY_A_PROXY, {"text": "never reached"}], "--model", "anthropic/claude-sonnet-5")
+ env = {**_env(machine), "OPENWORKER_RUN_RETRY_DELAYS": "600"}
+ proc = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ try:
+ shown = ""
+ deadline = time.time() + 60
+ while "retry 1/1 in 600s" not in shown:
+ assert time.time() < deadline, shown
+ if select.select([proc.stderr], [], [], 0.2)[0]:
+ shown += os.read(proc.stderr.fileno(), 4096).decode("utf-8", errors="replace")
+ time.sleep(0.5) # it is now waiting out the 600 seconds
+ proc.send_signal(signal.SIGTERM)
+ assert proc.wait(timeout=30) == 130 # it stopped; it did not wait
+ finally:
+ if proc.poll() is None:
+ proc.kill()
+ (folder,) = _session_folders(machine, ws)
+ assert json.loads((folder / "summary.json").read_text(encoding="utf-8"))["outcome"] == "interrupted"
+