Python
run: retry a failed model call after 2 s, then back off
Rohit C Prasad9de6b83c7fa0
diff --git a/coworker/headless/runner.py b/coworker/headless/runner.pyindex cc7fa287..10d8633a 100644--- a/coworker/headless/runner.py+++ b/coworker/headless/runner.py@@ -218,9 +218,14 @@ def normalize_model(model: str) -> str:# The provider clients retry a failed request a couple of times within seconds. A provider# outage lasting minutes outlives that, and the engine then ends the run with an ERROR# event; in an unattended run that is a lost run. So the runner waits with backoff and-# re-enters the conversation with a nudge, up to ~17 minutes in total. Permanent errors+# re-enters the conversation with a nudge, up to ~18 minutes in total. Permanent errors# (bad key, unknown model, context overflow) are not retried.-PROVIDER_RETRY_DELAYS: tuple[int, ...] = (30, 60, 120, 240, 300, 300)+#+# The first waits are short: the commonest failure is one reply cut off mid-stream, which+# the clients do not retry and which the very next request survives. Seen on OpenShell+# 0.1.2, which closes every open connection when a sandbox's settings change (10 s after+# each start, and on each policy update).+PROVIDER_RETRY_DELAYS: tuple[int, ...] = (2, 5, 10, 30, 60, 120, 240, 300, 300)PROVIDER_RETRY_NUDGE = ("The previous model request failed with a temporary provider error and has been ""retried. Continue the task from where you left off."diff --git a/docs/headless.md b/docs/headless.mdindex 5ebddd1f..2de97d53 100644--- a/docs/headless.md+++ b/docs/headless.md@@ -174,10 +174,12 @@ record states it.The trajectory and the exit code are the stable contract.A transient provider failure or an empty reply does not end the run: the runner waits-(30 s, 60 s, 120 s, 240 s, 300 s, 300 s) and re-enters the conversation with a nudge.+and re-enters the conversation with a nudge. The waits grow: 2 s, 5 s, 10 s, 30 s, 60 s,+120 s, 240 s, 300 s, 300 s. The first ones are short because the usual failure is one+dropped connection, which the next request gets past.Permanent errors (bad key, unknown model, context overflow) are not retried.Each retry prints a line with the error and its cause, for example`provider error (APIConnectionError: Connection error. Cause: the proxy answered 403-Forbidden); retry 1/6 in 30s`. A stop signal or Ctrl-C ends the run at once, also during+Forbidden); retry 1/9 in 2s`. A stop signal or Ctrl-C ends the run at once, also duringthe wait between retries, and the record is still written.diff --git a/tests/test_headless_run.py b/tests/test_headless_run.pyindex 467d5693..f9909f31 100644--- a/tests/test_headless_run.py+++ b/tests/test_headless_run.py@@ -243,6 +243,18 @@ def test_without_the_flags_the_provider_defaults_are_described(tmp_path: Path) -# -- transient provider errors ------------------------------------------------------------+def test_the_first_waits_are_short_and_the_waits_grow(monkeypatch: pytest.MonkeyPatch) -> None:+ # Seen inside OpenShell 0.1.2: one reply cut off mid-stream cost a 30 s wait, when the+ # next request would have worked at once.+ from coworker.headless import runner++ monkeypatch.delenv(runner.RETRY_DELAYS_ENV, raising=False)+ delays = runner.provider_retry_delays()+ assert delays[:4] == (2, 5, 10, 30)+ assert list(delays) == sorted(delays)+ assert 15 * 60 <= sum(delays) <= 20 * 60 # an outage of minutes is still waited out++def test_transient_provider_error_is_waited_out_and_the_run_completes(tmp_path: Path) -> None:fail = {"error": "Service unavailable", "error_type": "APIError"}write = {"text": "ok", "tool_calls": [{"name": "write_file", "arguments": {"path": "hello.txt", "content": "hello"}}]}