Skip to content
Python

run: retry a failed model call after 2 s, then back off

Rohit C Prasad9de6b83c7fa0
diff --git a/coworker/headless/runner.py b/coworker/headless/runner.py
index cc7fa287..10d8633a 100644
--- a/coworker/headless/runner.py
+++ b/coworker/headless/runner.py
@@ -218,9 +218,14 @@ def normalize_model(model: str) -> str:
# The provider clients retry a failed request a couple of times within seconds. A provider
# outage lasting minutes outlives that, and the engine then ends the run with an ERROR
# event; in an unattended run that is a lost run. So the runner waits with backoff and
-# re-enters the conversation with a nudge, up to ~17 minutes in total. Permanent errors
+# re-enters the conversation with a nudge, up to ~18 minutes in total. Permanent errors
# (bad key, unknown model, context overflow) are not retried.
-PROVIDER_RETRY_DELAYS: tuple[int, ...] = (30, 60, 120, 240, 300, 300)
+#
+# The first waits are short: the commonest failure is one reply cut off mid-stream, which
+# the clients do not retry and which the very next request survives. Seen on OpenShell
+# 0.1.2, which closes every open connection when a sandbox's settings change (10 s after
+# each start, and on each policy update).
+PROVIDER_RETRY_DELAYS: tuple[int, ...] = (2, 5, 10, 30, 60, 120, 240, 300, 300)
PROVIDER_RETRY_NUDGE = (
"The previous model request failed with a temporary provider error and has been "
"retried. Continue the task from where you left off."
diff --git a/docs/headless.md b/docs/headless.md
index 5ebddd1f..2de97d53 100644
--- a/docs/headless.md
+++ b/docs/headless.md
@@ -174,10 +174,12 @@ record states it.
The trajectory and the exit code are the stable contract.
A transient provider failure or an empty reply does not end the run: the runner waits
-(30 s, 60 s, 120 s, 240 s, 300 s, 300 s) and re-enters the conversation with a nudge.
+and re-enters the conversation with a nudge. The waits grow: 2 s, 5 s, 10 s, 30 s, 60 s,
+120 s, 240 s, 300 s, 300 s. The first ones are short because the usual failure is one
+dropped connection, which the next request gets past.
Permanent errors (bad key, unknown model, context overflow) are not retried.
Each retry prints a line with the error and its cause, for example
`provider error (APIConnectionError: Connection error. Cause: the proxy answered 403
-Forbidden); retry 1/6 in 30s`. A stop signal or Ctrl-C ends the run at once, also during
+Forbidden); retry 1/9 in 2s`. A stop signal or Ctrl-C ends the run at once, also during
the wait between retries, and the record is still written.
diff --git a/tests/test_headless_run.py b/tests/test_headless_run.py
index 467d5693..f9909f31 100644
--- a/tests/test_headless_run.py
+++ b/tests/test_headless_run.py
@@ -243,6 +243,18 @@ def test_without_the_flags_the_provider_defaults_are_described(tmp_path: Path) -
# -- transient provider errors ------------------------------------------------------------
+def test_the_first_waits_are_short_and_the_waits_grow(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Seen inside OpenShell 0.1.2: one reply cut off mid-stream cost a 30 s wait, when the
+ # next request would have worked at once.
+ from coworker.headless import runner
+
+ monkeypatch.delenv(runner.RETRY_DELAYS_ENV, raising=False)
+ delays = runner.provider_retry_delays()
+ assert delays[:4] == (2, 5, 10, 30)
+ assert list(delays) == sorted(delays)
+ assert 15 * 60 <= sum(delays) <= 20 * 60 # an outage of minutes is still waited out
+
+
def test_transient_provider_error_is_waited_out_and_the_run_completes(tmp_path: Path) -> None:
fail = {"error": "Service unavailable", "error_type": "APIError"}
write = {"text": "ok", "tool_calls": [{"name": "write_file", "arguments": {"path": "hello.txt", "content": "hello"}}]}