Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

fix(cli): crewai eval waits out AMP's rate limit instead of quitting (#8148)

João Mourab0c16c162aa3
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 56e26f40d..993212cfd 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -25,8 +25,10 @@ from collections.abc import Callable
import contextlib
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
+from email.utils import parsedate_to_datetime
from ipaddress import ip_address
import json
+import math
import os
from pathlib import Path
import sys
@@ -60,6 +62,18 @@ LAST_RUN_FILE = Path(".crewai") / "last_run.json"
TRACING_ENV_VAR = "CREWAI_TRACING_ENABLED"
POLL_SECONDS = 3.0
POLL_RETRIES = 5 # consecutive unreachable / 5xx polls before giving up; the evaluation keeps running
+# AMP limits evaluation requests per IP (30 a minute): a poll it answers 429
+# waits what its Retry-After asks, at most this long, and polls again — the
+# evaluation keeps running meanwhile, so being asked to slow down is no reason
+# to stop waiting for it.
+RATE_LIMIT_MAX_WAIT_SECONDS = 120.0
+# Polls that get no answer about the evaluation — rate-limited, failing,
+# unreachable, in any mix — for this long without a break end the wait: the
+# evaluation keeps running, and the sentence says where to follow it. One
+# monotonic deadline (`_clock`), reset only by AMP's answer (a 200), so a limit
+# that passes is ridden out and refusals taking turns cannot keep CI or an
+# agent waiting without end.
+UNANSWERED_FOR_AT_MOST_SECONDS = 15 * 60.0
# A run's spans reach AMP a little after the run ends — the exporter sends them
# as the process closes and AMP has its own queue behind that. A command that
@@ -812,6 +826,34 @@ def _wait(
misses = (
0 # AMP unreachable or answering 5xx: a blip is retried, a streak is reported
)
+ # When the polls stopped getting AMP's answer about the evaluation (a 200),
+ # by _clock, and what they got instead: one deadline over every kind of
+ # refusal, so 503s and 429s taking turns cannot keep the wait going.
+ unanswered_since: float | None = None
+ troubles: set[str] = set()
+
+ def unanswered_for(trouble: str) -> float:
+ nonlocal unanswered_since
+ troubles.add(trouble)
+ if unanswered_since is None:
+ unanswered_since = _clock()
+ return _clock() - unanswered_since
+
+ def stop_unanswered(response: httpx.Response | None) -> EvaluationStoppedError:
+ minutes = UNANSWERED_FOR_AT_MOST_SECONDS / 60
+ what = (
+ f"AMP kept rate-limiting this command for {minutes:.0f} minutes"
+ if troubles == {"rate_limited"}
+ else f"AMP gave this command no answer about the evaluation for {minutes:.0f} minutes "
+ "(rate limits, errors, no connection)"
+ )
+ return EvaluationStoppedError(
+ f"{what}, so it stopped waiting; the evaluation keeps running{where}.",
+ reason=_refusal_reason(response)
+ if response is not None
+ else "network_error",
+ )
+
while True:
try:
response = client.get_evaluation(evaluation_id)
@@ -822,6 +864,8 @@ def _wait(
f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}.",
reason="network_error",
) from error
+ if unanswered_for("unreachable") >= UNANSWERED_FOR_AT_MOST_SECONDS:
+ raise stop_unanswered(None) from error
time.sleep(POLL_SECONDS)
continue
if response.status_code >= 500:
@@ -831,13 +875,27 @@ def _wait(
_refusal_message(response, subject),
reason=_refusal_reason(response),
)
+ if unanswered_for("failing") >= UNANSWERED_FOR_AT_MOST_SECONDS:
+ raise stop_unanswered(response)
time.sleep(POLL_SECONDS)
continue
+ if response.status_code == 429:
+ misses = 0 # AMP answered: a streak of unreachable / 5xx polls is broken
+ waited = unanswered_for("rate_limited")
+ if waited >= UNANSWERED_FOR_AT_MOST_SECONDS:
+ raise stop_unanswered(response)
+ # never past the deadline: the last wait is what is left of it
+ time.sleep(
+ min(_retry_after(response), UNANSWERED_FOR_AT_MOST_SECONDS - waited)
+ )
+ continue
if response.status_code != 200:
raise EvaluationStoppedError(
_refusal_message(response, subject),
reason=_refusal_reason(response),
)
+ unanswered_since = None
+ troubles.clear()
misses = 0
payload = _payload(response) or {}
status = payload.get("status")
@@ -858,6 +916,31 @@ def _wait(
time.sleep(POLL_SECONDS)
+def _retry_after(response: httpx.Response) -> float:
+ """How long a 429 asks to wait before the next poll: its Retry-After —
+ delay-seconds or an HTTP-date, the two forms RFC 9110 allows — never less
+ than a poll's own interval nor more than RATE_LIMIT_MAX_WAIT_SECONDS. A
+ missing header, or one that is neither (a negative or non-finite number,
+ words), waits a minute; a date already past, a poll's interval."""
+ raw = response.headers.get("Retry-After", "").strip()
+ asked = 60.0
+ try:
+ number = float(raw)
+ except ValueError:
+ try:
+ when = parsedate_to_datetime(raw)
+ except (TypeError, ValueError, IndexError):
+ when = None
+ if when is not None:
+ if when.tzinfo is None:
+ when = when.replace(tzinfo=timezone.utc)
+ asked = (when - datetime.now(timezone.utc)).total_seconds()
+ else:
+ if math.isfinite(number) and number >= 0:
+ asked = number
+ return min(RATE_LIMIT_MAX_WAIT_SECONDS, max(POLL_SECONDS, asked))
+
+
def _carries_a_verdict(payload: dict[str, Any]) -> bool:
return _well_formed_verdict(payload.get("verdict"))
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index 4de661b51..8a511b11d 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+from datetime import datetime, timedelta, timezone
import json
import os
from pathlib import Path
@@ -526,6 +527,166 @@ def test_while_waiting_a_blip_is_retried_and_a_streak_is_reported(project, monke
assert f"Could not reach AMP while waiting (timed out); the evaluation keeps running at {URL}." in capsys.readouterr().out
+def test_a_rate_limited_poll_waits_what_amp_asks_and_keeps_waiting(project, monkeypatch, capsys):
+ """AMP limits evaluation requests per IP; a models comparison on prod
+ (2026-10-14) ended its CLI with exit 1 on one 429 while the deployment
+ went on running it. A 429 mid-poll is waited out — Retry-After, bounded —
+ and the poll goes on, however many come."""
+ directory, _ = project
+ record_last_run(directory)
+ slept: list[float] = []
+ monkeypatch.setattr(eval_module.time, "sleep", slept.append)
+ limited = httpx.Response(
+ 429, headers={"Retry-After": "60"},
+ json={"error": "rate_limit_exceeded", "message": "Too many evaluation requests. Please retry later."},
+ )
+ odd = httpx.Response(429, headers={"Retry-After": "soon"}, json={"error": "rate_limit_exceeded"})
+ long = httpx.Response(429, headers={"Retry-After": "3600"}, json={"error": "rate_limit_exceeded"})
+ amp = install(monkeypatch, FakeAMP(statuses=[limited] * 7 + [odd, long, done()]))
+
+ eval_module.eval_crew()
+
+ out = capsys.readouterr().out
+ assert "Goal gate: PASSED" in out and "Too many evaluation requests" not in out
+ assert amp.calls.count(("get", "ev-1")) == 10 # more 429s than POLL_RETRIES: never a give-up
+ assert slept == [60.0] * 7 + [60.0, eval_module.RATE_LIMIT_MAX_WAIT_SECONDS]
+
+
+def _clocked_sleep(monkeypatch) -> list[float]:
+ """time.sleep records each wait and moves the CLI's monotonic clock on by
+ it, so the unanswered deadline sees the time pass."""
+ slept: list[float] = []
+
+ def sleep(seconds: float) -> None:
+ slept.append(seconds)
+
+ monkeypatch.setattr(eval_module.time, "sleep", sleep)
+ monkeypatch.setattr(eval_module, "_clock", lambda: sum(slept))
+ return slept
+
+
+def test_a_rate_limit_that_never_lifts_ends_the_wait_and_a_passing_one_does_not(project, monkeypatch, capsys):
+ """Rate-limited without a break for UNANSWERED_FOR_AT_MOST_SECONDS the
+ wait stops, saying the evaluation keeps running; AMP's answer about the
+ evaluation starts the deadline again, so a limit that passes is ridden out."""
+ directory, _ = project
+ slept = _clocked_sleep(monkeypatch)
+ limited = httpx.Response(429, headers={"Retry-After": "60"}, json={"error": "rate_limit_exceeded"})
+ running = httpx.Response(200, json={"id": "ev-1", "status": "running", "url": URL})
+
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP(statuses=[limited] * 30))
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+ assert exit_.value.code == 1
+ assert (
+ f"AMP kept rate-limiting this command for 15 minutes, so it stopped waiting; the evaluation keeps running at {URL}."
+ in " ".join(capsys.readouterr().out.split())
+ )
+ assert amp.calls.count(("get", "ev-1")) == 16 # fifteen minutes of 60 s waits, then the one that stops
+ assert sum(slept) == eval_module.UNANSWERED_FOR_AT_MOST_SECONDS
+
+ slept.clear()
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP(statuses=[limited] * 14 + [running] + [limited] * 14 + [done()]))
+ eval_module.eval_crew()
+ assert "Goal gate: PASSED" in capsys.readouterr().out
+ assert amp.calls.count(("get", "ev-1")) == 30 # every 429 ridden out: the 200 between them reset the deadline
+
+
+def test_the_deadline_never_overruns_and_only_an_answer_restarts_it(project, monkeypatch, capsys):
+ """The last wait is what is left of UNANSWERED_FOR_AT_MOST_SECONDS, so the
+ wait never runs past it; a 5xx between 429s is no answer about the
+ evaluation, so it does not restart the deadline (only a 200 does)."""
+ directory, _ = project
+ slept = _clocked_sleep(monkeypatch)
+ long_wait = httpx.Response(429, headers={"Retry-After": "120"}, json={"error": "rate_limit_exceeded"})
+ blip = httpx.Response(503, json={"error": "service_unavailable", "message": "down for a moment"})
+
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(statuses=[long_wait] * 20))
+ with pytest.raises(SystemExit):
+ eval_module.eval_crew()
+ capsys.readouterr()
+ assert slept == [120.0] * 7 + [60.0] # 840 s, then the 60 s left of the fifteen minutes
+ assert sum(slept) == eval_module.UNANSWERED_FOR_AT_MOST_SECONDS
+
+ slept.clear()
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(statuses=[long_wait] * 7 + [blip] + [long_wait] * 7 + [done()]))
+ with pytest.raises(SystemExit):
+ eval_module.eval_crew()
+ assert "no answer about the evaluation for 15 minutes" in " ".join(capsys.readouterr().out.split())
+ assert sum(slept) == eval_module.UNANSWERED_FOR_AT_MOST_SECONDS # 840 + 3 (the 503) + the 57 s left
+
+
+def test_refusals_taking_turns_cannot_keep_the_wait_going(project, monkeypatch, capsys):
+ """iris on #8148: a 503 and a 429 taking turns broke each other's counts
+ (the 429 a streak of 5xx, the 5xx the rate-limit count), so callers could
+ wait without end. One deadline over every refusal stops them."""
+ directory, _ = project
+ slept = _clocked_sleep(monkeypatch)
+ blip = httpx.Response(503, json={"error": "service_unavailable", "message": "down for a moment"})
+ limited = httpx.Response(429, headers={"Retry-After": "60"}, json={"error": "rate_limit_exceeded"})
+
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP(statuses=[blip, limited] * 200))
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ assert (
+ "AMP gave this command no answer about the evaluation for 15 minutes (rate limits, errors, no connection), "
+ f"so it stopped waiting; the evaluation keeps running at {URL}."
+ ) in " ".join(capsys.readouterr().out.split())
+ assert sum(slept) <= eval_module.UNANSWERED_FOR_AT_MOST_SECONDS + eval_module.POLL_SECONDS
+ assert amp.calls.count(("get", "ev-1")) < 40
+
+
+def test_a_rate_limit_answer_breaks_a_streak_of_unreachable_polls(project, monkeypatch, capsys):
+ """POLL_RETRIES counts a streak of 5xx (or unreachable) polls; a 429 is
+ AMP answering, so it breaks the streak like any other answer."""
+ directory, _ = project
+ record_last_run(directory)
+ blip = httpx.Response(503, json={"error": "service_unavailable", "message": "down for a moment"})
+ limited = httpx.Response(429, headers={"Retry-After": "5"}, json={"error": "rate_limit_exceeded"})
+ streak = eval_module.POLL_RETRIES - 1
+ amp = install(monkeypatch, FakeAMP(statuses=[blip] * streak + [limited] + [blip] * streak + [done()]))
+
+ eval_module.eval_crew()
+
+ assert "Goal gate: PASSED" in capsys.readouterr().out
+ assert amp.calls.count(("get", "ev-1")) == 2 * streak + 2
+
+
+ ("header", "wait"),
+ [
+ ("90", 90.0),
+ ("0", eval_module.POLL_SECONDS),
+ ("3600", eval_module.RATE_LIMIT_MAX_WAIT_SECONDS),
+ ("-5", 60.0),
+ ("nan", 60.0),
+ ("inf", 60.0),
+ ("-inf", 60.0),
+ ("soon", 60.0),
+ (None, 60.0),
+ ("Wed, 21 Oct 2015 07:28:00 GMT", eval_module.POLL_SECONDS), # a date already past
+ ],
+)
+def test_retry_after_reads_seconds_or_a_date_and_anything_else_waits_a_minute(header, wait):
+ headers = {"Retry-After": header} if header is not None else {}
+ assert eval_module._retry_after(httpx.Response(429, headers=headers)) == wait
+
+
+def test_retry_after_waits_until_a_date_still_ahead():
+ from email.utils import format_datetime
+
+ ahead = datetime.now(timezone.utc) + timedelta(seconds=95)
+ waited = eval_module._retry_after(httpx.Response(429, headers={"Retry-After": format_datetime(ahead, usegmt=True)}))
+ assert 90.0 <= waited <= 95.0
+
+
def test_a_refusal_mid_poll_names_the_evaluation_and_an_unknown_status_stops_the_wait(project, monkeypatch, capsys):
directory, _ = project
record_last_run(directory)