Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(cli): record why an evaluation stopped before its verdict (#7963)
João Mourab3c4409bc59c
diff --git a/lib/cli/src/crewai_cli/crew_run_tui.py b/lib/cli/src/crewai_cli/crew_run_tui.pyindex 1ceeda579..aa8db5847 100644--- a/lib/cli/src/crewai_cli/crew_run_tui.py+++ b/lib/cli/src/crewai_cli/crew_run_tui.py@@ -1355,6 +1355,9 @@ FooterKey .footer-key--key {self._record_tui_button_click("evaluate")traced = self._traced_execution_id()if traced is None:+ from crewai_cli.experimental.eval_crew import record_stop++ record_stop("untraced")self.notify("This run was not traced, so there is nothing to evaluate. ""Turn tracing on and run it again.",@@ -1387,6 +1390,7 @@ FooterKey .footer-key--key {from crewai_cli.experimental.eval_crew import (EvaluationStoppedError,evaluate_run,+ record_stop,)def back(handler: Callable[..., Any], *args: Any) -> None:@@ -1401,21 +1405,24 @@ FooterKey .footer-key--key {note=lambda text: back(self._evaluation_note, text),)except EvaluationStoppedError as stopped:- back(self._evaluation_failed, str(stopped))- return+ reason, message = stopped.reason, str(stopped)except SystemExit as exit_:# Nothing on this path should exit any more — a stop is a value# now — but an exit carries a code, not a reason, and "1" on screen# is worse than saying plainly that the reason did not survive.- back(- self._evaluation_failed,- f"The evaluation stopped without saying why (exit {exit_.code}).",- )- return+ reason = "unexpected"+ message = f"The evaluation stopped without saying why (exit {exit_.code})."except Exception as error: # a client bug is still an answer to show- back(self._evaluation_failed, f"{type(error).__name__}: {error}")+ reason, message = "unexpected", f"{type(error).__name__}: {error}"+ else:+ back(self._evaluation_finished, finished)return- back(self._evaluation_finished, finished)+ # Answered for, even though it stopped before AMP started it: a+ # `crewai eval` waiting behind this screen would otherwise try the run+ # again, and count the same stop twice.+ self._say_what_became_of_the_evaluation(execution_id)+ record_stop(reason)+ back(self._evaluation_failed, message)def _evaluation_started(self, started: dict[str, Any]) -> None:if self._evaluation is None:diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex 20b77e477..80b252c29 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -30,7 +30,7 @@ import osfrom pathlib import Pathimport sysimport time-from typing import Any, NoReturn, cast+from typing import Any, Literal, NoReturn, castfrom urllib.parse import urlparseimport uuidimport webbrowser@@ -98,6 +98,41 @@ def _record_usage(*, logged_in: bool) -> None:pass+StopReason = Literal[+ "untraced",+ "declined",+ "no_project",+ "login_unreadable",+ "network_error",+ "amp_401",+ "amp_403",+ "amp_404",+ "amp_4xx",+ "amp_5xx",+ "invalid_response",+ "evaluation_failed",+ "unexpected",+]+++def record_stop(reason: StopReason) -> None:+ """Count an evaluation that was asked for and did not reach a verdict, and why.++ `cli_usage:eval` counts the evaluations that started; this is the other+ side, so the gap between pressing Evaluate and seeing a verdict has a cause.+ The reason is a word from a closed list and never the message, which can+ carry AMP's own words or a local path.+ """+ try:+ from crewai_core.telemetry import Telemetry++ telemetry = Telemetry()+ telemetry.set_tracer()+ telemetry.feature_usage_span(f"cli_usage:eval_stopped:{reason}")+ except Exception: # noqa: S110 - telemetry must never break a command+ pass++NOT_TRACED = ("The run finished but no trace was recorded: the run may have failed, sharing the ""trace was declined, or this project's crewai is older than the version that records "@@ -110,9 +145,14 @@ class EvaluationStoppedError(RuntimeError):Raised rather than printed-and-exited, because the same two functions servethe terminal and the run app: one of them owns the screen, and a line- printed underneath it is a smear nobody asked for.+ printed underneath it is a smear nobody asked for. REASON says why in one+ word, for the usage count; an unclassified stop is "unexpected"."""+ def __init__(self, message: str, *, reason: StopReason = "unexpected") -> None:+ super().__init__(message)+ self.reason: StopReason = reason+def _note(text: str, style: str = "dim") -> None:console.print(Text(text), style=style)@@ -139,7 +179,7 @@ def eval_crew(run_id: str | None = None) -> None:try:client = _amp_client(trusted)except EvaluationStoppedError as stopped:- _fail(str(stopped))+ _stop(stopped)recorded_amp = str(record.get("amp_base_url") or "").rstrip("/")if not run_id and recorded_amp and recorded_amp != client.base_url.rstrip("/"):console.print(@@ -156,7 +196,7 @@ def eval_crew(run_id: str | None = None) -> None:project_id=project_id,)except EvaluationStoppedError as stopped:- _fail(str(stopped))+ _stop(stopped)# After, not before: `cli_usage:eval` counts an evaluation, and a refused# request — a run AMP does not hold, a credential it will not take — is not# one. `_start_evaluation` raises rather than returning on those.@@ -174,12 +214,14 @@ def eval_crew(run_id: str | None = None) -> None:try:finished = _wait(client, started["id"], url)except EvaluationStoppedError as stopped:- _fail(str(stopped))+ _stop(stopped)except KeyboardInterrupt:console.print(Text(f"\nStill running{f' at {url}' if url else ''}."), style="yellow")raise SystemExit(130) from None+ if finished.get("status") == "failed":+ record_stop("evaluation_failed")_print_verdict(finished, url)_print_brief(finished)_say_where_the_criteria_live(write_eval_config(finished))@@ -362,6 +404,8 @@ def evaluate_run(record_evaluation_outcome(execution_id)on_started(started)finished = _wait(client, started["id"], started.get("url"), on_status=on_status)+ if finished.get("status") == "failed":+ record_stop("evaluation_failed")written = write_eval_config(finished)if written is not None:finished = {**finished, "wrote_eval_config": written.name}@@ -513,7 +557,8 @@ def saved_login() -> str | None:# take the sentence with it.raise EvaluationStoppedError(f"Could not read the saved login ({type(error).__name__}: {error}). "- "Run `crewai login` again, or `crewai eval` will not know who you are."+ "Run `crewai login` again, or `crewai eval` will not know who you are.",+ reason="login_unreadable",) from error@@ -526,6 +571,7 @@ def _run_and_let_the_app_evaluate() -> str | None:flow that took the terminal), and says so itself when nothing was traced."""if not Path("pyproject.toml").is_file():+ record_stop("no_project")_fail("No crewAI project here (no pyproject.toml). Run `crewai eval` from the project's ""directory, or name a run: `crewai eval --run EXECUTION_ID`."@@ -538,13 +584,16 @@ def _run_and_let_the_app_evaluate() -> str | None:if is_dmn_mode_enabled() or not sys.stdin.isatty():# `Text`, never markup: the reason may be an OS error's own words, and# its `[Errno 13]` would be read as a style tag.- console.print(Text(_nothing_traced_unattended() or steps), style="yellow")+ reason, message = _nothing_traced_unattended() or ("untraced", steps)+ record_stop(reason)+ console.print(Text(message), style="yellow")raise SystemExit(1)if not click.confirm("No traced run is recorded in this project. Turn tracing on and run the crew now? ""The run's trace is sent to CrewAI AMP.",default=True, # João, 2026-09-20: y/n with Y as the default — the prompt says what Enter does):+ record_stop("declined")console.print(steps, style="yellow")raise SystemExit(0)@@ -569,6 +618,7 @@ def _run_and_let_the_app_evaluate() -> str | None:if marker is not None:if marker.get("execution_id"):return None+ record_stop("untraced")console.print(NOT_TRACED, style="bold red")raise SystemExit(1)@@ -582,6 +632,7 @@ def _run_and_let_the_app_evaluate() -> str | None:if traced:return str(traced)+ record_stop("untraced")console.print(NOT_TRACED, style="bold red")raise SystemExit(1)@@ -599,7 +650,7 @@ def _recorded_since(record: dict[str, Any], began: datetime) -> bool:return when >= began - timedelta(seconds=1)-def _nothing_traced_unattended() -> str | None:+def _nothing_traced_unattended() -> tuple[StopReason, str] | None:"""Why a run with tracing on left nothing to evaluate, when nobody was there.An anonymous run asks before its trace leaves the machine, and a process with@@ -607,7 +658,7 @@ def _nothing_traced_unattended() -> str | None:not. Telling that user to turn tracing on sends them round the same loop;logging in is what makes an unattended run traced. None when tracing is offor there is a login: the ordinary steps are the right ones then. A login- that cannot be read says so instead.+ that cannot be read says so instead, and is counted as that."""if os.environ.get(TRACING_ENV_VAR, "").strip().lower() not in ("true", "1"):return None@@ -617,8 +668,8 @@ def _nothing_traced_unattended() -> str | None:except EvaluationStoppedError as unreadable:# A login that exists and cannot be read is the reason, and its# sentence says what to do about it.- return str(unreadable)- return (+ return unreadable.reason, str(unreadable)+ return "untraced", ("No traced run is recorded in this project. Tracing is on, but a run nobody is ""watching is only traced when you are logged in: run `crewai login`, then ""`crewai run` and `crewai eval` again."@@ -657,7 +708,8 @@ def _start_evaluation()except httpx.HTTPError as error:raise EvaluationStoppedError(- f"Could not reach AMP to start the evaluation: {error}"+ f"Could not reach AMP to start the evaluation: {error}",+ reason="network_error",) from error# The run finished moments ago and its spans are still on their way:# waiting is the answer, not a 404 the reader can do nothing with.@@ -699,10 +751,12 @@ def _accepted(payload["url"] = urlreturn payloadraise EvaluationStoppedError(- f"AMP answered without an evaluation id ({response.status_code})."+ f"AMP answered without an evaluation id ({response.status_code}).",+ reason="invalid_response",)raise EvaluationStoppedError(- _refusal_message(response, subject, about_a_deployment=about_a_deployment)+ _refusal_message(response, subject, about_a_deployment=about_a_deployment),+ reason=_refusal_reason(response),)@@ -764,30 +818,39 @@ def _wait(misses += 1if misses >= POLL_RETRIES:raise EvaluationStoppedError(- f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}."+ f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}.",+ reason="network_error",) from errortime.sleep(POLL_SECONDS)continueif response.status_code >= 500:misses += 1if misses >= POLL_RETRIES:- raise EvaluationStoppedError(_refusal_message(response, subject))+ raise EvaluationStoppedError(+ _refusal_message(response, subject),+ reason=_refusal_reason(response),+ )time.sleep(POLL_SECONDS)continueif response.status_code != 200:- raise EvaluationStoppedError(_refusal_message(response, subject))+ raise EvaluationStoppedError(+ _refusal_message(response, subject),+ reason=_refusal_reason(response),+ )misses = 0payload = _payload(response) or {}status = payload.get("status")if status == "done" and not well_formed(payload):raise EvaluationStoppedError(- f"AMP answered done without a {answer} (protocol error); follow it{where or ' on AMP'}."+ f"AMP answered done without a {answer} (protocol error); follow it{where or ' on AMP'}.",+ reason="invalid_response",)if status in FINISHED:return payloadif status not in STATUSES:raise EvaluationStoppedError(- f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}."+ f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}.",+ reason="invalid_response",)if on_status is not None:on_status(payload)@@ -1371,6 +1434,27 @@ def _refused(response: httpx.Response, subject: str) -> None:_fail(_refusal_message(response, subject))+def _refusal_reason(response: httpx.Response) -> StopReason:+ """AMP's refusal by status class, for the usage count."""+ code = response.status_code+ if code == 401:+ return "amp_401"+ if code == 403:+ return "amp_403"+ if code == 404:+ return "amp_404"+ if 400 <= code < 500:+ return "amp_4xx"+ if code >= 500:+ return "amp_5xx"+ return "invalid_response"+++def _stop(stopped: EvaluationStoppedError) -> NoReturn:+ record_stop(stopped.reason)+ _fail(str(stopped))++def _fail(message: str) -> NoReturn:# `Text`, because most of what reaches here is AMP's own sentence and a# `Console` parses square brackets. No caller relies on markup; the colourdiff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex fb76fe9cc..8c263326e 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -1520,7 +1520,7 @@ def test_a_model_without_its_provider_exits_one_before_anything_is_sent(deployedassert "provider/model" in capsys.readouterr().out-def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, capsys):+def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, capsys, usage):failed = httpx.Response(200, json={"id": "ev-9", "status": "failed", "error": "the deployment answered 500"})install(monkeypatch, FakeModelsAMP(statuses=[failed]))@@ -1529,6 +1529,8 @@ def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, caassert stopped.value.code == 1assert "Comparison failed: the deployment answered 500" in capsys.readouterr().out+ # `--models` compares models; it is not the evaluation the stop count is about+ assert not [feature for feature in usage if "eval_stopped" in feature]def test_done_without_a_comparison_is_a_protocol_error(deployed, monkeypatch, capsys):@@ -1773,3 +1775,301 @@ def test_the_help_tells_an_agent_it_gets_a_brief():) in help_textassert "That is when AMP provides the brief: otherwise its link, when AMP sends only that, else nothing more" in help_textassert "it prints a markdown brief after the comparison when AMP provides one" in help_text+++# --- why an evaluation that was asked for never reached a verdict -----------++_REAL_SAVED_LOGIN = eval_module.saved_login+++def usage(monkeypatch):+ """Every usage feature counted, in order."""+ features: list[str] = []++ class FakeTelemetry:+ def set_tracer(self) -> None:+ pass++ def feature_usage_span(self, feature, attributes=None) -> None:+ features.append(feature)++ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)+ return features+++ ("status", "reason"),+ [(401, "amp_401"), (403, "amp_403"), (404, "amp_404"), (422, "amp_4xx"), (503, "amp_5xx")],+)+def test_a_refused_start_counts_why_and_says_what_it_always_said(+ project, monkeypatch, capsys, usage, status, reason+):+ directory, _ = project+ record_last_run(directory) # not a run that just ended: a 404 is not waited out+ refusal = httpx.Response(status, json={"detail": "refused for a reason AMP states"})+ install(monkeypatch, FakeAMP(create=refusal))++ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew()++ assert exited.value.code == 1+ assert usage == [f"cli_usage:eval_stopped:{reason}"]+ expected = eval_module._refusal_message(refusal, f"run {EXECUTION_ID}")+ assert expected in capsys.readouterr().out+++def test_an_unreachable_amp_at_the_start_counts_as_a_network_error(project, monkeypatch, usage):+ directory, _ = project+ record_last_run(directory)++ class Unreachable(FakeAMP):+ def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):+ raise httpx.ConnectError("no route to host")++ install(monkeypatch, Unreachable())++ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval_stopped:network_error"]+++def test_a_start_without_an_evaluation_id_counts_as_an_invalid_response(project, monkeypatch, usage):+ directory, _ = project+ record_last_run(directory)+ install(monkeypatch, FakeAMP(create=httpx.Response(202, json={"status": "queued"})))++ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval_stopped:invalid_response"]+++ ("answer", "reason"),+ [+ (httpx.Response(200, json={"id": "ev-1", "status": "mystery"}), "invalid_response"),+ (httpx.Response(200, json={"id": "ev-1", "status": "done"}), "invalid_response"),+ (httpx.Response(403, json={}), "amp_403"),+ ],+)+def test_an_evaluation_that_stops_while_waiting_was_counted_and_says_why(+ project, monkeypatch, usage, answer, reason+):+ """It started, so `cli_usage:eval` counted it; the stop is the second half."""+ directory, _ = project+ record_last_run(directory)+ install(monkeypatch, FakeAMP(statuses=[answer]))++ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval", f"cli_usage:eval_stopped:{reason}"]+++def test_losing_amp_while_waiting_counts_as_a_network_error(project, monkeypatch, usage):+ directory, _ = project+ record_last_run(directory)++ class GoneQuiet(FakeAMP):+ def get_evaluation(self, evaluation_id):+ raise httpx.ReadTimeout("no answer")++ install(monkeypatch, GoneQuiet())++ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval", "cli_usage:eval_stopped:network_error"]+++def test_a_saved_login_that_cannot_be_read_counts_as_such(project, monkeypatch, usage):+ directory, _ = project+ record_last_run(directory)+ monkeypatch.setattr(eval_module, "saved_login", _REAL_SAVED_LOGIN)++ def broken_store():+ raise OSError("permission denied")++ monkeypatch.setattr(eval_module, "get_auth_token", broken_store)+ install(monkeypatch, FakeAMP())++ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval_stopped:login_unreadable"]+++def test_an_evaluation_amp_failed_is_counted_and_says_what_it_always_said(+ project, monkeypatch, capsys, usage+):+ directory, _ = project+ record_last_run(directory)+ failed = httpx.Response(200, json={"id": "ev-1", "status": "failed", "error": "the judge was unreachable"})+ install(monkeypatch, FakeAMP(statuses=[failed]))++ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew()++ assert exited.value.code == 1+ assert usage == ["cli_usage:eval", "cli_usage:eval_stopped:evaluation_failed"]+ assert "Evaluation failed: the judge was unreachable" in capsys.readouterr().out+++def test_an_evaluation_amp_failed_in_the_run_app_is_counted_and_still_returned(+ project, monkeypatch, usage+):+ failed = httpx.Response(200, json={"id": "ev-1", "status": "failed", "error": "the judge was unreachable"})+ install(monkeypatch, FakeAMP(statuses=[failed]))++ finished = eval_module.evaluate_run(EXECUTION_ID, on_started=lambda started: None)++ assert finished["status"] == "failed" # the app still shows AMP's reason+ assert usage == ["cli_usage:eval", "cli_usage:eval_stopped:evaluation_failed"]+++def test_an_unreadable_login_with_nobody_to_ask_counts_as_such(+ project, monkeypatch, capsys, usage+):+ """Tracing on, no terminal, and a saved login that cannot be read: the login+ is the reason shown, so it is the reason counted — not "untraced"."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")+ monkeypatch.setattr(eval_module, "saved_login", _REAL_SAVED_LOGIN)++ def broken_store():+ raise OSError("permission denied")++ monkeypatch.setattr(eval_module, "get_auth_token", broken_store)++ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval_stopped:login_unreadable"]+ assert "crewai login" in capsys.readouterr().out+++def test_a_start_the_run_app_was_refused_is_not_tried_again_by_the_command(+ project, monkeypatch, usage+):+ """`crewai eval` runs the crew, the app it opens tries the evaluation and AMP+ refuses the start. The app has shown the refusal and counted it once; the+ command waiting behind it must not start the same run again."""+ import contextlib++ from crewai_cli.crew_run_tui import CrewRunApp++ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: True)+ monkeypatch.setattr(eval_module, "_enable_tracing", lambda: None)+ amp = install(monkeypatch, FakeAMP(create=httpx.Response(401, json={"detail": "Bad credentials"})))+ monkeypatch.setattr(+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)+ )+ shown: list[str] = []+ monkeypatch.setattr(CrewRunApp, "_evaluation_failed", lambda self, message: shown.append(message))+ holder: dict[str, str | None] = {"execution_id": None}++ @contextlib.contextmanager+ def watched():+ yield holder++ def the_app_runs_and_evaluates() -> None:+ holder["execution_id"] = EXECUTION_ID # the app saw the run traced+ CrewRunApp()._evaluate_now(EXECUTION_ID)++ monkeypatch.setattr("crewai_cli.crew_run_tui.evaluating_after_run", watched)+ monkeypatch.setattr(run_crew_module, "run_crew", the_app_runs_and_evaluates)++ eval_module.eval_crew()++ assert [call for call in amp.calls if call[0] == "create"] == [("create", EXECUTION_ID)]+ assert usage == ["cli_usage:eval_stopped:amp_401"]+ assert len(shown) == 1+++def test_outside_a_project_counts_as_no_project(project, usage):+ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval_stopped:no_project"]+++def test_nothing_traced_with_nobody_to_ask_counts_as_untraced(project, monkeypatch, usage):+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)++ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval_stopped:untraced"]+++def test_declining_to_run_with_tracing_on_counts_as_declined(project, monkeypatch, usage):+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: False)++ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew()++ assert exited.value.code == 0+ assert usage == ["cli_usage:eval_stopped:declined"]++[email protected]("app_saw_the_run", [True, False])+def test_a_run_that_left_no_trace_counts_as_untraced_once(+ project, monkeypatch, usage, app_saw_the_run+):+ """Whether the run app looked and found nothing, or no app got to the run,+ the command is the one that counts it; the app counts nothing here."""+ import contextlib++ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: True)+ monkeypatch.setattr(eval_module, "_enable_tracing", lambda: None)+ monkeypatch.setattr(run_crew_module, "run_crew", lambda: None)++ @contextlib.contextmanager+ def watched():+ yield {"execution_id": None}++ monkeypatch.setattr("crewai_cli.crew_run_tui.evaluating_after_run", watched)+ monkeypatch.setattr(+ eval_module,+ "evaluation_marker",+ lambda after: {"execution_id": None} if app_saw_the_run else None,+ )++ with pytest.raises(SystemExit, match="1"):+ eval_module.eval_crew()++ assert usage == ["cli_usage:eval_stopped:untraced"]+++def test_a_failing_usage_count_never_changes_what_the_command_does(project, monkeypatch, capsys):+ directory, _ = project+ record_last_run(directory)++ class BrokenTelemetry:+ def set_tracer(self) -> None:+ raise RuntimeError("exporter is down")++ monkeypatch.setattr("crewai_core.telemetry.Telemetry", BrokenTelemetry)+ refusal = httpx.Response(403, json={"detail": "not yours"})+ install(monkeypatch, FakeAMP(create=refusal))++ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew()++ assert exited.value.code == 1+ assert eval_module._refusal_message(refusal, f"run {EXECUTION_ID}") in capsys.readouterr().outdiff --git a/lib/cli/tests/test_crew_run_tui.py b/lib/cli/tests/test_crew_run_tui.pyindex c4a4b4514..4a5809817 100644--- a/lib/cli/tests/test_crew_run_tui.py+++ b/lib/cli/tests/test_crew_run_tui.py@@ -1,5 +1,5 @@import contextvars-from datetime import datetime+from datetime import datetime, timezoneimport timefrom types import SimpleNamespacefrom unittest.mock import Mock@@ -2352,3 +2352,109 @@ def test_a_failing_usage_count_never_changes_the_deploy(monkeypatch, capsys) ->assert calls == ["create", "login", "create"]assert "Deploy failed: authentication is still required." in capsys.readouterr().out+++# --- why an evaluation asked for on this screen never reached a verdict ------+++def stops(monkeypatch) -> list[str]:+ """The `cli_usage:eval_stopped:*` features counted, in order."""+ features: list[str] = []++ class FakeTelemetry:+ def set_tracer(self) -> None:+ pass++ def feature_usage_span(self, feature, attributes=None) -> None:+ features.append(feature)++ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)+ return features+++def test_pressing_evaluate_on_an_untraced_run_counts_the_press_and_why_it_stopped(+ monkeypatch, stops+) -> None:+ app = CrewRunApp()+ app._status = "completed"+ app._telemetry = Mock()+ app._execution_uuid = "run-this-app"+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: False)+ monkeypatch.setattr(CrewRunApp, "notify", lambda self, *a, **k: None)++ app.action_evaluate_crew()++ app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:evaluate")+ assert stops == ["cli_usage:eval_stopped:untraced"]+++def test_a_run_started_by_crewai_eval_leaves_the_untraced_count_to_the_command(+ monkeypatch, stops+) -> None:+ """The command waiting behind this screen says "not traced" and counts it;+ counting it here as well would count one stop twice."""+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: False)+ monkeypatch.setattr(CrewRunApp, "notify", lambda self, *a, **k: None)+ with crew_run_tui.evaluating_after_run():+ app = CrewRunApp()+ app._execution_uuid = "run-this-app"+ app._evaluate_if_one_is_waiting()++ assert stops == []+++ ("raised", "reason"),+ [+ ("amp_403", "amp_403"),+ (SystemExit(1), "unexpected"),+ (RuntimeError("client bug"), "unexpected"),+ ],+)+def test_an_evaluation_that_stops_on_this_screen_counts_why_and_answers_for_the_run(+ monkeypatch, tmp_path, stops, raised, reason+) -> None:+ """The marker tells a `crewai eval` waiting behind this screen that the run+ was answered for, so it neither starts it again nor counts the stop twice."""+ from crewai_cli.experimental.eval_crew import (+ EvaluationStoppedError,+ evaluation_marker,+ )++ if isinstance(raised, str):+ raised = EvaluationStoppedError("AMP answered 403.", reason=raised)+ app = CrewRunApp()+ app._evaluation = {"state": "starting"}+ monkeypatch.setattr(+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)+ )+ monkeypatch.setattr(+ "crewai_cli.experimental.eval_crew.evaluate_run", Mock(side_effect=raised)+ )++ monkeypatch.chdir(tmp_path)+ began = datetime.now(timezone.utc)++ app._evaluate_now("run-this-app")++ assert app._evaluation["state"] == "failed"+ assert stops == [f"cli_usage:eval_stopped:{reason}"]+ marker = evaluation_marker(after=began)+ assert marker is not None and marker["execution_id"] == "run-this-app"+++def test_an_evaluation_that_finishes_counts_no_stop(monkeypatch, stops) -> None:+ app = CrewRunApp()+ app._evaluation = {"state": "starting"}+ monkeypatch.setattr(+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)+ )+ monkeypatch.setattr(+ "crewai_cli.experimental.eval_crew.evaluate_run",+ Mock(return_value={"status": "done", "verdict": {"gate": "passed"}}),+ )++ app._evaluate_now("run-this-app")++ assert stops == []