Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(cli): record why an evaluation stopped before its verdict (#7963)

João Mourab3c4409bc59c
diff --git a/lib/cli/src/crewai_cli/crew_run_tui.py b/lib/cli/src/crewai_cli/crew_run_tui.py
index 1ceeda579..aa8db5847 100644
--- a/lib/cli/src/crewai_cli/crew_run_tui.py
+++ b/lib/cli/src/crewai_cli/crew_run_tui.py
@@ -1355,6 +1355,9 @@ FooterKey .footer-key--key {
self._record_tui_button_click("evaluate")
traced = self._traced_execution_id()
if traced is None:
+ from crewai_cli.experimental.eval_crew import record_stop
+
+ record_stop("untraced")
self.notify(
"This run was not traced, so there is nothing to evaluate. "
"Turn tracing on and run it again.",
@@ -1387,6 +1390,7 @@ FooterKey .footer-key--key {
from crewai_cli.experimental.eval_crew import (
EvaluationStoppedError,
evaluate_run,
+ record_stop,
)
def back(handler: Callable[..., Any], *args: Any) -> None:
@@ -1401,21 +1405,24 @@ FooterKey .footer-key--key {
note=lambda text: back(self._evaluation_note, text),
)
except EvaluationStoppedError as stopped:
- back(self._evaluation_failed, str(stopped))
- return
+ reason, message = stopped.reason, str(stopped)
except SystemExit as exit_:
# Nothing on this path should exit any more — a stop is a value
# now — but an exit carries a code, not a reason, and "1" on screen
# is worse than saying plainly that the reason did not survive.
- back(
- self._evaluation_failed,
- f"The evaluation stopped without saying why (exit {exit_.code}).",
- )
- return
+ reason = "unexpected"
+ message = f"The evaluation stopped without saying why (exit {exit_.code})."
except Exception as error: # a client bug is still an answer to show
- back(self._evaluation_failed, f"{type(error).__name__}: {error}")
+ reason, message = "unexpected", f"{type(error).__name__}: {error}"
+ else:
+ back(self._evaluation_finished, finished)
return
- back(self._evaluation_finished, finished)
+ # Answered for, even though it stopped before AMP started it: a
+ # `crewai eval` waiting behind this screen would otherwise try the run
+ # again, and count the same stop twice.
+ self._say_what_became_of_the_evaluation(execution_id)
+ record_stop(reason)
+ back(self._evaluation_failed, message)
def _evaluation_started(self, started: dict[str, Any]) -> None:
if self._evaluation is None:
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 20b77e477..80b252c29 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -30,7 +30,7 @@ import os
from pathlib import Path
import sys
import time
-from typing import Any, NoReturn, cast
+from typing import Any, Literal, NoReturn, cast
from urllib.parse import urlparse
import uuid
import webbrowser
@@ -98,6 +98,41 @@ def _record_usage(*, logged_in: bool) -> None:
pass
+StopReason = Literal[
+ "untraced",
+ "declined",
+ "no_project",
+ "login_unreadable",
+ "network_error",
+ "amp_401",
+ "amp_403",
+ "amp_404",
+ "amp_4xx",
+ "amp_5xx",
+ "invalid_response",
+ "evaluation_failed",
+ "unexpected",
+]
+
+
+def record_stop(reason: StopReason) -> None:
+ """Count an evaluation that was asked for and did not reach a verdict, and why.
+
+ `cli_usage:eval` counts the evaluations that started; this is the other
+ side, so the gap between pressing Evaluate and seeing a verdict has a cause.
+ The reason is a word from a closed list and never the message, which can
+ carry AMP's own words or a local path.
+ """
+ try:
+ from crewai_core.telemetry import Telemetry
+
+ telemetry = Telemetry()
+ telemetry.set_tracer()
+ telemetry.feature_usage_span(f"cli_usage:eval_stopped:{reason}")
+ except Exception: # noqa: S110 - telemetry must never break a command
+ pass
+
+
NOT_TRACED = (
"The run finished but no trace was recorded: the run may have failed, sharing the "
"trace was declined, or this project's crewai is older than the version that records "
@@ -110,9 +145,14 @@ class EvaluationStoppedError(RuntimeError):
Raised rather than printed-and-exited, because the same two functions serve
the terminal and the run app: one of them owns the screen, and a line
- printed underneath it is a smear nobody asked for.
+ printed underneath it is a smear nobody asked for. REASON says why in one
+ word, for the usage count; an unclassified stop is "unexpected".
"""
+ def __init__(self, message: str, *, reason: StopReason = "unexpected") -> None:
+ super().__init__(message)
+ self.reason: StopReason = reason
+
def _note(text: str, style: str = "dim") -> None:
console.print(Text(text), style=style)
@@ -139,7 +179,7 @@ def eval_crew(run_id: str | None = None) -> None:
try:
client = _amp_client(trusted)
except EvaluationStoppedError as stopped:
- _fail(str(stopped))
+ _stop(stopped)
recorded_amp = str(record.get("amp_base_url") or "").rstrip("/")
if not run_id and recorded_amp and recorded_amp != client.base_url.rstrip("/"):
console.print(
@@ -156,7 +196,7 @@ def eval_crew(run_id: str | None = None) -> None:
project_id=project_id,
)
except EvaluationStoppedError as stopped:
- _fail(str(stopped))
+ _stop(stopped)
# After, not before: `cli_usage:eval` counts an evaluation, and a refused
# request — a run AMP does not hold, a credential it will not take — is not
# one. `_start_evaluation` raises rather than returning on those.
@@ -174,12 +214,14 @@ def eval_crew(run_id: str | None = None) -> None:
try:
finished = _wait(client, started["id"], url)
except EvaluationStoppedError as stopped:
- _fail(str(stopped))
+ _stop(stopped)
except KeyboardInterrupt:
console.print(
Text(f"\nStill running{f' at {url}' if url else ''}."), style="yellow"
)
raise SystemExit(130) from None
+ if finished.get("status") == "failed":
+ record_stop("evaluation_failed")
_print_verdict(finished, url)
_print_brief(finished)
_say_where_the_criteria_live(write_eval_config(finished))
@@ -362,6 +404,8 @@ def evaluate_run(
record_evaluation_outcome(execution_id)
on_started(started)
finished = _wait(client, started["id"], started.get("url"), on_status=on_status)
+ if finished.get("status") == "failed":
+ record_stop("evaluation_failed")
written = write_eval_config(finished)
if written is not None:
finished = {**finished, "wrote_eval_config": written.name}
@@ -513,7 +557,8 @@ def saved_login() -> str | None:
# take the sentence with it.
raise EvaluationStoppedError(
f"Could not read the saved login ({type(error).__name__}: {error}). "
- "Run `crewai login` again, or `crewai eval` will not know who you are."
+ "Run `crewai login` again, or `crewai eval` will not know who you are.",
+ reason="login_unreadable",
) from error
@@ -526,6 +571,7 @@ def _run_and_let_the_app_evaluate() -> str | None:
flow that took the terminal), and says so itself when nothing was traced.
"""
if not Path("pyproject.toml").is_file():
+ record_stop("no_project")
_fail(
"No crewAI project here (no pyproject.toml). Run `crewai eval` from the project's "
"directory, or name a run: `crewai eval --run EXECUTION_ID`."
@@ -538,13 +584,16 @@ def _run_and_let_the_app_evaluate() -> str | None:
if is_dmn_mode_enabled() or not sys.stdin.isatty():
# `Text`, never markup: the reason may be an OS error's own words, and
# its `[Errno 13]` would be read as a style tag.
- console.print(Text(_nothing_traced_unattended() or steps), style="yellow")
+ reason, message = _nothing_traced_unattended() or ("untraced", steps)
+ record_stop(reason)
+ console.print(Text(message), style="yellow")
raise SystemExit(1)
if not click.confirm(
"No traced run is recorded in this project. Turn tracing on and run the crew now? "
"The run's trace is sent to CrewAI AMP.",
default=True, # João, 2026-09-20: y/n with Y as the default — the prompt says what Enter does
):
+ record_stop("declined")
console.print(steps, style="yellow")
raise SystemExit(0)
@@ -569,6 +618,7 @@ def _run_and_let_the_app_evaluate() -> str | None:
if marker is not None:
if marker.get("execution_id"):
return None
+ record_stop("untraced")
console.print(NOT_TRACED, style="bold red")
raise SystemExit(1)
@@ -582,6 +632,7 @@ def _run_and_let_the_app_evaluate() -> str | None:
if traced:
return str(traced)
+ record_stop("untraced")
console.print(NOT_TRACED, style="bold red")
raise SystemExit(1)
@@ -599,7 +650,7 @@ def _recorded_since(record: dict[str, Any], began: datetime) -> bool:
return when >= began - timedelta(seconds=1)
-def _nothing_traced_unattended() -> str | None:
+def _nothing_traced_unattended() -> tuple[StopReason, str] | None:
"""Why a run with tracing on left nothing to evaluate, when nobody was there.
An anonymous run asks before its trace leaves the machine, and a process with
@@ -607,7 +658,7 @@ def _nothing_traced_unattended() -> str | None:
not. Telling that user to turn tracing on sends them round the same loop;
logging in is what makes an unattended run traced. None when tracing is off
or there is a login: the ordinary steps are the right ones then. A login
- that cannot be read says so instead.
+ that cannot be read says so instead, and is counted as that.
"""
if os.environ.get(TRACING_ENV_VAR, "").strip().lower() not in ("true", "1"):
return None
@@ -617,8 +668,8 @@ def _nothing_traced_unattended() -> str | None:
except EvaluationStoppedError as unreadable:
# A login that exists and cannot be read is the reason, and its
# sentence says what to do about it.
- return str(unreadable)
- return (
+ return unreadable.reason, str(unreadable)
+ return "untraced", (
"No traced run is recorded in this project. Tracing is on, but a run nobody is "
"watching is only traced when you are logged in: run `crewai login`, then "
"`crewai run` and `crewai eval` again."
@@ -657,7 +708,8 @@ def _start_evaluation(
)
except httpx.HTTPError as error:
raise EvaluationStoppedError(
- f"Could not reach AMP to start the evaluation: {error}"
+ f"Could not reach AMP to start the evaluation: {error}",
+ reason="network_error",
) from error
# The run finished moments ago and its spans are still on their way:
# waiting is the answer, not a 404 the reader can do nothing with.
@@ -699,10 +751,12 @@ def _accepted(
payload["url"] = url
return payload
raise EvaluationStoppedError(
- f"AMP answered without an evaluation id ({response.status_code})."
+ f"AMP answered without an evaluation id ({response.status_code}).",
+ reason="invalid_response",
)
raise EvaluationStoppedError(
- _refusal_message(response, subject, about_a_deployment=about_a_deployment)
+ _refusal_message(response, subject, about_a_deployment=about_a_deployment),
+ reason=_refusal_reason(response),
)
@@ -764,30 +818,39 @@ def _wait(
misses += 1
if misses >= POLL_RETRIES:
raise EvaluationStoppedError(
- f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}."
+ f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}.",
+ reason="network_error",
) from error
time.sleep(POLL_SECONDS)
continue
if response.status_code >= 500:
misses += 1
if misses >= POLL_RETRIES:
- raise EvaluationStoppedError(_refusal_message(response, subject))
+ raise EvaluationStoppedError(
+ _refusal_message(response, subject),
+ reason=_refusal_reason(response),
+ )
time.sleep(POLL_SECONDS)
continue
if response.status_code != 200:
- raise EvaluationStoppedError(_refusal_message(response, subject))
+ raise EvaluationStoppedError(
+ _refusal_message(response, subject),
+ reason=_refusal_reason(response),
+ )
misses = 0
payload = _payload(response) or {}
status = payload.get("status")
if status == "done" and not well_formed(payload):
raise EvaluationStoppedError(
- f"AMP answered done without a {answer} (protocol error); follow it{where or ' on AMP'}."
+ f"AMP answered done without a {answer} (protocol error); follow it{where or ' on AMP'}.",
+ reason="invalid_response",
)
if status in FINISHED:
return payload
if status not in STATUSES:
raise EvaluationStoppedError(
- f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}."
+ f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}.",
+ reason="invalid_response",
)
if on_status is not None:
on_status(payload)
@@ -1371,6 +1434,27 @@ def _refused(response: httpx.Response, subject: str) -> None:
_fail(_refusal_message(response, subject))
+def _refusal_reason(response: httpx.Response) -> StopReason:
+ """AMP's refusal by status class, for the usage count."""
+ code = response.status_code
+ if code == 401:
+ return "amp_401"
+ if code == 403:
+ return "amp_403"
+ if code == 404:
+ return "amp_404"
+ if 400 <= code < 500:
+ return "amp_4xx"
+ if code >= 500:
+ return "amp_5xx"
+ return "invalid_response"
+
+
+def _stop(stopped: EvaluationStoppedError) -> NoReturn:
+ record_stop(stopped.reason)
+ _fail(str(stopped))
+
+
def _fail(message: str) -> NoReturn:
# `Text`, because most of what reaches here is AMP's own sentence and a
# `Console` parses square brackets. No caller relies on markup; the colour
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index fb76fe9cc..8c263326e 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -1520,7 +1520,7 @@ def test_a_model_without_its_provider_exits_one_before_anything_is_sent(deployed
assert "provider/model" in capsys.readouterr().out
-def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, capsys):
+def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, capsys, usage):
failed = httpx.Response(200, json={"id": "ev-9", "status": "failed", "error": "the deployment answered 500"})
install(monkeypatch, FakeModelsAMP(statuses=[failed]))
@@ -1529,6 +1529,8 @@ def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, ca
assert stopped.value.code == 1
assert "Comparison failed: the deployment answered 500" in capsys.readouterr().out
+ # `--models` compares models; it is not the evaluation the stop count is about
+ assert not [feature for feature in usage if "eval_stopped" in feature]
def test_done_without_a_comparison_is_a_protocol_error(deployed, monkeypatch, capsys):
@@ -1773,3 +1775,301 @@ def test_the_help_tells_an_agent_it_gets_a_brief():
) in help_text
assert "That is when AMP provides the brief: otherwise its link, when AMP sends only that, else nothing more" in help_text
assert "it prints a markdown brief after the comparison when AMP provides one" in help_text
+
+
+# --- why an evaluation that was asked for never reached a verdict -----------
+
+_REAL_SAVED_LOGIN = eval_module.saved_login
+
+
+def usage(monkeypatch):
+ """Every usage feature counted, in order."""
+ features: list[str] = []
+
+ class FakeTelemetry:
+ def set_tracer(self) -> None:
+ pass
+
+ def feature_usage_span(self, feature, attributes=None) -> None:
+ features.append(feature)
+
+ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)
+ return features
+
+
+ ("status", "reason"),
+ [(401, "amp_401"), (403, "amp_403"), (404, "amp_404"), (422, "amp_4xx"), (503, "amp_5xx")],
+)
+def test_a_refused_start_counts_why_and_says_what_it_always_said(
+ project, monkeypatch, capsys, usage, status, reason
+):
+ directory, _ = project
+ record_last_run(directory) # not a run that just ended: a 404 is not waited out
+ refusal = httpx.Response(status, json={"detail": "refused for a reason AMP states"})
+ install(monkeypatch, FakeAMP(create=refusal))
+
+ with pytest.raises(SystemExit) as exited:
+ eval_module.eval_crew()
+
+ assert exited.value.code == 1
+ assert usage == [f"cli_usage:eval_stopped:{reason}"]
+ expected = eval_module._refusal_message(refusal, f"run {EXECUTION_ID}")
+ assert expected in capsys.readouterr().out
+
+
+def test_an_unreachable_amp_at_the_start_counts_as_a_network_error(project, monkeypatch, usage):
+ directory, _ = project
+ record_last_run(directory)
+
+ class Unreachable(FakeAMP):
+ def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):
+ raise httpx.ConnectError("no route to host")
+
+ install(monkeypatch, Unreachable())
+
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval_stopped:network_error"]
+
+
+def test_a_start_without_an_evaluation_id_counts_as_an_invalid_response(project, monkeypatch, usage):
+ directory, _ = project
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(create=httpx.Response(202, json={"status": "queued"})))
+
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval_stopped:invalid_response"]
+
+
+ ("answer", "reason"),
+ [
+ (httpx.Response(200, json={"id": "ev-1", "status": "mystery"}), "invalid_response"),
+ (httpx.Response(200, json={"id": "ev-1", "status": "done"}), "invalid_response"),
+ (httpx.Response(403, json={}), "amp_403"),
+ ],
+)
+def test_an_evaluation_that_stops_while_waiting_was_counted_and_says_why(
+ project, monkeypatch, usage, answer, reason
+):
+ """It started, so `cli_usage:eval` counted it; the stop is the second half."""
+ directory, _ = project
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(statuses=[answer]))
+
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval", f"cli_usage:eval_stopped:{reason}"]
+
+
+def test_losing_amp_while_waiting_counts_as_a_network_error(project, monkeypatch, usage):
+ directory, _ = project
+ record_last_run(directory)
+
+ class GoneQuiet(FakeAMP):
+ def get_evaluation(self, evaluation_id):
+ raise httpx.ReadTimeout("no answer")
+
+ install(monkeypatch, GoneQuiet())
+
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval", "cli_usage:eval_stopped:network_error"]
+
+
+def test_a_saved_login_that_cannot_be_read_counts_as_such(project, monkeypatch, usage):
+ directory, _ = project
+ record_last_run(directory)
+ monkeypatch.setattr(eval_module, "saved_login", _REAL_SAVED_LOGIN)
+
+ def broken_store():
+ raise OSError("permission denied")
+
+ monkeypatch.setattr(eval_module, "get_auth_token", broken_store)
+ install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval_stopped:login_unreadable"]
+
+
+def test_an_evaluation_amp_failed_is_counted_and_says_what_it_always_said(
+ project, monkeypatch, capsys, usage
+):
+ directory, _ = project
+ record_last_run(directory)
+ failed = httpx.Response(200, json={"id": "ev-1", "status": "failed", "error": "the judge was unreachable"})
+ install(monkeypatch, FakeAMP(statuses=[failed]))
+
+ with pytest.raises(SystemExit) as exited:
+ eval_module.eval_crew()
+
+ assert exited.value.code == 1
+ assert usage == ["cli_usage:eval", "cli_usage:eval_stopped:evaluation_failed"]
+ assert "Evaluation failed: the judge was unreachable" in capsys.readouterr().out
+
+
+def test_an_evaluation_amp_failed_in_the_run_app_is_counted_and_still_returned(
+ project, monkeypatch, usage
+):
+ failed = httpx.Response(200, json={"id": "ev-1", "status": "failed", "error": "the judge was unreachable"})
+ install(monkeypatch, FakeAMP(statuses=[failed]))
+
+ finished = eval_module.evaluate_run(EXECUTION_ID, on_started=lambda started: None)
+
+ assert finished["status"] == "failed" # the app still shows AMP's reason
+ assert usage == ["cli_usage:eval", "cli_usage:eval_stopped:evaluation_failed"]
+
+
+def test_an_unreadable_login_with_nobody_to_ask_counts_as_such(
+ project, monkeypatch, capsys, usage
+):
+ """Tracing on, no terminal, and a saved login that cannot be read: the login
+ is the reason shown, so it is the reason counted — not "untraced"."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
+ monkeypatch.setattr(eval_module, "saved_login", _REAL_SAVED_LOGIN)
+
+ def broken_store():
+ raise OSError("permission denied")
+
+ monkeypatch.setattr(eval_module, "get_auth_token", broken_store)
+
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval_stopped:login_unreadable"]
+ assert "crewai login" in capsys.readouterr().out
+
+
+def test_a_start_the_run_app_was_refused_is_not_tried_again_by_the_command(
+ project, monkeypatch, usage
+):
+ """`crewai eval` runs the crew, the app it opens tries the evaluation and AMP
+ refuses the start. The app has shown the refusal and counted it once; the
+ command waiting behind it must not start the same run again."""
+ import contextlib
+
+ from crewai_cli.crew_run_tui import CrewRunApp
+
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: True)
+ monkeypatch.setattr(eval_module, "_enable_tracing", lambda: None)
+ amp = install(monkeypatch, FakeAMP(create=httpx.Response(401, json={"detail": "Bad credentials"})))
+ monkeypatch.setattr(
+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)
+ )
+ shown: list[str] = []
+ monkeypatch.setattr(CrewRunApp, "_evaluation_failed", lambda self, message: shown.append(message))
+ holder: dict[str, str | None] = {"execution_id": None}
+
+ @contextlib.contextmanager
+ def watched():
+ yield holder
+
+ def the_app_runs_and_evaluates() -> None:
+ holder["execution_id"] = EXECUTION_ID # the app saw the run traced
+ CrewRunApp()._evaluate_now(EXECUTION_ID)
+
+ monkeypatch.setattr("crewai_cli.crew_run_tui.evaluating_after_run", watched)
+ monkeypatch.setattr(run_crew_module, "run_crew", the_app_runs_and_evaluates)
+
+ eval_module.eval_crew()
+
+ assert [call for call in amp.calls if call[0] == "create"] == [("create", EXECUTION_ID)]
+ assert usage == ["cli_usage:eval_stopped:amp_401"]
+ assert len(shown) == 1
+
+
+def test_outside_a_project_counts_as_no_project(project, usage):
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval_stopped:no_project"]
+
+
+def test_nothing_traced_with_nobody_to_ask_counts_as_untraced(project, monkeypatch, usage):
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
+
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval_stopped:untraced"]
+
+
+def test_declining_to_run_with_tracing_on_counts_as_declined(project, monkeypatch, usage):
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: False)
+
+ with pytest.raises(SystemExit) as exited:
+ eval_module.eval_crew()
+
+ assert exited.value.code == 0
+ assert usage == ["cli_usage:eval_stopped:declined"]
+
+
[email protected]("app_saw_the_run", [True, False])
+def test_a_run_that_left_no_trace_counts_as_untraced_once(
+ project, monkeypatch, usage, app_saw_the_run
+):
+ """Whether the run app looked and found nothing, or no app got to the run,
+ the command is the one that counts it; the app counts nothing here."""
+ import contextlib
+
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: True)
+ monkeypatch.setattr(eval_module, "_enable_tracing", lambda: None)
+ monkeypatch.setattr(run_crew_module, "run_crew", lambda: None)
+
+ @contextlib.contextmanager
+ def watched():
+ yield {"execution_id": None}
+
+ monkeypatch.setattr("crewai_cli.crew_run_tui.evaluating_after_run", watched)
+ monkeypatch.setattr(
+ eval_module,
+ "evaluation_marker",
+ lambda after: {"execution_id": None} if app_saw_the_run else None,
+ )
+
+ with pytest.raises(SystemExit, match="1"):
+ eval_module.eval_crew()
+
+ assert usage == ["cli_usage:eval_stopped:untraced"]
+
+
+def test_a_failing_usage_count_never_changes_what_the_command_does(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+
+ class BrokenTelemetry:
+ def set_tracer(self) -> None:
+ raise RuntimeError("exporter is down")
+
+ monkeypatch.setattr("crewai_core.telemetry.Telemetry", BrokenTelemetry)
+ refusal = httpx.Response(403, json={"detail": "not yours"})
+ install(monkeypatch, FakeAMP(create=refusal))
+
+ with pytest.raises(SystemExit) as exited:
+ eval_module.eval_crew()
+
+ assert exited.value.code == 1
+ assert eval_module._refusal_message(refusal, f"run {EXECUTION_ID}") in capsys.readouterr().out
diff --git a/lib/cli/tests/test_crew_run_tui.py b/lib/cli/tests/test_crew_run_tui.py
index c4a4b4514..4a5809817 100644
--- a/lib/cli/tests/test_crew_run_tui.py
+++ b/lib/cli/tests/test_crew_run_tui.py
@@ -1,5 +1,5 @@
import contextvars
-from datetime import datetime
+from datetime import datetime, timezone
import time
from types import SimpleNamespace
from unittest.mock import Mock
@@ -2352,3 +2352,109 @@ def test_a_failing_usage_count_never_changes_the_deploy(monkeypatch, capsys) ->
assert calls == ["create", "login", "create"]
assert "Deploy failed: authentication is still required." in capsys.readouterr().out
+
+
+# --- why an evaluation asked for on this screen never reached a verdict ------
+
+
+def stops(monkeypatch) -> list[str]:
+ """The `cli_usage:eval_stopped:*` features counted, in order."""
+ features: list[str] = []
+
+ class FakeTelemetry:
+ def set_tracer(self) -> None:
+ pass
+
+ def feature_usage_span(self, feature, attributes=None) -> None:
+ features.append(feature)
+
+ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)
+ return features
+
+
+def test_pressing_evaluate_on_an_untraced_run_counts_the_press_and_why_it_stopped(
+ monkeypatch, stops
+) -> None:
+ app = CrewRunApp()
+ app._status = "completed"
+ app._telemetry = Mock()
+ app._execution_uuid = "run-this-app"
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: False)
+ monkeypatch.setattr(CrewRunApp, "notify", lambda self, *a, **k: None)
+
+ app.action_evaluate_crew()
+
+ app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:evaluate")
+ assert stops == ["cli_usage:eval_stopped:untraced"]
+
+
+def test_a_run_started_by_crewai_eval_leaves_the_untraced_count_to_the_command(
+ monkeypatch, stops
+) -> None:
+ """The command waiting behind this screen says "not traced" and counts it;
+ counting it here as well would count one stop twice."""
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: False)
+ monkeypatch.setattr(CrewRunApp, "notify", lambda self, *a, **k: None)
+ with crew_run_tui.evaluating_after_run():
+ app = CrewRunApp()
+ app._execution_uuid = "run-this-app"
+ app._evaluate_if_one_is_waiting()
+
+ assert stops == []
+
+
+ ("raised", "reason"),
+ [
+ ("amp_403", "amp_403"),
+ (SystemExit(1), "unexpected"),
+ (RuntimeError("client bug"), "unexpected"),
+ ],
+)
+def test_an_evaluation_that_stops_on_this_screen_counts_why_and_answers_for_the_run(
+ monkeypatch, tmp_path, stops, raised, reason
+) -> None:
+ """The marker tells a `crewai eval` waiting behind this screen that the run
+ was answered for, so it neither starts it again nor counts the stop twice."""
+ from crewai_cli.experimental.eval_crew import (
+ EvaluationStoppedError,
+ evaluation_marker,
+ )
+
+ if isinstance(raised, str):
+ raised = EvaluationStoppedError("AMP answered 403.", reason=raised)
+ app = CrewRunApp()
+ app._evaluation = {"state": "starting"}
+ monkeypatch.setattr(
+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)
+ )
+ monkeypatch.setattr(
+ "crewai_cli.experimental.eval_crew.evaluate_run", Mock(side_effect=raised)
+ )
+
+ monkeypatch.chdir(tmp_path)
+ began = datetime.now(timezone.utc)
+
+ app._evaluate_now("run-this-app")
+
+ assert app._evaluation["state"] == "failed"
+ assert stops == [f"cli_usage:eval_stopped:{reason}"]
+ marker = evaluation_marker(after=began)
+ assert marker is not None and marker["execution_id"] == "run-this-app"
+
+
+def test_an_evaluation_that_finishes_counts_no_stop(monkeypatch, stops) -> None:
+ app = CrewRunApp()
+ app._evaluation = {"state": "starting"}
+ monkeypatch.setattr(
+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)
+ )
+ monkeypatch.setattr(
+ "crewai_cli.experimental.eval_crew.evaluate_run",
+ Mock(return_value={"status": "done", "verdict": {"gate": "passed"}}),
+ )
+
+ app._evaluate_now("run-this-app")
+
+ assert stops == []