Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix(cli): crewai eval exits 1 unless the gate passed, and says to log in when nothing was traced unattended (#7806)
João Moura19d2ebcc8c2f
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.pyindex 2c8f9e01f..04199729e 100644--- a/lib/cli/src/crewai_cli/cli.py+++ b/lib/cli/src/crewai_cli/cli.py@@ -693,7 +693,11 @@ def run(),)def eval_command(run_id: str | None) -> None:- """Evaluate the last traced run through CrewAI AMP."""+ """Evaluate the last traced run through CrewAI AMP.++ Exits 0 only when the goal gate PASSED, and 1 otherwise — a failed gate, no+ verdict, or an evaluation that could not run — so a CI job can gate on it.+ """eval_crew(run_id=run_id)diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex 5fcda3a29..1af0f69fc 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -168,10 +168,19 @@ def eval_crew(run_id: str | None = None) -> None:raise SystemExit(130) from None_print_verdict(finished, url)_say_where_the_criteria_live(write_eval_config(finished))- if finished.get("status") != "done":+ # The exit code is what a CI job reads, so it is the gate's: 0 only for a+ # run that PASSED. A failed gate, one without a verdict, and an evaluation+ # that stopped are all 1 — a pipeline that carried on past any of them would+ # ship what the evaluation did not vouch for.+ if not _gate_passed(finished):raise SystemExit(1)+def _gate_passed(finished: dict[str, Any]) -> bool:+ verdict = finished.get("verdict") if finished.get("status") == "done" else None+ return isinstance(verdict, dict) and str(verdict.get("gate")).lower() == "passed"++def _ran_just_now(record: dict[str, Any]) -> bool:"""Did the project record this run in the last few minutes?@@ -506,7 +515,9 @@ def _run_and_let_the_app_evaluate() -> str | None:f" 1. add {TRACING_ENV_VAR}=true to .env\n 2. crewai run\n 3. crewai eval")if is_dmn_mode_enabled() or not sys.stdin.isatty():- console.print(steps, style="yellow")+ # `Text`, never markup: the reason may be an OS error's own words, and+ # its `[Errno 13]` would be read as a style tag.+ console.print(Text(_nothing_traced_unattended() or steps), style="yellow")raise SystemExit(1)if not click.confirm("No traced run is recorded in this project. Turn tracing on and run the crew now? "@@ -567,6 +578,32 @@ def _recorded_since(record: dict[str, Any], began: datetime) -> bool:return when >= began - timedelta(seconds=1)+def _nothing_traced_unattended() -> str | None:+ """Why a run with tracing on left nothing to evaluate, when nobody was there.++ An anonymous run asks before its trace leaves the machine, and a process with+ no terminal has nobody to ask — so its trace is kept local, tracing on or+ not. Telling that user to turn tracing on sends them round the same loop;+ logging in is what makes an unattended run traced. None when tracing is off+ or there is a login: the ordinary steps are the right ones then. A login+ that cannot be read says so instead.+ """+ if os.environ.get(TRACING_ENV_VAR, "").strip().lower() not in ("true", "1"):+ return None+ try:+ if saved_login() is not None:+ return None+ except EvaluationStoppedError as unreadable:+ # A login that exists and cannot be read is the reason, and its+ # sentence says what to do about it.+ return str(unreadable)+ return (+ "No traced run is recorded in this project. Tracing is on, but a run nobody is "+ "watching is only traced when you are logged in: run `crewai login`, then "+ "`crewai run` and `crewai eval` again."+ )++def _enable_tracing() -> None:"""`CREWAI_TRACING_ENABLED=true` in the project's .env, and in this process for the run about to start."""env_file = Path.cwd() / ".env"diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex 1061b3fe1..5873d37e0 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -241,13 +241,33 @@ def test_run_names_another_execution_and_an_anonymous_caller_sends_no_token(projmonkeypatch.setattr(eval_module, "saved_login", lambda: None)amp = install(monkeypatch, FakeAMP(statuses=[done("failed")]))- eval_module.eval_crew(run_id="other-run")+ with pytest.raises(SystemExit): # a failed gate is exit 1+ eval_module.eval_crew(run_id="other-run")assert amp.api_key is Noneassert amp.calls[0] == ("create", "other-run")assert "Goal gate: FAILED" in capsys.readouterr().out+ "gate, code", [("passed", None), ("failed", 1), ("inconclusive", 1), ("unknown", 1)]+)+def test_the_exit_code_is_the_gates(project, monkeypatch, capsys, gate, code):+ """A CI job reads the exit code. Only a gate that PASSED is 0: a failed one,+ and one with no verdict, must stop the pipeline rather than wave it on."""+ directory, _ = project+ record_last_run(directory)+ install(monkeypatch, FakeAMP(statuses=[done(gate)]))++ if code is None:+ eval_module.eval_crew()+ else:+ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew()+ assert exited.value.code == code+ assert f"Goal gate: {gate.upper()}" in capsys.readouterr().out++def test_a_failed_evaluation_exits_one_with_amps_reason(project, monkeypatch, capsys):directory, _ = projectrecord_last_run(directory)@@ -1126,3 +1146,64 @@ def test_read_last_run_reads_the_record_crewai_writes(tmp_path):record_last_run(tmp_path)record = eval_module.read_last_run(tmp_path)assert record is not None and record["execution_id"] == EXECUTION_ID and record["amp_base_url"] == "https://amp.test"+++ "tracing, login, says_login",+ [+ ("true", None, True), # tracing on, nobody logged in: log in+ ("true", "tok", False), # logged in: the ordinary steps+ (None, None, False), # tracing off: turn it on+ ],+)+def test_an_unattended_run_with_nothing_traced_says_what_would_trace_it(+ project, monkeypatch, capsys, tracing, login, says_login+):+ """With no terminal, an anonymous run's trace stays on the machine even with+ tracing on. Telling that user to turn tracing on sends them round the same+ loop; logging in is what makes an unattended run traced."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)+ monkeypatch.setattr(eval_module, "saved_login", lambda: login)+ if tracing:+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", tracing)+ else:+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)++ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew()++ out = capsys.readouterr().out.replace("\n", " ")+ assert exited.value.code == 1+ assert ("run `crewai login`" in out) is says_login+ assert ("add CREWAI_TRACING_ENABLED=true" in out) is not says_login+++def test_an_unreadable_login_is_the_reason_given_when_nothing_was_traced(+ project, monkeypatch, capsys+):+ """A login that exists and cannot be read is why an unattended run was not+ traced, and its own sentence says what to do — not "turn tracing on"."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")++ def unreadable() -> None:+ raise eval_module.EvaluationStoppedError(+ "Could not read the saved login (PermissionError: [Errno 13] "+ "Permission denied: [/Users/me/.config/crewai]). Run `crewai login` again"+ )++ monkeypatch.setattr(eval_module, "saved_login", unreadable)++ with pytest.raises(SystemExit):+ eval_module.eval_crew()++ out = capsys.readouterr().out.replace("\n", " ")+ assert "Could not read the saved login" in out+ # printed as it is: `[/Users/…]` read as markup is a closing tag, and a crash+ assert "[/Users/me/.config/crewai]" in out+ assert "add CREWAI_TRACING_ENABLED=true" not in out+