Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

fix(cli): crewai eval exits 1 unless the gate passed, and says to log in when nothing was traced unattended (#7806)

João Moura19d2ebcc8c2f
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.py
index 2c8f9e01f..04199729e 100644
--- a/lib/cli/src/crewai_cli/cli.py
+++ b/lib/cli/src/crewai_cli/cli.py
@@ -693,7 +693,11 @@ def run(
),
)
def eval_command(run_id: str | None) -> None:
- """Evaluate the last traced run through CrewAI AMP."""
+ """Evaluate the last traced run through CrewAI AMP.
+
+ Exits 0 only when the goal gate PASSED, and 1 otherwise — a failed gate, no
+ verdict, or an evaluation that could not run — so a CI job can gate on it.
+ """
eval_crew(run_id=run_id)
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 5fcda3a29..1af0f69fc 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -168,10 +168,19 @@ def eval_crew(run_id: str | None = None) -> None:
raise SystemExit(130) from None
_print_verdict(finished, url)
_say_where_the_criteria_live(write_eval_config(finished))
- if finished.get("status") != "done":
+ # The exit code is what a CI job reads, so it is the gate's: 0 only for a
+ # run that PASSED. A failed gate, one without a verdict, and an evaluation
+ # that stopped are all 1 — a pipeline that carried on past any of them would
+ # ship what the evaluation did not vouch for.
+ if not _gate_passed(finished):
raise SystemExit(1)
+def _gate_passed(finished: dict[str, Any]) -> bool:
+ verdict = finished.get("verdict") if finished.get("status") == "done" else None
+ return isinstance(verdict, dict) and str(verdict.get("gate")).lower() == "passed"
+
+
def _ran_just_now(record: dict[str, Any]) -> bool:
"""Did the project record this run in the last few minutes?
@@ -506,7 +515,9 @@ def _run_and_let_the_app_evaluate() -> str | None:
f" 1. add {TRACING_ENV_VAR}=true to .env\n 2. crewai run\n 3. crewai eval"
)
if is_dmn_mode_enabled() or not sys.stdin.isatty():
- console.print(steps, style="yellow")
+ # `Text`, never markup: the reason may be an OS error's own words, and
+ # its `[Errno 13]` would be read as a style tag.
+ console.print(Text(_nothing_traced_unattended() or steps), style="yellow")
raise SystemExit(1)
if not click.confirm(
"No traced run is recorded in this project. Turn tracing on and run the crew now? "
@@ -567,6 +578,32 @@ def _recorded_since(record: dict[str, Any], began: datetime) -> bool:
return when >= began - timedelta(seconds=1)
+def _nothing_traced_unattended() -> str | None:
+ """Why a run with tracing on left nothing to evaluate, when nobody was there.
+
+ An anonymous run asks before its trace leaves the machine, and a process with
+ no terminal has nobody to ask — so its trace is kept local, tracing on or
+ not. Telling that user to turn tracing on sends them round the same loop;
+ logging in is what makes an unattended run traced. None when tracing is off
+ or there is a login: the ordinary steps are the right ones then. A login
+ that cannot be read says so instead.
+ """
+ if os.environ.get(TRACING_ENV_VAR, "").strip().lower() not in ("true", "1"):
+ return None
+ try:
+ if saved_login() is not None:
+ return None
+ except EvaluationStoppedError as unreadable:
+ # A login that exists and cannot be read is the reason, and its
+ # sentence says what to do about it.
+ return str(unreadable)
+ return (
+ "No traced run is recorded in this project. Tracing is on, but a run nobody is "
+ "watching is only traced when you are logged in: run `crewai login`, then "
+ "`crewai run` and `crewai eval` again."
+ )
+
+
def _enable_tracing() -> None:
"""`CREWAI_TRACING_ENABLED=true` in the project's .env, and in this process for the run about to start."""
env_file = Path.cwd() / ".env"
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index 1061b3fe1..5873d37e0 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -241,13 +241,33 @@ def test_run_names_another_execution_and_an_anonymous_caller_sends_no_token(proj
monkeypatch.setattr(eval_module, "saved_login", lambda: None)
amp = install(monkeypatch, FakeAMP(statuses=[done("failed")]))
- eval_module.eval_crew(run_id="other-run")
+ with pytest.raises(SystemExit): # a failed gate is exit 1
+ eval_module.eval_crew(run_id="other-run")
assert amp.api_key is None
assert amp.calls[0] == ("create", "other-run")
assert "Goal gate: FAILED" in capsys.readouterr().out
+ "gate, code", [("passed", None), ("failed", 1), ("inconclusive", 1), ("unknown", 1)]
+)
+def test_the_exit_code_is_the_gates(project, monkeypatch, capsys, gate, code):
+ """A CI job reads the exit code. Only a gate that PASSED is 0: a failed one,
+ and one with no verdict, must stop the pipeline rather than wave it on."""
+ directory, _ = project
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(statuses=[done(gate)]))
+
+ if code is None:
+ eval_module.eval_crew()
+ else:
+ with pytest.raises(SystemExit) as exited:
+ eval_module.eval_crew()
+ assert exited.value.code == code
+ assert f"Goal gate: {gate.upper()}" in capsys.readouterr().out
+
+
def test_a_failed_evaluation_exits_one_with_amps_reason(project, monkeypatch, capsys):
directory, _ = project
record_last_run(directory)
@@ -1126,3 +1146,64 @@ def test_read_last_run_reads_the_record_crewai_writes(tmp_path):
record_last_run(tmp_path)
record = eval_module.read_last_run(tmp_path)
assert record is not None and record["execution_id"] == EXECUTION_ID and record["amp_base_url"] == "https://amp.test"
+
+
+ "tracing, login, says_login",
+ [
+ ("true", None, True), # tracing on, nobody logged in: log in
+ ("true", "tok", False), # logged in: the ordinary steps
+ (None, None, False), # tracing off: turn it on
+ ],
+)
+def test_an_unattended_run_with_nothing_traced_says_what_would_trace_it(
+ project, monkeypatch, capsys, tracing, login, says_login
+):
+ """With no terminal, an anonymous run's trace stays on the machine even with
+ tracing on. Telling that user to turn tracing on sends them round the same
+ loop; logging in is what makes an unattended run traced."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
+ monkeypatch.setattr(eval_module, "saved_login", lambda: login)
+ if tracing:
+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", tracing)
+ else:
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+
+ with pytest.raises(SystemExit) as exited:
+ eval_module.eval_crew()
+
+ out = capsys.readouterr().out.replace("\n", " ")
+ assert exited.value.code == 1
+ assert ("run `crewai login`" in out) is says_login
+ assert ("add CREWAI_TRACING_ENABLED=true" in out) is not says_login
+
+
+def test_an_unreadable_login_is_the_reason_given_when_nothing_was_traced(
+ project, monkeypatch, capsys
+):
+ """A login that exists and cannot be read is why an unattended run was not
+ traced, and its own sentence says what to do — not "turn tracing on"."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
+
+ def unreadable() -> None:
+ raise eval_module.EvaluationStoppedError(
+ "Could not read the saved login (PermissionError: [Errno 13] "
+ "Permission denied: [/Users/me/.config/crewai]). Run `crewai login` again"
+ )
+
+ monkeypatch.setattr(eval_module, "saved_login", unreadable)
+
+ with pytest.raises(SystemExit):
+ eval_module.eval_crew()
+
+ out = capsys.readouterr().out.replace("\n", " ")
+ assert "Could not read the saved login" in out
+ # printed as it is: `[/Users/…]` read as markup is a closing tag, and a crash
+ assert "[/Users/me/.config/crewai]" in out
+ assert "add CREWAI_TRACING_ENABLED=true" not in out
+