Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

fix(cli): crewai eval says why before it asks, and its help says when it asks (#8301)

João Mouraaaa33e03e09e
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.py
index b68ba8654..05f3faeac 100644
--- a/lib/cli/src/crewai_cli/cli.py
+++ b/lib/cli/src/crewai_cli/cli.py
@@ -750,9 +750,12 @@ def eval_command(
It grades the project's last traced run. When there is none, or the project
changed since it, it runs the crew first and grades that run: the run is
- traced and its trace sent to CrewAI AMP, which grades it. Nothing is asked,
- with or without a terminal, so `crewai eval` alone takes a coding agent from
- a change to its verdict.
+ traced and its trace sent to CrewAI AMP, which grades it. The first time
+ tracing is turned on for the project, a person at a terminal is asked
+ first; with no terminal (a coding agent, a script) nothing is asked and the
+ command says it turned tracing on, so `crewai eval` alone takes an agent
+ from a change to its verdict. A project that turned tracing off is asked
+ at a terminal, and told how to turn it on without one.
Run by a script or coding agent (no terminal), it prints a markdown brief
after the verdict — what failed and the change to make — so an agent can act
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 621437c26..87718918c 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -173,6 +173,22 @@ RUNNING_THE_CREW: dict[RunReason, str] = {
),
"run_again": "That run was not traced: running the crew again to evaluate it.",
}
+# Said before the consent screen, at a terminal: that screen asks about a trace
+# before there is a run, so the person reads first why `crewai eval` needs one.
+BEFORE_ASKING: dict[RunReason, str] = {
+ "no_traced_run": (
+ "No traced run of this crew yet. To evaluate it, the crew runs now and its "
+ "trace goes to CrewAI AMP, which grades it."
+ ),
+ "project_changed": (
+ "The project changed since its last traced run. To evaluate what it does "
+ "now, the crew runs again and its trace goes to CrewAI AMP, which grades it."
+ ),
+ "run_again": (
+ "That run was not traced. To evaluate it, the crew runs again and its trace "
+ "goes to CrewAI AMP, which grades it."
+ ),
+}
TRACING_NOW_ON = (
"Tracing is now on for this project: its runs' traces are sent to CrewAI AMP, "
"which grades them."
@@ -886,12 +902,13 @@ def _at_a_terminal() -> bool:
return not is_dmn_mode_enabled() and sys.stdin.isatty()
-def _ask_before_the_first_enable() -> None:
- """Nobody turned tracing on yet, and a person is here: ask on crewAI's
- trace consent screen before turning it on. A no, or no answer before it
- times out, runs nothing."""
+def _ask_before_the_first_enable(why: RunReason) -> None:
+ """Nobody turned tracing on yet, and a person is here: say why the crew
+ runs, then ask on crewAI's trace consent screen before turning tracing on.
+ A no, or no answer before it times out, runs nothing."""
from crewai.events.listeners.tracing.utils import prompt_user_for_trace_viewing
+ console.print(Text(BEFORE_ASKING[why]), style="bold")
if not prompt_user_for_trace_viewing(sharing=True):
record_stop("declined")
console.print(NOT_SHARED, style="yellow")
@@ -958,20 +975,22 @@ def _run_the_crew_to_evaluate(
# script) the yes is inferred from this command and tracing is turned on
# for the project — said once, plainly, here and in the run app. From
# then on the project traces, and nothing more is asked or said.
- turned_on_now = False
+ turned_on_now = asked = False
if not tracing_already_on():
if _at_a_terminal():
- _ask_before_the_first_enable()
+ _ask_before_the_first_enable(why)
+ asked = True
else:
turn_tracing_on()
turned_on_now = True
_count_usage(f"cli_usage:eval_ran_crew:{why}")
- said = (
- f"{RUNNING_THE_CREW[why]} {TRACING_NOW_ON}"
- if turned_on_now
- else RUNNING_THE_CREW[why]
- )
- console.print(Text(said), style="bold")
+ if not asked: # the line before the consent screen already said why
+ said = (
+ f"{RUNNING_THE_CREW[why]} {TRACING_NOW_ON}"
+ if turned_on_now
+ else RUNNING_THE_CREW[why]
+ )
+ console.print(Text(said), style="bold")
from crewai_cli.crew_run_tui import evaluating_after_run
from crewai_cli.run_crew import run_crew
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index 971966d54..4d90d7d6b 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -46,6 +46,7 @@ def consent_screen(monkeypatch):
def prompt(timeout_seconds=20, **kwargs):
asked.append(kwargs)
+ print("[the consent screen]") # where it showed, in the captured output
return answer["yes"]
monkeypatch.setattr(
@@ -3127,6 +3128,14 @@ def test_at_a_terminal_tracing_is_turned_on_only_after_the_consent_screen(
assert consent_screen.asked == ([] if answer == "already on" else [{"sharing": True}])
assert "Tracing is now on" not in out # nothing inferred: they were asked, or had said
+ why = "No traced run of this crew yet. To evaluate it, the crew runs now and its trace goes to CrewAI AMP, which grades it."
+ if answer == "already on":
+ assert why not in out
+ assert "No traced run of this crew yet: running it now to evaluate it." in out
+ else:
+ # Why the crew runs is said before the screen asks, and not again after.
+ assert out.index(why) < out.index("[the consent screen]")
+ assert "running it now to evaluate it" not in out
if answer == "no":
assert ran == [] and amp.calls == []
assert not (directory / ".env").exists()