Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix(cli): crewai eval says why before it asks, and its help says when it asks (#8301)
João Mouraaaa33e03e09e
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.pyindex b68ba8654..05f3faeac 100644--- a/lib/cli/src/crewai_cli/cli.py+++ b/lib/cli/src/crewai_cli/cli.py@@ -750,9 +750,12 @@ def eval_command(It grades the project's last traced run. When there is none, or the projectchanged since it, it runs the crew first and grades that run: the run is- traced and its trace sent to CrewAI AMP, which grades it. Nothing is asked,- with or without a terminal, so `crewai eval` alone takes a coding agent from- a change to its verdict.+ traced and its trace sent to CrewAI AMP, which grades it. The first time+ tracing is turned on for the project, a person at a terminal is asked+ first; with no terminal (a coding agent, a script) nothing is asked and the+ command says it turned tracing on, so `crewai eval` alone takes an agent+ from a change to its verdict. A project that turned tracing off is asked+ at a terminal, and told how to turn it on without one.Run by a script or coding agent (no terminal), it prints a markdown briefafter the verdict — what failed and the change to make — so an agent can actdiff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex 621437c26..87718918c 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -173,6 +173,22 @@ RUNNING_THE_CREW: dict[RunReason, str] = {),"run_again": "That run was not traced: running the crew again to evaluate it.",}+# Said before the consent screen, at a terminal: that screen asks about a trace+# before there is a run, so the person reads first why `crewai eval` needs one.+BEFORE_ASKING: dict[RunReason, str] = {+ "no_traced_run": (+ "No traced run of this crew yet. To evaluate it, the crew runs now and its "+ "trace goes to CrewAI AMP, which grades it."+ ),+ "project_changed": (+ "The project changed since its last traced run. To evaluate what it does "+ "now, the crew runs again and its trace goes to CrewAI AMP, which grades it."+ ),+ "run_again": (+ "That run was not traced. To evaluate it, the crew runs again and its trace "+ "goes to CrewAI AMP, which grades it."+ ),+}TRACING_NOW_ON = ("Tracing is now on for this project: its runs' traces are sent to CrewAI AMP, ""which grades them."@@ -886,12 +902,13 @@ def _at_a_terminal() -> bool:return not is_dmn_mode_enabled() and sys.stdin.isatty()-def _ask_before_the_first_enable() -> None:- """Nobody turned tracing on yet, and a person is here: ask on crewAI's- trace consent screen before turning it on. A no, or no answer before it- times out, runs nothing."""+def _ask_before_the_first_enable(why: RunReason) -> None:+ """Nobody turned tracing on yet, and a person is here: say why the crew+ runs, then ask on crewAI's trace consent screen before turning tracing on.+ A no, or no answer before it times out, runs nothing."""from crewai.events.listeners.tracing.utils import prompt_user_for_trace_viewing+ console.print(Text(BEFORE_ASKING[why]), style="bold")if not prompt_user_for_trace_viewing(sharing=True):record_stop("declined")console.print(NOT_SHARED, style="yellow")@@ -958,20 +975,22 @@ def _run_the_crew_to_evaluate(# script) the yes is inferred from this command and tracing is turned on# for the project — said once, plainly, here and in the run app. From# then on the project traces, and nothing more is asked or said.- turned_on_now = False+ turned_on_now = asked = Falseif not tracing_already_on():if _at_a_terminal():- _ask_before_the_first_enable()+ _ask_before_the_first_enable(why)+ asked = Trueelse:turn_tracing_on()turned_on_now = True_count_usage(f"cli_usage:eval_ran_crew:{why}")- said = (- f"{RUNNING_THE_CREW[why]} {TRACING_NOW_ON}"- if turned_on_now- else RUNNING_THE_CREW[why]- )- console.print(Text(said), style="bold")+ if not asked: # the line before the consent screen already said why+ said = (+ f"{RUNNING_THE_CREW[why]} {TRACING_NOW_ON}"+ if turned_on_now+ else RUNNING_THE_CREW[why]+ )+ console.print(Text(said), style="bold")from crewai_cli.crew_run_tui import evaluating_after_runfrom crewai_cli.run_crew import run_crewdiff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex 971966d54..4d90d7d6b 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -46,6 +46,7 @@ def consent_screen(monkeypatch):def prompt(timeout_seconds=20, **kwargs):asked.append(kwargs)+ print("[the consent screen]") # where it showed, in the captured outputreturn answer["yes"]monkeypatch.setattr(@@ -3127,6 +3128,14 @@ def test_at_a_terminal_tracing_is_turned_on_only_after_the_consent_screen(assert consent_screen.asked == ([] if answer == "already on" else [{"sharing": True}])assert "Tracing is now on" not in out # nothing inferred: they were asked, or had said+ why = "No traced run of this crew yet. To evaluate it, the crew runs now and its trace goes to CrewAI AMP, which grades it."+ if answer == "already on":+ assert why not in out+ assert "No traced run of this crew yet: running it now to evaluate it." in out+ else:+ # Why the crew runs is said before the screen asks, and not again after.+ assert out.index(why) < out.index("[the consent screen]")+ assert "running it now to evaluate it" not in outif answer == "no":assert ran == [] and amp.calls == []assert not (directory / ".env").exists()