Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(cli): crewai eval prints a markdown brief when run by an agent (#7907)
João Moura00e2ff98e9e1
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.pyindex 7323e8207..c5c3d8bda 100644--- a/lib/cli/src/crewai_cli/cli.py+++ b/lib/cli/src/crewai_cli/cli.py@@ -707,7 +707,10 @@ def run(help=("Compare models on this project's deployment instead: ONE comma-separated "'list of provider/model, e.g. "openai/gpt-4o-mini,anthropic/claude-haiku-4-5". '- "The deployment runs once as deployed and once per model; needs `crewai login`."+ "The deployment runs once as deployed and once per model; needs `crewai login`. "+ "Run by a script or coding agent (no terminal), it prints a markdown brief "+ "after the comparison when AMP provides one — the model for each part and "+ "the changes to make — else its link, else nothing more."),)@click.option(@@ -727,6 +730,12 @@ def eval_command("""Evaluate the last traced run through CrewAI AMP, or compare models on theproject's deployment (--models).+ Run by a script or coding agent (no terminal), it prints a markdown brief+ after the verdict — what failed and the change to make — so an agent can act+ on it directly. That is when AMP provides the brief: otherwise its link, when+ AMP sends only that, else nothing more. It grades a run that already+ happened: after a change, `crewai run` again, then `crewai eval`.+A run's evaluation exits 0 only when the goal gate PASSED, and 1 otherwise — afailed gate, no verdict, or an evaluation that could not run — so a CI job cangate on it. A comparison exits 0 when it finished, 1 when it failed or coulddiff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex 4eb80b851..e7c41209d 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -181,6 +181,7 @@ def eval_crew(run_id: str | None = None) -> None:)raise SystemExit(130) from None_print_verdict(finished, url)+ _print_brief(finished)_say_where_the_criteria_live(write_eval_config(finished))# The exit code is what a CI job reads, so it is the gate's: 0 only for a# run that PASSED. A failed gate, one without a verdict, and an evaluation@@ -1051,6 +1052,7 @@ def eval_models(models_text: str, deployment_id: str | None = None) -> None:)raise SystemExit(130) from None_print_comparison(finished, url)+ _print_brief(finished)# The criteria the comparison was graded on, for the project to edit — only# when it has none, exactly as after a Mode 1 evaluation._say_where_the_criteria_live(write_eval_config(finished))@@ -1243,6 +1245,47 @@ def _print_verdict(finished: dict[str, Any], url: str | None) -> None:console.print(Text(f"Full report: {url}"))+def nobody_watching() -> bool:+ """Is the output going somewhere no person reads it as it prints?++ A coding agent or a script runs `crewai eval` with its stdout captured, so+ stdout — not stdin — is the question: the brief is for whoever READS the+ output. The enterprise non-interactive mode counts as nobody, as it does+ for every prompt this command would otherwise put.+ """+ if is_dmn_mode_enabled():+ return True+ try:+ return not sys.stdout.isatty()+ except Exception: # a replaced stream without isatty is not a terminal+ return True+++def _print_brief(finished: dict[str, Any]) -> None:+ """The markdown brief AMP wrote for a coding agent, when nobody is watching.++ It is what the report page's "Copy for your coding agent" button copies:+ what failed and the change to make (for `--models`, the model for each part+ and how sure each difference is). Written to stdout as-is, never through+ the Rich console, which would read `[Errno 13]` or `[link]` in it as markup.+ In a terminal nothing changes: the person has the report link. An older AMP+ sends neither field, and then there is nothing to print.+ """+ if not nobody_watching():+ return+ brief = finished.get("brief_markdown")+ if isinstance(brief, str) and brief.strip():+ text = brief+ else:+ link = _report_url(finished.get("brief_url"))+ if link is None:+ return+ text = f"Brief (markdown): {link}"+ console.file.flush()+ sys.stdout.write(text if text.endswith("\n") else f"{text}\n")+ sys.stdout.flush()++def _open(url: str) -> None:if is_dmn_mode_enabled():returndiff --git a/lib/cli/src/crewai_cli/templates/AGENTS.md b/lib/cli/src/crewai_cli/templates/AGENTS.mdindex 313a1befb..1c2a47b85 100644--- a/lib/cli/src/crewai_cli/templates/AGENTS.md+++ b/lib/cli/src/crewai_cli/templates/AGENTS.md@@ -101,6 +101,16 @@ Two rules:Full details: [Observability & Traces](#observability--traces) and [Deployment](#deployment).+<!-- crewai-eval:start -->+## Evaluating this crew/flow++Run `crewai eval` after a change: it grades the project's last traced run — a goal gate, a 1–5 grade per area, and a link to the full report. It grades a run that already happened, so after changing code run the project again first (`crewai run`, with `CREWAI_TRACING_ENABLED=true` in `.env`; a run with no terminal is only traced when logged in with `crewai login`), then `crewai eval`. In a terminal with no traced run recorded, it offers to run the project once with tracing on.++- Run by a coding agent or script (no terminal), it prints a **markdown brief** after the verdict: what failed and the change to make. Act on its "what failed" changes, then `crewai run` and `crewai eval` again. (An older AMP may print only a link to the brief, or nothing.)+- `crewai eval --models "provider/model,…"` compares models on the deployed project (needs `crewai login` and a deployment); the brief then names the model for each part and how sure each difference is.+- Exit codes: `crewai eval` exits 0 only when the goal gate passed; `--models` exits 0 when the comparison finished, whatever each model's grades. Criteria live in `eval.jsonc` at the project root (written after the first evaluation); edit it to say what good means.++<!-- crewai-eval:end -->## Quick Reference```bash@@ -135,6 +145,12 @@ crewai template listcrewai run # Run crew or flow (auto-detects from pyproject.toml)crewai flow kickoff # Deprecated compatibility alias for crewai run+<!-- crewai-eval:start -->+# Evaluation+crewai eval # Grade the last traced run; prints a markdown brief when run by an agent+crewai eval --models "openai/gpt-4o-mini,anthropic/claude-haiku-4-5" # Compare models on the deployment++<!-- crewai-eval:end --># Testing & trainingcrewai test # Test crew (default: 2 iterations, gpt-4o-mini)crewai test -n 5 -m gpt-4o # Custom iterations and modeldiff --git a/lib/cli/src/crewai_cli/templates/declarative_flow/AGENTS.md b/lib/cli/src/crewai_cli/templates/declarative_flow/AGENTS.mdindex 384a2c54c..bbb9cad88 100644--- a/lib/cli/src/crewai_cli/templates/declarative_flow/AGENTS.md+++ b/lib/cli/src/crewai_cli/templates/declarative_flow/AGENTS.md@@ -114,6 +114,14 @@ Dynamic value rules:- Do not ask agents to infer missing facts when accuracy matters. Tell them to mark missing dates, amounts, offers, logs, or constraints as unknown.- Do not set `config.stream: true` unless the caller is expected to consume a streaming result. For normal generated flows and CLI smoke tests, omit it.+## Evaluating this flow++Run `crewai eval` after a change: it grades the project's last traced run — a goal gate, a 1–5 grade per area, and a link to the full report. It grades a run that already happened, so after changing code run the flow again first (`crewai run`, with `CREWAI_TRACING_ENABLED=true` in `.env`; a run with no terminal is only traced when logged in with `crewai login`), then `crewai eval`. In a terminal with no traced run recorded, it offers to run the flow once with tracing on.++- Run by a coding agent or script (no terminal), it prints a **markdown brief** after the verdict: what failed and the change to make. Act on its "what failed" changes, then `crewai run` and `crewai eval` again. (An older AMP may print only a link to the brief, or nothing.)+- `crewai eval --models "provider/model,…"` compares models on the deployed project (needs `crewai login` and a deployment); the brief then names the model for each part and how sure each difference is.+- Exit codes: `crewai eval` exits 0 only when the goal gate passed; `--models` exits 0 when the comparison finished, whatever each model's grades. Criteria live in `eval.jsonc` at the project root (written after the first evaluation); edit it to say what good means.+## Examples### Crew review with routed follow-updiff --git a/lib/cli/src/crewai_cli/tools/main.py b/lib/cli/src/crewai_cli/tools/main.pyindex e45178533..fcf693eb3 100644--- a/lib/cli/src/crewai_cli/tools/main.py+++ b/lib/cli/src/crewai_cli/tools/main.py@@ -87,7 +87,7 @@ class ToolCommand(BaseCommand, PlusAPIMixin):project_root, "{{crewai_tools_dependency}}", get_crewai_tools_dependency())- copy_assistant_instructions(project_root)+ copy_assistant_instructions(project_root, evaluation=False)old_directory = os.getcwd()os.chdir(project_root)diff --git a/lib/cli/src/crewai_cli/utils.py b/lib/cli/src/crewai_cli/utils.pyindex 517293fab..91c16c14f 100644--- a/lib/cli/src/crewai_cli/utils.py+++ b/lib/cli/src/crewai_cli/utils.py@@ -106,9 +106,37 @@ def copy_assistant_imports(destination: Path) -> None:shutil.copy2(_TEMPLATES_DIR / name, destination / name)-def copy_assistant_instructions(destination: Path) -> None:- """Copy ``AGENTS.md`` and the files that import it into a project."""- shutil.copy2(_TEMPLATES_DIR / "AGENTS.md", destination / "AGENTS.md")+# Lines that open and close guidance only a crew or flow project can follow —+# `crewai eval` grades a traced `crewai run`, which a tool project has none of.+EVALUATION_START = "<!-- crewai-eval:start -->"+EVALUATION_END = "<!-- crewai-eval:end -->"+++def _assistant_instructions(text: str, *, evaluation: bool) -> str:+ """AGENTS.md with its evaluation blocks kept (markers dropped) or removed."""+ lines: list[str] = []+ inside = False+ for line in text.splitlines(keepends=True):+ marker = line.strip()+ if marker == EVALUATION_START:+ inside = True+ elif marker == EVALUATION_END:+ inside = False+ elif evaluation or not inside:+ lines.append(line)+ return "".join(lines)+++def copy_assistant_instructions(destination: Path, *, evaluation: bool = True) -> None:+ """Copy ``AGENTS.md`` and the files that import it into a project.++ EVALUATION: keep the `crewai eval` guidance — True for crew and flow+ projects, False for a tool project, which has no run to evaluate.+ """+ text = (_TEMPLATES_DIR / "AGENTS.md").read_text(encoding="utf-8")+ (destination / "AGENTS.md").write_text(+ _assistant_instructions(text, evaluation=evaluation), encoding="utf-8"+ )copy_assistant_imports(destination)diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex 8bcc5d594..fbdf50e4b 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -1600,3 +1600,112 @@ def test_without_crewais_catalog_every_model_is_other(monkeypatch):monkeypatch.setattr(eval_module, "_known_models_and_providers", lambda: (frozenset(), frozenset()))assert eval_module.telemetry_model_name("openai/gpt-4o-mini") == "other/other"+++# ── the brief for a coding agent ─────────────────────────────────────────────++BRIEF = (+ "# What failed\n\n"+ "1. task write — [Errno 13] the poem ignores [red]the topic[/red]\n"+ " change: set `expected_output` to \"Four lines about {topic}.\"\n"+)+BRIEF_URL = "https://evolve.crewai.test/e/ev-1.md"+++def _done_with(**fields):+ return httpx.Response(200, json={**done(gate="failed").json(), **fields})+++def test_with_nobody_watching_the_brief_follows_the_verdict_and_the_link_literally(project, monkeypatch, capsys):+ directory, _ = project+ record_last_run(directory)+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)+ install(monkeypatch, FakeAMP(statuses=[_done_with(brief_markdown=BRIEF, brief_url=BRIEF_URL)]))++ with pytest.raises(SystemExit) as exit_:+ eval_module.eval_crew()++ out = capsys.readouterr().out+ assert exit_.value.code == 1 # the gate's, unchanged+ assert BRIEF in out # brackets and all: never read as markup+ assert out.index("Goal gate: FAILED") < out.index(f"Full report: {URL}") < out.index(BRIEF)+ assert "Brief (markdown)" not in out+++def test_with_only_the_briefs_url_the_link_is_printed(project, monkeypatch, capsys):+ directory, _ = project+ record_last_run(directory)+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)+ install(monkeypatch, FakeAMP(statuses=[_done_with(brief_url=BRIEF_URL)]))++ with pytest.raises(SystemExit):+ eval_module.eval_crew()++ assert f"Brief (markdown): {BRIEF_URL}\n" in capsys.readouterr().out++[email protected]("brief_url", [None, "javascript:alert(1)"])+def test_an_older_server_adds_nothing_after_the_link(project, monkeypatch, capsys, brief_url):+ directory, _ = project+ record_last_run(directory)+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)+ fields = {"brief_url": brief_url} if brief_url else {}+ install(monkeypatch, FakeAMP(statuses=[_done_with(**fields)]))++ with pytest.raises(SystemExit):+ eval_module.eval_crew()++ out = capsys.readouterr().out+ assert out.endswith(f"Full report: {URL}\n") and "Brief" not in out+++def test_in_a_terminal_the_brief_is_not_printed(project, monkeypatch, capsys):+ directory, _ = project+ record_last_run(directory)+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: False)+ install(monkeypatch, FakeAMP(statuses=[_done_with(brief_markdown=BRIEF, brief_url=BRIEF_URL)]))++ with pytest.raises(SystemExit):+ eval_module.eval_crew()++ out = capsys.readouterr().out+ assert "What failed" not in out and "Brief" not in out+ assert out.endswith(f"Full report: {URL}\n")+++def test_nobody_is_watching_when_stdout_is_not_a_terminal(monkeypatch):+ monkeypatch.setattr(eval_module, "is_dmn_mode_enabled", lambda: False)+ monkeypatch.setattr(eval_module.sys, "stdout", SimpleNamespace(isatty=lambda: True))+ assert eval_module.nobody_watching() is False+ monkeypatch.setattr(eval_module.sys, "stdout", SimpleNamespace(isatty=lambda: False))+ assert eval_module.nobody_watching() is True+ monkeypatch.setattr(eval_module.sys, "stdout", SimpleNamespace()) # no isatty at all+ assert eval_module.nobody_watching() is True+ monkeypatch.setattr(eval_module.sys, "stdout", SimpleNamespace(isatty=lambda: True))+ monkeypatch.setattr(eval_module, "is_dmn_mode_enabled", lambda: True)+ assert eval_module.nobody_watching() is True++[email protected]("watching", [False, True])+def test_a_comparison_prints_its_brief_only_when_nobody_is_watching(deployed, monkeypatch, capsys, watching):+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: not watching)+ body = {**compared().json(), "brief_markdown": BRIEF, "brief_url": BRIEF_URL}+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json=body)]))++ eval_module.eval_models(MODELS) # a finished comparison exits 0, as before++ out = capsys.readouterr().out+ if watching:+ assert "What failed" not in out and out.endswith(f"Full report: {URL}\n")+ else:+ assert out.index(f"Full report: {URL}") < out.index(BRIEF)+++def test_the_help_tells_an_agent_it_gets_a_brief():+ help_text = " ".join(CliRunner().invoke(eval_command, ["--help"]).output.split())+ assert (+ "Run by a script or coding agent (no terminal), it prints a markdown brief "+ "after the verdict — what failed and the change to make — so an agent can act on it directly"+ ) in help_text+ assert "That is when AMP provides the brief: otherwise its link, when AMP sends only that, else nothing more" in help_text+ assert "it prints a markdown brief after the comparison when AMP provides one" in help_textdiff --git a/lib/cli/tests/test_create_crew.py b/lib/cli/tests/test_create_crew.pyindex abde05ca4..8feb4a91d 100644--- a/lib/cli/tests/test_create_crew.py+++ b/lib/cli/tests/test_create_crew.py@@ -1319,6 +1319,8 @@ def test_json_create_scaffolds_assistant_instructions(tmp_path, monkeypatch):project_root = tmp_path / "json_crew"agents_md = (project_root / "AGENTS.md").read_text(encoding="utf-8")assert "CrewAI Reference for AI Coding Assistants" in agents_md+ assert "## Evaluating this crew/flow" in agents_md+ assert "crewai-eval:" not in agents_mdassert "crew.jsonc" in agents_mdclaude_md = (project_root / "CLAUDE.md").read_text(encoding="utf-8")assert "@AGENTS.md" in claude_md.splitlines()@@ -1326,3 +1328,17 @@ def test_json_create_scaffolds_assistant_instructions(tmp_path, monkeypatch):assert "@AGENTS.md" in cursor_md.splitlines()gemini_md = (project_root / "GEMINI.md").read_text(encoding="utf-8")assert "@./AGENTS.md" in gemini_md.splitlines()+++def test_scaffolded_agents_md_tells_assistants_to_evaluate_with_crewai_eval(+ tmp_path, monkeypatch+):+ monkeypatch.chdir(tmp_path)+ create_crew("my-crew", skip_provider=True)++ agents_md = (tmp_path / "my_crew" / "AGENTS.md").read_text(encoding="utf-8")+ assert "## Evaluating this crew/flow" in agents_md+ assert "Run `crewai eval` after a change" in agents_md+ assert "it prints a **markdown brief** after the verdict" in agents_md+ assert 'crewai eval --models "provider/model,…"' in agents_md+ assert "crewai-eval:" not in agents_md # the markers never reach a projectdiff --git a/lib/cli/tests/test_create_flow.py b/lib/cli/tests/test_create_flow.pyindex 2ec1699ff..a9fb11ab8 100644--- a/lib/cli/tests/test_create_flow.py+++ b/lib/cli/tests/test_create_flow.py@@ -66,3 +66,20 @@ def test_create_flow_scaffolds_assistant_instructions(assert "@AGENTS.md" in claude_md.splitlines()gemini_md = (project_root / "GEMINI.md").read_text(encoding="utf-8")assert "@./AGENTS.md" in gemini_md.splitlines()+++def test_scaffolded_flow_agents_md_tells_assistants_to_evaluate_with_crewai_eval(+ tmp_path: Path, monkeypatch: MonkeyPatch+):+ monkeypatch.chdir(tmp_path)+ create_flow("Research Flow")+ create_flow("Declared Flow", declarative=True)++ for project in ("research_flow", "declared_flow"):+ agents_md = (tmp_path / project / "AGENTS.md").read_text(encoding="utf-8")+ assert "## Evaluating this" in agents_md+ assert "Run `crewai eval` after a change" in agents_md+ assert "it prints a **markdown brief** after the verdict" in agents_md+ assert 'crewai eval --models "provider/model,…"' in agents_md+ assert "{{name}}" not in agents_md+ assert "crewai-eval:" not in agents_mddiff --git a/lib/cli/tests/tools/test_main.py b/lib/cli/tests/tools/test_main.pyindex ee8591f9d..626187ee6 100644--- a/lib/cli/tests/tools/test_main.py+++ b/lib/cli/tests/tools/test_main.py@@ -81,6 +81,19 @@ def test_create_scaffolds_assistant_instructions(mock_subprocess, tool_command):assert "@./AGENTS.md" in gemini_md.splitlines()+@patch("crewai_cli.tools.main.subprocess.run")+def test_a_tool_project_gets_no_evaluation_guidance(mock_subprocess, tool_command):+ # A tool project has no `crewai run` and no traced run for `crewai eval`.+ with in_temp_dir():+ tool_command.create("test-tool")++ agents_md = Path("test_tool", "AGENTS.md").read_text(encoding="utf-8")+ assert "Evaluating this" not in agents_md+ assert "crewai eval" not in agents_md+ assert "crewai-eval:" not in agents_md+ assert "## Quick Reference" in agents_md++@patch("crewai_cli.tools.main.subprocess.run")@patch("crewai_cli.plus_api.PlusAPI.get_tool")@patch("crewai_cli.tools.main.ToolCommand._print_current_organization")