Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(cli): crewai eval prints a markdown brief when run by an agent (#7907)

João Moura00e2ff98e9e1
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.py
index 7323e8207..c5c3d8bda 100644
--- a/lib/cli/src/crewai_cli/cli.py
+++ b/lib/cli/src/crewai_cli/cli.py
@@ -707,7 +707,10 @@ def run(
help=(
"Compare models on this project's deployment instead: ONE comma-separated "
'list of provider/model, e.g. "openai/gpt-4o-mini,anthropic/claude-haiku-4-5". '
- "The deployment runs once as deployed and once per model; needs `crewai login`."
+ "The deployment runs once as deployed and once per model; needs `crewai login`. "
+ "Run by a script or coding agent (no terminal), it prints a markdown brief "
+ "after the comparison when AMP provides one — the model for each part and "
+ "the changes to make — else its link, else nothing more."
),
)
@click.option(
@@ -727,6 +730,12 @@ def eval_command(
"""Evaluate the last traced run through CrewAI AMP, or compare models on the
project's deployment (--models).
+ Run by a script or coding agent (no terminal), it prints a markdown brief
+ after the verdict — what failed and the change to make — so an agent can act
+ on it directly. That is when AMP provides the brief: otherwise its link, when
+ AMP sends only that, else nothing more. It grades a run that already
+ happened: after a change, `crewai run` again, then `crewai eval`.
+
A run's evaluation exits 0 only when the goal gate PASSED, and 1 otherwise — a
failed gate, no verdict, or an evaluation that could not run — so a CI job can
gate on it. A comparison exits 0 when it finished, 1 when it failed or could
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 4eb80b851..e7c41209d 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -181,6 +181,7 @@ def eval_crew(run_id: str | None = None) -> None:
)
raise SystemExit(130) from None
_print_verdict(finished, url)
+ _print_brief(finished)
_say_where_the_criteria_live(write_eval_config(finished))
# The exit code is what a CI job reads, so it is the gate's: 0 only for a
# run that PASSED. A failed gate, one without a verdict, and an evaluation
@@ -1051,6 +1052,7 @@ def eval_models(models_text: str, deployment_id: str | None = None) -> None:
)
raise SystemExit(130) from None
_print_comparison(finished, url)
+ _print_brief(finished)
# The criteria the comparison was graded on, for the project to edit — only
# when it has none, exactly as after a Mode 1 evaluation.
_say_where_the_criteria_live(write_eval_config(finished))
@@ -1243,6 +1245,47 @@ def _print_verdict(finished: dict[str, Any], url: str | None) -> None:
console.print(Text(f"Full report: {url}"))
+def nobody_watching() -> bool:
+ """Is the output going somewhere no person reads it as it prints?
+
+ A coding agent or a script runs `crewai eval` with its stdout captured, so
+ stdout — not stdin — is the question: the brief is for whoever READS the
+ output. The enterprise non-interactive mode counts as nobody, as it does
+ for every prompt this command would otherwise put.
+ """
+ if is_dmn_mode_enabled():
+ return True
+ try:
+ return not sys.stdout.isatty()
+ except Exception: # a replaced stream without isatty is not a terminal
+ return True
+
+
+def _print_brief(finished: dict[str, Any]) -> None:
+ """The markdown brief AMP wrote for a coding agent, when nobody is watching.
+
+ It is what the report page's "Copy for your coding agent" button copies:
+ what failed and the change to make (for `--models`, the model for each part
+ and how sure each difference is). Written to stdout as-is, never through
+ the Rich console, which would read `[Errno 13]` or `[link]` in it as markup.
+ In a terminal nothing changes: the person has the report link. An older AMP
+ sends neither field, and then there is nothing to print.
+ """
+ if not nobody_watching():
+ return
+ brief = finished.get("brief_markdown")
+ if isinstance(brief, str) and brief.strip():
+ text = brief
+ else:
+ link = _report_url(finished.get("brief_url"))
+ if link is None:
+ return
+ text = f"Brief (markdown): {link}"
+ console.file.flush()
+ sys.stdout.write(text if text.endswith("\n") else f"{text}\n")
+ sys.stdout.flush()
+
+
def _open(url: str) -> None:
if is_dmn_mode_enabled():
return
diff --git a/lib/cli/src/crewai_cli/templates/AGENTS.md b/lib/cli/src/crewai_cli/templates/AGENTS.md
index 313a1befb..1c2a47b85 100644
--- a/lib/cli/src/crewai_cli/templates/AGENTS.md
+++ b/lib/cli/src/crewai_cli/templates/AGENTS.md
@@ -101,6 +101,16 @@ Two rules:
Full details: [Observability & Traces](#observability--traces) and [Deployment](#deployment).
+<!-- crewai-eval:start -->
+## Evaluating this crew/flow
+
+Run `crewai eval` after a change: it grades the project's last traced run — a goal gate, a 1–5 grade per area, and a link to the full report. It grades a run that already happened, so after changing code run the project again first (`crewai run`, with `CREWAI_TRACING_ENABLED=true` in `.env`; a run with no terminal is only traced when logged in with `crewai login`), then `crewai eval`. In a terminal with no traced run recorded, it offers to run the project once with tracing on.
+
+- Run by a coding agent or script (no terminal), it prints a **markdown brief** after the verdict: what failed and the change to make. Act on its "what failed" changes, then `crewai run` and `crewai eval` again. (An older AMP may print only a link to the brief, or nothing.)
+- `crewai eval --models "provider/model,…"` compares models on the deployed project (needs `crewai login` and a deployment); the brief then names the model for each part and how sure each difference is.
+- Exit codes: `crewai eval` exits 0 only when the goal gate passed; `--models` exits 0 when the comparison finished, whatever each model's grades. Criteria live in `eval.jsonc` at the project root (written after the first evaluation); edit it to say what good means.
+
+<!-- crewai-eval:end -->
## Quick Reference
```bash
@@ -135,6 +145,12 @@ crewai template list
crewai run # Run crew or flow (auto-detects from pyproject.toml)
crewai flow kickoff # Deprecated compatibility alias for crewai run
+<!-- crewai-eval:start -->
+# Evaluation
+crewai eval # Grade the last traced run; prints a markdown brief when run by an agent
+crewai eval --models "openai/gpt-4o-mini,anthropic/claude-haiku-4-5" # Compare models on the deployment
+
+<!-- crewai-eval:end -->
# Testing & training
crewai test # Test crew (default: 2 iterations, gpt-4o-mini)
crewai test -n 5 -m gpt-4o # Custom iterations and model
diff --git a/lib/cli/src/crewai_cli/templates/declarative_flow/AGENTS.md b/lib/cli/src/crewai_cli/templates/declarative_flow/AGENTS.md
index 384a2c54c..bbb9cad88 100644
--- a/lib/cli/src/crewai_cli/templates/declarative_flow/AGENTS.md
+++ b/lib/cli/src/crewai_cli/templates/declarative_flow/AGENTS.md
@@ -114,6 +114,14 @@ Dynamic value rules:
- Do not ask agents to infer missing facts when accuracy matters. Tell them to mark missing dates, amounts, offers, logs, or constraints as unknown.
- Do not set `config.stream: true` unless the caller is expected to consume a streaming result. For normal generated flows and CLI smoke tests, omit it.
+## Evaluating this flow
+
+Run `crewai eval` after a change: it grades the project's last traced run — a goal gate, a 1–5 grade per area, and a link to the full report. It grades a run that already happened, so after changing code run the flow again first (`crewai run`, with `CREWAI_TRACING_ENABLED=true` in `.env`; a run with no terminal is only traced when logged in with `crewai login`), then `crewai eval`. In a terminal with no traced run recorded, it offers to run the flow once with tracing on.
+
+- Run by a coding agent or script (no terminal), it prints a **markdown brief** after the verdict: what failed and the change to make. Act on its "what failed" changes, then `crewai run` and `crewai eval` again. (An older AMP may print only a link to the brief, or nothing.)
+- `crewai eval --models "provider/model,…"` compares models on the deployed project (needs `crewai login` and a deployment); the brief then names the model for each part and how sure each difference is.
+- Exit codes: `crewai eval` exits 0 only when the goal gate passed; `--models` exits 0 when the comparison finished, whatever each model's grades. Criteria live in `eval.jsonc` at the project root (written after the first evaluation); edit it to say what good means.
+
## Examples
### Crew review with routed follow-up
diff --git a/lib/cli/src/crewai_cli/tools/main.py b/lib/cli/src/crewai_cli/tools/main.py
index e45178533..fcf693eb3 100644
--- a/lib/cli/src/crewai_cli/tools/main.py
+++ b/lib/cli/src/crewai_cli/tools/main.py
@@ -87,7 +87,7 @@ class ToolCommand(BaseCommand, PlusAPIMixin):
project_root, "{{crewai_tools_dependency}}", get_crewai_tools_dependency()
)
- copy_assistant_instructions(project_root)
+ copy_assistant_instructions(project_root, evaluation=False)
old_directory = os.getcwd()
os.chdir(project_root)
diff --git a/lib/cli/src/crewai_cli/utils.py b/lib/cli/src/crewai_cli/utils.py
index 517293fab..91c16c14f 100644
--- a/lib/cli/src/crewai_cli/utils.py
+++ b/lib/cli/src/crewai_cli/utils.py
@@ -106,9 +106,37 @@ def copy_assistant_imports(destination: Path) -> None:
shutil.copy2(_TEMPLATES_DIR / name, destination / name)
-def copy_assistant_instructions(destination: Path) -> None:
- """Copy ``AGENTS.md`` and the files that import it into a project."""
- shutil.copy2(_TEMPLATES_DIR / "AGENTS.md", destination / "AGENTS.md")
+# Lines that open and close guidance only a crew or flow project can follow —
+# `crewai eval` grades a traced `crewai run`, which a tool project has none of.
+EVALUATION_START = "<!-- crewai-eval:start -->"
+EVALUATION_END = "<!-- crewai-eval:end -->"
+
+
+def _assistant_instructions(text: str, *, evaluation: bool) -> str:
+ """AGENTS.md with its evaluation blocks kept (markers dropped) or removed."""
+ lines: list[str] = []
+ inside = False
+ for line in text.splitlines(keepends=True):
+ marker = line.strip()
+ if marker == EVALUATION_START:
+ inside = True
+ elif marker == EVALUATION_END:
+ inside = False
+ elif evaluation or not inside:
+ lines.append(line)
+ return "".join(lines)
+
+
+def copy_assistant_instructions(destination: Path, *, evaluation: bool = True) -> None:
+ """Copy ``AGENTS.md`` and the files that import it into a project.
+
+ EVALUATION: keep the `crewai eval` guidance — True for crew and flow
+ projects, False for a tool project, which has no run to evaluate.
+ """
+ text = (_TEMPLATES_DIR / "AGENTS.md").read_text(encoding="utf-8")
+ (destination / "AGENTS.md").write_text(
+ _assistant_instructions(text, evaluation=evaluation), encoding="utf-8"
+ )
copy_assistant_imports(destination)
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index 8bcc5d594..fbdf50e4b 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -1600,3 +1600,112 @@ def test_without_crewais_catalog_every_model_is_other(monkeypatch):
monkeypatch.setattr(eval_module, "_known_models_and_providers", lambda: (frozenset(), frozenset()))
assert eval_module.telemetry_model_name("openai/gpt-4o-mini") == "other/other"
+
+
+# ── the brief for a coding agent ─────────────────────────────────────────────
+
+BRIEF = (
+ "# What failed\n\n"
+ "1. task write — [Errno 13] the poem ignores [red]the topic[/red]\n"
+ " change: set `expected_output` to \"Four lines about {topic}.\"\n"
+)
+BRIEF_URL = "https://evolve.crewai.test/e/ev-1.md"
+
+
+def _done_with(**fields):
+ return httpx.Response(200, json={**done(gate="failed").json(), **fields})
+
+
+def test_with_nobody_watching_the_brief_follows_the_verdict_and_the_link_literally(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
+ install(monkeypatch, FakeAMP(statuses=[_done_with(brief_markdown=BRIEF, brief_url=BRIEF_URL)]))
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ out = capsys.readouterr().out
+ assert exit_.value.code == 1 # the gate's, unchanged
+ assert BRIEF in out # brackets and all: never read as markup
+ assert out.index("Goal gate: FAILED") < out.index(f"Full report: {URL}") < out.index(BRIEF)
+ assert "Brief (markdown)" not in out
+
+
+def test_with_only_the_briefs_url_the_link_is_printed(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
+ install(monkeypatch, FakeAMP(statuses=[_done_with(brief_url=BRIEF_URL)]))
+
+ with pytest.raises(SystemExit):
+ eval_module.eval_crew()
+
+ assert f"Brief (markdown): {BRIEF_URL}\n" in capsys.readouterr().out
+
+
[email protected]("brief_url", [None, "javascript:alert(1)"])
+def test_an_older_server_adds_nothing_after_the_link(project, monkeypatch, capsys, brief_url):
+ directory, _ = project
+ record_last_run(directory)
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
+ fields = {"brief_url": brief_url} if brief_url else {}
+ install(monkeypatch, FakeAMP(statuses=[_done_with(**fields)]))
+
+ with pytest.raises(SystemExit):
+ eval_module.eval_crew()
+
+ out = capsys.readouterr().out
+ assert out.endswith(f"Full report: {URL}\n") and "Brief" not in out
+
+
+def test_in_a_terminal_the_brief_is_not_printed(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: False)
+ install(monkeypatch, FakeAMP(statuses=[_done_with(brief_markdown=BRIEF, brief_url=BRIEF_URL)]))
+
+ with pytest.raises(SystemExit):
+ eval_module.eval_crew()
+
+ out = capsys.readouterr().out
+ assert "What failed" not in out and "Brief" not in out
+ assert out.endswith(f"Full report: {URL}\n")
+
+
+def test_nobody_is_watching_when_stdout_is_not_a_terminal(monkeypatch):
+ monkeypatch.setattr(eval_module, "is_dmn_mode_enabled", lambda: False)
+ monkeypatch.setattr(eval_module.sys, "stdout", SimpleNamespace(isatty=lambda: True))
+ assert eval_module.nobody_watching() is False
+ monkeypatch.setattr(eval_module.sys, "stdout", SimpleNamespace(isatty=lambda: False))
+ assert eval_module.nobody_watching() is True
+ monkeypatch.setattr(eval_module.sys, "stdout", SimpleNamespace()) # no isatty at all
+ assert eval_module.nobody_watching() is True
+ monkeypatch.setattr(eval_module.sys, "stdout", SimpleNamespace(isatty=lambda: True))
+ monkeypatch.setattr(eval_module, "is_dmn_mode_enabled", lambda: True)
+ assert eval_module.nobody_watching() is True
+
+
[email protected]("watching", [False, True])
+def test_a_comparison_prints_its_brief_only_when_nobody_is_watching(deployed, monkeypatch, capsys, watching):
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: not watching)
+ body = {**compared().json(), "brief_markdown": BRIEF, "brief_url": BRIEF_URL}
+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json=body)]))
+
+ eval_module.eval_models(MODELS) # a finished comparison exits 0, as before
+
+ out = capsys.readouterr().out
+ if watching:
+ assert "What failed" not in out and out.endswith(f"Full report: {URL}\n")
+ else:
+ assert out.index(f"Full report: {URL}") < out.index(BRIEF)
+
+
+def test_the_help_tells_an_agent_it_gets_a_brief():
+ help_text = " ".join(CliRunner().invoke(eval_command, ["--help"]).output.split())
+ assert (
+ "Run by a script or coding agent (no terminal), it prints a markdown brief "
+ "after the verdict — what failed and the change to make — so an agent can act on it directly"
+ ) in help_text
+ assert "That is when AMP provides the brief: otherwise its link, when AMP sends only that, else nothing more" in help_text
+ assert "it prints a markdown brief after the comparison when AMP provides one" in help_text
diff --git a/lib/cli/tests/test_create_crew.py b/lib/cli/tests/test_create_crew.py
index abde05ca4..8feb4a91d 100644
--- a/lib/cli/tests/test_create_crew.py
+++ b/lib/cli/tests/test_create_crew.py
@@ -1319,6 +1319,8 @@ def test_json_create_scaffolds_assistant_instructions(tmp_path, monkeypatch):
project_root = tmp_path / "json_crew"
agents_md = (project_root / "AGENTS.md").read_text(encoding="utf-8")
assert "CrewAI Reference for AI Coding Assistants" in agents_md
+ assert "## Evaluating this crew/flow" in agents_md
+ assert "crewai-eval:" not in agents_md
assert "crew.jsonc" in agents_md
claude_md = (project_root / "CLAUDE.md").read_text(encoding="utf-8")
assert "@AGENTS.md" in claude_md.splitlines()
@@ -1326,3 +1328,17 @@ def test_json_create_scaffolds_assistant_instructions(tmp_path, monkeypatch):
assert "@AGENTS.md" in cursor_md.splitlines()
gemini_md = (project_root / "GEMINI.md").read_text(encoding="utf-8")
assert "@./AGENTS.md" in gemini_md.splitlines()
+
+
+def test_scaffolded_agents_md_tells_assistants_to_evaluate_with_crewai_eval(
+ tmp_path, monkeypatch
+):
+ monkeypatch.chdir(tmp_path)
+ create_crew("my-crew", skip_provider=True)
+
+ agents_md = (tmp_path / "my_crew" / "AGENTS.md").read_text(encoding="utf-8")
+ assert "## Evaluating this crew/flow" in agents_md
+ assert "Run `crewai eval` after a change" in agents_md
+ assert "it prints a **markdown brief** after the verdict" in agents_md
+ assert 'crewai eval --models "provider/model,…"' in agents_md
+ assert "crewai-eval:" not in agents_md # the markers never reach a project
diff --git a/lib/cli/tests/test_create_flow.py b/lib/cli/tests/test_create_flow.py
index 2ec1699ff..a9fb11ab8 100644
--- a/lib/cli/tests/test_create_flow.py
+++ b/lib/cli/tests/test_create_flow.py
@@ -66,3 +66,20 @@ def test_create_flow_scaffolds_assistant_instructions(
assert "@AGENTS.md" in claude_md.splitlines()
gemini_md = (project_root / "GEMINI.md").read_text(encoding="utf-8")
assert "@./AGENTS.md" in gemini_md.splitlines()
+
+
+def test_scaffolded_flow_agents_md_tells_assistants_to_evaluate_with_crewai_eval(
+ tmp_path: Path, monkeypatch: MonkeyPatch
+):
+ monkeypatch.chdir(tmp_path)
+ create_flow("Research Flow")
+ create_flow("Declared Flow", declarative=True)
+
+ for project in ("research_flow", "declared_flow"):
+ agents_md = (tmp_path / project / "AGENTS.md").read_text(encoding="utf-8")
+ assert "## Evaluating this" in agents_md
+ assert "Run `crewai eval` after a change" in agents_md
+ assert "it prints a **markdown brief** after the verdict" in agents_md
+ assert 'crewai eval --models "provider/model,…"' in agents_md
+ assert "{{name}}" not in agents_md
+ assert "crewai-eval:" not in agents_md
diff --git a/lib/cli/tests/tools/test_main.py b/lib/cli/tests/tools/test_main.py
index ee8591f9d..626187ee6 100644
--- a/lib/cli/tests/tools/test_main.py
+++ b/lib/cli/tests/tools/test_main.py
@@ -81,6 +81,19 @@ def test_create_scaffolds_assistant_instructions(mock_subprocess, tool_command):
assert "@./AGENTS.md" in gemini_md.splitlines()
+@patch("crewai_cli.tools.main.subprocess.run")
+def test_a_tool_project_gets_no_evaluation_guidance(mock_subprocess, tool_command):
+ # A tool project has no `crewai run` and no traced run for `crewai eval`.
+ with in_temp_dir():
+ tool_command.create("test-tool")
+
+ agents_md = Path("test_tool", "AGENTS.md").read_text(encoding="utf-8")
+ assert "Evaluating this" not in agents_md
+ assert "crewai eval" not in agents_md
+ assert "crewai-eval:" not in agents_md
+ assert "## Quick Reference" in agents_md
+
+
@patch("crewai_cli.tools.main.subprocess.run")
@patch("crewai_cli.plus_api.PlusAPI.get_tool")
@patch("crewai_cli.tools.main.ToolCommand._print_current_organization")