Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(cli): crewai eval evaluates the last traced run through AMP (#7649)

João Moura0a3b8912b39f
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.py
index 4c922f84c..0250289d3 100644
--- a/lib/cli/src/crewai_cli/cli.py
+++ b/lib/cli/src/crewai_cli/cli.py
@@ -48,6 +48,12 @@ def run_crew(*args: Any, **kwargs: Any) -> Any:
return _run_crew(*args, **kwargs)
+def eval_crew(*args: Any, **kwargs: Any) -> Any:
+ from crewai_cli.eval_crew import eval_crew as _eval_crew
+
+ return _eval_crew(*args, **kwargs)
+
+
if TYPE_CHECKING:
# mypy sees the real classes; at runtime the shims below defer the
# heavy imports until a command actually instantiates them.
@@ -674,6 +680,23 @@ def run(
)
[email protected](name="eval")
+ "--run",
+ "run_id",
+ type=str,
+ default=None,
+ metavar="EXECUTION_ID",
+ help=(
+ "Evaluate this traced run instead of the last one. The execution id "
+ "crewAI recorded for the run."
+ ),
+)
+def eval_command(run_id: str | None) -> None:
+ """Evaluate the last traced run through CrewAI AMP."""
+ eval_crew(run_id=run_id)
+
+
@crewai.command()
def update() -> None:
"""Update the pyproject.toml of the Crew project to use uv."""
diff --git a/lib/cli/src/crewai_cli/eval_crew.py b/lib/cli/src/crewai_cli/eval_crew.py
new file mode 100644
index 000000000..308def5d6
--- /dev/null
+++ b/lib/cli/src/crewai_cli/eval_crew.py
@@ -0,0 +1,377 @@
+"""`crewai eval`: evaluate the last traced run through CrewAI AMP.
+
+crewAI records a traced run in `.crewai/last_run.json` when the run's spans
+reach Wharf. This command reads that record (or takes `--run EXECUTION_ID`),
+asks AMP to evaluate the run, prints and opens the URL AMP answers with,
+waits for the verdict and prints it. With no traced run recorded it offers
+to turn tracing on for the project and run the crew now.
+
+Who may evaluate what is AMP's decision: an anonymous run once without an
+account, then it needs one; a run traced while logged in for that
+organization's members; a deployment execution for members who may see its
+traces. The command sends the saved `crewai login` when there is one.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from ipaddress import ip_address
+import json
+import os
+from pathlib import Path
+import sys
+import time
+from typing import Any
+from urllib.parse import urlparse
+import webbrowser
+
+import click
+from crewai_core.constants import DEFAULT_CREWAI_ENTERPRISE_URL
+from crewai_core.settings import Settings
+from dotenv import load_dotenv, set_key
+import httpx
+from rich.console import Console
+
+from crewai_cli.authentication.token import AuthError, get_auth_token
+from crewai_cli.plus_api import PlusAPI
+from crewai_cli.utils import get_or_create_project_id, is_dmn_mode_enabled
+
+
+console = Console()
+
+LAST_RUN_FILE = Path(".crewai") / "last_run.json"
+TRACING_ENV_VAR = "CREWAI_TRACING_ENABLED"
+POLL_SECONDS = 3.0
+POLL_RETRIES = 5 # consecutive unreachable / 5xx polls before giving up; the evaluation keeps running
+FINISHED = {"done", "failed"}
+STATUSES = {"queued", "running"} | FINISHED
+
+
+def eval_crew(run_id: str | None = None) -> None:
+ """Evaluate the last traced run of this project, or the run RUN_ID."""
+ get_or_create_project_id()
+ # Read before the project's .env is loaded, so a project cannot add itself.
+ trusted = _trusted_amp_origins()
+ _load_project_env()
+ record = read_last_run() or {}
+ execution_id = run_id or record.get("execution_id")
+ if execution_id is None:
+ execution_id = _run_now_or_explain()
+ record = read_last_run() or {}
+
+ client = _amp_client(trusted)
+ recorded_amp = str(record.get("amp_base_url") or "").rstrip("/")
+ if not run_id and recorded_amp and recorded_amp != client.base_url.rstrip("/"):
+ console.print(
+ f"The run was traced to {recorded_amp}; evaluating at the configured AMP {client.base_url}.",
+ style="yellow",
+ )
+ started = _start_evaluation(client, execution_id)
+ url = started.get("url")
+ console.print(f"Evaluating run [bold]{execution_id}[/bold]")
+ if url:
+ console.print(f"Follow it at [cyan underline]{url}[/cyan underline]")
+ _open(url)
+
+ finished = _wait(client, str(started["id"]), url)
+ _print_verdict(finished, url)
+ if finished.get("status") != "done":
+ raise SystemExit(1)
+
+
+def _trusted_amp_origins() -> set[str]:
+ """Where the saved login may be sent: the AMP this machine is configured for
+ (`crewai enterprise configure`), one already exported in this shell, and
+ crewAI's own."""
+ candidates = (
+ os.environ.get("CREWAI_PLUS_URL"),
+ Settings().enterprise_base_url,
+ DEFAULT_CREWAI_ENTERPRISE_URL,
+ )
+ return {origin for origin in map(_origin, candidates) if origin}
+
+
+def _origin(url: str | None) -> str | None:
+ parsed = urlparse(str(url or ""))
+ return (
+ f"{parsed.scheme}://{parsed.netloc}".lower()
+ if parsed.scheme and parsed.netloc
+ else None
+ )
+
+
+def _encrypted(origin: str | None) -> bool:
+ """HTTPS, or plain HTTP to this machine — the rule `TraceGrantClient` already
+ applies to collector grants (localhost, its subdomains, loopback addresses)."""
+ parsed = urlparse(origin or "")
+ if parsed.scheme == "https":
+ return True
+ if parsed.scheme != "http":
+ return False
+ hostname = (parsed.hostname or "").rstrip(".")
+ if hostname == "localhost" or hostname.endswith(".localhost"):
+ return True
+ try:
+ return ip_address(hostname).is_loopback
+ except ValueError:
+ return False
+
+
+def _amp_client(trusted: set[str]) -> PlusAPI:
+ """The AMP to ask, and whether the saved login goes with it.
+
+ A project's `.env` may point `crewai eval` at another AMP — that is how a
+ self-hosted project is wired, and the run was traced there — so the request
+ follows it. The credential does not: it goes only to an AMP this machine is
+ logged in to, over a connection that encrypts it. Anywhere else the run is
+ read anonymously."""
+ client = PlusAPI(api_key=saved_login())
+ if client.api_key is None:
+ return client
+
+ origin = _origin(client.base_url)
+ if origin in trusted and _encrypted(origin):
+ return client
+
+ why = (
+ "is not an AMP this machine is logged in to"
+ if origin not in trusted
+ else "would carry the login over plain HTTP"
+ )
+ console.print(
+ f"Reading anonymously: {client.base_url} {why}. "
+ "Run `crewai enterprise configure <url>` to log in to it.",
+ style="yellow",
+ )
+ return PlusAPI()
+
+
+def _load_project_env() -> None:
+ """The project's .env, as `crewai run` loads it — so CREWAI_PLUS_URL here is the one the run used."""
+ env_file = Path.cwd() / ".env"
+ if env_file.is_file():
+ load_dotenv(env_file, override=True)
+
+
+def read_last_run(directory: Path | None = None) -> dict[str, Any] | None:
+ """The record crewAI wrote for the project's last traced run (execution_id,
+ tier, started_at, finished_at, amp_base_url), or None."""
+ path = (directory or Path.cwd()) / LAST_RUN_FILE
+ try:
+ loaded = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return None
+ if not isinstance(loaded, dict) or not loaded.get("execution_id"):
+ return None
+ loaded["execution_id"] = str(loaded["execution_id"])
+ return loaded
+
+
+def saved_login() -> str | None:
+ """The `crewai login` token, or None: AMP then treats the caller as anonymous.
+
+ Only "not logged in" reads as anonymous. A credential that exists but cannot
+ be read — a rotated key, a half-written store, a directory `sudo` left owned
+ by root — is said out loud: reading anonymously instead would quietly spend
+ the run's one anonymous read and then refuse a user who believes they are
+ logged in."""
+ try:
+ return get_auth_token()
+ except AuthError:
+ return None
+ except Exception as error:
+ _fail(
+ f"Could not read the saved login ({type(error).__name__}: {error}). "
+ "Run `crewai login` again, or `crewai eval` will not know who you are."
+ )
+ return None
+
+
+def _run_now_or_explain() -> str:
+ """No traced run recorded here: offer to turn tracing on and run the crew now."""
+ if not Path("pyproject.toml").is_file():
+ _fail(
+ "No crewAI project here (no pyproject.toml). Run `crewai eval` from the project's "
+ "directory, or name a run: `crewai eval --run EXECUTION_ID`."
+ )
+ steps = (
+ "No traced run is recorded in this project. Turn tracing on and run the crew, "
+ f"then come back:\n 1. add {TRACING_ENV_VAR}=true to .env\n 2. crewai run\n 3. crewai eval"
+ )
+ if is_dmn_mode_enabled() or not sys.stdin.isatty():
+ console.print(steps, style="yellow")
+ raise SystemExit(1)
+ if not click.confirm(
+ f"No traced run is recorded in this project. Turn tracing on ({TRACING_ENV_VAR}=true "
+ "stays in .env) and run the crew now?",
+ default=True, # João, 2026-09-20: y/n with Y as the default — the prompt says what Enter does
+ ):
+ console.print(steps, style="yellow")
+ raise SystemExit(0)
+
+ _enable_tracing()
+ from crewai_cli.run_crew import run_crew
+
+ run_crew()
+ record = read_last_run()
+ if record is None:
+ console.print(
+ "The run finished but no trace was recorded: the run may have failed, sharing the "
+ "trace was declined, or this project's crewai is older than the version that records "
+ f"the last run ({LAST_RUN_FILE}). Run the crew again and accept when asked, then `crewai eval`.",
+ style="bold red",
+ )
+ raise SystemExit(1)
+ return str(record["execution_id"])
+
+
+def _enable_tracing() -> None:
+ """`CREWAI_TRACING_ENABLED=true` in the project's .env, and in this process for the run about to start."""
+ env_file = Path.cwd() / ".env"
+ env_file.touch(exist_ok=True)
+ set_key(str(env_file), TRACING_ENV_VAR, "true", quote_mode="never")
+ os.environ[TRACING_ENV_VAR] = "true"
+ console.print(
+ f"Tracing is on for this project ({TRACING_ENV_VAR}=true in .env).",
+ style="green",
+ )
+
+
+def _start_evaluation(client: PlusAPI, execution_id: str) -> dict[str, Any]:
+ try:
+ response = client.create_evaluation(execution_id)
+ except httpx.HTTPError as error:
+ _fail(f"Could not reach AMP to start the evaluation: {error}")
+ if response.status_code in (200, 202):
+ payload = _payload(response)
+ if payload and payload.get("id"):
+ return payload
+ _fail(f"AMP answered without an evaluation id ({response.status_code}).")
+ _refused(response, f"run {execution_id}")
+ raise AssertionError("unreachable")
+
+
+def _wait(client: PlusAPI, evaluation_id: str, url: str | None) -> dict[str, Any]:
+ """Poll until the evaluation is done or failed; Ctrl-C leaves it running."""
+ console.print("Waiting for the verdict…", style="dim")
+ where = f" at {url}" if url else ""
+ subject = f"evaluation {evaluation_id}"
+ misses = (
+ 0 # AMP unreachable or answering 5xx: a blip is retried, a streak is reported
+ )
+ try:
+ while True:
+ try:
+ response = client.get_evaluation(evaluation_id)
+ except httpx.HTTPError as error:
+ misses += 1
+ if misses >= POLL_RETRIES:
+ _fail(
+ f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}."
+ )
+ time.sleep(POLL_SECONDS)
+ continue
+ if response.status_code >= 500:
+ misses += 1
+ if misses >= POLL_RETRIES:
+ _refused(response, subject)
+ time.sleep(POLL_SECONDS)
+ continue
+ if response.status_code != 200:
+ _refused(response, subject)
+ misses = 0
+ payload = _payload(response) or {}
+ status = payload.get("status")
+ if status == "done" and not _well_formed_verdict(payload.get("verdict")):
+ _fail(
+ f"AMP answered done without a verdict (protocol error); follow it{where or ' on AMP'}."
+ )
+ if status in FINISHED:
+ return payload
+ if status not in STATUSES:
+ _fail(
+ f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}."
+ )
+ time.sleep(POLL_SECONDS)
+ except KeyboardInterrupt:
+ console.print(f"\nStill running{where}.", style="yellow")
+ raise SystemExit(130) from None
+
+
+def _well_formed_verdict(verdict: Any) -> bool:
+ """`{"gate": "<word>", "grades": {area: 1..5 | null}}` — anything else is a
+ protocol error. A grade is an exact integer in range: `True` is an `int` to
+ Python and 6 is not a grade, and neither may print as one."""
+ return (
+ isinstance(verdict, dict)
+ and isinstance(verdict.get("gate"), str)
+ and isinstance(verdict.get("grades"), dict)
+ and all(_a_grade(grade) for grade in verdict["grades"].values())
+ )
+
+
+def _a_grade(grade: Any) -> bool:
+ return grade is None or (type(grade) is int and 1 <= grade <= 5)
+
+
+def _print_verdict(finished: dict[str, Any], url: str | None) -> None:
+ if finished.get("status") != "done":
+ console.print(
+ f"Evaluation failed: {finished.get('error') or 'no reason given'}",
+ style="bold red",
+ )
+ return
+ verdict = finished["verdict"] # _wait let only a well-formed one through
+ gate = str(verdict["gate"]).upper()
+ style = {"PASSED": "bold green", "FAILED": "bold red"}.get(gate, "bold yellow")
+ grades = verdict.get("grades") or {}
+ parts = [
+ f"{area} {grades[area]}/5"
+ if grades.get(area) is not None
+ else f"{area} not measured"
+ for area in ("goal", "quality", "process", "cost")
+ ]
+ console.print(f"Goal gate: [{style}]{gate}[/{style}] · " + " · ".join(parts))
+ if url:
+ console.print(f"Full report: {url}")
+
+
+def _open(url: str) -> None:
+ if is_dmn_mode_enabled():
+ return
+ with contextlib.suppress(Exception): # no browser is not an error
+ webbrowser.open(url)
+
+
+def _payload(response: httpx.Response) -> dict[str, Any] | None:
+ try:
+ loaded = response.json()
+ except ValueError:
+ return None
+ return loaded if isinstance(loaded, dict) else None
+
+
+def _refused(response: httpx.Response, subject: str) -> None:
+ """AMP's own words when it sent them, then exit 1. SUBJECT is "run <id>" or "evaluation <id>"."""
+ payload = _payload(response) or {}
+ message = str(payload.get("message") or "").strip()
+ error = str(payload.get("error") or "")
+ if response.status_code in (401, 403):
+ if error == "account_required" and message:
+ _fail(message)
+ _fail(
+ f"{message or 'AMP refused the credential'}. Log in with `crewai login` and try again."
+ )
+ if response.status_code == 404:
+ _fail(message or f"AMP answered 404 for {subject}.")
+ if response.status_code == 429:
+ retry = response.headers.get("Retry-After")
+ _fail(
+ f"{message or 'AMP is rate limiting this request'}{f' — retry after {retry}s' if retry else ''}."
+ )
+ _fail(f"AMP answered {response.status_code}{': ' + message if message else ''}.")
+
+
+def _fail(message: str) -> None:
+ console.print(message, style="bold red")
+ raise SystemExit(1)
diff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.py
index 6f94d96d3..f03661e5b 100644
--- a/lib/cli/src/crewai_cli/plus_api.py
+++ b/lib/cli/src/crewai_cli/plus_api.py
@@ -18,9 +18,32 @@ class PlusAPI(_CorePlusAPI):
The ZIP deployment methods live here as well as in newer crewai-core
versions so editable CLI installs still work when an older crewai-core is
- present in the runtime environment.
+ present in the runtime environment. The evaluation methods live here
+ because only the CLI calls them.
"""
+ EVALUATIONS_RESOURCE = f"{_CorePlusAPI.TRACING_RESOURCE}/evaluations"
+ # AMP reads the run's spans from Wharf inside the POST; a large run takes a while.
+ EVALUATION_START_TIMEOUT = 120.0
+ EVALUATION_POLL_TIMEOUT = 30.0
+
+ def create_evaluation(self, execution_id: str) -> httpx.Response:
+ """Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval)."""
+ return self._make_request(
+ "POST",
+ self.EVALUATIONS_RESOURCE,
+ json={"execution_id": execution_id},
+ timeout=self.EVALUATION_START_TIMEOUT,
+ )
+
+ def get_evaluation(self, evaluation_id: str) -> httpx.Response:
+ """The evaluation's status and, once done, its verdict."""
+ return self._make_request(
+ "GET",
+ f"{self.EVALUATIONS_RESOURCE}/{evaluation_id}",
+ timeout=self.EVALUATION_POLL_TIMEOUT,
+ )
+
def _make_multipart_request(
self,
method: HttpMethod,
diff --git a/lib/cli/tests/test_eval_crew.py b/lib/cli/tests/test_eval_crew.py
new file mode 100644
index 000000000..4c91e3fe3
--- /dev/null
+++ b/lib/cli/tests/test_eval_crew.py
@@ -0,0 +1,505 @@
+"""`crewai eval`: the last traced run, evaluated through AMP."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from types import SimpleNamespace
+
+from click.testing import CliRunner
+import httpx
+import pytest
+from rich.console import Console
+
+from crewai_cli import eval_crew as eval_module
+from crewai_cli.cli import eval_command
+
+
+EXECUTION_ID = "6f31fe1a-20bd-4bfe-a011-25d6b9341f62"
+URL = "https://evolve.crewai.test/e/ev-1"
+
+
+class FakeAMP:
+ """A PlusAPI double: scripted answers, calls recorded."""
+
+ def __init__(self, create=None, statuses=None):
+ self.create = create if create is not None else httpx.Response(
+ 202, json={"id": "ev-1", "url": URL, "status": "queued"}
+ )
+ self.statuses = list(statuses or [])
+ self.calls: list[tuple] = []
+ self.api_key = None
+
+ def create_evaluation(self, execution_id):
+ self.calls.append(("create", execution_id))
+ return self.create
+
+ def get_evaluation(self, evaluation_id):
+ self.calls.append(("get", evaluation_id))
+ return self.statuses.pop(0) if self.statuses else httpx.Response(200, json={"id": evaluation_id, "status": "running"})
+
+
+def done(gate="passed", grades=None):
+ return httpx.Response(200, json={
+ "id": "ev-1", "status": "done", "url": URL,
+ "verdict": {"gate": gate, "grades": grades if grades is not None else {"goal": 5, "quality": 4, "process": 5, "cost": None}},
+ })
+
+
+def project(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(eval_module, "console", Console(width=240)) # one sentence per line in the captured output
+ monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: None)
+ monkeypatch.setattr(eval_module, "saved_login", lambda: "login-token")
+ monkeypatch.setattr(eval_module.time, "sleep", lambda seconds: None)
+ monkeypatch.setattr(eval_module, "is_dmn_mode_enabled", lambda: False)
+ # This machine is logged in to https://amp.test (`crewai enterprise configure`).
+ monkeypatch.setattr(eval_module, "Settings", lambda: SimpleNamespace(enterprise_base_url="https://amp.test"))
+ monkeypatch.delenv("CREWAI_PLUS_URL", raising=False)
+ opened: list[str] = []
+ monkeypatch.setattr(eval_module.webbrowser, "open", lambda url: opened.append(url) or True)
+ return tmp_path, opened
+
+
+def record_last_run(directory: Path, execution_id: str = EXECUTION_ID, **fields) -> None:
+ (directory / ".crewai").mkdir(exist_ok=True)
+ record = {"execution_id": execution_id, "tier": "ephemeral", "amp_base_url": "https://amp.test", **fields}
+ (directory / ".crewai" / "last_run.json").write_text(json.dumps(record))
+
+
+def install(monkeypatch, amp: FakeAMP, configured_amp: str = "https://amp.test") -> FakeAMP:
+ def build(api_key=None, base_url=None):
+ # PlusAPI's own resolution: explicit, then CREWAI_PLUS_URL, then the saved settings.
+ amp.api_key = api_key
+ amp.base_url = base_url or os.environ.get("CREWAI_PLUS_URL") or configured_amp
+ return amp
+
+ monkeypatch.setattr(eval_module, "PlusAPI", build)
+ return amp
+
+
+def test_the_last_run_is_evaluated_the_url_opened_and_the_verdict_printed(project, monkeypatch, capsys):
+ directory, opened = project
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP(statuses=[httpx.Response(200, json={"id": "ev-1", "status": "running"}), done()]))
+
+ eval_module.eval_crew()
+
+ out = capsys.readouterr().out
+ assert amp.api_key == "login-token"
+ assert amp.calls == [("create", EXECUTION_ID), ("get", "ev-1"), ("get", "ev-1")]
+ assert opened == [URL]
+ assert EXECUTION_ID in out and URL in out
+ assert "Goal gate: PASSED" in out and "goal 5/5" in out and "cost not measured" in out
+
+
+def test_run_names_another_execution_and_an_anonymous_caller_sends_no_token(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ monkeypatch.setattr(eval_module, "saved_login", lambda: None)
+ amp = install(monkeypatch, FakeAMP(statuses=[done("failed")]))
+
+ eval_module.eval_crew(run_id="other-run")
+
+ assert amp.api_key is None
+ assert amp.calls[0] == ("create", "other-run")
+ assert "Goal gate: FAILED" in capsys.readouterr().out
+
+
+def test_a_failed_evaluation_exits_one_with_amps_reason(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(statuses=[httpx.Response(200, json={"id": "ev-1", "status": "failed", "error": "the judge was unreachable"})]))
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ assert "Evaluation failed: the judge was unreachable" in capsys.readouterr().out
+
+
+ ("response", "expected"),
+ [
+ (httpx.Response(401, json={"error": "account_required", "message": "Execution x was already read once without an account. Log in with `crewai login`, or create an account, to read it again."}),
+ "already read once without an account"),
+ (httpx.Response(401, json={"error": "bad_credentials", "message": "Bad credentials"}), "Bad credentials. Log in with `crewai login`"),
+ (httpx.Response(404, json={"error": "trace_not_found", "message": "No spans recorded for execution 6f31"}), "No spans recorded for execution 6f31"),
+ (httpx.Response(404, text="<html>Page not found</html>"), f"AMP answered 404 for run {EXECUTION_ID}."),
+ (httpx.Response(202, json={"url": URL}), "AMP answered without an evaluation id (202)."),
+ (httpx.Response(429, json={"error": "rate_limit_exceeded", "message": "Too many requests"}, headers={"Retry-After": "60"}), "Too many requests — retry after 60s"),
+ (httpx.Response(503, json={"error": "service_unavailable", "message": "Wharf could not list the spans"}), "AMP answered 503: Wharf could not list the spans"),
+ (httpx.Response(500, text="boom"), "AMP answered 500."),
+ ],
+)
+def test_amps_refusals_are_printed_in_its_words_and_exit_one(project, monkeypatch, capsys, response, expected):
+ directory, opened = project
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(create=response))
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ assert expected in capsys.readouterr().out
+ assert opened == []
+
+
+def test_the_credential_goes_only_to_the_configured_amp_never_to_an_address_off_the_record(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory, amp_base_url="https://evil.example/steal")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]), configured_amp="https://app.crewai.com")
+
+ eval_module.eval_crew()
+
+ assert amp.api_key == "login-token" and amp.base_url == "https://app.crewai.com" # PlusAPI got no base_url
+ out = capsys.readouterr().out
+ assert "The run was traced to https://evil.example/steal; evaluating at the configured AMP https://app.crewai.com." in out
+
+ # The project's .env is what `crewai run` traced with, so it is loaded first. This machine is
+ # logged in to that AMP, so the credential goes with the request and nothing is remarked on.
+ (directory / ".env").write_text("CREWAI_PLUS_URL=https://amp.test\n")
+ record_last_run(directory, amp_base_url="https://amp.test/")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+ eval_module.eval_crew()
+ assert eval_module.os.environ["CREWAI_PLUS_URL"] == "https://amp.test"
+ assert amp.api_key == "login-token" and amp.base_url == "https://amp.test"
+ out = capsys.readouterr().out
+ assert "was traced to" not in out and "Reading anonymously" not in out
+
+
+def test_a_project_may_point_at_another_amp_but_never_gets_the_saved_login(project, monkeypatch, capsys):
+ """A .env can send the request elsewhere — that is how a self-hosted project is wired —
+ but the token goes only to an AMP this machine is logged in to."""
+ directory, _ = project
+ (directory / ".env").write_text("CREWAI_PLUS_URL=https://evil.example\n")
+ record_last_run(directory, amp_base_url="https://evil.example")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew() # still works: AMP reads it anonymously
+
+ assert amp.base_url == "https://evil.example" # the request follows the project
+ assert amp.api_key is None # the credential does not
+ out = capsys.readouterr().out
+ assert "Reading anonymously: https://evil.example is not an AMP this machine is logged in to." in out
+ assert "crewai enterprise configure" in out
+
+
+def test_a_trusted_amp_over_plain_http_still_gets_no_credential(project, monkeypatch, capsys):
+ """A cleartext connection is not a place to put a bearer token, trusted or not."""
+ directory, _ = project
+ monkeypatch.setenv("CREWAI_PLUS_URL", "http://amp.internal") # exported, so it IS trusted
+ record_last_run(directory, amp_base_url="http://amp.internal")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.base_url == "http://amp.internal" and amp.api_key is None
+ assert "would carry the login over plain HTTP" in capsys.readouterr().out
+
+
+def test_plain_http_to_this_machine_is_fine_for_local_development(project, monkeypatch, capsys):
+ directory, _ = project
+ monkeypatch.setenv("CREWAI_PLUS_URL", "http://localhost:3000")
+ record_last_run(directory, amp_base_url="http://localhost:3000")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.api_key == "login-token"
+ assert "Reading anonymously" not in capsys.readouterr().out
+
+
+ ("origin", "encrypted"),
+ [
+ ("https://app.crewai.com", True),
+ ("http://localhost:3000", True),
+ ("http://127.0.0.1:8000", True),
+ ("http://[::1]:8000", True),
+ ("http://amp.localhost", True),
+ ("http://amp.internal", False),
+ ("http://169.254.169.254", False),
+ ("ftp://amp.test", False),
+ (None, False),
+ ],
+)
+def test_which_connections_may_carry_the_login(origin, encrypted):
+ assert eval_module._encrypted(origin) is encrypted
+
+
+def test_an_amp_exported_in_this_shell_is_trusted(project, monkeypatch, capsys):
+ directory, _ = project
+ monkeypatch.setenv("CREWAI_PLUS_URL", "https://shell.amp.test") # exported before the project is read
+ record_last_run(directory, amp_base_url="https://shell.amp.test")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.base_url == "https://shell.amp.test" and amp.api_key == "login-token"
+ assert "Reading anonymously" not in capsys.readouterr().out
+
+
+ ("url", "origin"),
+ [
+ ("https://amp.test", "https://amp.test"),
+ ("https://AMP.Test/crewai_plus/", "https://amp.test"),
+ ("http://localhost:8000/x", "http://localhost:8000"),
+ ("app.crewai.com", None), # no scheme: not an origin, never trusted
+ ("", None),
+ (None, None),
+ ],
+)
+def test_an_origin_is_scheme_and_host_only(url, origin):
+ assert eval_module._origin(url) == origin
+
+
+ "verdict",
+ [
+ None,
+ "passed",
+ [],
+ {"gate": "passed"},
+ {"gate": None, "grades": {}},
+ {"gate": "passed", "grades": "5/5"},
+ {"gate": "passed", "grades": {"goal": "five"}},
+ {"gate": "passed", "grades": {"goal": True}}, # a bool is an int to Python, never a grade
+ {"gate": "passed", "grades": {"goal": 6}}, # out of the 1..5 range
+ {"gate": "passed", "grades": {"goal": 0}},
+ {"gate": "passed", "grades": {"goal": 4.5}},
+ ],
+)
+def test_a_done_answer_without_a_well_formed_verdict_is_a_protocol_error(project, monkeypatch, capsys, verdict):
+ directory, _ = project
+ record_last_run(directory)
+ body = {"id": "ev-1", "status": "done", "url": URL}
+ if verdict is not None:
+ body["verdict"] = verdict
+ install(monkeypatch, FakeAMP(statuses=[httpx.Response(200, json=body)]))
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ assert f"AMP answered done without a verdict (protocol error); follow it at {URL}." in capsys.readouterr().out
+
+
+def test_amp_unreachable_at_the_start_is_a_sentence_not_a_traceback(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP())
+ monkeypatch.setattr(amp, "create_evaluation", lambda execution_id: (_ for _ in ()).throw(httpx.ConnectError("connection refused")))
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ assert "Could not reach AMP to start the evaluation: connection refused" in capsys.readouterr().out
+
+
+def test_while_waiting_a_blip_is_retried_and_a_streak_is_reported(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP(statuses=[httpx.Response(502), httpx.Response(503, json={"error": "service_unavailable", "message": "crew-optimize is down"}), done()]))
+
+ eval_module.eval_crew() # two bad polls, then the verdict
+
+ assert "Goal gate: PASSED" in capsys.readouterr().out
+ assert amp.calls.count(("get", "ev-1")) == 3
+
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP(statuses=[httpx.Response(503, json={"error": "service_unavailable", "message": "crew-optimize is down"})] * eval_module.POLL_RETRIES))
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+ assert exit_.value.code == 1
+ assert "AMP answered 503: crew-optimize is down" in capsys.readouterr().out
+ assert amp.calls.count(("get", "ev-1")) == eval_module.POLL_RETRIES
+
+ amp = install(monkeypatch, FakeAMP())
+ monkeypatch.setattr(amp, "get_evaluation", lambda evaluation_id: (_ for _ in ()).throw(httpx.ReadTimeout("timed out")))
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+ assert exit_.value.code == 1
+ assert f"Could not reach AMP while waiting (timed out); the evaluation keeps running at {URL}." in capsys.readouterr().out
+
+
+def test_a_refusal_mid_poll_names_the_evaluation_and_an_unknown_status_stops_the_wait(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(statuses=[httpx.Response(404, text="gone")]))
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+ assert exit_.value.code == 1 and "AMP answered 404 for evaluation ev-1." in capsys.readouterr().out
+
+ for odd in (httpx.Response(200, json={"id": "ev-1", "status": "cancelled"}), httpx.Response(200, json=[]), httpx.Response(200, text="<html>")):
+ install(monkeypatch, FakeAMP(statuses=[httpx.Response(200, json={"id": "ev-1", "status": "queued"}), odd]))
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+ assert exit_.value.code == 1
+ assert f"AMP answered without a known evaluation status" in capsys.readouterr().out
+
+
+def test_ctrl_c_leaves_the_evaluation_running_and_exits_130(project, monkeypatch, capsys):
+ directory, _ = project
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP())
+ monkeypatch.setattr(amp, "get_evaluation", lambda evaluation_id: (_ for _ in ()).throw(KeyboardInterrupt()))
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 130
+ assert f"Still running at {URL}." in capsys.readouterr().out
+
+
+def test_dmn_mode_prints_the_url_but_opens_no_browser(project, monkeypatch, capsys):
+ directory, opened = project
+ record_last_run(directory)
+ monkeypatch.setattr(eval_module, "is_dmn_mode_enabled", lambda: True)
+ install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert opened == [] and URL in capsys.readouterr().out
+
+
+def test_run_skips_the_offer_when_nothing_is_recorded(project, monkeypatch, capsys):
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: pytest.fail("no offer with --run"))
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew(run_id="named-run")
+
+ assert amp.calls[0] == ("create", "named-run")
+
+
+def test_outside_a_crewai_project_nothing_is_written_and_it_says_so(project, monkeypatch, capsys):
+ directory, _ = project
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: pytest.fail("no offer outside a project"))
+ amp = install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ assert "No crewAI project here (no pyproject.toml)" in capsys.readouterr().out
+ assert not (directory / ".env").exists() and amp.calls == []
+
+
+def test_without_a_traced_run_and_no_terminal_it_explains_and_exits(project, monkeypatch, capsys):
+ (project[0] / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module, "is_dmn_mode_enabled", lambda: True)
+ amp = install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ out = capsys.readouterr().out
+ assert "No traced run is recorded" in out and "CREWAI_TRACING_ENABLED=true" in out and "crewai run" in out
+ assert amp.calls == []
+
+
+def test_without_a_traced_run_it_offers_to_turn_tracing_on_and_run_the_crew(project, monkeypatch, capsys):
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ prompts: list[tuple] = []
+ monkeypatch.setattr(eval_module.click, "confirm", lambda text, **kwargs: prompts.append((text, kwargs)) or True)
+ ran: list[str] = []
+
+ def fake_run_crew() -> None:
+ ran.append("run")
+ assert eval_module.os.environ.get("CREWAI_TRACING_ENABLED") == "true"
+ record_last_run(directory, "fresh-run")
+
+ import crewai_cli.run_crew as run_crew_module
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert ran == ["run"]
+ text, kwargs = prompts[0]
+ assert "CREWAI_TRACING_ENABLED=true stays in .env" in text and kwargs == {"default": True} # Enter is yes (João's call); the prompt names both effects
+ assert "CREWAI_TRACING_ENABLED=true" in (directory / ".env").read_text()
+ assert amp.calls[0] == ("create", "fresh-run")
+ assert "Tracing is on for this project" in capsys.readouterr().out
+
+
+def test_declining_the_offer_exits_cleanly_with_the_steps(project, monkeypatch, capsys):
+ (project[0] / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: False)
+ amp = install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 0
+ assert "crewai eval" in capsys.readouterr().out and amp.calls == []
+
+
+def test_a_run_that_leaves_no_trace_behind_is_explained(project, monkeypatch, capsys):
+ (project[0] / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *args, **kwargs: True)
+ import crewai_cli.run_crew as run_crew_module
+
+ monkeypatch.setattr(run_crew_module, "run_crew", lambda: None)
+ install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ out = capsys.readouterr().out
+ assert "no trace was recorded" in out and "older than the version that records the last run" in out
+
+
+def test_the_cli_command_maps_to_the_implementation(monkeypatch):
+ calls = []
+ monkeypatch.setattr("crewai_cli.cli.eval_crew", lambda **kwargs: calls.append(kwargs))
+ runner = CliRunner()
+
+ assert runner.invoke(eval_command, []).exit_code == 0
+ assert runner.invoke(eval_command, ["--run", EXECUTION_ID]).exit_code == 0
+ assert calls == [{"run_id": None}, {"run_id": EXECUTION_ID}]
+ assert "Evaluate the last traced run" in runner.invoke(eval_command, ["--help"]).output
+
+
+def test_only_a_missing_login_reads_as_anonymous(monkeypatch, capsys):
+ """An unreadable credential store is not "anonymous": it is said out loud."""
+ from crewai_cli.authentication.token import AuthError
+
+ monkeypatch.setattr(eval_module, "get_auth_token", lambda: (_ for _ in ()).throw(AuthError("No token found")))
+ assert eval_module.saved_login() is None # not logged in: AMP treats the caller as anonymous
+
+ monkeypatch.setattr(eval_module, "get_auth_token", lambda: "login-token")
+ assert eval_module.saved_login() == "login-token"
+
+ for broken in (OSError(13, "Permission denied"), ValueError("Fernet key must be 32 url-safe base64-encoded bytes.")):
+ monkeypatch.setattr(eval_module, "get_auth_token", lambda error=broken: (_ for _ in ()).throw(error))
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.saved_login()
+ assert exit_.value.code == 1
+ out = capsys.readouterr().out
+ assert "Could not read the saved login" in out and type(broken).__name__ in out and "crewai login" in out
+
+
+def test_read_last_run_reads_the_record_crewai_writes(tmp_path):
+ assert eval_module.read_last_run(tmp_path) is None
+ (tmp_path / ".crewai").mkdir()
+ (tmp_path / ".crewai" / "last_run.json").write_text("not json")
+ assert eval_module.read_last_run(tmp_path) is None
+ (tmp_path / ".crewai" / "last_run.json").write_text(json.dumps({"tier": "ephemeral"}))
+ assert eval_module.read_last_run(tmp_path) is None
+ record_last_run(tmp_path)
+ record = eval_module.read_last_run(tmp_path)
+ assert record is not None and record["execution_id"] == EXECUTION_ID and record["amp_base_url"] == "https://amp.test"
diff --git a/lib/cli/tests/test_plus_api.py b/lib/cli/tests/test_plus_api.py
index 16cf684d5..e40cfdbc2 100644
--- a/lib/cli/tests/test_plus_api.py
+++ b/lib/cli/tests/test_plus_api.py
@@ -18,6 +18,33 @@ class TestPlusAPI(unittest.TestCase):
self.assertIn("CrewAI-CLI/", self.api.headers["User-Agent"])
self.assertTrue(self.api.headers["X-Crewai-Version"])
+ @patch("crewai_core.plus_api.PlusAPI._make_request")
+ def test_create_evaluation(self, mock_make_request):
+ mock_response = MagicMock()
+ mock_make_request.return_value = mock_response
+
+ response = self.api.create_evaluation("6f31fe1a-20bd-4bfe-a011-25d6b9341f62")
+
+ mock_make_request.assert_called_once_with(
+ "POST",
+ "/crewai_plus/api/v1/tracing/evaluations",
+ json={"execution_id": "6f31fe1a-20bd-4bfe-a011-25d6b9341f62"},
+ timeout=120.0, # AMP reads the run's spans inside this request
+ )
+ self.assertEqual(response, mock_response)
+
+ @patch("crewai_core.plus_api.PlusAPI._make_request")
+ def test_get_evaluation(self, mock_make_request):
+ mock_response = MagicMock()
+ mock_make_request.return_value = mock_response
+
+ response = self.api.get_evaluation("ev-1")
+
+ mock_make_request.assert_called_once_with(
+ "GET", "/crewai_plus/api/v1/tracing/evaluations/ev-1", timeout=30.0
+ )
+ self.assertEqual(response, mock_response)
+
@patch("crewai_core.plus_api.PlusAPI._make_request")
def test_login_to_tool_repository(self, mock_make_request):
mock_response = MagicMock()