Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(cli): crewai eval export downloads an evaluation as files (#8187)
João Moura1410318b2ed0
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.pyindex f0765e4dc..d4a555b2f 100644--- a/lib/cli/src/crewai_cli/cli.py+++ b/lib/cli/src/crewai_cli/cli.py@@ -60,6 +60,12 @@ def eval_models(*args: Any, **kwargs: Any) -> Any:return _eval_models(*args, **kwargs)+def eval_export(*args: Any, **kwargs: Any) -> Any:+ from crewai_cli.experimental.eval_crew import export_evaluation as _export++ return _export(*args, **kwargs)++if TYPE_CHECKING:# mypy sees the real classes; at runtime the shims below defer the# heavy imports until a command actually instantiates them.@@ -690,7 +696,7 @@ def run()[email protected](name="eval")[email protected](name="eval", invoke_without_command=True)@click.option("--run","run_id",@@ -728,8 +734,12 @@ def run("project id."),)[email protected]_contextdef eval_command(- run_id: str | None, models: str | None, deployment_id: str | None+ ctx: click.Context,+ run_id: str | None,+ models: str | None,+ deployment_id: str | None,) -> None:"""Evaluate the last traced run through CrewAI AMP, or compare models on theproject's deployment (--models).@@ -744,7 +754,19 @@ def eval_command(failed gate, no verdict, or an evaluation that could not run — so a CI job cangate on it. A comparison exits 0 when it finished, 1 when it failed or couldnot start.++ `crewai eval export ID` downloads an evaluation's full data as files, for a+ coding agent to dig into."""+ if ctx.invoked_subcommand is not None:+ # The subcommand does the work; an option meant for this body would+ # otherwise be dropped without a word.+ if run_id is not None or models is not None or deployment_id is not None:+ raise click.UsageError(+ f"`crewai eval {ctx.invoked_subcommand}` takes no --run, --models "+ "or --deployment; give those to `crewai eval` alone."+ )+ returnif models is not None:if run_id is not None:raise click.UsageError(@@ -760,6 +782,23 @@ def eval_command(eval_crew(run_id=run_id)+@eval_command.command(name="export")[email protected]("evaluation_id", metavar="ID")+def eval_export_command(evaluation_id: str) -> None:+ """Download evaluation ID's full data into ./crewai-eval-ID/.++ The report, every setup's outputs, every check with the judge's words, and+ the traces, as files for a coding agent to dig into: start with report.md.+ The folder gets a .gitignore of `*`, so the data is never committed by+ accident; exporting again overwrites the files.++ ID is the evaluation's UUID. It is read with your `crewai login` when there+ is one; a run's evaluation can be exported without one. Exits 0 when the+ files were written, 1 otherwise.+ """+ eval_export(evaluation_id)++@crewai.command()def update() -> None:"""Update the pyproject.toml of the Crew project to use uv."""diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex 993212cfd..847b1ad7d 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -17,20 +17,27 @@ deployed and once per model; each run is graded, and the comparison — the fourgrades, cost and time per model, and what would make the agents do better — isprinted when it is done. It always needs the login: it runs the deployment.Every evaluation, of either kind, is filed under the project id.++`crewai eval export ID` downloads an evaluation's full data — the report,+every setup's outputs, every check with the judge's words, and the traces —+into `./crewai-eval-<ID>/`, for a coding agent to dig into."""from __future__ import annotations-from collections.abc import Callable+from collections.abc import Callable, Iteratorimport contextlibfrom dataclasses import dataclassfrom datetime import datetime, timedelta, timezonefrom email.utils import parsedate_to_datetime+import errnofrom ipaddress import ip_addressimport jsonimport mathimport osfrom pathlib import Path+import re+import statimport sysimport timefrom typing import Any, Literal, NoReturn, cast@@ -1694,3 +1701,360 @@ def _fail(message: str) -> NoReturn:# comes from `style`.console.print(Text(message), style="bold red")raise SystemExit(1)+++# ── crewai eval export ───────────────────────────────────────────────────────++EXPORT_DIR_PREFIX = "crewai-eval-"+EXPORT_START_HERE = "report.md"+# One path segment of an exported file: plain names only. Everything else —+# a separator other than `/`, a drive, a space, a control character — is not+# a path this command writes.+_EXPORT_SEGMENT = re.compile(r"[A-Za-z0-9._-]+")+# The names Windows keeps for its devices, whatever the extension: there a+# file called `NUL.md` is the null device, and what is written to it is gone.+_WINDOWS_DEVICE_NAMES = frozenset(+ {"con", "prn", "aux", "nul"}+ | {f"{port}{n}" for port in ("com", "lpt") for n in range(10)}+)+# Linux and macOS open a file inside an open folder, and can refuse to go+# through a link while doing it, so the export is written that way: no path+# is checked first and opened again later, when another process may have put+# a link in its way (`_put_through_handles`). Windows cannot; there each path+# is resolved and checked to stay inside the folder before it is written+# (`_put_by_path`). That check can be raced: a process that can write inside+# the export folder could swap a junction in between, and any account can make+# one. The bounded loss, accepted: such a process already writes where this+# user does (the folder is in the user's own working directory), unless the+# CLI runs elevated in a folder a less trusted account may write to.+_WRITES_THROUGH_HANDLES = (+ hasattr(os, "O_NOFOLLOW")+ and hasattr(os, "O_DIRECTORY")+ and os.open in os.supports_dir_fd+ and os.mkdir in os.supports_dir_fd+ and os.unlink in os.supports_dir_fd+ and os.stat in os.supports_dir_fd+ and os.stat in os.supports_follow_symlinks+)+_FOLDER_FLAGS = (+ os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)+)+_NEW_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)+++def export_evaluation(evaluation_id: str) -> None:+ """Write evaluation EVALUATION_ID's full data into `./crewai-eval-<id>/`.++ The files are AMP's: the report, every setup's outputs and checks with the+ judge's words, and the traces. They are written as sent, one file per+ path, beside a `.gitignore` of `*` — this is a customer's data, and a+ folder dropped into a repository must not be committed by accident.+ Re-running overwrites them.+ """+ try:+ canonical = str(uuid.UUID(evaluation_id.strip()))+ except ValueError:+ _fail(f"{evaluation_id!r} is not an evaluation id: an evaluation id is a UUID.")+ # Read before the project's .env is loaded, so a project cannot add itself.+ trusted = _trusted_amp_origins()+ _load_project_env()+ try:+ client = _amp_client(trusted)+ except EvaluationStoppedError as stopped:+ _fail(str(stopped))+ try:+ response = client.export_evaluation(canonical)+ except httpx.HTTPError as error:+ _fail(f"Could not reach AMP to export evaluation {canonical}: {error}")+ if response.status_code != 200:+ _fail(+ _export_refusal(+ response,+ canonical,+ logged_in=client.api_key is not None,+ amp=client.base_url,+ )+ )+ files = (_payload(response) or {}).get("files")+ if not isinstance(files, dict):+ _fail(+ f"AMP answered without the files of evaluation {canonical} (protocol error)."+ )++ folder = Path(f"{EXPORT_DIR_PREFIX}{canonical}")+ written, skipped = _write_export(folder, files)+ line = f"Wrote {len(written)} file{'' if len(written) == 1 else 's'} to {folder}/"+ if EXPORT_START_HERE in written:+ line += f" — start with {EXPORT_START_HERE}"+ if skipped:+ line += f" (skipped {skipped}: an unsafe path, or content that is not text)"+ # One line whatever the width: an agent reads the folder off it, and a+ # pipe is 80 columns to Rich.+ console.print(Text(line), soft_wrap=True)+++def _export_refusal(+ response: httpx.Response, evaluation_id: str, *, logged_in: bool, amp: str+) -> str:+ """AMP's refusal of an export as one sentence, in AMP's own words when it sent them.++ A self-hosted install updates AMP, its evaluation service and this CLI+ apart, so a part older than exports says which one needs the update.+ """+ payload = _payload(response) or {}+ message = str(payload.get("message") or "").strip()+ error = str(payload.get("error") or "")+ status = response.status_code+ # AMP's 501: its evaluation service predates exports.+ if error == "export_unsupported":+ return message or (+ "This AMP's evaluation service does not export evaluations yet: "+ "it needs an update."+ )+ # AMP's 502 or the evaluation service's 413: an export too large to carry+ # is refused whole, never cut short.+ if error == "export_too_large":+ return message or f"Evaluation {evaluation_id} is too large to export."+ # AMP says `not_found` of an evaluation it does not hold or this login may+ # not read; a 404 without it is an AMP with no export route at all.+ if status == 404 and error != "not_found":+ return (+ f"The AMP at {amp.rstrip('/')} does not serve `crewai eval export` "+ "yet: it needs an update."+ )+ if status == 404:+ if logged_in:+ return message or (+ f"AMP holds no evaluation {evaluation_id} that this login may read."+ )+ found = message or (+ f"AMP holds no evaluation {evaluation_id} that can be read without a login"+ )+ if "crewai login" in found:+ return found+ return (+ f"{found.rstrip('.')}; if it is your account's, log in with "+ "`crewai login` and export it again."+ )+ if status == 409:+ return message or (+ f"Evaluation {evaluation_id} is still running; export it once it has finished."+ )+ if status == 410:+ return message or (+ f"The report of evaluation {evaluation_id} was opened already and is "+ "closed; log in with `crewai login` and evaluate the run again."+ )+ # 401 / 403 / 429 / 5xx read as every other evaluation call reads them.+ return _refusal_message(response, f"evaluation {evaluation_id}")+++def _export_target(name: str) -> list[str] | None:+ """The path segments the exported file NAME is written at, or None when it may not be.++ The paths come from whichever AMP answered — which a project's `.env` may+ choose — so a path is written only when it is relative, `/`-separated, and+ every segment a plain name (`[A-Za-z0-9._-]+`, never `.` or `..`). A `.git`+ segment is refused too: a `.git/config` there could name a command for the+ next `git status` in the folder to run, and a shell prompt runs one. The+ top `.gitignore` is this command's own. So is a name Windows keeps for a+ device (`CON`, `PRN`, `AUX`, `NUL`, `COM0`-`COM9`, `LPT0`-`LPT9`, with any+ extension), on every platform, since the folder may be opened on Windows+ later or sit on a drive it shares: there `NUL.md` is the null device, and+ the export would report a file written that nobody can read. All are+ compared as a case-insensitive filesystem that drops trailing dots would+ see them. Where the segments lead, through a link or not, is the writer's+ to check (`_export_folder`).+ """+ segments = name.split("/")+ for segment in segments:+ seen = _as_a_filesystem_sees_it(segment)+ if (+ not _EXPORT_SEGMENT.fullmatch(segment)+ or segment in {".", ".."}+ or seen == ".git"+ or seen.split(".", 1)[0] in _WINDOWS_DEVICE_NAMES+ ):+ return None+ if len(segments) == 1 and _as_a_filesystem_sees_it(segments[0]) == ".gitignore":+ return None+ return segments+++def _as_a_filesystem_sees_it(segment: str) -> str:+ """SEGMENT as macOS and Windows compare names: case ignored, trailing dots dropped."""+ return segment.rstrip(".").lower()+++def _write_export(folder: Path, files: dict[str, Any]) -> tuple[list[str], int]:+ """Write FILES under FOLDER: the names written, and how many were skipped.++ The `.gitignore` goes first, so the data is never in the folder without it.+ A file whose path would go through a link, or through a file where a+ folder belongs, is skipped like any other unsafe path. A file that cannot+ be written stops the export with the reason: half an export that says it+ is whole would mislead the agent reading it.+ """+ written: list[str] = []+ skipped = 0+ with _export_folder(folder) as put:+ try:+ ignored = put([".gitignore"], b"*\n")+ except OSError as error:+ _fail(f"Could not write the export into {folder}/: {error}")+ if not ignored:+ # The data never goes into the folder without it.+ _fail(+ f"{folder}/.gitignore is a folder, not this command's file: remove it "+ "and export again, so the data is never committed by accident."+ )+ for name, content in files.items():+ segments = _export_target(name)+ if segments is None or not isinstance(content, str):+ skipped += 1+ continue+ try:+ data = content.encode("utf-8")+ except UnicodeEncodeError: # a lone surrogate JSON allowed through+ skipped += 1+ continue+ try:+ put_there = put(segments, data)+ except OSError as error:+ _fail(f"Could not write {folder}/{name}: {error}")+ if put_there:+ written.append(name)+ else:+ skipped += 1+ return written, skipped+++def _export_folder(folder: Path) -> Iterator[Callable[[list[str], bytes], bool]]:+ """FOLDER, made when missing, as a function that writes one file into it.++ The function takes the file's path segments and its bytes, and answers+ False, with nothing written, when a link or a file stands where one of the+ path's folders goes. FOLDER, a name in the current directory, being a link+ itself stops the export: the export writes only into a folder of its own.+ """+ if not _WRITES_THROUGH_HANDLES:+ root = _export_root_by_path(folder)+ yield lambda segments, data: _put_by_path(root, segments, data)+ return+ handle = _export_root_handle(folder)+ try:+ yield lambda segments, data: _put_through_handles(handle, segments, data)+ finally:+ os.close(handle)+++def _export_root_is_a_link(folder: Path) -> NoReturn:+ """Stop: whoever made FOLDER a link chose where it leads, not this command."""+ _fail(f"{folder} is a link; the export writes only into a folder of its own.")+++def _export_root_handle(folder: Path) -> int:+ """A handle on FOLDER, made when missing, opened inside the current+ directory and never through a link."""+ try:+ here = os.open(".", _FOLDER_FLAGS)+ try:+ with contextlib.suppress(FileExistsError):+ os.mkdir(folder.name, dir_fd=here)+ return os.open(folder.name, _FOLDER_FLAGS, dir_fd=here)+ finally:+ os.close(here)+ except OSError as error:+ if folder.is_symlink():+ _export_root_is_a_link(folder)+ _fail(f"Could not write the export into {folder}/: {error}")+++def _put_through_handles(root: int, segments: list[str], data: bytes) -> bool:+ """Write DATA at SEGMENTS under the folder ROOT is a handle on, through no link.++ Each folder of the path is opened inside the one before it, and the file+ inside the last, so no path is checked first and opened again later: a+ link where a folder goes, left there or swapped in while the export runs,+ is never gone through, and the file is not written (False). The file's own+ place is emptied first and the file made anew, so a link there, symbolic+ or hard, is replaced rather than written through.+ """+ folder = root+ try:+ for segment in segments[:-1]:+ with contextlib.suppress(FileExistsError):+ os.mkdir(segment, dir_fd=folder)+ try:+ inner = os.open(segment, _FOLDER_FLAGS, dir_fd=folder)+ except OSError as error:+ if error.errno in (errno.ELOOP, errno.ENOTDIR):+ return False # a link, or a file, where a folder goes+ raise+ if folder != root:+ os.close(folder)+ folder = inner+ leaf = segments[-1]+ try:+ if stat.S_ISDIR(+ os.stat(leaf, dir_fd=folder, follow_symlinks=False).st_mode+ ):+ return False # a folder where a file goes+ os.unlink(leaf, dir_fd=folder)+ except FileNotFoundError:+ pass # nothing at the file's place yet: it is simply made+ descriptor = os.open(leaf, _NEW_FILE_FLAGS, 0o666, dir_fd=folder)+ try:+ out = open(descriptor, "wb")+ except BaseException:+ os.close(descriptor)+ raise+ with out:+ out.write(data)+ return True+ finally:+ if folder != root:+ os.close(folder)+++def _export_root_by_path(folder: Path) -> Path:+ """FOLDER, made when missing, as the absolute path the files go under.++ It must resolve to itself, here: a link, or a Windows junction (which+ `is_symlink` does not see, and which any account can make), would carry+ every file somewhere else.+ """+ if folder.is_symlink():+ _export_root_is_a_link(folder)+ try:+ folder.mkdir(exist_ok=True)+ root = folder.resolve()+ except OSError as error:+ _fail(f"Could not write the export into {folder}/: {error}")+ if os.path.normcase(root) != os.path.normcase(Path.cwd().resolve() / folder.name):+ _export_root_is_a_link(folder)+ return root+++def _put_by_path(root: Path, segments: list[str], data: bytes) -> bool:+ """Write DATA at SEGMENTS under ROOT, where a folder cannot be opened+ without following a link: the path is resolved first, and written only+ when its folder is inside ROOT (False otherwise, or when a file stands+ where a folder goes, or a folder where the file goes). Whatever is at the+ file's own place, a link symbolic or hard, is removed and the file made+ anew, so nothing is written through it."""+ parent = root.joinpath(*segments[:-1]).resolve()+ if not parent.is_relative_to(root):+ return False+ try:+ parent.mkdir(parents=True, exist_ok=True)+ except (FileExistsError, NotADirectoryError):+ return False+ target = parent / segments[-1]+ if target.is_dir() and not target.is_symlink():+ return False # a folder, or a junction, where a file goes+ target.unlink(missing_ok=True)+ with open(target, "xb") as out:+ out.write(data)+ return Truediff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.pyindex 9632e8a8f..9f099e4b4 100644--- a/lib/cli/src/crewai_cli/plus_api.py+++ b/lib/cli/src/crewai_cli/plus_api.py@@ -26,6 +26,9 @@ class PlusAPI(_CorePlusAPI):# AMP reads the run's spans from Wharf inside the POST; a large run takes a while.EVALUATION_START_TIMEOUT = 120.0EVALUATION_POLL_TIMEOUT = 30.0+ # The export carries every setup's outputs, checks and traces — tens of MB+ # for a large comparison — and AMP gathers them before it answers.+ EVALUATION_EXPORT_TIMEOUT = httpx.Timeout(EVALUATION_POLL_TIMEOUT, read=120.0)def create_evaluation(self,@@ -86,13 +89,29 @@ class PlusAPI(_CorePlusAPI):)def get_evaluation(self, evaluation_id: str) -> httpx.Response:- """The evaluation's status and, once done, its verdict."""+ """The evaluation's status and, once done, its verdict.++ `can_export=1` says this CLI has `crewai eval export`, so the brief may+ point at it. AMP passes it on to the evaluation service; an AMP from+ before exports drops it, so the brief names the command only when the+ CLI, AMP and the service all serve it.+ """return self._make_request("GET",f"{self.EVALUATIONS_RESOURCE}/{evaluation_id}",+ params={"can_export": "1"},timeout=self.EVALUATION_POLL_TIMEOUT,)+ def export_evaluation(self, evaluation_id: str) -> httpx.Response:+ """The evaluation's full data as files (crewai eval export):+ `{"id", "kind", "files": {relative path: text}}`."""+ return self._make_request(+ "GET",+ f"{self.EVALUATIONS_RESOURCE}/{evaluation_id}/export",+ timeout=self.EVALUATION_EXPORT_TIMEOUT,+ )+def _make_multipart_request(self,method: HttpMethod,diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex 8a511b11d..37a8ac803 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -2348,3 +2348,544 @@ def test_a_failing_usage_count_never_changes_what_the_command_does(project, monkassert exited.value.code == 1assert eval_module._refusal_message(refusal, f"run {EXECUTION_ID}") in capsys.readouterr().out+++# --- crewai eval export ------------------------------------------------------++EVALUATION_ID = "0b6c1f5e-8d2a-4f3b-9c4e-7a1d2e3f4a5b"+FOLDER = f"crewai-eval-{EVALUATION_ID}"+EXPORT_FILES = {+ "report.md": "# Report\n\nThe gate passed.\n",+ "run/outputs.md": "## Final output\n\nThree trends.\n",+ "run/checks.md": "- goal: passed — the output names three trends\n",+ "run/trace.json": '{"spans": []}\n',+ "gemini-4-argon/outputs.md": "## Final output\n\nTwo trends.\n",+}+SKIPPED = "(skipped 1: an unsafe path, or content that is not text)"+++class FakeExportAMP(FakeAMP):+ def __init__(self, answer=None):+ super().__init__()+ self.answer = answer if answer is not None else exported()++ def export_evaluation(self, evaluation_id):+ self.calls.append(("export", evaluation_id))+ if isinstance(self.answer, Exception):+ raise self.answer+ return self.answer+++def exported(files=None, kind="models"):+ return httpx.Response(+ 200,+ json={"id": EVALUATION_ID, "kind": kind, "files": EXPORT_FILES if files is None else files},+ )+++def files_in(folder: Path) -> dict[str, str]:+ return {+ p.relative_to(folder).as_posix(): p.read_text(encoding="utf-8")+ for p in folder.rglob("*")+ if p.is_file()+ }+++NO_EXPORT_ROUTE = "The AMP at https://amp.test does not serve `crewai eval export` yet: it needs an update."+handles_only = pytest.mark.skipif(+ not eval_module._WRITES_THROUGH_HANDLES,+ reason="this platform cannot open a file inside a folder handle without following a link",+)++[email protected](params=["handles", "paths"])+def writes(request, monkeypatch):+ """Both ways the export writes: through folder handles (Linux, macOS), and+ by a checked path where those are missing (Windows)."""+ if request.param == "handles" and not eval_module._WRITES_THROUGH_HANDLES:+ pytest.skip("this platform cannot open a file inside a folder handle without following a link")+ monkeypatch.setattr(eval_module, "_WRITES_THROUGH_HANDLES", request.param == "handles")+ return request.param+++def test_the_eval_group_keeps_every_old_invocation_and_adds_export(monkeypatch):+ from crewai_cli.cli import crewai++ calls = []+ monkeypatch.setattr("crewai_cli.cli.eval_crew", lambda **kwargs: calls.append(("run", kwargs)))+ monkeypatch.setattr("crewai_cli.cli.eval_models", lambda *args, **kwargs: calls.append(("models", args, kwargs)))+ monkeypatch.setattr("crewai_cli.cli.eval_export", lambda *args: calls.append(("export", args)))+ runner = CliRunner()++ assert runner.invoke(crewai, ["eval"]).exit_code == 0+ assert runner.invoke(crewai, ["eval", "--run", EXECUTION_ID]).exit_code == 0+ assert runner.invoke(crewai, ["eval", "--models", MODELS, "--deployment", DEPLOYMENT]).exit_code == 0+ assert runner.invoke(crewai, ["eval", "export", EVALUATION_ID]).exit_code == 0+ assert calls == [+ ("run", {"run_id": None}),+ ("run", {"run_id": EXECUTION_ID}),+ ("models", (MODELS,), {"deployment_id": DEPLOYMENT}),+ ("export", (EVALUATION_ID,)),+ ]++ # an option meant for `crewai eval` itself is refused, never dropped+ mixed = runner.invoke(eval_command, ["--run", EXECUTION_ID, "export", EVALUATION_ID])+ assert mixed.exit_code == 2 and "takes no --run, --models or --deployment" in mixed.output+ assert runner.invoke(eval_command, ["export"]).exit_code == 2 # the id is required+ assert len(calls) == 4++ help_text = " ".join(runner.invoke(eval_command, ["--help"]).output.split())+ assert "`crewai eval export ID` downloads an evaluation's full data as files" in help_text+ assert "export Download evaluation ID's full data into ./crewai-eval-ID/." in help_text+++def test_the_eval_group_keeps_every_old_exit_code(monkeypatch):+ """A CI job gates on `crewai eval`: the command's own exit code still comes+ out of the group, and what was a usage error still is one."""+ from crewai_cli.cli import crewai++ def exits_1(*args, **kwargs):+ raise SystemExit(1)++ monkeypatch.setattr("crewai_cli.cli.eval_crew", exits_1)+ monkeypatch.setattr("crewai_cli.cli.eval_models", exits_1)+ monkeypatch.setattr("crewai_cli.cli.eval_export", exits_1)+ runner = CliRunner()++ for args in (+ ["eval"],+ ["eval", "--run", EXECUTION_ID],+ ["eval", "--models", MODELS],+ ["eval", "--models", MODELS, "--deployment", DEPLOYMENT],+ ["eval", "export", EVALUATION_ID],+ ):+ assert runner.invoke(crewai, args).exit_code == 1, args+ for args in (+ ["eval", "--models", MODELS, "--run", EXECUTION_ID],+ ["eval", "--deployment", DEPLOYMENT],+ ["eval", "an-argument"],+ ):+ assert runner.invoke(crewai, args).exit_code == 2, args++[email protected]("writes")+def test_export_writes_every_file_and_a_gitignore_and_exits_0(project, monkeypatch):+ directory, opened = project+ amp = install(monkeypatch, FakeExportAMP())++ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])++ assert result.exit_code == 0, result.output+ assert amp.calls == [("export", EVALUATION_ID)]+ assert amp.api_key == "login-token"+ assert files_in(directory / FOLDER) == {**EXPORT_FILES, ".gitignore": "*\n"}+ assert result.output == f"Wrote 5 files to {FOLDER}/ — start with report.md\n"+ assert opened == []+++def test_the_id_is_sent_and_named_in_its_canonical_form(project, monkeypatch, capsys):+ directory, _ = project+ amp = install(monkeypatch, FakeExportAMP(exported({"report.md": "r"})))++ eval_module.export_evaluation(f" {EVALUATION_ID.upper()} ")++ assert amp.calls == [("export", EVALUATION_ID)]+ assert files_in(directory / FOLDER) == {"report.md": "r", ".gitignore": "*\n"}+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/ — start with report.md\n"++[email protected]("writes")+def test_exporting_again_overwrites_the_files(project, monkeypatch, capsys):+ directory, _ = project+ install(monkeypatch, FakeExportAMP(exported({"report.md": "first", "run/checks.md": "old"})))+ eval_module.export_evaluation(EVALUATION_ID)+ install(monkeypatch, FakeExportAMP(exported({"report.md": "second", "run/checks.md": "new"})))+ eval_module.export_evaluation(EVALUATION_ID)++ assert files_in(directory / FOLDER) == {"report.md": "second", "run/checks.md": "new", ".gitignore": "*\n"}++[email protected]("writes")+ "name",+ [+ "../x.md",+ "{tmp}/outside.md", # absolute+ "a/../../b.md",+ "run/../../b.md",+ "a\\b.md",+ "..\\x.md",+ "a//b.md", # empty segment+ "run/",+ "",+ "./report.md",+ "run/./checks.md",+ "C:/x.md",+ "run/a b.md",+ "run/x\n.md",+ ".git/config",+ ".Git./config",+ ".gitignore", # the folder's own+ ".GITIGNORE",+ # names Windows keeps for its devices, with any extension+ "NUL.md",+ "nul",+ "com1.txt",+ "Aux./x.md",+ "run/lpt9.tar.gz",+ "CON",+ ],+)+def test_an_unsafe_path_is_skipped_and_nothing_is_written_outside(project, monkeypatch, capsys, name):+ directory, _ = project+ name = name.replace("{tmp}", str(directory))+ install(monkeypatch, FakeExportAMP(exported({name: "!*\nnot yours", "report.md": "# Report\n"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert [p.name for p in directory.iterdir()] == [FOLDER]+ assert files_in(directory / FOLDER) == {"report.md": "# Report\n", ".gitignore": "*\n"}+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/ — start with report.md {SKIPPED}\n"++[email protected]("writes")+def test_a_name_that_only_starts_like_a_windows_device_is_written(project, monkeypatch, capsys):+ directory, _ = project+ names = {"console.md": "c", "nullable.md": "n", "com10.md": "10", "auxiliary/x.md": "x"}+ install(monkeypatch, FakeExportAMP(exported(names)))++ eval_module.export_evaluation(EVALUATION_ID)++ assert files_in(directory / FOLDER) == {**names, ".gitignore": "*\n"}+ assert capsys.readouterr().out == f"Wrote 4 files to {FOLDER}/\n"++[email protected]("writes")+def test_a_symlink_already_in_the_folder_cannot_lead_a_file_out(project, monkeypatch, capsys):+ directory, _ = project+ outside = directory / "outside"+ outside.mkdir()+ (directory / FOLDER).mkdir()+ (directory / FOLDER / "run").symlink_to(outside, target_is_directory=True)+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run/outputs.md": "o"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert list(outside.iterdir()) == []+ assert (directory / FOLDER / "report.md").read_text() == "r"+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/ — start with report.md {SKIPPED}\n"++[email protected]("writes")+def test_a_file_where_a_folder_goes_skips_what_goes_under_it(project, monkeypatch, capsys):+ directory, _ = project+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run": "a file", "run/outputs.md": "o"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert files_in(directory / FOLDER) == {"report.md": "r", "run": "a file", ".gitignore": "*\n"}+ assert capsys.readouterr().out == f"Wrote 2 files to {FOLDER}/ — start with report.md {SKIPPED}\n"++[email protected]("writes")+def test_a_folder_where_the_gitignore_goes_stops_the_export_before_any_data(project, monkeypatch):+ directory, _ = project+ (directory / FOLDER / ".gitignore").mkdir(parents=True)+ install(monkeypatch, FakeExportAMP())++ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])++ assert result.exit_code == 1+ assert result.output == (+ f"{FOLDER}/.gitignore is a folder, not this command's file: remove it "+ "and export again, so the data is never committed by accident.\n"+ )+ assert [p.name for p in (directory / FOLDER).iterdir()] == [".gitignore"] # no data written++[email protected]("writes")+def test_a_folder_where_a_file_goes_is_skipped_and_the_export_goes_on(project, monkeypatch, capsys):+ directory, _ = project+ (directory / FOLDER / "run").mkdir(parents=True)+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run": "a file", "notes.md": "n"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert (directory / FOLDER / "run").is_dir()+ assert files_in(directory / FOLDER) == {"report.md": "r", "notes.md": "n", ".gitignore": "*\n"}+ assert capsys.readouterr().out == f"Wrote 2 files to {FOLDER}/ — start with report.md {SKIPPED}\n"+++def test_an_export_folder_that_resolves_elsewhere_stops_the_export(project, monkeypatch):+ """A Windows junction is not a link to `is_symlink`, but resolves elsewhere:+ the path writer refuses any folder that does not resolve to itself."""+ directory, _ = project+ outside = directory / "outside"+ outside.mkdir()+ monkeypatch.setattr(eval_module, "_WRITES_THROUGH_HANDLES", False)+ real_resolve = Path.resolve+ monkeypatch.setattr(+ Path, "resolve", lambda self, *a, **k: real_resolve(outside) if self.name == FOLDER else real_resolve(self, *a, **k)+ )+ install(monkeypatch, FakeExportAMP())++ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])++ assert result.exit_code == 1+ assert result.output == f"{FOLDER} is a link; the export writes only into a folder of its own.\n"+ assert list(outside.iterdir()) == []++[email protected]("link", ["symbolic", "hard"])+def test_a_link_where_a_file_goes_is_replaced_never_written_through(project, monkeypatch, capsys, writes, link):+ directory, _ = project+ elsewhere = directory / "keep.md"+ elsewhere.write_text("not ours to change\n", encoding="utf-8")+ (directory / FOLDER / "run").mkdir(parents=True)+ placed = directory / FOLDER / "run" / "outputs.md"+ if link == "symbolic":+ placed.symlink_to(elsewhere)+ else:+ os.link(elsewhere, placed)+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run/outputs.md": "o"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert elsewhere.read_text(encoding="utf-8") == "not ours to change\n"+ assert not placed.is_symlink() and placed.read_text(encoding="utf-8") == "o"+ assert capsys.readouterr().out == f"Wrote 2 files to {FOLDER}/ — start with report.md\n"++[email protected]("writes")+def test_a_gitignore_link_already_in_the_folder_is_replaced_never_followed(+ project, monkeypatch+):+ directory, _ = project+ elsewhere = directory / "keep.txt"+ elsewhere.write_text("not ours to change\n", encoding="utf-8")+ (directory / FOLDER).mkdir()+ (directory / FOLDER / ".gitignore").symlink_to(elsewhere)+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert elsewhere.read_text(encoding="utf-8") == "not ours to change\n"+ gitignore = directory / FOLDER / ".gitignore"+ assert not gitignore.is_symlink()+ assert gitignore.read_text(encoding="utf-8") == "*\n"++[email protected]("writes")[email protected]("leads_to", ["a folder", "nowhere"])+def test_an_export_folder_that_is_a_link_stops_the_export(project, monkeypatch, leads_to):+ directory, _ = project+ outside = directory / "outside"+ if leads_to == "a folder":+ outside.mkdir()+ (directory / FOLDER).symlink_to(outside, target_is_directory=True)+ install(monkeypatch, FakeExportAMP())++ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])++ assert result.exit_code == 1+ assert result.output == f"{FOLDER} is a link; the export writes only into a folder of its own.\n"+ left = {FOLDER, "outside"} if leads_to == "a folder" else {FOLDER}+ assert {p.name for p in directory.iterdir()} == left # nothing made where the link leads+ if leads_to == "a folder":+ assert list(outside.iterdir()) == []+++@handles_only+def test_a_folder_swapped_for_a_link_while_the_export_runs_is_not_gone_through(project, monkeypatch, capsys):+ """The race a check-then-write loses: `run` is a folder when the first file+ goes in, and a link out of the export by the time the second one does,+ swapped in after the name was checked and before the folder is opened."""+ directory, _ = project+ outside = directory / "outside"+ outside.mkdir()+ real_mkdir = os.mkdir+ asked: list[str] = []++ def mkdir_then_swap(path, mode=0o777, *, dir_fd=None):+ asked.append(path)+ try:+ real_mkdir(path, mode, dir_fd=dir_fd)+ finally:+ if asked.count("run") == 2:+ (directory / FOLDER / "run").rename(directory / FOLDER / "moved")+ (directory / FOLDER / "run").symlink_to(outside, target_is_directory=True)++ monkeypatch.setattr(os, "mkdir", mkdir_then_swap)+ install(monkeypatch, FakeExportAMP(exported({"run/a.md": "a", "run/b.md": "b"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert asked.count("run") == 2 # the swap happened+ assert list(outside.iterdir()) == []+ assert files_in(directory / FOLDER) == {"moved/a.md": "a", ".gitignore": "*\n"}+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/ {SKIPPED}\n"+++@handles_only+def test_the_export_folder_swapped_for_a_link_while_the_export_runs_is_not_gone_through(project, monkeypatch):+ """The folder itself is held open: moved away and replaced by a link once+ the export has begun, the files still go into it and nowhere else."""+ directory, _ = project+ outside = directory / "outside"+ outside.mkdir()+ real_mkdir = os.mkdir++ def mkdir_then_swap(path, mode=0o777, *, dir_fd=None):+ if path == "run" and not (directory / "moved").exists():+ (directory / FOLDER).rename(directory / "moved")+ (directory / FOLDER).symlink_to(outside, target_is_directory=True)+ real_mkdir(path, mode, dir_fd=dir_fd)++ monkeypatch.setattr(os, "mkdir", mkdir_then_swap)+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run/a.md": "a"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert list(outside.iterdir()) == []+ assert files_in(directory / "moved") == {"report.md": "r", "run/a.md": "a", ".gitignore": "*\n"}+++@handles_only+def test_every_folder_handle_is_closed(project, monkeypatch):+ directory, _ = project+ outside = directory / "outside"+ outside.mkdir()+ (directory / FOLDER / "a").mkdir(parents=True)+ (directory / FOLDER / "a" / "link").symlink_to(outside, target_is_directory=True)+ install(monkeypatch, FakeExportAMP(exported({"a/b/c.md": "c", "a/link/x.md": "x", "report.md": "r"})))+ open_before = len(os.listdir("/dev/fd"))++ eval_module.export_evaluation(EVALUATION_ID)++ assert len(os.listdir("/dev/fd")) == open_before+ assert files_in(directory / FOLDER) == {"a/b/c.md": "c", "report.md": "r", ".gitignore": "*\n"}+++def test_content_that_is_not_text_is_skipped(project, monkeypatch, capsys):+ directory, _ = project+ body = (+ '{"id": "%s", "kind": "trace", "files": {"report.md": "# Report", "n.json": 5, "none.md": null,'+ ' "list.md": ["a"], "object.md": {"a": 1}, "half.md": "\\ud800"}}' % EVALUATION_ID+ )+ install(monkeypatch, FakeExportAMP(httpx.Response(200, content=body.encode())))++ eval_module.export_evaluation(EVALUATION_ID)++ assert files_in(directory / FOLDER) == {"report.md": "# Report", ".gitignore": "*\n"}+ assert capsys.readouterr().out == (+ f"Wrote 1 file to {FOLDER}/ — start with report.md "+ "(skipped 5: an unsafe path, or content that is not text)\n"+ )+++def test_an_export_without_a_report_does_not_point_at_one(project, monkeypatch, capsys):+ directory, _ = project+ install(monkeypatch, FakeExportAMP(exported({"run/outputs.md": "o"})))++ eval_module.export_evaluation(EVALUATION_ID)++ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/\n"++[email protected]("given", ["not-a-uuid", "", "../../etc", f"{EVALUATION_ID}0", "[red]x[/red]"])+def test_an_id_that_is_not_a_uuid_is_refused_without_a_request(project, monkeypatch, given):+ directory, _ = project+ amp = install(monkeypatch, FakeExportAMP())++ result = CliRunner().invoke(eval_command, ["export", given])++ assert result.exit_code == 1+ assert result.output == f"{given!r} is not an evaluation id: an evaluation id is a UUID.\n"+ assert amp.calls == []+ assert list(directory.iterdir()) == []+++ ("answer", "logged_in", "said"),+ [+ (httpx.Response(404, json={"error": "not_found", "message": "Evaluation not found."}), True,+ "Evaluation not found."),+ (httpx.Response(404, json={"error": "not_found"}), True,+ f"AMP holds no evaluation {EVALUATION_ID} that this login may read."),+ (httpx.Response(404, json={"error": "not_found"}), False,+ f"AMP holds no evaluation {EVALUATION_ID} that can be read without a login; "+ "if it is your account's, log in with `crewai login` and export it again."),+ # an AMP from before exports has no route: its 404 is not AMP's `not_found`+ (httpx.Response(404), True, NO_EXPORT_ROUTE),+ (httpx.Response(404), False, NO_EXPORT_ROUTE),+ (httpx.Response(404, json={"status": 404, "error": "Not Found"}), True, NO_EXPORT_ROUTE),+ (httpx.Response(404, content=b"<html>The page you were looking for doesn't exist.</html>"), True,+ NO_EXPORT_ROUTE),+ # an AMP whose evaluation service predates exports+ (httpx.Response(501, json={"error": "export_unsupported",+ "message": "This AMP's evaluation service cannot export yet; update it."}), True,+ "This AMP's evaluation service cannot export yet; update it."),+ (httpx.Response(501, json={"error": "export_unsupported"}), True,+ "This AMP's evaluation service does not export evaluations yet: it needs an update."),+ # an export over 64 MB is refused whole, by the evaluation service (413) or by AMP (502)+ (httpx.Response(413, json={"error": "export_too_large",+ "message": "This export is 80 MB; an export stops at 64 MB."}), True,+ "This export is 80 MB; an export stops at 64 MB."),+ (httpx.Response(502, json={"error": "export_too_large",+ "message": "The export is over 64 MB, more than AMP carries."}), False,+ "The export is over 64 MB, more than AMP carries."),+ (httpx.Response(413, json={"error": "export_too_large"}), True,+ f"Evaluation {EVALUATION_ID} is too large to export."),+ (httpx.Response(502, json={"error": "export_too_large"}), True,+ f"Evaluation {EVALUATION_ID} is too large to export."),+ (httpx.Response(502, json={"error": "bad_gateway", "message": "Upstream down"}), True,+ "AMP answered 502: Upstream down."),+ (httpx.Response(404, json={"error": "not_found", "message": "Evaluation not found."}), False,+ "Evaluation not found; if it is your account's, log in with `crewai login` and export it again."),+ (httpx.Response(409, json={"error": "not_finished", "message": "[b]Still[/b] grading."}), True,+ "[b]Still[/b] grading."),+ (httpx.Response(409), True,+ f"Evaluation {EVALUATION_ID} is still running; export it once it has finished."),+ (httpx.Response(410, json={"error": "report_closed", "message": "This report was opened and is closed."}), False,+ "This report was opened and is closed."),+ (httpx.Response(410), False,+ f"The report of evaluation {EVALUATION_ID} was opened already and is closed; "+ "log in with `crewai login` and evaluate the run again."),+ (httpx.Response(503, json={"error": "service_unavailable", "message": "Back soon"}), True,+ "AMP answered 503: Back soon."),+ (httpx.Response(429, json={"message": "Slow down"}, headers={"Retry-After": "30"}), True,+ "Slow down — retry after 30s."),+ (httpx.Response(401, json={"error": "bad_credentials", "message": "Token expired"}), True,+ "Token expired. Log in with `crewai login` and try again."),+ (httpx.ConnectError("connection refused"), True,+ f"Could not reach AMP to export evaluation {EVALUATION_ID}: connection refused"),+ (httpx.Response(200, json={"id": EVALUATION_ID, "kind": "trace"}), True,+ f"AMP answered without the files of evaluation {EVALUATION_ID} (protocol error)."),+ (httpx.Response(200, content=b"<html>"), True,+ f"AMP answered without the files of evaluation {EVALUATION_ID} (protocol error)."),+ ],+)+def test_a_refused_export_says_why_in_one_sentence_and_exits_1(project, monkeypatch, answer, logged_in, said):+ directory, _ = project+ if not logged_in:+ monkeypatch.setattr(eval_module, "saved_login", lambda: None)+ amp = install(monkeypatch, FakeExportAMP(answer))++ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])++ assert result.exit_code == 1+ assert result.output == f"{said}\n" # AMP's words printed as they are, brackets and all+ assert amp.calls == [("export", EVALUATION_ID)]+ assert (amp.api_key is not None) is logged_in+ if not logged_in:+ assert "X-Crewai-Organization-Id" not in amp.headers+ assert list(directory.iterdir()) == []+++def test_the_written_line_stays_one_line_in_a_narrow_pipe(project, monkeypatch, capsys):+ install(monkeypatch, FakeExportAMP())+ monkeypatch.setattr(eval_module, "console", Console(width=40))++ eval_module.export_evaluation(EVALUATION_ID)++ assert capsys.readouterr().out == f"Wrote 5 files to {FOLDER}/ — start with report.md\n"diff --git a/lib/cli/tests/test_plus_api.py b/lib/cli/tests/test_plus_api.pyindex 15abb8ba2..b797f29b5 100644--- a/lib/cli/tests/test_plus_api.py+++ b/lib/cli/tests/test_plus_api.py@@ -101,7 +101,24 @@ class TestPlusAPI(unittest.TestCase):response = self.api.get_evaluation("ev-1")mock_make_request.assert_called_once_with(- "GET", "/crewai_plus/api/v1/tracing/evaluations/ev-1", timeout=30.0+ "GET",+ "/crewai_plus/api/v1/tracing/evaluations/ev-1",+ params={"can_export": "1"},+ timeout=30.0,+ )+ self.assertEqual(response, mock_response)++ @patch("crewai_core.plus_api.PlusAPI._make_request")+ def test_export_evaluation(self, mock_make_request):+ mock_response = MagicMock()+ mock_make_request.return_value = mock_response++ response = self.api.export_evaluation("ev-1")++ mock_make_request.assert_called_once_with(+ "GET",+ "/crewai_plus/api/v1/tracing/evaluations/ev-1/export",+ timeout=httpx.Timeout(30.0, read=120.0),)self.assertEqual(response, mock_response)@@ -569,3 +586,28 @@ def test_requests_without_their_own_timeout_keep_the_httpx_default():assert request.url.path == "/crewai_plus/api/v1/agents/an_agent"assert request.extensions["timeout"]["read"] == 5.0+++def test_an_export_waits_two_minutes_for_its_large_answer():+ """The export carries every setup's outputs and traces, tens of MB: the+ httpx default of 5 s would cut it off."""+ (request,) = _requests_sent_by(lambda api: api.export_evaluation("ev-1"))++ assert request.method == "GET"+ assert request.url.path == "/crewai_plus/api/v1/tracing/evaluations/ev-1/export"+ assert request.extensions["timeout"] == {+ "connect": 30.0,+ "read": 120.0,+ "write": 30.0,+ "pool": 30.0,+ }+++def test_reading_an_evaluation_says_this_cli_can_export_it():+ """`can_export=1` lets the brief name `crewai eval export`: an AMP that does+ not know the parameter drops it, and the brief stays silent about it."""+ (request,) = _requests_sent_by(lambda api: api.get_evaluation("ev-1"))++ assert request.method == "GET"+ assert request.url.path == "/crewai_plus/api/v1/tracing/evaluations/ev-1"+ assert dict(request.url.params) == {"can_export": "1"}