Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(cli): crewai eval export downloads an evaluation as files (#8187)

João Moura1410318b2ed0
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.py
index f0765e4dc..d4a555b2f 100644
--- a/lib/cli/src/crewai_cli/cli.py
+++ b/lib/cli/src/crewai_cli/cli.py
@@ -60,6 +60,12 @@ def eval_models(*args: Any, **kwargs: Any) -> Any:
return _eval_models(*args, **kwargs)
+def eval_export(*args: Any, **kwargs: Any) -> Any:
+ from crewai_cli.experimental.eval_crew import export_evaluation as _export
+
+ return _export(*args, **kwargs)
+
+
if TYPE_CHECKING:
# mypy sees the real classes; at runtime the shims below defer the
# heavy imports until a command actually instantiates them.
@@ -690,7 +696,7 @@ def run(
)
[email protected](name="eval")
[email protected](name="eval", invoke_without_command=True)
@click.option(
"--run",
"run_id",
@@ -728,8 +734,12 @@ def run(
"project id."
),
)
def eval_command(
- run_id: str | None, models: str | None, deployment_id: str | None
+ ctx: click.Context,
+ run_id: str | None,
+ models: str | None,
+ deployment_id: str | None,
) -> None:
"""Evaluate the last traced run through CrewAI AMP, or compare models on the
project's deployment (--models).
@@ -744,7 +754,19 @@ def eval_command(
failed gate, no verdict, or an evaluation that could not run — so a CI job can
gate on it. A comparison exits 0 when it finished, 1 when it failed or could
not start.
+
+ `crewai eval export ID` downloads an evaluation's full data as files, for a
+ coding agent to dig into.
"""
+ if ctx.invoked_subcommand is not None:
+ # The subcommand does the work; an option meant for this body would
+ # otherwise be dropped without a word.
+ if run_id is not None or models is not None or deployment_id is not None:
+ raise click.UsageError(
+ f"`crewai eval {ctx.invoked_subcommand}` takes no --run, --models "
+ "or --deployment; give those to `crewai eval` alone."
+ )
+ return
if models is not None:
if run_id is not None:
raise click.UsageError(
@@ -760,6 +782,23 @@ def eval_command(
eval_crew(run_id=run_id)
+@eval_command.command(name="export")
[email protected]("evaluation_id", metavar="ID")
+def eval_export_command(evaluation_id: str) -> None:
+ """Download evaluation ID's full data into ./crewai-eval-ID/.
+
+ The report, every setup's outputs, every check with the judge's words, and
+ the traces, as files for a coding agent to dig into: start with report.md.
+ The folder gets a .gitignore of `*`, so the data is never committed by
+ accident; exporting again overwrites the files.
+
+ ID is the evaluation's UUID. It is read with your `crewai login` when there
+ is one; a run's evaluation can be exported without one. Exits 0 when the
+ files were written, 1 otherwise.
+ """
+ eval_export(evaluation_id)
+
+
@crewai.command()
def update() -> None:
"""Update the pyproject.toml of the Crew project to use uv."""
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 993212cfd..847b1ad7d 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -17,20 +17,27 @@ deployed and once per model; each run is graded, and the comparison — the four
grades, cost and time per model, and what would make the agents do better — is
printed when it is done. It always needs the login: it runs the deployment.
Every evaluation, of either kind, is filed under the project id.
+
+`crewai eval export ID` downloads an evaluation's full data — the report,
+every setup's outputs, every check with the judge's words, and the traces —
+into `./crewai-eval-<ID>/`, for a coding agent to dig into.
"""
from __future__ import annotations
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
import contextlib
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
+import errno
from ipaddress import ip_address
import json
import math
import os
from pathlib import Path
+import re
+import stat
import sys
import time
from typing import Any, Literal, NoReturn, cast
@@ -1694,3 +1701,360 @@ def _fail(message: str) -> NoReturn:
# comes from `style`.
console.print(Text(message), style="bold red")
raise SystemExit(1)
+
+
+# ── crewai eval export ───────────────────────────────────────────────────────
+
+EXPORT_DIR_PREFIX = "crewai-eval-"
+EXPORT_START_HERE = "report.md"
+# One path segment of an exported file: plain names only. Everything else —
+# a separator other than `/`, a drive, a space, a control character — is not
+# a path this command writes.
+_EXPORT_SEGMENT = re.compile(r"[A-Za-z0-9._-]+")
+# The names Windows keeps for its devices, whatever the extension: there a
+# file called `NUL.md` is the null device, and what is written to it is gone.
+_WINDOWS_DEVICE_NAMES = frozenset(
+ {"con", "prn", "aux", "nul"}
+ | {f"{port}{n}" for port in ("com", "lpt") for n in range(10)}
+)
+# Linux and macOS open a file inside an open folder, and can refuse to go
+# through a link while doing it, so the export is written that way: no path
+# is checked first and opened again later, when another process may have put
+# a link in its way (`_put_through_handles`). Windows cannot; there each path
+# is resolved and checked to stay inside the folder before it is written
+# (`_put_by_path`). That check can be raced: a process that can write inside
+# the export folder could swap a junction in between, and any account can make
+# one. The bounded loss, accepted: such a process already writes where this
+# user does (the folder is in the user's own working directory), unless the
+# CLI runs elevated in a folder a less trusted account may write to.
+_WRITES_THROUGH_HANDLES = (
+ hasattr(os, "O_NOFOLLOW")
+ and hasattr(os, "O_DIRECTORY")
+ and os.open in os.supports_dir_fd
+ and os.mkdir in os.supports_dir_fd
+ and os.unlink in os.supports_dir_fd
+ and os.stat in os.supports_dir_fd
+ and os.stat in os.supports_follow_symlinks
+)
+_FOLDER_FLAGS = (
+ os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+)
+_NEW_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+
+
+def export_evaluation(evaluation_id: str) -> None:
+ """Write evaluation EVALUATION_ID's full data into `./crewai-eval-<id>/`.
+
+ The files are AMP's: the report, every setup's outputs and checks with the
+ judge's words, and the traces. They are written as sent, one file per
+ path, beside a `.gitignore` of `*` — this is a customer's data, and a
+ folder dropped into a repository must not be committed by accident.
+ Re-running overwrites them.
+ """
+ try:
+ canonical = str(uuid.UUID(evaluation_id.strip()))
+ except ValueError:
+ _fail(f"{evaluation_id!r} is not an evaluation id: an evaluation id is a UUID.")
+ # Read before the project's .env is loaded, so a project cannot add itself.
+ trusted = _trusted_amp_origins()
+ _load_project_env()
+ try:
+ client = _amp_client(trusted)
+ except EvaluationStoppedError as stopped:
+ _fail(str(stopped))
+ try:
+ response = client.export_evaluation(canonical)
+ except httpx.HTTPError as error:
+ _fail(f"Could not reach AMP to export evaluation {canonical}: {error}")
+ if response.status_code != 200:
+ _fail(
+ _export_refusal(
+ response,
+ canonical,
+ logged_in=client.api_key is not None,
+ amp=client.base_url,
+ )
+ )
+ files = (_payload(response) or {}).get("files")
+ if not isinstance(files, dict):
+ _fail(
+ f"AMP answered without the files of evaluation {canonical} (protocol error)."
+ )
+
+ folder = Path(f"{EXPORT_DIR_PREFIX}{canonical}")
+ written, skipped = _write_export(folder, files)
+ line = f"Wrote {len(written)} file{'' if len(written) == 1 else 's'} to {folder}/"
+ if EXPORT_START_HERE in written:
+ line += f" — start with {EXPORT_START_HERE}"
+ if skipped:
+ line += f" (skipped {skipped}: an unsafe path, or content that is not text)"
+ # One line whatever the width: an agent reads the folder off it, and a
+ # pipe is 80 columns to Rich.
+ console.print(Text(line), soft_wrap=True)
+
+
+def _export_refusal(
+ response: httpx.Response, evaluation_id: str, *, logged_in: bool, amp: str
+) -> str:
+ """AMP's refusal of an export as one sentence, in AMP's own words when it sent them.
+
+ A self-hosted install updates AMP, its evaluation service and this CLI
+ apart, so a part older than exports says which one needs the update.
+ """
+ payload = _payload(response) or {}
+ message = str(payload.get("message") or "").strip()
+ error = str(payload.get("error") or "")
+ status = response.status_code
+ # AMP's 501: its evaluation service predates exports.
+ if error == "export_unsupported":
+ return message or (
+ "This AMP's evaluation service does not export evaluations yet: "
+ "it needs an update."
+ )
+ # AMP's 502 or the evaluation service's 413: an export too large to carry
+ # is refused whole, never cut short.
+ if error == "export_too_large":
+ return message or f"Evaluation {evaluation_id} is too large to export."
+ # AMP says `not_found` of an evaluation it does not hold or this login may
+ # not read; a 404 without it is an AMP with no export route at all.
+ if status == 404 and error != "not_found":
+ return (
+ f"The AMP at {amp.rstrip('/')} does not serve `crewai eval export` "
+ "yet: it needs an update."
+ )
+ if status == 404:
+ if logged_in:
+ return message or (
+ f"AMP holds no evaluation {evaluation_id} that this login may read."
+ )
+ found = message or (
+ f"AMP holds no evaluation {evaluation_id} that can be read without a login"
+ )
+ if "crewai login" in found:
+ return found
+ return (
+ f"{found.rstrip('.')}; if it is your account's, log in with "
+ "`crewai login` and export it again."
+ )
+ if status == 409:
+ return message or (
+ f"Evaluation {evaluation_id} is still running; export it once it has finished."
+ )
+ if status == 410:
+ return message or (
+ f"The report of evaluation {evaluation_id} was opened already and is "
+ "closed; log in with `crewai login` and evaluate the run again."
+ )
+ # 401 / 403 / 429 / 5xx read as every other evaluation call reads them.
+ return _refusal_message(response, f"evaluation {evaluation_id}")
+
+
+def _export_target(name: str) -> list[str] | None:
+ """The path segments the exported file NAME is written at, or None when it may not be.
+
+ The paths come from whichever AMP answered — which a project's `.env` may
+ choose — so a path is written only when it is relative, `/`-separated, and
+ every segment a plain name (`[A-Za-z0-9._-]+`, never `.` or `..`). A `.git`
+ segment is refused too: a `.git/config` there could name a command for the
+ next `git status` in the folder to run, and a shell prompt runs one. The
+ top `.gitignore` is this command's own. So is a name Windows keeps for a
+ device (`CON`, `PRN`, `AUX`, `NUL`, `COM0`-`COM9`, `LPT0`-`LPT9`, with any
+ extension), on every platform, since the folder may be opened on Windows
+ later or sit on a drive it shares: there `NUL.md` is the null device, and
+ the export would report a file written that nobody can read. All are
+ compared as a case-insensitive filesystem that drops trailing dots would
+ see them. Where the segments lead, through a link or not, is the writer's
+ to check (`_export_folder`).
+ """
+ segments = name.split("/")
+ for segment in segments:
+ seen = _as_a_filesystem_sees_it(segment)
+ if (
+ not _EXPORT_SEGMENT.fullmatch(segment)
+ or segment in {".", ".."}
+ or seen == ".git"
+ or seen.split(".", 1)[0] in _WINDOWS_DEVICE_NAMES
+ ):
+ return None
+ if len(segments) == 1 and _as_a_filesystem_sees_it(segments[0]) == ".gitignore":
+ return None
+ return segments
+
+
+def _as_a_filesystem_sees_it(segment: str) -> str:
+ """SEGMENT as macOS and Windows compare names: case ignored, trailing dots dropped."""
+ return segment.rstrip(".").lower()
+
+
+def _write_export(folder: Path, files: dict[str, Any]) -> tuple[list[str], int]:
+ """Write FILES under FOLDER: the names written, and how many were skipped.
+
+ The `.gitignore` goes first, so the data is never in the folder without it.
+ A file whose path would go through a link, or through a file where a
+ folder belongs, is skipped like any other unsafe path. A file that cannot
+ be written stops the export with the reason: half an export that says it
+ is whole would mislead the agent reading it.
+ """
+ written: list[str] = []
+ skipped = 0
+ with _export_folder(folder) as put:
+ try:
+ ignored = put([".gitignore"], b"*\n")
+ except OSError as error:
+ _fail(f"Could not write the export into {folder}/: {error}")
+ if not ignored:
+ # The data never goes into the folder without it.
+ _fail(
+ f"{folder}/.gitignore is a folder, not this command's file: remove it "
+ "and export again, so the data is never committed by accident."
+ )
+ for name, content in files.items():
+ segments = _export_target(name)
+ if segments is None or not isinstance(content, str):
+ skipped += 1
+ continue
+ try:
+ data = content.encode("utf-8")
+ except UnicodeEncodeError: # a lone surrogate JSON allowed through
+ skipped += 1
+ continue
+ try:
+ put_there = put(segments, data)
+ except OSError as error:
+ _fail(f"Could not write {folder}/{name}: {error}")
+ if put_there:
+ written.append(name)
+ else:
+ skipped += 1
+ return written, skipped
+
+
+def _export_folder(folder: Path) -> Iterator[Callable[[list[str], bytes], bool]]:
+ """FOLDER, made when missing, as a function that writes one file into it.
+
+ The function takes the file's path segments and its bytes, and answers
+ False, with nothing written, when a link or a file stands where one of the
+ path's folders goes. FOLDER, a name in the current directory, being a link
+ itself stops the export: the export writes only into a folder of its own.
+ """
+ if not _WRITES_THROUGH_HANDLES:
+ root = _export_root_by_path(folder)
+ yield lambda segments, data: _put_by_path(root, segments, data)
+ return
+ handle = _export_root_handle(folder)
+ try:
+ yield lambda segments, data: _put_through_handles(handle, segments, data)
+ finally:
+ os.close(handle)
+
+
+def _export_root_is_a_link(folder: Path) -> NoReturn:
+ """Stop: whoever made FOLDER a link chose where it leads, not this command."""
+ _fail(f"{folder} is a link; the export writes only into a folder of its own.")
+
+
+def _export_root_handle(folder: Path) -> int:
+ """A handle on FOLDER, made when missing, opened inside the current
+ directory and never through a link."""
+ try:
+ here = os.open(".", _FOLDER_FLAGS)
+ try:
+ with contextlib.suppress(FileExistsError):
+ os.mkdir(folder.name, dir_fd=here)
+ return os.open(folder.name, _FOLDER_FLAGS, dir_fd=here)
+ finally:
+ os.close(here)
+ except OSError as error:
+ if folder.is_symlink():
+ _export_root_is_a_link(folder)
+ _fail(f"Could not write the export into {folder}/: {error}")
+
+
+def _put_through_handles(root: int, segments: list[str], data: bytes) -> bool:
+ """Write DATA at SEGMENTS under the folder ROOT is a handle on, through no link.
+
+ Each folder of the path is opened inside the one before it, and the file
+ inside the last, so no path is checked first and opened again later: a
+ link where a folder goes, left there or swapped in while the export runs,
+ is never gone through, and the file is not written (False). The file's own
+ place is emptied first and the file made anew, so a link there, symbolic
+ or hard, is replaced rather than written through.
+ """
+ folder = root
+ try:
+ for segment in segments[:-1]:
+ with contextlib.suppress(FileExistsError):
+ os.mkdir(segment, dir_fd=folder)
+ try:
+ inner = os.open(segment, _FOLDER_FLAGS, dir_fd=folder)
+ except OSError as error:
+ if error.errno in (errno.ELOOP, errno.ENOTDIR):
+ return False # a link, or a file, where a folder goes
+ raise
+ if folder != root:
+ os.close(folder)
+ folder = inner
+ leaf = segments[-1]
+ try:
+ if stat.S_ISDIR(
+ os.stat(leaf, dir_fd=folder, follow_symlinks=False).st_mode
+ ):
+ return False # a folder where a file goes
+ os.unlink(leaf, dir_fd=folder)
+ except FileNotFoundError:
+ pass # nothing at the file's place yet: it is simply made
+ descriptor = os.open(leaf, _NEW_FILE_FLAGS, 0o666, dir_fd=folder)
+ try:
+ out = open(descriptor, "wb")
+ except BaseException:
+ os.close(descriptor)
+ raise
+ with out:
+ out.write(data)
+ return True
+ finally:
+ if folder != root:
+ os.close(folder)
+
+
+def _export_root_by_path(folder: Path) -> Path:
+ """FOLDER, made when missing, as the absolute path the files go under.
+
+ It must resolve to itself, here: a link, or a Windows junction (which
+ `is_symlink` does not see, and which any account can make), would carry
+ every file somewhere else.
+ """
+ if folder.is_symlink():
+ _export_root_is_a_link(folder)
+ try:
+ folder.mkdir(exist_ok=True)
+ root = folder.resolve()
+ except OSError as error:
+ _fail(f"Could not write the export into {folder}/: {error}")
+ if os.path.normcase(root) != os.path.normcase(Path.cwd().resolve() / folder.name):
+ _export_root_is_a_link(folder)
+ return root
+
+
+def _put_by_path(root: Path, segments: list[str], data: bytes) -> bool:
+ """Write DATA at SEGMENTS under ROOT, where a folder cannot be opened
+ without following a link: the path is resolved first, and written only
+ when its folder is inside ROOT (False otherwise, or when a file stands
+ where a folder goes, or a folder where the file goes). Whatever is at the
+ file's own place, a link symbolic or hard, is removed and the file made
+ anew, so nothing is written through it."""
+ parent = root.joinpath(*segments[:-1]).resolve()
+ if not parent.is_relative_to(root):
+ return False
+ try:
+ parent.mkdir(parents=True, exist_ok=True)
+ except (FileExistsError, NotADirectoryError):
+ return False
+ target = parent / segments[-1]
+ if target.is_dir() and not target.is_symlink():
+ return False # a folder, or a junction, where a file goes
+ target.unlink(missing_ok=True)
+ with open(target, "xb") as out:
+ out.write(data)
+ return True
diff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.py
index 9632e8a8f..9f099e4b4 100644
--- a/lib/cli/src/crewai_cli/plus_api.py
+++ b/lib/cli/src/crewai_cli/plus_api.py
@@ -26,6 +26,9 @@ class PlusAPI(_CorePlusAPI):
# AMP reads the run's spans from Wharf inside the POST; a large run takes a while.
EVALUATION_START_TIMEOUT = 120.0
EVALUATION_POLL_TIMEOUT = 30.0
+ # The export carries every setup's outputs, checks and traces — tens of MB
+ # for a large comparison — and AMP gathers them before it answers.
+ EVALUATION_EXPORT_TIMEOUT = httpx.Timeout(EVALUATION_POLL_TIMEOUT, read=120.0)
def create_evaluation(
self,
@@ -86,13 +89,29 @@ class PlusAPI(_CorePlusAPI):
)
def get_evaluation(self, evaluation_id: str) -> httpx.Response:
- """The evaluation's status and, once done, its verdict."""
+ """The evaluation's status and, once done, its verdict.
+
+ `can_export=1` says this CLI has `crewai eval export`, so the brief may
+ point at it. AMP passes it on to the evaluation service; an AMP from
+ before exports drops it, so the brief names the command only when the
+ CLI, AMP and the service all serve it.
+ """
return self._make_request(
"GET",
f"{self.EVALUATIONS_RESOURCE}/{evaluation_id}",
+ params={"can_export": "1"},
timeout=self.EVALUATION_POLL_TIMEOUT,
)
+ def export_evaluation(self, evaluation_id: str) -> httpx.Response:
+ """The evaluation's full data as files (crewai eval export):
+ `{"id", "kind", "files": {relative path: text}}`."""
+ return self._make_request(
+ "GET",
+ f"{self.EVALUATIONS_RESOURCE}/{evaluation_id}/export",
+ timeout=self.EVALUATION_EXPORT_TIMEOUT,
+ )
+
def _make_multipart_request(
self,
method: HttpMethod,
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index 8a511b11d..37a8ac803 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -2348,3 +2348,544 @@ def test_a_failing_usage_count_never_changes_what_the_command_does(project, monk
assert exited.value.code == 1
assert eval_module._refusal_message(refusal, f"run {EXECUTION_ID}") in capsys.readouterr().out
+
+
+# --- crewai eval export ------------------------------------------------------
+
+EVALUATION_ID = "0b6c1f5e-8d2a-4f3b-9c4e-7a1d2e3f4a5b"
+FOLDER = f"crewai-eval-{EVALUATION_ID}"
+EXPORT_FILES = {
+ "report.md": "# Report\n\nThe gate passed.\n",
+ "run/outputs.md": "## Final output\n\nThree trends.\n",
+ "run/checks.md": "- goal: passed — the output names three trends\n",
+ "run/trace.json": '{"spans": []}\n',
+ "gemini-4-argon/outputs.md": "## Final output\n\nTwo trends.\n",
+}
+SKIPPED = "(skipped 1: an unsafe path, or content that is not text)"
+
+
+class FakeExportAMP(FakeAMP):
+ def __init__(self, answer=None):
+ super().__init__()
+ self.answer = answer if answer is not None else exported()
+
+ def export_evaluation(self, evaluation_id):
+ self.calls.append(("export", evaluation_id))
+ if isinstance(self.answer, Exception):
+ raise self.answer
+ return self.answer
+
+
+def exported(files=None, kind="models"):
+ return httpx.Response(
+ 200,
+ json={"id": EVALUATION_ID, "kind": kind, "files": EXPORT_FILES if files is None else files},
+ )
+
+
+def files_in(folder: Path) -> dict[str, str]:
+ return {
+ p.relative_to(folder).as_posix(): p.read_text(encoding="utf-8")
+ for p in folder.rglob("*")
+ if p.is_file()
+ }
+
+
+NO_EXPORT_ROUTE = "The AMP at https://amp.test does not serve `crewai eval export` yet: it needs an update."
+handles_only = pytest.mark.skipif(
+ not eval_module._WRITES_THROUGH_HANDLES,
+ reason="this platform cannot open a file inside a folder handle without following a link",
+)
+
+
[email protected](params=["handles", "paths"])
+def writes(request, monkeypatch):
+ """Both ways the export writes: through folder handles (Linux, macOS), and
+ by a checked path where those are missing (Windows)."""
+ if request.param == "handles" and not eval_module._WRITES_THROUGH_HANDLES:
+ pytest.skip("this platform cannot open a file inside a folder handle without following a link")
+ monkeypatch.setattr(eval_module, "_WRITES_THROUGH_HANDLES", request.param == "handles")
+ return request.param
+
+
+def test_the_eval_group_keeps_every_old_invocation_and_adds_export(monkeypatch):
+ from crewai_cli.cli import crewai
+
+ calls = []
+ monkeypatch.setattr("crewai_cli.cli.eval_crew", lambda **kwargs: calls.append(("run", kwargs)))
+ monkeypatch.setattr("crewai_cli.cli.eval_models", lambda *args, **kwargs: calls.append(("models", args, kwargs)))
+ monkeypatch.setattr("crewai_cli.cli.eval_export", lambda *args: calls.append(("export", args)))
+ runner = CliRunner()
+
+ assert runner.invoke(crewai, ["eval"]).exit_code == 0
+ assert runner.invoke(crewai, ["eval", "--run", EXECUTION_ID]).exit_code == 0
+ assert runner.invoke(crewai, ["eval", "--models", MODELS, "--deployment", DEPLOYMENT]).exit_code == 0
+ assert runner.invoke(crewai, ["eval", "export", EVALUATION_ID]).exit_code == 0
+ assert calls == [
+ ("run", {"run_id": None}),
+ ("run", {"run_id": EXECUTION_ID}),
+ ("models", (MODELS,), {"deployment_id": DEPLOYMENT}),
+ ("export", (EVALUATION_ID,)),
+ ]
+
+ # an option meant for `crewai eval` itself is refused, never dropped
+ mixed = runner.invoke(eval_command, ["--run", EXECUTION_ID, "export", EVALUATION_ID])
+ assert mixed.exit_code == 2 and "takes no --run, --models or --deployment" in mixed.output
+ assert runner.invoke(eval_command, ["export"]).exit_code == 2 # the id is required
+ assert len(calls) == 4
+
+ help_text = " ".join(runner.invoke(eval_command, ["--help"]).output.split())
+ assert "`crewai eval export ID` downloads an evaluation's full data as files" in help_text
+ assert "export Download evaluation ID's full data into ./crewai-eval-ID/." in help_text
+
+
+def test_the_eval_group_keeps_every_old_exit_code(monkeypatch):
+ """A CI job gates on `crewai eval`: the command's own exit code still comes
+ out of the group, and what was a usage error still is one."""
+ from crewai_cli.cli import crewai
+
+ def exits_1(*args, **kwargs):
+ raise SystemExit(1)
+
+ monkeypatch.setattr("crewai_cli.cli.eval_crew", exits_1)
+ monkeypatch.setattr("crewai_cli.cli.eval_models", exits_1)
+ monkeypatch.setattr("crewai_cli.cli.eval_export", exits_1)
+ runner = CliRunner()
+
+ for args in (
+ ["eval"],
+ ["eval", "--run", EXECUTION_ID],
+ ["eval", "--models", MODELS],
+ ["eval", "--models", MODELS, "--deployment", DEPLOYMENT],
+ ["eval", "export", EVALUATION_ID],
+ ):
+ assert runner.invoke(crewai, args).exit_code == 1, args
+ for args in (
+ ["eval", "--models", MODELS, "--run", EXECUTION_ID],
+ ["eval", "--deployment", DEPLOYMENT],
+ ["eval", "an-argument"],
+ ):
+ assert runner.invoke(crewai, args).exit_code == 2, args
+
+
+def test_export_writes_every_file_and_a_gitignore_and_exits_0(project, monkeypatch):
+ directory, opened = project
+ amp = install(monkeypatch, FakeExportAMP())
+
+ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])
+
+ assert result.exit_code == 0, result.output
+ assert amp.calls == [("export", EVALUATION_ID)]
+ assert amp.api_key == "login-token"
+ assert files_in(directory / FOLDER) == {**EXPORT_FILES, ".gitignore": "*\n"}
+ assert result.output == f"Wrote 5 files to {FOLDER}/ — start with report.md\n"
+ assert opened == []
+
+
+def test_the_id_is_sent_and_named_in_its_canonical_form(project, monkeypatch, capsys):
+ directory, _ = project
+ amp = install(monkeypatch, FakeExportAMP(exported({"report.md": "r"})))
+
+ eval_module.export_evaluation(f" {EVALUATION_ID.upper()} ")
+
+ assert amp.calls == [("export", EVALUATION_ID)]
+ assert files_in(directory / FOLDER) == {"report.md": "r", ".gitignore": "*\n"}
+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/ — start with report.md\n"
+
+
+def test_exporting_again_overwrites_the_files(project, monkeypatch, capsys):
+ directory, _ = project
+ install(monkeypatch, FakeExportAMP(exported({"report.md": "first", "run/checks.md": "old"})))
+ eval_module.export_evaluation(EVALUATION_ID)
+ install(monkeypatch, FakeExportAMP(exported({"report.md": "second", "run/checks.md": "new"})))
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert files_in(directory / FOLDER) == {"report.md": "second", "run/checks.md": "new", ".gitignore": "*\n"}
+
+
+ "name",
+ [
+ "../x.md",
+ "{tmp}/outside.md", # absolute
+ "a/../../b.md",
+ "run/../../b.md",
+ "a\\b.md",
+ "..\\x.md",
+ "a//b.md", # empty segment
+ "run/",
+ "",
+ "./report.md",
+ "run/./checks.md",
+ "C:/x.md",
+ "run/a b.md",
+ "run/x\n.md",
+ ".git/config",
+ ".Git./config",
+ ".gitignore", # the folder's own
+ ".GITIGNORE",
+ # names Windows keeps for its devices, with any extension
+ "NUL.md",
+ "nul",
+ "com1.txt",
+ "Aux./x.md",
+ "run/lpt9.tar.gz",
+ "CON",
+ ],
+)
+def test_an_unsafe_path_is_skipped_and_nothing_is_written_outside(project, monkeypatch, capsys, name):
+ directory, _ = project
+ name = name.replace("{tmp}", str(directory))
+ install(monkeypatch, FakeExportAMP(exported({name: "!*\nnot yours", "report.md": "# Report\n"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert [p.name for p in directory.iterdir()] == [FOLDER]
+ assert files_in(directory / FOLDER) == {"report.md": "# Report\n", ".gitignore": "*\n"}
+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/ — start with report.md {SKIPPED}\n"
+
+
+def test_a_name_that_only_starts_like_a_windows_device_is_written(project, monkeypatch, capsys):
+ directory, _ = project
+ names = {"console.md": "c", "nullable.md": "n", "com10.md": "10", "auxiliary/x.md": "x"}
+ install(monkeypatch, FakeExportAMP(exported(names)))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert files_in(directory / FOLDER) == {**names, ".gitignore": "*\n"}
+ assert capsys.readouterr().out == f"Wrote 4 files to {FOLDER}/\n"
+
+
+def test_a_symlink_already_in_the_folder_cannot_lead_a_file_out(project, monkeypatch, capsys):
+ directory, _ = project
+ outside = directory / "outside"
+ outside.mkdir()
+ (directory / FOLDER).mkdir()
+ (directory / FOLDER / "run").symlink_to(outside, target_is_directory=True)
+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run/outputs.md": "o"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert list(outside.iterdir()) == []
+ assert (directory / FOLDER / "report.md").read_text() == "r"
+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/ — start with report.md {SKIPPED}\n"
+
+
+def test_a_file_where_a_folder_goes_skips_what_goes_under_it(project, monkeypatch, capsys):
+ directory, _ = project
+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run": "a file", "run/outputs.md": "o"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert files_in(directory / FOLDER) == {"report.md": "r", "run": "a file", ".gitignore": "*\n"}
+ assert capsys.readouterr().out == f"Wrote 2 files to {FOLDER}/ — start with report.md {SKIPPED}\n"
+
+
+def test_a_folder_where_the_gitignore_goes_stops_the_export_before_any_data(project, monkeypatch):
+ directory, _ = project
+ (directory / FOLDER / ".gitignore").mkdir(parents=True)
+ install(monkeypatch, FakeExportAMP())
+
+ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])
+
+ assert result.exit_code == 1
+ assert result.output == (
+ f"{FOLDER}/.gitignore is a folder, not this command's file: remove it "
+ "and export again, so the data is never committed by accident.\n"
+ )
+ assert [p.name for p in (directory / FOLDER).iterdir()] == [".gitignore"] # no data written
+
+
+def test_a_folder_where_a_file_goes_is_skipped_and_the_export_goes_on(project, monkeypatch, capsys):
+ directory, _ = project
+ (directory / FOLDER / "run").mkdir(parents=True)
+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run": "a file", "notes.md": "n"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert (directory / FOLDER / "run").is_dir()
+ assert files_in(directory / FOLDER) == {"report.md": "r", "notes.md": "n", ".gitignore": "*\n"}
+ assert capsys.readouterr().out == f"Wrote 2 files to {FOLDER}/ — start with report.md {SKIPPED}\n"
+
+
+def test_an_export_folder_that_resolves_elsewhere_stops_the_export(project, monkeypatch):
+ """A Windows junction is not a link to `is_symlink`, but resolves elsewhere:
+ the path writer refuses any folder that does not resolve to itself."""
+ directory, _ = project
+ outside = directory / "outside"
+ outside.mkdir()
+ monkeypatch.setattr(eval_module, "_WRITES_THROUGH_HANDLES", False)
+ real_resolve = Path.resolve
+ monkeypatch.setattr(
+ Path, "resolve", lambda self, *a, **k: real_resolve(outside) if self.name == FOLDER else real_resolve(self, *a, **k)
+ )
+ install(monkeypatch, FakeExportAMP())
+
+ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])
+
+ assert result.exit_code == 1
+ assert result.output == f"{FOLDER} is a link; the export writes only into a folder of its own.\n"
+ assert list(outside.iterdir()) == []
+
+
[email protected]("link", ["symbolic", "hard"])
+def test_a_link_where_a_file_goes_is_replaced_never_written_through(project, monkeypatch, capsys, writes, link):
+ directory, _ = project
+ elsewhere = directory / "keep.md"
+ elsewhere.write_text("not ours to change\n", encoding="utf-8")
+ (directory / FOLDER / "run").mkdir(parents=True)
+ placed = directory / FOLDER / "run" / "outputs.md"
+ if link == "symbolic":
+ placed.symlink_to(elsewhere)
+ else:
+ os.link(elsewhere, placed)
+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run/outputs.md": "o"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert elsewhere.read_text(encoding="utf-8") == "not ours to change\n"
+ assert not placed.is_symlink() and placed.read_text(encoding="utf-8") == "o"
+ assert capsys.readouterr().out == f"Wrote 2 files to {FOLDER}/ — start with report.md\n"
+
+
+def test_a_gitignore_link_already_in_the_folder_is_replaced_never_followed(
+ project, monkeypatch
+):
+ directory, _ = project
+ elsewhere = directory / "keep.txt"
+ elsewhere.write_text("not ours to change\n", encoding="utf-8")
+ (directory / FOLDER).mkdir()
+ (directory / FOLDER / ".gitignore").symlink_to(elsewhere)
+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert elsewhere.read_text(encoding="utf-8") == "not ours to change\n"
+ gitignore = directory / FOLDER / ".gitignore"
+ assert not gitignore.is_symlink()
+ assert gitignore.read_text(encoding="utf-8") == "*\n"
+
+
[email protected]("leads_to", ["a folder", "nowhere"])
+def test_an_export_folder_that_is_a_link_stops_the_export(project, monkeypatch, leads_to):
+ directory, _ = project
+ outside = directory / "outside"
+ if leads_to == "a folder":
+ outside.mkdir()
+ (directory / FOLDER).symlink_to(outside, target_is_directory=True)
+ install(monkeypatch, FakeExportAMP())
+
+ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])
+
+ assert result.exit_code == 1
+ assert result.output == f"{FOLDER} is a link; the export writes only into a folder of its own.\n"
+ left = {FOLDER, "outside"} if leads_to == "a folder" else {FOLDER}
+ assert {p.name for p in directory.iterdir()} == left # nothing made where the link leads
+ if leads_to == "a folder":
+ assert list(outside.iterdir()) == []
+
+
+@handles_only
+def test_a_folder_swapped_for_a_link_while_the_export_runs_is_not_gone_through(project, monkeypatch, capsys):
+ """The race a check-then-write loses: `run` is a folder when the first file
+ goes in, and a link out of the export by the time the second one does,
+ swapped in after the name was checked and before the folder is opened."""
+ directory, _ = project
+ outside = directory / "outside"
+ outside.mkdir()
+ real_mkdir = os.mkdir
+ asked: list[str] = []
+
+ def mkdir_then_swap(path, mode=0o777, *, dir_fd=None):
+ asked.append(path)
+ try:
+ real_mkdir(path, mode, dir_fd=dir_fd)
+ finally:
+ if asked.count("run") == 2:
+ (directory / FOLDER / "run").rename(directory / FOLDER / "moved")
+ (directory / FOLDER / "run").symlink_to(outside, target_is_directory=True)
+
+ monkeypatch.setattr(os, "mkdir", mkdir_then_swap)
+ install(monkeypatch, FakeExportAMP(exported({"run/a.md": "a", "run/b.md": "b"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert asked.count("run") == 2 # the swap happened
+ assert list(outside.iterdir()) == []
+ assert files_in(directory / FOLDER) == {"moved/a.md": "a", ".gitignore": "*\n"}
+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/ {SKIPPED}\n"
+
+
+@handles_only
+def test_the_export_folder_swapped_for_a_link_while_the_export_runs_is_not_gone_through(project, monkeypatch):
+ """The folder itself is held open: moved away and replaced by a link once
+ the export has begun, the files still go into it and nowhere else."""
+ directory, _ = project
+ outside = directory / "outside"
+ outside.mkdir()
+ real_mkdir = os.mkdir
+
+ def mkdir_then_swap(path, mode=0o777, *, dir_fd=None):
+ if path == "run" and not (directory / "moved").exists():
+ (directory / FOLDER).rename(directory / "moved")
+ (directory / FOLDER).symlink_to(outside, target_is_directory=True)
+ real_mkdir(path, mode, dir_fd=dir_fd)
+
+ monkeypatch.setattr(os, "mkdir", mkdir_then_swap)
+ install(monkeypatch, FakeExportAMP(exported({"report.md": "r", "run/a.md": "a"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert list(outside.iterdir()) == []
+ assert files_in(directory / "moved") == {"report.md": "r", "run/a.md": "a", ".gitignore": "*\n"}
+
+
+@handles_only
+def test_every_folder_handle_is_closed(project, monkeypatch):
+ directory, _ = project
+ outside = directory / "outside"
+ outside.mkdir()
+ (directory / FOLDER / "a").mkdir(parents=True)
+ (directory / FOLDER / "a" / "link").symlink_to(outside, target_is_directory=True)
+ install(monkeypatch, FakeExportAMP(exported({"a/b/c.md": "c", "a/link/x.md": "x", "report.md": "r"})))
+ open_before = len(os.listdir("/dev/fd"))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert len(os.listdir("/dev/fd")) == open_before
+ assert files_in(directory / FOLDER) == {"a/b/c.md": "c", "report.md": "r", ".gitignore": "*\n"}
+
+
+def test_content_that_is_not_text_is_skipped(project, monkeypatch, capsys):
+ directory, _ = project
+ body = (
+ '{"id": "%s", "kind": "trace", "files": {"report.md": "# Report", "n.json": 5, "none.md": null,'
+ ' "list.md": ["a"], "object.md": {"a": 1}, "half.md": "\\ud800"}}' % EVALUATION_ID
+ )
+ install(monkeypatch, FakeExportAMP(httpx.Response(200, content=body.encode())))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert files_in(directory / FOLDER) == {"report.md": "# Report", ".gitignore": "*\n"}
+ assert capsys.readouterr().out == (
+ f"Wrote 1 file to {FOLDER}/ — start with report.md "
+ "(skipped 5: an unsafe path, or content that is not text)\n"
+ )
+
+
+def test_an_export_without_a_report_does_not_point_at_one(project, monkeypatch, capsys):
+ directory, _ = project
+ install(monkeypatch, FakeExportAMP(exported({"run/outputs.md": "o"})))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert capsys.readouterr().out == f"Wrote 1 file to {FOLDER}/\n"
+
+
[email protected]("given", ["not-a-uuid", "", "../../etc", f"{EVALUATION_ID}0", "[red]x[/red]"])
+def test_an_id_that_is_not_a_uuid_is_refused_without_a_request(project, monkeypatch, given):
+ directory, _ = project
+ amp = install(monkeypatch, FakeExportAMP())
+
+ result = CliRunner().invoke(eval_command, ["export", given])
+
+ assert result.exit_code == 1
+ assert result.output == f"{given!r} is not an evaluation id: an evaluation id is a UUID.\n"
+ assert amp.calls == []
+ assert list(directory.iterdir()) == []
+
+
+ ("answer", "logged_in", "said"),
+ [
+ (httpx.Response(404, json={"error": "not_found", "message": "Evaluation not found."}), True,
+ "Evaluation not found."),
+ (httpx.Response(404, json={"error": "not_found"}), True,
+ f"AMP holds no evaluation {EVALUATION_ID} that this login may read."),
+ (httpx.Response(404, json={"error": "not_found"}), False,
+ f"AMP holds no evaluation {EVALUATION_ID} that can be read without a login; "
+ "if it is your account's, log in with `crewai login` and export it again."),
+ # an AMP from before exports has no route: its 404 is not AMP's `not_found`
+ (httpx.Response(404), True, NO_EXPORT_ROUTE),
+ (httpx.Response(404), False, NO_EXPORT_ROUTE),
+ (httpx.Response(404, json={"status": 404, "error": "Not Found"}), True, NO_EXPORT_ROUTE),
+ (httpx.Response(404, content=b"<html>The page you were looking for doesn't exist.</html>"), True,
+ NO_EXPORT_ROUTE),
+ # an AMP whose evaluation service predates exports
+ (httpx.Response(501, json={"error": "export_unsupported",
+ "message": "This AMP's evaluation service cannot export yet; update it."}), True,
+ "This AMP's evaluation service cannot export yet; update it."),
+ (httpx.Response(501, json={"error": "export_unsupported"}), True,
+ "This AMP's evaluation service does not export evaluations yet: it needs an update."),
+ # an export over 64 MB is refused whole, by the evaluation service (413) or by AMP (502)
+ (httpx.Response(413, json={"error": "export_too_large",
+ "message": "This export is 80 MB; an export stops at 64 MB."}), True,
+ "This export is 80 MB; an export stops at 64 MB."),
+ (httpx.Response(502, json={"error": "export_too_large",
+ "message": "The export is over 64 MB, more than AMP carries."}), False,
+ "The export is over 64 MB, more than AMP carries."),
+ (httpx.Response(413, json={"error": "export_too_large"}), True,
+ f"Evaluation {EVALUATION_ID} is too large to export."),
+ (httpx.Response(502, json={"error": "export_too_large"}), True,
+ f"Evaluation {EVALUATION_ID} is too large to export."),
+ (httpx.Response(502, json={"error": "bad_gateway", "message": "Upstream down"}), True,
+ "AMP answered 502: Upstream down."),
+ (httpx.Response(404, json={"error": "not_found", "message": "Evaluation not found."}), False,
+ "Evaluation not found; if it is your account's, log in with `crewai login` and export it again."),
+ (httpx.Response(409, json={"error": "not_finished", "message": "[b]Still[/b] grading."}), True,
+ "[b]Still[/b] grading."),
+ (httpx.Response(409), True,
+ f"Evaluation {EVALUATION_ID} is still running; export it once it has finished."),
+ (httpx.Response(410, json={"error": "report_closed", "message": "This report was opened and is closed."}), False,
+ "This report was opened and is closed."),
+ (httpx.Response(410), False,
+ f"The report of evaluation {EVALUATION_ID} was opened already and is closed; "
+ "log in with `crewai login` and evaluate the run again."),
+ (httpx.Response(503, json={"error": "service_unavailable", "message": "Back soon"}), True,
+ "AMP answered 503: Back soon."),
+ (httpx.Response(429, json={"message": "Slow down"}, headers={"Retry-After": "30"}), True,
+ "Slow down — retry after 30s."),
+ (httpx.Response(401, json={"error": "bad_credentials", "message": "Token expired"}), True,
+ "Token expired. Log in with `crewai login` and try again."),
+ (httpx.ConnectError("connection refused"), True,
+ f"Could not reach AMP to export evaluation {EVALUATION_ID}: connection refused"),
+ (httpx.Response(200, json={"id": EVALUATION_ID, "kind": "trace"}), True,
+ f"AMP answered without the files of evaluation {EVALUATION_ID} (protocol error)."),
+ (httpx.Response(200, content=b"<html>"), True,
+ f"AMP answered without the files of evaluation {EVALUATION_ID} (protocol error)."),
+ ],
+)
+def test_a_refused_export_says_why_in_one_sentence_and_exits_1(project, monkeypatch, answer, logged_in, said):
+ directory, _ = project
+ if not logged_in:
+ monkeypatch.setattr(eval_module, "saved_login", lambda: None)
+ amp = install(monkeypatch, FakeExportAMP(answer))
+
+ result = CliRunner().invoke(eval_command, ["export", EVALUATION_ID])
+
+ assert result.exit_code == 1
+ assert result.output == f"{said}\n" # AMP's words printed as they are, brackets and all
+ assert amp.calls == [("export", EVALUATION_ID)]
+ assert (amp.api_key is not None) is logged_in
+ if not logged_in:
+ assert "X-Crewai-Organization-Id" not in amp.headers
+ assert list(directory.iterdir()) == []
+
+
+def test_the_written_line_stays_one_line_in_a_narrow_pipe(project, monkeypatch, capsys):
+ install(monkeypatch, FakeExportAMP())
+ monkeypatch.setattr(eval_module, "console", Console(width=40))
+
+ eval_module.export_evaluation(EVALUATION_ID)
+
+ assert capsys.readouterr().out == f"Wrote 5 files to {FOLDER}/ — start with report.md\n"
diff --git a/lib/cli/tests/test_plus_api.py b/lib/cli/tests/test_plus_api.py
index 15abb8ba2..b797f29b5 100644
--- a/lib/cli/tests/test_plus_api.py
+++ b/lib/cli/tests/test_plus_api.py
@@ -101,7 +101,24 @@ class TestPlusAPI(unittest.TestCase):
response = self.api.get_evaluation("ev-1")
mock_make_request.assert_called_once_with(
- "GET", "/crewai_plus/api/v1/tracing/evaluations/ev-1", timeout=30.0
+ "GET",
+ "/crewai_plus/api/v1/tracing/evaluations/ev-1",
+ params={"can_export": "1"},
+ timeout=30.0,
+ )
+ self.assertEqual(response, mock_response)
+
+ @patch("crewai_core.plus_api.PlusAPI._make_request")
+ def test_export_evaluation(self, mock_make_request):
+ mock_response = MagicMock()
+ mock_make_request.return_value = mock_response
+
+ response = self.api.export_evaluation("ev-1")
+
+ mock_make_request.assert_called_once_with(
+ "GET",
+ "/crewai_plus/api/v1/tracing/evaluations/ev-1/export",
+ timeout=httpx.Timeout(30.0, read=120.0),
)
self.assertEqual(response, mock_response)
@@ -569,3 +586,28 @@ def test_requests_without_their_own_timeout_keep_the_httpx_default():
assert request.url.path == "/crewai_plus/api/v1/agents/an_agent"
assert request.extensions["timeout"]["read"] == 5.0
+
+
+def test_an_export_waits_two_minutes_for_its_large_answer():
+ """The export carries every setup's outputs and traces, tens of MB: the
+ httpx default of 5 s would cut it off."""
+ (request,) = _requests_sent_by(lambda api: api.export_evaluation("ev-1"))
+
+ assert request.method == "GET"
+ assert request.url.path == "/crewai_plus/api/v1/tracing/evaluations/ev-1/export"
+ assert request.extensions["timeout"] == {
+ "connect": 30.0,
+ "read": 120.0,
+ "write": 30.0,
+ "pool": 30.0,
+ }
+
+
+def test_reading_an_evaluation_says_this_cli_can_export_it():
+ """`can_export=1` lets the brief name `crewai eval export`: an AMP that does
+ not know the parameter drops it, and the brief stays silent about it."""
+ (request,) = _requests_sent_by(lambda api: api.get_evaluation("ev-1"))
+
+ assert request.method == "GET"
+ assert request.url.path == "/crewai_plus/api/v1/tracing/evaluations/ev-1"
+ assert dict(request.url.params) == {"can_export": "1"}