Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(cli): every crewai eval keeps the evaluation's full data in the project (#8326)
João Mouraa8877f0cdfbc
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex b7abe2cff..cfe10c9e8 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -40,6 +40,7 @@ import mathimport osfrom pathlib import Pathimport re+import shutilimport statimport sysimport time@@ -396,6 +397,7 @@ def eval_crew(record_finished(finished)_print_verdict(finished, url)_print_brief(finished)+ save_evaluation(client, finished)_say_where_the_criteria_live(write_eval_config(finished))# The exit code is what a CI job reads, so it is the gate's: 0 only for a# run that PASSED. A failed gate, one without a verdict, and an evaluation@@ -2245,6 +2247,130 @@ _FOLDER_FLAGS = (_NEW_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)+# Every evaluation `crewai eval` reads keeps its full data in the project. The+# report opens for one five-minute sitting after it is first read, and the+# verdict this command waits for is that read: a coding agent that wants a+# check's quotes or the trace an hour later, after its edits, would find it+# closed. The newest few are kept; older ones are removed.+SAVED_EVALUATIONS_DIR = "evaluations"+SAVED_EVALUATIONS_KEPT = 10+# Written last, and only when every file was: what says a folder was saved+# whole by this command — so a save cut short is tried again, and only such a+# folder is ever removed to keep the newest.+SAVED_MARKER = ".crewai-eval-saved"+# The verdict is already printed when the save starts, and one run's export is+# small: a slow save must not hold a CI job or an agent for export's two minutes.+SAVE_TIMEOUT = httpx.Timeout(10.0, read=20.0)+++def save_evaluation(client: PlusAPI, finished: dict[str, Any]) -> None:+ """Save evaluation FINISHED's full data (the export) into+ `.crewai/evaluations/<id>/`, and say where.++ Only inside a project, and never at the cost of the evaluation: a service+ that does not export yet says nothing, and any other failure is one line.+ A save cut short, or one that skipped a file, leaves no marker, says so,+ and is tried again the next time this evaluation is read.+ """+ try:+ canonical = str(uuid.UUID(str(finished.get("id"))))+ except ValueError:+ return+ if not Path("pyproject.toml").is_file():+ return # `crewai eval --run ID` from anywhere: no project to keep it in+ from crewai.telemetry.tracing.last_run import LAST_RUN_DIR++ folder = Path(LAST_RUN_DIR) / SAVED_EVALUATIONS_DIR / canonical+ if _saved_whole(folder):+ # Graded again with nothing changed: the same evaluation, saved when it+ # was first read — and its sitting may have closed since.+ console.print(Text(_saved_line(folder)), soft_wrap=True)+ return+ try:+ response = client.export_evaluation(canonical, timeout=SAVE_TIMEOUT)+ except Exception as error:+ console.print(Text(f"Its full data was not saved: {error}"), style="dim")+ return+ if response.status_code == 404:+ return # an evaluation service that does not export yet+ payload = _payload(response) or {}+ files = payload.get("files")+ if response.status_code != 200 or not isinstance(files, dict):+ said = payload.get("message")+ why = said if isinstance(said, str) and said.strip() else None+ why = why or f"AMP answered {response.status_code}."+ console.print(Text(f"Its full data was not saved: {why}"), style="dim")+ return+ try:+ _, skipped = _write_export(folder, files, marker=SAVED_MARKER)+ except ExportWriteError as error:+ console.print(Text(f"Its full data was not saved: {error}"), style="dim")+ return+ if skipped:+ plural = "" if skipped == 1 else "s"+ console.print(+ Text(+ f"Part of its data was saved in {folder}/: {skipped} file{plural} "+ "skipped (an unsafe path, or content that is not text)."+ ),+ style="dim",+ soft_wrap=True,+ )+ return+ _keep_newest_saved_evaluations(folder.parent)+ console.print(Text(_saved_line(folder)), soft_wrap=True)+++def _saved_line(folder: Path) -> str:+ return f"Its full data is saved in {folder}/ — start with {EXPORT_START_HERE}."+++def _saved_whole(folder: Path) -> bool:+ """Did this command save FOLDER whole? Its marker is there, a plain file."""+ try:+ return stat.S_ISREG((folder / SAVED_MARKER).lstat().st_mode)+ except OSError:+ return False+++def _is_a_link(path: Path) -> bool:+ """A symbolic link, or a Windows junction, which `is_symlink` misses."""+ if path.is_symlink():+ return True+ is_junction = getattr(path, "is_junction", None) # Python 3.12++ return bool(is_junction and is_junction())+++def _keep_newest_saved_evaluations(folder: Path) -> None:+ """Remove all but the SAVED_EVALUATIONS_KEPT newest evaluations this+ command saved whole: only a real folder named by an evaluation id with the+ marker in it — never a link, never a folder it did not write. Nothing is+ removed when FOLDER, or the `.crewai/` above it, is a link."""+ with contextlib.suppress(OSError):+ if _is_a_link(folder) or _is_a_link(folder.parent):+ return+ if os.path.normcase(folder.resolve()) != os.path.normcase(+ Path.cwd().resolve() / folder+ ):+ return+ saved = []+ for entry in folder.iterdir():+ try:+ uuid.UUID(entry.name)+ except ValueError:+ continue+ if _is_a_link(entry) or not entry.is_dir() or not _saved_whole(entry):+ continue+ saved.append((entry.stat().st_mtime, entry))+ saved.sort(reverse=True)+ for _, old in saved[SAVED_EVALUATIONS_KEPT:]:+ shutil.rmtree(old, ignore_errors=True)+++class ExportWriteError(Exception):+ """An export could not be written where it was going; the message says why."""++def export_evaluation(evaluation_id: str) -> None:"""Write evaluation EVALUATION_ID's full data into `./crewai-eval-<id>/`.@@ -2285,7 +2411,10 @@ def export_evaluation(evaluation_id: str) -> None:)folder = Path(f"{EXPORT_DIR_PREFIX}{canonical}")- written, skipped = _write_export(folder, files)+ try:+ written, skipped = _write_export(folder, files)+ except ExportWriteError as error:+ _fail(str(error))line = f"Wrote {len(written)} file{'' if len(written) == 1 else 's'} to {folder}/"if EXPORT_START_HERE in written:line += f" — start with {EXPORT_START_HERE}"@@ -2360,7 +2489,9 @@ def _export_target(name: str) -> list[str] | None:every segment a plain name (`[A-Za-z0-9._-]+`, never `.` or `..`). A `.git`segment is refused too: a `.git/config` there could name a command for thenext `git status` in the folder to run, and a shell prompt runs one. The- top `.gitignore` is this command's own. So is a name Windows keeps for a+ top `.gitignore` is this command's own, and so is the top SAVED_MARKER: a+ marker AMP sent would say a save was whole when it was not. So is a name+ Windows keeps for adevice (`CON`, `PRN`, `AUX`, `NUL`, `COM0`-`COM9`, `LPT0`-`LPT9`, with anyextension), on every platform, since the folder may be opened on Windowslater or sit on a drive it shares: there `NUL.md` is the null device, and@@ -2379,18 +2510,26 @@ def _export_target(name: str) -> list[str] | None:or seen.split(".", 1)[0] in _WINDOWS_DEVICE_NAMES):return None- if len(segments) == 1 and _as_a_filesystem_sees_it(segments[0]) == ".gitignore":+ if len(segments) == 1 and _as_a_filesystem_sees_it(segments[0]) in _RESERVED_TOP:return Nonereturn segments+# The folder's own files, which no exported file may be: `.gitignore` keeps the+# data out of a commit, the marker says the save was whole.+_RESERVED_TOP = frozenset({".gitignore", SAVED_MARKER})++def _as_a_filesystem_sees_it(segment: str) -> str:"""SEGMENT as macOS and Windows compare names: case ignored, trailing dots dropped."""return segment.rstrip(".").lower()-def _write_export(folder: Path, files: dict[str, Any]) -> tuple[list[str], int]:+def _write_export(+ folder: Path, files: dict[str, Any], *, marker: str | None = None+) -> tuple[list[str], int]:"""Write FILES under FOLDER: the names written, and how many were skipped.+ MARKER, when given, is written last, and only when nothing was skipped.The `.gitignore` goes first, so the data is never in the folder without it.A file whose path would go through a link, or through a file where a@@ -2404,10 +2543,12 @@ def _write_export(folder: Path, files: dict[str, Any]) -> tuple[list[str], int]:try:ignored = put([".gitignore"], b"*\n")except OSError as error:- _fail(f"Could not write the export into {folder}/: {error}")+ raise ExportWriteError(+ f"Could not write the export into {folder}/: {error}"+ ) from errorif not ignored:# The data never goes into the folder without it.- _fail(+ raise ExportWriteError(f"{folder}/.gitignore is a folder, not this command's file: remove it ""and export again, so the data is never committed by accident.")@@ -2424,11 +2565,20 @@ def _write_export(folder: Path, files: dict[str, Any]) -> tuple[list[str], int]:try:put_there = put(segments, data)except OSError as error:- _fail(f"Could not write {folder}/{name}: {error}")+ raise ExportWriteError(+ f"Could not write {folder}/{name}: {error}"+ ) from errorif put_there:written.append(name)else:skipped += 1+ if marker is not None and not skipped:+ try:+ put([marker], b"Saved whole by `crewai eval`.\n")+ except OSError as error:+ raise ExportWriteError(+ f"Could not write {folder}/{marker}: {error}"+ ) from errorreturn written, skipped@@ -2454,24 +2604,38 @@ def _export_folder(folder: Path) -> Iterator[Callable[[list[str], bytes], bool]]def _export_root_is_a_link(folder: Path) -> NoReturn:"""Stop: whoever made FOLDER a link chose where it leads, not this command."""- _fail(f"{folder} is a link; the export writes only into a folder of its own.")+ raise ExportWriteError(+ f"{folder} is a link; the export writes only into a folder of its own."+ )def _export_root_handle(folder: Path) -> int:- """A handle on FOLDER, made when missing, opened inside the current- directory and never through a link."""+ """A handle on FOLDER, a path relative to the current directory: each of+ its folders made when missing and opened inside the one before it, never+ through a link."""try:- here = os.open(".", _FOLDER_FLAGS)- try:+ handle = os.open(".", _FOLDER_FLAGS)+ except OSError as error:+ raise ExportWriteError(+ f"Could not write the export into {folder}/: {error}"+ ) from error+ try:+ for part in folder.parts:with contextlib.suppress(FileExistsError):- os.mkdir(folder.name, dir_fd=here)- return os.open(folder.name, _FOLDER_FLAGS, dir_fd=here)- finally:- os.close(here)+ os.mkdir(part, dir_fd=handle)+ inner = os.open(part, _FOLDER_FLAGS, dir_fd=handle)+ os.close(handle)+ handle = innerexcept OSError as error:- if folder.is_symlink():+ os.close(handle)+ if any(+ Path(*folder.parts[: n + 1]).is_symlink() for n in range(len(folder.parts))+ ):_export_root_is_a_link(folder)- _fail(f"Could not write the export into {folder}/: {error}")+ raise ExportWriteError(+ f"Could not write the export into {folder}/: {error}"+ ) from error+ return handledef _put_through_handles(root: int, segments: list[str], data: bytes) -> bool:@@ -2528,14 +2692,18 @@ def _export_root_by_path(folder: Path) -> Path:`is_symlink` does not see, and which any account can make), would carryevery file somewhere else."""- if folder.is_symlink():- _export_root_is_a_link(folder)+ # Every folder on the way, before anything is made through one of them.+ for n in range(len(folder.parts)):+ if _is_a_link(Path(*folder.parts[: n + 1])):+ _export_root_is_a_link(folder)try:- folder.mkdir(exist_ok=True)+ folder.mkdir(parents=True, exist_ok=True)root = folder.resolve()except OSError as error:- _fail(f"Could not write the export into {folder}/: {error}")- if os.path.normcase(root) != os.path.normcase(Path.cwd().resolve() / folder.name):+ raise ExportWriteError(+ f"Could not write the export into {folder}/: {error}"+ ) from error+ if os.path.normcase(root) != os.path.normcase(Path.cwd().resolve() / folder):_export_root_is_a_link(folder)return rootdiff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.pyindex 401fde87f..70db689c0 100644--- a/lib/cli/src/crewai_cli/plus_api.py+++ b/lib/cli/src/crewai_cli/plus_api.py@@ -103,13 +103,16 @@ class PlusAPI(_CorePlusAPI):timeout=self.EVALUATION_POLL_TIMEOUT,)- def export_evaluation(self, evaluation_id: str) -> httpx.Response:+ def export_evaluation(+ self, evaluation_id: str, *, timeout: httpx.Timeout | None = None+ ) -> httpx.Response:"""The evaluation's full data as files (crewai eval export):- `{"id", "kind", "files": {relative path: text}}`."""+ `{"id", "kind", "files": {relative path: text}}`. TIMEOUT replaces the+ default, which allows the two minutes a large comparison may take."""return self._make_request("GET",f"{self.EVALUATIONS_RESOURCE}/{evaluation_id}/export",- timeout=self.EVALUATION_EXPORT_TIMEOUT,+ timeout=timeout or self.EVALUATION_EXPORT_TIMEOUT,)def _make_multipart_request(diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex dd6768440..dd2fd68aa 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -6,6 +6,7 @@ import contextlibfrom datetime import datetime, timedelta, timezoneimport jsonimport os+import uuidfrom pathlib import Pathfrom types import SimpleNamespace@@ -2388,8 +2389,9 @@ class FakeExportAMP(FakeAMP):super().__init__()self.answer = answer if answer is not None else exported()- def export_evaluation(self, evaluation_id):+ def export_evaluation(self, evaluation_id, **kwargs):self.calls.append(("export", evaluation_id))+ self.export_kwargs = kwargsif isinstance(self.answer, Exception):raise self.answerreturn self.answer@@ -2546,6 +2548,8 @@ def test_exporting_again_overwrites_the_files(project, monkeypatch, capsys):".Git./config",".gitignore", # the folder's own".GITIGNORE",+ ".crewai-eval-saved", # the saved evaluation's marker, never AMP's to send+ ".CREWAI-EVAL-SAVED.",# names Windows keeps for its devices, with any extension"NUL.md","nul",@@ -3460,6 +3464,208 @@ def test_a_project_that_turned_tracing_off_is_not_asked_to_trace_toward_an_untruassert "CREWAI_TRACING_ENABLED=false" in (directory / ".env").read_text()+# ── every evaluation keeps its full data in the project ─────────────────────++SAVED = Path(".crewai") / "evaluations"+++def _verdict_of(evaluation_id: str, gate: str = "passed") -> httpx.Response:+ payload = done(gate=gate).json()+ return httpx.Response(200, json={**payload, "id": evaluation_id})+++def _graded(monkeypatch, directory: Path, answer=None, evaluation_id: str = EVALUATION_ID, gate: str = "passed"):+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ amp = FakeExportAMP(answer=answer if answer is not None else exported(kind="trace"))+ amp.statuses = [_verdict_of(evaluation_id, gate)]+ return install(monkeypatch, amp)+++def test_the_verdict_leaves_the_evaluations_full_data_in_the_project(project, monkeypatch, capsys):+ """The report opens for one five-minute sitting, and waiting for the verdict+ is that read: an agent that wants a check's quotes or the trace after its+ edits finds them here instead."""+ directory, _ = project+ amp = _graded(monkeypatch, directory)++ eval_module.eval_crew(run_id="named-run")++ saved = directory / SAVED / EVALUATION_ID+ assert files_in(saved) == {+ ".gitignore": "*\n",+ **EXPORT_FILES,+ eval_module.SAVED_MARKER: "Saved whole by `crewai eval`.\n",+ }+ assert ("export", EVALUATION_ID) in amp.calls+ # its own short wait: the verdict is already out, and CI waits on the command+ assert amp.export_kwargs == {"timeout": eval_module.SAVE_TIMEOUT}+ said = " ".join(capsys.readouterr().out.split())+ assert f"Its full data is saved in {SAVED / EVALUATION_ID}/ — start with report.md." in said+++def test_a_service_that_does_not_export_yet_costs_nothing_and_says_nothing(project, monkeypatch, capsys):+ directory, _ = project+ _graded(monkeypatch, directory, answer=httpx.Response(404, json={"error": "not_found"}))++ eval_module.eval_crew(run_id="named-run")++ assert not (directory / SAVED).exists()+ assert "full data" not in capsys.readouterr().out+++ "answer",+ [httpx.ConnectError("no route"), httpx.Response(500, json={"error": "boom"})],+)+def test_a_save_that_fails_is_one_line_and_never_the_verdicts(project, monkeypatch, capsys, answer):+ """The exit code stays the gate's: a failed gate is still 1, a passed one 0."""+ directory, _ = project+ _graded(monkeypatch, directory, answer=answer, gate="failed")++ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew(run_id="named-run")++ assert exited.value.code == 1 # the failed gate, not the save+ said = " ".join(capsys.readouterr().out.split())+ assert said.count("Its full data was not saved:") == 1+ assert "Goal gate: FAILED" in said+++def test_only_the_newest_saved_evaluations_are_kept(project, monkeypatch):+ directory, _ = project+ older = [str(uuid.UUID(int=n)) for n in range(1, 13)]+ for n, name in enumerate(older):+ folder = directory / SAVED / name+ folder.mkdir(parents=True)+ (folder / "report.md").write_text("old")+ (folder / eval_module.SAVED_MARKER).write_text("saved\n")+ os.utime(folder, (1_000_000 + n, 1_000_000 + n))+ (directory / SAVED / "notes").mkdir() # not an evaluation: never removed+ # named like an evaluation, but not saved by this command: never removed+ unmarked = str(uuid.UUID(int=99))+ (directory / SAVED / unmarked).mkdir()+ os.utime(directory / SAVED / unmarked, (1, 1))+ _graded(monkeypatch, directory)++ eval_module.eval_crew(run_id="named-run")++ kept = sorted(p.name for p in (directory / SAVED).iterdir())+ assert EVALUATION_ID in kept and "notes" in kept and unmarked in kept+ assert len([name for name in kept if name not in ("notes", unmarked)]) == eval_module.SAVED_EVALUATIONS_KEPT+ assert older[0] not in kept and older[-1] in kept # the oldest went first+++def test_outside_a_project_nothing_is_saved(project, monkeypatch):+ directory, _ = project+ amp = _graded(monkeypatch, directory)+ (directory / "pyproject.toml").unlink()++ eval_module.eval_crew(run_id="named-run")++ assert not (directory / ".crewai" / "evaluations").exists()+ assert ("export", EVALUATION_ID) not in amp.calls+++def test_graded_again_unchanged_it_points_at_what_it_saved_and_asks_nothing(project, monkeypatch, capsys):+ """The same evaluation, read again after its sitting closed: what was saved+ the first time is where the data is, and AMP is not asked a second time."""+ directory, _ = project+ saved = directory / SAVED / EVALUATION_ID+ saved.mkdir(parents=True)+ (saved / "report.md").write_text("# Report\n")+ (saved / eval_module.SAVED_MARKER).write_text("saved\n")+ amp = _graded(monkeypatch, directory, answer=httpx.Response(410, json={"error": "report_closed", "message": "read already"}))++ eval_module.eval_crew(run_id="named-run")++ assert ("export", EVALUATION_ID) not in amp.calls+ assert "Its full data is saved in" in " ".join(capsys.readouterr().out.split())+++def test_a_refused_save_says_amps_own_words(project, monkeypatch, capsys):+ directory, _ = project+ message = f"Evaluation {EVALUATION_ID} has been read already. Evaluate the run again for a fresh report."+ _graded(monkeypatch, directory, answer=httpx.Response(410, json={"error": "report_closed", "message": message}))++ eval_module.eval_crew(run_id="named-run")++ assert f"Its full data was not saved: {message}" in " ".join(capsys.readouterr().out.split())+++def test_a_save_that_skipped_a_file_says_so_and_is_tried_again(project, monkeypatch, capsys):+ """Missing checks or a missing trace must not read as the full data."""+ directory, _ = project+ files = {**EXPORT_FILES, "../outside.md": "no"}+ amp = _graded(monkeypatch, directory, answer=exported(files=files, kind="trace"))++ eval_module.eval_crew(run_id="named-run")++ said = " ".join(capsys.readouterr().out.split())+ assert f"Part of its data was saved in {SAVED / EVALUATION_ID}/: 1 file skipped" in said+ assert "Its full data is saved" not in said+ assert not (directory / SAVED / EVALUATION_ID / eval_module.SAVED_MARKER).exists()++ amp.statuses = [_verdict_of(EVALUATION_ID)]+ eval_module.eval_crew(run_id="named-run")+ assert amp.calls.count(("export", EVALUATION_ID)) == 2 # not taken as saved+++def test_a_save_cut_short_after_the_report_is_tried_again(project, monkeypatch, capsys):+ """report.md alone is not the evaluation saved: no marker, so the next+ read saves it again instead of pointing at half of it."""+ directory, _ = project+ amp = _graded(monkeypatch, directory)+ real = {name: getattr(eval_module, name) for name in ("_put_through_handles", "_put_by_path")}++ def failing(name):+ def put(root, segments, data):+ if segments == ["run", "trace.json"]:+ raise OSError("disk full")+ return real[name](root, segments, data)+ return put++ with monkeypatch.context() as patched:+ for name in real:+ patched.setattr(eval_module, name, failing(name))+ eval_module.eval_crew(run_id="named-run")++ said = " ".join(capsys.readouterr().out.split())+ assert "Its full data was not saved: Could not write" in said and "disk full" in said+ saved = directory / SAVED / EVALUATION_ID+ assert (saved / "report.md").is_file() and not (saved / eval_module.SAVED_MARKER).exists()++ amp.statuses = [_verdict_of(EVALUATION_ID)]+ eval_module.eval_crew(run_id="named-run")+ assert amp.calls.count(("export", EVALUATION_ID)) == 2+ assert (saved / eval_module.SAVED_MARKER).is_file()++[email protected](os.name == "nt", reason="symbolic links need privileges on Windows")[email protected]("linked", [".crewai", ".crewai/evaluations"])[email protected]("through_handles", [True, False])+def test_a_link_on_the_way_keeps_the_data_out_of_wherever_it_leads(+ project, monkeypatch, capsys, tmp_path_factory, linked, through_handles+):+ """Whoever made `.crewai` or `.crewai/evaluations` a link chose where it+ leads: nothing is written there, by either way of writing an export."""+ if through_handles and not eval_module._WRITES_THROUGH_HANDLES:+ pytest.skip("this platform writes by path")+ monkeypatch.setattr(eval_module, "_WRITES_THROUGH_HANDLES", through_handles)+ directory, _ = project+ elsewhere = tmp_path_factory.mktemp("elsewhere")+ if linked == ".crewai/evaluations":+ (directory / ".crewai").mkdir(exist_ok=True)+ (directory / linked).symlink_to(elsewhere, target_is_directory=True)+ _graded(monkeypatch, directory)++ eval_module.eval_crew(run_id="named-run")++ assert [p for p in elsewhere.rglob("*") if p.is_file()] == []+ said = " ".join(capsys.readouterr().out.split())+ assert "Its full data was not saved:" in said and "is a link" in said+ assert "Goal gate: PASSED" in said++# ── with no terminal, the run's own output goes to a log, not the agent ────@@ -3652,3 +3858,23 @@ def test_putting_back_goes_on_past_a_failure_and_keeps_the_runs_own_error(tmp_pafinally:real_dup2(stdout_copy, 1) # the test puts stdout back itselfos.close(stdout_copy)+++def test_a_marker_in_the_payload_never_marks_a_save_that_was_not_whole(project, monkeypatch, capsys):+ """AMP cannot send the marker: with one file skipped, a forged marker would+ have made a partial save read as whole, never tried again and kept over a+ complete one."""+ directory, _ = project+ files = {**EXPORT_FILES, eval_module.SAVED_MARKER: "forged\n", "../outside.md": "no"}+ amp = _graded(monkeypatch, directory, answer=exported(files=files, kind="trace"))++ eval_module.eval_crew(run_id="named-run")++ saved = directory / SAVED / EVALUATION_ID+ assert not (saved / eval_module.SAVED_MARKER).exists()+ said = " ".join(capsys.readouterr().out.split())+ assert "Part of its data was saved" in said and "2 files skipped" in said++ amp.statuses = [_verdict_of(EVALUATION_ID)]+ eval_module.eval_crew(run_id="named-run")+ assert amp.calls.count(("export", EVALUATION_ID)) == 2 # tried again