Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(cli): with no terminal, crewai eval sends the crew's own output to a log (#8325)
João Moura94508c971f02
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex 87718918c..b7abe2cff 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -43,7 +43,7 @@ import reimport statimport sysimport time-from typing import Any, Literal, NoReturn, cast+from typing import Any, BinaryIO, Literal, NoReturn, castfrom urllib.parse import urlparseimport uuidimport webbrowser@@ -161,6 +161,13 @@ NOT_TRACED = (f"({LAST_RUN_FILE}).")+# With no terminal, what the crew prints while it runs (its panels, the final+# answer, every tool call) goes to this file in the project's `.crewai/`, not to+# the coding agent's screen: in one measured run it was 341 of 378 lines, all+# before the verdict, and an agent pays for every line or has the end cut off.+RUN_LOG_FILE = "last_run.log"+RUN_LOG_TAIL_LINES = 40+# Why `crewai eval` ran the crew before evaluating, a word from a closed list:# nothing traced yet, the project changed since its last traced run, or the run# app's Evaluate pressed on a run that was not traced.@@ -1002,11 +1009,23 @@ def _run_the_crew_to_evaluate(# The handshake (`crewai_core.eval_handshake`) carries what this command# decided before the project's `.env` was loaded: the AMP origins its saved# login may go to, and whether it just turned tracing on.+ # Whoever reads the output decides, as for the brief: stdout, not stdin.+ run_log = _run_log_path() if nobody_watching() else None+ if run_log is not None:+ console.print(+ f"The crew's output goes to {_shown_path(run_log)} while it runs.",+ style="dim",+ )began = datetime.now(timezone.utc)- with evaluating_after_run(- trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now- ) as watched:+ started = time.monotonic()+ with (+ evaluating_after_run(+ trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now+ ) as watched,+ _run_output_to(run_log) as run_log,+ ):run_crew(**run_args)+ seconds = time.monotonic() - started# The app says what it did with the run it watched, and its word settles# this: it evaluated it (nothing left to do), or it had nothing to evaluate@@ -1018,7 +1037,8 @@ def _run_the_crew_to_evaluate(remember_project_state(str(marker["execution_id"]))return Nonerecord_stop("untraced_after_run")- console.print(NOT_TRACED, style="bold red")+ _say_what_the_run_left(run_log, seconds, traced=False)+ console.print(_not_traced(run_log), style="bold red")raise SystemExit(1)# The project's record, only when it was written after this run began: a@@ -1029,13 +1049,157 @@ def _run_the_crew_to_evaluate(record.get("execution_id") if _recorded_since(record, began) else None)if traced:+ _say_what_the_run_left(run_log, seconds, traced=True)return str(traced)record_stop("untraced_after_run")- console.print(NOT_TRACED, style="bold red")+ _say_what_the_run_left(run_log, seconds, traced=False)+ console.print(_not_traced(run_log), style="bold red")raise SystemExit(1)+def _not_traced(run_log: Path | None) -> str:+ """NOT_TRACED, pointing at where the run's output went."""+ if run_log is None:+ return NOT_TRACED+ return NOT_TRACED.replace("(its output is above)", "(its last lines are above)")+++def _run_log_path() -> Path | None:+ """Where a run with no terminal sends its output, or None when the+ project's `.crewai/` cannot be written (the output then stays on screen)."""+ from crewai.telemetry.tracing.last_run import LAST_RUN_DIR, project_dir++ try:+ path = project_dir() / LAST_RUN_DIR / RUN_LOG_FILE+ path.parent.mkdir(parents=True, exist_ok=True)+ except OSError:+ return None+ return path+++def _run_output_to(path: Path | None) -> Iterator[Path | None]:+ """Send everything printed inside the block to PATH, and yield it; with no+ PATH, yield None and change nothing.++ It moves this process's own stdout and stderr (file descriptors 1 and 2),+ which every child the run starts inherits, so each way a project runs —+ `uv run`, the JSON crew runner, a declarative flow — is covered without+ touching any of them. Anything that keeps them from moving leaves the+ output where it was: a quiet log is a nicety, never a failed run. Both are+ always put back after, one by one, whatever failed: a stream left on the+ log would swallow the verdict, and the run's own error is never hidden by+ a failure to put them back."""+ log = _open_run_log(path) if path is not None else None+ if log is None:+ yield None+ return+ with log:+ _flush_std_streams()+ saved: dict[int, int] = {}+ moved: list[int] = []+ try:+ for fd in (1, 2):+ saved[fd] = os.dup(fd)+ for fd in (1, 2):+ os.dup2(log.fileno(), fd)+ moved.append(fd)+ except OSError:+ _put_back(saved, moved)+ yield None+ return+ _point_windows_std_handles_at(1, 2)+ try:+ yield path+ finally:+ _flush_std_streams()+ _put_back(saved, moved)+ _point_windows_std_handles_at(1, 2)+++def _open_run_log(path: Path) -> BinaryIO | None:+ """The run log, opened for writing owner-only and never through a link:+ it holds final answers, tool output and tracebacks that can carry a+ credential. A log an earlier run left is emptied and its mode tightened.+ None when it cannot be opened that way."""+ flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)+ try:+ if path.is_symlink():+ return None # a link someone planted does not choose where it goes+ fd = os.open(path, flags, 0o600)+ except OSError:+ return None+ with contextlib.suppress(OSError, AttributeError):+ os.fchmod(fd, 0o600)+ return os.fdopen(fd, "wb")+++def _flush_std_streams() -> None:+ for stream in (sys.stdout, sys.stderr):+ with contextlib.suppress(Exception):+ stream.flush()+++def _put_back(saved: dict[int, int], moved: list[int]) -> None:+ """Point every MOVED descriptor at its SAVED original again, and close+ every saved copy: each on its own, so one failure stops none of the rest."""+ for fd in moved:+ with contextlib.suppress(OSError):+ os.dup2(saved[fd], fd)+ for copy in saved.values():+ _close_quietly(copy)+++def _close_quietly(fd: int) -> None:+ with contextlib.suppress(OSError):+ os.close(fd)+++def _point_windows_std_handles_at(out_fd: int, err_fd: int) -> None:+ """On Windows a child takes the process's standard handles, not its file+ descriptors: point them where descriptors 1 and 2 now go."""+ if os.name != "nt":+ return+ with contextlib.suppress(Exception):+ import ctypes+ import msvcrt++ kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]+ handle_of = msvcrt.get_osfhandle # type: ignore[attr-defined]+ kernel32.SetStdHandle(-11, handle_of(out_fd)) # STD_OUTPUT_HANDLE+ kernel32.SetStdHandle(-12, handle_of(err_fd)) # STD_ERROR_HANDLE+++def _say_what_the_run_left(log: Path | None, seconds: float, traced: bool) -> None:+ """After a quiet run: where its output is, and — when it left nothing to+ evaluate — its last lines, which usually hold the error."""+ if log is None:+ return+ where = _shown_path(log)+ if traced:+ console.print(+ f"The crew ran in {seconds:.0f}s; its output is in {where}.", style="dim"+ )+ return+ with contextlib.suppress(OSError):+ lines = log.read_text(encoding="utf-8", errors="replace").splitlines()+ tail = lines[-RUN_LOG_TAIL_LINES:]+ if tail:+ console.print(+ f"The crew's output is in {where}; its last {len(tail)} lines:",+ style="bold",+ )+ console.print(Text("\n".join(tail)))+++def _shown_path(path: Path) -> str:+ try:+ return str(path.relative_to(Path.cwd()))+ except ValueError:+ return str(path)++def _recorded_since(record: dict[str, Any], began: datetime) -> bool:"""Was RECORD written after BEGAN? A record without a readable, zoned stampis not assumed to be this run's."""diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex 4d90d7d6b..dd6768440 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -3458,3 +3458,197 @@ def test_a_project_that_turned_tracing_off_is_not_asked_to_trace_toward_an_untruassert exited.value.code == 1assert usage == ["cli_usage:eval_stopped:untrusted_amp"]assert "CREWAI_TRACING_ENABLED=false" in (directory / ".env").read_text()+++# ── with no terminal, the run's own output goes to a log, not the agent ────+++def _a_tracing_project(directory: Path) -> None:+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ (directory / ".env").write_text("CREWAI_TRACING_ENABLED=true\n")+++def test_with_no_terminal_the_runs_own_output_goes_to_a_log_not_the_agent(+ project, monkeypatch, capfd+):+ """A coding agent reads every line `crewai eval` prints, and pays for it:+ the crew's panels, its final answer and its tool calls go to+ `.crewai/last_run.log`, and the agent gets one line saying so, then the+ verdict."""+ directory, _ = project+ _a_tracing_project(directory)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)++ def fake_run_crew() -> None:+ # what the run's child processes print: straight to the descriptors+ os.write(1, b"Crew Execution Started panel\n")+ os.write(2, b"a warning on stderr\n")+ record_last_run(directory, "the-new-run", recorded_at=_now())++ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ out, err = capfd.readouterr()+ log = (directory / ".crewai" / "last_run.log").read_text()+ assert "Crew Execution Started panel" in log and "a warning on stderr" in log+ assert "Crew Execution Started panel" not in out + err+ assert "a warning on stderr" not in out + err+ said = " ".join(out.split())+ assert "The crew's output goes to .crewai/last_run.log while it runs." in said+ assert "its output is in .crewai/last_run.log." in said+ assert "Goal gate: PASSED" in said+++def test_with_no_terminal_a_run_that_left_no_trace_shows_its_last_lines(+ project, monkeypatch, capfd, usage+):+ """The error a failed run printed is what the agent needs to fix it: its+ last lines are shown, and the sentence says where the rest is."""+ directory, _ = project+ _a_tracing_project(directory)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)++ def fake_run_crew() -> None:+ os.write(1, b"".join(f"line {n}\n".encode() for n in range(100)))+ os.write(2, b"KeyError: 'topic'\n") # and no trace recorded++ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ amp = install(monkeypatch, FakeAMP())++ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew()++ assert exited.value.code == 1+ said = " ".join(capfd.readouterr().out.split())+ assert "its last 40 lines:" in said+ assert "KeyError: 'topic'" in said and "line 99" in said+ assert "line 50 " not in said # only the tail+ assert "(its last lines are above)" in said+ assert amp.calls == [] and usage[-1] == "cli_usage:eval_stopped:untraced_after_run"+++def test_at_a_terminal_the_run_prints_where_it_always_did(project, monkeypatch, capfd):+ directory, _ = project+ _a_tracing_project(directory)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: False) # a person reads stdout++ def fake_run_crew() -> None:+ os.write(1, b"Crew Execution Started panel\n")+ record_last_run(directory, "the-new-run", recorded_at=_now())++ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ out, _ = capfd.readouterr()+ assert "Crew Execution Started panel" in out+ assert "last_run.log" not in out+ assert not (directory / ".crewai" / "last_run.log").exists()+++def test_what_decides_is_whether_anyone_reads_stdout_not_stdin(project, monkeypatch, capfd):+ """A coding agent can hand the command a terminal for stdin and still read+ a captured stdout: the transcript stays out of what it reads."""+ directory, _ = project+ _a_tracing_project(directory)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)++ def fake_run_crew() -> None:+ os.write(1, b"Crew Execution Started panel\n")+ record_last_run(directory, "the-new-run", recorded_at=_now())++ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert "Crew Execution Started panel" not in capfd.readouterr().out+ assert "Crew Execution Started panel" in (directory / ".crewai" / "last_run.log").read_text()++[email protected](os.name == "nt", reason="POSIX file modes")+def test_the_run_log_is_the_owners_alone_even_one_left_by_an_earlier_run(tmp_path):+ log = tmp_path / "last_run.log"+ log.write_text("an earlier run\n")+ log.chmod(0o644)++ with eval_module._run_output_to(log) as where:+ assert where == log+ os.write(1, b"this run\n")++ assert log.read_text() == "this run\n" # emptied, then written+ assert log.stat().st_mode & 0o777 == 0o600++[email protected](os.name == "nt", reason="symbolic links need privileges on Windows")+def test_a_link_at_the_logs_place_leaves_the_output_on_screen(tmp_path, capfd):+ elsewhere = tmp_path / "elsewhere.txt"+ elsewhere.write_text("")+ log = tmp_path / "last_run.log"+ log.symlink_to(elsewhere)++ with eval_module._run_output_to(log) as where:+ assert where is None+ os.write(1, b"on screen\n")++ assert "on screen" in capfd.readouterr().out+ assert elsewhere.read_text() == ""+++def test_a_move_that_fails_halfway_puts_back_what_it_moved(tmp_path, monkeypatch, capfd):+ """Descriptor 1 moved, 2 refused: 1 comes back before the block runs, so+ the verdict printed after it still reaches the screen."""+ real_dup2 = os.dup2+ targets: list[int] = []++ def dup2(src: int, dst: int, inheritable: bool = True) -> int:+ targets.append(dst)+ if dst == 2 and len(targets) == 2:+ raise OSError("no room for stderr")+ return real_dup2(src, dst, inheritable)++ with monkeypatch.context() as patched:+ patched.setattr(eval_module.os, "dup2", dup2)+ with eval_module._run_output_to(tmp_path / "last_run.log") as where:+ assert where is None+ os.write(1, b"the verdict\n")++ assert "the verdict" in capfd.readouterr().out+ assert (tmp_path / "last_run.log").read_text() == ""+++def test_putting_back_goes_on_past_a_failure_and_keeps_the_runs_own_error(tmp_path, monkeypatch, capfd):+ """Putting stdout back fails; stderr is put back anyway, every saved copy+ is closed, and the error the run raised is the one that surfaces."""+ real_dup2 = os.dup2+ stdout_copy = os.dup(1)+ calls: list[int] = []++ def dup2(src: int, dst: int, inheritable: bool = True) -> int:+ calls.append(dst)+ if len(calls) == 3: # the first put-back: stdout+ raise OSError("cannot put stdout back")+ return real_dup2(src, dst, inheritable)++ def the_run() -> None:+ raise ValueError("the run's own error")++ try:+ with monkeypatch.context() as patched:+ patched.setattr(eval_module.os, "dup2", dup2)+ with pytest.raises(ValueError, match="the run's own error"):+ with eval_module._run_output_to(tmp_path / "last_run.log"):+ the_run()+ assert calls == [1, 2, 1, 2] # stderr was put back after stdout failed+ os.write(2, b"stderr is back\n")+ assert "stderr is back" in capfd.readouterr().err+ finally:+ real_dup2(stdout_copy, 1) # the test puts stdout back itself+ os.close(stdout_copy)