Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(cli): with no terminal, crewai eval sends the crew's own output to a log (#8325)

João Moura94508c971f02
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 87718918c..b7abe2cff 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -43,7 +43,7 @@ import re
import stat
import sys
import time
-from typing import Any, Literal, NoReturn, cast
+from typing import Any, BinaryIO, Literal, NoReturn, cast
from urllib.parse import urlparse
import uuid
import webbrowser
@@ -161,6 +161,13 @@ NOT_TRACED = (
f"({LAST_RUN_FILE})."
)
+# With no terminal, what the crew prints while it runs (its panels, the final
+# answer, every tool call) goes to this file in the project's `.crewai/`, not to
+# the coding agent's screen: in one measured run it was 341 of 378 lines, all
+# before the verdict, and an agent pays for every line or has the end cut off.
+RUN_LOG_FILE = "last_run.log"
+RUN_LOG_TAIL_LINES = 40
+
# Why `crewai eval` ran the crew before evaluating, a word from a closed list:
# nothing traced yet, the project changed since its last traced run, or the run
# app's Evaluate pressed on a run that was not traced.
@@ -1002,11 +1009,23 @@ def _run_the_crew_to_evaluate(
# The handshake (`crewai_core.eval_handshake`) carries what this command
# decided before the project's `.env` was loaded: the AMP origins its saved
# login may go to, and whether it just turned tracing on.
+ # Whoever reads the output decides, as for the brief: stdout, not stdin.
+ run_log = _run_log_path() if nobody_watching() else None
+ if run_log is not None:
+ console.print(
+ f"The crew's output goes to {_shown_path(run_log)} while it runs.",
+ style="dim",
+ )
began = datetime.now(timezone.utc)
- with evaluating_after_run(
- trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now
- ) as watched:
+ started = time.monotonic()
+ with (
+ evaluating_after_run(
+ trusted_amp_origins=trusted or (), tracing_turned_on=turned_on_now
+ ) as watched,
+ _run_output_to(run_log) as run_log,
+ ):
run_crew(**run_args)
+ seconds = time.monotonic() - started
# The app says what it did with the run it watched, and its word settles
# this: it evaluated it (nothing left to do), or it had nothing to evaluate
@@ -1018,7 +1037,8 @@ def _run_the_crew_to_evaluate(
remember_project_state(str(marker["execution_id"]))
return None
record_stop("untraced_after_run")
- console.print(NOT_TRACED, style="bold red")
+ _say_what_the_run_left(run_log, seconds, traced=False)
+ console.print(_not_traced(run_log), style="bold red")
raise SystemExit(1)
# The project's record, only when it was written after this run began: a
@@ -1029,13 +1049,157 @@ def _run_the_crew_to_evaluate(
record.get("execution_id") if _recorded_since(record, began) else None
)
if traced:
+ _say_what_the_run_left(run_log, seconds, traced=True)
return str(traced)
record_stop("untraced_after_run")
- console.print(NOT_TRACED, style="bold red")
+ _say_what_the_run_left(run_log, seconds, traced=False)
+ console.print(_not_traced(run_log), style="bold red")
raise SystemExit(1)
+def _not_traced(run_log: Path | None) -> str:
+ """NOT_TRACED, pointing at where the run's output went."""
+ if run_log is None:
+ return NOT_TRACED
+ return NOT_TRACED.replace("(its output is above)", "(its last lines are above)")
+
+
+def _run_log_path() -> Path | None:
+ """Where a run with no terminal sends its output, or None when the
+ project's `.crewai/` cannot be written (the output then stays on screen)."""
+ from crewai.telemetry.tracing.last_run import LAST_RUN_DIR, project_dir
+
+ try:
+ path = project_dir() / LAST_RUN_DIR / RUN_LOG_FILE
+ path.parent.mkdir(parents=True, exist_ok=True)
+ except OSError:
+ return None
+ return path
+
+
+def _run_output_to(path: Path | None) -> Iterator[Path | None]:
+ """Send everything printed inside the block to PATH, and yield it; with no
+ PATH, yield None and change nothing.
+
+ It moves this process's own stdout and stderr (file descriptors 1 and 2),
+ which every child the run starts inherits, so each way a project runs —
+ `uv run`, the JSON crew runner, a declarative flow — is covered without
+ touching any of them. Anything that keeps them from moving leaves the
+ output where it was: a quiet log is a nicety, never a failed run. Both are
+ always put back after, one by one, whatever failed: a stream left on the
+ log would swallow the verdict, and the run's own error is never hidden by
+ a failure to put them back."""
+ log = _open_run_log(path) if path is not None else None
+ if log is None:
+ yield None
+ return
+ with log:
+ _flush_std_streams()
+ saved: dict[int, int] = {}
+ moved: list[int] = []
+ try:
+ for fd in (1, 2):
+ saved[fd] = os.dup(fd)
+ for fd in (1, 2):
+ os.dup2(log.fileno(), fd)
+ moved.append(fd)
+ except OSError:
+ _put_back(saved, moved)
+ yield None
+ return
+ _point_windows_std_handles_at(1, 2)
+ try:
+ yield path
+ finally:
+ _flush_std_streams()
+ _put_back(saved, moved)
+ _point_windows_std_handles_at(1, 2)
+
+
+def _open_run_log(path: Path) -> BinaryIO | None:
+ """The run log, opened for writing owner-only and never through a link:
+ it holds final answers, tool output and tracebacks that can carry a
+ credential. A log an earlier run left is emptied and its mode tightened.
+ None when it cannot be opened that way."""
+ flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)
+ try:
+ if path.is_symlink():
+ return None # a link someone planted does not choose where it goes
+ fd = os.open(path, flags, 0o600)
+ except OSError:
+ return None
+ with contextlib.suppress(OSError, AttributeError):
+ os.fchmod(fd, 0o600)
+ return os.fdopen(fd, "wb")
+
+
+def _flush_std_streams() -> None:
+ for stream in (sys.stdout, sys.stderr):
+ with contextlib.suppress(Exception):
+ stream.flush()
+
+
+def _put_back(saved: dict[int, int], moved: list[int]) -> None:
+ """Point every MOVED descriptor at its SAVED original again, and close
+ every saved copy: each on its own, so one failure stops none of the rest."""
+ for fd in moved:
+ with contextlib.suppress(OSError):
+ os.dup2(saved[fd], fd)
+ for copy in saved.values():
+ _close_quietly(copy)
+
+
+def _close_quietly(fd: int) -> None:
+ with contextlib.suppress(OSError):
+ os.close(fd)
+
+
+def _point_windows_std_handles_at(out_fd: int, err_fd: int) -> None:
+ """On Windows a child takes the process's standard handles, not its file
+ descriptors: point them where descriptors 1 and 2 now go."""
+ if os.name != "nt":
+ return
+ with contextlib.suppress(Exception):
+ import ctypes
+ import msvcrt
+
+ kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
+ handle_of = msvcrt.get_osfhandle # type: ignore[attr-defined]
+ kernel32.SetStdHandle(-11, handle_of(out_fd)) # STD_OUTPUT_HANDLE
+ kernel32.SetStdHandle(-12, handle_of(err_fd)) # STD_ERROR_HANDLE
+
+
+def _say_what_the_run_left(log: Path | None, seconds: float, traced: bool) -> None:
+ """After a quiet run: where its output is, and — when it left nothing to
+ evaluate — its last lines, which usually hold the error."""
+ if log is None:
+ return
+ where = _shown_path(log)
+ if traced:
+ console.print(
+ f"The crew ran in {seconds:.0f}s; its output is in {where}.", style="dim"
+ )
+ return
+ with contextlib.suppress(OSError):
+ lines = log.read_text(encoding="utf-8", errors="replace").splitlines()
+ tail = lines[-RUN_LOG_TAIL_LINES:]
+ if tail:
+ console.print(
+ f"The crew's output is in {where}; its last {len(tail)} lines:",
+ style="bold",
+ )
+ console.print(Text("\n".join(tail)))
+
+
+def _shown_path(path: Path) -> str:
+ try:
+ return str(path.relative_to(Path.cwd()))
+ except ValueError:
+ return str(path)
+
+
def _recorded_since(record: dict[str, Any], began: datetime) -> bool:
"""Was RECORD written after BEGAN? A record without a readable, zoned stamp
is not assumed to be this run's."""
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index 4d90d7d6b..dd6768440 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -3458,3 +3458,197 @@ def test_a_project_that_turned_tracing_off_is_not_asked_to_trace_toward_an_untru
assert exited.value.code == 1
assert usage == ["cli_usage:eval_stopped:untrusted_amp"]
assert "CREWAI_TRACING_ENABLED=false" in (directory / ".env").read_text()
+
+
+# ── with no terminal, the run's own output goes to a log, not the agent ────
+
+
+def _a_tracing_project(directory: Path) -> None:
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ (directory / ".env").write_text("CREWAI_TRACING_ENABLED=true\n")
+
+
+def test_with_no_terminal_the_runs_own_output_goes_to_a_log_not_the_agent(
+ project, monkeypatch, capfd
+):
+ """A coding agent reads every line `crewai eval` prints, and pays for it:
+ the crew's panels, its final answer and its tool calls go to
+ `.crewai/last_run.log`, and the agent gets one line saying so, then the
+ verdict."""
+ directory, _ = project
+ _a_tracing_project(directory)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
+
+ def fake_run_crew() -> None:
+ # what the run's child processes print: straight to the descriptors
+ os.write(1, b"Crew Execution Started panel\n")
+ os.write(2, b"a warning on stderr\n")
+ record_last_run(directory, "the-new-run", recorded_at=_now())
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ out, err = capfd.readouterr()
+ log = (directory / ".crewai" / "last_run.log").read_text()
+ assert "Crew Execution Started panel" in log and "a warning on stderr" in log
+ assert "Crew Execution Started panel" not in out + err
+ assert "a warning on stderr" not in out + err
+ said = " ".join(out.split())
+ assert "The crew's output goes to .crewai/last_run.log while it runs." in said
+ assert "its output is in .crewai/last_run.log." in said
+ assert "Goal gate: PASSED" in said
+
+
+def test_with_no_terminal_a_run_that_left_no_trace_shows_its_last_lines(
+ project, monkeypatch, capfd, usage
+):
+ """The error a failed run printed is what the agent needs to fix it: its
+ last lines are shown, and the sentence says where the rest is."""
+ directory, _ = project
+ _a_tracing_project(directory)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: False)
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
+
+ def fake_run_crew() -> None:
+ os.write(1, b"".join(f"line {n}\n".encode() for n in range(100)))
+ os.write(2, b"KeyError: 'topic'\n") # and no trace recorded
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ amp = install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit) as exited:
+ eval_module.eval_crew()
+
+ assert exited.value.code == 1
+ said = " ".join(capfd.readouterr().out.split())
+ assert "its last 40 lines:" in said
+ assert "KeyError: 'topic'" in said and "line 99" in said
+ assert "line 50 " not in said # only the tail
+ assert "(its last lines are above)" in said
+ assert amp.calls == [] and usage[-1] == "cli_usage:eval_stopped:untraced_after_run"
+
+
+def test_at_a_terminal_the_run_prints_where_it_always_did(project, monkeypatch, capfd):
+ directory, _ = project
+ _a_tracing_project(directory)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: False) # a person reads stdout
+
+ def fake_run_crew() -> None:
+ os.write(1, b"Crew Execution Started panel\n")
+ record_last_run(directory, "the-new-run", recorded_at=_now())
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ out, _ = capfd.readouterr()
+ assert "Crew Execution Started panel" in out
+ assert "last_run.log" not in out
+ assert not (directory / ".crewai" / "last_run.log").exists()
+
+
+def test_what_decides_is_whether_anyone_reads_stdout_not_stdin(project, monkeypatch, capfd):
+ """A coding agent can hand the command a terminal for stdin and still read
+ a captured stdout: the transcript stays out of what it reads."""
+ directory, _ = project
+ _a_tracing_project(directory)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(eval_module, "nobody_watching", lambda: True)
+
+ def fake_run_crew() -> None:
+ os.write(1, b"Crew Execution Started panel\n")
+ record_last_run(directory, "the-new-run", recorded_at=_now())
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert "Crew Execution Started panel" not in capfd.readouterr().out
+ assert "Crew Execution Started panel" in (directory / ".crewai" / "last_run.log").read_text()
+
+
[email protected](os.name == "nt", reason="POSIX file modes")
+def test_the_run_log_is_the_owners_alone_even_one_left_by_an_earlier_run(tmp_path):
+ log = tmp_path / "last_run.log"
+ log.write_text("an earlier run\n")
+ log.chmod(0o644)
+
+ with eval_module._run_output_to(log) as where:
+ assert where == log
+ os.write(1, b"this run\n")
+
+ assert log.read_text() == "this run\n" # emptied, then written
+ assert log.stat().st_mode & 0o777 == 0o600
+
+
[email protected](os.name == "nt", reason="symbolic links need privileges on Windows")
+def test_a_link_at_the_logs_place_leaves_the_output_on_screen(tmp_path, capfd):
+ elsewhere = tmp_path / "elsewhere.txt"
+ elsewhere.write_text("")
+ log = tmp_path / "last_run.log"
+ log.symlink_to(elsewhere)
+
+ with eval_module._run_output_to(log) as where:
+ assert where is None
+ os.write(1, b"on screen\n")
+
+ assert "on screen" in capfd.readouterr().out
+ assert elsewhere.read_text() == ""
+
+
+def test_a_move_that_fails_halfway_puts_back_what_it_moved(tmp_path, monkeypatch, capfd):
+ """Descriptor 1 moved, 2 refused: 1 comes back before the block runs, so
+ the verdict printed after it still reaches the screen."""
+ real_dup2 = os.dup2
+ targets: list[int] = []
+
+ def dup2(src: int, dst: int, inheritable: bool = True) -> int:
+ targets.append(dst)
+ if dst == 2 and len(targets) == 2:
+ raise OSError("no room for stderr")
+ return real_dup2(src, dst, inheritable)
+
+ with monkeypatch.context() as patched:
+ patched.setattr(eval_module.os, "dup2", dup2)
+ with eval_module._run_output_to(tmp_path / "last_run.log") as where:
+ assert where is None
+ os.write(1, b"the verdict\n")
+
+ assert "the verdict" in capfd.readouterr().out
+ assert (tmp_path / "last_run.log").read_text() == ""
+
+
+def test_putting_back_goes_on_past_a_failure_and_keeps_the_runs_own_error(tmp_path, monkeypatch, capfd):
+ """Putting stdout back fails; stderr is put back anyway, every saved copy
+ is closed, and the error the run raised is the one that surfaces."""
+ real_dup2 = os.dup2
+ stdout_copy = os.dup(1)
+ calls: list[int] = []
+
+ def dup2(src: int, dst: int, inheritable: bool = True) -> int:
+ calls.append(dst)
+ if len(calls) == 3: # the first put-back: stdout
+ raise OSError("cannot put stdout back")
+ return real_dup2(src, dst, inheritable)
+
+ def the_run() -> None:
+ raise ValueError("the run's own error")
+
+ try:
+ with monkeypatch.context() as patched:
+ patched.setattr(eval_module.os, "dup2", dup2)
+ with pytest.raises(ValueError, match="the run's own error"):
+ with eval_module._run_output_to(tmp_path / "last_run.log"):
+ the_run()
+ assert calls == [1, 2, 1, 2] # stderr was put back after stdout failed
+ os.write(2, b"stderr is back\n")
+ assert "stderr is back" in capfd.readouterr().err
+ finally:
+ real_dup2(stdout_copy, 1) # the test puts stdout back itself
+ os.close(stdout_copy)