Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

fix(tracing): turning tracing on is the answer, and an Evaluate button in the TUI (#7765)

João Moura4dcd19aa2ef9
diff --git a/lib/cli/src/crewai_cli/crew_run_tui.py b/lib/cli/src/crewai_cli/crew_run_tui.py
index 3aad179b4..1ceeda579 100644
--- a/lib/cli/src/crewai_cli/crew_run_tui.py
+++ b/lib/cli/src/crewai_cli/crew_run_tui.py
@@ -5,8 +5,15 @@ Two-column layout: left sidebar (tasks/agents/tokens) + main content
"""
import asyncio
+from collections.abc import Callable, Iterator
+import contextlib
+from contextlib import contextmanager
+from contextvars import ContextVar
import json as _json
+import os
+from pathlib import Path
import re
+import secrets
import threading
import time
from typing import Any, ClassVar
@@ -289,6 +296,116 @@ class TraceConsentScreen(ModalScreen[bool]):
self.dismiss(False)
+_AUTO_EVAL: ContextVar[dict[str, str | None] | None] = ContextVar(
+ "crewai_tui_auto_eval", default=None
+)
+
+# The same word, for the app that runs in a CHILD process. A project's crew and
+# flow are run through `uv run …` in the project's own environment, and nothing
+# in this process's memory reaches that app — the environment does, and it is
+# the only thing that does. Set and cleared by `evaluating_after_run()` alone:
+# an internal handshake between two parts of one command, never a setting for
+# anyone to turn on.
+#
+# The child loads the project's `.env` over its environment, so the variable
+# alone would be a switch any project could flip — an evaluation started, with
+# the machine's `crewai login`, on a plain `crewai run`. Its value is therefore a
+# random token, and it counts only while the file of that name exists in this
+# user's crewAI data directory: the command writes it before the run and removes
+# it after, and a `.env` cannot create a file.
+_AWAITING_EVAL_ENV = "CREWAI_EVAL_AWAITING_RUN"
+_AWAITING_TOKEN = re.compile(r"[0-9a-f]{32}")
+
+
+def _awaiting_dir() -> Path:
+ import appdirs
+
+ return Path(appdirs.user_data_dir("crewai", "CrewAI")) / "eval-awaiting"
+
+
+@contextmanager
+def evaluating_after_run() -> Iterator[dict[str, str | None]]:
+ """Run the crew for an evaluation that is already under way.
+
+ `crewai eval` with nothing traced offers to run the crew first. The app it
+ opens would otherwise sit there until somebody quits it, with the command
+ waiting behind — so inside this block the app closes itself when the run
+ ends and leaves its execution id in the holder. It does NOT chain into an
+ evaluation of its own: the command that opened it is the one evaluating.
+
+ A run in the same process leaves its id here; a run in a child process
+ leaves it in the project's last-run record, which the caller reads when the
+ holder comes back empty.
+ """
+ holder: dict[str, str | None] = {"execution_id": None}
+ token = _AUTO_EVAL.set(holder)
+ before = os.environ.get(_AWAITING_EVAL_ENV)
+ # Without the file a child app cannot be told, and the command grades the
+ # run itself once the app closes — slower to the verdict, never wrong.
+ marker: Path | None = None
+ with contextlib.suppress(OSError):
+ nonce = secrets.token_hex(16)
+ _awaiting_dir().mkdir(parents=True, exist_ok=True)
+ (_awaiting_dir() / nonce).touch(exist_ok=False)
+ marker = _awaiting_dir() / nonce
+ os.environ[_AWAITING_EVAL_ENV] = nonce
+ try:
+ yield holder
+ finally:
+ _AUTO_EVAL.reset(token)
+ if marker is not None:
+ with contextlib.suppress(OSError):
+ marker.unlink()
+ if before is None:
+ os.environ.pop(_AWAITING_EVAL_ENV, None)
+ else:
+ os.environ[_AWAITING_EVAL_ENV] = before
+
+
+def _an_evaluation_is_waiting() -> dict[str, str | None] | None:
+ """The evaluation this run was started for, if there is one.
+
+ In this process the holder itself; in a child process the environment says
+ an evaluation is waiting and the holder is a local one — the id travels
+ back through the project's last-run record instead, which the child writes
+ when its trace is exported.
+ """
+ holder = _AUTO_EVAL.get()
+ if holder is not None:
+ return holder
+
+ value = os.environ.get(_AWAITING_EVAL_ENV) or ""
+ if not _AWAITING_TOKEN.fullmatch(value):
+ return None
+ try:
+ waiting = (_awaiting_dir() / value).is_file()
+ except OSError:
+ return None
+ return {"execution_id": None} if waiting else None
+
+
+def _recorded_run(execution_uuid: str) -> dict[str, Any] | None:
+ """This execution's own record, if its trace reached AMP.
+
+ crewAI records a run when its spans are exported, so the record answers the
+ one question the uuid alone cannot: whether there is anything to grade, and
+ where AMP will show it. It is matched by id rather than trusted as "the
+ last run", which it is only until another run in the project finishes.
+ """
+ # a missing or unreadable record simply means "not traced"
+ with contextlib.suppress(Exception):
+ from crewai.telemetry.tracing.last_run import read_last_run
+
+ record = read_last_run() or {}
+ if str(record.get("execution_id") or "") == execution_uuid:
+ return record
+ return None
+
+
+def _trace_was_recorded(execution_uuid: str) -> bool:
+ return _recorded_run(execution_uuid) is not None
+
+
class CrewRunApp(App[Any]):
TITLE = "CrewAI"
@@ -408,16 +525,25 @@ FooterKey .footer-key--key {
text-style: bold;
}
-#btn-traces {
+#btn-eval {
background: #1F7982;
color: #e0e0e0;
border: none;
}
-#btn-traces:hover {
+#btn-eval:hover {
background: #28969f;
}
+
+#btn-traces {
+ background: #2b2b2b;
+ color: #e0e0e0;
+ border: none;
+}
+#btn-traces:hover {
+ background: #3d3d3d;
+}
#btn-traces:disabled {
- background: #1a4a50;
+ background: #202020;
color: #888888;
}
@@ -437,6 +563,7 @@ FooterKey .footer-key--key {
Binding("s", "toggle_sidebar", "Sidebar"),
Binding("l", "toggle_logs", "Logs"),
Binding("t", "view_traces", "Traces", show=False),
+ Binding("e", "evaluate_crew", "Evaluate", show=False),
Binding("d", "deploy_crew", "Deploy", show=False),
Binding("down", "log_down", "Log ↓", show=False),
Binding("up", "log_up", "Log ↑", show=False),
@@ -529,6 +656,19 @@ FooterKey .footer-key--key {
self._current_method: str | None = None
self._elapsed_frozen: float | None = None
self._want_deploy: bool = False
+ # This app's own execution, captured from the run while it runs. The
+ # project-wide record is the last run to FINISH anywhere, which another
+ # run in the same project can replace between this one ending and its
+ # button being pressed.
+ self._execution_uuid: str | None = None
+ # The evaluation of this run, while it happens and after: it runs from
+ # inside the app, on the app's own screen, rather than sending anybody
+ # back to a terminal to watch a link appear.
+ self._evaluation: dict[str, Any] | None = None
+ # Read here, on the thread that built the app, because that is where
+ # `crewai eval` set it — or in the environment it passed to this
+ # process, when the run is a child of that command.
+ self._auto_eval: dict[str, str | None] | None = _an_evaluation_is_waiting()
self._consent_screen: TraceConsentScreen | None = None
self._trace_consent_pending: threading.Event | None = None
self._discard_trace_on_exit = False
@@ -556,6 +696,7 @@ FooterKey .footer-key--key {
with VerticalScroll(id="sidebar"):
yield Static(id="sidebar-content")
with Vertical(id="sidebar-actions"):
+ yield Button("Evaluate", id="btn-eval", classes="action-btn")
yield Button("View Traces", id="btn-traces", classes="action-btn")
yield Button("Deploy", id="btn-deploy", classes="action-btn")
with Vertical(id="main-panel"):
@@ -746,6 +887,41 @@ FooterKey .footer-key--key {
self._tick_timer.stop()
self._tick_timer = self.set_interval(1 / 2, self._tick)
self._unsubscribe_if_no_running_memory_save(wait_for_queued=True)
+ self._evaluate_if_one_is_waiting()
+
+ def _evaluate_if_one_is_waiting(self) -> None:
+ """Start the evaluation `crewai eval` opened this run for.
+
+ The evaluation is the errand; the run was the prerequisite. It begins
+ here, on this screen, rather than by closing the app and leaving
+ somebody in a bare terminal watching for a link: the app has the room
+ to show the link, the progress and the verdict, and the reader leaves
+ when they are done reading, not when the command is.
+ """
+ if self._auto_eval is None or self._evaluation is not None:
+ return
+
+ traced = self._traced_execution_id()
+ self._auto_eval["execution_id"] = traced
+ if traced is None:
+ # Said to the command waiting behind this screen as well: it cannot
+ # see this notice, and "no marker" is how it tells a run no app got
+ # to from a run an app looked at and found nothing to grade.
+ self._say_what_became_of_the_evaluation(None)
+ self.notify(
+ "This run was not traced, so there is nothing to evaluate.",
+ title="Evaluate",
+ severity="warning",
+ )
+ return
+
+ self._start_evaluation(traced)
+
+ def _say_what_became_of_the_evaluation(self, execution_id: str | None) -> None:
+ with contextlib.suppress(Exception):
+ from crewai_cli.experimental.eval_crew import record_evaluation_outcome
+
+ record_evaluation_outcome(execution_id)
def _on_crew_failed(self, error: str) -> None:
with self._lock:
@@ -772,6 +948,10 @@ FooterKey .footer-key--key {
self._tick_timer.stop()
self._tick_timer = self.set_interval(1 / 2, self._tick)
self._unsubscribe_if_no_running_memory_save(wait_for_queued=True)
+ # A run that failed is still a run to grade — where it went wrong is
+ # what somebody asking for an evaluation wants to know. If nothing was
+ # traced, the same notice says so and nothing is asked of AMP.
+ self._evaluate_if_one_is_waiting()
# ── Conversational flow execution ───────────────────────
@@ -996,9 +1176,16 @@ FooterKey .footer-key--key {
self.exit(self._crew_result)
def _request_trace_consent(self) -> bool:
- """Wait in the execution worker while the UI asks for upload consent."""
+ """Wait in the execution worker while the UI asks for upload consent.
+
+ Unless the user already answered: turning tracing on — in the project's
+ .env, or with ``tracing=True`` — is the yes, and a modal at the end of
+ the run would be the same question a second time.
+ """
if self._discard_trace_on_exit:
return False
+ if self._tracing_was_asked_for():
+ return True
done = threading.Event()
decision: list[bool] = []
self._trace_consent_pending = done
@@ -1028,15 +1215,84 @@ FooterKey .footer-key--key {
self.call_from_thread(self._dismiss_consent_modal)
def action_view_traces(self) -> None:
+ """Open AMP's view of this run's traces.
+
+ crewAI prints that link after a run, but never under a TUI — its
+ console is kept out of this layout — so the link reaches here the way
+ everything else about the run does: through the record crewAI writes
+ when the spans are exported, matched to THIS app's execution.
+ """
if self._status != "completed":
return
# Recorded here rather than in on_button_pressed so the `t` key binding
# is counted too, and only once the action can actually do something.
self._record_tui_button_click("view_traces")
+ record = (
+ _recorded_run(self._execution_uuid) if self._execution_uuid else None
+ ) or {}
+ url = str(record.get("trace_url") or "")
+ if url:
+ self._open_report(url)
+ self.notify(url, title="Execution traces")
+ return
+
self.notify(
- "Trace sharing is requested when the execution finishes. "
- "A trace link is not available for this run.",
+ "This run was not traced, so there are no traces to view. "
+ "Turn tracing on and run it again."
+ if not record
+ else "This run was traced, but AMP granted no link to view it.",
title="Execution traces",
+ severity="warning",
+ )
+
+ def _capture_execution_uuid(self) -> None:
+ """Remember which execution this app is watching.
+
+ Read as the run announces itself, from the run's own context, and kept:
+ the uuid is gone by the time the kickoff returns, and the project's
+ record cannot say which run was THIS one.
+ """
+ if self._execution_uuid is not None:
+ return
+ try:
+ from crewai.execution import get_execution_uuid
+
+ self._execution_uuid = get_execution_uuid()
+ except Exception: # a run without tracing simply has no execution
+ self._execution_uuid = None
+
+ def _tracing_was_asked_for(self) -> bool:
+ """Did somebody turn tracing on for this run?
+
+ Read here rather than through `tracing_asked_for()`, because the
+ ContextVar that helper reads is set where the crew is CONSTRUCTED — not
+ this worker thread — and a flow never sets it at all. So the declaration
+ is read off the object itself.
+
+ Only ONE of the helper's guards is dropped: it refuses while tracing
+ messages are suppressed, which this app does deliberately to keep
+ crewAI's console out of its own layout, and that guard is about a prompt
+ nobody would see. The others hold. A suite under test, or a TUI with no
+ interactive user behind it — embedded, redirected — has nobody whose yes
+ this could be, and gets the modal's own fail-closed answer instead.
+ """
+ import os
+
+ from crewai.events.listeners.tracing.utils import (
+ _is_interactive_terminal,
+ _is_test_environment,
+ )
+
+ if _is_test_environment() or not _is_interactive_terminal():
+ return False
+
+ if os.getenv("CREWAI_TRACING_ENABLED", "").lower() in ("true", "1"):
+ return True
+
+ return any(
+ getattr(target, "tracing", None) is True
+ for target in (self._crew, self._flow)
+ if target is not None
)
def _dismiss_consent_modal(self) -> None:
@@ -1057,6 +1313,211 @@ FooterKey .footer-key--key {
self._unsubscribe()
self.exit(self._crew_result)
+ def _traced_execution_id(self) -> str | None:
+ """This app's own execution, if it was traced.
+
+ Taken from the session the run itself opened, not from the project's
+ last-run record: that record is whichever run finished last anywhere in
+ the project, and a second TUI in the same project can replace it between
+ this run ending and its button being pressed. A run with no session was
+ not traced, and there is nothing to grade.
+ """
+ if self._execution_uuid is None:
+ return None
+
+ return (
+ self._execution_uuid if _trace_was_recorded(self._execution_uuid) else None
+ )
+
+ def action_evaluate_crew(self) -> None:
+ """Grade the run that just finished, here.
+
+ Unlike Deploy, this does not leave: the evaluation takes a few minutes,
+ it has a link worth showing and a verdict worth reading, and a person
+ sent back to a bare terminal to wait for them has been given the worst
+ of both. The report opens in the browser; the app keeps the link and
+ says what came back.
+ """
+ if self._status != "completed":
+ return
+
+ # An evaluation already in flight, or already answered: the button
+ # re-opens what it made rather than paying for a second one. One that
+ # STOPPED is a different matter — a timeout or a 502 is worth another
+ # press, and the run is still there to grade.
+ state = str((self._evaluation or {}).get("state"))
+ if self._evaluation is not None and state != "failed":
+ url = self._evaluation.get("url")
+ if url:
+ self._open_report(str(url))
+ return
+
+ self._record_tui_button_click("evaluate")
+ traced = self._traced_execution_id()
+ if traced is None:
+ self.notify(
+ "This run was not traced, so there is nothing to evaluate. "
+ "Turn tracing on and run it again.",
+ title="Evaluate",
+ severity="warning",
+ )
+ return
+
+ self._start_evaluation(traced)
+
+ def _start_evaluation(self, execution_id: str) -> None:
+ self._evaluation = {"state": "starting", "execution_id": execution_id}
+ # The run is over and the clock slowed to twice a second; something is
+ # moving again, so the screen moves with it.
+ self._set_tick_rate(1 / 8)
+ self._refresh_eval_button()
+ self._evaluate_worker(execution_id)
+
+ def _set_tick_rate(self, interval: float) -> None:
+ with contextlib.suppress(Exception):
+ self._tick_timer.stop()
+ self._tick_timer = self.set_interval(interval, self._tick)
+
+ @work(thread=True, exclusive=True, group="evaluation")
+ def _evaluate_worker(self, execution_id: str) -> None:
+ self._evaluate_now(execution_id)
+
+ def _evaluate_now(self, execution_id: str) -> None:
+ """AMP evaluates the run; this thread only carries the answers back."""
+ from crewai_cli.experimental.eval_crew import (
+ EvaluationStoppedError,
+ evaluate_run,
+ )
+
+ def back(handler: Callable[..., Any], *args: Any) -> None:
+ with contextlib.suppress(Exception): # the app may be gone by now
+ self.call_from_thread(handler, *args)
+
+ try:
+ finished = evaluate_run(
+ execution_id,
+ on_started=lambda started: back(self._evaluation_started, started),
+ on_status=lambda payload: back(self._evaluation_progress, payload),
+ note=lambda text: back(self._evaluation_note, text),
+ )
+ except EvaluationStoppedError as stopped:
+ back(self._evaluation_failed, str(stopped))
+ return
+ except SystemExit as exit_:
+ # Nothing on this path should exit any more — a stop is a value
+ # now — but an exit carries a code, not a reason, and "1" on screen
+ # is worse than saying plainly that the reason did not survive.
+ back(
+ self._evaluation_failed,
+ f"The evaluation stopped without saying why (exit {exit_.code}).",
+ )
+ return
+ except Exception as error: # a client bug is still an answer to show
+ back(self._evaluation_failed, f"{type(error).__name__}: {error}")
+ return
+ back(self._evaluation_finished, finished)
+
+ def _evaluation_started(self, started: dict[str, Any]) -> None:
+ if self._evaluation is None:
+ return
+ url = started.get("url")
+ self._evaluation.update(
+ {"state": "running", "id": started.get("id"), "url": url, "note": None}
+ )
+ self._refresh_eval_button()
+ if isinstance(url, str) and url:
+ self._open_report(url)
+
+ def _evaluation_progress(self, payload: dict[str, Any]) -> None:
+ """What the service says it is doing, in its own counting.
+
+ The report page counts the same way — one event per subject judged —
+ so a person watching both sees one number, not two.
+ """
+ if self._evaluation is None:
+ return
+ judged = planned = None
+ with contextlib.suppress(Exception):
+ events = payload.get("events") or []
+ judged = sum(
+ 1 for event in events if (event.get("data") or {}).get("subject")
+ )
+ planned = next(
+ (
+ (event.get("data") or {}).get("planned")
+ for event in reversed(events)
+ if (event.get("data") or {}).get("planned")
+ ),
+ None,
+ )
+ self._evaluation["note"] = (
+ f"judged {judged} of {planned}"
+ if judged and planned
+ else (f"judged {judged}" if judged else None)
+ )
+
+ def _evaluation_note(self, text: str) -> None:
+ if self._evaluation is not None:
+ self._evaluation["note"] = text
+
+ def _evaluation_finished(self, finished: dict[str, Any]) -> None:
+ if self._evaluation is None:
+ return
+ if finished.get("status") == "done":
+ self._evaluation.update(
+ {
+ "state": "done",
+ "verdict": finished.get("verdict") or {},
+ "note": None,
+ # the file the evaluation wrote, for the line printed once
+ # the app has gone (`_print_evaluation_line`)
+ "wrote_config": finished.get("wrote_eval_config"),
+ }
+ )
+ else:
+ self._evaluation.update(
+ {
+ "state": "failed",
+ "error": str(finished.get("error") or "no reason given"),
+ "note": None,
+ }
+ )
+ self._set_tick_rate(1 / 2)
+ self._refresh_eval_button()
+
+ def _evaluation_failed(self, message: str) -> None:
+ if self._evaluation is None:
+ return
+ self._evaluation.update({"state": "failed", "error": message, "note": None})
+ self._set_tick_rate(1 / 2)
+ self._refresh_eval_button()
+
+ def _open_report(self, url: str) -> None:
+ import webbrowser
+
+ with contextlib.suppress(Exception):
+ webbrowser.open(url)
+
+ def _evaluating(self) -> bool:
+ return (self._evaluation or {}).get("state") in ("starting", "running")
+
+ def _refresh_eval_button(self) -> None:
+ """The button while the evaluation runs: still teal, still pressable —
+ the report page is where the progress is — and spinning, because a
+ label that never moves reads as a screen that has stopped."""
+ labels = {
+ "done": "Open eval report",
+ "failed": "Evaluate",
+ }
+ with contextlib.suppress(Exception):
+ button = self.query_one("#btn-eval", Button)
+ state = str((self._evaluation or {}).get("state"))
+ button.label = (
+ f"{self._spinner()} Evaluating…"
+ if self._evaluating()
+ else labels.get(state, "Evaluate")
+ )
+
def _record_tui_button_click(self, button_name: str) -> None:
try:
if self._telemetry is None:
@@ -1069,6 +1530,8 @@ FooterKey .footer-key--key {
def on_button_pressed(self, event: Button.Pressed) -> None:
if event.button.id == "btn-traces":
self.action_view_traces()
+ elif event.button.id == "btn-eval":
+ self.action_evaluate_crew()
elif event.button.id == "btn-deploy":
self.action_deploy_crew()
@@ -1125,6 +1588,8 @@ FooterKey .footer-key--key {
try:
with self._lock:
+ if self._evaluating():
+ self._refresh_eval_button()
self._render_sidebar()
self._render_task_header()
self._render_main_content()
@@ -1255,6 +1720,46 @@ FooterKey .footer-key--key {
# ── Task header rendering ───────────────────────────────
+ def _append_evaluation(self, t: Text) -> None:
+ """The evaluation under the run's own line: what it is doing, where to
+ read it, and what it answered. Everything here came over the wire, so
+ it is appended to the Text and never parsed as markup."""
+ evaluation = self._evaluation
+ if not evaluation:
+ return
+
+ state = str(evaluation.get("state"))
+ url = str(evaluation.get("url") or "")
+ t.append("\n")
+ if state in ("starting", "running"):
+ t.append(f"{self._spinner()} ", style=_C_TEAL)
+ t.append("Evaluating this run", style=f"bold {_C_TEAL}")
+ note = evaluation.get("note")
+ if note:
+ t.append(f" {note}", style=_C_DIM)
+ elif state == "done":
+ verdict = evaluation.get("verdict") or {}
+ gate = str(verdict.get("gate") or "").upper()
+ mark, style = {
+ "PASSED": ("✔ ", f"bold {_C_GREEN}"),
+ "FAILED": ("✘ ", f"bold {_C_RED}"),
+ }.get(gate, ("• ", f"bold {_C_TEAL}"))
+ t.append(mark, style=style)
+ t.append(f"Goal gate {gate or 'graded'}", style=style)
+ grades = verdict.get("grades")
+ if isinstance(grades, dict):
+ for area, grade in grades.items():
+ t.append(f" {area} ", style=_C_DIM)
+ t.append(f"{grade if grade is not None else '—'}/5", style=_C_TEXT)
+ else:
+ t.append("✘ ", style=f"bold {_C_RED}")
+ t.append("Evaluation stopped", style=f"bold {_C_RED}")
+ t.append(f" {evaluation.get('error') or ''}", style=_C_DIM)
+
+ if url:
+ t.append("\n")
+ t.append(url, style=f"{_C_TEAL} underline")
+
def _render_task_header(self) -> None:
widget = self.query_one("#task-header", Static)
t = Text()
@@ -1289,11 +1794,13 @@ FooterKey .footer-key--key {
parts.append(f"↓{out:,}")
if parts:
t.append(f" {' '.join(parts)} tokens", style=_C_DIM)
+ self._append_evaluation(t)
elif self._status == "failed":
t.append("✘ ", style=f"bold {_C_RED}")
t.append("Failed", style=f"bold {_C_RED}")
if self._error:
t.append(f"\n{self._error[:120]}", style=_C_RED)
+ self._append_evaluation(t)
elif self._current_method:
paused = any(
s["name"] == self._current_method and s["status"] == "paused"
@@ -1338,12 +1845,14 @@ FooterKey .footer-key--key {
parts.append(f"↓{out:,}")
if parts:
t.append(f" {' '.join(parts)} tokens", style=_C_DIM)
+ self._append_evaluation(t)
elif self._status == "failed":
t.append("✘ ", style=f"bold {_C_RED}")
t.append("Failed", style=f"bold {_C_RED}")
if self._error:
t.append(f"\n{self._error[:120]}", style=_C_RED)
+ self._append_evaluation(t)
elif self._current_task_idx > 0:
t.append(
@@ -1977,6 +2486,7 @@ FooterKey .footer-key--key {
@crewai_event_bus.on(CrewKickoffStartedEvent)
def on_crew_started(source: Any, event: CrewKickoffStartedEvent) -> None:
+ self._capture_execution_uuid()
with self._lock:
# In flow mode the app is named for the flow; a nested crew's
# kickoff (a `call: crew` step) must not rename it.
@@ -1990,6 +2500,7 @@ FooterKey .footer-key--key {
# ── Declarative-flow method events → STEPS panel ────────
@crewai_event_bus.on(FlowStartedEvent)
def on_flow_started(source: Any, event: FlowStartedEvent) -> None:
+ self._capture_execution_uuid()
with self._lock:
self._status = "working"
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 6ef6b5869..5fcda3a29 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -14,7 +14,9 @@ traces. The command sends the saved `crewai login` when there is one.
from __future__ import annotations
+from collections.abc import Callable
import contextlib
+from datetime import datetime, timedelta, timezone
from ipaddress import ip_address
import json
import os
@@ -44,10 +46,65 @@ LAST_RUN_FILE = Path(".crewai") / "last_run.json"
TRACING_ENV_VAR = "CREWAI_TRACING_ENABLED"
POLL_SECONDS = 3.0
POLL_RETRIES = 5 # consecutive unreachable / 5xx polls before giving up; the evaluation keeps running
+
+# A run's spans reach AMP a little after the run ends — the exporter sends them
+# as the process closes and AMP has its own queue behind that. A command that
+# just watched the run would otherwise ask for a trace that is still in flight
+# and be told, correctly and uselessly, that there is nothing there. So a run
+# we know is fresh gets a wait; an id somebody typed does not, and fails at
+# once as it always has.
+SPANS_WAIT_SECONDS = 120.0
+SPANS_POLL_SECONDS = 5.0
+RUN_IS_FRESH_SECONDS = 600.0
FINISHED = {"done", "failed"}
STATUSES = {"queued", "running"} | FINISHED
+def _record_usage(*, logged_in: bool) -> None:
+ """Count an evaluation that is actually starting, and whether the caller was
+ logged in.
+
+ Usage stats are anonymous, so nothing that names the run or the account is
+ sent — no execution id, no organization, and nothing about the run's content.
+ Which runs were evaluated, and by whom, is AMP's record, not these stats'.
+
+ The TUI's button counts `cli_usage:evaluate` when it is pressed, so the
+ difference between that and `cli_usage:eval` is intent that never became an
+ evaluation.
+ """
+ try:
+ from crewai_core.telemetry import Telemetry
+
+ telemetry = Telemetry()
+ telemetry.set_tracer()
+ telemetry.feature_usage_span(
+ "cli_usage:eval",
+ {"authenticated": "true" if logged_in else "false"},
+ )
+ except Exception: # noqa: S110 - telemetry must never break a command
+ pass
+
+
+NOT_TRACED = (
+ "The run finished but no trace was recorded: the run may have failed, sharing the "
+ "trace was declined, or this project's crewai is older than the version that records "
+ f"the last run ({LAST_RUN_FILE}). Run the crew again and accept when asked, then `crewai eval`."
+)
+
+
+class EvaluationStoppedError(RuntimeError):
+ """The evaluation cannot go on, in words meant for a reader.
+
+ Raised rather than printed-and-exited, because the same two functions serve
+ the terminal and the run app: one of them owns the screen, and a line
+ printed underneath it is a smear nobody asked for.
+ """
+
+
+def _note(text: str, style: str = "dim") -> None:
+ console.print(Text(text), style=style)
+
+
def eval_crew(run_id: str | None = None) -> None:
"""Evaluate the last traced run of this project, or the run RUN_ID."""
get_or_create_project_id()
@@ -57,10 +114,19 @@ def eval_crew(run_id: str | None = None) -> None:
record = read_last_run() or {}
execution_id = run_id or record.get("execution_id")
if execution_id is None:
- execution_id = _run_now_or_explain()
+ # Nothing traced here: the crew runs first, and the app that runs it
+ # carries the evaluation on its own screen — link, progress, verdict.
+ # It comes back with the run to grade when the app did NOT get to it: a
+ # conversational session, or a flow that took the terminal instead.
+ execution_id = _run_and_let_the_app_evaluate()
+ if execution_id is None:
+ return
record = read_last_run() or {}
- client = _amp_client(trusted)
+ try:
+ client = _amp_client(trusted)
+ except EvaluationStoppedError as stopped:
+ _fail(str(stopped))
recorded_amp = str(record.get("amp_base_url") or "").rstrip("/")
if not run_id and recorded_amp and recorded_amp != client.base_url.rstrip("/"):
console.print(
@@ -69,7 +135,18 @@ def eval_crew(run_id: str | None = None) -> None:
),
style="yellow",
)
- started = _start_evaluation(client, execution_id)
+ try:
+ started = _start_evaluation(
+ client,
+ execution_id,
+ wait_for_spans=run_id is None and _ran_just_now(record),
+ )
+ except EvaluationStoppedError as stopped:
+ _fail(str(stopped))
+ # After, not before: `cli_usage:eval` counts an evaluation, and a refused
+ # request — a run AMP does not hold, a credential it will not take — is not
+ # one. `_start_evaluation` raises rather than returning on those.
+ _record_usage(logged_in=client.api_key is not None)
url = started.get("url")
console.print(Text("Evaluating run ").append(execution_id, style="bold"))
if url:
@@ -79,12 +156,202 @@ def eval_crew(run_id: str | None = None) -> None:
console.print(Text("Follow it at ").append(url, style="cyan underline"))
_open(url)
- finished = _wait(client, started["id"], url)
+ console.print("Waiting for the verdict…", style="dim")
+ try:
+ finished = _wait(client, started["id"], url)
+ except EvaluationStoppedError as stopped:
+ _fail(str(stopped))
+ except KeyboardInterrupt:
+ console.print(
+ Text(f"\nStill running{f' at {url}' if url else ''}."), style="yellow"
+ )
+ raise SystemExit(130) from None
_print_verdict(finished, url)
+ _say_where_the_criteria_live(write_eval_config(finished))
if finished.get("status") != "done":
raise SystemExit(1)
+def _ran_just_now(record: dict[str, Any]) -> bool:
+ """Did the project record this run in the last few minutes?
+
+ The one thing that tells a run still landing in AMP apart from a run AMP
+ does not hold: when it finished. A record without a readable stamp is not
+ assumed fresh — the wait is for the case we can name.
+ """
+ stamp = str(record.get("recorded_at") or record.get("finished_at") or "")
+ if not stamp:
+ return False
+
+ try:
+ when = datetime.fromisoformat(stamp)
+ except ValueError:
+ return False
+
+ now = datetime.now(when.tzinfo) if when.tzinfo else datetime.now()
+ return 0 <= (now - when).total_seconds() <= RUN_IS_FRESH_SECONDS
+
+
+EVAL_CONFIG_FILE = "eval.jsonc"
+# A project's criteria are criteria, not a corpus — the same bound AMP holds
+# the field to, checked here so a file that will be refused is not uploaded.
+MAX_EVAL_CONFIG_BYTES = 64 * 1024
+
+
+def project_eval_config(*, note: Callable[[str], None] = _note) -> str | None:
+ """What this project says good means, if it has said.
+
+ Read from the project's own directory, beside `pyproject.toml`, because
+ that is where the team keeps the things they argue about in review. Not
+ parsed here: what a criterion means is the grader's to say, and a document
+ this CLI could not read is one the grader will refuse with the line that
+ is wrong.
+ """
+ path = Path.cwd() / EVAL_CONFIG_FILE
+ try:
+ if not path.is_file():
+ return None
+ text = path.read_text(encoding="utf-8")
+ except OSError:
+ return None
+
+ if len(text.encode("utf-8")) > MAX_EVAL_CONFIG_BYTES:
+ # Through NOTE, which the run app routes onto its own screen: a print
+ # would land under its layout.
+ note(
+ f"{EVAL_CONFIG_FILE} is larger than "
+ f"{MAX_EVAL_CONFIG_BYTES // 1024}KB and was not sent; this run is graded on "
+ "the crew's own expectations."
+ )
+ return None
+ return text or None
+
+
+def write_eval_config(finished: dict[str, Any]) -> Path | None:
+ """Write the criteria this run was graded on, for the project to edit.
+
+ Only when the project has none: after that the file is theirs, and the one
+ thing worse than no criteria is criteria that get overwritten every time
+ somebody runs an evaluation.
+ """
+ text = finished.get("eval_config")
+ if not isinstance(text, str) or not text.strip():
+ return None
+
+ path = Path.cwd() / EVAL_CONFIG_FILE
+ try:
+ if path.exists() or not path.parent.is_dir():
+ return None
+ path.write_text(text, encoding="utf-8")
+ except OSError:
+ return None
+ return path
+
+
+EVAL_MARKER_FILE = "last_eval.json"
+
+
+def _marker_path() -> Path:
+ from crewai.telemetry.tracing.last_run import LAST_RUN_DIR, project_dir
+
+ return project_dir() / LAST_RUN_DIR / EVAL_MARKER_FILE
+
+
+def record_evaluation_outcome(execution_id: str | None) -> None:
+ """What the run app did with the evaluation it was opened for.
+
+ The app runs in a child process for most projects, so nothing in memory
+ reaches the command waiting behind it. Two things that command must know
+ are both here: that this run has been evaluated already, and — when the app
+ had nothing to evaluate — that it was the app saying so rather than no app
+ at all. A conversational session and a flow that takes the terminal never
+ get here, leave no marker, and the command grades the run itself.
+
+ Never raises: a marker that cannot be written costs a second evaluation,
+ not the run.
+ """
+ with contextlib.suppress(Exception):
+ path = _marker_path()
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(
+ json.dumps(
+ {
+ "execution_id": execution_id,
+ "at": datetime.now(timezone.utc).isoformat(),
+ }
+ ),
+ encoding="utf-8",
+ )
+
+
+def evaluation_marker(after: datetime) -> dict[str, Any] | None:
+ """The app's word about the run this command just watched, if it left one.
+
+ AFTER is when the run was started, and the marker must be newer: a project
+ keeps one marker, and the one from an earlier `crewai eval` is not about
+ this run. Matched on time rather than on the project's last-run record,
+ which another run finishing in the same project can replace while this app
+ is still open.
+
+ A second of slack, because the two stamps are written by two processes and
+ the question being asked is "this run or an older one", which a second
+ cannot confuse.
+ """
+ with contextlib.suppress(Exception):
+ marker = json.loads(_marker_path().read_text(encoding="utf-8"))
+ when = datetime.fromisoformat(str(marker.get("at")))
+ if isinstance(marker, dict) and when >= after - timedelta(seconds=1):
+ return dict(marker)
+ return None
+
+
+def evaluate_run(
+ execution_id: str,
+ *,
+ on_started: Callable[[dict[str, Any]], None],
+ on_status: Callable[[dict[str, Any]], None] | None = None,
+ note: Callable[[str], None] = _note,
+) -> dict[str, Any]:
+ """Start the evaluation of EXECUTION_ID and wait for its verdict.
+
+ The run app's door into this command: it has a screen of its own to show
+ the link and the verdict on, so nothing here prints and nothing exits — the
+ url reaches ON_STARTED as soon as AMP gives it, every unfinished answer
+ reaches ON_STATUS, and anything that stops the evaluation is an
+ `EvaluationStoppedError` carrying the sentence to show.
+
+ The spans of a run that just ended may still be in flight, which is the
+ whole reason this is called from inside the app that ran it, so the wait
+ for them is always on here.
+ """
+ client = _amp_client(_machine_amp_origins(), note=note)
+ started = _start_evaluation(client, execution_id, wait_for_spans=True, note=note)
+ # After the start, exactly as the command counts it: `cli_usage:eval` is the
+ # count of evaluations that began, and an evaluation the app runs is one.
+ _record_usage(logged_in=client.api_key is not None)
+ record_evaluation_outcome(execution_id)
+ on_started(started)
+ finished = _wait(client, started["id"], started.get("url"), on_status=on_status)
+ written = write_eval_config(finished)
+ if written is not None:
+ finished = {**finished, "wrote_eval_config": written.name}
+ return finished
+
+
+def _say_where_the_criteria_live(path: Path | None) -> None:
+ """One line, once: the file exists now, and editing it changes the next grade."""
+ if path is None:
+ return
+
+ console.print(
+ Text(
+ f"Wrote {path.name} — say what good means for this crew there, and the next "
+ "`crewai eval` is graded on it."
+ ),
+ style="green",
+ )
+
+
def _trusted_amp_origins() -> set[str]:
"""Where the saved login may be sent: the AMP this machine is configured for
(`crewai enterprise configure`), one already exported in this shell, and
@@ -97,6 +364,20 @@ def _trusted_amp_origins() -> set[str]:
return {origin for origin in map(_origin, candidates) if origin}
+def _machine_amp_origins() -> set[str]:
+ """The same set, for a caller whose environment can no longer be trusted.
+
+ `crewai eval` reads `CREWAI_PLUS_URL` BEFORE the project's `.env` is loaded,
+ so a shell export counts there and a project cannot introduce one. Inside
+ the run app there is no such "before": the crew has already run and its
+ `.env` was loaded for it, so the variable is left out of the set entirely.
+ The request still follows it — that is how a self-hosted project is wired —
+ and the credential still does not.
+ """
+ candidates = (Settings().enterprise_base_url, DEFAULT_CREWAI_ENTERPRISE_URL)
+ return {origin for origin in map(_origin, candidates) if origin}
+
+
def _origin(url: str | None) -> str | None:
parsed = urlparse(str(url or ""))
return (
@@ -123,7 +404,7 @@ def _encrypted(origin: str | None) -> bool:
return False
-def _amp_client(trusted: set[str]) -> PlusAPI:
+def _amp_client(trusted: set[str], *, note: Callable[[str], None] = _note) -> PlusAPI:
"""The AMP to ask, and whether the saved login goes with it.
A project's `.env` may point `crewai eval` at another AMP — that is how a
@@ -133,7 +414,7 @@ def _amp_client(trusted: set[str]) -> PlusAPI:
read anonymously."""
client = PlusAPI(api_key=saved_login())
if client.api_key is None:
- return client
+ return _anonymous(client)
origin = _origin(client.base_url)
if origin in trusted and _encrypted(origin):
@@ -144,14 +425,23 @@ def _amp_client(trusted: set[str]) -> PlusAPI:
if origin not in trusted
else "would carry the login over plain HTTP"
)
- console.print(
- Text(
- f"Reading anonymously: {client.base_url} {why}. "
- "Run `crewai enterprise configure <url>` to log in to it."
- ),
- style="yellow",
+ note(
+ f"Reading anonymously: {client.base_url} {why}. "
+ "Run `crewai enterprise configure <url>` to log in to it."
)
- return PlusAPI()
+ return _anonymous(PlusAPI())
+
+
+def _anonymous(client: PlusAPI) -> PlusAPI:
+ """A client that says nothing about who is asking.
+
+ `PlusAPI` adds the saved organization to every request, and AMP reads it
+ only beside a credential — so without one it tells AMP nothing, and tells
+ an AMP this machine is not logged in to which organization is. Dropped, as
+ anonymous trace grants drop it.
+ """
+ client.headers.pop("X-Crewai-Organization-Id", None)
+ return client
def _load_project_env() -> None:
@@ -188,49 +478,93 @@ def saved_login() -> str | None:
except AuthError:
return None
except Exception as error:
- _fail(
+ # Raised, not printed-and-exited: the run app shows this on its own
+ # screen, where a print would land under the layout and an exit would
+ # take the sentence with it.
+ raise EvaluationStoppedError(
f"Could not read the saved login ({type(error).__name__}: {error}). "
"Run `crewai login` again, or `crewai eval` will not know who you are."
- )
- return None
+ ) from error
+
+def _run_and_let_the_app_evaluate() -> str | None:
+ """No traced run recorded here: offer to turn tracing on and run the crew.
-def _run_now_or_explain() -> str:
- """No traced run recorded here: offer to turn tracing on and run the crew now."""
+ The run app evaluates what it ran, on its own screen, so this returns None
+ when it did — the reader has already seen the link and the verdict. It
+ returns the run to grade when no app got to it (a conversational session, a
+ flow that took the terminal), and says so itself when nothing was traced.
+ """
if not Path("pyproject.toml").is_file():
_fail(
"No crewAI project here (no pyproject.toml). Run `crewai eval` from the project's "
"directory, or name a run: `crewai eval --run EXECUTION_ID`."
)
steps = (
- "No traced run is recorded in this project. Turn tracing on and run the crew, "
- f"then come back:\n 1. add {TRACING_ENV_VAR}=true to .env\n 2. crewai run\n 3. crewai eval"
+ "No traced run is recorded in this project. Turn tracing on — its runs are then "
+ "traced to CrewAI AMP — and run the crew, then come back:\n"
+ f" 1. add {TRACING_ENV_VAR}=true to .env\n 2. crewai run\n 3. crewai eval"
)
if is_dmn_mode_enabled() or not sys.stdin.isatty():
console.print(steps, style="yellow")
raise SystemExit(1)
if not click.confirm(
- f"No traced run is recorded in this project. Turn tracing on ({TRACING_ENV_VAR}=true "
- "stays in .env) and run the crew now?",
+ "No traced run is recorded in this project. Turn tracing on and run the crew now? "
+ "The run's trace is sent to CrewAI AMP.",
default=True, # João, 2026-09-20: y/n with Y as the default — the prompt says what Enter does
):
console.print(steps, style="yellow")
raise SystemExit(0)
_enable_tracing()
+ from crewai_cli.crew_run_tui import evaluating_after_run
from crewai_cli.run_crew import run_crew
- run_crew()
- record = read_last_run()
- if record is None:
- console.print(
- "The run finished but no trace was recorded: the run may have failed, sharing the "
- "trace was declined, or this project's crewai is older than the version that records "
- f"the last run ({LAST_RUN_FILE}). Run the crew again and accept when asked, then `crewai eval`.",
- style="bold red",
- )
+ # The app is told an evaluation is waiting and starts it itself when the run
+ # ends. It runs in this process for some projects and in a child for others,
+ # so what says a run was traced is either the holder's id or the record the
+ # run wrote.
+ began = datetime.now(timezone.utc)
+ with evaluating_after_run() as watched:
+ run_crew()
+
+ # The app says what it did with the run it watched, and its word settles
+ # this: it evaluated it (nothing left to do), or it had nothing to evaluate
+ # (say so once, in the terminal the screen has left). Only when no app got
+ # here at all — a conversational session, a flow that took the terminal —
+ # is there a run for this command to grade.
+ marker = evaluation_marker(after=began)
+ if marker is not None:
+ if marker.get("execution_id"):
+ return None
+ console.print(NOT_TRACED, style="bold red")
raise SystemExit(1)
- return str(record["execution_id"])
+
+ # The project's record, only when it was written after this run began: a
+ # record is the project's LAST run, and another run finishing in the same
+ # project would otherwise be graded in this one's place.
+ record = read_last_run() or {}
+ traced = watched["execution_id"] or (
+ record.get("execution_id") if _recorded_since(record, began) else None
+ )
+ if traced:
+ return str(traced)
+
+ console.print(NOT_TRACED, style="bold red")
+ raise SystemExit(1)
+
+
+def _recorded_since(record: dict[str, Any], began: datetime) -> bool:
+ """Was RECORD written after BEGAN? A record without a readable, zoned stamp
+ is not assumed to be this run's."""
+ stamp = str(record.get("recorded_at") or record.get("finished_at") or "")
+ try:
+ when = datetime.fromisoformat(stamp)
+ except ValueError:
+ return False
+ if when.tzinfo is None:
+ return False
+ return when >= began - timedelta(seconds=1)
def _enable_tracing() -> None:
@@ -240,16 +574,42 @@ def _enable_tracing() -> None:
set_key(str(env_file), TRACING_ENV_VAR, "true", quote_mode="never")
os.environ[TRACING_ENV_VAR] = "true"
console.print(
- f"Tracing is on for this project ({TRACING_ENV_VAR}=true in .env).",
+ "Tracing is on for this project — its runs are traced to CrewAI AMP.",
style="green",
)
-def _start_evaluation(client: PlusAPI, execution_id: str) -> dict[str, Any]:
- try:
- response = client.create_evaluation(execution_id)
- except httpx.HTTPError as error:
- _fail(f"Could not reach AMP to start the evaluation: {error}")
+def _start_evaluation(
+ client: PlusAPI,
+ execution_id: str,
+ *,
+ wait_for_spans: bool = False,
+ note: Callable[[str], None] = _note,
+) -> dict[str, Any]:
+ deadline = time.monotonic() + SPANS_WAIT_SECONDS if wait_for_spans else 0.0
+ said = False
+ # Read once: the file does not change while the spans are in flight, and a
+ # warning about it belongs on the screen once, not on every retry.
+ eval_config = project_eval_config(note=note)
+ while True:
+ try:
+ response = client.create_evaluation(execution_id, eval_config=eval_config)
+ except httpx.HTTPError as error:
+ raise EvaluationStoppedError(
+ f"Could not reach AMP to start the evaluation: {error}"
+ ) from error
+ # The run finished moments ago and its spans are still on their way:
+ # waiting is the answer, not a 404 the reader can do nothing with.
+ if response.status_code == 404 and time.monotonic() < deadline:
+ if not said:
+ note(
+ "The run's trace has not reached CrewAI AMP yet — waiting up to "
+ f"{int(SPANS_WAIT_SECONDS // 60)} minutes for it…"
+ )
+ said = True
+ time.sleep(SPANS_POLL_SECONDS)
+ continue
+ break
if response.status_code in (200, 202):
payload = _payload(response)
# The id is what every later call is made with, so a missing or
@@ -258,67 +618,102 @@ def _start_evaluation(client: PlusAPI, execution_id: str) -> dict[str, Any]:
# costs the link and nothing else: the evaluation is already running
# and its verdict is what the user came for.
if payload and isinstance(payload.get("id"), str) and payload["id"]:
- if not isinstance(payload.get("url"), str):
- if payload.get("url") is not None:
- console.print(
- Text(
- "AMP answered with a report url that is not a string; "
- "the link is unavailable for this run."
- ),
- style="yellow",
- )
- payload["url"] = None
+ url = _report_url(payload.get("url"))
+ if url is None and payload.get("url") is not None:
+ note(
+ "AMP answered with a report url that is not a web address this "
+ "command will open; the link is unavailable for this run."
+ )
+ payload["url"] = url
return payload
- _fail(f"AMP answered without an evaluation id ({response.status_code}).")
- _refused(response, f"run {execution_id}")
- raise AssertionError("unreachable")
-
-
-def _wait(client: PlusAPI, evaluation_id: str, url: str | None) -> dict[str, Any]:
- """Poll until the evaluation is done or failed; Ctrl-C leaves it running."""
- console.print("Waiting for the verdict…", style="dim")
+ raise EvaluationStoppedError(
+ f"AMP answered without an evaluation id ({response.status_code})."
+ )
+ raise EvaluationStoppedError(_refusal_message(response, f"run {execution_id}"))
+
+
+def _report_url(value: Any) -> str | None:
+ """The report link, if it is one a browser may be sent to.
+
+ It is printed as a link and opened without a click, and it comes from
+ whichever AMP answered — which a project's `.env` may choose. So it must be
+ an absolute web address: HTTPS, or plain HTTP to this machine, with no
+ credentials in it and no control characters to restyle the terminal. The
+ host is not pinned: the report is served by the evaluation service, not by
+ AMP, and a self-hosted AMP names its own.
+ """
+ if (
+ not isinstance(value, str)
+ or not value
+ or any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in value)
+ ):
+ return None
+ try:
+ parsed = urlparse(value)
+ if (
+ parsed.hostname
+ and parsed.username is None
+ and parsed.password is None
+ and _encrypted(f"{parsed.scheme}://{parsed.netloc}")
+ ):
+ return value
+ except ValueError:
+ pass
+ return None
+
+
+def _wait(
+ client: PlusAPI,
+ evaluation_id: str,
+ url: str | None,
+ *,
+ on_status: Callable[[dict[str, Any]], None] | None = None,
+) -> dict[str, Any]:
+ """Poll until the evaluation is done or failed.
+
+ Every unfinished answer goes to ON_STATUS, so a caller that has somewhere to
+ show progress can show it; the caller decides what a Ctrl-C means.
+ """
where = f" at {url}" if url else ""
subject = f"evaluation {evaluation_id}"
misses = (
0 # AMP unreachable or answering 5xx: a blip is retried, a streak is reported
)
- try:
- while True:
- try:
- response = client.get_evaluation(evaluation_id)
- except httpx.HTTPError as error:
- misses += 1
- if misses >= POLL_RETRIES:
- _fail(
- f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}."
- )
- time.sleep(POLL_SECONDS)
- continue
- if response.status_code >= 500:
- misses += 1
- if misses >= POLL_RETRIES:
- _refused(response, subject)
- time.sleep(POLL_SECONDS)
- continue
- if response.status_code != 200:
- _refused(response, subject)
- misses = 0
- payload = _payload(response) or {}
- status = payload.get("status")
- if status == "done" and not _well_formed_verdict(payload.get("verdict")):
- _fail(
- f"AMP answered done without a verdict (protocol error); follow it{where or ' on AMP'}."
- )
- if status in FINISHED:
- return payload
- if status not in STATUSES:
- _fail(
- f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}."
- )
+ while True:
+ try:
+ response = client.get_evaluation(evaluation_id)
+ except httpx.HTTPError as error:
+ misses += 1
+ if misses >= POLL_RETRIES:
+ raise EvaluationStoppedError(
+ f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}."
+ ) from error
time.sleep(POLL_SECONDS)
- except KeyboardInterrupt:
- console.print(Text(f"\nStill running{where}."), style="yellow")
- raise SystemExit(130) from None
+ continue
+ if response.status_code >= 500:
+ misses += 1
+ if misses >= POLL_RETRIES:
+ raise EvaluationStoppedError(_refusal_message(response, subject))
+ time.sleep(POLL_SECONDS)
+ continue
+ if response.status_code != 200:
+ raise EvaluationStoppedError(_refusal_message(response, subject))
+ misses = 0
+ payload = _payload(response) or {}
+ status = payload.get("status")
+ if status == "done" and not _well_formed_verdict(payload.get("verdict")):
+ raise EvaluationStoppedError(
+ f"AMP answered done without a verdict (protocol error); follow it{where or ' on AMP'}."
+ )
+ if status in FINISHED:
+ return payload
+ if status not in STATUSES:
+ raise EvaluationStoppedError(
+ f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}."
+ )
+ if on_status is not None:
+ on_status(payload)
+ time.sleep(POLL_SECONDS)
def _well_formed_verdict(verdict: Any) -> bool:
@@ -385,25 +780,30 @@ def _payload(response: httpx.Response) -> dict[str, Any] | None:
return loaded if isinstance(loaded, dict) else None
-def _refused(response: httpx.Response, subject: str) -> None:
- """AMP's own words when it sent them, then exit 1. SUBJECT is "run <id>" or "evaluation <id>"."""
+def _refusal_message(response: httpx.Response, subject: str) -> str:
+ """AMP's own words when it sent them. SUBJECT is "run <id>" or "evaluation <id>".
+
+ A sentence, not a print: the terminal and the run app both show it, and only
+ one of them shows it by printing.
+ """
payload = _payload(response) or {}
message = str(payload.get("message") or "").strip()
error = str(payload.get("error") or "")
if response.status_code in (401, 403):
if error == "account_required" and message:
- _fail(message)
- _fail(
- f"{message or 'AMP refused the credential'}. Log in with `crewai login` and try again."
- )
+ return message
+ return f"{message or 'AMP refused the credential'}. Log in with `crewai login` and try again."
if response.status_code == 404:
- _fail(message or f"AMP answered 404 for {subject}.")
+ return message or f"AMP answered 404 for {subject}."
if response.status_code == 429:
retry = response.headers.get("Retry-After")
- _fail(
- f"{message or 'AMP is rate limiting this request'}{f' — retry after {retry}s' if retry else ''}."
- )
- _fail(f"AMP answered {response.status_code}{': ' + message if message else ''}.")
+ return f"{message or 'AMP is rate limiting this request'}{f' — retry after {retry}s' if retry else ''}."
+ return f"AMP answered {response.status_code}{': ' + message if message else ''}."
+
+
+def _refused(response: httpx.Response, subject: str) -> None:
+ """The same words, printed, then exit 1."""
+ _fail(_refusal_message(response, subject))
def _fail(message: str) -> None:
diff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.py
index f03661e5b..0256f7c30 100644
--- a/lib/cli/src/crewai_cli/plus_api.py
+++ b/lib/cli/src/crewai_cli/plus_api.py
@@ -27,12 +27,22 @@ class PlusAPI(_CorePlusAPI):
EVALUATION_START_TIMEOUT = 120.0
EVALUATION_POLL_TIMEOUT = 30.0
- def create_evaluation(self, execution_id: str) -> httpx.Response:
- """Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval)."""
+ def create_evaluation(
+ self, execution_id: str, *, eval_config: str | None = None
+ ) -> httpx.Response:
+ """Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval).
+
+ EVAL_CONFIG is the project's own `eval.jsonc` when it has one: what
+ good means for this crew, in its own words. Sent as it was written,
+ comments and all, and read by the grader rather than here.
+ """
+ body: dict[str, str] = {"execution_id": execution_id}
+ if eval_config:
+ body["eval_config"] = eval_config
return self._make_request(
"POST",
self.EVALUATIONS_RESOURCE,
- json={"execution_id": execution_id},
+ json=body,
timeout=self.EVALUATION_START_TIMEOUT,
)
diff --git a/lib/cli/src/crewai_cli/run_crew.py b/lib/cli/src/crewai_cli/run_crew.py
index 903112a98..9597b75a4 100644
--- a/lib/cli/src/crewai_cli/run_crew.py
+++ b/lib/cli/src/crewai_cli/run_crew.py
@@ -600,6 +600,48 @@ def _print_post_tui_summary(app: CrewRunApp) -> None:
)
)
+ _print_evaluation_line(app, console, crewai_teal)
+
+
+def _print_evaluation_line(app: CrewRunApp, console: Any, teal: str) -> None:
+ """The evaluation's link, once the app that showed it has gone.
+
+ An evaluation started inside the app is read there; the terminal is what is
+ left afterwards, and a link that only ever existed on a screen that is now
+ closed is a link nobody can open again.
+ """
+ evaluation = getattr(app, "_evaluation", None) or {}
+ url = str(evaluation.get("url") or "")
+ if not url:
+ return
+
+ from rich.text import Text
+
+ state = str(evaluation.get("state"))
+ line = Text("\n ")
+ if state == "done":
+ verdict = evaluation.get("verdict") or {}
+ line.append("Evaluated: ", style="dim")
+ line.append(
+ f"goal gate {str(verdict.get('gate') or '').upper()} ", style="bold"
+ )
+ elif state == "failed":
+ line.append("Evaluation stopped — the report has what it got: ", style="dim")
+ else:
+ line.append("Evaluation still running at ", style="dim")
+ line.append(url, style=f"{teal} underline")
+ console.print(line)
+
+ wrote = evaluation.get("wrote_config")
+ if wrote:
+ note = Text(" ")
+ note.append(f"Wrote {wrote}", style="bold")
+ note.append(
+ " — say what good means for this crew there, and the next evaluation is graded on it.",
+ style="dim",
+ )
+ console.print(note)
+
def run_crew(
trained_agents_file: str | None = None,
diff --git a/lib/cli/src/crewai_cli/run_declarative_flow.py b/lib/cli/src/crewai_cli/run_declarative_flow.py
index 3042d00c0..4fbd2ca4f 100644
--- a/lib/cli/src/crewai_cli/run_declarative_flow.py
+++ b/lib/cli/src/crewai_cli/run_declarative_flow.py
@@ -328,6 +328,10 @@ def _print_flow_post_tui_summary(app: Any) -> None:
)
)
+ from crewai_cli.run_crew import _print_evaluation_line
+
+ _print_evaluation_line(app, console, crewai_teal)
+
def _resolve_flow_inputs(flow: Any, provided: dict[str, Any]) -> dict[str, Any]:
"""Resolve kickoff inputs from the flow's state schema.
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index ce5007059..1061b3fe1 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -12,10 +12,21 @@ import httpx
import pytest
from rich.console import Console
+from crewai_cli import run_crew as run_crew_module
from crewai_cli.experimental import eval_crew as eval_module
from crewai_cli.cli import eval_command
[email protected](autouse=True)
+def _awaiting_files_stay_in_tmp(tmp_path, monkeypatch):
+ """The waiting-evaluation token file belongs to the user's crewAI data
+ directory; a test keeps it in its own."""
+ monkeypatch.setattr(
+ "crewai_cli.crew_run_tui._awaiting_dir", lambda: tmp_path / "eval-awaiting"
+ )
+
+
+
EXECUTION_ID = "6f31fe1a-20bd-4bfe-a011-25d6b9341f62"
URL = "https://evolve.crewai.test/e/ev-1"
@@ -29,10 +40,15 @@ class FakeAMP:
)
self.statuses = list(statuses or [])
self.calls: list[tuple] = []
+ self.sent_config = None
self.api_key = None
+ self.headers: dict[str, str] = {"X-Crewai-Organization-Id": "org-42"}
- def create_evaluation(self, execution_id):
+ def create_evaluation(self, execution_id, *, eval_config=None):
self.calls.append(("create", execution_id))
+ self.sent_config = eval_config
+ if isinstance(self.create, list):
+ return self.create.pop(0) if len(self.create) > 1 else self.create[0]
return self.create
def get_evaluation(self, evaluation_id):
@@ -40,11 +56,14 @@ class FakeAMP:
return self.statuses.pop(0) if self.statuses else httpx.Response(200, json={"id": evaluation_id, "status": "running"})
-def done(gate="passed", grades=None):
- return httpx.Response(200, json={
+def done(gate="passed", grades=None, eval_config=None):
+ payload = {
"id": "ev-1", "status": "done", "url": URL,
"verdict": {"gate": gate, "grades": grades if grades is not None else {"goal": 5, "quality": 4, "process": 5, "cost": None}},
- })
+ }
+ if eval_config is not None:
+ payload["eval_config"] = eval_config
+ return httpx.Response(200, json=payload)
@pytest.fixture
@@ -69,6 +88,12 @@ def record_last_run(directory: Path, execution_id: str = EXECUTION_ID, **fields)
(directory / ".crewai" / "last_run.json").write_text(json.dumps(record))
+def _now() -> str:
+ from datetime import datetime, timezone
+
+ return datetime.now(timezone.utc).isoformat()
+
+
def install(monkeypatch, amp: FakeAMP, configured_amp: str = "https://amp.test") -> FakeAMP:
def build(api_key=None, base_url=None):
# PlusAPI's own resolution: explicit, then CREWAI_PLUS_URL, then the saved settings.
@@ -137,13 +162,15 @@ def test_the_follow_link_cannot_be_retargeted_by_the_url_amp_sends(project, monk
# command to let markup through.
directory, _ = project
record_last_run(directory)
- hostile = "[link=http://attacker.test/]https://app.crewai.com/e/ev-1[/link]"
+ # A web address that carries markup inside it: printed as text, never
+ # interpreted (one that IS markup is not a web address, and is dropped).
+ hostile = "https://app.crewai.com/e/[link=http://attacker.test/]ev-1[/link]"
created = httpx.Response(202, json={"id": "ev-1", "url": hostile, "status": "queued"})
install(monkeypatch, FakeAMP(create=created, statuses=[done()]))
eval_module.eval_crew()
- out = capsys.readouterr().out
+ out = capsys.readouterr().out.replace("\n", "")
assert "[link=http://attacker.test/]" in out # printed, not followed
@@ -160,7 +187,7 @@ def test_a_url_that_is_not_a_string_costs_the_link_and_nothing_else(project, mon
eval_module.eval_crew()
out = capsys.readouterr().out
- assert "report url that is not a string" in out # said, not swallowed
+ assert "not a web address" in out # said, not swallowed
assert "Goal gate: PASSED" in out # and the verdict still arrives
assert opened == [] # nothing was handed to a browser
assert "Follow it at" not in out
@@ -300,6 +327,44 @@ def test_a_project_may_point_at_another_amp_but_never_gets_the_saved_login(proje
assert "crewai enterprise configure" in out
+def test_the_app_never_trusts_an_amp_the_project_env_introduced(project, monkeypatch):
+ """`crewai eval` reads CREWAI_PLUS_URL before the project's `.env` is
+ loaded, so a shell export is trusted there and a project cannot slip one
+ in. Inside the run app there is no such "before" — the crew has already run
+ and its `.env` was loaded for it — so an exported variable buys nothing and
+ the saved login stays home."""
+ directory, _ = project
+ # what a project's .env would leave behind, indistinguishable by then
+ monkeypatch.setenv("CREWAI_PLUS_URL", "https://evil.example")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+ notes: list[str] = []
+
+ eval_module.evaluate_run(
+ EXECUTION_ID, on_started=lambda started: None, note=notes.append
+ )
+
+ assert amp.base_url == "https://evil.example" # the request still follows it
+ assert amp.api_key is None # the credential does not
+ assert any("Reading anonymously" in note for note in notes)
+ # and it is said through the caller's own note, never printed over its screen
+ assert notes and all(isinstance(note, str) for note in notes)
+
+
+def test_the_command_still_trusts_an_amp_exported_in_the_shell(project, monkeypatch, capsys):
+ """The command's own reading is unchanged: a shell export is this machine
+ speaking, and the login goes with it."""
+ directory, _ = project
+ monkeypatch.setenv("CREWAI_PLUS_URL", "https://amp.exported")
+ record_last_run(directory, amp_base_url="https://amp.exported")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.base_url == "https://amp.exported"
+ assert amp.api_key == "login-token"
+ assert "Reading anonymously" not in capsys.readouterr().out
+
+
def test_a_trusted_amp_over_plain_http_still_gets_no_credential(project, monkeypatch, capsys):
"""A cleartext connection is not a place to put a bearer token, trusted or not."""
directory, _ = project
@@ -405,7 +470,7 @@ def test_amp_unreachable_at_the_start_is_a_sentence_not_a_traceback(project, mon
directory, _ = project
record_last_run(directory)
amp = install(monkeypatch, FakeAMP())
- monkeypatch.setattr(amp, "create_evaluation", lambda execution_id: (_ for _ in ()).throw(httpx.ConnectError("connection refused")))
+ monkeypatch.setattr(amp, "create_evaluation", lambda execution_id, **_: (_ for _ in ()).throw(httpx.ConnectError("connection refused")))
with pytest.raises(SystemExit) as exit_:
eval_module.eval_crew()
@@ -529,10 +594,12 @@ def test_without_a_traced_run_it_offers_to_turn_tracing_on_and_run_the_crew(proj
ran.append("run")
assert eval_module.os.environ.get("CREWAI_TRACING_ENABLED") == "true"
record_last_run(directory, "fresh-run")
+ eval_module.record_evaluation_outcome("fresh-run") # what the app does
import crewai_cli.run_crew as run_crew_module
monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
amp = install(monkeypatch, FakeAMP(statuses=[done()]))
@@ -540,12 +607,433 @@ def test_without_a_traced_run_it_offers_to_turn_tracing_on_and_run_the_crew(proj
assert ran == ["run"]
text, kwargs = prompts[0]
- assert "CREWAI_TRACING_ENABLED=true stays in .env" in text and kwargs == {"default": True} # Enter is yes (João's call); the prompt names both effects
+ # the offer is about what happens, not about the variable that makes it happen
+ assert "Turn tracing on and run the crew now?" in text and kwargs == {"default": True} # Enter is yes (João's call)
+ assert "sent to CrewAI AMP" in text # what saying yes means, in the words that matter
+ assert "CREWAI_TRACING_ENABLED" not in text # and not the variable that carries it
assert "CREWAI_TRACING_ENABLED=true" in (directory / ".env").read_text()
- assert amp.calls[0] == ("create", "fresh-run")
+ # the app that ran the crew evaluates it on its own screen; the command that
+ # opened it does not ask AMP a second time
+ assert amp.calls == []
assert "Tracing is on for this project" in capsys.readouterr().out
[email protected]("logged_in", [True, False])
+def test_the_command_counts_the_evaluation_and_nothing_that_names_it(
+ project, monkeypatch, logged_in
+):
+ """`cli_usage:eval` is every evaluation; the TUI's own `cli_usage:evaluate`
+ is intent, and the difference is intent that never became one. Usage stats
+ are anonymous, so the span says whether the caller was logged in — never
+ which run, which organization, or anything about the run's content."""
+ directory, _ = project
+ record_last_run(directory, "counted-run")
+ spans: list[tuple[str, dict[str, str]]] = []
+
+ class FakeTelemetry:
+ def set_tracer(self) -> None:
+ pass
+
+ def feature_usage_span(self, feature, attributes=None) -> None:
+ spans.append((feature, attributes or {}))
+
+ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)
+ monkeypatch.setattr(eval_module, "saved_login", lambda: "tok" if logged_in else None)
+ install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert spans == [
+ ("cli_usage:eval", {"authenticated": "true" if logged_in else "false"})
+ ]
+
+
+def test_an_oversized_config_is_said_once_and_through_the_callers_note(
+ project, monkeypatch
+):
+ """The run app routes notes onto its own screen, and the spans of a fresh
+ run can take several 404s to land — the warning belongs there, once."""
+ directory, _ = project
+ (directory / "eval.jsonc").write_text("x" * (eval_module.MAX_EVAL_CONFIG_BYTES + 1))
+ sent: list[str | None] = []
+ answers = iter([httpx.Response(404, json={}), httpx.Response(404, json={}), QUEUED])
+
+ class Client:
+ def create_evaluation(self, execution_id, *, eval_config=None):
+ sent.append(eval_config)
+ return next(answers)
+
+ monkeypatch.setattr(eval_module.time, "sleep", lambda _s: None)
+ notes: list[str] = []
+
+ eval_module._start_evaluation(
+ Client(), "run-1", wait_for_spans=True, note=notes.append
+ )
+
+ assert sent == [None, None, None]
+ assert [n for n in notes if "larger than" in n] == [notes[0]]
+
+
+def test_the_fallback_grades_only_a_run_recorded_after_it_began(
+ project, monkeypatch, capsys
+):
+ """No app evaluated the run, so the command reads the project's record — but
+ that record is the project's LAST run, and one written before this run
+ began belongs to some other run."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(
+ run_crew_module,
+ "run_crew",
+ lambda: record_last_run(
+ directory, "somebody-elses-run", recorded_at="2020-01-01T00:00:00+00:00"
+ ),
+ )
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ with pytest.raises(SystemExit) as exited:
+ eval_module.eval_crew()
+
+ assert exited.value.code == 1
+ assert amp.calls == []
+ assert "no trace was recorded" in capsys.readouterr().out
+
+
+ "url, opened",
+ [
+ ("https://optimize.crewai.test/e/ev-1", True),
+ ("http://localhost:3000/e/ev-1", True),
+ ("http://evil.test/e/ev-1", False),
+ ("file:///etc/passwd", False),
+ ("javascript:alert(1)", False),
+ ("https://user:[email protected]/e/ev-1", False),
+ ("https://optimize.crewai.test/e/ev-1\x1b[31m", False),
+ ("/e/ev-1", False),
+ ],
+)
+def test_only_a_web_address_is_printed_and_opened(url, opened):
+ """The report link comes from whichever AMP answered — which a project's
+ `.env` may choose — and is opened without a click. Its host is not pinned
+ (the report lives on the evaluation service), but it must be a web address."""
+ assert (eval_module._report_url(url) == url) is opened
+
+
+def test_a_report_url_that_is_not_a_web_address_is_dropped_with_a_note(monkeypatch):
+ class Client:
+ def create_evaluation(self, execution_id, *, eval_config=None):
+ return httpx.Response(
+ 202, json={"id": "ev-1", "url": "file:///etc/passwd", "status": "queued"}
+ )
+
+ notes: list[str] = []
+ started = eval_module._start_evaluation(Client(), "run-1", note=notes.append)
+
+ assert started["url"] is None
+ assert "link is unavailable" in notes[0]
+
+
[email protected]("login", [None, "tok"])
+def test_a_request_without_the_login_names_no_organization(project, monkeypatch, login):
+ """AMP reads the organization header only beside a credential, so without
+ one it says nothing — except, to an AMP this machine is not logged in to,
+ which organization is asking."""
+ monkeypatch.setenv("CREWAI_PLUS_URL", "https://not-logged-in.test")
+ monkeypatch.setattr(eval_module, "saved_login", lambda: login)
+ monkeypatch.setattr(
+ "crewai_core.plus_api.Settings",
+ lambda: SimpleNamespace(org_uuid="org-42", enterprise_base_url=None),
+ )
+
+ client = eval_module._amp_client({"https://app.crewai.com"}, note=lambda _t: None)
+
+ assert client.api_key is None
+ assert "X-Crewai-Organization-Id" not in client.headers
+
+
+def test_telemetry_never_breaks_the_command(project, monkeypatch):
+ directory, _ = project
+ record_last_run(directory, "counted-run")
+
+ def explode() -> None:
+ raise RuntimeError("no tracer here")
+
+ monkeypatch.setattr("crewai_core.telemetry.Telemetry", lambda: explode())
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.calls[0] == ("create", "counted-run")
+
+
+def test_a_project_that_says_what_good_means_is_graded_on_it(project, monkeypatch):
+ """The file travels as it was written — comments and all — because what a
+ criterion means is the grader's to read, not this command's."""
+ directory, _ = project
+ record_last_run(directory)
+ written = '// ours\n{"dataset": [{"id": "c1", "criteria": ["cites a source"]}]}\n'
+ (directory / "eval.jsonc").write_text(written)
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.sent_config == written
+
+
+def test_a_project_with_nothing_to_say_sends_nothing(project, monkeypatch):
+ directory, _ = project
+ record_last_run(directory)
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.sent_config is None
+
+
+def test_a_config_too_large_to_be_criteria_is_left_behind(project, monkeypatch, capsys):
+ """A file that AMP would refuse is not uploaded, and the run is still graded."""
+ directory, _ = project
+ record_last_run(directory)
+ (directory / "eval.jsonc").write_text("// " + "x" * (64 * 1024))
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.sent_config is None
+ assert amp.calls[0][0] == "create" # graded anyway
+ assert "was not sent" in capsys.readouterr().out
+
+
+def test_the_first_evaluation_leaves_the_criteria_behind_as_a_file(
+ project, monkeypatch, capsys
+):
+ """A project cannot say what good means until it has somewhere to say it."""
+ directory, _ = project
+ record_last_run(directory)
+ install(monkeypatch, FakeAMP(statuses=[done(eval_config='// yours\n{"dataset": []}\n')]))
+
+ eval_module.eval_crew()
+
+ assert (directory / "eval.jsonc").read_text() == '// yours\n{"dataset": []}\n'
+ assert "Wrote eval.jsonc" in capsys.readouterr().out
+
+
+def test_a_file_a_project_already_has_is_never_overwritten(project, monkeypatch, capsys):
+ """After the first one the file is theirs — the one thing worse than no
+ criteria is criteria that vanish every time somebody evaluates."""
+ directory, _ = project
+ record_last_run(directory)
+ (directory / "eval.jsonc").write_text("// mine, edited\n{\"dataset\": []}\n")
+ install(monkeypatch, FakeAMP(statuses=[done(eval_config='// theirs\n{}')]))
+
+ eval_module.eval_crew()
+
+ assert (directory / "eval.jsonc").read_text() == "// mine, edited\n{\"dataset\": []}\n"
+ assert "Wrote eval.jsonc" not in capsys.readouterr().out
+
+
+def _write_marker(directory: Path, execution_id: str | None, at: str) -> None:
+ (directory / ".crewai").mkdir(exist_ok=True)
+ (directory / ".crewai" / "last_eval.json").write_text(
+ json.dumps({"execution_id": execution_id, "at": at})
+ )
+
+
+def _seconds_ago(seconds: float) -> str:
+ from datetime import datetime, timedelta, timezone
+
+ return (datetime.now(timezone.utc) - timedelta(seconds=seconds)).isoformat()
+
+
+NOT_FOUND = httpx.Response(
+ 404,
+ json={
+ "error": "trace_not_found",
+ "message": "No spans recorded for execution 6f31fe1a",
+ },
+)
+QUEUED = httpx.Response(202, json={"id": "ev-1", "url": URL, "status": "queued"})
+
+
+def test_the_app_it_opens_is_told_an_evaluation_is_waiting(project, monkeypatch):
+ """With nothing traced, the command runs the crew — and the app it opens is
+ told to evaluate what it ran, so nobody quits a screen to reach it."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+
+ def fake_run_crew() -> None:
+ from crewai_cli.crew_run_tui import _AUTO_EVAL
+
+ assert _AUTO_EVAL.get() is not None, "the app must be told in this process"
+ # and in a child process, where a ContextVar cannot reach: a token, and
+ # the file of that name that makes it count
+ from crewai_cli.crew_run_tui import _awaiting_dir
+
+ assert (_awaiting_dir() / os.environ["CREWAI_EVAL_AWAITING_RUN"]).is_file()
+ record_last_run(directory, "run-it-just-did")
+ eval_module.record_evaluation_outcome("run-it-just-did")
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.calls == [] # the app did it, on its own screen
+ assert "CREWAI_EVAL_AWAITING_RUN" not in os.environ
+ from crewai_cli.crew_run_tui import _awaiting_dir
+
+ assert not any(_awaiting_dir().iterdir()) # and the token went with it
+
+
+def test_a_run_no_app_evaluated_is_graded_by_the_command(project, monkeypatch):
+ """Not every run goes through the app: a conversational session never ends
+ the way a crew does, and a flow with human feedback takes the terminal
+ instead. Nothing marked the run as evaluated, so the command grades it
+ rather than exiting as if somebody had."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(
+ run_crew_module,
+ "run_crew",
+ lambda: record_last_run(directory, "run-nobody-graded", recorded_at=_now()),
+ )
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.calls[0] == ("create", "run-nobody-graded")
+
+
+def test_a_marker_from_an_earlier_run_never_counts_for_this_one(project, monkeypatch):
+ """A project keeps one marker, and the one from an earlier `crewai eval` is
+ not about the run this command just watched — it is older than it."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ _write_marker(directory, "some-older-run", _seconds_ago(3600))
+
+ def fake_run_crew() -> None:
+ record_last_run(directory, "the-new-run", recorded_at=_now())
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.calls[0] == ("create", "the-new-run")
+
+
+def test_the_app_saying_it_had_nothing_to_grade_is_not_a_run_to_grade(
+ project, monkeypatch, capsys
+):
+ """An app that looked and found no trace has answered; the command says so
+ once — the screen that showed it is gone — and grades nothing."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+
+ def fake_run_crew() -> None:
+ # another run in the same project finishes while this app is open
+ record_last_run(directory, "somebody-elses-run")
+ eval_module.record_evaluation_outcome(None) # what the app says
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ amp = install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit) as exit_code:
+ eval_module.eval_crew()
+
+ assert exit_code.value.code == 1
+ assert amp.calls == [] # never somebody else's run
+ assert "no trace was recorded" in capsys.readouterr().out
+
+
+def test_a_concurrent_run_cannot_send_the_command_after_the_wrong_one(
+ project, monkeypatch
+):
+ """The record is the project's last finish, which another run can replace
+ while this app is open. The app's own marker is what settles it."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+
+ def fake_run_crew() -> None:
+ eval_module.record_evaluation_outcome("the-run-the-app-watched")
+ record_last_run(directory, "a-run-that-finished-later")
+
+ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.calls == [] # the app graded its own run; nothing is graded twice
+
+
+def test_a_run_that_recorded_nothing_says_so(project, monkeypatch, capsys):
+ """Nothing was traced, so nothing was evaluated and the app had nothing to
+ show: this is the one thing the command still has to say."""
+ directory, _ = project
+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)
+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
+ monkeypatch.setattr(run_crew_module, "run_crew", lambda: None)
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ amp = install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit) as exit_code:
+ eval_module.eval_crew()
+
+ assert exit_code.value.code == 1
+ assert amp.calls == []
+ assert "no trace was recorded" in capsys.readouterr().out
+
+
+def test_a_run_from_hours_ago_is_not_waited_for(project, monkeypatch, capsys):
+ """Nothing is in flight: AMP does not hold this run, and saying so at once
+ is the useful answer."""
+ directory, _ = project
+ record_last_run(directory, recorded_at=_seconds_ago(7200))
+ amp = install(monkeypatch, FakeAMP(create=[NOT_FOUND, QUEUED]))
+
+ with pytest.raises(SystemExit):
+ eval_module.eval_crew()
+
+ assert [call for call in amp.calls if call[0] == "create"] == [
+ ("create", EXECUTION_ID)
+ ]
+ assert "No spans recorded" in capsys.readouterr().out
+
+
+def test_an_id_somebody_typed_is_answered_at_once(project, monkeypatch):
+ """`--run <id>` is a question about a named run, not about this project's
+ last one, and a typo should not cost two minutes."""
+ directory, _ = project
+ record_last_run(directory, recorded_at=_seconds_ago(5))
+ amp = install(monkeypatch, FakeAMP(create=[NOT_FOUND, QUEUED]))
+
+ with pytest.raises(SystemExit):
+ eval_module.eval_crew(run_id="1d6b3f70-0000-4000-8000-000000000000")
+
+ assert [call for call in amp.calls if call[0] == "create"] == [
+ ("create", "1d6b3f70-0000-4000-8000-000000000000")
+ ]
+
+
def test_declining_the_offer_exits_cleanly_with_the_steps(project, monkeypatch, capsys):
(project[0] / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)
@@ -597,13 +1085,35 @@ def test_only_a_missing_login_reads_as_anonymous(monkeypatch, capsys):
monkeypatch.setattr(eval_module, "get_auth_token", lambda: "login-token")
assert eval_module.saved_login() == "login-token"
+ # It stops the evaluation as a VALUE, not as an exit: the same sentence has
+ # to reach a terminal and a screen, and only one of them is a terminal.
for broken in (OSError(13, "Permission denied"), ValueError("Fernet key must be 32 url-safe base64-encoded bytes.")):
monkeypatch.setattr(eval_module, "get_auth_token", lambda error=broken: (_ for _ in ()).throw(error))
- with pytest.raises(SystemExit) as exit_:
+ with pytest.raises(eval_module.EvaluationStoppedError) as stopped:
eval_module.saved_login()
- assert exit_.value.code == 1
- out = capsys.readouterr().out
- assert "Could not read the saved login" in out and type(broken).__name__ in out and "crewai login" in out
+ said = str(stopped.value)
+ assert "Could not read the saved login" in said and type(broken).__name__ in said and "crewai login" in said
+
+
+def test_the_command_still_prints_that_sentence_and_exits(project, monkeypatch, capsys):
+ """What the app shows on its screen, the command says in the terminal."""
+ directory, _ = project
+ record_last_run(directory)
+ stopped = eval_module.EvaluationStoppedError(
+ "Could not read the saved login (OSError: Permission denied). "
+ "Run `crewai login` again, or `crewai eval` will not know who you are."
+ )
+ monkeypatch.setattr(
+ eval_module, "saved_login", lambda: (_ for _ in ()).throw(stopped)
+ )
+ amp = install(monkeypatch, FakeAMP())
+
+ with pytest.raises(SystemExit) as exit_:
+ eval_module.eval_crew()
+
+ assert exit_.value.code == 1
+ assert amp.calls == [] # stopped before anything was asked of AMP
+ assert "Could not read the saved login" in capsys.readouterr().out
def test_read_last_run_reads_the_record_crewai_writes(tmp_path):
diff --git a/lib/cli/tests/test_crew_run_tui.py b/lib/cli/tests/test_crew_run_tui.py
index 77ae40f7f..611a41e3d 100644
--- a/lib/cli/tests/test_crew_run_tui.py
+++ b/lib/cli/tests/test_crew_run_tui.py
@@ -1,3 +1,4 @@
+import contextvars
from datetime import datetime
import time
from types import SimpleNamespace
@@ -32,7 +33,7 @@ from crewai.events.types.tool_usage_events import (
ToolUsageFinishedEvent,
ToolUsageStartedEvent,
)
-from crewai_cli import run_crew
+from crewai_cli import crew_run_tui, run_crew
from crewai_cli.command import AuthenticationRequiredError
from crewai_cli.crew_run_tui import (
_LOG_ARGS_TEXT_LIMIT,
@@ -43,6 +44,17 @@ from crewai_cli.crew_run_tui import (
_try_parse_structured,
)
import pytest
+from rich.text import Text
+
+
[email protected](autouse=True)
+def _awaiting_files_stay_in_tmp(tmp_path, monkeypatch):
+ """The waiting-evaluation token file belongs to the user's crewAI data
+ directory; a test keeps it in its own."""
+ monkeypatch.setattr(
+ "crewai_cli.crew_run_tui._awaiting_dir", lambda: tmp_path / "eval-awaiting"
+ )
+
def _app_with_plan() -> CrewRunApp:
@@ -137,21 +149,56 @@ def test_chain_deploy_does_not_login_for_deploy_exit(monkeypatch, capsys) -> Non
assert "Deploy failed with exit code 42" in capsys.readouterr().out
-def test_view_traces_button_click_records_telemetry(monkeypatch) -> None:
+def test_view_traces_opens_the_link_crewai_recorded_for_this_run(monkeypatch) -> None:
+ """crewAI prints that link after a run but never under a TUI, so the button
+ reads it off the record crewAI wrote — matched to THIS app's execution."""
app = CrewRunApp()
app._status = "completed"
app._telemetry = Mock()
- notice = Mock()
- monkeypatch.setattr(app, "notify", notice)
+ app._execution_uuid = "run-this-app"
+ monkeypatch.setattr(
+ crew_run_tui,
+ "_recorded_run",
+ lambda uuid: {"execution_id": uuid, "trace_url": "https://amp.example/t/1"}
+ if uuid == "run-this-app"
+ else None,
+ )
+ opened: list[str] = []
+ monkeypatch.setattr(CrewRunApp, "_open_report", lambda self, url: opened.append(url))
+ monkeypatch.setattr(app, "notify", Mock())
app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-traces")))
app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:view_traces")
- notice.assert_called_once_with(
- "Trace sharing is requested when the execution finishes. "
- "A trace link is not available for this run.",
- title="Execution traces",
+ assert opened == ["https://amp.example/t/1"]
+
+
+ ("record", "said"),
+ [
+ (None, "not traced"),
+ ({"execution_id": "run-this-app"}, "granted no link"),
+ ],
+)
+def test_view_traces_says_which_of_the_two_reasons_there_is_no_link(
+ monkeypatch, record, said
+) -> None:
+ """A run nobody traced and a traced run AMP gave no link for are different
+ problems, and only one of them is the reader's to fix."""
+ app = CrewRunApp()
+ app._status = "completed"
+ app._telemetry = Mock()
+ app._execution_uuid = "run-this-app"
+ monkeypatch.setattr(crew_run_tui, "_recorded_run", lambda uuid: record)
+ monkeypatch.setattr(
+ CrewRunApp, "_open_report", lambda self, url: pytest.fail("nothing to open")
)
+ notices: list[str] = []
+ monkeypatch.setattr(app, "notify", lambda message, **kwargs: notices.append(message))
+
+ app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-traces")))
+
+ assert said in notices[0]
def test_deploy_button_click_records_telemetry() -> None:
@@ -170,6 +217,363 @@ def test_deploy_button_click_records_telemetry() -> None:
assert exits == [app._crew_result]
+def test_evaluate_button_evaluates_here_and_does_not_leave(monkeypatch) -> None:
+ """The evaluation takes minutes, has a link worth showing and a verdict
+ worth reading. Sending somebody to a bare terminal to wait for them is the
+ worst of both, so it runs on this screen — and it names THIS app's
+ execution, not whichever run finished last in the project."""
+ app = CrewRunApp()
+ app._status = "completed"
+ app._telemetry = Mock()
+ app._execution_uuid = "run-this-app"
+ app.exit = lambda result=None: pytest.fail("the app must stay open") # type: ignore[method-assign]
+ started: list[str] = []
+ monkeypatch.setattr(
+ CrewRunApp, "_evaluate_worker", lambda self, execution_id: started.append(execution_id)
+ )
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: uuid == "run-this-app")
+
+ app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-eval")))
+
+ app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:evaluate")
+ assert started == ["run-this-app"]
+ assert app._evaluation == {"state": "starting", "execution_id": "run-this-app"}
+
+
+def test_the_button_spins_while_it_evaluates_and_still_opens_the_report(
+ monkeypatch,
+) -> None:
+ """A label that never moves reads as a screen that has stopped — and the
+ report page is where the progress is, so the button stays pressable."""
+ app = CrewRunApp()
+ app._status = "completed"
+ app._telemetry = Mock()
+ app._execution_uuid = "run-this-app"
+ monkeypatch.setattr(CrewRunApp, "_evaluate_worker", lambda self, execution_id: None)
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)
+ labels: list[str] = []
+ button = SimpleNamespace(label="Evaluate", disabled=False)
+ monkeypatch.setattr(CrewRunApp, "query_one", lambda self, *a: button)
+ monkeypatch.setattr(CrewRunApp, "_open_report", lambda self, url: None)
+
+ app.action_evaluate_crew()
+ labels.append(str(button.label))
+ app._frame += 1
+ app._refresh_eval_button()
+ labels.append(str(button.label))
+
+ assert all(label.endswith("Evaluating…") for label in labels)
+ assert labels[0] != labels[1] # it moves
+ assert button.disabled is False # and it is not a dead button
+
+ app._evaluation_started({"id": "ev-1", "url": "https://optimize.example/e/1"})
+ app._evaluation_finished({"status": "done", "verdict": {"gate": "passed"}})
+ assert str(button.label) == "Open eval report"
+
+
+def test_pressing_evaluate_again_opens_the_report_it_already_made(monkeypatch) -> None:
+ """One evaluation per run: the second press is somebody looking for the
+ report, not asking to pay for another."""
+ app = CrewRunApp()
+ app._status = "completed"
+ app._telemetry = Mock()
+ app._evaluation = {"state": "done", "url": "https://optimize.example/e/1"}
+ monkeypatch.setattr(
+ CrewRunApp, "_evaluate_worker", lambda self, execution_id: pytest.fail("evaluated twice")
+ )
+ opened: list[str] = []
+ monkeypatch.setattr(CrewRunApp, "_open_report", lambda self, url: opened.append(url))
+
+ app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-eval")))
+
+ assert opened == ["https://optimize.example/e/1"]
+ app._telemetry.feature_usage_span.assert_not_called()
+
+
+def test_an_evaluation_that_stopped_can_be_tried_again(monkeypatch) -> None:
+ """A timeout or a 502 is worth another press, and the run is still there to
+ grade — only a finished one sends the reader to its report instead."""
+ app = CrewRunApp()
+ app._status = "completed"
+ app._telemetry = Mock()
+ app._execution_uuid = "run-this-app"
+ app._evaluation = {"state": "failed", "error": "AMP answered 502."}
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)
+ started: list[str] = []
+ monkeypatch.setattr(
+ CrewRunApp,
+ "_evaluate_worker",
+ lambda self, execution_id: started.append(execution_id),
+ )
+
+ app.action_evaluate_crew()
+
+ assert started == ["run-this-app"]
+
+
+def test_a_login_that_cannot_be_read_is_shown_on_screen(monkeypatch) -> None:
+ """A credential that exists but cannot be read stops the evaluation, and
+ the sentence has to reach the screen: printing it would land under the
+ layout, and exiting would take it away."""
+ from crewai_cli.experimental import eval_crew
+
+ app = CrewRunApp()
+ app._evaluation = {"state": "starting"}
+ monkeypatch.setattr(
+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)
+ )
+
+ def broken_store():
+ raise OSError("permission denied")
+
+ monkeypatch.setattr(eval_crew, "get_auth_token", broken_store)
+
+ app._evaluate_now("run-this-app")
+
+ assert app._evaluation["state"] == "failed"
+ assert "Could not read the saved login" in app._evaluation["error"]
+ assert "crewai login" in app._evaluation["error"]
+
+
+def test_an_exit_with_no_reason_says_that_rather_than_its_code(monkeypatch) -> None:
+ """Nothing on this path should exit any more, but an exit carries a code
+ and `1` on screen is not an answer."""
+ app = CrewRunApp()
+ app._evaluation = {"state": "starting"}
+ monkeypatch.setattr(
+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)
+ )
+ monkeypatch.setattr(
+ "crewai_cli.experimental.eval_crew.evaluate_run",
+ Mock(side_effect=SystemExit(1)),
+ )
+
+ app._evaluate_now("run-this-app")
+
+ assert app._evaluation["state"] == "failed"
+ assert app._evaluation["error"] == (
+ "The evaluation stopped without saying why (exit 1)."
+ )
+
+
+def test_the_evaluation_shows_its_link_its_progress_and_its_verdict() -> None:
+ """Everything the reader came for is on the run's own screen."""
+ app = CrewRunApp()
+ app._evaluation = {"state": "starting"}
+ app._evaluation_started({"id": "ev-1", "url": "https://optimize.example/e/1"})
+ app._open_report = lambda url: None # type: ignore[method-assign]
+
+ line = Text()
+ app._append_evaluation(line)
+ assert "Evaluating this run" in line.plain
+ assert "https://optimize.example/e/1" in line.plain
+
+ app._evaluation_progress({"events": [{"data": {"subject": "the final output", "planned": 9}}]})
+ line = Text()
+ app._append_evaluation(line)
+ assert "judged 1 of 9" in line.plain
+
+ app._evaluation_finished(
+ {"status": "done", "verdict": {"gate": "passed", "grades": {"goal": 5, "tools": None}}}
+ )
+ line = Text()
+ app._append_evaluation(line)
+ assert "Goal gate PASSED" in line.plain
+ assert "goal 5/5" in line.plain and "tools —/5" in line.plain
+ assert "https://optimize.example/e/1" in line.plain
+
+
+def test_an_evaluation_that_stopped_says_so_where_it_was_running() -> None:
+ app = CrewRunApp()
+ app._evaluation = {"state": "running", "url": "https://optimize.example/e/1"}
+ app._evaluation_failed("AMP answered 404 for run x.")
+
+ line = Text()
+ app._append_evaluation(line)
+ assert "Evaluation stopped" in line.plain
+ assert "AMP answered 404 for run x." in line.plain
+
+
[email protected]("why", ["not traced at all", "another run replaced the record"])
+def test_evaluate_says_so_when_this_run_was_not_traced(monkeypatch, why) -> None:
+ """A run with no session of its own, or whose trace never reached AMP, has
+ nothing to grade — and a record naming a DIFFERENT run is not evidence that
+ this one was traced."""
+ app = CrewRunApp()
+ app._status = "completed"
+ app._telemetry = Mock()
+ app._execution_uuid = None if why == "not traced at all" else "run-this-app"
+ notices: list[str] = []
+ app.notify = lambda message, **kwargs: notices.append(message) # type: ignore[method-assign]
+ app.exit = lambda result=None: pytest.fail("the TUI must not leave for nothing") # type: ignore[method-assign]
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: False)
+
+ app.action_evaluate_crew()
+
+ assert app._evaluation is None
+ assert "not traced" in notices[0]
+
+
+def test_tracing_asked_for_reads_the_declaration_but_keeps_the_other_guards(
+ monkeypatch,
+) -> None:
+ """The ContextVar is set where the crew is CONSTRUCTED, not on the worker
+ thread that later asks — but an app with nobody in front of it is still
+ nobody's yes."""
+ utils = "crewai.events.listeners.tracing.utils"
+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)
+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)
+ app = CrewRunApp()
+ app._crew = SimpleNamespace(tracing=True)
+
+ assert app._tracing_was_asked_for() is True
+ assert app._request_trace_consent() is True
+
+ app._crew = SimpleNamespace(tracing=None)
+ assert app._tracing_was_asked_for() is False
+
+ # embedded, redirected, or under test: the declaration is not a person
+ app._crew = SimpleNamespace(tracing=True)
+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: False)
+ assert app._tracing_was_asked_for() is False
+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)
+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: True)
+ assert app._tracing_was_asked_for() is False
+
+
+def test_a_run_started_for_an_evaluation_evaluates_it_here(monkeypatch) -> None:
+ """`crewai eval` runs the crew when nothing is traced. The evaluation it
+ came for starts on this screen when the run ends — the app does not close
+ and leave somebody in a bare terminal watching for a link."""
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)
+ started: list[str] = []
+ monkeypatch.setattr(
+ CrewRunApp,
+ "_evaluate_worker",
+ lambda self, execution_id: started.append(execution_id),
+ )
+ with crew_run_tui.evaluating_after_run() as watched:
+ app = CrewRunApp()
+ app._execution_uuid = "run-this-app"
+ app.exit = lambda result=None: pytest.fail("the app must stay open") # type: ignore[method-assign]
+
+ app._evaluate_if_one_is_waiting()
+
+ assert started == ["run-this-app"]
+ assert watched["execution_id"] == "run-this-app"
+
+
+def test_a_child_process_run_reads_the_waiting_evaluation_off_its_environment(
+ monkeypatch,
+) -> None:
+ """A project's crew runs through `uv run …`, so the app is in another
+ process — the one thing that crosses is the environment the command passed
+ it."""
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)
+ started: list[str] = []
+ monkeypatch.setattr(
+ CrewRunApp,
+ "_evaluate_worker",
+ lambda self, execution_id: started.append(execution_id),
+ )
+
+ def the_child() -> None:
+ # a fresh context: that variable, and nothing this process holds in memory
+ assert crew_run_tui._AUTO_EVAL.get() is None
+ app = CrewRunApp()
+ app._execution_uuid = "run-in-the-child"
+ app.exit = lambda result=None: pytest.fail("the app must stay open") # type: ignore[method-assign]
+ app._evaluate_if_one_is_waiting()
+
+ with crew_run_tui.evaluating_after_run():
+ contextvars.Context().run(the_child)
+
+ assert started == ["run-in-the-child"]
+
+
[email protected]("value", ["1", "true", "0" * 32])
+def test_a_project_env_cannot_say_an_evaluation_is_waiting(monkeypatch, value) -> None:
+ """The child loads the project's `.env` over its environment. A value the
+ command did not mint — with no token file behind it — is no handshake, so
+ a plain `crewai run` never starts an evaluation on its own."""
+ monkeypatch.setenv("CREWAI_EVAL_AWAITING_RUN", value)
+
+ assert crew_run_tui._an_evaluation_is_waiting() is None
+
+
+def test_the_token_stops_counting_when_the_command_is_done(monkeypatch) -> None:
+ with crew_run_tui.evaluating_after_run():
+ passed = crew_run_tui.os.environ["CREWAI_EVAL_AWAITING_RUN"]
+
+ monkeypatch.setenv("CREWAI_EVAL_AWAITING_RUN", passed)
+ assert crew_run_tui._an_evaluation_is_waiting() is None
+
+
+def test_a_run_with_no_trace_says_so_instead_of_evaluating(monkeypatch) -> None:
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: False)
+ marked: list[str | None] = []
+ monkeypatch.setattr(
+ "crewai_cli.experimental.eval_crew.record_evaluation_outcome", marked.append
+ )
+ monkeypatch.setattr(
+ CrewRunApp,
+ "_evaluate_worker",
+ lambda self, execution_id: pytest.fail("nothing to grade"),
+ )
+ with crew_run_tui.evaluating_after_run() as watched:
+ app = CrewRunApp()
+ app._execution_uuid = "run-this-app"
+ notices: list[str] = []
+ app.notify = lambda message, **kwargs: notices.append(message) # type: ignore[method-assign]
+
+ app._evaluate_if_one_is_waiting()
+
+ assert watched["execution_id"] is None
+ assert "not traced" in notices[0]
+ # and the command waiting behind the screen is told, since it cannot see it
+ assert marked == [None]
+
+
+def test_a_run_that_failed_is_still_evaluated(monkeypatch) -> None:
+ """Where a run went wrong is what somebody asking for an evaluation wants
+ to know, and the trace it exported is still there."""
+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)
+ started: list[str] = []
+ monkeypatch.setattr(
+ CrewRunApp,
+ "_evaluate_worker",
+ lambda self, execution_id: started.append(execution_id),
+ )
+ with crew_run_tui.evaluating_after_run():
+ app = CrewRunApp()
+ app._status = "failed"
+ app._execution_uuid = "the-run-that-broke"
+
+ app._evaluate_if_one_is_waiting()
+
+ assert started == ["the-run-that-broke"]
+
+
+def test_a_run_nobody_is_waiting_on_evaluates_nothing() -> None:
+ app = CrewRunApp()
+ app.exit = lambda result=None: pytest.fail("the app must stay open") # type: ignore[method-assign]
+
+ app._evaluate_if_one_is_waiting()
+
+ assert app._evaluation is None
+
+
+def test_evaluate_before_completion_records_nothing() -> None:
+ app = CrewRunApp()
+ app._status = "running"
+ app._telemetry = Mock()
+
+ app.action_evaluate_crew()
+
+ app._telemetry.feature_usage_span.assert_not_called()
+ assert app._evaluation is None
+
+
def test_conversation_turn_done_records_assistant_message() -> None:
class RawResult:
raw = "hello from the flow"
diff --git a/lib/cli/tests/test_run_declarative_flow.py b/lib/cli/tests/test_run_declarative_flow.py
index e6c1e1ff8..2fde34a34 100644
--- a/lib/cli/tests/test_run_declarative_flow.py
+++ b/lib/cli/tests/test_run_declarative_flow.py
@@ -418,7 +418,7 @@ def test_id_restore_still_drops_unknown_keys(
# ── TUI vs terminal (headless/deploy) routing ──────────────────────
-def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False):
+def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False, evaluation=None):
"""Replace CrewRunApp/EventListener/summary so _run_declarative_flow_tui is
driven by a controllable fake app."""
@@ -430,6 +430,7 @@ def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False):
self._crew_name = crew_name
self._status = status
self._want_deploy = want_deploy
+ self._evaluation = evaluation
self._crew_result = "result"
def run(self):
@@ -541,6 +542,34 @@ def test_run_declarative_flow_tui_no_deploy_when_not_requested(
assert deploy_calls == []
+def test_run_declarative_flow_tui_leaves_the_report_link_behind(capsys) -> None:
+ """A declarative flow evaluates inside the app like a crew does. What the
+ terminal is for is afterwards: the link, once the screen that held it is
+ gone."""
+ app = SimpleNamespace(
+ _crew_name="Flow",
+ _status="completed",
+ _elapsed_frozen=1.0,
+ _start_time=0.0,
+ _input_tokens=0,
+ _output_tokens=0,
+ _live_out_tokens=0,
+ _final_output=None,
+ _error=None,
+ _evaluation={
+ "state": "done",
+ "url": "https://optimize.example/e/1",
+ "verdict": {"gate": "passed"},
+ },
+ )
+
+ run_declarative_flow_module._print_flow_post_tui_summary(app)
+
+ printed = capsys.readouterr().out
+ assert "goal gate PASSED" in printed
+ assert "https://optimize.example/e/1" in printed
+
+
def test_run_declarative_flow_tui_enables_flow_events(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/lib/cli/tests/test_trace_consent_session.py b/lib/cli/tests/test_trace_consent_session.py
index 1d9579d12..a4c37f0be 100644
--- a/lib/cli/tests/test_trace_consent_session.py
+++ b/lib/cli/tests/test_trace_consent_session.py
@@ -18,10 +18,22 @@ class ConsentApp(CrewRunApp):
event.prevent_default()
+def _say_a_person_is_here(monkeypatch):
+ """The rule only holds where a prompt could have been shown, and a suite is
+ one of the places it could not. These cases are about a real session."""
+ utils = "crewai.events.listeners.tracing.utils"
+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)
+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)
+
+
async def wait_for_consent(app, pilot):
for _ in range(50):
await pilot.pause(0.01)
- if isinstance(app.screen, TraceConsentScreen):
+ # A pushed screen is not a laid-out one, and a click lands on the
+ # region the buttons have — zero-sized until the first layout pass.
+ if isinstance(app.screen, TraceConsentScreen) and all(
+ button.region for button in app.screen.query(Button)
+ ):
return app.screen
raise AssertionError("The execution worker did not open the consent prompt")
@@ -63,6 +75,53 @@ async def test_session_consent_buttons_and_keys_resolve_execution_worker(
assert app._trace_consent_pending is None
[email protected]("value", ["true", "1"])
+async def test_tracing_asked_for_needs_no_modal(monkeypatch, value):
+ """The user turned tracing on for this project; the modal would be the same
+ question a second time."""
+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", value)
+ _say_a_person_is_here(monkeypatch)
+ app = ConsentApp()
+ async with app.run_test(size=(100, 40)) as pilot:
+ assert await asyncio.to_thread(app._request_trace_consent) is True
+ await pilot.pause()
+ assert not isinstance(app.screen, TraceConsentScreen)
+ assert app._trace_consent_pending is None
+
+
+async def test_the_app_suppresses_messages_and_still_needs_no_modal(monkeypatch):
+ """This app silences crewAI's console to keep it out of its own layout. That
+ is a reason a PROMPT cannot be shown, not a reason to ask again — the app
+ has a screen and somebody in front of it."""
+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
+ _say_a_person_is_here(monkeypatch)
+ monkeypatch.setattr(
+ "crewai.events.listeners.tracing.utils.should_suppress_tracing_messages",
+ lambda: True,
+ )
+ app = ConsentApp()
+ async with app.run_test(size=(100, 40)) as pilot:
+ assert await asyncio.to_thread(app._request_trace_consent) is True
+ await pilot.pause()
+ assert not isinstance(app.screen, TraceConsentScreen)
+
+
+async def test_a_declaration_is_the_same_yes_as_the_variable(monkeypatch):
+ """`tracing=True` never reaches this worker's context — the crew set it on
+ the thread it was built on — so the declaration itself is read."""
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+ _say_a_person_is_here(monkeypatch)
+ app = ConsentApp()
+ app._crew = SimpleNamespace(tracing=True)
+ async with app.run_test(size=(100, 40)) as pilot:
+ assert await asyncio.to_thread(app._request_trace_consent) is True
+ await pilot.pause()
+ assert not isinstance(app.screen, TraceConsentScreen)
+
+
@pytest.mark.asyncio
async def test_consent_timeout_returns_false_and_dismisses_modal(monkeypatch):
app = ConsentApp()
diff --git a/lib/crewai-core/src/crewai_core/telemetry.py b/lib/crewai-core/src/crewai_core/telemetry.py
index 3d86d1d39..a5e4aca26 100644
--- a/lib/crewai-core/src/crewai_core/telemetry.py
+++ b/lib/crewai-core/src/crewai_core/telemetry.py
@@ -572,8 +572,29 @@ class Telemetry:
self._safe_telemetry_procedure(_operation)
- def feature_usage_span(self, feature: str) -> None:
- """Records that a feature was used. One span = one count."""
+ # What a caller may add to a feature span, per feature. Anything else is
+ # dropped: the span counts a use, and a key nobody agreed on is how run
+ # content — a prompt, an output, a path — or an identifier reaches stats
+ # that promise to hold neither. The dimensions every span already carries
+ # are never on a list: a caller that could overwrite `feature` could file
+ # its use under somebody else's name.
+ FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {
+ "cli_usage:eval": frozenset({"authenticated"}),
+ }
+
+ def feature_usage_span(
+ self, feature: str, attributes: dict[str, str] | None = None
+ ) -> None:
+ """Records that a feature was used. One span = one count.
+
+ Args:
+ feature: Feature identifier, e.g. ``"cli_usage:eval"``.
+ attributes: What a caller knows about THIS use that the common
+ attributes cannot know. Only the keys ``FEATURE_ATTRIBUTES`` lists
+ for this feature are kept; anything else is dropped, including
+ the dimensions every span already carries (the project id, the
+ runtime and the version).
+ """
from crewai_core.version import get_crewai_version
def _operation() -> None:
@@ -581,6 +602,10 @@ class Telemetry:
span = tracer.start_span("Feature Usage")
self._add_attribute(span, "crewai_version", get_crewai_version())
self._add_attribute(span, "feature", feature)
+ allowed = self.FEATURE_ATTRIBUTES.get(feature, frozenset())
+ for key, value in (attributes or {}).items():
+ if key in allowed:
+ self._add_attribute(span, key, str(value))
close_span(span)
self._safe_telemetry_procedure(_operation)
diff --git a/lib/crewai-core/tests/test_smoke.py b/lib/crewai-core/tests/test_smoke.py
index beba3a868..22097086a 100644
--- a/lib/crewai-core/tests/test_smoke.py
+++ b/lib/crewai-core/tests/test_smoke.py
@@ -269,6 +269,48 @@ def test_core_telemetry_records_feature_usage(
span.end.assert_called_once()
+def test_core_feature_span_keeps_only_what_the_feature_may_send(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Each feature names what it may add. A key off its list — an id, run
+ content, or a dimension every span already carries — is dropped, and a
+ feature with no list sends nothing extra."""
+ from crewai_core.telemetry import Telemetry
+
+ Telemetry._instance = None
+ monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)
+ monkeypatch.delenv("CREWAI_DISABLE_TELEMETRY", raising=False)
+ monkeypatch.delenv("CREWAI_DISABLE_TRACKING", raising=False)
+
+ tracer = Mock()
+ span = Mock()
+ tracer.start_span.return_value = span
+ monkeypatch.setattr(
+ "crewai_core.telemetry.TracerProvider",
+ lambda **_kwargs: Mock(get_tracer=Mock(return_value=tracer)),
+ )
+ offered = {
+ "authenticated": "true",
+ "execution_id": "run-1",
+ "organization_id": "org-1",
+ "feature": "someone_else",
+ "crewai_version": "0.0.0",
+ }
+
+ telemetry = Telemetry()
+ telemetry.feature_usage_span("cli_usage:eval", offered)
+ sent = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}
+ assert sent["authenticated"] == "true"
+ assert sent["feature"] == "cli_usage:eval"
+ assert sent["crewai_version"] != "0.0.0"
+ assert "execution_id" not in sent and "organization_id" not in sent
+
+ span.set_attribute.reset_mock()
+ telemetry.feature_usage_span("cli_usage:deploy", offered)
+ sent = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}
+ assert "authenticated" not in sent and sent["feature"] == "cli_usage:deploy"
+
+
def test_core_telemetry_records_flow_creation_version(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/lib/crewai/src/crewai/events/listeners/tracing/utils.py b/lib/crewai/src/crewai/events/listeners/tracing/utils.py
index 0b434c265..c58125441 100644
--- a/lib/crewai/src/crewai/events/listeners/tracing/utils.py
+++ b/lib/crewai/src/crewai/events/listeners/tracing/utils.py
@@ -55,6 +55,7 @@ __all__ = [
"should_auto_collect_first_time_traces",
"should_enable_tracing",
"should_suppress_tracing_messages",
+ "tracing_asked_for",
"truncate_messages",
"update_user_data",
]
@@ -105,6 +106,32 @@ def should_suppress_tracing_messages() -> bool:
return _suppress_tracing_messages.get()
+def tracing_asked_for() -> bool:
+ """True when a PERSON turned tracing on for this run — `CREWAI_TRACING_ENABLED`,
+ or `tracing=True` on the crew or flow in this context — and is there to have
+ meant it.
+
+ Turning tracing on is the answer to "may we collect this?"; asking again when
+ the run ends is asking the same question twice. First-time auto-collection is
+ the case that still has to ask, because nobody asked for it.
+
+ Where the prompt could not be shown, there is nothing to answer. A copied
+ `.env` that reaches CI, a container or a server carries the variable without
+ carrying the person; a suite under test, and a host that asked for tracing
+ messages to be suppressed, are the same case. An ephemeral run has always
+ failed closed in all three rather than upload what nobody approved in front
+ of a screen, and it still does — this turns a second question into an
+ answer, and changes nothing about where the first one is asked.
+ """
+ if not (
+ is_tracing_enabled_in_context()
+ or os.getenv("CREWAI_TRACING_ENABLED", "").lower() in ("true", "1")
+ ):
+ return False
+
+ return _prompt_can_be_shown()
+
+
def should_enable_tracing(*, override: bool | None = None) -> bool:
"""Determine if tracing should be enabled.
@@ -486,6 +513,25 @@ def _is_interactive_terminal() -> bool:
return False
+def _prompt_can_be_shown() -> bool:
+ """Could a question about traces be put to somebody at all?
+
+ Three cases where it cannot: a suite under test, a host that asked for
+ tracing messages to be suppressed, and a process with no terminal (CI, API
+ servers, Docker) where a prompt would block for twenty seconds with nobody
+ to answer it. `prompt_user_for_trace_viewing` returns False in each, which
+ is what keeps an unapproved ephemeral trace on the machine — and
+ `tracing_asked_for` asks the same question, so the two cannot drift apart.
+ """
+ if _is_test_environment():
+ return False
+
+ if should_suppress_tracing_messages():
+ return False
+
+ return _is_interactive_terminal()
+
+
def prompt_user_for_trace_viewing(
timeout_seconds: int = 20, *, sharing: bool = False
) -> bool:
@@ -494,15 +540,7 @@ def prompt_user_for_trace_viewing(
Returns True if user agrees, False otherwise. ``sharing`` explicitly asks
permission to upload locally buffered spans rather than merely view a trace.
"""
- if _is_test_environment():
- return False
-
- if should_suppress_tracing_messages():
- return False
-
- # Skip prompt in non-interactive contexts (CI, API servers, Docker, etc.)
- # This avoids blocking for 20 seconds when no one can respond
- if not _is_interactive_terminal():
+ if not _prompt_can_be_shown():
return False
try:
diff --git a/lib/crewai/src/crewai/telemetry/telemetry.py b/lib/crewai/src/crewai/telemetry/telemetry.py
index 1e4e0041f..a34fbe4ca 100644
--- a/lib/crewai/src/crewai/telemetry/telemetry.py
+++ b/lib/crewai/src/crewai/telemetry/telemetry.py
@@ -18,7 +18,7 @@ import os
import platform
import signal
import threading
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
from crewai_core.telemetry import (
CommonAttributesSpanProcessor,
@@ -1285,13 +1285,29 @@ class Telemetry:
self._safe_telemetry_operation(_operation)
- def feature_usage_span(self, feature: str) -> None:
+ # What a caller may add to a feature span, per feature. Anything else is
+ # dropped: the span counts a use, and a key nobody agreed on is how run
+ # content — a prompt, an output, a path — or an identifier reaches stats
+ # that promise to hold neither. The dimensions every span already carries
+ # are never on a list: a caller that could overwrite `feature` could file
+ # its use under somebody else's name.
+ FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {
+ "cli_usage:eval": frozenset({"authenticated"}),
+ }
+
+ def feature_usage_span(
+ self, feature: str, attributes: dict[str, str] | None = None
+ ) -> None:
"""Records that a feature was used. One span = one count.
Args:
feature: Feature identifier, e.g. "planning:creation",
"mcp:connection", "a2a:delegation",
"hooks:pre_tool_call", "hooks:aborted".
+ attributes: What a caller knows about THIS use that the common
+ attributes cannot know. Only the keys ``FEATURE_ATTRIBUTES`` lists
+ for this feature are kept; anything else is dropped, including
+ the dimensions every span already carries.
"""
def _operation() -> None:
@@ -1299,6 +1315,10 @@ class Telemetry:
span = tracer.start_span("Feature Usage")
self._add_attribute(span, "crewai_version", version("crewai"))
self._add_attribute(span, "feature", feature)
+ allowed = self.FEATURE_ATTRIBUTES.get(feature, frozenset())
+ for key, value in (attributes or {}).items():
+ if key in allowed:
+ self._add_attribute(span, key, str(value))
close_span(span)
self._safe_telemetry_operation(_operation)
diff --git a/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py b/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py
index 0c160a895..4c870a519 100644
--- a/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py
+++ b/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py
@@ -19,7 +19,10 @@ from opentelemetry.sdk.trace.export import (
SpanExporter,
)
-from crewai.events.listeners.tracing.utils import prompt_user_for_trace_viewing
+from crewai.events.listeners.tracing.utils import (
+ prompt_user_for_trace_viewing,
+ tracing_asked_for,
+)
from crewai.telemetry.tracing.grants import (
GrantSpanExporter,
TraceGrantClient,
@@ -122,10 +125,18 @@ class EphemeralSpanBuffer(SpanExporter):
)
if not buffered:
return
+ # A host's consent callback stays authoritative — it is policy, and
+ # an embedder may answer no for reasons of its own. What it replaces
+ # is the PROMPT, and a prompt is what a user who turned tracing on
+ # has already answered: CREWAI_TRACING_ENABLED, or tracing=True, is
+ # the yes. (A callback that opens its own prompt — the TUI's modal —
+ # asks `tracing_asked_for()` for itself.)
consent = self._consent or _trace_consent.get()
approved = (
consent()
if consent is not None
+ else True
+ if tracing_asked_for()
else prompt_user_for_trace_viewing(sharing=True)
)
if first_time:
diff --git a/lib/crewai/src/crewai/telemetry/tracing/grants.py b/lib/crewai/src/crewai/telemetry/tracing/grants.py
index 9678c3097..a9dcc3071 100644
--- a/lib/crewai/src/crewai/telemetry/tracing/grants.py
+++ b/lib/crewai/src/crewai/telemetry/tracing/grants.py
@@ -298,6 +298,7 @@ class GrantSpanExporter(SpanExporter):
started_at_ns=self._first_start_ns,
finished_at_ns=self._last_end_ns,
amp_base_url=getattr(api, "base_url", None),
+ trace_url=self._trace_url,
)
logger.debug("Traces exported for execution %s", execution_uuid)
self._show_trace_link()
diff --git a/lib/crewai/src/crewai/telemetry/tracing/last_run.py b/lib/crewai/src/crewai/telemetry/tracing/last_run.py
index 53e137dbd..e8b78464f 100644
--- a/lib/crewai/src/crewai/telemetry/tracing/last_run.py
+++ b/lib/crewai/src/crewai/telemetry/tracing/last_run.py
@@ -72,6 +72,7 @@ def record_last_run(
started_at_ns: int | None,
finished_at_ns: int | None,
amp_base_url: str | None,
+ trace_url: str | None = None,
) -> Path | None:
"""Write the record atomically; the path, or None when recording is off
or the write failed. Never raises — a run is never failed by this.
@@ -90,6 +91,11 @@ def record_last_run(
"recorded_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
"amp_base_url": amp_base_url,
}
+ # AMP's viewer link for this run, when it granted one. The run app shows no
+ # console line — crewAI's own link is silenced under a TUI — so the record
+ # is the only way its View Traces button can know where to go.
+ if trace_url:
+ record["trace_url"] = trace_url
path: Path | None = None
try:
path = last_run_path()
diff --git a/lib/crewai/tests/telemetry/test_deferred_trace_transitions.py b/lib/crewai/tests/telemetry/test_deferred_trace_transitions.py
index d97615e38..f7a8b5cc6 100644
--- a/lib/crewai/tests/telemetry/test_deferred_trace_transitions.py
+++ b/lib/crewai/tests/telemetry/test_deferred_trace_transitions.py
@@ -115,6 +115,69 @@ def test_disabling_deferred_trace_discards_without_consent(
assert get_trace_session() is None and get_execution_uuid() is None
[email protected]("how", ["env", "flag"])
+def test_tracing_asked_for_is_the_answer_and_the_run_is_not_asked_again(
+ traces, monkeypatch, how
+):
+ """Turning tracing on IS consent. The prompt at the end of a run is for the
+ first-time collection nobody asked for — not for a user who said collect it."""
+ buffers, grants, recorders, prompt = traces
+ # the rule only holds where a prompt could have been shown, and a suite is
+ # one of the places it could not — so say a person is here
+ utils = "crewai.events.listeners.tracing.utils"
+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)
+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)
+ if how == "env":
+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
+ else: # the flag alone must carry it, or this case proves nothing
+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)
+
+ class Conversation(Flow):
+ @start()
+ def turn(self):
+ return "said out loud"
+
+ flow = Conversation(tracing=(how == "flag") or None)
+ assert flow.kickoff() == "said out loud"
+ assert crewai_event_bus.flush()
+
+ prompt.assert_not_called() # asked once, not twice
+ assert grants and recorders # and the trace went where it was told to go
+
+
+ "closed", ["no terminal", "under test", "messages suppressed"]
+)
+def test_where_the_prompt_could_not_be_shown_nothing_is_uploaded(
+ traces, monkeypatch, closed
+):
+ """The switch answers a question; where the question could not have been
+ put to anybody, there is nothing to answer. A copied `.env` reaching CI
+ carries the variable, not the person — and a suite under test, and a host
+ that suppressed tracing messages, are the same case. All three have always
+ failed closed, and still do."""
+ buffers, grants, recorders, prompt = traces
+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")
+ utils = "crewai.events.listeners.tracing.utils"
+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: closed != "no terminal")
+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: closed == "under test")
+ monkeypatch.setattr(
+ f"{utils}.should_suppress_tracing_messages",
+ lambda: closed == "messages suppressed",
+ )
+ prompt.return_value = False # what the prompt answers where nobody can answer
+
+ class Conversation(Flow):
+ @start()
+ def turn(self):
+ return "said in a container"
+
+ assert Conversation(tracing=True).kickoff() == "said in a container"
+ assert crewai_event_bus.flush()
+
+ assert not grants and not recorders # nothing left the machine
+
+
def test_enabling_deferred_trace_opens_a_new_flow_root(traces):
buffers, grants, recorders, prompt = traces
sessions = []
diff --git a/lib/crewai/tests/telemetry/test_session_trace_export.py b/lib/crewai/tests/telemetry/test_session_trace_export.py
index db2ac76d6..ee39e61a0 100644
--- a/lib/crewai/tests/telemetry/test_session_trace_export.py
+++ b/lib/crewai/tests/telemetry/test_session_trace_export.py
@@ -223,6 +223,10 @@ def test_the_viewer_link_is_shown_the_id_is_not_and_a_successful_export_is_recor
assert written["tier"] == ("authenticated" if authenticated else "ephemeral")
assert written["started_at"] and written["finished_at"] and written["recorded_at"]
assert written["amp_base_url"] == collector.url
+ # The link is recorded even where it is never printed: under a TUI the
+ # console line is silenced, and the run app's View Traces button has
+ # nowhere else to read it from.
+ assert written["trace_url"] == url
else:
assert written is None
diff --git a/lib/crewai/tests/telemetry/test_trace_lifecycle.py b/lib/crewai/tests/telemetry/test_trace_lifecycle.py
index 5e62866e5..fce79e34b 100644
--- a/lib/crewai/tests/telemetry/test_trace_lifecycle.py
+++ b/lib/crewai/tests/telemetry/test_trace_lifecycle.py
@@ -118,6 +118,9 @@ def test_pause_resume_exports_traces_and_preserves_deferred_root(
def finish(self, feedback):
return feedback.feedback
+ utils = "crewai.events.listeners.tracing.utils"
+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)
+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)
with patch(
"crewai.telemetry.tracing.ephemeral.prompt_user_for_trace_viewing",
return_value=True,
@@ -142,7 +145,8 @@ def test_pause_resume_exports_traces_and_preserves_deferred_root(
assert all(
span.status.status_code != trace.StatusCode.ERROR for span in paused
)
- assert prompt.call_count == (0 if authenticated or deferred else 1)
+ # `tracing=True` IS the consent: nobody is asked again mid-flight
+ assert prompt.call_count == 0
assert get_trace_session() is None and get_execution_uuid() is None
resumed = (
@@ -159,7 +163,7 @@ def test_pause_resume_exports_traces_and_preserves_deferred_root(
flow.finalize_session_traces()
flow.finalize_session_traces()
- assert prompt.call_count == (0 if authenticated else (1 if deferred else 2))
+ assert prompt.call_count == 0 # … nor when the deferred trace is finalised
assert flow._deferred_execution_trace is None
assert len(issued) == (1 if deferred else 2)
diff --git a/lib/crewai/tests/telemetry/test_tracer_isolation.py b/lib/crewai/tests/telemetry/test_tracer_isolation.py
index 9ee96383d..3783988f7 100644
--- a/lib/crewai/tests/telemetry/test_tracer_isolation.py
+++ b/lib/crewai/tests/telemetry/test_tracer_isolation.py
@@ -94,6 +94,30 @@ def test_our_own_spans_still_reach_our_exporter(telemetry_with_exporter):
assert [span.name for span in exporter.get_finished_spans()] == ["Feature Usage"]
+def test_a_feature_span_keeps_only_what_the_feature_may_send(telemetry_with_exporter):
+ """The same policy as crewai-core's: each feature names what it may add, and
+ the dimensions every span already carries cannot be overwritten."""
+ telemetry, exporter = telemetry_with_exporter
+ telemetry.set_tracer()
+ offered = {
+ "authenticated": "false",
+ "execution_id": "run-1",
+ "organization_id": "org-1",
+ "feature": "someone_else",
+ }
+
+ telemetry.feature_usage_span("cli_usage:eval", offered)
+ telemetry.feature_usage_span("cli_usage:deploy", offered)
+
+ eval_span, deploy_span = exporter.get_finished_spans()
+ assert eval_span.attributes["authenticated"] == "false"
+ assert eval_span.attributes["feature"] == "cli_usage:eval"
+ assert "execution_id" not in eval_span.attributes
+ assert "organization_id" not in eval_span.attributes
+ assert "authenticated" not in deploy_span.attributes
+ assert deploy_span.attributes["feature"] == "cli_usage:deploy"
+
+
def test_our_spans_are_unaffected_by_an_application_provider(telemetry_with_exporter):
"""An app that installs its own provider must not divert our telemetry.