Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix(tracing): turning tracing on is the answer, and an Evaluate button in the TUI (#7765)
João Moura4dcd19aa2ef9
diff --git a/lib/cli/src/crewai_cli/crew_run_tui.py b/lib/cli/src/crewai_cli/crew_run_tui.pyindex 3aad179b4..1ceeda579 100644--- a/lib/cli/src/crewai_cli/crew_run_tui.py+++ b/lib/cli/src/crewai_cli/crew_run_tui.py@@ -5,8 +5,15 @@ Two-column layout: left sidebar (tasks/agents/tokens) + main content"""import asyncio+from collections.abc import Callable, Iterator+import contextlib+from contextlib import contextmanager+from contextvars import ContextVarimport json as _json+import os+from pathlib import Pathimport re+import secretsimport threadingimport timefrom typing import Any, ClassVar@@ -289,6 +296,116 @@ class TraceConsentScreen(ModalScreen[bool]):self.dismiss(False)+_AUTO_EVAL: ContextVar[dict[str, str | None] | None] = ContextVar(+ "crewai_tui_auto_eval", default=None+)++# The same word, for the app that runs in a CHILD process. A project's crew and+# flow are run through `uv run …` in the project's own environment, and nothing+# in this process's memory reaches that app — the environment does, and it is+# the only thing that does. Set and cleared by `evaluating_after_run()` alone:+# an internal handshake between two parts of one command, never a setting for+# anyone to turn on.+#+# The child loads the project's `.env` over its environment, so the variable+# alone would be a switch any project could flip — an evaluation started, with+# the machine's `crewai login`, on a plain `crewai run`. Its value is therefore a+# random token, and it counts only while the file of that name exists in this+# user's crewAI data directory: the command writes it before the run and removes+# it after, and a `.env` cannot create a file.+_AWAITING_EVAL_ENV = "CREWAI_EVAL_AWAITING_RUN"+_AWAITING_TOKEN = re.compile(r"[0-9a-f]{32}")+++def _awaiting_dir() -> Path:+ import appdirs++ return Path(appdirs.user_data_dir("crewai", "CrewAI")) / "eval-awaiting"+++@contextmanager+def evaluating_after_run() -> Iterator[dict[str, str | None]]:+ """Run the crew for an evaluation that is already under way.++ `crewai eval` with nothing traced offers to run the crew first. The app it+ opens would otherwise sit there until somebody quits it, with the command+ waiting behind — so inside this block the app closes itself when the run+ ends and leaves its execution id in the holder. It does NOT chain into an+ evaluation of its own: the command that opened it is the one evaluating.++ A run in the same process leaves its id here; a run in a child process+ leaves it in the project's last-run record, which the caller reads when the+ holder comes back empty.+ """+ holder: dict[str, str | None] = {"execution_id": None}+ token = _AUTO_EVAL.set(holder)+ before = os.environ.get(_AWAITING_EVAL_ENV)+ # Without the file a child app cannot be told, and the command grades the+ # run itself once the app closes — slower to the verdict, never wrong.+ marker: Path | None = None+ with contextlib.suppress(OSError):+ nonce = secrets.token_hex(16)+ _awaiting_dir().mkdir(parents=True, exist_ok=True)+ (_awaiting_dir() / nonce).touch(exist_ok=False)+ marker = _awaiting_dir() / nonce+ os.environ[_AWAITING_EVAL_ENV] = nonce+ try:+ yield holder+ finally:+ _AUTO_EVAL.reset(token)+ if marker is not None:+ with contextlib.suppress(OSError):+ marker.unlink()+ if before is None:+ os.environ.pop(_AWAITING_EVAL_ENV, None)+ else:+ os.environ[_AWAITING_EVAL_ENV] = before+++def _an_evaluation_is_waiting() -> dict[str, str | None] | None:+ """The evaluation this run was started for, if there is one.++ In this process the holder itself; in a child process the environment says+ an evaluation is waiting and the holder is a local one — the id travels+ back through the project's last-run record instead, which the child writes+ when its trace is exported.+ """+ holder = _AUTO_EVAL.get()+ if holder is not None:+ return holder++ value = os.environ.get(_AWAITING_EVAL_ENV) or ""+ if not _AWAITING_TOKEN.fullmatch(value):+ return None+ try:+ waiting = (_awaiting_dir() / value).is_file()+ except OSError:+ return None+ return {"execution_id": None} if waiting else None+++def _recorded_run(execution_uuid: str) -> dict[str, Any] | None:+ """This execution's own record, if its trace reached AMP.++ crewAI records a run when its spans are exported, so the record answers the+ one question the uuid alone cannot: whether there is anything to grade, and+ where AMP will show it. It is matched by id rather than trusted as "the+ last run", which it is only until another run in the project finishes.+ """+ # a missing or unreadable record simply means "not traced"+ with contextlib.suppress(Exception):+ from crewai.telemetry.tracing.last_run import read_last_run++ record = read_last_run() or {}+ if str(record.get("execution_id") or "") == execution_uuid:+ return record+ return None+++def _trace_was_recorded(execution_uuid: str) -> bool:+ return _recorded_run(execution_uuid) is not None++class CrewRunApp(App[Any]):TITLE = "CrewAI"@@ -408,16 +525,25 @@ FooterKey .footer-key--key {text-style: bold;}-#btn-traces {+#btn-eval {background: #1F7982;color: #e0e0e0;border: none;}-#btn-traces:hover {+#btn-eval:hover {background: #28969f;}++#btn-traces {+ background: #2b2b2b;+ color: #e0e0e0;+ border: none;+}+#btn-traces:hover {+ background: #3d3d3d;+}#btn-traces:disabled {- background: #1a4a50;+ background: #202020;color: #888888;}@@ -437,6 +563,7 @@ FooterKey .footer-key--key {Binding("s", "toggle_sidebar", "Sidebar"),Binding("l", "toggle_logs", "Logs"),Binding("t", "view_traces", "Traces", show=False),+ Binding("e", "evaluate_crew", "Evaluate", show=False),Binding("d", "deploy_crew", "Deploy", show=False),Binding("down", "log_down", "Log ↓", show=False),Binding("up", "log_up", "Log ↑", show=False),@@ -529,6 +656,19 @@ FooterKey .footer-key--key {self._current_method: str | None = Noneself._elapsed_frozen: float | None = Noneself._want_deploy: bool = False+ # This app's own execution, captured from the run while it runs. The+ # project-wide record is the last run to FINISH anywhere, which another+ # run in the same project can replace between this one ending and its+ # button being pressed.+ self._execution_uuid: str | None = None+ # The evaluation of this run, while it happens and after: it runs from+ # inside the app, on the app's own screen, rather than sending anybody+ # back to a terminal to watch a link appear.+ self._evaluation: dict[str, Any] | None = None+ # Read here, on the thread that built the app, because that is where+ # `crewai eval` set it — or in the environment it passed to this+ # process, when the run is a child of that command.+ self._auto_eval: dict[str, str | None] | None = _an_evaluation_is_waiting()self._consent_screen: TraceConsentScreen | None = Noneself._trace_consent_pending: threading.Event | None = Noneself._discard_trace_on_exit = False@@ -556,6 +696,7 @@ FooterKey .footer-key--key {with VerticalScroll(id="sidebar"):yield Static(id="sidebar-content")with Vertical(id="sidebar-actions"):+ yield Button("Evaluate", id="btn-eval", classes="action-btn")yield Button("View Traces", id="btn-traces", classes="action-btn")yield Button("Deploy", id="btn-deploy", classes="action-btn")with Vertical(id="main-panel"):@@ -746,6 +887,41 @@ FooterKey .footer-key--key {self._tick_timer.stop()self._tick_timer = self.set_interval(1 / 2, self._tick)self._unsubscribe_if_no_running_memory_save(wait_for_queued=True)+ self._evaluate_if_one_is_waiting()++ def _evaluate_if_one_is_waiting(self) -> None:+ """Start the evaluation `crewai eval` opened this run for.++ The evaluation is the errand; the run was the prerequisite. It begins+ here, on this screen, rather than by closing the app and leaving+ somebody in a bare terminal watching for a link: the app has the room+ to show the link, the progress and the verdict, and the reader leaves+ when they are done reading, not when the command is.+ """+ if self._auto_eval is None or self._evaluation is not None:+ return++ traced = self._traced_execution_id()+ self._auto_eval["execution_id"] = traced+ if traced is None:+ # Said to the command waiting behind this screen as well: it cannot+ # see this notice, and "no marker" is how it tells a run no app got+ # to from a run an app looked at and found nothing to grade.+ self._say_what_became_of_the_evaluation(None)+ self.notify(+ "This run was not traced, so there is nothing to evaluate.",+ title="Evaluate",+ severity="warning",+ )+ return++ self._start_evaluation(traced)++ def _say_what_became_of_the_evaluation(self, execution_id: str | None) -> None:+ with contextlib.suppress(Exception):+ from crewai_cli.experimental.eval_crew import record_evaluation_outcome++ record_evaluation_outcome(execution_id)def _on_crew_failed(self, error: str) -> None:with self._lock:@@ -772,6 +948,10 @@ FooterKey .footer-key--key {self._tick_timer.stop()self._tick_timer = self.set_interval(1 / 2, self._tick)self._unsubscribe_if_no_running_memory_save(wait_for_queued=True)+ # A run that failed is still a run to grade — where it went wrong is+ # what somebody asking for an evaluation wants to know. If nothing was+ # traced, the same notice says so and nothing is asked of AMP.+ self._evaluate_if_one_is_waiting()# ── Conversational flow execution ───────────────────────@@ -996,9 +1176,16 @@ FooterKey .footer-key--key {self.exit(self._crew_result)def _request_trace_consent(self) -> bool:- """Wait in the execution worker while the UI asks for upload consent."""+ """Wait in the execution worker while the UI asks for upload consent.++ Unless the user already answered: turning tracing on — in the project's+ .env, or with ``tracing=True`` — is the yes, and a modal at the end of+ the run would be the same question a second time.+ """if self._discard_trace_on_exit:return False+ if self._tracing_was_asked_for():+ return Truedone = threading.Event()decision: list[bool] = []self._trace_consent_pending = done@@ -1028,15 +1215,84 @@ FooterKey .footer-key--key {self.call_from_thread(self._dismiss_consent_modal)def action_view_traces(self) -> None:+ """Open AMP's view of this run's traces.++ crewAI prints that link after a run, but never under a TUI — its+ console is kept out of this layout — so the link reaches here the way+ everything else about the run does: through the record crewAI writes+ when the spans are exported, matched to THIS app's execution.+ """if self._status != "completed":return# Recorded here rather than in on_button_pressed so the `t` key binding# is counted too, and only once the action can actually do something.self._record_tui_button_click("view_traces")+ record = (+ _recorded_run(self._execution_uuid) if self._execution_uuid else None+ ) or {}+ url = str(record.get("trace_url") or "")+ if url:+ self._open_report(url)+ self.notify(url, title="Execution traces")+ return+self.notify(- "Trace sharing is requested when the execution finishes. "- "A trace link is not available for this run.",+ "This run was not traced, so there are no traces to view. "+ "Turn tracing on and run it again."+ if not record+ else "This run was traced, but AMP granted no link to view it.",title="Execution traces",+ severity="warning",+ )++ def _capture_execution_uuid(self) -> None:+ """Remember which execution this app is watching.++ Read as the run announces itself, from the run's own context, and kept:+ the uuid is gone by the time the kickoff returns, and the project's+ record cannot say which run was THIS one.+ """+ if self._execution_uuid is not None:+ return+ try:+ from crewai.execution import get_execution_uuid++ self._execution_uuid = get_execution_uuid()+ except Exception: # a run without tracing simply has no execution+ self._execution_uuid = None++ def _tracing_was_asked_for(self) -> bool:+ """Did somebody turn tracing on for this run?++ Read here rather than through `tracing_asked_for()`, because the+ ContextVar that helper reads is set where the crew is CONSTRUCTED — not+ this worker thread — and a flow never sets it at all. So the declaration+ is read off the object itself.++ Only ONE of the helper's guards is dropped: it refuses while tracing+ messages are suppressed, which this app does deliberately to keep+ crewAI's console out of its own layout, and that guard is about a prompt+ nobody would see. The others hold. A suite under test, or a TUI with no+ interactive user behind it — embedded, redirected — has nobody whose yes+ this could be, and gets the modal's own fail-closed answer instead.+ """+ import os++ from crewai.events.listeners.tracing.utils import (+ _is_interactive_terminal,+ _is_test_environment,+ )++ if _is_test_environment() or not _is_interactive_terminal():+ return False++ if os.getenv("CREWAI_TRACING_ENABLED", "").lower() in ("true", "1"):+ return True++ return any(+ getattr(target, "tracing", None) is True+ for target in (self._crew, self._flow)+ if target is not None)def _dismiss_consent_modal(self) -> None:@@ -1057,6 +1313,211 @@ FooterKey .footer-key--key {self._unsubscribe()self.exit(self._crew_result)+ def _traced_execution_id(self) -> str | None:+ """This app's own execution, if it was traced.++ Taken from the session the run itself opened, not from the project's+ last-run record: that record is whichever run finished last anywhere in+ the project, and a second TUI in the same project can replace it between+ this run ending and its button being pressed. A run with no session was+ not traced, and there is nothing to grade.+ """+ if self._execution_uuid is None:+ return None++ return (+ self._execution_uuid if _trace_was_recorded(self._execution_uuid) else None+ )++ def action_evaluate_crew(self) -> None:+ """Grade the run that just finished, here.++ Unlike Deploy, this does not leave: the evaluation takes a few minutes,+ it has a link worth showing and a verdict worth reading, and a person+ sent back to a bare terminal to wait for them has been given the worst+ of both. The report opens in the browser; the app keeps the link and+ says what came back.+ """+ if self._status != "completed":+ return++ # An evaluation already in flight, or already answered: the button+ # re-opens what it made rather than paying for a second one. One that+ # STOPPED is a different matter — a timeout or a 502 is worth another+ # press, and the run is still there to grade.+ state = str((self._evaluation or {}).get("state"))+ if self._evaluation is not None and state != "failed":+ url = self._evaluation.get("url")+ if url:+ self._open_report(str(url))+ return++ self._record_tui_button_click("evaluate")+ traced = self._traced_execution_id()+ if traced is None:+ self.notify(+ "This run was not traced, so there is nothing to evaluate. "+ "Turn tracing on and run it again.",+ title="Evaluate",+ severity="warning",+ )+ return++ self._start_evaluation(traced)++ def _start_evaluation(self, execution_id: str) -> None:+ self._evaluation = {"state": "starting", "execution_id": execution_id}+ # The run is over and the clock slowed to twice a second; something is+ # moving again, so the screen moves with it.+ self._set_tick_rate(1 / 8)+ self._refresh_eval_button()+ self._evaluate_worker(execution_id)++ def _set_tick_rate(self, interval: float) -> None:+ with contextlib.suppress(Exception):+ self._tick_timer.stop()+ self._tick_timer = self.set_interval(interval, self._tick)++ @work(thread=True, exclusive=True, group="evaluation")+ def _evaluate_worker(self, execution_id: str) -> None:+ self._evaluate_now(execution_id)++ def _evaluate_now(self, execution_id: str) -> None:+ """AMP evaluates the run; this thread only carries the answers back."""+ from crewai_cli.experimental.eval_crew import (+ EvaluationStoppedError,+ evaluate_run,+ )++ def back(handler: Callable[..., Any], *args: Any) -> None:+ with contextlib.suppress(Exception): # the app may be gone by now+ self.call_from_thread(handler, *args)++ try:+ finished = evaluate_run(+ execution_id,+ on_started=lambda started: back(self._evaluation_started, started),+ on_status=lambda payload: back(self._evaluation_progress, payload),+ note=lambda text: back(self._evaluation_note, text),+ )+ except EvaluationStoppedError as stopped:+ back(self._evaluation_failed, str(stopped))+ return+ except SystemExit as exit_:+ # Nothing on this path should exit any more — a stop is a value+ # now — but an exit carries a code, not a reason, and "1" on screen+ # is worse than saying plainly that the reason did not survive.+ back(+ self._evaluation_failed,+ f"The evaluation stopped without saying why (exit {exit_.code}).",+ )+ return+ except Exception as error: # a client bug is still an answer to show+ back(self._evaluation_failed, f"{type(error).__name__}: {error}")+ return+ back(self._evaluation_finished, finished)++ def _evaluation_started(self, started: dict[str, Any]) -> None:+ if self._evaluation is None:+ return+ url = started.get("url")+ self._evaluation.update(+ {"state": "running", "id": started.get("id"), "url": url, "note": None}+ )+ self._refresh_eval_button()+ if isinstance(url, str) and url:+ self._open_report(url)++ def _evaluation_progress(self, payload: dict[str, Any]) -> None:+ """What the service says it is doing, in its own counting.++ The report page counts the same way — one event per subject judged —+ so a person watching both sees one number, not two.+ """+ if self._evaluation is None:+ return+ judged = planned = None+ with contextlib.suppress(Exception):+ events = payload.get("events") or []+ judged = sum(+ 1 for event in events if (event.get("data") or {}).get("subject")+ )+ planned = next(+ (+ (event.get("data") or {}).get("planned")+ for event in reversed(events)+ if (event.get("data") or {}).get("planned")+ ),+ None,+ )+ self._evaluation["note"] = (+ f"judged {judged} of {planned}"+ if judged and planned+ else (f"judged {judged}" if judged else None)+ )++ def _evaluation_note(self, text: str) -> None:+ if self._evaluation is not None:+ self._evaluation["note"] = text++ def _evaluation_finished(self, finished: dict[str, Any]) -> None:+ if self._evaluation is None:+ return+ if finished.get("status") == "done":+ self._evaluation.update(+ {+ "state": "done",+ "verdict": finished.get("verdict") or {},+ "note": None,+ # the file the evaluation wrote, for the line printed once+ # the app has gone (`_print_evaluation_line`)+ "wrote_config": finished.get("wrote_eval_config"),+ }+ )+ else:+ self._evaluation.update(+ {+ "state": "failed",+ "error": str(finished.get("error") or "no reason given"),+ "note": None,+ }+ )+ self._set_tick_rate(1 / 2)+ self._refresh_eval_button()++ def _evaluation_failed(self, message: str) -> None:+ if self._evaluation is None:+ return+ self._evaluation.update({"state": "failed", "error": message, "note": None})+ self._set_tick_rate(1 / 2)+ self._refresh_eval_button()++ def _open_report(self, url: str) -> None:+ import webbrowser++ with contextlib.suppress(Exception):+ webbrowser.open(url)++ def _evaluating(self) -> bool:+ return (self._evaluation or {}).get("state") in ("starting", "running")++ def _refresh_eval_button(self) -> None:+ """The button while the evaluation runs: still teal, still pressable —+ the report page is where the progress is — and spinning, because a+ label that never moves reads as a screen that has stopped."""+ labels = {+ "done": "Open eval report",+ "failed": "Evaluate",+ }+ with contextlib.suppress(Exception):+ button = self.query_one("#btn-eval", Button)+ state = str((self._evaluation or {}).get("state"))+ button.label = (+ f"{self._spinner()} Evaluating…"+ if self._evaluating()+ else labels.get(state, "Evaluate")+ )+def _record_tui_button_click(self, button_name: str) -> None:try:if self._telemetry is None:@@ -1069,6 +1530,8 @@ FooterKey .footer-key--key {def on_button_pressed(self, event: Button.Pressed) -> None:if event.button.id == "btn-traces":self.action_view_traces()+ elif event.button.id == "btn-eval":+ self.action_evaluate_crew()elif event.button.id == "btn-deploy":self.action_deploy_crew()@@ -1125,6 +1588,8 @@ FooterKey .footer-key--key {try:with self._lock:+ if self._evaluating():+ self._refresh_eval_button()self._render_sidebar()self._render_task_header()self._render_main_content()@@ -1255,6 +1720,46 @@ FooterKey .footer-key--key {# ── Task header rendering ───────────────────────────────+ def _append_evaluation(self, t: Text) -> None:+ """The evaluation under the run's own line: what it is doing, where to+ read it, and what it answered. Everything here came over the wire, so+ it is appended to the Text and never parsed as markup."""+ evaluation = self._evaluation+ if not evaluation:+ return++ state = str(evaluation.get("state"))+ url = str(evaluation.get("url") or "")+ t.append("\n")+ if state in ("starting", "running"):+ t.append(f"{self._spinner()} ", style=_C_TEAL)+ t.append("Evaluating this run", style=f"bold {_C_TEAL}")+ note = evaluation.get("note")+ if note:+ t.append(f" {note}", style=_C_DIM)+ elif state == "done":+ verdict = evaluation.get("verdict") or {}+ gate = str(verdict.get("gate") or "").upper()+ mark, style = {+ "PASSED": ("✔ ", f"bold {_C_GREEN}"),+ "FAILED": ("✘ ", f"bold {_C_RED}"),+ }.get(gate, ("• ", f"bold {_C_TEAL}"))+ t.append(mark, style=style)+ t.append(f"Goal gate {gate or 'graded'}", style=style)+ grades = verdict.get("grades")+ if isinstance(grades, dict):+ for area, grade in grades.items():+ t.append(f" {area} ", style=_C_DIM)+ t.append(f"{grade if grade is not None else '—'}/5", style=_C_TEXT)+ else:+ t.append("✘ ", style=f"bold {_C_RED}")+ t.append("Evaluation stopped", style=f"bold {_C_RED}")+ t.append(f" {evaluation.get('error') or ''}", style=_C_DIM)++ if url:+ t.append("\n")+ t.append(url, style=f"{_C_TEAL} underline")+def _render_task_header(self) -> None:widget = self.query_one("#task-header", Static)t = Text()@@ -1289,11 +1794,13 @@ FooterKey .footer-key--key {parts.append(f"↓{out:,}")if parts:t.append(f" {' '.join(parts)} tokens", style=_C_DIM)+ self._append_evaluation(t)elif self._status == "failed":t.append("✘ ", style=f"bold {_C_RED}")t.append("Failed", style=f"bold {_C_RED}")if self._error:t.append(f"\n{self._error[:120]}", style=_C_RED)+ self._append_evaluation(t)elif self._current_method:paused = any(s["name"] == self._current_method and s["status"] == "paused"@@ -1338,12 +1845,14 @@ FooterKey .footer-key--key {parts.append(f"↓{out:,}")if parts:t.append(f" {' '.join(parts)} tokens", style=_C_DIM)+ self._append_evaluation(t)elif self._status == "failed":t.append("✘ ", style=f"bold {_C_RED}")t.append("Failed", style=f"bold {_C_RED}")if self._error:t.append(f"\n{self._error[:120]}", style=_C_RED)+ self._append_evaluation(t)elif self._current_task_idx > 0:t.append(@@ -1977,6 +2486,7 @@ FooterKey .footer-key--key {@crewai_event_bus.on(CrewKickoffStartedEvent)def on_crew_started(source: Any, event: CrewKickoffStartedEvent) -> None:+ self._capture_execution_uuid()with self._lock:# In flow mode the app is named for the flow; a nested crew's# kickoff (a `call: crew` step) must not rename it.@@ -1990,6 +2500,7 @@ FooterKey .footer-key--key {# ── Declarative-flow method events → STEPS panel ────────@crewai_event_bus.on(FlowStartedEvent)def on_flow_started(source: Any, event: FlowStartedEvent) -> None:+ self._capture_execution_uuid()with self._lock:self._status = "working"diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex 6ef6b5869..5fcda3a29 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -14,7 +14,9 @@ traces. The command sends the saved `crewai login` when there is one.from __future__ import annotations+from collections.abc import Callableimport contextlib+from datetime import datetime, timedelta, timezonefrom ipaddress import ip_addressimport jsonimport os@@ -44,10 +46,65 @@ LAST_RUN_FILE = Path(".crewai") / "last_run.json"TRACING_ENV_VAR = "CREWAI_TRACING_ENABLED"POLL_SECONDS = 3.0POLL_RETRIES = 5 # consecutive unreachable / 5xx polls before giving up; the evaluation keeps running++# A run's spans reach AMP a little after the run ends — the exporter sends them+# as the process closes and AMP has its own queue behind that. A command that+# just watched the run would otherwise ask for a trace that is still in flight+# and be told, correctly and uselessly, that there is nothing there. So a run+# we know is fresh gets a wait; an id somebody typed does not, and fails at+# once as it always has.+SPANS_WAIT_SECONDS = 120.0+SPANS_POLL_SECONDS = 5.0+RUN_IS_FRESH_SECONDS = 600.0FINISHED = {"done", "failed"}STATUSES = {"queued", "running"} | FINISHED+def _record_usage(*, logged_in: bool) -> None:+ """Count an evaluation that is actually starting, and whether the caller was+ logged in.++ Usage stats are anonymous, so nothing that names the run or the account is+ sent — no execution id, no organization, and nothing about the run's content.+ Which runs were evaluated, and by whom, is AMP's record, not these stats'.++ The TUI's button counts `cli_usage:evaluate` when it is pressed, so the+ difference between that and `cli_usage:eval` is intent that never became an+ evaluation.+ """+ try:+ from crewai_core.telemetry import Telemetry++ telemetry = Telemetry()+ telemetry.set_tracer()+ telemetry.feature_usage_span(+ "cli_usage:eval",+ {"authenticated": "true" if logged_in else "false"},+ )+ except Exception: # noqa: S110 - telemetry must never break a command+ pass+++NOT_TRACED = (+ "The run finished but no trace was recorded: the run may have failed, sharing the "+ "trace was declined, or this project's crewai is older than the version that records "+ f"the last run ({LAST_RUN_FILE}). Run the crew again and accept when asked, then `crewai eval`."+)+++class EvaluationStoppedError(RuntimeError):+ """The evaluation cannot go on, in words meant for a reader.++ Raised rather than printed-and-exited, because the same two functions serve+ the terminal and the run app: one of them owns the screen, and a line+ printed underneath it is a smear nobody asked for.+ """+++def _note(text: str, style: str = "dim") -> None:+ console.print(Text(text), style=style)++def eval_crew(run_id: str | None = None) -> None:"""Evaluate the last traced run of this project, or the run RUN_ID."""get_or_create_project_id()@@ -57,10 +114,19 @@ def eval_crew(run_id: str | None = None) -> None:record = read_last_run() or {}execution_id = run_id or record.get("execution_id")if execution_id is None:- execution_id = _run_now_or_explain()+ # Nothing traced here: the crew runs first, and the app that runs it+ # carries the evaluation on its own screen — link, progress, verdict.+ # It comes back with the run to grade when the app did NOT get to it: a+ # conversational session, or a flow that took the terminal instead.+ execution_id = _run_and_let_the_app_evaluate()+ if execution_id is None:+ returnrecord = read_last_run() or {}- client = _amp_client(trusted)+ try:+ client = _amp_client(trusted)+ except EvaluationStoppedError as stopped:+ _fail(str(stopped))recorded_amp = str(record.get("amp_base_url") or "").rstrip("/")if not run_id and recorded_amp and recorded_amp != client.base_url.rstrip("/"):console.print(@@ -69,7 +135,18 @@ def eval_crew(run_id: str | None = None) -> None:),style="yellow",)- started = _start_evaluation(client, execution_id)+ try:+ started = _start_evaluation(+ client,+ execution_id,+ wait_for_spans=run_id is None and _ran_just_now(record),+ )+ except EvaluationStoppedError as stopped:+ _fail(str(stopped))+ # After, not before: `cli_usage:eval` counts an evaluation, and a refused+ # request — a run AMP does not hold, a credential it will not take — is not+ # one. `_start_evaluation` raises rather than returning on those.+ _record_usage(logged_in=client.api_key is not None)url = started.get("url")console.print(Text("Evaluating run ").append(execution_id, style="bold"))if url:@@ -79,12 +156,202 @@ def eval_crew(run_id: str | None = None) -> None:console.print(Text("Follow it at ").append(url, style="cyan underline"))_open(url)- finished = _wait(client, started["id"], url)+ console.print("Waiting for the verdict…", style="dim")+ try:+ finished = _wait(client, started["id"], url)+ except EvaluationStoppedError as stopped:+ _fail(str(stopped))+ except KeyboardInterrupt:+ console.print(+ Text(f"\nStill running{f' at {url}' if url else ''}."), style="yellow"+ )+ raise SystemExit(130) from None_print_verdict(finished, url)+ _say_where_the_criteria_live(write_eval_config(finished))if finished.get("status") != "done":raise SystemExit(1)+def _ran_just_now(record: dict[str, Any]) -> bool:+ """Did the project record this run in the last few minutes?++ The one thing that tells a run still landing in AMP apart from a run AMP+ does not hold: when it finished. A record without a readable stamp is not+ assumed fresh — the wait is for the case we can name.+ """+ stamp = str(record.get("recorded_at") or record.get("finished_at") or "")+ if not stamp:+ return False++ try:+ when = datetime.fromisoformat(stamp)+ except ValueError:+ return False++ now = datetime.now(when.tzinfo) if when.tzinfo else datetime.now()+ return 0 <= (now - when).total_seconds() <= RUN_IS_FRESH_SECONDS+++EVAL_CONFIG_FILE = "eval.jsonc"+# A project's criteria are criteria, not a corpus — the same bound AMP holds+# the field to, checked here so a file that will be refused is not uploaded.+MAX_EVAL_CONFIG_BYTES = 64 * 1024+++def project_eval_config(*, note: Callable[[str], None] = _note) -> str | None:+ """What this project says good means, if it has said.++ Read from the project's own directory, beside `pyproject.toml`, because+ that is where the team keeps the things they argue about in review. Not+ parsed here: what a criterion means is the grader's to say, and a document+ this CLI could not read is one the grader will refuse with the line that+ is wrong.+ """+ path = Path.cwd() / EVAL_CONFIG_FILE+ try:+ if not path.is_file():+ return None+ text = path.read_text(encoding="utf-8")+ except OSError:+ return None++ if len(text.encode("utf-8")) > MAX_EVAL_CONFIG_BYTES:+ # Through NOTE, which the run app routes onto its own screen: a print+ # would land under its layout.+ note(+ f"{EVAL_CONFIG_FILE} is larger than "+ f"{MAX_EVAL_CONFIG_BYTES // 1024}KB and was not sent; this run is graded on "+ "the crew's own expectations."+ )+ return None+ return text or None+++def write_eval_config(finished: dict[str, Any]) -> Path | None:+ """Write the criteria this run was graded on, for the project to edit.++ Only when the project has none: after that the file is theirs, and the one+ thing worse than no criteria is criteria that get overwritten every time+ somebody runs an evaluation.+ """+ text = finished.get("eval_config")+ if not isinstance(text, str) or not text.strip():+ return None++ path = Path.cwd() / EVAL_CONFIG_FILE+ try:+ if path.exists() or not path.parent.is_dir():+ return None+ path.write_text(text, encoding="utf-8")+ except OSError:+ return None+ return path+++EVAL_MARKER_FILE = "last_eval.json"+++def _marker_path() -> Path:+ from crewai.telemetry.tracing.last_run import LAST_RUN_DIR, project_dir++ return project_dir() / LAST_RUN_DIR / EVAL_MARKER_FILE+++def record_evaluation_outcome(execution_id: str | None) -> None:+ """What the run app did with the evaluation it was opened for.++ The app runs in a child process for most projects, so nothing in memory+ reaches the command waiting behind it. Two things that command must know+ are both here: that this run has been evaluated already, and — when the app+ had nothing to evaluate — that it was the app saying so rather than no app+ at all. A conversational session and a flow that takes the terminal never+ get here, leave no marker, and the command grades the run itself.++ Never raises: a marker that cannot be written costs a second evaluation,+ not the run.+ """+ with contextlib.suppress(Exception):+ path = _marker_path()+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(+ json.dumps(+ {+ "execution_id": execution_id,+ "at": datetime.now(timezone.utc).isoformat(),+ }+ ),+ encoding="utf-8",+ )+++def evaluation_marker(after: datetime) -> dict[str, Any] | None:+ """The app's word about the run this command just watched, if it left one.++ AFTER is when the run was started, and the marker must be newer: a project+ keeps one marker, and the one from an earlier `crewai eval` is not about+ this run. Matched on time rather than on the project's last-run record,+ which another run finishing in the same project can replace while this app+ is still open.++ A second of slack, because the two stamps are written by two processes and+ the question being asked is "this run or an older one", which a second+ cannot confuse.+ """+ with contextlib.suppress(Exception):+ marker = json.loads(_marker_path().read_text(encoding="utf-8"))+ when = datetime.fromisoformat(str(marker.get("at")))+ if isinstance(marker, dict) and when >= after - timedelta(seconds=1):+ return dict(marker)+ return None+++def evaluate_run(+ execution_id: str,+ *,+ on_started: Callable[[dict[str, Any]], None],+ on_status: Callable[[dict[str, Any]], None] | None = None,+ note: Callable[[str], None] = _note,+) -> dict[str, Any]:+ """Start the evaluation of EXECUTION_ID and wait for its verdict.++ The run app's door into this command: it has a screen of its own to show+ the link and the verdict on, so nothing here prints and nothing exits — the+ url reaches ON_STARTED as soon as AMP gives it, every unfinished answer+ reaches ON_STATUS, and anything that stops the evaluation is an+ `EvaluationStoppedError` carrying the sentence to show.++ The spans of a run that just ended may still be in flight, which is the+ whole reason this is called from inside the app that ran it, so the wait+ for them is always on here.+ """+ client = _amp_client(_machine_amp_origins(), note=note)+ started = _start_evaluation(client, execution_id, wait_for_spans=True, note=note)+ # After the start, exactly as the command counts it: `cli_usage:eval` is the+ # count of evaluations that began, and an evaluation the app runs is one.+ _record_usage(logged_in=client.api_key is not None)+ record_evaluation_outcome(execution_id)+ on_started(started)+ finished = _wait(client, started["id"], started.get("url"), on_status=on_status)+ written = write_eval_config(finished)+ if written is not None:+ finished = {**finished, "wrote_eval_config": written.name}+ return finished+++def _say_where_the_criteria_live(path: Path | None) -> None:+ """One line, once: the file exists now, and editing it changes the next grade."""+ if path is None:+ return++ console.print(+ Text(+ f"Wrote {path.name} — say what good means for this crew there, and the next "+ "`crewai eval` is graded on it."+ ),+ style="green",+ )++def _trusted_amp_origins() -> set[str]:"""Where the saved login may be sent: the AMP this machine is configured for(`crewai enterprise configure`), one already exported in this shell, and@@ -97,6 +364,20 @@ def _trusted_amp_origins() -> set[str]:return {origin for origin in map(_origin, candidates) if origin}+def _machine_amp_origins() -> set[str]:+ """The same set, for a caller whose environment can no longer be trusted.++ `crewai eval` reads `CREWAI_PLUS_URL` BEFORE the project's `.env` is loaded,+ so a shell export counts there and a project cannot introduce one. Inside+ the run app there is no such "before": the crew has already run and its+ `.env` was loaded for it, so the variable is left out of the set entirely.+ The request still follows it — that is how a self-hosted project is wired —+ and the credential still does not.+ """+ candidates = (Settings().enterprise_base_url, DEFAULT_CREWAI_ENTERPRISE_URL)+ return {origin for origin in map(_origin, candidates) if origin}++def _origin(url: str | None) -> str | None:parsed = urlparse(str(url or ""))return (@@ -123,7 +404,7 @@ def _encrypted(origin: str | None) -> bool:return False-def _amp_client(trusted: set[str]) -> PlusAPI:+def _amp_client(trusted: set[str], *, note: Callable[[str], None] = _note) -> PlusAPI:"""The AMP to ask, and whether the saved login goes with it.A project's `.env` may point `crewai eval` at another AMP — that is how a@@ -133,7 +414,7 @@ def _amp_client(trusted: set[str]) -> PlusAPI:read anonymously."""client = PlusAPI(api_key=saved_login())if client.api_key is None:- return client+ return _anonymous(client)origin = _origin(client.base_url)if origin in trusted and _encrypted(origin):@@ -144,14 +425,23 @@ def _amp_client(trusted: set[str]) -> PlusAPI:if origin not in trustedelse "would carry the login over plain HTTP")- console.print(- Text(- f"Reading anonymously: {client.base_url} {why}. "- "Run `crewai enterprise configure <url>` to log in to it."- ),- style="yellow",+ note(+ f"Reading anonymously: {client.base_url} {why}. "+ "Run `crewai enterprise configure <url>` to log in to it.")- return PlusAPI()+ return _anonymous(PlusAPI())+++def _anonymous(client: PlusAPI) -> PlusAPI:+ """A client that says nothing about who is asking.++ `PlusAPI` adds the saved organization to every request, and AMP reads it+ only beside a credential — so without one it tells AMP nothing, and tells+ an AMP this machine is not logged in to which organization is. Dropped, as+ anonymous trace grants drop it.+ """+ client.headers.pop("X-Crewai-Organization-Id", None)+ return clientdef _load_project_env() -> None:@@ -188,49 +478,93 @@ def saved_login() -> str | None:except AuthError:return Noneexcept Exception as error:- _fail(+ # Raised, not printed-and-exited: the run app shows this on its own+ # screen, where a print would land under the layout and an exit would+ # take the sentence with it.+ raise EvaluationStoppedError(f"Could not read the saved login ({type(error).__name__}: {error}). ""Run `crewai login` again, or `crewai eval` will not know who you are."- )- return None+ ) from error++def _run_and_let_the_app_evaluate() -> str | None:+ """No traced run recorded here: offer to turn tracing on and run the crew.-def _run_now_or_explain() -> str:- """No traced run recorded here: offer to turn tracing on and run the crew now."""+ The run app evaluates what it ran, on its own screen, so this returns None+ when it did — the reader has already seen the link and the verdict. It+ returns the run to grade when no app got to it (a conversational session, a+ flow that took the terminal), and says so itself when nothing was traced.+ """if not Path("pyproject.toml").is_file():_fail("No crewAI project here (no pyproject.toml). Run `crewai eval` from the project's ""directory, or name a run: `crewai eval --run EXECUTION_ID`.")steps = (- "No traced run is recorded in this project. Turn tracing on and run the crew, "- f"then come back:\n 1. add {TRACING_ENV_VAR}=true to .env\n 2. crewai run\n 3. crewai eval"+ "No traced run is recorded in this project. Turn tracing on — its runs are then "+ "traced to CrewAI AMP — and run the crew, then come back:\n"+ f" 1. add {TRACING_ENV_VAR}=true to .env\n 2. crewai run\n 3. crewai eval")if is_dmn_mode_enabled() or not sys.stdin.isatty():console.print(steps, style="yellow")raise SystemExit(1)if not click.confirm(- f"No traced run is recorded in this project. Turn tracing on ({TRACING_ENV_VAR}=true "- "stays in .env) and run the crew now?",+ "No traced run is recorded in this project. Turn tracing on and run the crew now? "+ "The run's trace is sent to CrewAI AMP.",default=True, # João, 2026-09-20: y/n with Y as the default — the prompt says what Enter does):console.print(steps, style="yellow")raise SystemExit(0)_enable_tracing()+ from crewai_cli.crew_run_tui import evaluating_after_runfrom crewai_cli.run_crew import run_crew- run_crew()- record = read_last_run()- if record is None:- console.print(- "The run finished but no trace was recorded: the run may have failed, sharing the "- "trace was declined, or this project's crewai is older than the version that records "- f"the last run ({LAST_RUN_FILE}). Run the crew again and accept when asked, then `crewai eval`.",- style="bold red",- )+ # The app is told an evaluation is waiting and starts it itself when the run+ # ends. It runs in this process for some projects and in a child for others,+ # so what says a run was traced is either the holder's id or the record the+ # run wrote.+ began = datetime.now(timezone.utc)+ with evaluating_after_run() as watched:+ run_crew()++ # The app says what it did with the run it watched, and its word settles+ # this: it evaluated it (nothing left to do), or it had nothing to evaluate+ # (say so once, in the terminal the screen has left). Only when no app got+ # here at all — a conversational session, a flow that took the terminal —+ # is there a run for this command to grade.+ marker = evaluation_marker(after=began)+ if marker is not None:+ if marker.get("execution_id"):+ return None+ console.print(NOT_TRACED, style="bold red")raise SystemExit(1)- return str(record["execution_id"])++ # The project's record, only when it was written after this run began: a+ # record is the project's LAST run, and another run finishing in the same+ # project would otherwise be graded in this one's place.+ record = read_last_run() or {}+ traced = watched["execution_id"] or (+ record.get("execution_id") if _recorded_since(record, began) else None+ )+ if traced:+ return str(traced)++ console.print(NOT_TRACED, style="bold red")+ raise SystemExit(1)+++def _recorded_since(record: dict[str, Any], began: datetime) -> bool:+ """Was RECORD written after BEGAN? A record without a readable, zoned stamp+ is not assumed to be this run's."""+ stamp = str(record.get("recorded_at") or record.get("finished_at") or "")+ try:+ when = datetime.fromisoformat(stamp)+ except ValueError:+ return False+ if when.tzinfo is None:+ return False+ return when >= began - timedelta(seconds=1)def _enable_tracing() -> None:@@ -240,16 +574,42 @@ def _enable_tracing() -> None:set_key(str(env_file), TRACING_ENV_VAR, "true", quote_mode="never")os.environ[TRACING_ENV_VAR] = "true"console.print(- f"Tracing is on for this project ({TRACING_ENV_VAR}=true in .env).",+ "Tracing is on for this project — its runs are traced to CrewAI AMP.",style="green",)-def _start_evaluation(client: PlusAPI, execution_id: str) -> dict[str, Any]:- try:- response = client.create_evaluation(execution_id)- except httpx.HTTPError as error:- _fail(f"Could not reach AMP to start the evaluation: {error}")+def _start_evaluation(+ client: PlusAPI,+ execution_id: str,+ *,+ wait_for_spans: bool = False,+ note: Callable[[str], None] = _note,+) -> dict[str, Any]:+ deadline = time.monotonic() + SPANS_WAIT_SECONDS if wait_for_spans else 0.0+ said = False+ # Read once: the file does not change while the spans are in flight, and a+ # warning about it belongs on the screen once, not on every retry.+ eval_config = project_eval_config(note=note)+ while True:+ try:+ response = client.create_evaluation(execution_id, eval_config=eval_config)+ except httpx.HTTPError as error:+ raise EvaluationStoppedError(+ f"Could not reach AMP to start the evaluation: {error}"+ ) from error+ # The run finished moments ago and its spans are still on their way:+ # waiting is the answer, not a 404 the reader can do nothing with.+ if response.status_code == 404 and time.monotonic() < deadline:+ if not said:+ note(+ "The run's trace has not reached CrewAI AMP yet — waiting up to "+ f"{int(SPANS_WAIT_SECONDS // 60)} minutes for it…"+ )+ said = True+ time.sleep(SPANS_POLL_SECONDS)+ continue+ breakif response.status_code in (200, 202):payload = _payload(response)# The id is what every later call is made with, so a missing or@@ -258,67 +618,102 @@ def _start_evaluation(client: PlusAPI, execution_id: str) -> dict[str, Any]:# costs the link and nothing else: the evaluation is already running# and its verdict is what the user came for.if payload and isinstance(payload.get("id"), str) and payload["id"]:- if not isinstance(payload.get("url"), str):- if payload.get("url") is not None:- console.print(- Text(- "AMP answered with a report url that is not a string; "- "the link is unavailable for this run."- ),- style="yellow",- )- payload["url"] = None+ url = _report_url(payload.get("url"))+ if url is None and payload.get("url") is not None:+ note(+ "AMP answered with a report url that is not a web address this "+ "command will open; the link is unavailable for this run."+ )+ payload["url"] = urlreturn payload- _fail(f"AMP answered without an evaluation id ({response.status_code}).")- _refused(response, f"run {execution_id}")- raise AssertionError("unreachable")---def _wait(client: PlusAPI, evaluation_id: str, url: str | None) -> dict[str, Any]:- """Poll until the evaluation is done or failed; Ctrl-C leaves it running."""- console.print("Waiting for the verdict…", style="dim")+ raise EvaluationStoppedError(+ f"AMP answered without an evaluation id ({response.status_code})."+ )+ raise EvaluationStoppedError(_refusal_message(response, f"run {execution_id}"))+++def _report_url(value: Any) -> str | None:+ """The report link, if it is one a browser may be sent to.++ It is printed as a link and opened without a click, and it comes from+ whichever AMP answered — which a project's `.env` may choose. So it must be+ an absolute web address: HTTPS, or plain HTTP to this machine, with no+ credentials in it and no control characters to restyle the terminal. The+ host is not pinned: the report is served by the evaluation service, not by+ AMP, and a self-hosted AMP names its own.+ """+ if (+ not isinstance(value, str)+ or not value+ or any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in value)+ ):+ return None+ try:+ parsed = urlparse(value)+ if (+ parsed.hostname+ and parsed.username is None+ and parsed.password is None+ and _encrypted(f"{parsed.scheme}://{parsed.netloc}")+ ):+ return value+ except ValueError:+ pass+ return None+++def _wait(+ client: PlusAPI,+ evaluation_id: str,+ url: str | None,+ *,+ on_status: Callable[[dict[str, Any]], None] | None = None,+) -> dict[str, Any]:+ """Poll until the evaluation is done or failed.++ Every unfinished answer goes to ON_STATUS, so a caller that has somewhere to+ show progress can show it; the caller decides what a Ctrl-C means.+ """where = f" at {url}" if url else ""subject = f"evaluation {evaluation_id}"misses = (0 # AMP unreachable or answering 5xx: a blip is retried, a streak is reported)- try:- while True:- try:- response = client.get_evaluation(evaluation_id)- except httpx.HTTPError as error:- misses += 1- if misses >= POLL_RETRIES:- _fail(- f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}."- )- time.sleep(POLL_SECONDS)- continue- if response.status_code >= 500:- misses += 1- if misses >= POLL_RETRIES:- _refused(response, subject)- time.sleep(POLL_SECONDS)- continue- if response.status_code != 200:- _refused(response, subject)- misses = 0- payload = _payload(response) or {}- status = payload.get("status")- if status == "done" and not _well_formed_verdict(payload.get("verdict")):- _fail(- f"AMP answered done without a verdict (protocol error); follow it{where or ' on AMP'}."- )- if status in FINISHED:- return payload- if status not in STATUSES:- _fail(- f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}."- )+ while True:+ try:+ response = client.get_evaluation(evaluation_id)+ except httpx.HTTPError as error:+ misses += 1+ if misses >= POLL_RETRIES:+ raise EvaluationStoppedError(+ f"Could not reach AMP while waiting ({error}); the evaluation keeps running{where}."+ ) from errortime.sleep(POLL_SECONDS)- except KeyboardInterrupt:- console.print(Text(f"\nStill running{where}."), style="yellow")- raise SystemExit(130) from None+ continue+ if response.status_code >= 500:+ misses += 1+ if misses >= POLL_RETRIES:+ raise EvaluationStoppedError(_refusal_message(response, subject))+ time.sleep(POLL_SECONDS)+ continue+ if response.status_code != 200:+ raise EvaluationStoppedError(_refusal_message(response, subject))+ misses = 0+ payload = _payload(response) or {}+ status = payload.get("status")+ if status == "done" and not _well_formed_verdict(payload.get("verdict")):+ raise EvaluationStoppedError(+ f"AMP answered done without a verdict (protocol error); follow it{where or ' on AMP'}."+ )+ if status in FINISHED:+ return payload+ if status not in STATUSES:+ raise EvaluationStoppedError(+ f"AMP answered without a known evaluation status ({status!r}); follow it{where or ' on AMP'}."+ )+ if on_status is not None:+ on_status(payload)+ time.sleep(POLL_SECONDS)def _well_formed_verdict(verdict: Any) -> bool:@@ -385,25 +780,30 @@ def _payload(response: httpx.Response) -> dict[str, Any] | None:return loaded if isinstance(loaded, dict) else None-def _refused(response: httpx.Response, subject: str) -> None:- """AMP's own words when it sent them, then exit 1. SUBJECT is "run <id>" or "evaluation <id>"."""+def _refusal_message(response: httpx.Response, subject: str) -> str:+ """AMP's own words when it sent them. SUBJECT is "run <id>" or "evaluation <id>".++ A sentence, not a print: the terminal and the run app both show it, and only+ one of them shows it by printing.+ """payload = _payload(response) or {}message = str(payload.get("message") or "").strip()error = str(payload.get("error") or "")if response.status_code in (401, 403):if error == "account_required" and message:- _fail(message)- _fail(- f"{message or 'AMP refused the credential'}. Log in with `crewai login` and try again."- )+ return message+ return f"{message or 'AMP refused the credential'}. Log in with `crewai login` and try again."if response.status_code == 404:- _fail(message or f"AMP answered 404 for {subject}.")+ return message or f"AMP answered 404 for {subject}."if response.status_code == 429:retry = response.headers.get("Retry-After")- _fail(- f"{message or 'AMP is rate limiting this request'}{f' — retry after {retry}s' if retry else ''}."- )- _fail(f"AMP answered {response.status_code}{': ' + message if message else ''}.")+ return f"{message or 'AMP is rate limiting this request'}{f' — retry after {retry}s' if retry else ''}."+ return f"AMP answered {response.status_code}{': ' + message if message else ''}."+++def _refused(response: httpx.Response, subject: str) -> None:+ """The same words, printed, then exit 1."""+ _fail(_refusal_message(response, subject))def _fail(message: str) -> None:diff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.pyindex f03661e5b..0256f7c30 100644--- a/lib/cli/src/crewai_cli/plus_api.py+++ b/lib/cli/src/crewai_cli/plus_api.py@@ -27,12 +27,22 @@ class PlusAPI(_CorePlusAPI):EVALUATION_START_TIMEOUT = 120.0EVALUATION_POLL_TIMEOUT = 30.0- def create_evaluation(self, execution_id: str) -> httpx.Response:- """Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval)."""+ def create_evaluation(+ self, execution_id: str, *, eval_config: str | None = None+ ) -> httpx.Response:+ """Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval).++ EVAL_CONFIG is the project's own `eval.jsonc` when it has one: what+ good means for this crew, in its own words. Sent as it was written,+ comments and all, and read by the grader rather than here.+ """+ body: dict[str, str] = {"execution_id": execution_id}+ if eval_config:+ body["eval_config"] = eval_configreturn self._make_request("POST",self.EVALUATIONS_RESOURCE,- json={"execution_id": execution_id},+ json=body,timeout=self.EVALUATION_START_TIMEOUT,)diff --git a/lib/cli/src/crewai_cli/run_crew.py b/lib/cli/src/crewai_cli/run_crew.pyindex 903112a98..9597b75a4 100644--- a/lib/cli/src/crewai_cli/run_crew.py+++ b/lib/cli/src/crewai_cli/run_crew.py@@ -600,6 +600,48 @@ def _print_post_tui_summary(app: CrewRunApp) -> None:))+ _print_evaluation_line(app, console, crewai_teal)+++def _print_evaluation_line(app: CrewRunApp, console: Any, teal: str) -> None:+ """The evaluation's link, once the app that showed it has gone.++ An evaluation started inside the app is read there; the terminal is what is+ left afterwards, and a link that only ever existed on a screen that is now+ closed is a link nobody can open again.+ """+ evaluation = getattr(app, "_evaluation", None) or {}+ url = str(evaluation.get("url") or "")+ if not url:+ return++ from rich.text import Text++ state = str(evaluation.get("state"))+ line = Text("\n ")+ if state == "done":+ verdict = evaluation.get("verdict") or {}+ line.append("Evaluated: ", style="dim")+ line.append(+ f"goal gate {str(verdict.get('gate') or '').upper()} ", style="bold"+ )+ elif state == "failed":+ line.append("Evaluation stopped — the report has what it got: ", style="dim")+ else:+ line.append("Evaluation still running at ", style="dim")+ line.append(url, style=f"{teal} underline")+ console.print(line)++ wrote = evaluation.get("wrote_config")+ if wrote:+ note = Text(" ")+ note.append(f"Wrote {wrote}", style="bold")+ note.append(+ " — say what good means for this crew there, and the next evaluation is graded on it.",+ style="dim",+ )+ console.print(note)+def run_crew(trained_agents_file: str | None = None,diff --git a/lib/cli/src/crewai_cli/run_declarative_flow.py b/lib/cli/src/crewai_cli/run_declarative_flow.pyindex 3042d00c0..4fbd2ca4f 100644--- a/lib/cli/src/crewai_cli/run_declarative_flow.py+++ b/lib/cli/src/crewai_cli/run_declarative_flow.py@@ -328,6 +328,10 @@ def _print_flow_post_tui_summary(app: Any) -> None:))+ from crewai_cli.run_crew import _print_evaluation_line++ _print_evaluation_line(app, console, crewai_teal)+def _resolve_flow_inputs(flow: Any, provided: dict[str, Any]) -> dict[str, Any]:"""Resolve kickoff inputs from the flow's state schema.diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex ce5007059..1061b3fe1 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -12,10 +12,21 @@ import httpximport pytestfrom rich.console import Console+from crewai_cli import run_crew as run_crew_modulefrom crewai_cli.experimental import eval_crew as eval_modulefrom crewai_cli.cli import eval_command[email protected](autouse=True)+def _awaiting_files_stay_in_tmp(tmp_path, monkeypatch):+ """The waiting-evaluation token file belongs to the user's crewAI data+ directory; a test keeps it in its own."""+ monkeypatch.setattr(+ "crewai_cli.crew_run_tui._awaiting_dir", lambda: tmp_path / "eval-awaiting"+ )+++EXECUTION_ID = "6f31fe1a-20bd-4bfe-a011-25d6b9341f62"URL = "https://evolve.crewai.test/e/ev-1"@@ -29,10 +40,15 @@ class FakeAMP:)self.statuses = list(statuses or [])self.calls: list[tuple] = []+ self.sent_config = Noneself.api_key = None+ self.headers: dict[str, str] = {"X-Crewai-Organization-Id": "org-42"}- def create_evaluation(self, execution_id):+ def create_evaluation(self, execution_id, *, eval_config=None):self.calls.append(("create", execution_id))+ self.sent_config = eval_config+ if isinstance(self.create, list):+ return self.create.pop(0) if len(self.create) > 1 else self.create[0]return self.createdef get_evaluation(self, evaluation_id):@@ -40,11 +56,14 @@ class FakeAMP:return self.statuses.pop(0) if self.statuses else httpx.Response(200, json={"id": evaluation_id, "status": "running"})-def done(gate="passed", grades=None):- return httpx.Response(200, json={+def done(gate="passed", grades=None, eval_config=None):+ payload = {"id": "ev-1", "status": "done", "url": URL,"verdict": {"gate": gate, "grades": grades if grades is not None else {"goal": 5, "quality": 4, "process": 5, "cost": None}},- })+ }+ if eval_config is not None:+ payload["eval_config"] = eval_config+ return httpx.Response(200, json=payload)@pytest.fixture@@ -69,6 +88,12 @@ def record_last_run(directory: Path, execution_id: str = EXECUTION_ID, **fields)(directory / ".crewai" / "last_run.json").write_text(json.dumps(record))+def _now() -> str:+ from datetime import datetime, timezone++ return datetime.now(timezone.utc).isoformat()++def install(monkeypatch, amp: FakeAMP, configured_amp: str = "https://amp.test") -> FakeAMP:def build(api_key=None, base_url=None):# PlusAPI's own resolution: explicit, then CREWAI_PLUS_URL, then the saved settings.@@ -137,13 +162,15 @@ def test_the_follow_link_cannot_be_retargeted_by_the_url_amp_sends(project, monk# command to let markup through.directory, _ = projectrecord_last_run(directory)- hostile = "[link=http://attacker.test/]https://app.crewai.com/e/ev-1[/link]"+ # A web address that carries markup inside it: printed as text, never+ # interpreted (one that IS markup is not a web address, and is dropped).+ hostile = "https://app.crewai.com/e/[link=http://attacker.test/]ev-1[/link]"created = httpx.Response(202, json={"id": "ev-1", "url": hostile, "status": "queued"})install(monkeypatch, FakeAMP(create=created, statuses=[done()]))eval_module.eval_crew()- out = capsys.readouterr().out+ out = capsys.readouterr().out.replace("\n", "")assert "[link=http://attacker.test/]" in out # printed, not followed@@ -160,7 +187,7 @@ def test_a_url_that_is_not_a_string_costs_the_link_and_nothing_else(project, moneval_module.eval_crew()out = capsys.readouterr().out- assert "report url that is not a string" in out # said, not swallowed+ assert "not a web address" in out # said, not swallowedassert "Goal gate: PASSED" in out # and the verdict still arrivesassert opened == [] # nothing was handed to a browserassert "Follow it at" not in out@@ -300,6 +327,44 @@ def test_a_project_may_point_at_another_amp_but_never_gets_the_saved_login(projeassert "crewai enterprise configure" in out+def test_the_app_never_trusts_an_amp_the_project_env_introduced(project, monkeypatch):+ """`crewai eval` reads CREWAI_PLUS_URL before the project's `.env` is+ loaded, so a shell export is trusted there and a project cannot slip one+ in. Inside the run app there is no such "before" — the crew has already run+ and its `.env` was loaded for it — so an exported variable buys nothing and+ the saved login stays home."""+ directory, _ = project+ # what a project's .env would leave behind, indistinguishable by then+ monkeypatch.setenv("CREWAI_PLUS_URL", "https://evil.example")+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))+ notes: list[str] = []++ eval_module.evaluate_run(+ EXECUTION_ID, on_started=lambda started: None, note=notes.append+ )++ assert amp.base_url == "https://evil.example" # the request still follows it+ assert amp.api_key is None # the credential does not+ assert any("Reading anonymously" in note for note in notes)+ # and it is said through the caller's own note, never printed over its screen+ assert notes and all(isinstance(note, str) for note in notes)+++def test_the_command_still_trusts_an_amp_exported_in_the_shell(project, monkeypatch, capsys):+ """The command's own reading is unchanged: a shell export is this machine+ speaking, and the login goes with it."""+ directory, _ = project+ monkeypatch.setenv("CREWAI_PLUS_URL", "https://amp.exported")+ record_last_run(directory, amp_base_url="https://amp.exported")+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.base_url == "https://amp.exported"+ assert amp.api_key == "login-token"+ assert "Reading anonymously" not in capsys.readouterr().out++def test_a_trusted_amp_over_plain_http_still_gets_no_credential(project, monkeypatch, capsys):"""A cleartext connection is not a place to put a bearer token, trusted or not."""directory, _ = project@@ -405,7 +470,7 @@ def test_amp_unreachable_at_the_start_is_a_sentence_not_a_traceback(project, mondirectory, _ = projectrecord_last_run(directory)amp = install(monkeypatch, FakeAMP())- monkeypatch.setattr(amp, "create_evaluation", lambda execution_id: (_ for _ in ()).throw(httpx.ConnectError("connection refused")))+ monkeypatch.setattr(amp, "create_evaluation", lambda execution_id, **_: (_ for _ in ()).throw(httpx.ConnectError("connection refused")))with pytest.raises(SystemExit) as exit_:eval_module.eval_crew()@@ -529,10 +594,12 @@ def test_without_a_traced_run_it_offers_to_turn_tracing_on_and_run_the_crew(projran.append("run")assert eval_module.os.environ.get("CREWAI_TRACING_ENABLED") == "true"record_last_run(directory, "fresh-run")+ eval_module.record_evaluation_outcome("fresh-run") # what the app doesimport crewai_cli.run_crew as run_crew_modulemonkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)amp = install(monkeypatch, FakeAMP(statuses=[done()]))@@ -540,12 +607,433 @@ def test_without_a_traced_run_it_offers_to_turn_tracing_on_and_run_the_crew(projassert ran == ["run"]text, kwargs = prompts[0]- assert "CREWAI_TRACING_ENABLED=true stays in .env" in text and kwargs == {"default": True} # Enter is yes (João's call); the prompt names both effects+ # the offer is about what happens, not about the variable that makes it happen+ assert "Turn tracing on and run the crew now?" in text and kwargs == {"default": True} # Enter is yes (João's call)+ assert "sent to CrewAI AMP" in text # what saying yes means, in the words that matter+ assert "CREWAI_TRACING_ENABLED" not in text # and not the variable that carries itassert "CREWAI_TRACING_ENABLED=true" in (directory / ".env").read_text()- assert amp.calls[0] == ("create", "fresh-run")+ # the app that ran the crew evaluates it on its own screen; the command that+ # opened it does not ask AMP a second time+ assert amp.calls == []assert "Tracing is on for this project" in capsys.readouterr().out[email protected]("logged_in", [True, False])+def test_the_command_counts_the_evaluation_and_nothing_that_names_it(+ project, monkeypatch, logged_in+):+ """`cli_usage:eval` is every evaluation; the TUI's own `cli_usage:evaluate`+ is intent, and the difference is intent that never became one. Usage stats+ are anonymous, so the span says whether the caller was logged in — never+ which run, which organization, or anything about the run's content."""+ directory, _ = project+ record_last_run(directory, "counted-run")+ spans: list[tuple[str, dict[str, str]]] = []++ class FakeTelemetry:+ def set_tracer(self) -> None:+ pass++ def feature_usage_span(self, feature, attributes=None) -> None:+ spans.append((feature, attributes or {}))++ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)+ monkeypatch.setattr(eval_module, "saved_login", lambda: "tok" if logged_in else None)+ install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert spans == [+ ("cli_usage:eval", {"authenticated": "true" if logged_in else "false"})+ ]+++def test_an_oversized_config_is_said_once_and_through_the_callers_note(+ project, monkeypatch+):+ """The run app routes notes onto its own screen, and the spans of a fresh+ run can take several 404s to land — the warning belongs there, once."""+ directory, _ = project+ (directory / "eval.jsonc").write_text("x" * (eval_module.MAX_EVAL_CONFIG_BYTES + 1))+ sent: list[str | None] = []+ answers = iter([httpx.Response(404, json={}), httpx.Response(404, json={}), QUEUED])++ class Client:+ def create_evaluation(self, execution_id, *, eval_config=None):+ sent.append(eval_config)+ return next(answers)++ monkeypatch.setattr(eval_module.time, "sleep", lambda _s: None)+ notes: list[str] = []++ eval_module._start_evaluation(+ Client(), "run-1", wait_for_spans=True, note=notes.append+ )++ assert sent == [None, None, None]+ assert [n for n in notes if "larger than" in n] == [notes[0]]+++def test_the_fallback_grades_only_a_run_recorded_after_it_began(+ project, monkeypatch, capsys+):+ """No app evaluated the run, so the command reads the project's record — but+ that record is the project's LAST run, and one written before this run+ began belongs to some other run."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ monkeypatch.setattr(+ run_crew_module,+ "run_crew",+ lambda: record_last_run(+ directory, "somebody-elses-run", recorded_at="2020-01-01T00:00:00+00:00"+ ),+ )+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ with pytest.raises(SystemExit) as exited:+ eval_module.eval_crew()++ assert exited.value.code == 1+ assert amp.calls == []+ assert "no trace was recorded" in capsys.readouterr().out+++ "url, opened",+ [+ ("https://optimize.crewai.test/e/ev-1", True),+ ("http://localhost:3000/e/ev-1", True),+ ("http://evil.test/e/ev-1", False),+ ("file:///etc/passwd", False),+ ("javascript:alert(1)", False),+ ("https://user:[email protected]/e/ev-1", False),+ ("https://optimize.crewai.test/e/ev-1\x1b[31m", False),+ ("/e/ev-1", False),+ ],+)+def test_only_a_web_address_is_printed_and_opened(url, opened):+ """The report link comes from whichever AMP answered — which a project's+ `.env` may choose — and is opened without a click. Its host is not pinned+ (the report lives on the evaluation service), but it must be a web address."""+ assert (eval_module._report_url(url) == url) is opened+++def test_a_report_url_that_is_not_a_web_address_is_dropped_with_a_note(monkeypatch):+ class Client:+ def create_evaluation(self, execution_id, *, eval_config=None):+ return httpx.Response(+ 202, json={"id": "ev-1", "url": "file:///etc/passwd", "status": "queued"}+ )++ notes: list[str] = []+ started = eval_module._start_evaluation(Client(), "run-1", note=notes.append)++ assert started["url"] is None+ assert "link is unavailable" in notes[0]++[email protected]("login", [None, "tok"])+def test_a_request_without_the_login_names_no_organization(project, monkeypatch, login):+ """AMP reads the organization header only beside a credential, so without+ one it says nothing — except, to an AMP this machine is not logged in to,+ which organization is asking."""+ monkeypatch.setenv("CREWAI_PLUS_URL", "https://not-logged-in.test")+ monkeypatch.setattr(eval_module, "saved_login", lambda: login)+ monkeypatch.setattr(+ "crewai_core.plus_api.Settings",+ lambda: SimpleNamespace(org_uuid="org-42", enterprise_base_url=None),+ )++ client = eval_module._amp_client({"https://app.crewai.com"}, note=lambda _t: None)++ assert client.api_key is None+ assert "X-Crewai-Organization-Id" not in client.headers+++def test_telemetry_never_breaks_the_command(project, monkeypatch):+ directory, _ = project+ record_last_run(directory, "counted-run")++ def explode() -> None:+ raise RuntimeError("no tracer here")++ monkeypatch.setattr("crewai_core.telemetry.Telemetry", lambda: explode())+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.calls[0] == ("create", "counted-run")+++def test_a_project_that_says_what_good_means_is_graded_on_it(project, monkeypatch):+ """The file travels as it was written — comments and all — because what a+ criterion means is the grader's to read, not this command's."""+ directory, _ = project+ record_last_run(directory)+ written = '// ours\n{"dataset": [{"id": "c1", "criteria": ["cites a source"]}]}\n'+ (directory / "eval.jsonc").write_text(written)+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.sent_config == written+++def test_a_project_with_nothing_to_say_sends_nothing(project, monkeypatch):+ directory, _ = project+ record_last_run(directory)+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.sent_config is None+++def test_a_config_too_large_to_be_criteria_is_left_behind(project, monkeypatch, capsys):+ """A file that AMP would refuse is not uploaded, and the run is still graded."""+ directory, _ = project+ record_last_run(directory)+ (directory / "eval.jsonc").write_text("// " + "x" * (64 * 1024))+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.sent_config is None+ assert amp.calls[0][0] == "create" # graded anyway+ assert "was not sent" in capsys.readouterr().out+++def test_the_first_evaluation_leaves_the_criteria_behind_as_a_file(+ project, monkeypatch, capsys+):+ """A project cannot say what good means until it has somewhere to say it."""+ directory, _ = project+ record_last_run(directory)+ install(monkeypatch, FakeAMP(statuses=[done(eval_config='// yours\n{"dataset": []}\n')]))++ eval_module.eval_crew()++ assert (directory / "eval.jsonc").read_text() == '// yours\n{"dataset": []}\n'+ assert "Wrote eval.jsonc" in capsys.readouterr().out+++def test_a_file_a_project_already_has_is_never_overwritten(project, monkeypatch, capsys):+ """After the first one the file is theirs — the one thing worse than no+ criteria is criteria that vanish every time somebody evaluates."""+ directory, _ = project+ record_last_run(directory)+ (directory / "eval.jsonc").write_text("// mine, edited\n{\"dataset\": []}\n")+ install(monkeypatch, FakeAMP(statuses=[done(eval_config='// theirs\n{}')]))++ eval_module.eval_crew()++ assert (directory / "eval.jsonc").read_text() == "// mine, edited\n{\"dataset\": []}\n"+ assert "Wrote eval.jsonc" not in capsys.readouterr().out+++def _write_marker(directory: Path, execution_id: str | None, at: str) -> None:+ (directory / ".crewai").mkdir(exist_ok=True)+ (directory / ".crewai" / "last_eval.json").write_text(+ json.dumps({"execution_id": execution_id, "at": at})+ )+++def _seconds_ago(seconds: float) -> str:+ from datetime import datetime, timedelta, timezone++ return (datetime.now(timezone.utc) - timedelta(seconds=seconds)).isoformat()+++NOT_FOUND = httpx.Response(+ 404,+ json={+ "error": "trace_not_found",+ "message": "No spans recorded for execution 6f31fe1a",+ },+)+QUEUED = httpx.Response(202, json={"id": "ev-1", "url": URL, "status": "queued"})+++def test_the_app_it_opens_is_told_an_evaluation_is_waiting(project, monkeypatch):+ """With nothing traced, the command runs the crew — and the app it opens is+ told to evaluate what it ran, so nobody quits a screen to reach it."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)++ def fake_run_crew() -> None:+ from crewai_cli.crew_run_tui import _AUTO_EVAL++ assert _AUTO_EVAL.get() is not None, "the app must be told in this process"+ # and in a child process, where a ContextVar cannot reach: a token, and+ # the file of that name that makes it count+ from crewai_cli.crew_run_tui import _awaiting_dir++ assert (_awaiting_dir() / os.environ["CREWAI_EVAL_AWAITING_RUN"]).is_file()+ record_last_run(directory, "run-it-just-did")+ eval_module.record_evaluation_outcome("run-it-just-did")++ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.calls == [] # the app did it, on its own screen+ assert "CREWAI_EVAL_AWAITING_RUN" not in os.environ+ from crewai_cli.crew_run_tui import _awaiting_dir++ assert not any(_awaiting_dir().iterdir()) # and the token went with it+++def test_a_run_no_app_evaluated_is_graded_by_the_command(project, monkeypatch):+ """Not every run goes through the app: a conversational session never ends+ the way a crew does, and a flow with human feedback takes the terminal+ instead. Nothing marked the run as evaluated, so the command grades it+ rather than exiting as if somebody had."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ monkeypatch.setattr(+ run_crew_module,+ "run_crew",+ lambda: record_last_run(directory, "run-nobody-graded", recorded_at=_now()),+ )+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.calls[0] == ("create", "run-nobody-graded")+++def test_a_marker_from_an_earlier_run_never_counts_for_this_one(project, monkeypatch):+ """A project keeps one marker, and the one from an earlier `crewai eval` is+ not about the run this command just watched — it is older than it."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ _write_marker(directory, "some-older-run", _seconds_ago(3600))++ def fake_run_crew() -> None:+ record_last_run(directory, "the-new-run", recorded_at=_now())++ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.calls[0] == ("create", "the-new-run")+++def test_the_app_saying_it_had_nothing_to_grade_is_not_a_run_to_grade(+ project, monkeypatch, capsys+):+ """An app that looked and found no trace has answered; the command says so+ once — the screen that showed it is gone — and grades nothing."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)++ def fake_run_crew() -> None:+ # another run in the same project finishes while this app is open+ record_last_run(directory, "somebody-elses-run")+ eval_module.record_evaluation_outcome(None) # what the app says++ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)+ amp = install(monkeypatch, FakeAMP())++ with pytest.raises(SystemExit) as exit_code:+ eval_module.eval_crew()++ assert exit_code.value.code == 1+ assert amp.calls == [] # never somebody else's run+ assert "no trace was recorded" in capsys.readouterr().out+++def test_a_concurrent_run_cannot_send_the_command_after_the_wrong_one(+ project, monkeypatch+):+ """The record is the project's last finish, which another run can replace+ while this app is open. The app's own marker is what settles it."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)++ def fake_run_crew() -> None:+ eval_module.record_evaluation_outcome("the-run-the-app-watched")+ record_last_run(directory, "a-run-that-finished-later")++ monkeypatch.setattr(run_crew_module, "run_crew", fake_run_crew)+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.calls == [] # the app graded its own run; nothing is graded twice+++def test_a_run_that_recorded_nothing_says_so(project, monkeypatch, capsys):+ """Nothing was traced, so nothing was evaluated and the app had nothing to+ show: this is the one thing the command still has to say."""+ directory, _ = project+ (directory / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr(eval_module.click, "confirm", lambda *a, **k: True)+ monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)+ monkeypatch.setattr(run_crew_module, "run_crew", lambda: None)+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)+ amp = install(monkeypatch, FakeAMP())++ with pytest.raises(SystemExit) as exit_code:+ eval_module.eval_crew()++ assert exit_code.value.code == 1+ assert amp.calls == []+ assert "no trace was recorded" in capsys.readouterr().out+++def test_a_run_from_hours_ago_is_not_waited_for(project, monkeypatch, capsys):+ """Nothing is in flight: AMP does not hold this run, and saying so at once+ is the useful answer."""+ directory, _ = project+ record_last_run(directory, recorded_at=_seconds_ago(7200))+ amp = install(monkeypatch, FakeAMP(create=[NOT_FOUND, QUEUED]))++ with pytest.raises(SystemExit):+ eval_module.eval_crew()++ assert [call for call in amp.calls if call[0] == "create"] == [+ ("create", EXECUTION_ID)+ ]+ assert "No spans recorded" in capsys.readouterr().out+++def test_an_id_somebody_typed_is_answered_at_once(project, monkeypatch):+ """`--run <id>` is a question about a named run, not about this project's+ last one, and a typo should not cost two minutes."""+ directory, _ = project+ record_last_run(directory, recorded_at=_seconds_ago(5))+ amp = install(monkeypatch, FakeAMP(create=[NOT_FOUND, QUEUED]))++ with pytest.raises(SystemExit):+ eval_module.eval_crew(run_id="1d6b3f70-0000-4000-8000-000000000000")++ assert [call for call in amp.calls if call[0] == "create"] == [+ ("create", "1d6b3f70-0000-4000-8000-000000000000")+ ]++def test_declining_the_offer_exits_cleanly_with_the_steps(project, monkeypatch, capsys):(project[0] / "pyproject.toml").write_text("[project]\nname = 'demo'\n")monkeypatch.setattr(eval_module.sys.stdin, "isatty", lambda: True)@@ -597,13 +1085,35 @@ def test_only_a_missing_login_reads_as_anonymous(monkeypatch, capsys):monkeypatch.setattr(eval_module, "get_auth_token", lambda: "login-token")assert eval_module.saved_login() == "login-token"+ # It stops the evaluation as a VALUE, not as an exit: the same sentence has+ # to reach a terminal and a screen, and only one of them is a terminal.for broken in (OSError(13, "Permission denied"), ValueError("Fernet key must be 32 url-safe base64-encoded bytes.")):monkeypatch.setattr(eval_module, "get_auth_token", lambda error=broken: (_ for _ in ()).throw(error))- with pytest.raises(SystemExit) as exit_:+ with pytest.raises(eval_module.EvaluationStoppedError) as stopped:eval_module.saved_login()- assert exit_.value.code == 1- out = capsys.readouterr().out- assert "Could not read the saved login" in out and type(broken).__name__ in out and "crewai login" in out+ said = str(stopped.value)+ assert "Could not read the saved login" in said and type(broken).__name__ in said and "crewai login" in said+++def test_the_command_still_prints_that_sentence_and_exits(project, monkeypatch, capsys):+ """What the app shows on its screen, the command says in the terminal."""+ directory, _ = project+ record_last_run(directory)+ stopped = eval_module.EvaluationStoppedError(+ "Could not read the saved login (OSError: Permission denied). "+ "Run `crewai login` again, or `crewai eval` will not know who you are."+ )+ monkeypatch.setattr(+ eval_module, "saved_login", lambda: (_ for _ in ()).throw(stopped)+ )+ amp = install(monkeypatch, FakeAMP())++ with pytest.raises(SystemExit) as exit_:+ eval_module.eval_crew()++ assert exit_.value.code == 1+ assert amp.calls == [] # stopped before anything was asked of AMP+ assert "Could not read the saved login" in capsys.readouterr().outdef test_read_last_run_reads_the_record_crewai_writes(tmp_path):diff --git a/lib/cli/tests/test_crew_run_tui.py b/lib/cli/tests/test_crew_run_tui.pyindex 77ae40f7f..611a41e3d 100644--- a/lib/cli/tests/test_crew_run_tui.py+++ b/lib/cli/tests/test_crew_run_tui.py@@ -1,3 +1,4 @@+import contextvarsfrom datetime import datetimeimport timefrom types import SimpleNamespace@@ -32,7 +33,7 @@ from crewai.events.types.tool_usage_events import (ToolUsageFinishedEvent,ToolUsageStartedEvent,)-from crewai_cli import run_crew+from crewai_cli import crew_run_tui, run_crewfrom crewai_cli.command import AuthenticationRequiredErrorfrom crewai_cli.crew_run_tui import (_LOG_ARGS_TEXT_LIMIT,@@ -43,6 +44,17 @@ from crewai_cli.crew_run_tui import (_try_parse_structured,)import pytest+from rich.text import Text++[email protected](autouse=True)+def _awaiting_files_stay_in_tmp(tmp_path, monkeypatch):+ """The waiting-evaluation token file belongs to the user's crewAI data+ directory; a test keeps it in its own."""+ monkeypatch.setattr(+ "crewai_cli.crew_run_tui._awaiting_dir", lambda: tmp_path / "eval-awaiting"+ )+def _app_with_plan() -> CrewRunApp:@@ -137,21 +149,56 @@ def test_chain_deploy_does_not_login_for_deploy_exit(monkeypatch, capsys) -> Nonassert "Deploy failed with exit code 42" in capsys.readouterr().out-def test_view_traces_button_click_records_telemetry(monkeypatch) -> None:+def test_view_traces_opens_the_link_crewai_recorded_for_this_run(monkeypatch) -> None:+ """crewAI prints that link after a run but never under a TUI, so the button+ reads it off the record crewAI wrote — matched to THIS app's execution."""app = CrewRunApp()app._status = "completed"app._telemetry = Mock()- notice = Mock()- monkeypatch.setattr(app, "notify", notice)+ app._execution_uuid = "run-this-app"+ monkeypatch.setattr(+ crew_run_tui,+ "_recorded_run",+ lambda uuid: {"execution_id": uuid, "trace_url": "https://amp.example/t/1"}+ if uuid == "run-this-app"+ else None,+ )+ opened: list[str] = []+ monkeypatch.setattr(CrewRunApp, "_open_report", lambda self, url: opened.append(url))+ monkeypatch.setattr(app, "notify", Mock())app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-traces")))app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:view_traces")- notice.assert_called_once_with(- "Trace sharing is requested when the execution finishes. "- "A trace link is not available for this run.",- title="Execution traces",+ assert opened == ["https://amp.example/t/1"]+++ ("record", "said"),+ [+ (None, "not traced"),+ ({"execution_id": "run-this-app"}, "granted no link"),+ ],+)+def test_view_traces_says_which_of_the_two_reasons_there_is_no_link(+ monkeypatch, record, said+) -> None:+ """A run nobody traced and a traced run AMP gave no link for are different+ problems, and only one of them is the reader's to fix."""+ app = CrewRunApp()+ app._status = "completed"+ app._telemetry = Mock()+ app._execution_uuid = "run-this-app"+ monkeypatch.setattr(crew_run_tui, "_recorded_run", lambda uuid: record)+ monkeypatch.setattr(+ CrewRunApp, "_open_report", lambda self, url: pytest.fail("nothing to open"))+ notices: list[str] = []+ monkeypatch.setattr(app, "notify", lambda message, **kwargs: notices.append(message))++ app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-traces")))++ assert said in notices[0]def test_deploy_button_click_records_telemetry() -> None:@@ -170,6 +217,363 @@ def test_deploy_button_click_records_telemetry() -> None:assert exits == [app._crew_result]+def test_evaluate_button_evaluates_here_and_does_not_leave(monkeypatch) -> None:+ """The evaluation takes minutes, has a link worth showing and a verdict+ worth reading. Sending somebody to a bare terminal to wait for them is the+ worst of both, so it runs on this screen — and it names THIS app's+ execution, not whichever run finished last in the project."""+ app = CrewRunApp()+ app._status = "completed"+ app._telemetry = Mock()+ app._execution_uuid = "run-this-app"+ app.exit = lambda result=None: pytest.fail("the app must stay open") # type: ignore[method-assign]+ started: list[str] = []+ monkeypatch.setattr(+ CrewRunApp, "_evaluate_worker", lambda self, execution_id: started.append(execution_id)+ )+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: uuid == "run-this-app")++ app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-eval")))++ app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:evaluate")+ assert started == ["run-this-app"]+ assert app._evaluation == {"state": "starting", "execution_id": "run-this-app"}+++def test_the_button_spins_while_it_evaluates_and_still_opens_the_report(+ monkeypatch,+) -> None:+ """A label that never moves reads as a screen that has stopped — and the+ report page is where the progress is, so the button stays pressable."""+ app = CrewRunApp()+ app._status = "completed"+ app._telemetry = Mock()+ app._execution_uuid = "run-this-app"+ monkeypatch.setattr(CrewRunApp, "_evaluate_worker", lambda self, execution_id: None)+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)+ labels: list[str] = []+ button = SimpleNamespace(label="Evaluate", disabled=False)+ monkeypatch.setattr(CrewRunApp, "query_one", lambda self, *a: button)+ monkeypatch.setattr(CrewRunApp, "_open_report", lambda self, url: None)++ app.action_evaluate_crew()+ labels.append(str(button.label))+ app._frame += 1+ app._refresh_eval_button()+ labels.append(str(button.label))++ assert all(label.endswith("Evaluating…") for label in labels)+ assert labels[0] != labels[1] # it moves+ assert button.disabled is False # and it is not a dead button++ app._evaluation_started({"id": "ev-1", "url": "https://optimize.example/e/1"})+ app._evaluation_finished({"status": "done", "verdict": {"gate": "passed"}})+ assert str(button.label) == "Open eval report"+++def test_pressing_evaluate_again_opens_the_report_it_already_made(monkeypatch) -> None:+ """One evaluation per run: the second press is somebody looking for the+ report, not asking to pay for another."""+ app = CrewRunApp()+ app._status = "completed"+ app._telemetry = Mock()+ app._evaluation = {"state": "done", "url": "https://optimize.example/e/1"}+ monkeypatch.setattr(+ CrewRunApp, "_evaluate_worker", lambda self, execution_id: pytest.fail("evaluated twice")+ )+ opened: list[str] = []+ monkeypatch.setattr(CrewRunApp, "_open_report", lambda self, url: opened.append(url))++ app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-eval")))++ assert opened == ["https://optimize.example/e/1"]+ app._telemetry.feature_usage_span.assert_not_called()+++def test_an_evaluation_that_stopped_can_be_tried_again(monkeypatch) -> None:+ """A timeout or a 502 is worth another press, and the run is still there to+ grade — only a finished one sends the reader to its report instead."""+ app = CrewRunApp()+ app._status = "completed"+ app._telemetry = Mock()+ app._execution_uuid = "run-this-app"+ app._evaluation = {"state": "failed", "error": "AMP answered 502."}+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)+ started: list[str] = []+ monkeypatch.setattr(+ CrewRunApp,+ "_evaluate_worker",+ lambda self, execution_id: started.append(execution_id),+ )++ app.action_evaluate_crew()++ assert started == ["run-this-app"]+++def test_a_login_that_cannot_be_read_is_shown_on_screen(monkeypatch) -> None:+ """A credential that exists but cannot be read stops the evaluation, and+ the sentence has to reach the screen: printing it would land under the+ layout, and exiting would take it away."""+ from crewai_cli.experimental import eval_crew++ app = CrewRunApp()+ app._evaluation = {"state": "starting"}+ monkeypatch.setattr(+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)+ )++ def broken_store():+ raise OSError("permission denied")++ monkeypatch.setattr(eval_crew, "get_auth_token", broken_store)++ app._evaluate_now("run-this-app")++ assert app._evaluation["state"] == "failed"+ assert "Could not read the saved login" in app._evaluation["error"]+ assert "crewai login" in app._evaluation["error"]+++def test_an_exit_with_no_reason_says_that_rather_than_its_code(monkeypatch) -> None:+ """Nothing on this path should exit any more, but an exit carries a code+ and `1` on screen is not an answer."""+ app = CrewRunApp()+ app._evaluation = {"state": "starting"}+ monkeypatch.setattr(+ CrewRunApp, "call_from_thread", lambda self, handler, *args: handler(*args)+ )+ monkeypatch.setattr(+ "crewai_cli.experimental.eval_crew.evaluate_run",+ Mock(side_effect=SystemExit(1)),+ )++ app._evaluate_now("run-this-app")++ assert app._evaluation["state"] == "failed"+ assert app._evaluation["error"] == (+ "The evaluation stopped without saying why (exit 1)."+ )+++def test_the_evaluation_shows_its_link_its_progress_and_its_verdict() -> None:+ """Everything the reader came for is on the run's own screen."""+ app = CrewRunApp()+ app._evaluation = {"state": "starting"}+ app._evaluation_started({"id": "ev-1", "url": "https://optimize.example/e/1"})+ app._open_report = lambda url: None # type: ignore[method-assign]++ line = Text()+ app._append_evaluation(line)+ assert "Evaluating this run" in line.plain+ assert "https://optimize.example/e/1" in line.plain++ app._evaluation_progress({"events": [{"data": {"subject": "the final output", "planned": 9}}]})+ line = Text()+ app._append_evaluation(line)+ assert "judged 1 of 9" in line.plain++ app._evaluation_finished(+ {"status": "done", "verdict": {"gate": "passed", "grades": {"goal": 5, "tools": None}}}+ )+ line = Text()+ app._append_evaluation(line)+ assert "Goal gate PASSED" in line.plain+ assert "goal 5/5" in line.plain and "tools —/5" in line.plain+ assert "https://optimize.example/e/1" in line.plain+++def test_an_evaluation_that_stopped_says_so_where_it_was_running() -> None:+ app = CrewRunApp()+ app._evaluation = {"state": "running", "url": "https://optimize.example/e/1"}+ app._evaluation_failed("AMP answered 404 for run x.")++ line = Text()+ app._append_evaluation(line)+ assert "Evaluation stopped" in line.plain+ assert "AMP answered 404 for run x." in line.plain++[email protected]("why", ["not traced at all", "another run replaced the record"])+def test_evaluate_says_so_when_this_run_was_not_traced(monkeypatch, why) -> None:+ """A run with no session of its own, or whose trace never reached AMP, has+ nothing to grade — and a record naming a DIFFERENT run is not evidence that+ this one was traced."""+ app = CrewRunApp()+ app._status = "completed"+ app._telemetry = Mock()+ app._execution_uuid = None if why == "not traced at all" else "run-this-app"+ notices: list[str] = []+ app.notify = lambda message, **kwargs: notices.append(message) # type: ignore[method-assign]+ app.exit = lambda result=None: pytest.fail("the TUI must not leave for nothing") # type: ignore[method-assign]+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: False)++ app.action_evaluate_crew()++ assert app._evaluation is None+ assert "not traced" in notices[0]+++def test_tracing_asked_for_reads_the_declaration_but_keeps_the_other_guards(+ monkeypatch,+) -> None:+ """The ContextVar is set where the crew is CONSTRUCTED, not on the worker+ thread that later asks — but an app with nobody in front of it is still+ nobody's yes."""+ utils = "crewai.events.listeners.tracing.utils"+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)+ app = CrewRunApp()+ app._crew = SimpleNamespace(tracing=True)++ assert app._tracing_was_asked_for() is True+ assert app._request_trace_consent() is True++ app._crew = SimpleNamespace(tracing=None)+ assert app._tracing_was_asked_for() is False++ # embedded, redirected, or under test: the declaration is not a person+ app._crew = SimpleNamespace(tracing=True)+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: False)+ assert app._tracing_was_asked_for() is False+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: True)+ assert app._tracing_was_asked_for() is False+++def test_a_run_started_for_an_evaluation_evaluates_it_here(monkeypatch) -> None:+ """`crewai eval` runs the crew when nothing is traced. The evaluation it+ came for starts on this screen when the run ends — the app does not close+ and leave somebody in a bare terminal watching for a link."""+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)+ started: list[str] = []+ monkeypatch.setattr(+ CrewRunApp,+ "_evaluate_worker",+ lambda self, execution_id: started.append(execution_id),+ )+ with crew_run_tui.evaluating_after_run() as watched:+ app = CrewRunApp()+ app._execution_uuid = "run-this-app"+ app.exit = lambda result=None: pytest.fail("the app must stay open") # type: ignore[method-assign]++ app._evaluate_if_one_is_waiting()++ assert started == ["run-this-app"]+ assert watched["execution_id"] == "run-this-app"+++def test_a_child_process_run_reads_the_waiting_evaluation_off_its_environment(+ monkeypatch,+) -> None:+ """A project's crew runs through `uv run …`, so the app is in another+ process — the one thing that crosses is the environment the command passed+ it."""+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)+ started: list[str] = []+ monkeypatch.setattr(+ CrewRunApp,+ "_evaluate_worker",+ lambda self, execution_id: started.append(execution_id),+ )++ def the_child() -> None:+ # a fresh context: that variable, and nothing this process holds in memory+ assert crew_run_tui._AUTO_EVAL.get() is None+ app = CrewRunApp()+ app._execution_uuid = "run-in-the-child"+ app.exit = lambda result=None: pytest.fail("the app must stay open") # type: ignore[method-assign]+ app._evaluate_if_one_is_waiting()++ with crew_run_tui.evaluating_after_run():+ contextvars.Context().run(the_child)++ assert started == ["run-in-the-child"]++[email protected]("value", ["1", "true", "0" * 32])+def test_a_project_env_cannot_say_an_evaluation_is_waiting(monkeypatch, value) -> None:+ """The child loads the project's `.env` over its environment. A value the+ command did not mint — with no token file behind it — is no handshake, so+ a plain `crewai run` never starts an evaluation on its own."""+ monkeypatch.setenv("CREWAI_EVAL_AWAITING_RUN", value)++ assert crew_run_tui._an_evaluation_is_waiting() is None+++def test_the_token_stops_counting_when_the_command_is_done(monkeypatch) -> None:+ with crew_run_tui.evaluating_after_run():+ passed = crew_run_tui.os.environ["CREWAI_EVAL_AWAITING_RUN"]++ monkeypatch.setenv("CREWAI_EVAL_AWAITING_RUN", passed)+ assert crew_run_tui._an_evaluation_is_waiting() is None+++def test_a_run_with_no_trace_says_so_instead_of_evaluating(monkeypatch) -> None:+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: False)+ marked: list[str | None] = []+ monkeypatch.setattr(+ "crewai_cli.experimental.eval_crew.record_evaluation_outcome", marked.append+ )+ monkeypatch.setattr(+ CrewRunApp,+ "_evaluate_worker",+ lambda self, execution_id: pytest.fail("nothing to grade"),+ )+ with crew_run_tui.evaluating_after_run() as watched:+ app = CrewRunApp()+ app._execution_uuid = "run-this-app"+ notices: list[str] = []+ app.notify = lambda message, **kwargs: notices.append(message) # type: ignore[method-assign]++ app._evaluate_if_one_is_waiting()++ assert watched["execution_id"] is None+ assert "not traced" in notices[0]+ # and the command waiting behind the screen is told, since it cannot see it+ assert marked == [None]+++def test_a_run_that_failed_is_still_evaluated(monkeypatch) -> None:+ """Where a run went wrong is what somebody asking for an evaluation wants+ to know, and the trace it exported is still there."""+ monkeypatch.setattr(crew_run_tui, "_trace_was_recorded", lambda uuid: True)+ started: list[str] = []+ monkeypatch.setattr(+ CrewRunApp,+ "_evaluate_worker",+ lambda self, execution_id: started.append(execution_id),+ )+ with crew_run_tui.evaluating_after_run():+ app = CrewRunApp()+ app._status = "failed"+ app._execution_uuid = "the-run-that-broke"++ app._evaluate_if_one_is_waiting()++ assert started == ["the-run-that-broke"]+++def test_a_run_nobody_is_waiting_on_evaluates_nothing() -> None:+ app = CrewRunApp()+ app.exit = lambda result=None: pytest.fail("the app must stay open") # type: ignore[method-assign]++ app._evaluate_if_one_is_waiting()++ assert app._evaluation is None+++def test_evaluate_before_completion_records_nothing() -> None:+ app = CrewRunApp()+ app._status = "running"+ app._telemetry = Mock()++ app.action_evaluate_crew()++ app._telemetry.feature_usage_span.assert_not_called()+ assert app._evaluation is None++def test_conversation_turn_done_records_assistant_message() -> None:class RawResult:raw = "hello from the flow"diff --git a/lib/cli/tests/test_run_declarative_flow.py b/lib/cli/tests/test_run_declarative_flow.pyindex e6c1e1ff8..2fde34a34 100644--- a/lib/cli/tests/test_run_declarative_flow.py+++ b/lib/cli/tests/test_run_declarative_flow.py@@ -418,7 +418,7 @@ def test_id_restore_still_drops_unknown_keys(# ── TUI vs terminal (headless/deploy) routing ──────────────────────-def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False):+def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False, evaluation=None):"""Replace CrewRunApp/EventListener/summary so _run_declarative_flow_tui isdriven by a controllable fake app."""@@ -430,6 +430,7 @@ def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False):self._crew_name = crew_nameself._status = statusself._want_deploy = want_deploy+ self._evaluation = evaluationself._crew_result = "result"def run(self):@@ -541,6 +542,34 @@ def test_run_declarative_flow_tui_no_deploy_when_not_requested(assert deploy_calls == []+def test_run_declarative_flow_tui_leaves_the_report_link_behind(capsys) -> None:+ """A declarative flow evaluates inside the app like a crew does. What the+ terminal is for is afterwards: the link, once the screen that held it is+ gone."""+ app = SimpleNamespace(+ _crew_name="Flow",+ _status="completed",+ _elapsed_frozen=1.0,+ _start_time=0.0,+ _input_tokens=0,+ _output_tokens=0,+ _live_out_tokens=0,+ _final_output=None,+ _error=None,+ _evaluation={+ "state": "done",+ "url": "https://optimize.example/e/1",+ "verdict": {"gate": "passed"},+ },+ )++ run_declarative_flow_module._print_flow_post_tui_summary(app)++ printed = capsys.readouterr().out+ assert "goal gate PASSED" in printed+ assert "https://optimize.example/e/1" in printed++def test_run_declarative_flow_tui_enables_flow_events(monkeypatch: pytest.MonkeyPatch,) -> None:diff --git a/lib/cli/tests/test_trace_consent_session.py b/lib/cli/tests/test_trace_consent_session.pyindex 1d9579d12..a4c37f0be 100644--- a/lib/cli/tests/test_trace_consent_session.py+++ b/lib/cli/tests/test_trace_consent_session.py@@ -18,10 +18,22 @@ class ConsentApp(CrewRunApp):event.prevent_default()+def _say_a_person_is_here(monkeypatch):+ """The rule only holds where a prompt could have been shown, and a suite is+ one of the places it could not. These cases are about a real session."""+ utils = "crewai.events.listeners.tracing.utils"+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)++async def wait_for_consent(app, pilot):for _ in range(50):await pilot.pause(0.01)- if isinstance(app.screen, TraceConsentScreen):+ # A pushed screen is not a laid-out one, and a click lands on the+ # region the buttons have — zero-sized until the first layout pass.+ if isinstance(app.screen, TraceConsentScreen) and all(+ button.region for button in app.screen.query(Button)+ ):return app.screenraise AssertionError("The execution worker did not open the consent prompt")@@ -63,6 +75,53 @@ async def test_session_consent_buttons_and_keys_resolve_execution_worker(assert app._trace_consent_pending is None[email protected]("value", ["true", "1"])+async def test_tracing_asked_for_needs_no_modal(monkeypatch, value):+ """The user turned tracing on for this project; the modal would be the same+ question a second time."""+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", value)+ _say_a_person_is_here(monkeypatch)+ app = ConsentApp()+ async with app.run_test(size=(100, 40)) as pilot:+ assert await asyncio.to_thread(app._request_trace_consent) is True+ await pilot.pause()+ assert not isinstance(app.screen, TraceConsentScreen)+ assert app._trace_consent_pending is None+++async def test_the_app_suppresses_messages_and_still_needs_no_modal(monkeypatch):+ """This app silences crewAI's console to keep it out of its own layout. That+ is a reason a PROMPT cannot be shown, not a reason to ask again — the app+ has a screen and somebody in front of it."""+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")+ _say_a_person_is_here(monkeypatch)+ monkeypatch.setattr(+ "crewai.events.listeners.tracing.utils.should_suppress_tracing_messages",+ lambda: True,+ )+ app = ConsentApp()+ async with app.run_test(size=(100, 40)) as pilot:+ assert await asyncio.to_thread(app._request_trace_consent) is True+ await pilot.pause()+ assert not isinstance(app.screen, TraceConsentScreen)+++async def test_a_declaration_is_the_same_yes_as_the_variable(monkeypatch):+ """`tracing=True` never reaches this worker's context — the crew set it on+ the thread it was built on — so the declaration itself is read."""+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)+ _say_a_person_is_here(monkeypatch)+ app = ConsentApp()+ app._crew = SimpleNamespace(tracing=True)+ async with app.run_test(size=(100, 40)) as pilot:+ assert await asyncio.to_thread(app._request_trace_consent) is True+ await pilot.pause()+ assert not isinstance(app.screen, TraceConsentScreen)++@pytest.mark.asyncioasync def test_consent_timeout_returns_false_and_dismisses_modal(monkeypatch):app = ConsentApp()diff --git a/lib/crewai-core/src/crewai_core/telemetry.py b/lib/crewai-core/src/crewai_core/telemetry.pyindex 3d86d1d39..a5e4aca26 100644--- a/lib/crewai-core/src/crewai_core/telemetry.py+++ b/lib/crewai-core/src/crewai_core/telemetry.py@@ -572,8 +572,29 @@ class Telemetry:self._safe_telemetry_procedure(_operation)- def feature_usage_span(self, feature: str) -> None:- """Records that a feature was used. One span = one count."""+ # What a caller may add to a feature span, per feature. Anything else is+ # dropped: the span counts a use, and a key nobody agreed on is how run+ # content — a prompt, an output, a path — or an identifier reaches stats+ # that promise to hold neither. The dimensions every span already carries+ # are never on a list: a caller that could overwrite `feature` could file+ # its use under somebody else's name.+ FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {+ "cli_usage:eval": frozenset({"authenticated"}),+ }++ def feature_usage_span(+ self, feature: str, attributes: dict[str, str] | None = None+ ) -> None:+ """Records that a feature was used. One span = one count.++ Args:+ feature: Feature identifier, e.g. ``"cli_usage:eval"``.+ attributes: What a caller knows about THIS use that the common+ attributes cannot know. Only the keys ``FEATURE_ATTRIBUTES`` lists+ for this feature are kept; anything else is dropped, including+ the dimensions every span already carries (the project id, the+ runtime and the version).+ """from crewai_core.version import get_crewai_versiondef _operation() -> None:@@ -581,6 +602,10 @@ class Telemetry:span = tracer.start_span("Feature Usage")self._add_attribute(span, "crewai_version", get_crewai_version())self._add_attribute(span, "feature", feature)+ allowed = self.FEATURE_ATTRIBUTES.get(feature, frozenset())+ for key, value in (attributes or {}).items():+ if key in allowed:+ self._add_attribute(span, key, str(value))close_span(span)self._safe_telemetry_procedure(_operation)diff --git a/lib/crewai-core/tests/test_smoke.py b/lib/crewai-core/tests/test_smoke.pyindex beba3a868..22097086a 100644--- a/lib/crewai-core/tests/test_smoke.py+++ b/lib/crewai-core/tests/test_smoke.py@@ -269,6 +269,48 @@ def test_core_telemetry_records_feature_usage(span.end.assert_called_once()+def test_core_feature_span_keeps_only_what_the_feature_may_send(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Each feature names what it may add. A key off its list — an id, run+ content, or a dimension every span already carries — is dropped, and a+ feature with no list sends nothing extra."""+ from crewai_core.telemetry import Telemetry++ Telemetry._instance = None+ monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)+ monkeypatch.delenv("CREWAI_DISABLE_TELEMETRY", raising=False)+ monkeypatch.delenv("CREWAI_DISABLE_TRACKING", raising=False)++ tracer = Mock()+ span = Mock()+ tracer.start_span.return_value = span+ monkeypatch.setattr(+ "crewai_core.telemetry.TracerProvider",+ lambda **_kwargs: Mock(get_tracer=Mock(return_value=tracer)),+ )+ offered = {+ "authenticated": "true",+ "execution_id": "run-1",+ "organization_id": "org-1",+ "feature": "someone_else",+ "crewai_version": "0.0.0",+ }++ telemetry = Telemetry()+ telemetry.feature_usage_span("cli_usage:eval", offered)+ sent = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}+ assert sent["authenticated"] == "true"+ assert sent["feature"] == "cli_usage:eval"+ assert sent["crewai_version"] != "0.0.0"+ assert "execution_id" not in sent and "organization_id" not in sent++ span.set_attribute.reset_mock()+ telemetry.feature_usage_span("cli_usage:deploy", offered)+ sent = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}+ assert "authenticated" not in sent and sent["feature"] == "cli_usage:deploy"++def test_core_telemetry_records_flow_creation_version(monkeypatch: pytest.MonkeyPatch,) -> None:diff --git a/lib/crewai/src/crewai/events/listeners/tracing/utils.py b/lib/crewai/src/crewai/events/listeners/tracing/utils.pyindex 0b434c265..c58125441 100644--- a/lib/crewai/src/crewai/events/listeners/tracing/utils.py+++ b/lib/crewai/src/crewai/events/listeners/tracing/utils.py@@ -55,6 +55,7 @@ __all__ = ["should_auto_collect_first_time_traces","should_enable_tracing","should_suppress_tracing_messages",+ "tracing_asked_for","truncate_messages","update_user_data",]@@ -105,6 +106,32 @@ def should_suppress_tracing_messages() -> bool:return _suppress_tracing_messages.get()+def tracing_asked_for() -> bool:+ """True when a PERSON turned tracing on for this run — `CREWAI_TRACING_ENABLED`,+ or `tracing=True` on the crew or flow in this context — and is there to have+ meant it.++ Turning tracing on is the answer to "may we collect this?"; asking again when+ the run ends is asking the same question twice. First-time auto-collection is+ the case that still has to ask, because nobody asked for it.++ Where the prompt could not be shown, there is nothing to answer. A copied+ `.env` that reaches CI, a container or a server carries the variable without+ carrying the person; a suite under test, and a host that asked for tracing+ messages to be suppressed, are the same case. An ephemeral run has always+ failed closed in all three rather than upload what nobody approved in front+ of a screen, and it still does — this turns a second question into an+ answer, and changes nothing about where the first one is asked.+ """+ if not (+ is_tracing_enabled_in_context()+ or os.getenv("CREWAI_TRACING_ENABLED", "").lower() in ("true", "1")+ ):+ return False++ return _prompt_can_be_shown()++def should_enable_tracing(*, override: bool | None = None) -> bool:"""Determine if tracing should be enabled.@@ -486,6 +513,25 @@ def _is_interactive_terminal() -> bool:return False+def _prompt_can_be_shown() -> bool:+ """Could a question about traces be put to somebody at all?++ Three cases where it cannot: a suite under test, a host that asked for+ tracing messages to be suppressed, and a process with no terminal (CI, API+ servers, Docker) where a prompt would block for twenty seconds with nobody+ to answer it. `prompt_user_for_trace_viewing` returns False in each, which+ is what keeps an unapproved ephemeral trace on the machine — and+ `tracing_asked_for` asks the same question, so the two cannot drift apart.+ """+ if _is_test_environment():+ return False++ if should_suppress_tracing_messages():+ return False++ return _is_interactive_terminal()++def prompt_user_for_trace_viewing(timeout_seconds: int = 20, *, sharing: bool = False) -> bool:@@ -494,15 +540,7 @@ def prompt_user_for_trace_viewing(Returns True if user agrees, False otherwise. ``sharing`` explicitly askspermission to upload locally buffered spans rather than merely view a trace."""- if _is_test_environment():- return False-- if should_suppress_tracing_messages():- return False-- # Skip prompt in non-interactive contexts (CI, API servers, Docker, etc.)- # This avoids blocking for 20 seconds when no one can respond- if not _is_interactive_terminal():+ if not _prompt_can_be_shown():return Falsetry:diff --git a/lib/crewai/src/crewai/telemetry/telemetry.py b/lib/crewai/src/crewai/telemetry/telemetry.pyindex 1e4e0041f..a34fbe4ca 100644--- a/lib/crewai/src/crewai/telemetry/telemetry.py+++ b/lib/crewai/src/crewai/telemetry/telemetry.py@@ -18,7 +18,7 @@ import osimport platformimport signalimport threading-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, ClassVarfrom crewai_core.telemetry import (CommonAttributesSpanProcessor,@@ -1285,13 +1285,29 @@ class Telemetry:self._safe_telemetry_operation(_operation)- def feature_usage_span(self, feature: str) -> None:+ # What a caller may add to a feature span, per feature. Anything else is+ # dropped: the span counts a use, and a key nobody agreed on is how run+ # content — a prompt, an output, a path — or an identifier reaches stats+ # that promise to hold neither. The dimensions every span already carries+ # are never on a list: a caller that could overwrite `feature` could file+ # its use under somebody else's name.+ FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {+ "cli_usage:eval": frozenset({"authenticated"}),+ }++ def feature_usage_span(+ self, feature: str, attributes: dict[str, str] | None = None+ ) -> None:"""Records that a feature was used. One span = one count.Args:feature: Feature identifier, e.g. "planning:creation","mcp:connection", "a2a:delegation","hooks:pre_tool_call", "hooks:aborted".+ attributes: What a caller knows about THIS use that the common+ attributes cannot know. Only the keys ``FEATURE_ATTRIBUTES`` lists+ for this feature are kept; anything else is dropped, including+ the dimensions every span already carries."""def _operation() -> None:@@ -1299,6 +1315,10 @@ class Telemetry:span = tracer.start_span("Feature Usage")self._add_attribute(span, "crewai_version", version("crewai"))self._add_attribute(span, "feature", feature)+ allowed = self.FEATURE_ATTRIBUTES.get(feature, frozenset())+ for key, value in (attributes or {}).items():+ if key in allowed:+ self._add_attribute(span, key, str(value))close_span(span)self._safe_telemetry_operation(_operation)diff --git a/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py b/lib/crewai/src/crewai/telemetry/tracing/ephemeral.pyindex 0c160a895..4c870a519 100644--- a/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py+++ b/lib/crewai/src/crewai/telemetry/tracing/ephemeral.py@@ -19,7 +19,10 @@ from opentelemetry.sdk.trace.export import (SpanExporter,)-from crewai.events.listeners.tracing.utils import prompt_user_for_trace_viewing+from crewai.events.listeners.tracing.utils import (+ prompt_user_for_trace_viewing,+ tracing_asked_for,+)from crewai.telemetry.tracing.grants import (GrantSpanExporter,TraceGrantClient,@@ -122,10 +125,18 @@ class EphemeralSpanBuffer(SpanExporter):)if not buffered:return+ # A host's consent callback stays authoritative — it is policy, and+ # an embedder may answer no for reasons of its own. What it replaces+ # is the PROMPT, and a prompt is what a user who turned tracing on+ # has already answered: CREWAI_TRACING_ENABLED, or tracing=True, is+ # the yes. (A callback that opens its own prompt — the TUI's modal —+ # asks `tracing_asked_for()` for itself.)consent = self._consent or _trace_consent.get()approved = (consent()if consent is not None+ else True+ if tracing_asked_for()else prompt_user_for_trace_viewing(sharing=True))if first_time:diff --git a/lib/crewai/src/crewai/telemetry/tracing/grants.py b/lib/crewai/src/crewai/telemetry/tracing/grants.pyindex 9678c3097..a9dcc3071 100644--- a/lib/crewai/src/crewai/telemetry/tracing/grants.py+++ b/lib/crewai/src/crewai/telemetry/tracing/grants.py@@ -298,6 +298,7 @@ class GrantSpanExporter(SpanExporter):started_at_ns=self._first_start_ns,finished_at_ns=self._last_end_ns,amp_base_url=getattr(api, "base_url", None),+ trace_url=self._trace_url,)logger.debug("Traces exported for execution %s", execution_uuid)self._show_trace_link()diff --git a/lib/crewai/src/crewai/telemetry/tracing/last_run.py b/lib/crewai/src/crewai/telemetry/tracing/last_run.pyindex 53e137dbd..e8b78464f 100644--- a/lib/crewai/src/crewai/telemetry/tracing/last_run.py+++ b/lib/crewai/src/crewai/telemetry/tracing/last_run.py@@ -72,6 +72,7 @@ def record_last_run(started_at_ns: int | None,finished_at_ns: int | None,amp_base_url: str | None,+ trace_url: str | None = None,) -> Path | None:"""Write the record atomically; the path, or None when recording is offor the write failed. Never raises — a run is never failed by this.@@ -90,6 +91,11 @@ def record_last_run("recorded_at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),"amp_base_url": amp_base_url,}+ # AMP's viewer link for this run, when it granted one. The run app shows no+ # console line — crewAI's own link is silenced under a TUI — so the record+ # is the only way its View Traces button can know where to go.+ if trace_url:+ record["trace_url"] = trace_urlpath: Path | None = Nonetry:path = last_run_path()diff --git a/lib/crewai/tests/telemetry/test_deferred_trace_transitions.py b/lib/crewai/tests/telemetry/test_deferred_trace_transitions.pyindex d97615e38..f7a8b5cc6 100644--- a/lib/crewai/tests/telemetry/test_deferred_trace_transitions.py+++ b/lib/crewai/tests/telemetry/test_deferred_trace_transitions.py@@ -115,6 +115,69 @@ def test_disabling_deferred_trace_discards_without_consent(assert get_trace_session() is None and get_execution_uuid() is None[email protected]("how", ["env", "flag"])+def test_tracing_asked_for_is_the_answer_and_the_run_is_not_asked_again(+ traces, monkeypatch, how+):+ """Turning tracing on IS consent. The prompt at the end of a run is for the+ first-time collection nobody asked for — not for a user who said collect it."""+ buffers, grants, recorders, prompt = traces+ # the rule only holds where a prompt could have been shown, and a suite is+ # one of the places it could not — so say a person is here+ utils = "crewai.events.listeners.tracing.utils"+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)+ if how == "env":+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")+ else: # the flag alone must carry it, or this case proves nothing+ monkeypatch.delenv("CREWAI_TRACING_ENABLED", raising=False)++ class Conversation(Flow):+ @start()+ def turn(self):+ return "said out loud"++ flow = Conversation(tracing=(how == "flag") or None)+ assert flow.kickoff() == "said out loud"+ assert crewai_event_bus.flush()++ prompt.assert_not_called() # asked once, not twice+ assert grants and recorders # and the trace went where it was told to go+++ "closed", ["no terminal", "under test", "messages suppressed"]+)+def test_where_the_prompt_could_not_be_shown_nothing_is_uploaded(+ traces, monkeypatch, closed+):+ """The switch answers a question; where the question could not have been+ put to anybody, there is nothing to answer. A copied `.env` reaching CI+ carries the variable, not the person — and a suite under test, and a host+ that suppressed tracing messages, are the same case. All three have always+ failed closed, and still do."""+ buffers, grants, recorders, prompt = traces+ monkeypatch.setenv("CREWAI_TRACING_ENABLED", "true")+ utils = "crewai.events.listeners.tracing.utils"+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: closed != "no terminal")+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: closed == "under test")+ monkeypatch.setattr(+ f"{utils}.should_suppress_tracing_messages",+ lambda: closed == "messages suppressed",+ )+ prompt.return_value = False # what the prompt answers where nobody can answer++ class Conversation(Flow):+ @start()+ def turn(self):+ return "said in a container"++ assert Conversation(tracing=True).kickoff() == "said in a container"+ assert crewai_event_bus.flush()++ assert not grants and not recorders # nothing left the machine++def test_enabling_deferred_trace_opens_a_new_flow_root(traces):buffers, grants, recorders, prompt = tracessessions = []diff --git a/lib/crewai/tests/telemetry/test_session_trace_export.py b/lib/crewai/tests/telemetry/test_session_trace_export.pyindex db2ac76d6..ee39e61a0 100644--- a/lib/crewai/tests/telemetry/test_session_trace_export.py+++ b/lib/crewai/tests/telemetry/test_session_trace_export.py@@ -223,6 +223,10 @@ def test_the_viewer_link_is_shown_the_id_is_not_and_a_successful_export_is_recorassert written["tier"] == ("authenticated" if authenticated else "ephemeral")assert written["started_at"] and written["finished_at"] and written["recorded_at"]assert written["amp_base_url"] == collector.url+ # The link is recorded even where it is never printed: under a TUI the+ # console line is silenced, and the run app's View Traces button has+ # nowhere else to read it from.+ assert written["trace_url"] == urlelse:assert written is Nonediff --git a/lib/crewai/tests/telemetry/test_trace_lifecycle.py b/lib/crewai/tests/telemetry/test_trace_lifecycle.pyindex 5e62866e5..fce79e34b 100644--- a/lib/crewai/tests/telemetry/test_trace_lifecycle.py+++ b/lib/crewai/tests/telemetry/test_trace_lifecycle.py@@ -118,6 +118,9 @@ def test_pause_resume_exports_traces_and_preserves_deferred_root(def finish(self, feedback):return feedback.feedback+ utils = "crewai.events.listeners.tracing.utils"+ monkeypatch.setattr(f"{utils}._is_interactive_terminal", lambda: True)+ monkeypatch.setattr(f"{utils}._is_test_environment", lambda: False)with patch("crewai.telemetry.tracing.ephemeral.prompt_user_for_trace_viewing",return_value=True,@@ -142,7 +145,8 @@ def test_pause_resume_exports_traces_and_preserves_deferred_root(assert all(span.status.status_code != trace.StatusCode.ERROR for span in paused)- assert prompt.call_count == (0 if authenticated or deferred else 1)+ # `tracing=True` IS the consent: nobody is asked again mid-flight+ assert prompt.call_count == 0assert get_trace_session() is None and get_execution_uuid() is Noneresumed = (@@ -159,7 +163,7 @@ def test_pause_resume_exports_traces_and_preserves_deferred_root(flow.finalize_session_traces()flow.finalize_session_traces()- assert prompt.call_count == (0 if authenticated else (1 if deferred else 2))+ assert prompt.call_count == 0 # … nor when the deferred trace is finalisedassert flow._deferred_execution_trace is Noneassert len(issued) == (1 if deferred else 2)diff --git a/lib/crewai/tests/telemetry/test_tracer_isolation.py b/lib/crewai/tests/telemetry/test_tracer_isolation.pyindex 9ee96383d..3783988f7 100644--- a/lib/crewai/tests/telemetry/test_tracer_isolation.py+++ b/lib/crewai/tests/telemetry/test_tracer_isolation.py@@ -94,6 +94,30 @@ def test_our_own_spans_still_reach_our_exporter(telemetry_with_exporter):assert [span.name for span in exporter.get_finished_spans()] == ["Feature Usage"]+def test_a_feature_span_keeps_only_what_the_feature_may_send(telemetry_with_exporter):+ """The same policy as crewai-core's: each feature names what it may add, and+ the dimensions every span already carries cannot be overwritten."""+ telemetry, exporter = telemetry_with_exporter+ telemetry.set_tracer()+ offered = {+ "authenticated": "false",+ "execution_id": "run-1",+ "organization_id": "org-1",+ "feature": "someone_else",+ }++ telemetry.feature_usage_span("cli_usage:eval", offered)+ telemetry.feature_usage_span("cli_usage:deploy", offered)++ eval_span, deploy_span = exporter.get_finished_spans()+ assert eval_span.attributes["authenticated"] == "false"+ assert eval_span.attributes["feature"] == "cli_usage:eval"+ assert "execution_id" not in eval_span.attributes+ assert "organization_id" not in eval_span.attributes+ assert "authenticated" not in deploy_span.attributes+ assert deploy_span.attributes["feature"] == "cli_usage:deploy"++def test_our_spans_are_unaffected_by_an_application_provider(telemetry_with_exporter):"""An app that installs its own provider must not divert our telemetry.