Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

crewai eval --models: per-run cost and time, and no repeated progress lines (#7965)

João Mouraed71a6f4d631
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index e7c41209d..20b77e477 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -1032,12 +1032,15 @@ def eval_models(models_text: str, deployment_id: str | None = None) -> None:
_open(url)
console.print("Waiting for the comparison…", style="dim")
- shown: list[str] = []
+ # Every line once: setups run in parallel, so consecutive polls alternate
+ # between them ("judging the final output", "0 of 3 setups done", again),
+ # and a line said before carries nothing new.
+ shown: set[str] = set()
def show_progress(payload: dict[str, Any]) -> None:
line = _progress_line(payload)
- if line and (not shown or shown[-1] != line):
- shown.append(line)
+ if line and line not in shown:
+ shown.add(line)
_note(line)
try:
@@ -1125,13 +1128,15 @@ def _print_comparison(finished: dict[str, Any], url: str | None) -> None:
rows: list[dict[str, Any]] = comparison["models"]
grades = {area: [_grade_of(row, area) for row in rows] for area in GRADE_COLUMNS}
- costs = [_number(row.get("cost_usd")) for row in rows]
- seconds = [_number(row.get("seconds")) for row in rows]
+ # Per run, as the brief and the report say it; the totals over every run of
+ # a setup only from a service that sends no per-run figure.
+ costs, cost_header = _per_run(rows, "cost_per_run", "cost_usd", "cost")
+ seconds, time_header = _per_run(rows, "seconds_per_run", "seconds", "time")
best = {area: _best(values, max) for area, values in grades.items()}
cheapest, fastest = _best(costs, min), _best(seconds, min)
table = Table(show_edge=False, pad_edge=False)
- for header in ("model", *GRADE_COLUMNS, "cost", "time"):
+ for header in ("model", *GRADE_COLUMNS, cost_header, time_header):
table.add_column(header, justify="left" if header == "model" else "right")
for n, row in enumerate(rows):
label = Text(str(row.get("label") or row.get("key") or "?"))
@@ -1152,6 +1157,14 @@ def _print_comparison(finished: dict[str, Any], url: str | None) -> None:
)
table.add_row(*cells)
console.print(table)
+ # A dash is a model the service could not price, or a setup with no time
+ # recorded — said once, so a `/ run` column with a gap is not a mystery.
+ if any(cost is None for cost in costs):
+ console.print(Text("— in cost: no price is known for that model", style="dim"))
+ if any(took is None for took in seconds):
+ console.print(
+ Text("— in time: no time was recorded for that setup", style="dim")
+ )
suggestions = _top_suggestions(comparison.get("suggestions"))
if suggestions:
@@ -1175,6 +1188,26 @@ def _print_comparison(finished: dict[str, Any], url: str | None) -> None:
console.print(Text(f"Full report: {url}"))
+def _per_run(
+ rows: list[dict[str, Any]], per_run: str, total: str, noun: str
+) -> tuple[list[float | None], str]:
+ """One column's values and header: the per-run figure when every row that
+ has a value has one (`noun / run`), else every row's total (`noun`) — never
+ a column that mixes the two. A row with neither (a model the service could
+ not price) does not decide the unit: one unpriced model would otherwise turn
+ the whole column back into totals, which the brief beside it does not use.
+ Its dash is explained under the table."""
+ totals = [_number(row.get(total)) for row in rows]
+ runs = [_number(row.get(per_run)) for row in rows]
+ if any(value is not None for value in runs) and all(
+ run is not None
+ for run, whole in zip(runs, totals, strict=True)
+ if whole is not None
+ ):
+ return runs, f"{noun} / run"
+ return totals, noun
+
+
def _grade_of(row: dict[str, Any], area: str) -> int | None:
grades = row.get("grades")
grade = grades.get(area) if isinstance(grades, dict) else None
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index fbdf50e4b..fb76fe9cc 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -1331,6 +1331,70 @@ def test_models_are_compared_on_the_deployment_and_the_table_printed(deployed, m
assert "a fourth" not in out
+def test_cost_and_time_are_per_run_when_the_service_sends_them(deployed, monkeypatch, capsys):
+ per_run = comparison()
+ per_run["models"][0].update(cost_per_run=0.0020, seconds_per_run=4.5)
+ per_run["models"][1].update(cost_per_run=0.0004, seconds_per_run=2.5)
+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(
+ 200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": per_run})]))
+
+ eval_module.eval_models(MODELS)
+
+ out = capsys.readouterr().out
+ assert "cost / run" in out and "time / run" in out
+ mini = next(line for line in out.splitlines() if "Poem composer: openai/gpt-4o-mini" in line)
+ base = next(line for line in out.splitlines() if "gpt-5.6-sol" in line)
+ assert "$0.0004 ★" in mini and "2.5s ★" in mini
+ assert "$0.0020" in base and "4.5s" in base
+ # the totals are not what the column says
+ assert "$0.0012" not in out and "9.5s" not in out
+
+
+def test_a_column_never_mixes_per_run_and_totals(deployed, monkeypatch, capsys):
+ partial = comparison()
+ partial["models"][1].update(cost_per_run=0.0004) # one row only: the totals stand
+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(
+ 200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": partial})]))
+
+ eval_module.eval_models(MODELS)
+
+ out = capsys.readouterr().out
+ assert "cost / run" not in out and "time / run" not in out
+ assert "$0.0012 ★" in out and "$0.0004" not in out
+
+
+def test_an_unpriced_model_keeps_the_per_run_unit_and_its_dash_is_explained(deployed, monkeypatch, capsys):
+ unpriced = comparison()
+ unpriced["models"][0].update(cost_per_run=0.0020, seconds_per_run=4.5)
+ unpriced["models"][1].update(cost_usd=None, cost_per_run=None, seconds_per_run=2.5)
+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(
+ 200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": unpriced})]))
+
+ eval_module.eval_models(MODELS)
+
+ out = capsys.readouterr().out
+ assert "cost / run" in out and "time / run" in out
+ mini = next(line for line in out.splitlines() if "Poem composer: openai/gpt-4o-mini" in line)
+ assert "—" in mini and "$0.0020" in out
+ assert "— in cost: no price is known for that model" in out
+ assert "— in time" not in out
+
+
+def test_a_progress_line_is_said_once_even_when_parallel_setups_alternate(deployed, monkeypatch, capsys):
+ def at(message):
+ return httpx.Response(200, json={"id": "ev-9", "status": "running", "progress": {"message": message}})
+
+ alternating = [at("judging the final output"), at("0 of 3 setups done")] * 4
+ install(monkeypatch, FakeModelsAMP(statuses=[*alternating, at("1 of 3 setups done"), compared()]))
+
+ eval_module.eval_models(MODELS)
+
+ out = capsys.readouterr().out
+ assert out.count("judging the final output") == 1
+ assert out.count("0 of 3 setups done") == 1
+ assert out.index("0 of 3 setups done") < out.index("1 of 3 setups done")
+
+
def test_the_comparison_is_counted_with_the_models_and_nothing_that_names_it(deployed, monkeypatch):
spans: list[tuple[str, dict[str, str]]] = []