Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
crewai eval --models: per-run cost and time, and no repeated progress lines (#7965)
João Mouraed71a6f4d631
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex e7c41209d..20b77e477 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -1032,12 +1032,15 @@ def eval_models(models_text: str, deployment_id: str | None = None) -> None:_open(url)console.print("Waiting for the comparison…", style="dim")- shown: list[str] = []+ # Every line once: setups run in parallel, so consecutive polls alternate+ # between them ("judging the final output", "0 of 3 setups done", again),+ # and a line said before carries nothing new.+ shown: set[str] = set()def show_progress(payload: dict[str, Any]) -> None:line = _progress_line(payload)- if line and (not shown or shown[-1] != line):- shown.append(line)+ if line and line not in shown:+ shown.add(line)_note(line)try:@@ -1125,13 +1128,15 @@ def _print_comparison(finished: dict[str, Any], url: str | None) -> None:rows: list[dict[str, Any]] = comparison["models"]grades = {area: [_grade_of(row, area) for row in rows] for area in GRADE_COLUMNS}- costs = [_number(row.get("cost_usd")) for row in rows]- seconds = [_number(row.get("seconds")) for row in rows]+ # Per run, as the brief and the report say it; the totals over every run of+ # a setup only from a service that sends no per-run figure.+ costs, cost_header = _per_run(rows, "cost_per_run", "cost_usd", "cost")+ seconds, time_header = _per_run(rows, "seconds_per_run", "seconds", "time")best = {area: _best(values, max) for area, values in grades.items()}cheapest, fastest = _best(costs, min), _best(seconds, min)table = Table(show_edge=False, pad_edge=False)- for header in ("model", *GRADE_COLUMNS, "cost", "time"):+ for header in ("model", *GRADE_COLUMNS, cost_header, time_header):table.add_column(header, justify="left" if header == "model" else "right")for n, row in enumerate(rows):label = Text(str(row.get("label") or row.get("key") or "?"))@@ -1152,6 +1157,14 @@ def _print_comparison(finished: dict[str, Any], url: str | None) -> None:)table.add_row(*cells)console.print(table)+ # A dash is a model the service could not price, or a setup with no time+ # recorded — said once, so a `/ run` column with a gap is not a mystery.+ if any(cost is None for cost in costs):+ console.print(Text("— in cost: no price is known for that model", style="dim"))+ if any(took is None for took in seconds):+ console.print(+ Text("— in time: no time was recorded for that setup", style="dim")+ )suggestions = _top_suggestions(comparison.get("suggestions"))if suggestions:@@ -1175,6 +1188,26 @@ def _print_comparison(finished: dict[str, Any], url: str | None) -> None:console.print(Text(f"Full report: {url}"))+def _per_run(+ rows: list[dict[str, Any]], per_run: str, total: str, noun: str+) -> tuple[list[float | None], str]:+ """One column's values and header: the per-run figure when every row that+ has a value has one (`noun / run`), else every row's total (`noun`) — never+ a column that mixes the two. A row with neither (a model the service could+ not price) does not decide the unit: one unpriced model would otherwise turn+ the whole column back into totals, which the brief beside it does not use.+ Its dash is explained under the table."""+ totals = [_number(row.get(total)) for row in rows]+ runs = [_number(row.get(per_run)) for row in rows]+ if any(value is not None for value in runs) and all(+ run is not None+ for run, whole in zip(runs, totals, strict=True)+ if whole is not None+ ):+ return runs, f"{noun} / run"+ return totals, noun++def _grade_of(row: dict[str, Any], area: str) -> int | None:grades = row.get("grades")grade = grades.get(area) if isinstance(grades, dict) else Nonediff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex fbdf50e4b..fb76fe9cc 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -1331,6 +1331,70 @@ def test_models_are_compared_on_the_deployment_and_the_table_printed(deployed, massert "a fourth" not in out+def test_cost_and_time_are_per_run_when_the_service_sends_them(deployed, monkeypatch, capsys):+ per_run = comparison()+ per_run["models"][0].update(cost_per_run=0.0020, seconds_per_run=4.5)+ per_run["models"][1].update(cost_per_run=0.0004, seconds_per_run=2.5)+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(+ 200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": per_run})]))++ eval_module.eval_models(MODELS)++ out = capsys.readouterr().out+ assert "cost / run" in out and "time / run" in out+ mini = next(line for line in out.splitlines() if "Poem composer: openai/gpt-4o-mini" in line)+ base = next(line for line in out.splitlines() if "gpt-5.6-sol" in line)+ assert "$0.0004 ★" in mini and "2.5s ★" in mini+ assert "$0.0020" in base and "4.5s" in base+ # the totals are not what the column says+ assert "$0.0012" not in out and "9.5s" not in out+++def test_a_column_never_mixes_per_run_and_totals(deployed, monkeypatch, capsys):+ partial = comparison()+ partial["models"][1].update(cost_per_run=0.0004) # one row only: the totals stand+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(+ 200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": partial})]))++ eval_module.eval_models(MODELS)++ out = capsys.readouterr().out+ assert "cost / run" not in out and "time / run" not in out+ assert "$0.0012 ★" in out and "$0.0004" not in out+++def test_an_unpriced_model_keeps_the_per_run_unit_and_its_dash_is_explained(deployed, monkeypatch, capsys):+ unpriced = comparison()+ unpriced["models"][0].update(cost_per_run=0.0020, seconds_per_run=4.5)+ unpriced["models"][1].update(cost_usd=None, cost_per_run=None, seconds_per_run=2.5)+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(+ 200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": unpriced})]))++ eval_module.eval_models(MODELS)++ out = capsys.readouterr().out+ assert "cost / run" in out and "time / run" in out+ mini = next(line for line in out.splitlines() if "Poem composer: openai/gpt-4o-mini" in line)+ assert "—" in mini and "$0.0020" in out+ assert "— in cost: no price is known for that model" in out+ assert "— in time" not in out+++def test_a_progress_line_is_said_once_even_when_parallel_setups_alternate(deployed, monkeypatch, capsys):+ def at(message):+ return httpx.Response(200, json={"id": "ev-9", "status": "running", "progress": {"message": message}})++ alternating = [at("judging the final output"), at("0 of 3 setups done")] * 4+ install(monkeypatch, FakeModelsAMP(statuses=[*alternating, at("1 of 3 setups done"), compared()]))++ eval_module.eval_models(MODELS)++ out = capsys.readouterr().out+ assert out.count("judging the final output") == 1+ assert out.count("0 of 3 setups done") == 1+ assert out.index("0 of 3 setups done") < out.index("1 of 3 setups done")++def test_the_comparison_is_counted_with_the_models_and_nothing_that_names_it(deployed, monkeypatch):spans: list[tuple[str, dict[str, str]]] = []