Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

feat(cli): crewai eval --models, and llm_overlay swaps models as well as roles (#7811)

João Moura1b9bbccbed63
diff --git a/.github/workflows/vulnerability-scan.yml b/.github/workflows/vulnerability-scan.yml
index 4052f1e77..8d853b662 100644
--- a/.github/workflows/vulnerability-scan.yml
+++ b/.github/workflows/vulnerability-scan.yml
@@ -117,6 +117,16 @@ jobs:
# patched release; keep the ignore list in sync with
# .pre-commit-config.yaml.
--ignore-vuln GHSA-4j2p-28q2-5m79
+ # oauthlib <4.0.0: GHSA-xpv3-w29h-x7cv: timing side channel in the
+ # server-side PKCE code_verifier check (authorization_code grant).
+ # The fix, 4.0.0 (published 2026-09-28), is inside the 3-day
+ # exclude-newer cooldown. Transitive via chromadb -> kubernetes ->
+ # requests-oauthlib, which uses oauthlib as an OAuth client only;
+ # CrewAI never runs an oauthlib authorization server.
+ # TODO: drop this ignore and run `uv lock --upgrade-package oauthlib`
+ # once 4.0.0 clears the cooldown (after 2026-10-01 06:01 UTC); keep
+ # the ignore list in sync with .pre-commit-config.yaml.
+ --ignore-vuln GHSA-xpv3-w29h-x7cv
)
uv run pip-audit "${pip_audit_args[@]}"
continue-on-error: true
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 64ec22459..cffd7dc99 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -31,6 +31,7 @@ repos:
# Keep this ignore list in sync with .github/workflows/vulnerability-scan.yml.
# TODO: drop --ignore-vuln GHSA-8mgp-746c-j5xp when bumping nltk past 3.10.3.
# TODO: drop --ignore-vuln GHSA-4j2p-28q2-5m79 when bumping accelerate past 1.14.0.
+ # TODO: drop --ignore-vuln GHSA-xpv3-w29h-x7cv when bumping oauthlib to 4.0.0 (after 2026-10-01 06:01 UTC).
entry: >-
bash -c 'case "$OSTYPE" in msys*|cygwin*|win32*) source .venv/Scripts/activate ;; *) source .venv/bin/activate ;; esac && uv run pip-audit --skip-editable
--ignore-vuln PYSEC-2024-277
@@ -63,7 +64,8 @@ repos:
--ignore-vuln GHSA-36p7-vc44-83pf
--ignore-vuln GHSA-xph7-9rjv-w5fr
--ignore-vuln GHSA-8mgp-746c-j5xp
- --ignore-vuln GHSA-4j2p-28q2-5m79' --
+ --ignore-vuln GHSA-4j2p-28q2-5m79
+ --ignore-vuln GHSA-xpv3-w29h-x7cv' --
language: system
pass_filenames: false
stages: [pre-push, manual]
diff --git a/lib/cli/pyproject.toml b/lib/cli/pyproject.toml
index 84188fdb2..a48c761ef 100644
--- a/lib/cli/pyproject.toml
+++ b/lib/cli/pyproject.toml
@@ -15,7 +15,7 @@ dependencies = [
"appdirs~=1.4.4",
"cryptography>=42.0",
"httpx~=0.28.1",
- "pyjwt>=2.13.0,<3",
+ "pyjwt>=2.15.0,<3",
"rich>=13.7.1",
"tomli~=2.0.2",
"tomli-w~=1.1.0",
diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.py
index 04199729e..7323e8207 100644
--- a/lib/cli/src/crewai_cli/cli.py
+++ b/lib/cli/src/crewai_cli/cli.py
@@ -54,6 +54,12 @@ def eval_crew(*args: Any, **kwargs: Any) -> Any:
return _eval_crew(*args, **kwargs)
+def eval_models(*args: Any, **kwargs: Any) -> Any:
+ from crewai_cli.experimental.eval_crew import eval_models as _eval_models
+
+ return _eval_models(*args, **kwargs)
+
+
if TYPE_CHECKING:
# mypy sees the real classes; at runtime the shims below defer the
# heavy imports until a command actually instantiates them.
@@ -692,12 +698,52 @@ def run(
"crewAI recorded for the run."
),
)
-def eval_command(run_id: str | None) -> None:
- """Evaluate the last traced run through CrewAI AMP.
+ "--models",
+ "models",
+ type=str,
+ default=None,
+ metavar="LIST",
+ help=(
+ "Compare models on this project's deployment instead: ONE comma-separated "
+ 'list of provider/model, e.g. "openai/gpt-4o-mini,anthropic/claude-haiku-4-5". '
+ "The deployment runs once as deployed and once per model; needs `crewai login`."
+ ),
+)
+ "--deployment",
+ "deployment_id",
+ type=str,
+ default=None,
+ metavar="UUID",
+ help=(
+ "With --models: the deployment to run, when AMP cannot tell it from the "
+ "project id."
+ ),
+)
+def eval_command(
+ run_id: str | None, models: str | None, deployment_id: str | None
+) -> None:
+ """Evaluate the last traced run through CrewAI AMP, or compare models on the
+ project's deployment (--models).
- Exits 0 only when the goal gate PASSED, and 1 otherwise — a failed gate, no
- verdict, or an evaluation that could not run — so a CI job can gate on it.
+ A run's evaluation exits 0 only when the goal gate PASSED, and 1 otherwise — a
+ failed gate, no verdict, or an evaluation that could not run — so a CI job can
+ gate on it. A comparison exits 0 when it finished, 1 when it failed or could
+ not start.
"""
+ if models is not None:
+ if run_id is not None:
+ raise click.UsageError(
+ "--run grades a run that already happened; --models runs the "
+ "deployment again. Give one of them."
+ )
+ eval_models(models, deployment_id=deployment_id)
+ return
+ if deployment_id is not None:
+ raise click.UsageError(
+ "--deployment names the deployment --models runs; add --models LIST."
+ )
eval_crew(run_id=run_id)
diff --git a/lib/cli/src/crewai_cli/deploy/main.py b/lib/cli/src/crewai_cli/deploy/main.py
index 17e1f1249..85d652cfa 100644
--- a/lib/cli/src/crewai_cli/deploy/main.py
+++ b/lib/cli/src/crewai_cli/deploy/main.py
@@ -15,7 +15,11 @@ from crewai_cli.command import BaseCommand, PlusAPIMixin
from crewai_cli.constants import DEFAULT_CREWAI_ENTERPRISE_URL
from crewai_cli.deploy.archive import ArchiveError, create_project_zip
from crewai_cli.deploy.validate import DeployValidator, Severity, render_report
-from crewai_cli.utils import fetch_and_json_env_file, get_project_name
+from crewai_cli.utils import (
+ fetch_and_json_env_file,
+ get_project_id,
+ get_project_name,
+)
console = Console()
@@ -232,6 +236,14 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
BaseCommand.__init__(self)
PlusAPIMixin.__init__(self, telemetry=self._telemetry)
self.project_name = get_project_name(require=True)
+ # Sent with every create and push, so AMP knows which project a
+ # deployment runs — `crewai eval --models` finds the deployment by it.
+ # Read, never minted: a deploy does not rewrite pyproject.toml.
+ self.project_id = get_project_id()
+
+ def _project(self) -> dict[str, str]:
+ """`project_id=` for a create or push request, when the project has one."""
+ return {"project_id": self.project_id} if self.project_id else {}
def _standard_no_param_error_message(self) -> None:
"""
@@ -424,10 +436,10 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
env_vars = fetch_and_json_env_file()
return self._update_crew_from_zip(deployment_uuid, repository, env_vars)
if uuid:
- return self.plus_api_client.deploy_by_uuid(uuid)
+ return self.plus_api_client.deploy_by_uuid(uuid, **self._project())
if not project_name:
raise ValueError("project_name is required to deploy by name")
- return self.plus_api_client.deploy_by_name(project_name)
+ return self.plus_api_client.deploy_by_name(project_name, **self._project())
def _deploy_from_local_source(
self,
@@ -438,9 +450,9 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
) -> Any | None:
"""Deploy using local origin, as before AMP zip_deployment existed."""
if remote_repo_url and uuid:
- return self.plus_api_client.deploy_by_uuid(uuid)
+ return self.plus_api_client.deploy_by_uuid(uuid, **self._project())
if remote_repo_url and project_name:
- return self.plus_api_client.deploy_by_name(project_name)
+ return self.plus_api_client.deploy_by_name(project_name, **self._project())
if uuid:
_display_git_remote_help()
env_vars = fetch_and_json_env_file()
@@ -627,6 +639,7 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
zip_file_path,
name=self.project_name,
env=env_vars,
+ **self._project(),
)
finally:
zip_file_path.unlink(missing_ok=True)
@@ -649,6 +662,7 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
uuid,
zip_file_path,
env=env_vars,
+ **self._project(),
)
finally:
zip_file_path.unlink(missing_ok=True)
@@ -692,13 +706,16 @@ class DeployCommand(BaseCommand, PlusAPIMixin):
"""
if not self.project_name:
raise ValueError("project_name is required to create a deployment payload")
- return {
+ payload: CreateCrewPayload = {
"deploy": {
"name": self.project_name,
"repo_clone_url": remote_repo_url,
"env": env_vars,
}
}
+ if self.project_id:
+ payload["project_id"] = self.project_id
+ return payload
def _display_creation_success(self, json_response: dict[str, Any]) -> None:
"""
diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.py
index 1af0f69fc..4eb80b851 100644
--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py
+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py
@@ -10,6 +10,13 @@ Who may evaluate what is AMP's decision: an anonymous run once without an
account, then it needs one; a run traced while logged in for that
organization's members; a deployment execution for members who may see its
traces. The command sends the saved `crewai login` when there is one.
+
+`crewai eval --models "a,b"` compares models instead: AMP finds the project's
+deployment by `[tool.crewai].project_id` (or `--deployment`), which runs once as
+deployed and once per model; each run is graded, and the comparison — the four
+grades, cost and time per model, and what would make the agents do better — is
+printed when it is done. It always needs the login: it runs the deployment.
+Every evaluation, of either kind, is filed under the project id.
"""
from __future__ import annotations
@@ -23,8 +30,9 @@ import os
from pathlib import Path
import sys
import time
-from typing import Any
+from typing import Any, NoReturn, cast
from urllib.parse import urlparse
+import uuid
import webbrowser
import click
@@ -33,11 +41,16 @@ from crewai_core.settings import Settings
from dotenv import load_dotenv, set_key
import httpx
from rich.console import Console
+from rich.table import Table
from rich.text import Text
from crewai_cli.authentication.token import AuthError, get_auth_token
from crewai_cli.plus_api import PlusAPI
-from crewai_cli.utils import get_or_create_project_id, is_dmn_mode_enabled
+from crewai_cli.utils import (
+ get_or_create_project_id,
+ get_project_id,
+ is_dmn_mode_enabled,
+)
console = Console()
@@ -107,7 +120,7 @@ def _note(text: str, style: str = "dim") -> None:
def eval_crew(run_id: str | None = None) -> None:
"""Evaluate the last traced run of this project, or the run RUN_ID."""
- get_or_create_project_id()
+ project_id = get_or_create_project_id()
# Read before the project's .env is loaded, so a project cannot add itself.
trusted = _trusted_amp_origins()
_load_project_env()
@@ -140,6 +153,7 @@ def eval_crew(run_id: str | None = None) -> None:
client,
execution_id,
wait_for_spans=run_id is None and _ran_just_now(record),
+ project_id=project_id,
)
except EvaluationStoppedError as stopped:
_fail(str(stopped))
@@ -334,7 +348,13 @@ def evaluate_run(
for them is always on here.
"""
client = _amp_client(_machine_amp_origins(), note=note)
- started = _start_evaluation(client, execution_id, wait_for_spans=True, note=note)
+ started = _start_evaluation(
+ client,
+ execution_id,
+ wait_for_spans=True,
+ note=note,
+ project_id=get_project_id(),
+ )
# After the start, exactly as the command counts it: `cli_usage:eval` is the
# count of evaluations that began, and an evaluation the app runs is one.
_record_usage(logged_in=client.api_key is not None)
@@ -622,6 +642,7 @@ def _start_evaluation(
*,
wait_for_spans: bool = False,
note: Callable[[str], None] = _note,
+ project_id: str | None = None,
) -> dict[str, Any]:
deadline = time.monotonic() + SPANS_WAIT_SECONDS if wait_for_spans else 0.0
said = False
@@ -630,7 +651,9 @@ def _start_evaluation(
eval_config = project_eval_config(note=note)
while True:
try:
- response = client.create_evaluation(execution_id, eval_config=eval_config)
+ response = client.create_evaluation(
+ execution_id, eval_config=eval_config, project_id=project_id
+ )
except httpx.HTTPError as error:
raise EvaluationStoppedError(
f"Could not reach AMP to start the evaluation: {error}"
@@ -647,6 +670,17 @@ def _start_evaluation(
time.sleep(SPANS_POLL_SECONDS)
continue
break
+ return _accepted(response, f"run {execution_id}", note=note)
+
+
+def _accepted(
+ response: httpx.Response,
+ subject: str,
+ *,
+ note: Callable[[str], None] = _note,
+ about_a_deployment: bool = False,
+) -> dict[str, Any]:
+ """AMP's answer to a start: the evaluation it opened, or its refusal as a sentence."""
if response.status_code in (200, 202):
payload = _payload(response)
# The id is what every later call is made with, so a missing or
@@ -666,7 +700,9 @@ def _start_evaluation(
raise EvaluationStoppedError(
f"AMP answered without an evaluation id ({response.status_code})."
)
- raise EvaluationStoppedError(_refusal_message(response, f"run {execution_id}"))
+ raise EvaluationStoppedError(
+ _refusal_message(response, subject, about_a_deployment=about_a_deployment)
+ )
def _report_url(value: Any) -> str | None:
@@ -705,12 +741,16 @@ def _wait(
url: str | None,
*,
on_status: Callable[[dict[str, Any]], None] | None = None,
+ answer: str = "verdict",
) -> dict[str, Any]:
"""Poll until the evaluation is done or failed.
Every unfinished answer goes to ON_STATUS, so a caller that has somewhere to
- show progress can show it; the caller decides what a Ctrl-C means.
+ show progress can show it; the caller decides what a Ctrl-C means. A done
+ answer must carry a well-formed ANSWER: a run's `verdict`, or a models
+ evaluation's `comparison`.
"""
+ well_formed = _WELL_FORMED[answer]
where = f" at {url}" if url else ""
subject = f"evaluation {evaluation_id}"
misses = (
@@ -738,9 +778,9 @@ def _wait(
misses = 0
payload = _payload(response) or {}
status = payload.get("status")
- if status == "done" and not _well_formed_verdict(payload.get("verdict")):
+ if status == "done" and not well_formed(payload):
raise EvaluationStoppedError(
- f"AMP answered done without a verdict (protocol error); follow it{where or ' on AMP'}."
+ f"AMP answered done without a {answer} (protocol error); follow it{where or ' on AMP'}."
)
if status in FINISHED:
return payload
@@ -753,6 +793,10 @@ def _wait(
time.sleep(POLL_SECONDS)
+def _carries_a_verdict(payload: dict[str, Any]) -> bool:
+ return _well_formed_verdict(payload.get("verdict"))
+
+
def _well_formed_verdict(verdict: Any) -> bool:
"""`{"gate": "<word>", "grades": {area: 1..5 | null}}` — anything else is a
protocol error. A grade is an exact integer in range: `True` is an `int` to
@@ -769,6 +813,403 @@ def _a_grade(grade: Any) -> bool:
return grade is None or (type(grade) is int and 1 <= grade <= 5)
+def _carries_a_comparison(payload: dict[str, Any]) -> bool:
+ """`{"comparison": {"models": [{...}, ...]}}` — the rows are read cell by
+ cell when printed, and a cell that is not what it should be prints as "—";
+ a comparison with no rows to print is a protocol error."""
+ comparison = payload.get("comparison")
+ return (
+ isinstance(comparison, dict)
+ and isinstance(comparison.get("models"), list)
+ and bool(comparison["models"])
+ and all(isinstance(row, dict) for row in comparison["models"])
+ )
+
+
+_WELL_FORMED: dict[str, Callable[[dict[str, Any]], bool]] = {
+ "verdict": _carries_a_verdict,
+ "comparison": _carries_a_comparison,
+}
+
+
+# ── crewai eval --models ─────────────────────────────────────────────────────
+
+MAX_MODELS = 5
+MAX_MODEL_CHARS = 200
+MODELS_EXAMPLE = '--models "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"'
+LOGIN_REQUIRED = (
+ "Comparing models runs your deployment, so it needs your CrewAI AMP account: "
+ "log in with `crewai login` and run it again."
+)
+GRADE_COLUMNS = ("goal", "tasks", "agents", "tools")
+TOP_SUGGESTIONS = 3
+
+
+def parse_models(text: str | None) -> list[str]:
+ """The models to compare, from ONE comma-separated list.
+
+ Items are stripped and a repeat is dropped. Each must name its provider
+ (`provider/model`) — the deployment builds the model from exactly that
+ string, and a bare `gpt-4o` would be a guess about which provider bills it.
+ """
+ models: list[str] = []
+ for raw in (text or "").split(","):
+ item = raw.strip()
+ if not item or item in models:
+ continue
+ provider, separator, name = item.partition("/")
+ if not (separator and provider.strip() and name.strip()):
+ raise EvaluationStoppedError(
+ f"{item!r} names no provider: write each model as provider/model, "
+ f"e.g. {MODELS_EXAMPLE}."
+ )
+ if len(item) > MAX_MODEL_CHARS:
+ raise EvaluationStoppedError(
+ f"{item[:40]!r}… is longer than {MAX_MODEL_CHARS} characters; "
+ "that is not a model name."
+ )
+ models.append(item)
+ if not models:
+ raise EvaluationStoppedError(
+ f"--models names no model: give one to {MAX_MODELS}, e.g. {MODELS_EXAMPLE}."
+ )
+ if len(models) > MAX_MODELS:
+ raise EvaluationStoppedError(
+ f"--models names {len(models)} models; compare at most {MAX_MODELS} at once."
+ )
+ return models
+
+
+def _deployment_id(value: str | None) -> str | None:
+ """`--deployment` as the UUID AMP knows the deployment by, or None."""
+ if value is None:
+ return None
+ try:
+ return str(uuid.UUID(value.strip()))
+ except ValueError:
+ raise EvaluationStoppedError(
+ f"--deployment {value!r} is not a deployment id; it is the UUID AMP "
+ "shows for the deployment."
+ ) from None
+
+
+# Model names that are the same for everyone who types them are public. What
+# else a `--models` item can carry is a customer's: a fine-tune id
+# (`openai/ft:gpt-4o-mini:acme-corp::abc`), an Azure deployment name, a
+# self-hosted model or host. Those are sent as `<provider>/other`.
+OTHER_MODEL = "other"
+# The vendor segment of an aggregator id (`openrouter/openai/gpt-4o-mini`)
+# that is itself public.
+_PUBLIC_VENDORS = frozenset(
+ {"openai", "anthropic", "google", "meta-llama", "mistralai", "deepseek", "qwen"}
+)
+
+
+def telemetry_model_name(model: str) -> str:
+ """MODEL as the usage stats may carry it: as typed when it is a model crewAI
+ knows, else `<provider>/other` — and `other/other` for a provider crewAI does
+ not know, since a provider string can name a host too.
+
+ Known means an exact entry of crewAI's model catalog (the context-window
+ tables every provider resolves against), never a prefix of one: a fine-tune
+ or a deployment named after a public model is still the customer's.
+ """
+ catalog, providers = _known_models_and_providers()
+ provider, _, name = model.partition("/")
+ if provider not in providers:
+ return f"{OTHER_MODEL}/{OTHER_MODEL}"
+ vendor, nested, tail = name.partition("/")
+ public = (
+ name in catalog
+ or model in catalog
+ or (nested and vendor in _PUBLIC_VENDORS and tail in catalog)
+ )
+ if not public or any(part.startswith("ft:") for part in model.split("/")):
+ return f"{provider}/{OTHER_MODEL}"
+ return model
+
+
+def _known_models_and_providers() -> tuple[frozenset[str], frozenset[str]]:
+ """crewAI's catalog of models and the providers it routes; empty when this
+ environment's crewai cannot say, so every model is then sent as "other"."""
+ from crewai_cli.constants import PROVIDERS
+
+ try:
+ from crewai.llm import SUPPORTED_NATIVE_PROVIDERS
+ from crewai.llms.context_window import LLM_CONTEXT_WINDOW_SIZES
+ except Exception:
+ return frozenset(), frozenset()
+ return (
+ frozenset(LLM_CONTEXT_WINDOW_SIZES),
+ frozenset(SUPPORTED_NATIVE_PROVIDERS) | frozenset(PROVIDERS),
+ )
+
+
+def _record_models_usage(models: list[str]) -> None:
+ """Count a comparison that is actually starting, and which models it compares.
+
+ The models go through `telemetry_model_name`: a public model by name, any
+ other as `<provider>/other`. Nothing names the run, the deployment or the
+ organization. Always logged in: a comparison cannot start without the
+ account.
+ """
+ try:
+ from crewai_core.telemetry import Telemetry
+
+ telemetry = Telemetry()
+ telemetry.set_tracer()
+ telemetry.feature_usage_span(
+ "cli_usage:eval_models",
+ {
+ "authenticated": "true",
+ "models": ",".join(telemetry_model_name(m) for m in models),
+ "models_count": str(len(models)),
+ },
+ )
+ except Exception: # noqa: S110 - telemetry must never break a command
+ pass
+
+
+def eval_models(models_text: str, deployment_id: str | None = None) -> None:
+ """Run this project's deployment once as deployed and once per model, and compare.
+
+ The deployment is AMP's to find, by the project id, unless DEPLOYMENT_ID
+ names one; its own models are the baseline, read off the deployment rather
+ than this checkout, which may differ from what was deployed.
+ """
+ try:
+ models = parse_models(models_text)
+ deployment = _deployment_id(deployment_id)
+ except EvaluationStoppedError as stopped:
+ _fail(str(stopped))
+ if not Path("pyproject.toml").is_file():
+ _fail(
+ "No crewAI project here (no pyproject.toml). Run `crewai eval --models` "
+ "from the directory of the project you deployed."
+ )
+ project_id = get_or_create_project_id()
+ if not project_id:
+ _fail(
+ "Could not read or write [tool.crewai].project_id in pyproject.toml, which "
+ "is how AMP finds this project's deployment."
+ )
+ # Read before the project's .env is loaded, so a project cannot add itself.
+ trusted = _trusted_amp_origins()
+ _load_project_env()
+ try:
+ client = _amp_client(trusted)
+ except EvaluationStoppedError as stopped:
+ _fail(str(stopped))
+ if client.api_key is None:
+ _fail(LOGIN_REQUIRED)
+
+ try:
+ response = client.create_models_evaluation(
+ models,
+ project_id=project_id,
+ eval_config=project_eval_config(),
+ deployment_id=deployment,
+ )
+ except httpx.HTTPError as error:
+ _fail(f"Could not reach AMP to start the comparison: {error}")
+ try:
+ started = _accepted(response, "the comparison", about_a_deployment=True)
+ except EvaluationStoppedError as stopped:
+ _fail(str(stopped))
+ # After, not before, as `cli_usage:eval` is counted: a refused request —
+ # no deployment, one this account may not run — is not a comparison.
+ _record_models_usage(models)
+ url = started.get("url")
+ console.print(
+ Text("Comparing ")
+ .append(", ".join(models), style="bold")
+ .append(" with the deployed models")
+ )
+ if url:
+ # Appended, never interpolated — the same reason as `eval_crew`'s link.
+ console.print(Text("Follow it at ").append(url, style="cyan underline"))
+ _open(url)
+
+ console.print("Waiting for the comparison…", style="dim")
+ shown: list[str] = []
+
+ def show_progress(payload: dict[str, Any]) -> None:
+ line = _progress_line(payload)
+ if line and (not shown or shown[-1] != line):
+ shown.append(line)
+ _note(line)
+
+ try:
+ finished = _wait(
+ client, started["id"], url, on_status=show_progress, answer="comparison"
+ )
+ except EvaluationStoppedError as stopped:
+ _fail(str(stopped))
+ except KeyboardInterrupt:
+ console.print(
+ Text(f"\nStill running{f' at {url}' if url else ''}."), style="yellow"
+ )
+ raise SystemExit(130) from None
+ _print_comparison(finished, url)
+ # The criteria the comparison was graded on, for the project to edit — only
+ # when it has none, exactly as after a Mode 1 evaluation.
+ _say_where_the_criteria_live(write_eval_config(finished))
+ if finished.get("status") != "done":
+ raise SystemExit(1)
+
+
+def _progress_line(payload: dict[str, Any]) -> str | None:
+ """One line for where the comparison is, from the poll's `progress` (or the
+ last of its `events`): which model is running, "model 2 of 3", and the
+ subject being judged. What is not there is left out, never guessed."""
+ progress = payload.get("progress")
+ events = payload.get("events")
+ if progress is None and isinstance(events, list) and events:
+ progress = events[-1]
+ if isinstance(progress, str):
+ return progress.strip() or None
+ if not isinstance(progress, dict):
+ return None
+ if isinstance(progress.get("message"), str) and progress["message"].strip():
+ return str(progress["message"]).strip()
+ event = progress.get("event")
+ detail = progress.get("payload")
+ if not isinstance(detail, dict):
+ detail = progress
+ index, total = detail.get("index"), detail.get("total")
+ position = (
+ f"model {index + 1} of {total}"
+ if type(index) is int and type(total) is int and 0 <= index < total
+ else None
+ )
+ name = next(
+ (
+ str(detail[key])
+ for key in ("label", "key")
+ if isinstance(detail.get(key), str) and detail[key]
+ ),
+ None,
+ )
+ subject = detail.get("subject")
+ if event == "judging" or isinstance(subject, str):
+ parts = [f"judging {subject}" if isinstance(subject, str) else "judging"]
+ elif event == "configuration_done":
+ parts = ["graded"]
+ else:
+ parts = ["running"]
+ parts += [part for part in (position, name) if part]
+ known = event is not None or isinstance(subject, str) or len(parts) > 1
+ return " · ".join(parts) if known else None
+
+
+def _print_comparison(finished: dict[str, Any], url: str | None) -> None:
+ """The models side by side, then what would make them better.
+
+ Every cell came over the wire, so each is a `Text`: a label such as
+ `Writer: [red]x[/red]` prints as written. The baseline — the deployment as
+ it is — is marked, and the best value in each column is starred: the highest
+ grade, the lowest cost and time. A column where every model is the same, or
+ only one has a value, stars nothing: there is no "best" to point at.
+ """
+ if finished.get("status") != "done":
+ console.print(
+ Text(f"Comparison failed: {finished.get('error') or 'no reason given'}"),
+ style="bold red",
+ )
+ if url:
+ console.print(Text(f"Report: {url}"))
+ return
+ comparison = finished["comparison"] # _wait let only a well-formed one through
+ rows: list[dict[str, Any]] = comparison["models"]
+
+ grades = {area: [_grade_of(row, area) for row in rows] for area in GRADE_COLUMNS}
+ costs = [_number(row.get("cost_usd")) for row in rows]
+ seconds = [_number(row.get("seconds")) for row in rows]
+ best = {area: _best(values, max) for area, values in grades.items()}
+ cheapest, fastest = _best(costs, min), _best(seconds, min)
+
+ table = Table(show_edge=False, pad_edge=False)
+ for header in ("model", *GRADE_COLUMNS, "cost", "time"):
+ table.add_column(header, justify="left" if header == "model" else "right")
+ for n, row in enumerate(rows):
+ label = Text(str(row.get("label") or row.get("key") or "?"))
+ if row.get("baseline") is True:
+ label.append(" (deployed)", style="dim")
+ cells = [label]
+ for area in GRADE_COLUMNS:
+ grade = grades[area][n]
+ cells.append(
+ _starred(f"{grade}/5" if grade is not None else "—", grade, best[area])
+ )
+ cost, took = costs[n], seconds[n]
+ cells.append(
+ _starred(f"${cost:.4f}" if cost is not None else "—", cost, cheapest)
+ )
+ cells.append(
+ _starred(f"{took:.1f}s" if took is not None else "—", took, fastest)
+ )
+ table.add_row(*cells)
+ console.print(table)
+
+ suggestions = _top_suggestions(comparison.get("suggestions"))
+ if suggestions:
+ console.print(Text("What would make it better", style="bold"))
+ for n, item in enumerate(suggestions, 1):
+ where = " — ".join(
+ str(item[key])
+ for key in ("subject", "field")
+ if isinstance(item.get(key), str)
+ )
+ line = Text(f"{n}. ")
+ if where:
+ line.append(where, style="bold").append(": ")
+ line.append(str(item.get("problem") or ""))
+ if item.get("shared") is True:
+ line.append(" (every model)", style="dim")
+ console.print(line)
+ if isinstance(item.get("change"), str) and item["change"].strip():
+ console.print(Text(f" change: {item['change'].strip()}"))
+ if url:
+ console.print(Text(f"Full report: {url}"))
+
+
+def _grade_of(row: dict[str, Any], area: str) -> int | None:
+ grades = row.get("grades")
+ grade = grades.get(area) if isinstance(grades, dict) else None
+ return grade if grade is not None and _a_grade(grade) else None
+
+
+def _number(value: Any) -> float | None:
+ """A non-negative number, or None: `True` is an `int` to Python and not a cost."""
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return None
+ return float(value) if value >= 0 else None
+
+
+def _best(values: list[Any], pick: Callable[[list[Any]], Any]) -> float | int | None:
+ known = [value for value in values if value is not None]
+ if len(known) < 2 or len(set(known)) == 1:
+ return None
+ return cast(float | int, pick(known))
+
+
+def _starred(text: str, value: Any, best: Any) -> Text:
+ cell = Text(text)
+ if best is not None and value == best:
+ cell.append(" ★", style="yellow")
+ return cell
+
+
+def _top_suggestions(value: Any) -> list[dict[str, Any]]:
+ """The first few suggestions, the ones every model needed first — a problem
+ every model had points at the prompt, not at a model."""
+ if not isinstance(value, list):
+ return []
+ items = [item for item in value if isinstance(item, dict) and item.get("problem")]
+ items.sort(key=lambda item: item.get("shared") is not True)
+ return items[:TOP_SUGGESTIONS]
+
+
def _print_verdict(finished: dict[str, Any], url: str | None) -> None:
"""Everything printed here came over the wire, so it is composed as `Text`
and never as markup: a `Console` parses square brackets, and an area named
@@ -817,15 +1258,26 @@ def _payload(response: httpx.Response) -> dict[str, Any] | None:
return loaded if isinstance(loaded, dict) else None
-def _refusal_message(response: httpx.Response, subject: str) -> str:
+def _refusal_message(
+ response: httpx.Response, subject: str, *, about_a_deployment: bool = False
+) -> str:
"""AMP's own words when it sent them. SUBJECT is "run <id>" or "evaluation <id>".
A sentence, not a print: the terminal and the run app both show it, and only
- one of them shows it by printing.
+ one of them shows it by printing. ABOUT_A_DEPLOYMENT: a 403 that is not about
+ the credential — this account may not run that deployment — is AMP's
+ sentence alone, because logging in again changes nothing there.
"""
payload = _payload(response) or {}
message = str(payload.get("message") or "").strip()
error = str(payload.get("error") or "")
+ if (
+ about_a_deployment
+ and response.status_code == 403
+ and error not in {"bad_credentials", "account_required"}
+ and message
+ ):
+ return message
if response.status_code in (401, 403):
if error == "account_required" and message:
return message
@@ -843,7 +1295,7 @@ def _refused(response: httpx.Response, subject: str) -> None:
_fail(_refusal_message(response, subject))
-def _fail(message: str) -> None:
+def _fail(message: str) -> NoReturn:
# `Text`, because most of what reaches here is AMP's own sentence and a
# `Console` parses square brackets. No caller relies on markup; the colour
# comes from `style`.
diff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.py
index 0256f7c30..b9b2f1d28 100644
--- a/lib/cli/src/crewai_cli/plus_api.py
+++ b/lib/cli/src/crewai_cli/plus_api.py
@@ -28,17 +28,56 @@ class PlusAPI(_CorePlusAPI):
EVALUATION_POLL_TIMEOUT = 30.0
def create_evaluation(
- self, execution_id: str, *, eval_config: str | None = None
+ self,
+ execution_id: str,
+ *,
+ eval_config: str | None = None,
+ project_id: str | None = None,
) -> httpx.Response:
"""Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval).
EVAL_CONFIG is the project's own `eval.jsonc` when it has one: what
good means for this crew, in its own words. Sent as it was written,
- comments and all, and read by the grader rather than here.
+ comments and all, and read by the grader rather than here. PROJECT_ID
+ is `[tool.crewai].project_id`, so the evaluation is filed under the
+ project it is about.
"""
body: dict[str, str] = {"execution_id": execution_id}
if eval_config:
body["eval_config"] = eval_config
+ if project_id:
+ body["project_id"] = project_id
+ return self._make_request(
+ "POST",
+ self.EVALUATIONS_RESOURCE,
+ json=body,
+ timeout=self.EVALUATION_START_TIMEOUT,
+ )
+
+ def create_models_evaluation(
+ self,
+ models: list[str],
+ *,
+ project_id: str,
+ eval_config: str | None = None,
+ deployment_id: str | None = None,
+ ) -> httpx.Response:
+ """Ask AMP to run the project's deployment once per model and compare them.
+
+ AMP finds the deployment by PROJECT_ID (`[tool.crewai].project_id`)
+ unless DEPLOYMENT_ID names one. MODELS are `provider/model` strings;
+ the deployment's own models are the baseline, so they are not sent.
+ EVAL_CONFIG is the project's `eval.jsonc`, as `create_evaluation` sends it.
+ """
+ body: dict[str, Any] = {
+ "kind": "models",
+ "project_id": project_id,
+ "models": list(models),
+ }
+ if eval_config:
+ body["eval_config"] = eval_config
+ if deployment_id:
+ body["deployment_id"] = deployment_id
return self._make_request(
"POST",
self.EVALUATIONS_RESOURCE,
@@ -90,11 +129,18 @@ class PlusAPI(_CorePlusAPI):
*,
name: str | None = None,
env: dict[str, str] | None = None,
+ project_id: str | None = None,
) -> httpx.Response:
- """Create a crew deployment from a local project ZIP archive."""
+ """Create a crew deployment from a local project ZIP archive.
+
+ PROJECT_ID is `[tool.crewai].project_id`, so AMP knows which project the
+ deployment runs; sent only when the project has one.
+ """
data: dict[str, str] = {}
if name:
data["name"] = name
+ if project_id:
+ data["project_id"] = project_id
if env:
data.update({f"env[{key}]": value for key, value in env.items()})
return self._make_multipart_request(
@@ -111,9 +157,15 @@ class PlusAPI(_CorePlusAPI):
zip_file_path: str | Path,
*,
env: dict[str, str] | None = None,
+ project_id: str | None = None,
) -> httpx.Response:
- """Update an existing crew deployment from a local project ZIP archive."""
+ """Update an existing crew deployment from a local project ZIP archive.
+
+ PROJECT_ID as for `create_crew_from_zip`.
+ """
data: dict[str, str] = {}
+ if project_id:
+ data["project_id"] = project_id
if env:
data.update({f"env[{key}]": value for key, value in env.items()})
return self._make_multipart_request(
diff --git a/lib/cli/tests/deploy/test_deploy_main.py b/lib/cli/tests/deploy/test_deploy_main.py
index 08b95a6c5..a815cca9b 100644
--- a/lib/cli/tests/deploy/test_deploy_main.py
+++ b/lib/cli/tests/deploy/test_deploy_main.py
@@ -1316,3 +1316,128 @@ class TestDeployCommand(unittest.TestCase):
from crewai_cli.version import get_crewai_version
assert isinstance(get_crewai_version(), str)
+
+
+PROJECT_ID = "1f0c6a3e-8d0b-4c4e-9a52-6a9f6d1d2b7e"
+
+
+class TestDeploySendsTheProjectId(unittest.TestCase):
+ """Every create and push carries `[tool.crewai].project_id` when the
+ project has one, so AMP knows which project a deployment runs."""
+
+ @patch("crewai_cli.command.get_auth_token", return_value="test_token")
+ @patch("crewai_cli.deploy.main.get_project_id", return_value=PROJECT_ID)
+ @patch("crewai_cli.deploy.main.get_project_name", return_value="test_project")
+ @patch("crewai_cli.command.PlusAPI")
+ def setUp(self, _plus_api, _name, _project_id, _token):
+ self.deploy_command = deploy_main.DeployCommand()
+ self.mock_client = self.deploy_command.plus_api_client
+ self.mock_client.base_url = "https://app.crewai.com"
+ browser = patch("crewai_cli.deploy.main.webbrowser.open")
+ browser.start()
+ self.addCleanup(browser.stop)
+
+ def _created(self, uuid: str = "new-uuid") -> MagicMock:
+ response = MagicMock()
+ response.status_code = 201
+ response.is_success = True
+ response.json.return_value = {"uuid": uuid, "status": "created"}
+ return response
+
+ def _status(self, zip_deployment: bool) -> MagicMock:
+ response = MagicMock()
+ response.is_success = True
+ response.json.return_value = {"uuid": "test-uuid", "zip_deployment": zip_deployment}
+ return response
+
+ def test_the_command_reads_the_project_id(self):
+ self.assertEqual(self.deploy_command.project_id, PROJECT_ID)
+
+ @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})
+ @patch("crewai_cli.deploy.main.git.Repository")
+ def test_create_from_git_sends_it(self, mock_repository, _env):
+ mock_repository.return_value.origin_url.return_value = "https://github.com/t/r.git"
+ mock_repository.return_value.create_initial_commit_if_needed.return_value = False
+ self.mock_client.create_crew.return_value = self._created()
+
+ with patch("sys.stdout", new=StringIO()):
+ self.deploy_command.create_crew(confirm=True, skip_validate=True)
+
+ payload = self.mock_client.create_crew.call_args.args[0]
+ # Once, at the top level, where AMP reads it on every deploy route.
+ self.assertEqual(payload["project_id"], PROJECT_ID)
+ self.assertNotIn("project_id", payload["deploy"])
+
+ @patch("crewai_cli.deploy.main.create_project_zip", return_value=Path("/tmp/p.zip"))
+ @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})
+ @patch("crewai_cli.deploy.main.git.Repository")
+ def test_create_from_a_zip_sends_it(self, mock_repository, _env, _zip):
+ mock_repository.return_value.origin_url.return_value = None
+ mock_repository.return_value.create_initial_commit_if_needed.return_value = False
+ self.mock_client.create_crew_from_zip.return_value = self._created()
+
+ with patch("sys.stdout", new=StringIO()):
+ self.deploy_command.create_crew(confirm=True, skip_validate=True)
+
+ self.mock_client.create_crew_from_zip.assert_called_once_with(
+ Path("/tmp/p.zip"), name="test_project", env={}, project_id=PROJECT_ID
+ )
+
+ @patch("crewai_cli.deploy.main.git.Repository")
+ @patch("crewai_cli.deploy.main.DeployCommand._display_deployment_info")
+ def test_a_push_from_git_sends_it(self, _display, mock_repository):
+ mock_repository.return_value.origin_url.return_value = "https://github.com/t/r.git"
+ mock_repository.return_value.create_initial_commit_if_needed.return_value = False
+ self.mock_client.crew_status_by_uuid.return_value = self._status(False)
+ self.mock_client.crew_status_by_name.return_value = self._status(False)
+
+ self.deploy_command.deploy(uuid="test-uuid", skip_validate=True)
+ self.deploy_command.deploy(skip_validate=True)
+
+ self.mock_client.deploy_by_uuid.assert_called_once_with(
+ "test-uuid", project_id=PROJECT_ID
+ )
+ self.mock_client.deploy_by_name.assert_called_once_with(
+ "test_project", project_id=PROJECT_ID
+ )
+
+ @patch("crewai_cli.deploy.main.create_project_zip", return_value=Path("/tmp/p.zip"))
+ @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})
+ @patch("crewai_cli.deploy.main.git.Repository")
+ @patch("crewai_cli.deploy.main.DeployCommand._display_deployment_info")
+ def test_a_push_from_a_zip_sends_it(self, _display, mock_repository, _env, _zip):
+ mock_repository.return_value.origin_url.return_value = None
+ mock_repository.return_value.create_initial_commit_if_needed.return_value = False
+ self.mock_client.crew_status_by_uuid.return_value = self._status(True)
+
+ self.deploy_command.deploy(uuid="test-uuid", skip_validate=True)
+
+ self.mock_client.update_crew_from_zip.assert_called_once_with(
+ "test-uuid", Path("/tmp/p.zip"), env={}, project_id=PROJECT_ID
+ )
+
+
+def test_a_project_without_an_id_sends_none(monkeypatch, tmp_path: Path):
+ """Read, never minted: a deploy does not rewrite pyproject.toml."""
+ monkeypatch.chdir(tmp_path)
+ (tmp_path / "pyproject.toml").write_text("[project]\nname = 'demo'\n")
+ monkeypatch.setattr("crewai_cli.command.get_auth_token", lambda: "t")
+ monkeypatch.setattr("crewai_cli.command.PlusAPI", MagicMock())
+ monkeypatch.setattr(deploy_main, "get_project_name", lambda require=False: "demo")
+
+ command = deploy_main.DeployCommand()
+
+ assert command.project_id is None and command._project() == {}
+ assert "project_id" not in (tmp_path / "pyproject.toml").read_text()
+
+
+def test_the_project_id_is_read_from_pyproject(monkeypatch, tmp_path: Path):
+ monkeypatch.chdir(tmp_path)
+ (tmp_path / "pyproject.toml").write_text(
+ f"[project]\nname = 'demo'\n\n[tool.crewai]\nproject_id = \"{PROJECT_ID}\"\n"
+ )
+ monkeypatch.setattr("crewai_cli.command.get_auth_token", lambda: "t")
+ monkeypatch.setattr("crewai_cli.command.PlusAPI", MagicMock())
+ monkeypatch.setattr(deploy_main, "get_project_name", lambda require=False: "demo")
+
+ assert deploy_main.DeployCommand()._project() == {"project_id": PROJECT_ID}
diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.py
index 5873d37e0..8bcc5d594 100644
--- a/lib/cli/tests/experimental/test_eval_crew.py
+++ b/lib/cli/tests/experimental/test_eval_crew.py
@@ -44,9 +44,10 @@ class FakeAMP:
self.api_key = None
self.headers: dict[str, str] = {"X-Crewai-Organization-Id": "org-42"}
- def create_evaluation(self, execution_id, *, eval_config=None):
+ def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):
self.calls.append(("create", execution_id))
self.sent_config = eval_config
+ self.sent_project_id = project_id
if isinstance(self.create, list):
return self.create.pop(0) if len(self.create) > 1 else self.create[0]
return self.create
@@ -679,7 +680,7 @@ def test_an_oversized_config_is_said_once_and_through_the_callers_note(
answers = iter([httpx.Response(404, json={}), httpx.Response(404, json={}), QUEUED])
class Client:
- def create_evaluation(self, execution_id, *, eval_config=None):
+ def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):
sent.append(eval_config)
return next(answers)
@@ -744,7 +745,7 @@ def test_only_a_web_address_is_printed_and_opened(url, opened):
def test_a_report_url_that_is_not_a_web_address_is_dropped_with_a_note(monkeypatch):
class Client:
- def create_evaluation(self, execution_id, *, eval_config=None):
+ def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):
return httpx.Response(
202, json={"id": "ev-1", "url": "file:///etc/passwd", "status": "queued"}
)
@@ -1207,3 +1208,395 @@ def test_an_unreadable_login_is_the_reason_given_when_nothing_was_traced(
assert "[/Users/me/.config/crewai]" in out
assert "add CREWAI_TRACING_ENABLED=true" not in out
+
+# ── Mode 1 files the evaluation under the project ───────────────────────────
+
+
+def test_the_evaluation_is_filed_under_the_project(project, monkeypatch):
+ directory, _ = project
+ record_last_run(directory)
+ monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: "proj-1")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.eval_crew()
+
+ assert amp.sent_project_id == "proj-1"
+
+
+def test_the_app_files_its_evaluation_under_the_project_too(project, monkeypatch):
+ """The run app's door reads the id without minting one: it runs inside a
+ crew's process, where rewriting pyproject.toml is not its business."""
+ monkeypatch.setattr(eval_module, "get_project_id", lambda: "proj-2")
+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))
+
+ eval_module.evaluate_run(EXECUTION_ID, on_started=lambda started: None)
+
+ assert amp.sent_project_id == "proj-2"
+
+
+# ── crewai eval --models ─────────────────────────────────────────────────────
+
+MODELS = "openai/gpt-4o-mini, openrouter/meta-llama/llama-4-maverick"
+DEPLOYMENT = "9b0a4a53-3f7c-4a8e-9a55-2f1c1e0f8e11"
+
+
+def comparison(**overrides):
+ body = {
+ "models": [
+ {
+ "key": "base", "label": "Poem composer: openai/gpt-5.6-sol", "baseline": True,
+ "grades": {"goal": 5, "tasks": 4, "agents": 3, "tools": None},
+ "gate": "passed", "cost_usd": 0.0040, "seconds": 9.5, "tokens": 900,
+ },
+ {
+ "key": "mini", "label": "Poem composer: openai/gpt-4o-mini", "baseline": False,
+ "grades": {"goal": 5, "tasks": 5, "agents": 2, "tools": None},
+ "gate": "passed", "cost_usd": 0.0012, "seconds": 7.6, "tokens": 736,
+ },
+ ],
+ "suggestions": [
+ {"subject": "agent:Poem composer", "field": "backstory", "problem": "only one model had it",
+ "evidence": "x", "change": "only-one change", "models_affected": ["mini"], "shared": False},
+ {"subject": "agent:Poem composer", "field": "goal", "problem": "the poem ignores the topic",
+ "evidence": "y", "change": "Write a poem about {topic}.", "models_affected": ["base", "mini"],
+ "shared": True},
+ {"subject": "task:write", "field": "expected_output", "problem": "no length", "change": "4 lines"},
+ {"subject": "agent:Poem composer", "field": "tools", "problem": "a fourth", "change": "never shown"},
+ ],
+ }
+ body.update(overrides)
+ return body
+
+
+def compared(**overrides):
+ return httpx.Response(200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": comparison(**overrides)})
+
+
+class FakeModelsAMP(FakeAMP):
+ def __init__(self, create=None, statuses=None):
+ super().__init__(create=create if create is not None else httpx.Response(
+ 202, json={"id": "ev-9", "url": URL, "status": "queued"}
+ ), statuses=statuses)
+ self.sent: dict | None = None
+
+ def create_models_evaluation(self, models, *, project_id, eval_config=None, deployment_id=None):
+ self.calls.append(("create_models", tuple(models)))
+ self.sent = {"models": models, "project_id": project_id, "eval_config": eval_config,
+ "deployment_id": deployment_id}
+ return self.create
+
+
+def deployed(project, monkeypatch):
+ directory, opened = project
+ (directory / "pyproject.toml").write_text('[tool.crewai]\nproject_id = "proj-1"\n')
+ monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: "proj-1")
+ return directory, opened
+
+
+def test_models_are_compared_on_the_deployment_and_the_table_printed(deployed, monkeypatch, capsys):
+ directory, opened = deployed
+ (directory / "eval.jsonc").write_text('{"dataset": []}')
+ running = httpx.Response(200, json={"id": "ev-9", "status": "running", "progress": {
+ "event": "configuration_started", "payload": {"index": 1, "total": 3, "key": "mini"}}})
+ judging = httpx.Response(200, json={"id": "ev-9", "status": "running", "progress": {
+ "event": "judging", "payload": {"subject": "agent 'Poem composer'", "index": 1, "total": 3}}})
+ amp = install(monkeypatch, FakeModelsAMP(statuses=[running, running, judging, compared()]))
+
+ eval_module.eval_models(MODELS)
+
+ out = capsys.readouterr().out
+ assert amp.api_key == "login-token"
+ assert amp.sent == {
+ "models": ["openai/gpt-4o-mini", "openrouter/meta-llama/llama-4-maverick"],
+ "project_id": "proj-1", "eval_config": '{"dataset": []}', "deployment_id": None,
+ }
+ assert opened == [URL] and URL in out
+ # Progress: said once per change, never once per poll.
+ assert out.count("running · model 2 of 3 · mini") == 1
+ assert "judging agent 'Poem composer' · model 2 of 3" in out
+ lines = out.splitlines()
+ base = next(line for line in lines if "gpt-5.6-sol" in line)
+ mini = next(line for line in lines if "Poem composer: openai/gpt-4o-mini" in line)
+ assert "(deployed)" in base and "(deployed)" not in mini
+ # The best per column: tasks and agents differ, goal is a tie, tools nothing.
+ assert "5/5 ★" in mini and "4/5" in base and "4/5 ★" not in base
+ assert "3/5 ★" in base and "2/5 ★" not in mini
+ assert "$0.0012 ★" in mini and "7.6s ★" in mini and "9.5s ★" not in base
+ assert "5/5 ★" not in base # goal: both 5, nothing to point at
+ # The top three suggestions, the one every model needed first.
+ assert "What would make it better" in out
+ assert out.index("the poem ignores the topic") < out.index("only one model had it")
+ assert "change: Write a poem about {topic}." in out
+ assert "a fourth" not in out
+
+
+def test_the_comparison_is_counted_with_the_models_and_nothing_that_names_it(deployed, monkeypatch):
+ spans: list[tuple[str, dict[str, str]]] = []
+
+ class FakeTelemetry:
+ def set_tracer(self) -> None:
+ pass
+
+ def feature_usage_span(self, feature, attributes=None) -> None:
+ spans.append((feature, attributes or {}))
+
+ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)
+ install(monkeypatch, FakeModelsAMP(statuses=[compared()]))
+
+ eval_module.eval_models(MODELS)
+
+ assert spans == [("cli_usage:eval_models", {
+ "authenticated": "true",
+ # A public model by name; one crewAI's catalog does not list, by provider.
+ "models": "openai/gpt-4o-mini,openrouter/other",
+ "models_count": "2",
+ })]
+
+
+def test_a_refused_comparison_is_not_counted(deployed, monkeypatch, capsys):
+ spans: list[str] = []
+
+ class FakeTelemetry:
+ def set_tracer(self) -> None:
+ pass
+
+ def feature_usage_span(self, feature, attributes=None) -> None:
+ spans.append(feature)
+
+ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)
+ refused = httpx.Response(404, json={"error": "deployment_not_found", "message": (
+ "No deployment of this project; deploy it with `crewai deploy create`, or pass --deployment.")})
+ install(monkeypatch, FakeModelsAMP(create=refused))
+
+ with pytest.raises(SystemExit) as stopped:
+ eval_module.eval_models(MODELS)
+
+ assert stopped.value.code == 1 and spans == []
+ assert "deploy it with `crewai deploy create`, or pass --deployment." in capsys.readouterr().out
+
+
+def test_the_deployment_named_is_sent(deployed, monkeypatch):
+ amp = install(monkeypatch, FakeModelsAMP(statuses=[compared()]))
+
+ eval_module.eval_models("openai/gpt-4o-mini", deployment_id=f" {DEPLOYMENT.upper()} ")
+
+ assert amp.sent["deployment_id"] == DEPLOYMENT
+
+
+def test_a_deployment_that_is_not_a_uuid_is_refused_before_anything_is_sent(deployed, monkeypatch, capsys):
+ amp = install(monkeypatch, FakeModelsAMP())
+
+ with pytest.raises(SystemExit) as stopped:
+ eval_module.eval_models("openai/gpt-4o-mini", deployment_id="poem_creator")
+
+ assert stopped.value.code == 1 and amp.calls == []
+ assert "is not a deployment id" in capsys.readouterr().out
+
+
+def test_comparing_models_needs_the_login(deployed, monkeypatch, capsys):
+ monkeypatch.setattr(eval_module, "saved_login", lambda: None)
+ amp = install(monkeypatch, FakeModelsAMP())
+
+ with pytest.raises(SystemExit) as stopped:
+ eval_module.eval_models(MODELS)
+
+ assert stopped.value.code == 1 and amp.calls == []
+ assert "log in with `crewai login`" in capsys.readouterr().out
+
+
+def test_comparing_models_happens_from_the_project(project, monkeypatch, capsys):
+ amp = install(monkeypatch, FakeModelsAMP())
+
+ with pytest.raises(SystemExit) as stopped:
+ eval_module.eval_models(MODELS)
+
+ assert stopped.value.code == 1 and amp.calls == []
+ assert "No crewAI project here" in capsys.readouterr().out
+
+
+ ("text", "expected"),
+ [
+ ("openai/gpt-4o-mini", ["openai/gpt-4o-mini"]),
+ (" openai/gpt-4o-mini , openai/gpt-4o-mini,anthropic/claude-haiku-4-5, ",
+ ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"]),
+ ("openrouter/meta-llama/llama-4-maverick", ["openrouter/meta-llama/llama-4-maverick"]),
+ ],
+)
+def test_the_models_are_one_list_stripped_with_repeats_dropped(text, expected):
+ assert eval_module.parse_models(text) == expected
+
+
+ ("text", "said"),
+ [
+ ("gpt-4o-mini", "names no provider"),
+ ("openai/", "names no provider"),
+ ("/gpt-4o", "names no provider"),
+ (" , ", "names no model"),
+ (",".join(f"openai/m{n}" for n in range(6)), "at most 5"),
+ ("openai/" + "m" * 200, "longer than 200"),
+ ],
+)
+def test_a_list_amp_would_refuse_is_refused_here_with_an_example(text, said):
+ with pytest.raises(eval_module.EvaluationStoppedError, match=said) as refused:
+ eval_module.parse_models(text)
+ if "provider" in said or "names no model" in said:
+ assert "openai/gpt-4o-mini" in str(refused.value)
+
+
+def test_a_model_without_its_provider_exits_one_before_anything_is_sent(deployed, monkeypatch, capsys):
+ amp = install(monkeypatch, FakeModelsAMP())
+
+ with pytest.raises(SystemExit) as stopped:
+ eval_module.eval_models("gpt-4o-mini")
+
+ assert stopped.value.code == 1 and amp.calls == []
+ assert "provider/model" in capsys.readouterr().out
+
+
+def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, capsys):
+ failed = httpx.Response(200, json={"id": "ev-9", "status": "failed", "error": "the deployment answered 500"})
+ install(monkeypatch, FakeModelsAMP(statuses=[failed]))
+
+ with pytest.raises(SystemExit) as stopped:
+ eval_module.eval_models(MODELS)
+
+ assert stopped.value.code == 1
+ assert "Comparison failed: the deployment answered 500" in capsys.readouterr().out
+
+
+def test_done_without_a_comparison_is_a_protocol_error(deployed, monkeypatch, capsys):
+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json={"id": "ev-9", "status": "done"})]))
+
+ with pytest.raises(SystemExit) as stopped:
+ eval_module.eval_models(MODELS)
+
+ assert stopped.value.code == 1
+ assert "done without a comparison (protocol error)" in capsys.readouterr().out
+
+
+def test_a_deployment_this_account_may_not_run_is_amps_sentence_alone(deployed, monkeypatch, capsys):
+ """Logging in again changes nothing about a deployment one may not run."""
+ refused = httpx.Response(403, json={"error": "not_allowed", "message": "You may not run poem_creator."})
+ install(monkeypatch, FakeModelsAMP(create=refused))
+
+ with pytest.raises(SystemExit):
+ eval_module.eval_models(MODELS)
+
+ out = capsys.readouterr().out
+ assert "You may not run poem_creator." in out and "crewai login" not in out
+
+
+def test_everything_from_the_wire_prints_literally(deployed, monkeypatch, capsys):
+ hostile = comparison(
+ models=[{"key": "k", "label": "[red]Writer[/red]: [link=https://evil.test]x[/link]", "baseline": True,
+ "grades": {"goal": 9, "tasks": True, "agents": "5", "tools": None}, "cost_usd": True,
+ "seconds": -1}],
+ suggestions=[{"subject": "[bold]agent[/bold]", "field": "goal", "problem": "[red]p[/red]",
+ "change": "[link=https://evil.test]c[/link]"}],
+ )
+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(
+ 200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": hostile})]))
+
+ eval_module.eval_models(MODELS)
+
+ out = capsys.readouterr().out
+ assert "[red]Writer[/red]: [link=https://evil.test]x[/link]" in out
+ assert "[red]p[/red]" in out and "[link=https://evil.test]c[/link]" in out
+ # Not a grade, a cost or a time: each prints as "—", never as a value.
+ row = next(line for line in out.splitlines() if "Writer" in line)
+ assert "9/5" not in row and "True" not in row and "-1" not in row and "5/5" not in row
+ assert row.count("—") == 6
+
+
+ ("progress", "line"),
+ [
+ ("Setting up the deployment", "Setting up the deployment"),
+ ({"message": "model 2 of 3: mini"}, "model 2 of 3: mini"),
+ ({"event": "configuration_done", "payload": {"index": 0, "total": 2, "key": "base"}},
+ "graded · model 1 of 2 · base"),
+ ({"subject": "the final output"}, "judging the final output"),
+ ({"unknown": "shape"}, None),
+ (["not", "a", "dict"], None),
+ ],
+)
+def test_the_progress_line_says_what_is_there_and_guesses_nothing(progress, line):
+ assert eval_module._progress_line({"progress": progress}) == line
+
+
+def test_the_last_event_stands_in_for_progress():
+ events = [{"event": "configuration_started", "payload": {"index": 0, "total": 2, "key": "base"}},
+ {"event": "judging", "payload": {"subject": "task 'write'"}}]
+
+ assert eval_module._progress_line({"events": events}) == "judging task 'write'"
+
+
+def test_the_cli_maps_models_and_deployment_to_the_comparison(monkeypatch):
+ calls = []
+ monkeypatch.setattr("crewai_cli.cli.eval_models", lambda *args, **kwargs: calls.append((args, kwargs)))
+ monkeypatch.setattr("crewai_cli.cli.eval_crew", lambda **kwargs: calls.append(("run", kwargs)))
+ runner = CliRunner()
+
+ assert runner.invoke(eval_command, ["--models", MODELS]).exit_code == 0
+ assert runner.invoke(eval_command, ["--models", "openai/gpt-4o", "--deployment", DEPLOYMENT]).exit_code == 0
+ assert calls == [((MODELS,), {"deployment_id": None}), (("openai/gpt-4o",), {"deployment_id": DEPLOYMENT})]
+
+ both = runner.invoke(eval_command, ["--models", MODELS, "--run", EXECUTION_ID])
+ alone = runner.invoke(eval_command, ["--deployment", DEPLOYMENT])
+ assert both.exit_code == 2 and "Give one of them" in both.output
+ assert alone.exit_code == 2 and "add --models LIST" in alone.output
+ assert len(calls) == 2
+ assert "--models" in runner.invoke(eval_command, ["--help"]).output
+
+
+def test_a_comparison_leaves_its_criteria_behind_when_the_project_has_none(deployed, monkeypatch, capsys):
+ directory, _ = deployed
+ answer = compared()
+ body = {**answer.json(), "eval_config": '// the criteria\n{"dataset": []}\n'}
+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json=body)]))
+
+ eval_module.eval_models(MODELS)
+
+ assert (directory / "eval.jsonc").read_text() == '// the criteria\n{"dataset": []}\n'
+ assert "Wrote eval.jsonc" in capsys.readouterr().out
+
+
+def test_a_comparison_never_overwrites_the_projects_criteria(deployed, monkeypatch, capsys):
+ directory, _ = deployed
+ (directory / "eval.jsonc").write_text("// ours\n{}\n")
+ body = {**compared().json(), "eval_config": "// theirs\n{}\n"}
+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json=body)]))
+
+ eval_module.eval_models(MODELS)
+
+ assert (directory / "eval.jsonc").read_text() == "// ours\n{}\n"
+ assert "Wrote eval.jsonc" not in capsys.readouterr().out
+
+
+ ("typed", "sent"),
+ [
+ ("openai/gpt-4o-mini", "openai/gpt-4o-mini"),
+ ("anthropic/claude-haiku-4-5", "anthropic/claude-haiku-4-5"),
+ ("openrouter/openai/gpt-4o-mini", "openrouter/openai/gpt-4o-mini"),
+ # A customer's own: a fine-tune, a deployment name, a model named after
+ # a public one, an unknown name — by provider only.
+ ("openai/ft:gpt-4o-mini:acme-corp::abc", "openai/other"),
+ ("azure/my-deployment", "azure/other"),
+ ("openai/gpt-4o-acme", "openai/other"),
+ ("ollama/acme-llama", "ollama/other"),
+ # A provider crewAI does not know can name a host.
+ ("llm.acme.internal/llama-3", "other/other"),
+ ],
+)
+def test_usage_stats_name_a_model_only_when_it_is_a_public_one(typed, sent):
+ assert eval_module.telemetry_model_name(typed) == sent
+
+
+def test_without_crewais_catalog_every_model_is_other(monkeypatch):
+ monkeypatch.setattr(eval_module, "_known_models_and_providers", lambda: (frozenset(), frozenset()))
+
+ assert eval_module.telemetry_model_name("openai/gpt-4o-mini") == "other/other"
diff --git a/lib/cli/tests/test_plus_api.py b/lib/cli/tests/test_plus_api.py
index e40cfdbc2..0a3b721da 100644
--- a/lib/cli/tests/test_plus_api.py
+++ b/lib/cli/tests/test_plus_api.py
@@ -33,6 +33,60 @@ class TestPlusAPI(unittest.TestCase):
)
self.assertEqual(response, mock_response)
+ @patch("crewai_core.plus_api.PlusAPI._make_request")
+ def test_create_evaluation_files_it_under_the_project(self, mock_make_request):
+ self.api.create_evaluation("run-1", eval_config="{}", project_id="proj-1")
+
+ mock_make_request.assert_called_once_with(
+ "POST",
+ "/crewai_plus/api/v1/tracing/evaluations",
+ json={"execution_id": "run-1", "eval_config": "{}", "project_id": "proj-1"},
+ timeout=120.0,
+ )
+
+ @patch("crewai_core.plus_api.PlusAPI._make_request")
+ def test_create_models_evaluation(self, mock_make_request):
+ mock_response = MagicMock()
+ mock_make_request.return_value = mock_response
+
+ response = self.api.create_models_evaluation(
+ ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"],
+ project_id="proj-1",
+ eval_config="// ours\n{}",
+ deployment_id="dep-1",
+ )
+
+ mock_make_request.assert_called_once_with(
+ "POST",
+ "/crewai_plus/api/v1/tracing/evaluations",
+ json={
+ "kind": "models",
+ "project_id": "proj-1",
+ "models": ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"],
+ "eval_config": "// ours\n{}",
+ "deployment_id": "dep-1",
+ },
+ timeout=120.0,
+ )
+ self.assertEqual(response, mock_response)
+
+ @patch("crewai_core.plus_api.PlusAPI._make_request")
+ def test_create_models_evaluation_sends_only_what_it_was_given(
+ self, mock_make_request
+ ):
+ self.api.create_models_evaluation(["openai/gpt-4o-mini"], project_id="proj-1")
+
+ mock_make_request.assert_called_once_with(
+ "POST",
+ "/crewai_plus/api/v1/tracing/evaluations",
+ json={
+ "kind": "models",
+ "project_id": "proj-1",
+ "models": ["openai/gpt-4o-mini"],
+ },
+ timeout=120.0,
+ )
+
@patch("crewai_core.plus_api.PlusAPI._make_request")
def test_get_evaluation(self, mock_make_request):
mock_response = MagicMock()
@@ -252,6 +306,42 @@ class TestPlusAPI(unittest.TestCase):
"POST", "/crewai_plus/api/v1/crews/test_uuid/deploy"
)
+ @patch("crewai_core.plus_api.PlusAPI._make_request")
+ def test_deploy_sends_the_project_id_when_there_is_one(self, mock_make_request):
+ self.api.deploy_by_name("test_project", project_id="proj-1")
+ mock_make_request.assert_called_with(
+ "POST",
+ "/crewai_plus/api/v1/crews/by-name/test_project/deploy",
+ json={"project_id": "proj-1"},
+ )
+ self.api.deploy_by_uuid("test_uuid", project_id="proj-1")
+ mock_make_request.assert_called_with(
+ "POST",
+ "/crewai_plus/api/v1/crews/test_uuid/deploy",
+ json={"project_id": "proj-1"},
+ )
+
+ @patch("crewai_cli.plus_api.PlusAPI._make_multipart_request")
+ def test_zip_deploys_send_the_project_id_when_there_is_one(
+ self, mock_make_multipart_request
+ ):
+ self.api.create_crew_from_zip("/tmp/test.zip", name="c", project_id="proj-1")
+ mock_make_multipart_request.assert_called_with(
+ "POST",
+ "/crewai_plus/api/v1/crews/zip",
+ zip_file_path="/tmp/test.zip",
+ data={"name": "c", "project_id": "proj-1"},
+ timeout=300,
+ )
+ self.api.update_crew_from_zip("u", "/tmp/test.zip", project_id="proj-1")
+ mock_make_multipart_request.assert_called_with(
+ "POST",
+ "/crewai_plus/api/v1/crews/u/zip_update",
+ zip_file_path="/tmp/test.zip",
+ data={"project_id": "proj-1"},
+ timeout=300,
+ )
+
@patch("crewai_core.plus_api.PlusAPI._make_request")
def test_crew_status_by_name(self, mock_make_request):
self.api.crew_status_by_name("test_project")
diff --git a/lib/crewai-core/pyproject.toml b/lib/crewai-core/pyproject.toml
index f6b790ef9..dd7068a68 100644
--- a/lib/crewai-core/pyproject.toml
+++ b/lib/crewai-core/pyproject.toml
@@ -13,7 +13,7 @@ dependencies = [
"httpx~=0.28.1",
"packaging>=23.0",
"portalocker~=2.7.0",
- "pyjwt>=2.13.0,<3",
+ "pyjwt>=2.15.0,<3",
"pydantic>=2.11.9,<2.13",
"rich>=13.7.1",
"opentelemetry-api>=1.42,<2",
diff --git a/lib/crewai-core/src/crewai_core/plus_api.py b/lib/crewai-core/src/crewai_core/plus_api.py
index c1acbad4e..28c8968d6 100644
--- a/lib/crewai-core/src/crewai_core/plus_api.py
+++ b/lib/crewai-core/src/crewai_core/plus_api.py
@@ -62,6 +62,9 @@ class CrewDeploymentSpec(TypedDict):
class CreateCrewPayload(TypedDict):
deploy: CrewDeploymentSpec
+ # ``[tool.crewai].project_id``, at the top level as every other deploy
+ # request sends it (AMP reads it there on every route); only when set.
+ project_id: NotRequired[str]
class _WithUserIdentifier(TypedDict):
@@ -320,13 +323,29 @@ class PlusAPI:
params=params or None,
)
- def deploy_by_name(self, project_name: str) -> httpx.Response:
- return self._make_request(
- "POST", f"{self.CREWS_RESOURCE}/by-name/{project_name}/deploy"
- )
+ def deploy_by_name(
+ self, project_name: str, *, project_id: str | None = None
+ ) -> httpx.Response:
+ """Redeploy by name, with PROJECT_ID (`[tool.crewai].project_id`) when set.
- def deploy_by_uuid(self, uuid: str) -> httpx.Response:
- return self._make_request("POST", f"{self.CREWS_RESOURCE}/{uuid}/deploy")
+ The body is sent only when there is a project id. An AMP that predates
+ project ids ignores it: this route has read its body through Rails'
+ ``params.permit`` since the v1 API (2024-08), which drops a key it does
+ not name, and AMP never made unpermitted keys an error.
+ """
+ endpoint = f"{self.CREWS_RESOURCE}/by-name/{project_name}/deploy"
+ if project_id:
+ return self._make_request("POST", endpoint, json={"project_id": project_id})
+ return self._make_request("POST", endpoint)
+
+ def deploy_by_uuid(
+ self, uuid: str, *, project_id: str | None = None
+ ) -> httpx.Response:
+ """Redeploy by uuid, with PROJECT_ID as for `deploy_by_name`."""
+ endpoint = f"{self.CREWS_RESOURCE}/{uuid}/deploy"
+ if project_id:
+ return self._make_request("POST", endpoint, json={"project_id": project_id})
+ return self._make_request("POST", endpoint)
def crew_status_by_name(self, project_name: str) -> httpx.Response:
return self._make_request(
@@ -368,11 +387,18 @@ class PlusAPI:
*,
name: str | None = None,
env: dict[str, str] | None = None,
+ project_id: str | None = None,
) -> httpx.Response:
- """Create a crew deployment from a local project ZIP archive."""
+ """Create a crew deployment from a local project ZIP archive.
+
+ PROJECT_ID is `[tool.crewai].project_id`, so AMP knows which project the
+ deployment runs; sent only when the project has one.
+ """
data: dict[str, str] = {}
if name:
data["name"] = name
+ if project_id:
+ data["project_id"] = project_id
if env:
data.update({f"env[{key}]": value for key, value in env.items()})
return self._make_multipart_request(
@@ -389,9 +415,15 @@ class PlusAPI:
zip_file_path: str | Path,
*,
env: dict[str, str] | None = None,
+ project_id: str | None = None,
) -> httpx.Response:
- """Update an existing crew deployment from a local project ZIP archive."""
+ """Update an existing crew deployment from a local project ZIP archive.
+
+ PROJECT_ID as for `create_crew_from_zip`.
+ """
data: dict[str, str] = {}
+ if project_id:
+ data["project_id"] = project_id
if env:
data.update({f"env[{key}]": value for key, value in env.items()})
return self._make_multipart_request(
diff --git a/lib/crewai-core/src/crewai_core/telemetry.py b/lib/crewai-core/src/crewai_core/telemetry.py
index a5e4aca26..a8de2f025 100644
--- a/lib/crewai-core/src/crewai_core/telemetry.py
+++ b/lib/crewai-core/src/crewai_core/telemetry.py
@@ -580,6 +580,11 @@ class Telemetry:
# its use under somebody else's name.
FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {
"cli_usage:eval": frozenset({"authenticated"}),
+ # The models compared: a model in crewAI's own catalog by name (anyone
+ # can read it off a price list), any other — a fine-tune, a deployment
+ # name, a self-hosted model — as `<provider>/other`; the CLI decides
+ # (`telemetry_model_name`). Never a run, an output or an organization.
+ "cli_usage:eval_models": frozenset({"authenticated", "models", "models_count"}),
}
def feature_usage_span(
diff --git a/lib/crewai-core/tests/test_smoke.py b/lib/crewai-core/tests/test_smoke.py
index 22097086a..4deeb99df 100644
--- a/lib/crewai-core/tests/test_smoke.py
+++ b/lib/crewai-core/tests/test_smoke.py
@@ -311,6 +311,45 @@ def test_core_feature_span_keeps_only_what_the_feature_may_send(
assert "authenticated" not in sent and sent["feature"] == "cli_usage:deploy"
+def test_core_models_span_carries_the_models_compared_and_nothing_else(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """`cli_usage:eval_models` names the models and how many — provider/model
+ names, not a run — and still drops an id or run content."""
+ from crewai_core.telemetry import Telemetry
+
+ Telemetry._instance = None
+ monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)
+ monkeypatch.delenv("CREWAI_DISABLE_TELEMETRY", raising=False)
+ monkeypatch.delenv("CREWAI_DISABLE_TRACKING", raising=False)
+
+ tracer = Mock()
+ span = Mock()
+ tracer.start_span.return_value = span
+ monkeypatch.setattr(
+ "crewai_core.telemetry.TracerProvider",
+ lambda **_kwargs: Mock(get_tracer=Mock(return_value=tracer)),
+ )
+
+ Telemetry().feature_usage_span(
+ "cli_usage:eval_models",
+ {
+ "authenticated": "true",
+ "models": "openai/gpt-4o-mini,anthropic/claude-haiku-4-5",
+ "models_count": "2",
+ "evaluation_id": "ev-1",
+ "organization_id": "org-1",
+ },
+ )
+
+ sent = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}
+ assert sent["feature"] == "cli_usage:eval_models"
+ assert sent["authenticated"] == "true"
+ assert sent["models"] == "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"
+ assert sent["models_count"] == "2"
+ assert "evaluation_id" not in sent and "organization_id" not in sent
+
+
def test_core_telemetry_records_flow_creation_version(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/lib/crewai/pyproject.toml b/lib/crewai/pyproject.toml
index ffb6c8d0d..02ab9c30e 100644
--- a/lib/crewai/pyproject.toml
+++ b/lib/crewai/pyproject.toml
@@ -29,7 +29,7 @@ dependencies = [
"openpyxl~=3.1.5",
# Authentication and Security
"python-dotenv>=1.2.2,<2",
- "pyjwt>=2.13.0,<3",
+ "pyjwt>=2.15.0,<3",
# Configuration and Utils
"click>=8.1.7,<9",
"appdirs~=1.4.4",
diff --git a/lib/crewai/src/crewai/agent/core.py b/lib/crewai/src/crewai/agent/core.py
index 3a43cee93..1ce5f6392 100644
--- a/lib/crewai/src/crewai/agent/core.py
+++ b/lib/crewai/src/crewai/agent/core.py
@@ -79,7 +79,7 @@ from crewai.hooks.dispatch import HookAborted
from crewai.knowledge.knowledge import Knowledge
from crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource
from crewai.lite_agent_output import LiteAgentOutput
-from crewai.llm_overlay import active as overlay_active, overlay_model_for
+from crewai.llm_overlay import active as overlay_active
from crewai.llms.base_llm import BaseLLM
from crewai.mcp.config import MCPServerConfig
from crewai.rag.embeddings.types import EmbedderConfig
@@ -114,7 +114,7 @@ from crewai.utilities.env import get_env_context
from crewai.utilities.guardrail import process_guardrail, serialize_guardrail_for_json
from crewai.utilities.guardrail_types import GuardrailCallable, GuardrailType
from crewai.utilities.i18n import I18N_DEFAULT
-from crewai.utilities.llm_utils import create_llm, create_llm_like
+from crewai.utilities.llm_utils import create_llm, overlay_llm_for
from crewai.utilities.prompts import Prompts, StandardPromptResult, SystemPromptResult
from crewai.utilities.pydantic_schema_utils import generate_model_description
from crewai.utilities.string_utils import sanitize_tool_name
@@ -420,13 +420,7 @@ class Agent(BaseAgent):
else:
declared = create_llm(self.llm)
self._declared_llm = declared
- overlay_model = overlay_model_for(self.role)
- if overlay_model:
- self.llm = self._overlay_built = create_llm_like(
- overlay_model, declared
- )
- else:
- self.llm = declared
+ self.llm = self._resolved_overlay(declared)
self._overlay_read = True
if self.function_calling_llm and not isinstance(
self.function_calling_llm, BaseLLM
@@ -523,15 +517,22 @@ class Agent(BaseAgent):
if self.role == role_before or overlay_active.get() is None:
return
previous = self.llm
- declared = self._declared_now()
- overlay_model = overlay_model_for(self.role)
- if overlay_model:
- self.llm = self._overlay_built = create_llm_like(overlay_model, declared)
- else:
- self.llm = declared
+ self.llm = self._resolved_overlay(self._declared_now())
if isinstance(previous, BaseLLM) and isinstance(self.llm, BaseLLM):
self.llm.stream = previous.stream
+ def _resolved_overlay(self, declared: BaseLLM | None) -> BaseLLM | None:
+ """The llm ``llm_overlay`` puts this agent on, given its ``declared`` one.
+
+ The role's key wins over the model keys; with neither, ``declared``.
+ What the overlay built is remembered, so a later miss can tell it from
+ an llm the caller assigned.
+ """
+ resolved = overlay_llm_for(self.role, declared)
+ if resolved is not declared:
+ self._overlay_built = resolved
+ return resolved
+
def _declared_now(self) -> BaseLLM | None:
"""The llm this agent is declared with, as of now.
diff --git a/lib/crewai/src/crewai/lite_agent.py b/lib/crewai/src/crewai/lite_agent.py
index 0fa370a9c..9699e2e5d 100644
--- a/lib/crewai/src/crewai/lite_agent.py
+++ b/lib/crewai/src/crewai/lite_agent.py
@@ -69,7 +69,6 @@ from crewai.hooks.types import (
)
from crewai.lite_agent_output import LiteAgentOutput
from crewai.llm import LLM
-from crewai.llm_overlay import overlay_model_for
from crewai.llms.base_llm import BaseLLM
from crewai.tools.base_tool import BaseTool
from crewai.tools.structured_tool import CrewStructuredTool
@@ -102,7 +101,7 @@ from crewai.utilities.converter import (
from crewai.utilities.guardrail import process_guardrail, serialize_guardrail_for_json
from crewai.utilities.guardrail_types import GuardrailCallable, GuardrailType
from crewai.utilities.i18n import I18N_DEFAULT
-from crewai.utilities.llm_utils import create_llm, create_llm_like
+from crewai.utilities.llm_utils import create_llm, overlay_llm_for
from crewai.utilities.pydantic_schema_utils import (
generate_model_description,
serialize_model_class,
@@ -320,11 +319,7 @@ class LiteAgent(FlowTrackable, BaseModel):
@model_validator(mode="after")
def setup_llm(self) -> Self:
"""Set up the LLM and other components after initialization."""
- declared = create_llm(self.llm)
- overlay_model = overlay_model_for(self.role)
- self.llm = (
- create_llm_like(overlay_model, declared) if overlay_model else declared
- )
+ self.llm = overlay_llm_for(self.role, create_llm(self.llm))
if not isinstance(self.llm, BaseLLM):
raise ValueError(
f"Expected LLM instance of type BaseLLM, got {type(self.llm).__name__}"
diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py
index 569296bb8..2ba1e6409 100644
--- a/lib/crewai/src/crewai/llm.py
+++ b/lib/crewai/src/crewai/llm.py
@@ -2,6 +2,7 @@ from __future__ import annotations
from collections import defaultdict
from collections.abc import Callable, Mapping
+from contextvars import ContextVar
from datetime import datetime
import json
import logging
@@ -33,6 +34,7 @@ from crewai.events.types.tool_usage_events import (
ToolUsageStartedEvent,
)
from crewai.hooks.dispatch import HookAborted
+from crewai.llm_overlay import overlay_model_for_model
from crewai.llms._finish_reason_utils import extract_choices_finish_reason_and_id
from crewai.llms.base_llm import (
BaseLLM,
@@ -88,6 +90,10 @@ logger = logging.getLogger(__name__)
_litellm_loaded = False
LITELLM_AVAILABLE = False
+# The instance ``LLM.__new__`` just built on a model ``llm_overlay`` mapped to,
+# whose ``__init__`` call with the caller's arguments must be skipped.
+_overlay_built: ContextVar[LLM | None] = ContextVar("llm_overlay_built", default=None)
+
if TYPE_CHECKING:
import litellm
from litellm.litellm_core_utils.get_supported_openai_params import (
@@ -194,6 +200,30 @@ SUPPORTED_NATIVE_PROVIDERS: Final[list[str]] = [
]
+# The provider a ``<prefix>/<model>`` string routes to, by prefix: the names
+# crewAI accepts for each native provider. ``llm_overlay`` compares model
+# names through the same table, so ``google/x`` and ``gemini/x`` are one model.
+PROVIDER_ALIASES: Final[dict[str, str]] = {
+ "openai": "openai",
+ "anthropic": "anthropic",
+ "claude": "anthropic",
+ "azure": "azure",
+ "azure_openai": "azure",
+ "google": "gemini",
+ "gemini": "gemini",
+ "bedrock": "bedrock",
+ "aws": "bedrock",
+ "openrouter": "openrouter",
+ "deepseek": "deepseek",
+ "ollama": "ollama",
+ "ollama_chat": "ollama_chat",
+ "hosted_vllm": "hosted_vllm",
+ "cerebras": "cerebras",
+ "dashscope": "dashscope",
+ "snowflake": "snowflake",
+}
+
+
class Delta(TypedDict):
content: str | None
role: str | None
@@ -266,6 +296,11 @@ class LLM(BaseLLM):
if not model or not isinstance(model, str):
raise ValueError("Model must be a non-empty string")
+ if cls is LLM:
+ mapped = cls._overlay_mapped(model, is_litellm, kwargs)
+ if mapped is not None:
+ return mapped
+
route = cls._resolve_route(model, kwargs)
if route.native_class is not None and not is_litellm:
try:
@@ -310,6 +345,39 @@ class LLM(BaseLLM):
return object.__new__(cls)
+ def __init__(self, model: str, is_litellm: bool = False, **kwargs: Any) -> None:
+ # Python calls ``__init__`` on whatever ``__new__`` returns when it is an
+ # ``LLM``; an instance ``llm_overlay`` built on a mapped model is already
+ # initialized, and initializing it again with the caller's arguments
+ # would put the declared model back.
+ if _overlay_built.get() is self:
+ _overlay_built.set(None)
+ return
+ super().__init__(model=model, is_litellm=is_litellm, **kwargs)
+
+ @classmethod
+ def _overlay_mapped(
+ cls, model: str, is_litellm: bool, kwargs: dict[str, Any]
+ ) -> LLM | None:
+ """``LLM(model, **kwargs)`` built on the model an ``llm_overlay`` key maps it to.
+
+ ``None`` when no model key of the active overlay matches ``model``. The
+ provider a bare model routes to is put beside it, so a key written
+ ``model:openai/gpt-4o`` matches ``LLM(model="gpt-4o")``.
+ """
+ provider = kwargs.get("provider")
+ if not provider and "/" not in model:
+ provider = cls._infer_provider_from_model(model)
+ mapped_model = overlay_model_for_model(model, provider)
+ if mapped_model is None:
+ return None
+ from crewai.utilities.llm_utils import create_llm_from_kwargs_like
+
+ built = create_llm_from_kwargs_like(mapped_model, model, kwargs, is_litellm)
+ if isinstance(built, LLM):
+ _overlay_built.set(built)
+ return cast(LLM, built)
+
@classmethod
def _matches_provider_pattern(cls, model: str, provider: str) -> bool:
"""Check if a model name matches provider-specific patterns.
@@ -416,27 +484,7 @@ class LLM(BaseLLM):
elif "/" in model:
prefix, _, model_part = model.partition("/")
- provider_mapping = {
- "openai": "openai",
- "anthropic": "anthropic",
- "claude": "anthropic",
- "azure": "azure",
- "azure_openai": "azure",
- "google": "gemini",
- "gemini": "gemini",
- "bedrock": "bedrock",
- "aws": "bedrock",
- "openrouter": "openrouter",
- "deepseek": "deepseek",
- "ollama": "ollama",
- "ollama_chat": "ollama_chat",
- "hosted_vllm": "hosted_vllm",
- "cerebras": "cerebras",
- "dashscope": "dashscope",
- "snowflake": "snowflake",
- }
-
- canonical_provider = provider_mapping.get(prefix.lower())
+ canonical_provider = PROVIDER_ALIASES.get(prefix.lower())
valid_native_model = bool(
canonical_provider
diff --git a/lib/crewai/src/crewai/llm_overlay.py b/lib/crewai/src/crewai/llm_overlay.py
index 8cb520d6a..fde469432 100644
--- a/lib/crewai/src/crewai/llm_overlay.py
+++ b/lib/crewai/src/crewai/llm_overlay.py
@@ -1,10 +1,37 @@
-"""Process-context ``role -> model`` overlay for agents.
+"""Process-context ``role -> model`` and ``model -> model`` overlay.
-``llm_overlay`` lets a per-run caller make specific agents use a different
-model without editing the code that builds them. While the block is active,
-an ``Agent`` or ``LiteAgent`` whose ``role`` is a key of the mapping is built
-with the mapped model instead of its declared ``llm``. Roles that are not
-keys, and every agent built outside the block, keep their own model.
+``llm_overlay`` lets a per-run caller make specific agents, or specific
+models, use a different model without editing the code that builds them.
+While the block is active, an ``Agent`` or ``LiteAgent`` whose ``role`` is a
+key of the mapping is built with the mapped model instead of its declared
+``llm``. Roles that are not keys, and every agent built outside the block,
+keep their own model.
+
+A key that starts with :data:`MODEL_KEY_PREFIX` names a model instead of a
+role: ``"model:openai/gpt-4o"`` maps every LLM built from that model string
+inside the block, and ``"model:*"`` every LLM built from any model string.
+That is what reaches an LLM no role names — a flow step's own
+``LLM(model="openai/gpt-4o").call(...)`` — and the simplest way to say "this
+whole run on model X". A model key is read wherever an LLM is built from a
+model string: ``LLM(model=...)`` (and so ``create_llm`` and an agent's
+``llm="..."``, which go through it), and an agent's declared ``llm`` instance
+when it was built outside the block. A model is compared as the caller wrote
+it and without the provider prefix native providers strip, so
+``"model:openai/gpt-4o"`` matches ``LLM(model="gpt-4o")`` and an instance
+whose ``model`` reads ``"gpt-4o"``, and ``"model:gpt-4o"`` matches
+``LLM(model="openai/gpt-4o")``; an exact key wins over a stripped one, and
+both over ``"model:*"``. Only the model's own provider prefix is ever
+stripped: an aggregator's route such as ``openrouter/openai/gpt-4o`` is that
+aggregator's model and matches only a key naming it whole. The mapped model is built once, never looked up
+again: ``{"model:a": "b", "model:b": "c"}`` puts an ``a`` on ``b``. For an
+agent a role key wins over every model key, so ``{"Researcher": x,
+"model:*": y}`` runs the Researcher on ``x`` and everything else on ``y`` —
+and ``x`` is built like the llm the caller declared, even when a model key
+already mapped that llm as it was built inside the block.
+The settings the caller passed to ``LLM(...)`` are carried to the mapped model
+by the same rule as a declared ``llm``'s (below). Only ``LLM`` itself is
+mapped: a subclass of it, or a provider class built directly, is the caller's
+own choice of class and keeps its model.
Roles are matched exactly, by the text they have when the overlay is read;
only whitespace around a role or a key is ignored, so a role a YAML file
@@ -46,6 +73,9 @@ registers an agent when it first emits) does not read it again, so an agent
built outside a block keeps its model through a kickoff inside one.
Example:
+ >>> with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+ ... flow.kickoff() # every LLM built in the run is on gpt-4o-mini
+
>>> with llm_overlay({"Researcher": "openai/gpt-4o"}):
... crew = build_crew() # the Researcher agent is built on gpt-4o
... crew.kickoff()
@@ -60,24 +90,57 @@ from __future__ import annotations
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
+from typing import Any, Final, TypeVar
+import weakref
+
+
+MODEL_KEY_PREFIX: Final[str] = "model:"
+"""A key starting with this names a model (``"model:openai/gpt-4o"``), not a role."""
+
+_T = TypeVar("_T")
+
+ANY_MODEL: Final[str] = "*"
+"""The model part of the key that maps every LLM built from a model string."""
active: ContextVar[dict[str, str] | None] = ContextVar(
"llm_overlay_active", default=None
)
+# Set while the overlay builds the model it mapped to, so that build is not
+# mapped again (a chain of model keys, or ``model:*`` under a role's model).
+_building_mapped: ContextVar[bool] = ContextVar(
+ "llm_overlay_building_mapped", default=False
+)
+
+# The LLMs a model key already mapped, by id. An agent reading its declared
+# ``llm`` skips one of these: its model is the mapped one, never a key again.
+_mapped_llms: weakref.WeakValueDictionary[int, Any] = weakref.WeakValueDictionary()
+
+# What the caller declared for an LLM the overlay built: the declared instance,
+# or ``LLM(model, **kwargs)``'s ``(model, kwargs, is_litellm)``. Dropped with
+# the LLM it describes.
+Declared = Any
+_declared_before: dict[int, Declared] = {}
+
@contextmanager
def llm_overlay(mapping: dict[str, str] | None) -> Iterator[None]:
- """Route agent roles to models for the duration of the block.
+ """Route agent roles and models to models for the duration of the block.
Args:
- mapping: ``{role: model}`` for the block; ``None`` clears any active
- overlay for the block. Whitespace around each role is dropped from
- the copy the block uses, so a key matches a role that differs from
- it only by its surrounding whitespace; the mapping passed in is
- left as it is. The previous value is always restored on exit,
- including when the block raises.
+ mapping: ``{role: model}`` and ``{"model:<model>": model}`` (or
+ ``{"model:*": model}``) for the block; ``None`` clears any active
+ overlay for the block. Whitespace around each role, and around the
+ model a model key names, is dropped from the copy the block uses,
+ so a key matches a role that differs from it only by its
+ surrounding whitespace; the mapping passed in is left as it is. The
+ previous value is always restored on exit, including when the
+ block raises.
+
+ Raises:
+ ValueError: Two keys name one role or one model with different
+ models, or a model key names no model.
"""
token = active.set(_stripped(mapping))
try:
@@ -90,7 +153,8 @@ def overlay_model_for(role: str | None) -> str | None:
"""The model the active overlay assigns to ``role``.
Whitespace around ``role`` is ignored, as it was around the keys when the
- overlay was set. An empty or ``None`` role matches nothing.
+ overlay was set. An empty or ``None`` role matches nothing, and a role is
+ never matched against a model key.
Returns:
The mapped model string, or ``None`` when no overlay is active or
@@ -100,22 +164,166 @@ def overlay_model_for(role: str | None) -> str | None:
if not mapping or role is None:
return None
role = role.strip()
- return mapping.get(role) if role else None
+ if not role or role.startswith(MODEL_KEY_PREFIX):
+ return None
+ return mapping.get(role)
+
+
+def overlay_model_for_model(
+ model: str | None, provider: str | None = None
+) -> str | None:
+ """The model the active overlay's model keys assign to an LLM on ``model``.
+
+ ``model`` is compared as written and, when a native provider strips its own
+ prefix, without it: ``openai/gpt-4o`` also as ``gpt-4o``, and ``gpt-4o``
+ routed to ``provider`` also as ``openai/gpt-4o``. A prefix is only ever the
+ model's OWN provider: in an aggregator's route (``openrouter/openai/gpt-4o``)
+ the rest is that aggregator's model id, never the model a key for
+ ``openai/gpt-4o`` or ``gpt-4o`` names, so it is compared whole. An existing
+ instance of such a route records ``model="openai/gpt-4o"`` under
+ ``provider="openrouter"``, and is compared as ``openrouter/openai/gpt-4o``.
+ ``model:*`` matches every model. Nothing matches while the overlay is
+ building a model it mapped.
+
+ Args:
+ model: The model string an LLM is being built from, or the ``model``
+ of an existing instance.
+ provider: The provider the LLM routes to, when ``model`` does not say.
+
+ Returns:
+ The mapped model string, or ``None`` when no overlay is active, it has
+ no model key for ``model``, or a mapped model is being built.
+ """
+ mapping = active.get()
+ if not mapping or not model or _building_mapped.get():
+ return None
+ forms = _model_forms(model.strip(), provider)
+ for form in forms:
+ mapped = mapping.get(MODEL_KEY_PREFIX + form)
+ if mapped is not None:
+ return mapped
+ # The same model under another of its provider's names: a key written
+ # `google/x` for an llm that routed `gemini/x`, or the other way round.
+ aliases = _aliases()
+ canonical = {_canonical(form, aliases) for form in forms}
+ for key, mapped in mapping.items():
+ named = (
+ key[len(MODEL_KEY_PREFIX) :] if key.startswith(MODEL_KEY_PREFIX) else None
+ )
+ if named and named != ANY_MODEL and _canonical(named, aliases) in canonical:
+ return mapped
+ return mapping.get(MODEL_KEY_PREFIX + ANY_MODEL)
+
+
+def _model_forms(model: str, provider: str | None) -> list[str]:
+ """``model`` in the forms a key may name it by, the one as written first."""
+ if provider and not model.startswith(f"{provider}/"):
+ routed = f"{provider}/{model}"
+ # A bare name is the native one with its prefix stripped; a name that
+ # still has a slash is the provider's own route id and only means
+ # something beside the provider.
+ forms = [routed] if "/" in model else [model, routed]
+ else:
+ forms = [model]
+ prefix, separator, rest = forms[-1].partition("/")
+ if separator and "/" not in rest and prefix.lower() in _aliases():
+ if rest not in forms:
+ forms.append(rest)
+ return forms
+
+
+def _aliases() -> dict[str, str]:
+ """The router's provider aliases (``crewai.llm.PROVIDER_ALIASES``): every
+ prefix ``LLM`` strips from a model string it routes natively, to the
+ provider it names."""
+ from crewai.llm import PROVIDER_ALIASES
+
+ return PROVIDER_ALIASES
+
+
+def _canonical(form: str, aliases: dict[str, str]) -> str:
+ """``form`` with its provider prefix under the provider's one name."""
+ prefix, separator, rest = form.partition("/")
+ provider = aliases.get(prefix.lower()) if separator else None
+ return f"{provider}/{rest}" if provider else form
+
+
+def overlay_model_for_llm(llm: Any) -> str | None:
+ """The model the active overlay's model keys assign to an existing ``llm``.
+
+ An agent's declared ``llm`` built outside the block is looked up by its
+ ``model`` and ``provider``; one a model key already mapped when it was
+ built is not looked up again.
+ """
+ if llm is None or _mapped_llms.get(id(llm)) is llm:
+ return None
+ return overlay_model_for_model(
+ getattr(llm, "model", None), getattr(llm, "provider", None)
+ )
+
+
+def declared_before_overlay(llm: Any) -> Declared | None:
+ """What the caller declared for ``llm`` before the overlay mapped it, or None."""
+ if llm is None or _mapped_llms.get(id(llm)) is not llm:
+ return None
+ return _declared_before.get(id(llm))
+
+
+@contextmanager
+def building_mapped_model() -> Iterator[None]:
+ """Build the model the overlay mapped to, without mapping it again."""
+ token = _building_mapped.set(True)
+ try:
+ yield
+ finally:
+ _building_mapped.reset(token)
+
+
+def mark_mapped(llm: _T, declared: Declared | None = None) -> _T:
+ """Record ``llm`` as built by the overlay, so no model key maps it again.
+
+ ``declared`` is what the caller declared before the overlay mapped it —
+ the declared instance, or ``LLM(model, **kwargs)``'s arguments as
+ ``(model, kwargs, is_litellm)`` — so a role key can still be built from
+ the caller's own declaration (:func:`declared_before_overlay`).
+ """
+ try:
+ _mapped_llms[id(llm)] = llm
+ if declared is not None:
+ _declared_before[id(llm)] = declared
+ weakref.finalize(llm, _declared_before.pop, id(llm), None)
+ except TypeError:
+ # Only a weakly-referenceable object can be remembered (every
+ # ``BaseLLM`` is). Anything else is returned as built: the one cost is
+ # that an agent declared with it looks its model up again, as it would
+ # for an llm built outside the block — never a failed build.
+ pass
+ return llm
def _stripped(mapping: dict[str, str] | None) -> dict[str, str] | None:
"""A copy of ``mapping`` with the whitespace around each role dropped.
- Two keys that differ only by whitespace name one role. When they name the
- same model the copy holds it once; when they name different models the
- mapping is ambiguous and is refused, so the model an agent runs on never
- depends on dictionary order.
+ Two keys that differ only by whitespace name one role (or, after the
+ :data:`MODEL_KEY_PREFIX`, one model). When they name the same model the
+ copy holds it once; when they name different models the mapping is
+ ambiguous and is refused, so the model an agent runs on never depends on
+ dictionary order.
"""
if mapping is None:
return None
stripped: dict[str, str] = {}
for role, model in mapping.items():
key = role.strip()
+ if key.startswith(MODEL_KEY_PREFIX):
+ named = key[len(MODEL_KEY_PREFIX) :].strip()
+ if not named:
+ raise ValueError(
+ f"llm_overlay: the key {role!r} names no model; write "
+ f"'{MODEL_KEY_PREFIX}<provider/model>' or "
+ f"'{MODEL_KEY_PREFIX}{ANY_MODEL}'"
+ )
+ key = MODEL_KEY_PREFIX + named
if key in stripped and stripped[key] != model:
raise ValueError(
f"llm_overlay: role {key!r} is mapped twice with different models "
diff --git a/lib/crewai/src/crewai/telemetry/telemetry.py b/lib/crewai/src/crewai/telemetry/telemetry.py
index a34fbe4ca..4de559bd9 100644
--- a/lib/crewai/src/crewai/telemetry/telemetry.py
+++ b/lib/crewai/src/crewai/telemetry/telemetry.py
@@ -1293,6 +1293,11 @@ class Telemetry:
# its use under somebody else's name.
FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {
"cli_usage:eval": frozenset({"authenticated"}),
+ # The models compared: a model in crewAI's own catalog by name (anyone
+ # can read it off a price list), any other — a fine-tune, a deployment
+ # name, a self-hosted model — as `<provider>/other`; the CLI decides
+ # (`telemetry_model_name`). Never a run, an output or an organization.
+ "cli_usage:eval_models": frozenset({"authenticated", "models", "models_count"}),
}
def feature_usage_span(
diff --git a/lib/crewai/src/crewai/utilities/llm_utils.py b/lib/crewai/src/crewai/utilities/llm_utils.py
index 57509bd67..a6c0bb376 100644
--- a/lib/crewai/src/crewai/utilities/llm_utils.py
+++ b/lib/crewai/src/crewai/utilities/llm_utils.py
@@ -6,6 +6,13 @@ from pydantic import ValidationError
from crewai.constants import DEFAULT_LLM_MODEL, ENV_VARS, LITELLM_PARAMS
from crewai.llm import LLM
+from crewai.llm_overlay import (
+ building_mapped_model,
+ declared_before_overlay,
+ mark_mapped,
+ overlay_model_for,
+ overlay_model_for_llm,
+)
from crewai.llms.base_llm import BaseLLM
@@ -208,22 +215,114 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:
Returns:
A new instance for ``model``; never ``base`` itself.
"""
- if not isinstance(base, BaseLLM):
- return LLM(model=model)
-
- # The declared endpoint decides where an unknown ``openai/`` model routes (a
- # custom OpenAI-compatible endpoint), so the same-provider question is asked
- # with it: a self-hosted model mapped to another model on the same endpoint
- # must keep that endpoint.
- route = LLM._resolve_route(model, _configured_settings(base, _ENDPOINT_NAMES))
- carried = _configured_settings(base, GENERATION_SETTINGS)
- if _same_provider(base, route):
- carried.update(_configured_settings(base, PROVIDER_SETTINGS))
- target = LLM._resolve_route(model, carried).native_class or LLM
- if type(base) is LLM and base.is_litellm:
+ with building_mapped_model():
+ if not isinstance(base, BaseLLM):
+ return mark_mapped(LLM(model=model))
+
+ # The declared endpoint decides where an unknown ``openai/`` model routes (a
+ # custom OpenAI-compatible endpoint), so the same-provider question is asked
+ # with it: a self-hosted model mapped to another model on the same endpoint
+ # must keep that endpoint.
+ route = LLM._resolve_route(model, _configured_settings(base, _ENDPOINT_NAMES))
+ carried = _configured_settings(base, GENERATION_SETTINGS)
+ if _same_provider(base, route):
+ carried.update(_configured_settings(base, PROVIDER_SETTINGS))
# The declared llm runs through LiteLLM — by the caller's choice or
# because no native class knew its model; either way that is the
# environment its callbacks and extra kwargs were written for.
+ through_litellm = type(base) is LLM and base.is_litellm
+ return mark_mapped(
+ _build_like(model, carried, type(base), through_litellm), base
+ )
+
+
+def overlay_llm_for(role: str | None, declared: BaseLLM | None) -> BaseLLM | None:
+ """The llm an agent with ``role`` and ``declared`` llm runs on under ``llm_overlay``.
+
+ A role key wins, and its model is built from the caller's declaration even
+ when a model key already mapped ``declared`` on its way in; else a model
+ key matching ``declared``'s model (one a model key already mapped when it
+ was built is not looked up again); else ``declared`` itself. A mapped model
+ is built like ``declared`` (:func:`create_llm_like`). Outside any block
+ this is ``declared``.
+ """
+ role_model = overlay_model_for(role)
+ if role_model:
+ # The role wins, built from what the caller declared: when a model key
+ # already mapped the declared llm, its declaration, not the mapping.
+ before = declared_before_overlay(declared)
+ if isinstance(before, tuple):
+ declared_model, kwargs, is_litellm = before
+ return create_llm_from_kwargs_like(
+ role_model, declared_model, kwargs, is_litellm
+ )
+ base = before if isinstance(before, BaseLLM) else declared
+ return create_llm_like(role_model, base)
+ model = overlay_model_for_llm(declared)
+ return create_llm_like(model, declared) if model else declared
+
+
+def create_llm_from_kwargs_like(
+ model: str, declared_model: str, kwargs: dict[str, Any], is_litellm: bool
+) -> BaseLLM:
+ """Build ``model`` configured like ``LLM(declared_model, **kwargs)`` would be.
+
+ What ``llm_overlay``'s model keys build when ``LLM(model=...)`` is called
+ inside the block: the settings the caller passed are carried by the rule of
+ :func:`create_llm_like` — generation settings to any class that has them,
+ credentials and endpoints only to the same provider — without building the
+ declared model first (its provider's key may not be in this environment).
+ Everything the caller passed is theirs, so nothing counts as derived.
+ """
+ declaration = (declared_model, dict(kwargs), is_litellm)
+ with building_mapped_model():
+ settings = {
+ k: v
+ for k, v in kwargs.items()
+ if v is not None and not (isinstance(v, (list, dict)) and not v)
+ }
+ if isinstance(settings.get("endpoint"), str):
+ settings["endpoint"] = settings["endpoint"].split(_AZURE_DEPLOYMENT_PATH)[0]
+ endpoint = {k: settings[k] for k in _ENDPOINT_NAMES if k in settings}
+ route = LLM._resolve_route(model, endpoint)
+ carried = {k: settings[k] for k in GENERATION_SETTINGS if k in settings}
+ try:
+ declared = LLM._resolve_route(declared_model, kwargs)
+ except ImportError:
+ # The declared model's SDK is not installed here. Nothing is built
+ # on it, so that is no reason to fail; its credentials, though,
+ # cannot be matched to the new provider, so none are carried. A
+ # caller who chose LiteLLM keeps it.
+ return mark_mapped(
+ _build_like(model, carried, None, is_litellm), declaration
+ )
+ declared_class = (
+ LLM
+ if is_litellm or declared.native_class is None
+ else declared.native_class
+ )
+ if _same_route_provider(declared_class, declared.provider, route):
+ carried.update({k: settings[k] for k in PROVIDER_SETTINGS if k in settings})
+ return mark_mapped(
+ _build_like(model, carried, declared_class, declared_class is LLM),
+ declaration,
+ )
+
+
+def _build_like(
+ model: str,
+ carried: dict[str, Any],
+ base_class: type[BaseLLM] | None,
+ through_litellm: bool,
+) -> BaseLLM:
+ """Build ``model`` with the ``carried`` settings its class accepts.
+
+ ``base_class`` is the declared llm's class, ``None`` when it is not known;
+ ``additional_params`` travel only within one class.
+ """
+ route = LLM._resolve_route(model, carried)
+ target = route.native_class or LLM
+ if through_litellm:
target = LLM
carried["is_litellm"] = True
accepted = {
@@ -231,7 +330,7 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:
for k, v in carried.items()
if k in target.model_fields or k == "is_litellm"
}
- if type(base) is not target:
+ if base_class is not target:
accepted.pop("additional_params", None)
_carry_cap_under_the_targets_name(carried, accepted, target)
if route.provider == "anthropic" and "temperature" in accepted:
@@ -239,6 +338,34 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:
return _build(model, accepted)
+def _same_route_provider(
+ declared_class: type[BaseLLM], declared_provider: str, route: Any
+) -> bool:
+ """:func:`_same_provider` for a declared llm known by its route, not an instance.
+
+ The same rule, case for case: a native declared class compares by class, a
+ LiteLLM-routed ``LLM`` by its provider — to a native route through the class
+ that provider names, so ``LLM(model="openai/gpt-4o", is_litellm=True)``
+ mapped to a native OpenAI model keeps its key and endpoint.
+ """
+ if route.native_class is None:
+ return declared_class is LLM and route.provider == declared_provider
+ if declared_class is route.native_class:
+ return not _serves_several_providers(route.native_class) or (
+ route.provider == declared_provider
+ )
+ if declared_class is not LLM:
+ return False
+ try:
+ named = LLM._get_native_provider(declared_provider or "")
+ except ImportError:
+ return False
+ return named is route.native_class and (
+ not _serves_several_providers(route.native_class)
+ or route.provider == declared_provider
+ )
+
+
def _same_provider(base: BaseLLM, route: Any) -> bool:
"""Whether ``route`` lands where ``base``'s credentials and endpoint belong.
diff --git a/lib/crewai/tests/llms/openai/test_openai.py b/lib/crewai/tests/llms/openai/test_openai.py
index 5a06f4676..4ab77b962 100644
--- a/lib/crewai/tests/llms/openai/test_openai.py
+++ b/lib/crewai/tests/llms/openai/test_openai.py
@@ -214,6 +214,15 @@ def test_openai_completion_module_is_imported(monkeypatch):
"""
module_name = "crewai.llms.providers.openai.completion"
+ # Re-importing binds the fresh module on its package too, and monkeypatch
+ # only restores sys.modules: without also restoring the attribute, every
+ # later test in this worker sees two module objects for one name, and a
+ # `patch("crewai.llms.providers.openai.completion.X")` patches the one that
+ # `from ... import X` does not read.
+ import crewai.llms.providers.openai as openai_package
+
+ if hasattr(openai_package, "completion"):
+ monkeypatch.setattr(openai_package, "completion", openai_package.completion)
monkeypatch.delitem(sys.modules, module_name, raising=False)
LLM(model="gpt-4o")
diff --git a/lib/crewai/tests/telemetry/test_tracer_isolation.py b/lib/crewai/tests/telemetry/test_tracer_isolation.py
index 3783988f7..645ea5645 100644
--- a/lib/crewai/tests/telemetry/test_tracer_isolation.py
+++ b/lib/crewai/tests/telemetry/test_tracer_isolation.py
@@ -118,6 +118,32 @@ def test_a_feature_span_keeps_only_what_the_feature_may_send(telemetry_with_expo
assert deploy_span.attributes["feature"] == "cli_usage:deploy"
+def test_the_models_span_carries_the_models_compared_and_nothing_else(
+ telemetry_with_exporter,
+):
+ telemetry, exporter = telemetry_with_exporter
+ telemetry.set_tracer()
+
+ telemetry.feature_usage_span(
+ "cli_usage:eval_models",
+ {
+ "authenticated": "true",
+ "models": "openai/gpt-4o-mini,anthropic/claude-haiku-4-5",
+ "models_count": "2",
+ "evaluation_id": "ev-1",
+ "organization_id": "org-1",
+ },
+ )
+
+ (span,) = exporter.get_finished_spans()
+ assert span.attributes["feature"] == "cli_usage:eval_models"
+ assert span.attributes["authenticated"] == "true"
+ assert span.attributes["models"] == "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"
+ assert span.attributes["models_count"] == "2"
+ assert "evaluation_id" not in span.attributes
+ assert "organization_id" not in span.attributes
+
+
def test_our_spans_are_unaffected_by_an_application_provider(telemetry_with_exporter):
"""An app that installs its own provider must not divert our telemetry.
diff --git a/lib/crewai/tests/test_llm_overlay.py b/lib/crewai/tests/test_llm_overlay.py
index 0e329bd3d..3642da3be 100644
--- a/lib/crewai/tests/test_llm_overlay.py
+++ b/lib/crewai/tests/test_llm_overlay.py
@@ -1,10 +1,13 @@
-"""`llm_overlay` swaps an agent's model by role, for the calling context only.
+"""`llm_overlay` swaps an agent's model by role, and any LLM's by model, for
+the calling context only.
-The overlay is read in exactly three places: the validators where `Agent` and
-`LiteAgent` resolve their `llm`, and `Agent.interpolate_inputs`, which looks the
+The overlay is read in exactly four places: the validators where `Agent` and
+`LiteAgent` resolve their `llm`, `Agent.interpolate_inputs`, which looks the
interpolated role up again because a templated role only becomes a key once a
-kickoff fills its placeholders in. So these tests build agents, interpolate
-them, and look at the model they end up with. No LLM is ever called.
+kickoff fills its placeholders in, and `LLM.__new__`, where a `model:` key maps
+an LLM built from a model string. So these tests build agents and LLMs,
+interpolate them, and look at the model they end up with. No LLM is ever called
+over the network.
`create_llm("openai/gpt-4o")` returns the native OpenAI provider, which strips
the `openai/` prefix, so the resolved model reads `"gpt-4o"`.
@@ -19,8 +22,14 @@ from typing import Any
from crewai import Agent, Crew, Task
from crewai.lite_agent import LiteAgent
from crewai.llm import LLM
-from crewai.llm_overlay import active, llm_overlay, overlay_model_for
+from crewai.llm_overlay import (
+ MODEL_KEY_PREFIX,
+ active,
+ llm_overlay,
+ overlay_model_for,
+)
from crewai.llms.base_llm import BaseLLM
+from crewai.utilities.llm_utils import create_llm
import pytest
@@ -567,3 +576,315 @@ def test_re_validation_keeps_the_llm_a_kickoff_time_swap_set() -> None:
RuntimeState(root=[agent])
assert agent.llm is swapped
+
+
+# ── model keys: `model:<provider/model>` and `model:*` ───────────────────────
+
+
+def test_the_model_key_prefix_is_a_public_constant() -> None:
+ """Another package feature-detects the model-for-model form on it."""
+ assert MODEL_KEY_PREFIX == "model:"
+
+
+def test_a_bare_llm_call_in_a_flow_step_runs_on_the_mapped_model(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The case roles cannot reach: a flow step's own `LLM(...).call()`."""
+ from crewai.flow.flow import Flow, start
+
+ openai_class = type(LLM(model="openai/gpt-4o-mini"))
+ monkeypatch.setattr(
+ openai_class,
+ "call",
+ lambda self, messages, *args, **kwargs: f"answered by {self.model}",
+ )
+
+ class Poem(Flow): # type: ignore[type-arg]
+ @start()
+ def write(self) -> str:
+ return LLM(model="openai/gpt-5.6-sol").call("a poem")
+
+ with llm_overlay({"model:openai/gpt-5.6-sol": "openai/gpt-4o-mini"}):
+ assert Poem().kickoff() == "answered by gpt-4o-mini"
+
+ assert Poem().kickoff() == "answered by gpt-5.6-sol"
+
+
+def test_a_model_key_matches_with_and_without_the_provider_prefix() -> None:
+ """Native providers strip `openai/`; a key written either way matches both."""
+ with llm_overlay({"model:openai/gpt-4o": "openai/gpt-4.1"}):
+ assert LLM(model="gpt-4o").model == "gpt-4.1"
+ assert LLM(model="openai/gpt-4o").model == "gpt-4.1"
+ assert LLM(model="openai/gpt-4o-mini").model == "gpt-4o-mini"
+ with llm_overlay({"model:gpt-4o": "openai/gpt-4.1"}):
+ assert LLM(model="openai/gpt-4o").model == "gpt-4.1"
+
+
+def test_an_exact_model_key_wins_over_a_stripped_one_and_over_the_wildcard() -> (
+ None
+):
+ with llm_overlay(
+ {
+ "model:gpt-4o": "openai/gpt-4.1-nano",
+ "model:openai/gpt-4o": "openai/gpt-4.1",
+ "model:*": "openai/gpt-4o-mini",
+ }
+ ):
+ assert LLM(model="openai/gpt-4o").model == "gpt-4.1"
+ assert LLM(model="gpt-4o").model == "gpt-4.1-nano"
+ assert LLM(model="openai/o3-mini").model == "gpt-4o-mini"
+
+
+def test_the_wildcard_maps_every_llm_built_from_a_model_string() -> None:
+ with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+ built = [
+ LLM(model="openai/gpt-4o"),
+ create_llm("anthropic/claude-haiku-4-5"),
+ LLM(model="gpt-5.6-sol"),
+ ]
+
+ assert [(type(b).__name__, b.model) for b in built] == [
+ ("OpenAICompletion", "gpt-4o-mini")
+ ] * 3
+ assert LLM(model="openai/gpt-4o").model == "gpt-4o"
+
+
+def test_a_mapped_model_is_not_mapped_again() -> None:
+ """A chain of keys is one step: the model a key maps to is built as it is."""
+ with llm_overlay(
+ {"model:openai/gpt-4o": "openai/gpt-4.1", "model:openai/gpt-4.1": "openai/o3"}
+ ):
+ assert LLM(model="openai/gpt-4o").model == "gpt-4.1"
+
+
+def test_the_caller_settings_follow_the_mapped_model_by_the_declared_llm_rule() -> (
+ None
+):
+ """Generation settings go to any provider; a key and an endpoint only to
+ the provider they were issued for."""
+ with llm_overlay({"model:*": "openai/gpt-4o"}):
+ same = LLM(model="openai/gpt-4o-mini", **CONFIGURATION)
+ with llm_overlay({"model:*": "anthropic/claude-haiku-4-5"}):
+ other = LLM(model="openai/gpt-4o-mini", **CONFIGURATION)
+
+ assert type(same).__name__ == "OpenAICompletion" and same.model == "gpt-4o"
+ assert _configuration_of(same) == CONFIGURATION
+ assert type(other).__name__ == "AnthropicCompletion"
+ assert other.model == "claude-haiku-4-5"
+ assert other.timeout == 42 and other.temperature == 0.1
+ assert other.max_tokens == 77
+ assert other.api_key != "k" and other.base_url is None
+
+
+def test_a_model_mapped_onto_litellm_is_not_initialized_again() -> None:
+ """`LLM.__new__` returning an `LLM` makes Python call `__init__` with the
+ caller's arguments; the mapped instance must keep the mapped model."""
+ pytest.importorskip("litellm")
+ with llm_overlay({"model:*": "groq/llama-3.1-8b-instant"}):
+ built = LLM(model="openai/gpt-4o-mini", temperature=0.3, api_key="k")
+
+ assert type(built).__name__ == "LLM" and built.is_litellm
+ assert built.model == "groq/llama-3.1-8b-instant"
+ assert built.temperature == 0.3 and built.api_key != "k"
+
+
+def test_an_agent_declared_with_a_model_string_runs_on_the_mapped_model() -> None:
+ with llm_overlay({"model:openai/gpt-4o-mini": "openai/gpt-4o"}):
+ agent = _agent("Writer")
+
+ assert agent.llm.model == "gpt-4o"
+
+
+def test_an_agent_whose_llm_was_built_outside_the_block_is_mapped_by_its_model() -> (
+ None
+):
+ """The declared instance is looked up by its model, with the configuration
+ carried like a role's swap."""
+ declared = _configured_llm()
+ with llm_overlay({"model:openai/gpt-4o-mini": "openai/gpt-4o"}):
+ agent = Agent(role="Writer", goal="g", backstory="b", llm=declared)
+ lite = LiteAgent(role="Writer", goal="g", backstory="b", llm=declared)
+
+ for built in (agent.llm, lite.llm):
+ assert built is not declared and built.model == "gpt-4o"
+ assert _configuration_of(built) == CONFIGURATION
+
+
+def test_a_role_key_wins_over_model_keys_for_that_agent() -> None:
+ with llm_overlay(
+ {
+ "Researcher": "openai/gpt-4o",
+ "model:*": "openai/gpt-4.1-nano",
+ }
+ ):
+ researcher = _agent("Researcher")
+ writer = _agent("Writer")
+ outside = Agent(
+ role="Researcher", goal="g", backstory="b", llm=_configured_llm()
+ )
+
+ assert researcher.llm.model == "gpt-4o"
+ assert writer.llm.model == "gpt-4.1-nano"
+ assert outside.llm.model == "gpt-4o"
+
+
+def test_a_role_that_looks_like_a_model_key_is_never_one() -> None:
+ with llm_overlay({"model:*": "openai/gpt-4o"}):
+ assert overlay_model_for("model:*") is None
+
+
+def test_a_model_key_that_names_no_model_is_refused() -> None:
+ with pytest.raises(ValueError, match="names no model"):
+ with llm_overlay({"model: ": "openai/gpt-4o"}):
+ pass
+
+
+def test_a_subclass_of_llm_keeps_its_model() -> None:
+ """Only `LLM` itself routes; a subclass is the caller's own choice of class."""
+ pytest.importorskip("litellm")
+
+ class Mine(LLM):
+ pass
+
+ with llm_overlay({"model:*": "openai/gpt-4o"}):
+ mine = Mine(model="groq/llama-3.1-8b-instant")
+
+ assert type(mine) is Mine and mine.model == "groq/llama-3.1-8b-instant"
+
+
+def test_a_declared_model_whose_sdk_is_missing_still_maps(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Nothing is built on the declared model, so its SDK need not be here; its
+ key cannot be matched to the new provider, so it stays behind."""
+ native = LLM._get_native_provider.__func__ # type: ignore[attr-defined]
+
+ def without_anthropic(cls: type[LLM], provider: str) -> Any:
+ if provider in ("anthropic", "claude"):
+ raise ImportError("Anthropic native provider not available")
+ return native(cls, provider)
+
+ monkeypatch.setattr(LLM, "_get_native_provider", classmethod(without_anthropic))
+ with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+ built = LLM(model="anthropic/claude-haiku-4-5", api_key="k", temperature=0.2)
+
+ assert type(built).__name__ == "OpenAICompletion" and built.model == "gpt-4o-mini"
+ assert built.temperature == 0.2 and built.api_key != "k"
+
+
+def test_a_litellm_routed_llm_keeps_its_key_on_the_same_provider_only() -> None:
+ """Same provider is a question about the provider, not the class: an
+ `LLM(...)` the caller routed through LiteLLM keeps its key and endpoint
+ when mapped to another model of that provider, and another provider's
+ model never gets them."""
+ pytest.importorskip("litellm")
+ declared = {"api_key": "k", "base_url": "http://localhost:9999/v1"}
+ with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+ same = LLM(model="openai/gpt-4o", is_litellm=True, **declared)
+ with llm_overlay({"model:*": "anthropic/claude-haiku-4-5"}):
+ other = LLM(model="openai/gpt-4o", is_litellm=True, **declared)
+
+ assert "gpt-4o-mini" in same.model
+ assert same.api_key == "k" and same.base_url == "http://localhost:9999/v1"
+ assert "claude-haiku-4-5" in other.model
+ assert other.api_key != "k" and other.base_url != "http://localhost:9999/v1"
+
+
+ ("built", "key"),
+ [
+ ("openrouter/openai/gpt-4o", "model:openai/gpt-4o"),
+ ("openrouter/openai/gpt-4o", "model:gpt-4o"),
+ ],
+)
+def test_an_aggregators_route_is_never_the_native_model_it_names(
+ built: str, key: str
+) -> None:
+ """`openrouter/openai/gpt-4o` is OpenRouter's model, not OpenAI's: a key
+ written for the native model leaves it alone."""
+ with llm_overlay({key: "openai/gpt-4.1"}):
+ llm = LLM(model=built)
+
+ assert type(llm).__name__ == "OpenAICompatibleCompletion"
+ assert llm.model == "openai/gpt-4o"
+
+
+def test_an_aggregators_route_is_matched_by_its_own_full_name() -> None:
+ declared = LLM(model="openrouter/openai/gpt-4o")
+ with llm_overlay({"model:openrouter/openai/gpt-4o": "openai/gpt-4.1"}):
+ built = LLM(model="openrouter/openai/gpt-4o")
+ agent = Agent(role="Writer", goal="g", backstory="b", llm=declared)
+ with llm_overlay({"model:openai/gpt-4o": "openai/gpt-4.1"}):
+ untouched = Agent(role="Writer", goal="g", backstory="b", llm=declared)
+
+ assert built.model == "gpt-4.1" and agent.llm.model == "gpt-4.1"
+ assert untouched.llm is declared
+
+
+def test_a_native_provider_prefix_is_still_its_own() -> None:
+ with llm_overlay({"model:llama3": "ollama/qwen3"}):
+ assert LLM(model="ollama/llama3").model == "qwen3"
+
+
+def test_a_role_key_is_built_from_the_declaration_a_model_key_mapped() -> None:
+ """Role wins, with the caller's declared settings: the llm a model key
+ swapped on the way in is not what the role's model is built like."""
+ overlay = {"Researcher": "openai/gpt-4o", "model:*": "anthropic/claude-haiku-4-5"}
+ with llm_overlay(overlay):
+ declared = _configured_llm() # mapped to Anthropic, without the key
+ researcher = Agent(role="Researcher", goal="g", backstory="b", llm=declared)
+ by_string = Agent(
+ role="Researcher", goal="g", backstory="b", llm="openai/gpt-4o-mini"
+ )
+ writer = Agent(role="Writer", goal="g", backstory="b", llm=declared)
+
+ assert type(declared).__name__ == "AnthropicCompletion"
+ assert type(researcher.llm).__name__ == "OpenAICompletion"
+ assert researcher.llm.model == "gpt-4o"
+ assert _configuration_of(researcher.llm) == CONFIGURATION
+ assert by_string.llm.model == "gpt-4o"
+ assert writer.llm is declared
+
+
+def test_a_providers_other_name_is_the_same_model_on_both_paths(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """`google/x` routes as `gemini/x`, so a key written either way matches an
+ llm built in the block and one built before it, whichever name it used."""
+ pytest.importorskip("google.genai")
+ monkeypatch.setenv("GEMINI_API_KEY", "g")
+ outside = LLM(model="google/gemini-2.5-pro") # records provider "gemini"
+ for key in ("model:google/gemini-2.5-pro", "model:gemini/gemini-2.5-pro"):
+ with llm_overlay({key: "openai/gpt-4o-mini"}):
+ built = [LLM(model="google/gemini-2.5-pro"), LLM(model="gemini/gemini-2.5-pro")]
+ agent = Agent(role="Writer", goal="g", backstory="b", llm=outside)
+
+ assert [b.model for b in built] == ["gpt-4o-mini", "gpt-4o-mini"], key
+ assert agent.llm.model == "gpt-4o-mini", key
+
+
+def test_an_aggregator_route_under_an_alias_still_is_not_the_native_model() -> None:
+ with llm_overlay({"model:google/gemini-2.5-pro": "openai/gpt-4o-mini"}):
+ llm = LLM(model="openrouter/google/gemini-2.5-pro")
+
+ assert llm.model == "google/gemini-2.5-pro"
+
+
+def test_a_caller_who_chose_litellm_keeps_it_when_the_declared_sdk_is_missing(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ pytest.importorskip("litellm")
+ native = LLM._get_native_provider.__func__ # type: ignore[attr-defined]
+
+ def without_anthropic(cls: type[LLM], provider: str) -> Any:
+ if provider in ("anthropic", "claude"):
+ raise ImportError("Anthropic native provider not available")
+ return native(cls, provider)
+
+ monkeypatch.setattr(LLM, "_get_native_provider", classmethod(without_anthropic))
+ with llm_overlay({"model:*": "openai/gpt-4o-mini"}):
+ chosen = LLM(model="anthropic/claude-haiku-4-5", is_litellm=True)
+ default = LLM(model="anthropic/claude-haiku-4-5")
+
+ assert type(chosen).__name__ == "LLM" and chosen.is_litellm
+ assert type(default).__name__ == "OpenAICompletion"
diff --git a/pyproject.toml b/pyproject.toml
index 5ad51f65e..97a0e38b6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -209,6 +209,9 @@ exclude-newer-package = { msgpack = "2026-06-20T00:00:00Z", pydantic-settings =
# global 3-day cutoff is now far later than that date, so the per-package pin only blocked the fix.
# pyasn1 <0.6.4 has GHSA-8ppf-4f7h-5ppj and GHSA-hm4w-wwcw-mr6r; force 0.6.4+.
# urllib3 <2.7.0 has GHSA-qccp-gfcp-xxvc (ProxyManager cross-origin redirect leaks Authorization/Cookie) and GHSA-mf9v-mfxr-j63j (streaming decompression-bomb bypass); force 2.7.0+.
+# urllib3 <2.8.0 has GHSA-8988-9cw3-xx77 (HTTPS proxy TLS configuration ignored), GHSA-gh4c-6fx4-qh6g (chunked
+# deflate infinite loop) and GHSA-vxq7-64xx-v4gw (unbounded chunk-size line); force 2.8.0+.
+# virtualenv <21.7.13 has PYSEC-2026-4011..4014 (unchecked seed wheels, pyvenv.cfg and activator injection); force 21.7.13+.
# langsmith <0.8.18 has GHSA-3644-q5cj-c5c7 (public prompt manifest deserialization, SSRF/secret disclosure)
# and GHSA-f4xh-w4cj-qxq8; force 0.8.18+.
# authlib <1.6.12 has GHSA-jj8c-mmj3-mmgv (CSRF bypass in cache-based state storage) and PYSEC-2026-188.
@@ -269,7 +272,8 @@ override-dependencies = [
"pillow>=12.3.0",
"langchain-core>=1.3.3,<2",
"langchain-text-splitters>=1.1.2,<2",
- "urllib3>=2.7.0",
+ "urllib3>=2.8.0",
+ "virtualenv>=21.7.13",
"transformers>=5.4.0; python_version >= '3.10'",
"cryptography>=50.0.0",
"pypdf>=6.16.1,<7",
diff --git a/uv.lock b/uv.lock
index b589d1790..ffb2d49cc 100644
--- a/uv.lock
+++ b/uv.lock
@@ -64,8 +64,9 @@ overrides = [
{ name = "starlette", specifier = ">=1.3.1" },
{ name = "torch", specifier = ">=2.13.0" },
{ name = "transformers", marker = "python_full_version >= '3.10'", specifier = ">=5.4.0" },
- { name = "urllib3", specifier = ">=2.7.0" },
+ { name = "urllib3", specifier = ">=2.8.0" },
{ name = "uv", specifier = ">=0.11.15,<1" },
+ { name = "virtualenv", specifier = ">=21.7.13" },
]
[manifest.dependency-groups]
@@ -1571,7 +1572,7 @@ requires-dist = [
{ name = "portalocker", specifier = "~=2.7.0" },
{ name = "pydantic", specifier = ">=2.11.9,<2.13" },
{ name = "pydantic-settings", specifier = ">=2.14.2,<3" },
- { name = "pyjwt", specifier = ">=2.13.0,<3" },
+ { name = "pyjwt", specifier = ">=2.15.0,<3" },
{ name = "python-dotenv", specifier = ">=1.2.2,<2" },
{ name = "pyyaml", specifier = "~=6.0" },
{ name = "qdrant-client", extras = ["fastembed"], marker = "extra == 'qdrant'", specifier = "~=1.14.3" },
@@ -1618,7 +1619,7 @@ requires-dist = [
{ name = "packaging", specifier = ">=23.0" },
{ name = "pydantic", specifier = ">=2.11.9,<2.13" },
{ name = "pydantic-settings", specifier = ">=2.14.2,<3" },
- { name = "pyjwt", specifier = ">=2.13.0,<3" },
+ { name = "pyjwt", specifier = ">=2.15.0,<3" },
{ name = "python-dotenv", specifier = ">=1.2.2,<2" },
{ name = "rich", specifier = ">=13.7.1" },
{ name = "textual", specifier = ">=7.5.0" },
@@ -1656,7 +1657,7 @@ requires-dist = [
{ name = "packaging", specifier = ">=23.0" },
{ name = "portalocker", specifier = "~=2.7.0" },
{ name = "pydantic", specifier = ">=2.11.9,<2.13" },
- { name = "pyjwt", specifier = ">=2.13.0,<3" },
+ { name = "pyjwt", specifier = ">=2.15.0,<3" },
{ name = "rich", specifier = ">=13.7.1" },
{ name = "tomli", specifier = "~=2.0.2" },
]
@@ -7353,14 +7354,14 @@ wheels = [
[[package]]
name = "pyjwt"
-version = "2.13.0"
+version = "2.15.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/02/a5/5197bfd06417837ac079921c66fa6393f1dea3557272a263cebfef69e432/pyjwt-2.15.0.tar.gz", hash = "sha256:b11c5f9791d7bf51c2b39a81ed669f6b2dbbd669df2942f6c60167e9e3d1abe4", size = 120513, upload-time = "2026-09-23T16:56:00.689Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/55/40e45bf052ee8ee12a4dfd785519660f8effa7b065442b91646ec6828619/pyjwt-2.15.0-py3-none-any.whl", hash = "sha256:7a3742debf6b879e912dbb9819ceec1594be812452b78c5f2e2dfc56564954f8", size = 33680, upload-time = "2026-09-23T16:55:59.241Z" },
]
[package.optional-dependencies]
@@ -7727,15 +7728,14 @@ wheels = [
[[package]]
name = "python-discovery"
-version = "1.4.0"
+version = "1.6.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filelock" },
- { name = "platformdirs" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/a6/12/38c1a0b1e64806780c9563e3fc9f6e472251839662587cfbe9bfaf2ae10a/python_discovery-1.4.0.tar.gz", hash = "sha256:eb8bc7daad3c226c147e45bb4e970a1feb1bf4048ee178e6db59e197b8010ce3", size = 68455, upload-time = "2026-05-28T01:15:37.639Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/0c/57/250bd238b966cece44328235eb85290045d059265fdaf7527a3a958123db/python_discovery-1.6.1.tar.gz", hash = "sha256:cf87d3627dfb4412437fdd5b13eae402607722998d21567993aedbc59b23c15e", size = 84338, upload-time = "2026-09-18T01:31:53.971Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/c8/8d/3d316429f65029532bb1e28ff77b797d86b5ac3915bb44ca4e19aa283d43/python_discovery-1.4.0-py3-none-any.whl", hash = "sha256:26ed78d703e234879a66244c7d4114563fb13ec5cd30a2d1357e5fb4850782da", size = 33217, upload-time = "2026-05-28T01:15:36.573Z" },
+ { url = "https://files.pythonhosted.org/packages/16/7d/e9ffbadfbf89c93848412d04594135c4ae8c1d37d9e053b9c3ed718fabc4/python_discovery-1.6.1-py3-none-any.whl", hash = "sha256:d43fcdef879fe795352bd13ccf8d185ba5a9f86f36cfcd00529f596e737442b3", size = 38664, upload-time = "2026-09-18T01:31:52.448Z" },
]
[[package]]
@@ -10152,11 +10152,11 @@ wheels = [
[[package]]
name = "urllib3"
-version = "2.7.0"
+version = "2.8.0"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" },
+ { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" },
]
[[package]]
@@ -10351,7 +10351,7 @@ wheels = [
[[package]]
name = "virtualenv"
-version = "21.4.2"
+version = "21.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "distlib" },
@@ -10360,9 +10360,9 @@ dependencies = [
{ name = "python-discovery" },
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/e1/0d/4e93c8e6d1001a75763f87d8f5ecda8ebc7f4aa2153dddfaf4ae8892821a/virtualenv-21.4.2.tar.gz", hash = "sha256:38e6ee0a555615c0ea9da2ac7e9998fe8dc3b911dd33ad8eaad2020957653b0c", size = 7613326, upload-time = "2026-05-31T17:01:22.827Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/31/0b/825cbfd46beb2cc96c46403141081190c1afd1a348d0232a4ebcb6dcd362/virtualenv-21.13.0.tar.gz", hash = "sha256:e8aa144aabba43ffd9058e1fd1a02012b1915f8cb915acc16ad50aba9c13bfdc", size = 5399552, upload-time = "2026-09-26T16:42:09.321Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/bf/c4/557dc082be035381b85fdb2b74e21d3d21b57750b74f2b47a32f3a639ff9/virtualenv-21.4.2-py3-none-any.whl", hash = "sha256:854210ca524a1a4d0d744734f4acbc721c3ffe163b85bbf5d56d14d5ae2f0fae", size = 7594079, upload-time = "2026-05-31T17:01:20.735Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/2c/8b107f9e029da006c39af38a0a150dde57ecce48b495c79917d1ad963821/virtualenv-21.13.0-py3-none-any.whl", hash = "sha256:ca16ceba89e58d8e2f376769349c63f3ed66f79881ae68bc13e19a9d50e0e987", size = 5486498, upload-time = "2026-09-26T16:42:07.106Z" },
]
[[package]]