Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(cli): crewai eval --models, and llm_overlay swaps models as well as roles (#7811)
João Moura1b9bbccbed63
diff --git a/.github/workflows/vulnerability-scan.yml b/.github/workflows/vulnerability-scan.ymlindex 4052f1e77..8d853b662 100644--- a/.github/workflows/vulnerability-scan.yml+++ b/.github/workflows/vulnerability-scan.yml@@ -117,6 +117,16 @@ jobs:# patched release; keep the ignore list in sync with# .pre-commit-config.yaml.--ignore-vuln GHSA-4j2p-28q2-5m79+ # oauthlib <4.0.0: GHSA-xpv3-w29h-x7cv: timing side channel in the+ # server-side PKCE code_verifier check (authorization_code grant).+ # The fix, 4.0.0 (published 2026-09-28), is inside the 3-day+ # exclude-newer cooldown. Transitive via chromadb -> kubernetes ->+ # requests-oauthlib, which uses oauthlib as an OAuth client only;+ # CrewAI never runs an oauthlib authorization server.+ # TODO: drop this ignore and run `uv lock --upgrade-package oauthlib`+ # once 4.0.0 clears the cooldown (after 2026-10-01 06:01 UTC); keep+ # the ignore list in sync with .pre-commit-config.yaml.+ --ignore-vuln GHSA-xpv3-w29h-x7cv)uv run pip-audit "${pip_audit_args[@]}"continue-on-error: truediff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 64ec22459..cffd7dc99 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -31,6 +31,7 @@ repos:# Keep this ignore list in sync with .github/workflows/vulnerability-scan.yml.# TODO: drop --ignore-vuln GHSA-8mgp-746c-j5xp when bumping nltk past 3.10.3.# TODO: drop --ignore-vuln GHSA-4j2p-28q2-5m79 when bumping accelerate past 1.14.0.+ # TODO: drop --ignore-vuln GHSA-xpv3-w29h-x7cv when bumping oauthlib to 4.0.0 (after 2026-10-01 06:01 UTC).entry: >-bash -c 'case "$OSTYPE" in msys*|cygwin*|win32*) source .venv/Scripts/activate ;; *) source .venv/bin/activate ;; esac && uv run pip-audit --skip-editable--ignore-vuln PYSEC-2024-277@@ -63,7 +64,8 @@ repos:--ignore-vuln GHSA-36p7-vc44-83pf--ignore-vuln GHSA-xph7-9rjv-w5fr--ignore-vuln GHSA-8mgp-746c-j5xp- --ignore-vuln GHSA-4j2p-28q2-5m79' --+ --ignore-vuln GHSA-4j2p-28q2-5m79+ --ignore-vuln GHSA-xpv3-w29h-x7cv' --language: systempass_filenames: falsestages: [pre-push, manual]diff --git a/lib/cli/pyproject.toml b/lib/cli/pyproject.tomlindex 84188fdb2..a48c761ef 100644--- a/lib/cli/pyproject.toml+++ b/lib/cli/pyproject.toml@@ -15,7 +15,7 @@ dependencies = ["appdirs~=1.4.4","cryptography>=42.0","httpx~=0.28.1",- "pyjwt>=2.13.0,<3",+ "pyjwt>=2.15.0,<3","rich>=13.7.1","tomli~=2.0.2","tomli-w~=1.1.0",diff --git a/lib/cli/src/crewai_cli/cli.py b/lib/cli/src/crewai_cli/cli.pyindex 04199729e..7323e8207 100644--- a/lib/cli/src/crewai_cli/cli.py+++ b/lib/cli/src/crewai_cli/cli.py@@ -54,6 +54,12 @@ def eval_crew(*args: Any, **kwargs: Any) -> Any:return _eval_crew(*args, **kwargs)+def eval_models(*args: Any, **kwargs: Any) -> Any:+ from crewai_cli.experimental.eval_crew import eval_models as _eval_models++ return _eval_models(*args, **kwargs)++if TYPE_CHECKING:# mypy sees the real classes; at runtime the shims below defer the# heavy imports until a command actually instantiates them.@@ -692,12 +698,52 @@ def run("crewAI recorded for the run."),)-def eval_command(run_id: str | None) -> None:- """Evaluate the last traced run through CrewAI AMP.+ "--models",+ "models",+ type=str,+ default=None,+ metavar="LIST",+ help=(+ "Compare models on this project's deployment instead: ONE comma-separated "+ 'list of provider/model, e.g. "openai/gpt-4o-mini,anthropic/claude-haiku-4-5". '+ "The deployment runs once as deployed and once per model; needs `crewai login`."+ ),+)+ "--deployment",+ "deployment_id",+ type=str,+ default=None,+ metavar="UUID",+ help=(+ "With --models: the deployment to run, when AMP cannot tell it from the "+ "project id."+ ),+)+def eval_command(+ run_id: str | None, models: str | None, deployment_id: str | None+) -> None:+ """Evaluate the last traced run through CrewAI AMP, or compare models on the+ project's deployment (--models).- Exits 0 only when the goal gate PASSED, and 1 otherwise — a failed gate, no- verdict, or an evaluation that could not run — so a CI job can gate on it.+ A run's evaluation exits 0 only when the goal gate PASSED, and 1 otherwise — a+ failed gate, no verdict, or an evaluation that could not run — so a CI job can+ gate on it. A comparison exits 0 when it finished, 1 when it failed or could+ not start."""+ if models is not None:+ if run_id is not None:+ raise click.UsageError(+ "--run grades a run that already happened; --models runs the "+ "deployment again. Give one of them."+ )+ eval_models(models, deployment_id=deployment_id)+ return+ if deployment_id is not None:+ raise click.UsageError(+ "--deployment names the deployment --models runs; add --models LIST."+ )eval_crew(run_id=run_id)diff --git a/lib/cli/src/crewai_cli/deploy/main.py b/lib/cli/src/crewai_cli/deploy/main.pyindex 17e1f1249..85d652cfa 100644--- a/lib/cli/src/crewai_cli/deploy/main.py+++ b/lib/cli/src/crewai_cli/deploy/main.py@@ -15,7 +15,11 @@ from crewai_cli.command import BaseCommand, PlusAPIMixinfrom crewai_cli.constants import DEFAULT_CREWAI_ENTERPRISE_URLfrom crewai_cli.deploy.archive import ArchiveError, create_project_zipfrom crewai_cli.deploy.validate import DeployValidator, Severity, render_report-from crewai_cli.utils import fetch_and_json_env_file, get_project_name+from crewai_cli.utils import (+ fetch_and_json_env_file,+ get_project_id,+ get_project_name,+)console = Console()@@ -232,6 +236,14 @@ class DeployCommand(BaseCommand, PlusAPIMixin):BaseCommand.__init__(self)PlusAPIMixin.__init__(self, telemetry=self._telemetry)self.project_name = get_project_name(require=True)+ # Sent with every create and push, so AMP knows which project a+ # deployment runs — `crewai eval --models` finds the deployment by it.+ # Read, never minted: a deploy does not rewrite pyproject.toml.+ self.project_id = get_project_id()++ def _project(self) -> dict[str, str]:+ """`project_id=` for a create or push request, when the project has one."""+ return {"project_id": self.project_id} if self.project_id else {}def _standard_no_param_error_message(self) -> None:"""@@ -424,10 +436,10 @@ class DeployCommand(BaseCommand, PlusAPIMixin):env_vars = fetch_and_json_env_file()return self._update_crew_from_zip(deployment_uuid, repository, env_vars)if uuid:- return self.plus_api_client.deploy_by_uuid(uuid)+ return self.plus_api_client.deploy_by_uuid(uuid, **self._project())if not project_name:raise ValueError("project_name is required to deploy by name")- return self.plus_api_client.deploy_by_name(project_name)+ return self.plus_api_client.deploy_by_name(project_name, **self._project())def _deploy_from_local_source(self,@@ -438,9 +450,9 @@ class DeployCommand(BaseCommand, PlusAPIMixin):) -> Any | None:"""Deploy using local origin, as before AMP zip_deployment existed."""if remote_repo_url and uuid:- return self.plus_api_client.deploy_by_uuid(uuid)+ return self.plus_api_client.deploy_by_uuid(uuid, **self._project())if remote_repo_url and project_name:- return self.plus_api_client.deploy_by_name(project_name)+ return self.plus_api_client.deploy_by_name(project_name, **self._project())if uuid:_display_git_remote_help()env_vars = fetch_and_json_env_file()@@ -627,6 +639,7 @@ class DeployCommand(BaseCommand, PlusAPIMixin):zip_file_path,name=self.project_name,env=env_vars,+ **self._project(),)finally:zip_file_path.unlink(missing_ok=True)@@ -649,6 +662,7 @@ class DeployCommand(BaseCommand, PlusAPIMixin):uuid,zip_file_path,env=env_vars,+ **self._project(),)finally:zip_file_path.unlink(missing_ok=True)@@ -692,13 +706,16 @@ class DeployCommand(BaseCommand, PlusAPIMixin):"""if not self.project_name:raise ValueError("project_name is required to create a deployment payload")- return {+ payload: CreateCrewPayload = {"deploy": {"name": self.project_name,"repo_clone_url": remote_repo_url,"env": env_vars,}}+ if self.project_id:+ payload["project_id"] = self.project_id+ return payloaddef _display_creation_success(self, json_response: dict[str, Any]) -> None:"""diff --git a/lib/cli/src/crewai_cli/experimental/eval_crew.py b/lib/cli/src/crewai_cli/experimental/eval_crew.pyindex 1af0f69fc..4eb80b851 100644--- a/lib/cli/src/crewai_cli/experimental/eval_crew.py+++ b/lib/cli/src/crewai_cli/experimental/eval_crew.py@@ -10,6 +10,13 @@ Who may evaluate what is AMP's decision: an anonymous run once without anaccount, then it needs one; a run traced while logged in for thatorganization's members; a deployment execution for members who may see itstraces. The command sends the saved `crewai login` when there is one.++`crewai eval --models "a,b"` compares models instead: AMP finds the project's+deployment by `[tool.crewai].project_id` (or `--deployment`), which runs once as+deployed and once per model; each run is graded, and the comparison — the four+grades, cost and time per model, and what would make the agents do better — is+printed when it is done. It always needs the login: it runs the deployment.+Every evaluation, of either kind, is filed under the project id."""from __future__ import annotations@@ -23,8 +30,9 @@ import osfrom pathlib import Pathimport sysimport time-from typing import Any+from typing import Any, NoReturn, castfrom urllib.parse import urlparse+import uuidimport webbrowserimport click@@ -33,11 +41,16 @@ from crewai_core.settings import Settingsfrom dotenv import load_dotenv, set_keyimport httpxfrom rich.console import Console+from rich.table import Tablefrom rich.text import Textfrom crewai_cli.authentication.token import AuthError, get_auth_tokenfrom crewai_cli.plus_api import PlusAPI-from crewai_cli.utils import get_or_create_project_id, is_dmn_mode_enabled+from crewai_cli.utils import (+ get_or_create_project_id,+ get_project_id,+ is_dmn_mode_enabled,+)console = Console()@@ -107,7 +120,7 @@ def _note(text: str, style: str = "dim") -> None:def eval_crew(run_id: str | None = None) -> None:"""Evaluate the last traced run of this project, or the run RUN_ID."""- get_or_create_project_id()+ project_id = get_or_create_project_id()# Read before the project's .env is loaded, so a project cannot add itself.trusted = _trusted_amp_origins()_load_project_env()@@ -140,6 +153,7 @@ def eval_crew(run_id: str | None = None) -> None:client,execution_id,wait_for_spans=run_id is None and _ran_just_now(record),+ project_id=project_id,)except EvaluationStoppedError as stopped:_fail(str(stopped))@@ -334,7 +348,13 @@ def evaluate_run(for them is always on here."""client = _amp_client(_machine_amp_origins(), note=note)- started = _start_evaluation(client, execution_id, wait_for_spans=True, note=note)+ started = _start_evaluation(+ client,+ execution_id,+ wait_for_spans=True,+ note=note,+ project_id=get_project_id(),+ )# After the start, exactly as the command counts it: `cli_usage:eval` is the# count of evaluations that began, and an evaluation the app runs is one._record_usage(logged_in=client.api_key is not None)@@ -622,6 +642,7 @@ def _start_evaluation(*,wait_for_spans: bool = False,note: Callable[[str], None] = _note,+ project_id: str | None = None,) -> dict[str, Any]:deadline = time.monotonic() + SPANS_WAIT_SECONDS if wait_for_spans else 0.0said = False@@ -630,7 +651,9 @@ def _start_evaluation(eval_config = project_eval_config(note=note)while True:try:- response = client.create_evaluation(execution_id, eval_config=eval_config)+ response = client.create_evaluation(+ execution_id, eval_config=eval_config, project_id=project_id+ )except httpx.HTTPError as error:raise EvaluationStoppedError(f"Could not reach AMP to start the evaluation: {error}"@@ -647,6 +670,17 @@ def _start_evaluation(time.sleep(SPANS_POLL_SECONDS)continuebreak+ return _accepted(response, f"run {execution_id}", note=note)+++def _accepted(+ response: httpx.Response,+ subject: str,+ *,+ note: Callable[[str], None] = _note,+ about_a_deployment: bool = False,+) -> dict[str, Any]:+ """AMP's answer to a start: the evaluation it opened, or its refusal as a sentence."""if response.status_code in (200, 202):payload = _payload(response)# The id is what every later call is made with, so a missing or@@ -666,7 +700,9 @@ def _start_evaluation(raise EvaluationStoppedError(f"AMP answered without an evaluation id ({response.status_code}).")- raise EvaluationStoppedError(_refusal_message(response, f"run {execution_id}"))+ raise EvaluationStoppedError(+ _refusal_message(response, subject, about_a_deployment=about_a_deployment)+ )def _report_url(value: Any) -> str | None:@@ -705,12 +741,16 @@ def _wait(url: str | None,*,on_status: Callable[[dict[str, Any]], None] | None = None,+ answer: str = "verdict",) -> dict[str, Any]:"""Poll until the evaluation is done or failed.Every unfinished answer goes to ON_STATUS, so a caller that has somewhere to- show progress can show it; the caller decides what a Ctrl-C means.+ show progress can show it; the caller decides what a Ctrl-C means. A done+ answer must carry a well-formed ANSWER: a run's `verdict`, or a models+ evaluation's `comparison`."""+ well_formed = _WELL_FORMED[answer]where = f" at {url}" if url else ""subject = f"evaluation {evaluation_id}"misses = (@@ -738,9 +778,9 @@ def _wait(misses = 0payload = _payload(response) or {}status = payload.get("status")- if status == "done" and not _well_formed_verdict(payload.get("verdict")):+ if status == "done" and not well_formed(payload):raise EvaluationStoppedError(- f"AMP answered done without a verdict (protocol error); follow it{where or ' on AMP'}."+ f"AMP answered done without a {answer} (protocol error); follow it{where or ' on AMP'}.")if status in FINISHED:return payload@@ -753,6 +793,10 @@ def _wait(time.sleep(POLL_SECONDS)+def _carries_a_verdict(payload: dict[str, Any]) -> bool:+ return _well_formed_verdict(payload.get("verdict"))++def _well_formed_verdict(verdict: Any) -> bool:"""`{"gate": "<word>", "grades": {area: 1..5 | null}}` — anything else is aprotocol error. A grade is an exact integer in range: `True` is an `int` to@@ -769,6 +813,403 @@ def _a_grade(grade: Any) -> bool:return grade is None or (type(grade) is int and 1 <= grade <= 5)+def _carries_a_comparison(payload: dict[str, Any]) -> bool:+ """`{"comparison": {"models": [{...}, ...]}}` — the rows are read cell by+ cell when printed, and a cell that is not what it should be prints as "—";+ a comparison with no rows to print is a protocol error."""+ comparison = payload.get("comparison")+ return (+ isinstance(comparison, dict)+ and isinstance(comparison.get("models"), list)+ and bool(comparison["models"])+ and all(isinstance(row, dict) for row in comparison["models"])+ )+++_WELL_FORMED: dict[str, Callable[[dict[str, Any]], bool]] = {+ "verdict": _carries_a_verdict,+ "comparison": _carries_a_comparison,+}+++# ── crewai eval --models ─────────────────────────────────────────────────────++MAX_MODELS = 5+MAX_MODEL_CHARS = 200+MODELS_EXAMPLE = '--models "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"'+LOGIN_REQUIRED = (+ "Comparing models runs your deployment, so it needs your CrewAI AMP account: "+ "log in with `crewai login` and run it again."+)+GRADE_COLUMNS = ("goal", "tasks", "agents", "tools")+TOP_SUGGESTIONS = 3+++def parse_models(text: str | None) -> list[str]:+ """The models to compare, from ONE comma-separated list.++ Items are stripped and a repeat is dropped. Each must name its provider+ (`provider/model`) — the deployment builds the model from exactly that+ string, and a bare `gpt-4o` would be a guess about which provider bills it.+ """+ models: list[str] = []+ for raw in (text or "").split(","):+ item = raw.strip()+ if not item or item in models:+ continue+ provider, separator, name = item.partition("/")+ if not (separator and provider.strip() and name.strip()):+ raise EvaluationStoppedError(+ f"{item!r} names no provider: write each model as provider/model, "+ f"e.g. {MODELS_EXAMPLE}."+ )+ if len(item) > MAX_MODEL_CHARS:+ raise EvaluationStoppedError(+ f"{item[:40]!r}… is longer than {MAX_MODEL_CHARS} characters; "+ "that is not a model name."+ )+ models.append(item)+ if not models:+ raise EvaluationStoppedError(+ f"--models names no model: give one to {MAX_MODELS}, e.g. {MODELS_EXAMPLE}."+ )+ if len(models) > MAX_MODELS:+ raise EvaluationStoppedError(+ f"--models names {len(models)} models; compare at most {MAX_MODELS} at once."+ )+ return models+++def _deployment_id(value: str | None) -> str | None:+ """`--deployment` as the UUID AMP knows the deployment by, or None."""+ if value is None:+ return None+ try:+ return str(uuid.UUID(value.strip()))+ except ValueError:+ raise EvaluationStoppedError(+ f"--deployment {value!r} is not a deployment id; it is the UUID AMP "+ "shows for the deployment."+ ) from None+++# Model names that are the same for everyone who types them are public. What+# else a `--models` item can carry is a customer's: a fine-tune id+# (`openai/ft:gpt-4o-mini:acme-corp::abc`), an Azure deployment name, a+# self-hosted model or host. Those are sent as `<provider>/other`.+OTHER_MODEL = "other"+# The vendor segment of an aggregator id (`openrouter/openai/gpt-4o-mini`)+# that is itself public.+_PUBLIC_VENDORS = frozenset(+ {"openai", "anthropic", "google", "meta-llama", "mistralai", "deepseek", "qwen"}+)+++def telemetry_model_name(model: str) -> str:+ """MODEL as the usage stats may carry it: as typed when it is a model crewAI+ knows, else `<provider>/other` — and `other/other` for a provider crewAI does+ not know, since a provider string can name a host too.++ Known means an exact entry of crewAI's model catalog (the context-window+ tables every provider resolves against), never a prefix of one: a fine-tune+ or a deployment named after a public model is still the customer's.+ """+ catalog, providers = _known_models_and_providers()+ provider, _, name = model.partition("/")+ if provider not in providers:+ return f"{OTHER_MODEL}/{OTHER_MODEL}"+ vendor, nested, tail = name.partition("/")+ public = (+ name in catalog+ or model in catalog+ or (nested and vendor in _PUBLIC_VENDORS and tail in catalog)+ )+ if not public or any(part.startswith("ft:") for part in model.split("/")):+ return f"{provider}/{OTHER_MODEL}"+ return model+++def _known_models_and_providers() -> tuple[frozenset[str], frozenset[str]]:+ """crewAI's catalog of models and the providers it routes; empty when this+ environment's crewai cannot say, so every model is then sent as "other"."""+ from crewai_cli.constants import PROVIDERS++ try:+ from crewai.llm import SUPPORTED_NATIVE_PROVIDERS+ from crewai.llms.context_window import LLM_CONTEXT_WINDOW_SIZES+ except Exception:+ return frozenset(), frozenset()+ return (+ frozenset(LLM_CONTEXT_WINDOW_SIZES),+ frozenset(SUPPORTED_NATIVE_PROVIDERS) | frozenset(PROVIDERS),+ )+++def _record_models_usage(models: list[str]) -> None:+ """Count a comparison that is actually starting, and which models it compares.++ The models go through `telemetry_model_name`: a public model by name, any+ other as `<provider>/other`. Nothing names the run, the deployment or the+ organization. Always logged in: a comparison cannot start without the+ account.+ """+ try:+ from crewai_core.telemetry import Telemetry++ telemetry = Telemetry()+ telemetry.set_tracer()+ telemetry.feature_usage_span(+ "cli_usage:eval_models",+ {+ "authenticated": "true",+ "models": ",".join(telemetry_model_name(m) for m in models),+ "models_count": str(len(models)),+ },+ )+ except Exception: # noqa: S110 - telemetry must never break a command+ pass+++def eval_models(models_text: str, deployment_id: str | None = None) -> None:+ """Run this project's deployment once as deployed and once per model, and compare.++ The deployment is AMP's to find, by the project id, unless DEPLOYMENT_ID+ names one; its own models are the baseline, read off the deployment rather+ than this checkout, which may differ from what was deployed.+ """+ try:+ models = parse_models(models_text)+ deployment = _deployment_id(deployment_id)+ except EvaluationStoppedError as stopped:+ _fail(str(stopped))+ if not Path("pyproject.toml").is_file():+ _fail(+ "No crewAI project here (no pyproject.toml). Run `crewai eval --models` "+ "from the directory of the project you deployed."+ )+ project_id = get_or_create_project_id()+ if not project_id:+ _fail(+ "Could not read or write [tool.crewai].project_id in pyproject.toml, which "+ "is how AMP finds this project's deployment."+ )+ # Read before the project's .env is loaded, so a project cannot add itself.+ trusted = _trusted_amp_origins()+ _load_project_env()+ try:+ client = _amp_client(trusted)+ except EvaluationStoppedError as stopped:+ _fail(str(stopped))+ if client.api_key is None:+ _fail(LOGIN_REQUIRED)++ try:+ response = client.create_models_evaluation(+ models,+ project_id=project_id,+ eval_config=project_eval_config(),+ deployment_id=deployment,+ )+ except httpx.HTTPError as error:+ _fail(f"Could not reach AMP to start the comparison: {error}")+ try:+ started = _accepted(response, "the comparison", about_a_deployment=True)+ except EvaluationStoppedError as stopped:+ _fail(str(stopped))+ # After, not before, as `cli_usage:eval` is counted: a refused request —+ # no deployment, one this account may not run — is not a comparison.+ _record_models_usage(models)+ url = started.get("url")+ console.print(+ Text("Comparing ")+ .append(", ".join(models), style="bold")+ .append(" with the deployed models")+ )+ if url:+ # Appended, never interpolated — the same reason as `eval_crew`'s link.+ console.print(Text("Follow it at ").append(url, style="cyan underline"))+ _open(url)++ console.print("Waiting for the comparison…", style="dim")+ shown: list[str] = []++ def show_progress(payload: dict[str, Any]) -> None:+ line = _progress_line(payload)+ if line and (not shown or shown[-1] != line):+ shown.append(line)+ _note(line)++ try:+ finished = _wait(+ client, started["id"], url, on_status=show_progress, answer="comparison"+ )+ except EvaluationStoppedError as stopped:+ _fail(str(stopped))+ except KeyboardInterrupt:+ console.print(+ Text(f"\nStill running{f' at {url}' if url else ''}."), style="yellow"+ )+ raise SystemExit(130) from None+ _print_comparison(finished, url)+ # The criteria the comparison was graded on, for the project to edit — only+ # when it has none, exactly as after a Mode 1 evaluation.+ _say_where_the_criteria_live(write_eval_config(finished))+ if finished.get("status") != "done":+ raise SystemExit(1)+++def _progress_line(payload: dict[str, Any]) -> str | None:+ """One line for where the comparison is, from the poll's `progress` (or the+ last of its `events`): which model is running, "model 2 of 3", and the+ subject being judged. What is not there is left out, never guessed."""+ progress = payload.get("progress")+ events = payload.get("events")+ if progress is None and isinstance(events, list) and events:+ progress = events[-1]+ if isinstance(progress, str):+ return progress.strip() or None+ if not isinstance(progress, dict):+ return None+ if isinstance(progress.get("message"), str) and progress["message"].strip():+ return str(progress["message"]).strip()+ event = progress.get("event")+ detail = progress.get("payload")+ if not isinstance(detail, dict):+ detail = progress+ index, total = detail.get("index"), detail.get("total")+ position = (+ f"model {index + 1} of {total}"+ if type(index) is int and type(total) is int and 0 <= index < total+ else None+ )+ name = next(+ (+ str(detail[key])+ for key in ("label", "key")+ if isinstance(detail.get(key), str) and detail[key]+ ),+ None,+ )+ subject = detail.get("subject")+ if event == "judging" or isinstance(subject, str):+ parts = [f"judging {subject}" if isinstance(subject, str) else "judging"]+ elif event == "configuration_done":+ parts = ["graded"]+ else:+ parts = ["running"]+ parts += [part for part in (position, name) if part]+ known = event is not None or isinstance(subject, str) or len(parts) > 1+ return " · ".join(parts) if known else None+++def _print_comparison(finished: dict[str, Any], url: str | None) -> None:+ """The models side by side, then what would make them better.++ Every cell came over the wire, so each is a `Text`: a label such as+ `Writer: [red]x[/red]` prints as written. The baseline — the deployment as+ it is — is marked, and the best value in each column is starred: the highest+ grade, the lowest cost and time. A column where every model is the same, or+ only one has a value, stars nothing: there is no "best" to point at.+ """+ if finished.get("status") != "done":+ console.print(+ Text(f"Comparison failed: {finished.get('error') or 'no reason given'}"),+ style="bold red",+ )+ if url:+ console.print(Text(f"Report: {url}"))+ return+ comparison = finished["comparison"] # _wait let only a well-formed one through+ rows: list[dict[str, Any]] = comparison["models"]++ grades = {area: [_grade_of(row, area) for row in rows] for area in GRADE_COLUMNS}+ costs = [_number(row.get("cost_usd")) for row in rows]+ seconds = [_number(row.get("seconds")) for row in rows]+ best = {area: _best(values, max) for area, values in grades.items()}+ cheapest, fastest = _best(costs, min), _best(seconds, min)++ table = Table(show_edge=False, pad_edge=False)+ for header in ("model", *GRADE_COLUMNS, "cost", "time"):+ table.add_column(header, justify="left" if header == "model" else "right")+ for n, row in enumerate(rows):+ label = Text(str(row.get("label") or row.get("key") or "?"))+ if row.get("baseline") is True:+ label.append(" (deployed)", style="dim")+ cells = [label]+ for area in GRADE_COLUMNS:+ grade = grades[area][n]+ cells.append(+ _starred(f"{grade}/5" if grade is not None else "—", grade, best[area])+ )+ cost, took = costs[n], seconds[n]+ cells.append(+ _starred(f"${cost:.4f}" if cost is not None else "—", cost, cheapest)+ )+ cells.append(+ _starred(f"{took:.1f}s" if took is not None else "—", took, fastest)+ )+ table.add_row(*cells)+ console.print(table)++ suggestions = _top_suggestions(comparison.get("suggestions"))+ if suggestions:+ console.print(Text("What would make it better", style="bold"))+ for n, item in enumerate(suggestions, 1):+ where = " — ".join(+ str(item[key])+ for key in ("subject", "field")+ if isinstance(item.get(key), str)+ )+ line = Text(f"{n}. ")+ if where:+ line.append(where, style="bold").append(": ")+ line.append(str(item.get("problem") or ""))+ if item.get("shared") is True:+ line.append(" (every model)", style="dim")+ console.print(line)+ if isinstance(item.get("change"), str) and item["change"].strip():+ console.print(Text(f" change: {item['change'].strip()}"))+ if url:+ console.print(Text(f"Full report: {url}"))+++def _grade_of(row: dict[str, Any], area: str) -> int | None:+ grades = row.get("grades")+ grade = grades.get(area) if isinstance(grades, dict) else None+ return grade if grade is not None and _a_grade(grade) else None+++def _number(value: Any) -> float | None:+ """A non-negative number, or None: `True` is an `int` to Python and not a cost."""+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ return None+ return float(value) if value >= 0 else None+++def _best(values: list[Any], pick: Callable[[list[Any]], Any]) -> float | int | None:+ known = [value for value in values if value is not None]+ if len(known) < 2 or len(set(known)) == 1:+ return None+ return cast(float | int, pick(known))+++def _starred(text: str, value: Any, best: Any) -> Text:+ cell = Text(text)+ if best is not None and value == best:+ cell.append(" ★", style="yellow")+ return cell+++def _top_suggestions(value: Any) -> list[dict[str, Any]]:+ """The first few suggestions, the ones every model needed first — a problem+ every model had points at the prompt, not at a model."""+ if not isinstance(value, list):+ return []+ items = [item for item in value if isinstance(item, dict) and item.get("problem")]+ items.sort(key=lambda item: item.get("shared") is not True)+ return items[:TOP_SUGGESTIONS]++def _print_verdict(finished: dict[str, Any], url: str | None) -> None:"""Everything printed here came over the wire, so it is composed as `Text`and never as markup: a `Console` parses square brackets, and an area named@@ -817,15 +1258,26 @@ def _payload(response: httpx.Response) -> dict[str, Any] | None:return loaded if isinstance(loaded, dict) else None-def _refusal_message(response: httpx.Response, subject: str) -> str:+def _refusal_message(+ response: httpx.Response, subject: str, *, about_a_deployment: bool = False+) -> str:"""AMP's own words when it sent them. SUBJECT is "run <id>" or "evaluation <id>".A sentence, not a print: the terminal and the run app both show it, and only- one of them shows it by printing.+ one of them shows it by printing. ABOUT_A_DEPLOYMENT: a 403 that is not about+ the credential — this account may not run that deployment — is AMP's+ sentence alone, because logging in again changes nothing there."""payload = _payload(response) or {}message = str(payload.get("message") or "").strip()error = str(payload.get("error") or "")+ if (+ about_a_deployment+ and response.status_code == 403+ and error not in {"bad_credentials", "account_required"}+ and message+ ):+ return messageif response.status_code in (401, 403):if error == "account_required" and message:return message@@ -843,7 +1295,7 @@ def _refused(response: httpx.Response, subject: str) -> None:_fail(_refusal_message(response, subject))-def _fail(message: str) -> None:+def _fail(message: str) -> NoReturn:# `Text`, because most of what reaches here is AMP's own sentence and a# `Console` parses square brackets. No caller relies on markup; the colour# comes from `style`.diff --git a/lib/cli/src/crewai_cli/plus_api.py b/lib/cli/src/crewai_cli/plus_api.pyindex 0256f7c30..b9b2f1d28 100644--- a/lib/cli/src/crewai_cli/plus_api.py+++ b/lib/cli/src/crewai_cli/plus_api.py@@ -28,17 +28,56 @@ class PlusAPI(_CorePlusAPI):EVALUATION_POLL_TIMEOUT = 30.0def create_evaluation(- self, execution_id: str, *, eval_config: str | None = None+ self,+ execution_id: str,+ *,+ eval_config: str | None = None,+ project_id: str | None = None,) -> httpx.Response:"""Ask AMP to evaluate the traced run EXECUTION_ID (crewai eval).EVAL_CONFIG is the project's own `eval.jsonc` when it has one: whatgood means for this crew, in its own words. Sent as it was written,- comments and all, and read by the grader rather than here.+ comments and all, and read by the grader rather than here. PROJECT_ID+ is `[tool.crewai].project_id`, so the evaluation is filed under the+ project it is about."""body: dict[str, str] = {"execution_id": execution_id}if eval_config:body["eval_config"] = eval_config+ if project_id:+ body["project_id"] = project_id+ return self._make_request(+ "POST",+ self.EVALUATIONS_RESOURCE,+ json=body,+ timeout=self.EVALUATION_START_TIMEOUT,+ )++ def create_models_evaluation(+ self,+ models: list[str],+ *,+ project_id: str,+ eval_config: str | None = None,+ deployment_id: str | None = None,+ ) -> httpx.Response:+ """Ask AMP to run the project's deployment once per model and compare them.++ AMP finds the deployment by PROJECT_ID (`[tool.crewai].project_id`)+ unless DEPLOYMENT_ID names one. MODELS are `provider/model` strings;+ the deployment's own models are the baseline, so they are not sent.+ EVAL_CONFIG is the project's `eval.jsonc`, as `create_evaluation` sends it.+ """+ body: dict[str, Any] = {+ "kind": "models",+ "project_id": project_id,+ "models": list(models),+ }+ if eval_config:+ body["eval_config"] = eval_config+ if deployment_id:+ body["deployment_id"] = deployment_idreturn self._make_request("POST",self.EVALUATIONS_RESOURCE,@@ -90,11 +129,18 @@ class PlusAPI(_CorePlusAPI):*,name: str | None = None,env: dict[str, str] | None = None,+ project_id: str | None = None,) -> httpx.Response:- """Create a crew deployment from a local project ZIP archive."""+ """Create a crew deployment from a local project ZIP archive.++ PROJECT_ID is `[tool.crewai].project_id`, so AMP knows which project the+ deployment runs; sent only when the project has one.+ """data: dict[str, str] = {}if name:data["name"] = name+ if project_id:+ data["project_id"] = project_idif env:data.update({f"env[{key}]": value for key, value in env.items()})return self._make_multipart_request(@@ -111,9 +157,15 @@ class PlusAPI(_CorePlusAPI):zip_file_path: str | Path,*,env: dict[str, str] | None = None,+ project_id: str | None = None,) -> httpx.Response:- """Update an existing crew deployment from a local project ZIP archive."""+ """Update an existing crew deployment from a local project ZIP archive.++ PROJECT_ID as for `create_crew_from_zip`.+ """data: dict[str, str] = {}+ if project_id:+ data["project_id"] = project_idif env:data.update({f"env[{key}]": value for key, value in env.items()})return self._make_multipart_request(diff --git a/lib/cli/tests/deploy/test_deploy_main.py b/lib/cli/tests/deploy/test_deploy_main.pyindex 08b95a6c5..a815cca9b 100644--- a/lib/cli/tests/deploy/test_deploy_main.py+++ b/lib/cli/tests/deploy/test_deploy_main.py@@ -1316,3 +1316,128 @@ class TestDeployCommand(unittest.TestCase):from crewai_cli.version import get_crewai_versionassert isinstance(get_crewai_version(), str)+++PROJECT_ID = "1f0c6a3e-8d0b-4c4e-9a52-6a9f6d1d2b7e"+++class TestDeploySendsTheProjectId(unittest.TestCase):+ """Every create and push carries `[tool.crewai].project_id` when the+ project has one, so AMP knows which project a deployment runs."""++ @patch("crewai_cli.command.get_auth_token", return_value="test_token")+ @patch("crewai_cli.deploy.main.get_project_id", return_value=PROJECT_ID)+ @patch("crewai_cli.deploy.main.get_project_name", return_value="test_project")+ @patch("crewai_cli.command.PlusAPI")+ def setUp(self, _plus_api, _name, _project_id, _token):+ self.deploy_command = deploy_main.DeployCommand()+ self.mock_client = self.deploy_command.plus_api_client+ self.mock_client.base_url = "https://app.crewai.com"+ browser = patch("crewai_cli.deploy.main.webbrowser.open")+ browser.start()+ self.addCleanup(browser.stop)++ def _created(self, uuid: str = "new-uuid") -> MagicMock:+ response = MagicMock()+ response.status_code = 201+ response.is_success = True+ response.json.return_value = {"uuid": uuid, "status": "created"}+ return response++ def _status(self, zip_deployment: bool) -> MagicMock:+ response = MagicMock()+ response.is_success = True+ response.json.return_value = {"uuid": "test-uuid", "zip_deployment": zip_deployment}+ return response++ def test_the_command_reads_the_project_id(self):+ self.assertEqual(self.deploy_command.project_id, PROJECT_ID)++ @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})+ @patch("crewai_cli.deploy.main.git.Repository")+ def test_create_from_git_sends_it(self, mock_repository, _env):+ mock_repository.return_value.origin_url.return_value = "https://github.com/t/r.git"+ mock_repository.return_value.create_initial_commit_if_needed.return_value = False+ self.mock_client.create_crew.return_value = self._created()++ with patch("sys.stdout", new=StringIO()):+ self.deploy_command.create_crew(confirm=True, skip_validate=True)++ payload = self.mock_client.create_crew.call_args.args[0]+ # Once, at the top level, where AMP reads it on every deploy route.+ self.assertEqual(payload["project_id"], PROJECT_ID)+ self.assertNotIn("project_id", payload["deploy"])++ @patch("crewai_cli.deploy.main.create_project_zip", return_value=Path("/tmp/p.zip"))+ @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})+ @patch("crewai_cli.deploy.main.git.Repository")+ def test_create_from_a_zip_sends_it(self, mock_repository, _env, _zip):+ mock_repository.return_value.origin_url.return_value = None+ mock_repository.return_value.create_initial_commit_if_needed.return_value = False+ self.mock_client.create_crew_from_zip.return_value = self._created()++ with patch("sys.stdout", new=StringIO()):+ self.deploy_command.create_crew(confirm=True, skip_validate=True)++ self.mock_client.create_crew_from_zip.assert_called_once_with(+ Path("/tmp/p.zip"), name="test_project", env={}, project_id=PROJECT_ID+ )++ @patch("crewai_cli.deploy.main.git.Repository")+ @patch("crewai_cli.deploy.main.DeployCommand._display_deployment_info")+ def test_a_push_from_git_sends_it(self, _display, mock_repository):+ mock_repository.return_value.origin_url.return_value = "https://github.com/t/r.git"+ mock_repository.return_value.create_initial_commit_if_needed.return_value = False+ self.mock_client.crew_status_by_uuid.return_value = self._status(False)+ self.mock_client.crew_status_by_name.return_value = self._status(False)++ self.deploy_command.deploy(uuid="test-uuid", skip_validate=True)+ self.deploy_command.deploy(skip_validate=True)++ self.mock_client.deploy_by_uuid.assert_called_once_with(+ "test-uuid", project_id=PROJECT_ID+ )+ self.mock_client.deploy_by_name.assert_called_once_with(+ "test_project", project_id=PROJECT_ID+ )++ @patch("crewai_cli.deploy.main.create_project_zip", return_value=Path("/tmp/p.zip"))+ @patch("crewai_cli.deploy.main.fetch_and_json_env_file", return_value={})+ @patch("crewai_cli.deploy.main.git.Repository")+ @patch("crewai_cli.deploy.main.DeployCommand._display_deployment_info")+ def test_a_push_from_a_zip_sends_it(self, _display, mock_repository, _env, _zip):+ mock_repository.return_value.origin_url.return_value = None+ mock_repository.return_value.create_initial_commit_if_needed.return_value = False+ self.mock_client.crew_status_by_uuid.return_value = self._status(True)++ self.deploy_command.deploy(uuid="test-uuid", skip_validate=True)++ self.mock_client.update_crew_from_zip.assert_called_once_with(+ "test-uuid", Path("/tmp/p.zip"), env={}, project_id=PROJECT_ID+ )+++def test_a_project_without_an_id_sends_none(monkeypatch, tmp_path: Path):+ """Read, never minted: a deploy does not rewrite pyproject.toml."""+ monkeypatch.chdir(tmp_path)+ (tmp_path / "pyproject.toml").write_text("[project]\nname = 'demo'\n")+ monkeypatch.setattr("crewai_cli.command.get_auth_token", lambda: "t")+ monkeypatch.setattr("crewai_cli.command.PlusAPI", MagicMock())+ monkeypatch.setattr(deploy_main, "get_project_name", lambda require=False: "demo")++ command = deploy_main.DeployCommand()++ assert command.project_id is None and command._project() == {}+ assert "project_id" not in (tmp_path / "pyproject.toml").read_text()+++def test_the_project_id_is_read_from_pyproject(monkeypatch, tmp_path: Path):+ monkeypatch.chdir(tmp_path)+ (tmp_path / "pyproject.toml").write_text(+ f"[project]\nname = 'demo'\n\n[tool.crewai]\nproject_id = \"{PROJECT_ID}\"\n"+ )+ monkeypatch.setattr("crewai_cli.command.get_auth_token", lambda: "t")+ monkeypatch.setattr("crewai_cli.command.PlusAPI", MagicMock())+ monkeypatch.setattr(deploy_main, "get_project_name", lambda require=False: "demo")++ assert deploy_main.DeployCommand()._project() == {"project_id": PROJECT_ID}diff --git a/lib/cli/tests/experimental/test_eval_crew.py b/lib/cli/tests/experimental/test_eval_crew.pyindex 5873d37e0..8bcc5d594 100644--- a/lib/cli/tests/experimental/test_eval_crew.py+++ b/lib/cli/tests/experimental/test_eval_crew.py@@ -44,9 +44,10 @@ class FakeAMP:self.api_key = Noneself.headers: dict[str, str] = {"X-Crewai-Organization-Id": "org-42"}- def create_evaluation(self, execution_id, *, eval_config=None):+ def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):self.calls.append(("create", execution_id))self.sent_config = eval_config+ self.sent_project_id = project_idif isinstance(self.create, list):return self.create.pop(0) if len(self.create) > 1 else self.create[0]return self.create@@ -679,7 +680,7 @@ def test_an_oversized_config_is_said_once_and_through_the_callers_note(answers = iter([httpx.Response(404, json={}), httpx.Response(404, json={}), QUEUED])class Client:- def create_evaluation(self, execution_id, *, eval_config=None):+ def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):sent.append(eval_config)return next(answers)@@ -744,7 +745,7 @@ def test_only_a_web_address_is_printed_and_opened(url, opened):def test_a_report_url_that_is_not_a_web_address_is_dropped_with_a_note(monkeypatch):class Client:- def create_evaluation(self, execution_id, *, eval_config=None):+ def create_evaluation(self, execution_id, *, eval_config=None, project_id=None):return httpx.Response(202, json={"id": "ev-1", "url": "file:///etc/passwd", "status": "queued"})@@ -1207,3 +1208,395 @@ def test_an_unreadable_login_is_the_reason_given_when_nothing_was_traced(assert "[/Users/me/.config/crewai]" in outassert "add CREWAI_TRACING_ENABLED=true" not in out++# ── Mode 1 files the evaluation under the project ───────────────────────────+++def test_the_evaluation_is_filed_under_the_project(project, monkeypatch):+ directory, _ = project+ record_last_run(directory)+ monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: "proj-1")+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.eval_crew()++ assert amp.sent_project_id == "proj-1"+++def test_the_app_files_its_evaluation_under_the_project_too(project, monkeypatch):+ """The run app's door reads the id without minting one: it runs inside a+ crew's process, where rewriting pyproject.toml is not its business."""+ monkeypatch.setattr(eval_module, "get_project_id", lambda: "proj-2")+ amp = install(monkeypatch, FakeAMP(statuses=[done()]))++ eval_module.evaluate_run(EXECUTION_ID, on_started=lambda started: None)++ assert amp.sent_project_id == "proj-2"+++# ── crewai eval --models ─────────────────────────────────────────────────────++MODELS = "openai/gpt-4o-mini, openrouter/meta-llama/llama-4-maverick"+DEPLOYMENT = "9b0a4a53-3f7c-4a8e-9a55-2f1c1e0f8e11"+++def comparison(**overrides):+ body = {+ "models": [+ {+ "key": "base", "label": "Poem composer: openai/gpt-5.6-sol", "baseline": True,+ "grades": {"goal": 5, "tasks": 4, "agents": 3, "tools": None},+ "gate": "passed", "cost_usd": 0.0040, "seconds": 9.5, "tokens": 900,+ },+ {+ "key": "mini", "label": "Poem composer: openai/gpt-4o-mini", "baseline": False,+ "grades": {"goal": 5, "tasks": 5, "agents": 2, "tools": None},+ "gate": "passed", "cost_usd": 0.0012, "seconds": 7.6, "tokens": 736,+ },+ ],+ "suggestions": [+ {"subject": "agent:Poem composer", "field": "backstory", "problem": "only one model had it",+ "evidence": "x", "change": "only-one change", "models_affected": ["mini"], "shared": False},+ {"subject": "agent:Poem composer", "field": "goal", "problem": "the poem ignores the topic",+ "evidence": "y", "change": "Write a poem about {topic}.", "models_affected": ["base", "mini"],+ "shared": True},+ {"subject": "task:write", "field": "expected_output", "problem": "no length", "change": "4 lines"},+ {"subject": "agent:Poem composer", "field": "tools", "problem": "a fourth", "change": "never shown"},+ ],+ }+ body.update(overrides)+ return body+++def compared(**overrides):+ return httpx.Response(200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": comparison(**overrides)})+++class FakeModelsAMP(FakeAMP):+ def __init__(self, create=None, statuses=None):+ super().__init__(create=create if create is not None else httpx.Response(+ 202, json={"id": "ev-9", "url": URL, "status": "queued"}+ ), statuses=statuses)+ self.sent: dict | None = None++ def create_models_evaluation(self, models, *, project_id, eval_config=None, deployment_id=None):+ self.calls.append(("create_models", tuple(models)))+ self.sent = {"models": models, "project_id": project_id, "eval_config": eval_config,+ "deployment_id": deployment_id}+ return self.create+++def deployed(project, monkeypatch):+ directory, opened = project+ (directory / "pyproject.toml").write_text('[tool.crewai]\nproject_id = "proj-1"\n')+ monkeypatch.setattr(eval_module, "get_or_create_project_id", lambda: "proj-1")+ return directory, opened+++def test_models_are_compared_on_the_deployment_and_the_table_printed(deployed, monkeypatch, capsys):+ directory, opened = deployed+ (directory / "eval.jsonc").write_text('{"dataset": []}')+ running = httpx.Response(200, json={"id": "ev-9", "status": "running", "progress": {+ "event": "configuration_started", "payload": {"index": 1, "total": 3, "key": "mini"}}})+ judging = httpx.Response(200, json={"id": "ev-9", "status": "running", "progress": {+ "event": "judging", "payload": {"subject": "agent 'Poem composer'", "index": 1, "total": 3}}})+ amp = install(monkeypatch, FakeModelsAMP(statuses=[running, running, judging, compared()]))++ eval_module.eval_models(MODELS)++ out = capsys.readouterr().out+ assert amp.api_key == "login-token"+ assert amp.sent == {+ "models": ["openai/gpt-4o-mini", "openrouter/meta-llama/llama-4-maverick"],+ "project_id": "proj-1", "eval_config": '{"dataset": []}', "deployment_id": None,+ }+ assert opened == [URL] and URL in out+ # Progress: said once per change, never once per poll.+ assert out.count("running · model 2 of 3 · mini") == 1+ assert "judging agent 'Poem composer' · model 2 of 3" in out+ lines = out.splitlines()+ base = next(line for line in lines if "gpt-5.6-sol" in line)+ mini = next(line for line in lines if "Poem composer: openai/gpt-4o-mini" in line)+ assert "(deployed)" in base and "(deployed)" not in mini+ # The best per column: tasks and agents differ, goal is a tie, tools nothing.+ assert "5/5 ★" in mini and "4/5" in base and "4/5 ★" not in base+ assert "3/5 ★" in base and "2/5 ★" not in mini+ assert "$0.0012 ★" in mini and "7.6s ★" in mini and "9.5s ★" not in base+ assert "5/5 ★" not in base # goal: both 5, nothing to point at+ # The top three suggestions, the one every model needed first.+ assert "What would make it better" in out+ assert out.index("the poem ignores the topic") < out.index("only one model had it")+ assert "change: Write a poem about {topic}." in out+ assert "a fourth" not in out+++def test_the_comparison_is_counted_with_the_models_and_nothing_that_names_it(deployed, monkeypatch):+ spans: list[tuple[str, dict[str, str]]] = []++ class FakeTelemetry:+ def set_tracer(self) -> None:+ pass++ def feature_usage_span(self, feature, attributes=None) -> None:+ spans.append((feature, attributes or {}))++ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)+ install(monkeypatch, FakeModelsAMP(statuses=[compared()]))++ eval_module.eval_models(MODELS)++ assert spans == [("cli_usage:eval_models", {+ "authenticated": "true",+ # A public model by name; one crewAI's catalog does not list, by provider.+ "models": "openai/gpt-4o-mini,openrouter/other",+ "models_count": "2",+ })]+++def test_a_refused_comparison_is_not_counted(deployed, monkeypatch, capsys):+ spans: list[str] = []++ class FakeTelemetry:+ def set_tracer(self) -> None:+ pass++ def feature_usage_span(self, feature, attributes=None) -> None:+ spans.append(feature)++ monkeypatch.setattr("crewai_core.telemetry.Telemetry", FakeTelemetry)+ refused = httpx.Response(404, json={"error": "deployment_not_found", "message": (+ "No deployment of this project; deploy it with `crewai deploy create`, or pass --deployment.")})+ install(monkeypatch, FakeModelsAMP(create=refused))++ with pytest.raises(SystemExit) as stopped:+ eval_module.eval_models(MODELS)++ assert stopped.value.code == 1 and spans == []+ assert "deploy it with `crewai deploy create`, or pass --deployment." in capsys.readouterr().out+++def test_the_deployment_named_is_sent(deployed, monkeypatch):+ amp = install(monkeypatch, FakeModelsAMP(statuses=[compared()]))++ eval_module.eval_models("openai/gpt-4o-mini", deployment_id=f" {DEPLOYMENT.upper()} ")++ assert amp.sent["deployment_id"] == DEPLOYMENT+++def test_a_deployment_that_is_not_a_uuid_is_refused_before_anything_is_sent(deployed, monkeypatch, capsys):+ amp = install(monkeypatch, FakeModelsAMP())++ with pytest.raises(SystemExit) as stopped:+ eval_module.eval_models("openai/gpt-4o-mini", deployment_id="poem_creator")++ assert stopped.value.code == 1 and amp.calls == []+ assert "is not a deployment id" in capsys.readouterr().out+++def test_comparing_models_needs_the_login(deployed, monkeypatch, capsys):+ monkeypatch.setattr(eval_module, "saved_login", lambda: None)+ amp = install(monkeypatch, FakeModelsAMP())++ with pytest.raises(SystemExit) as stopped:+ eval_module.eval_models(MODELS)++ assert stopped.value.code == 1 and amp.calls == []+ assert "log in with `crewai login`" in capsys.readouterr().out+++def test_comparing_models_happens_from_the_project(project, monkeypatch, capsys):+ amp = install(monkeypatch, FakeModelsAMP())++ with pytest.raises(SystemExit) as stopped:+ eval_module.eval_models(MODELS)++ assert stopped.value.code == 1 and amp.calls == []+ assert "No crewAI project here" in capsys.readouterr().out+++ ("text", "expected"),+ [+ ("openai/gpt-4o-mini", ["openai/gpt-4o-mini"]),+ (" openai/gpt-4o-mini , openai/gpt-4o-mini,anthropic/claude-haiku-4-5, ",+ ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"]),+ ("openrouter/meta-llama/llama-4-maverick", ["openrouter/meta-llama/llama-4-maverick"]),+ ],+)+def test_the_models_are_one_list_stripped_with_repeats_dropped(text, expected):+ assert eval_module.parse_models(text) == expected+++ ("text", "said"),+ [+ ("gpt-4o-mini", "names no provider"),+ ("openai/", "names no provider"),+ ("/gpt-4o", "names no provider"),+ (" , ", "names no model"),+ (",".join(f"openai/m{n}" for n in range(6)), "at most 5"),+ ("openai/" + "m" * 200, "longer than 200"),+ ],+)+def test_a_list_amp_would_refuse_is_refused_here_with_an_example(text, said):+ with pytest.raises(eval_module.EvaluationStoppedError, match=said) as refused:+ eval_module.parse_models(text)+ if "provider" in said or "names no model" in said:+ assert "openai/gpt-4o-mini" in str(refused.value)+++def test_a_model_without_its_provider_exits_one_before_anything_is_sent(deployed, monkeypatch, capsys):+ amp = install(monkeypatch, FakeModelsAMP())++ with pytest.raises(SystemExit) as stopped:+ eval_module.eval_models("gpt-4o-mini")++ assert stopped.value.code == 1 and amp.calls == []+ assert "provider/model" in capsys.readouterr().out+++def test_a_failed_comparison_exits_one_with_the_reason(deployed, monkeypatch, capsys):+ failed = httpx.Response(200, json={"id": "ev-9", "status": "failed", "error": "the deployment answered 500"})+ install(monkeypatch, FakeModelsAMP(statuses=[failed]))++ with pytest.raises(SystemExit) as stopped:+ eval_module.eval_models(MODELS)++ assert stopped.value.code == 1+ assert "Comparison failed: the deployment answered 500" in capsys.readouterr().out+++def test_done_without_a_comparison_is_a_protocol_error(deployed, monkeypatch, capsys):+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json={"id": "ev-9", "status": "done"})]))++ with pytest.raises(SystemExit) as stopped:+ eval_module.eval_models(MODELS)++ assert stopped.value.code == 1+ assert "done without a comparison (protocol error)" in capsys.readouterr().out+++def test_a_deployment_this_account_may_not_run_is_amps_sentence_alone(deployed, monkeypatch, capsys):+ """Logging in again changes nothing about a deployment one may not run."""+ refused = httpx.Response(403, json={"error": "not_allowed", "message": "You may not run poem_creator."})+ install(monkeypatch, FakeModelsAMP(create=refused))++ with pytest.raises(SystemExit):+ eval_module.eval_models(MODELS)++ out = capsys.readouterr().out+ assert "You may not run poem_creator." in out and "crewai login" not in out+++def test_everything_from_the_wire_prints_literally(deployed, monkeypatch, capsys):+ hostile = comparison(+ models=[{"key": "k", "label": "[red]Writer[/red]: [link=https://evil.test]x[/link]", "baseline": True,+ "grades": {"goal": 9, "tasks": True, "agents": "5", "tools": None}, "cost_usd": True,+ "seconds": -1}],+ suggestions=[{"subject": "[bold]agent[/bold]", "field": "goal", "problem": "[red]p[/red]",+ "change": "[link=https://evil.test]c[/link]"}],+ )+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(+ 200, json={"id": "ev-9", "status": "done", "url": URL, "comparison": hostile})]))++ eval_module.eval_models(MODELS)++ out = capsys.readouterr().out+ assert "[red]Writer[/red]: [link=https://evil.test]x[/link]" in out+ assert "[red]p[/red]" in out and "[link=https://evil.test]c[/link]" in out+ # Not a grade, a cost or a time: each prints as "—", never as a value.+ row = next(line for line in out.splitlines() if "Writer" in line)+ assert "9/5" not in row and "True" not in row and "-1" not in row and "5/5" not in row+ assert row.count("—") == 6+++ ("progress", "line"),+ [+ ("Setting up the deployment", "Setting up the deployment"),+ ({"message": "model 2 of 3: mini"}, "model 2 of 3: mini"),+ ({"event": "configuration_done", "payload": {"index": 0, "total": 2, "key": "base"}},+ "graded · model 1 of 2 · base"),+ ({"subject": "the final output"}, "judging the final output"),+ ({"unknown": "shape"}, None),+ (["not", "a", "dict"], None),+ ],+)+def test_the_progress_line_says_what_is_there_and_guesses_nothing(progress, line):+ assert eval_module._progress_line({"progress": progress}) == line+++def test_the_last_event_stands_in_for_progress():+ events = [{"event": "configuration_started", "payload": {"index": 0, "total": 2, "key": "base"}},+ {"event": "judging", "payload": {"subject": "task 'write'"}}]++ assert eval_module._progress_line({"events": events}) == "judging task 'write'"+++def test_the_cli_maps_models_and_deployment_to_the_comparison(monkeypatch):+ calls = []+ monkeypatch.setattr("crewai_cli.cli.eval_models", lambda *args, **kwargs: calls.append((args, kwargs)))+ monkeypatch.setattr("crewai_cli.cli.eval_crew", lambda **kwargs: calls.append(("run", kwargs)))+ runner = CliRunner()++ assert runner.invoke(eval_command, ["--models", MODELS]).exit_code == 0+ assert runner.invoke(eval_command, ["--models", "openai/gpt-4o", "--deployment", DEPLOYMENT]).exit_code == 0+ assert calls == [((MODELS,), {"deployment_id": None}), (("openai/gpt-4o",), {"deployment_id": DEPLOYMENT})]++ both = runner.invoke(eval_command, ["--models", MODELS, "--run", EXECUTION_ID])+ alone = runner.invoke(eval_command, ["--deployment", DEPLOYMENT])+ assert both.exit_code == 2 and "Give one of them" in both.output+ assert alone.exit_code == 2 and "add --models LIST" in alone.output+ assert len(calls) == 2+ assert "--models" in runner.invoke(eval_command, ["--help"]).output+++def test_a_comparison_leaves_its_criteria_behind_when_the_project_has_none(deployed, monkeypatch, capsys):+ directory, _ = deployed+ answer = compared()+ body = {**answer.json(), "eval_config": '// the criteria\n{"dataset": []}\n'}+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json=body)]))++ eval_module.eval_models(MODELS)++ assert (directory / "eval.jsonc").read_text() == '// the criteria\n{"dataset": []}\n'+ assert "Wrote eval.jsonc" in capsys.readouterr().out+++def test_a_comparison_never_overwrites_the_projects_criteria(deployed, monkeypatch, capsys):+ directory, _ = deployed+ (directory / "eval.jsonc").write_text("// ours\n{}\n")+ body = {**compared().json(), "eval_config": "// theirs\n{}\n"}+ install(monkeypatch, FakeModelsAMP(statuses=[httpx.Response(200, json=body)]))++ eval_module.eval_models(MODELS)++ assert (directory / "eval.jsonc").read_text() == "// ours\n{}\n"+ assert "Wrote eval.jsonc" not in capsys.readouterr().out+++ ("typed", "sent"),+ [+ ("openai/gpt-4o-mini", "openai/gpt-4o-mini"),+ ("anthropic/claude-haiku-4-5", "anthropic/claude-haiku-4-5"),+ ("openrouter/openai/gpt-4o-mini", "openrouter/openai/gpt-4o-mini"),+ # A customer's own: a fine-tune, a deployment name, a model named after+ # a public one, an unknown name — by provider only.+ ("openai/ft:gpt-4o-mini:acme-corp::abc", "openai/other"),+ ("azure/my-deployment", "azure/other"),+ ("openai/gpt-4o-acme", "openai/other"),+ ("ollama/acme-llama", "ollama/other"),+ # A provider crewAI does not know can name a host.+ ("llm.acme.internal/llama-3", "other/other"),+ ],+)+def test_usage_stats_name_a_model_only_when_it_is_a_public_one(typed, sent):+ assert eval_module.telemetry_model_name(typed) == sent+++def test_without_crewais_catalog_every_model_is_other(monkeypatch):+ monkeypatch.setattr(eval_module, "_known_models_and_providers", lambda: (frozenset(), frozenset()))++ assert eval_module.telemetry_model_name("openai/gpt-4o-mini") == "other/other"diff --git a/lib/cli/tests/test_plus_api.py b/lib/cli/tests/test_plus_api.pyindex e40cfdbc2..0a3b721da 100644--- a/lib/cli/tests/test_plus_api.py+++ b/lib/cli/tests/test_plus_api.py@@ -33,6 +33,60 @@ class TestPlusAPI(unittest.TestCase):)self.assertEqual(response, mock_response)+ @patch("crewai_core.plus_api.PlusAPI._make_request")+ def test_create_evaluation_files_it_under_the_project(self, mock_make_request):+ self.api.create_evaluation("run-1", eval_config="{}", project_id="proj-1")++ mock_make_request.assert_called_once_with(+ "POST",+ "/crewai_plus/api/v1/tracing/evaluations",+ json={"execution_id": "run-1", "eval_config": "{}", "project_id": "proj-1"},+ timeout=120.0,+ )++ @patch("crewai_core.plus_api.PlusAPI._make_request")+ def test_create_models_evaluation(self, mock_make_request):+ mock_response = MagicMock()+ mock_make_request.return_value = mock_response++ response = self.api.create_models_evaluation(+ ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"],+ project_id="proj-1",+ eval_config="// ours\n{}",+ deployment_id="dep-1",+ )++ mock_make_request.assert_called_once_with(+ "POST",+ "/crewai_plus/api/v1/tracing/evaluations",+ json={+ "kind": "models",+ "project_id": "proj-1",+ "models": ["openai/gpt-4o-mini", "anthropic/claude-haiku-4-5"],+ "eval_config": "// ours\n{}",+ "deployment_id": "dep-1",+ },+ timeout=120.0,+ )+ self.assertEqual(response, mock_response)++ @patch("crewai_core.plus_api.PlusAPI._make_request")+ def test_create_models_evaluation_sends_only_what_it_was_given(+ self, mock_make_request+ ):+ self.api.create_models_evaluation(["openai/gpt-4o-mini"], project_id="proj-1")++ mock_make_request.assert_called_once_with(+ "POST",+ "/crewai_plus/api/v1/tracing/evaluations",+ json={+ "kind": "models",+ "project_id": "proj-1",+ "models": ["openai/gpt-4o-mini"],+ },+ timeout=120.0,+ )+@patch("crewai_core.plus_api.PlusAPI._make_request")def test_get_evaluation(self, mock_make_request):mock_response = MagicMock()@@ -252,6 +306,42 @@ class TestPlusAPI(unittest.TestCase):"POST", "/crewai_plus/api/v1/crews/test_uuid/deploy")+ @patch("crewai_core.plus_api.PlusAPI._make_request")+ def test_deploy_sends_the_project_id_when_there_is_one(self, mock_make_request):+ self.api.deploy_by_name("test_project", project_id="proj-1")+ mock_make_request.assert_called_with(+ "POST",+ "/crewai_plus/api/v1/crews/by-name/test_project/deploy",+ json={"project_id": "proj-1"},+ )+ self.api.deploy_by_uuid("test_uuid", project_id="proj-1")+ mock_make_request.assert_called_with(+ "POST",+ "/crewai_plus/api/v1/crews/test_uuid/deploy",+ json={"project_id": "proj-1"},+ )++ @patch("crewai_cli.plus_api.PlusAPI._make_multipart_request")+ def test_zip_deploys_send_the_project_id_when_there_is_one(+ self, mock_make_multipart_request+ ):+ self.api.create_crew_from_zip("/tmp/test.zip", name="c", project_id="proj-1")+ mock_make_multipart_request.assert_called_with(+ "POST",+ "/crewai_plus/api/v1/crews/zip",+ zip_file_path="/tmp/test.zip",+ data={"name": "c", "project_id": "proj-1"},+ timeout=300,+ )+ self.api.update_crew_from_zip("u", "/tmp/test.zip", project_id="proj-1")+ mock_make_multipart_request.assert_called_with(+ "POST",+ "/crewai_plus/api/v1/crews/u/zip_update",+ zip_file_path="/tmp/test.zip",+ data={"project_id": "proj-1"},+ timeout=300,+ )+@patch("crewai_core.plus_api.PlusAPI._make_request")def test_crew_status_by_name(self, mock_make_request):self.api.crew_status_by_name("test_project")diff --git a/lib/crewai-core/pyproject.toml b/lib/crewai-core/pyproject.tomlindex f6b790ef9..dd7068a68 100644--- a/lib/crewai-core/pyproject.toml+++ b/lib/crewai-core/pyproject.toml@@ -13,7 +13,7 @@ dependencies = ["httpx~=0.28.1","packaging>=23.0","portalocker~=2.7.0",- "pyjwt>=2.13.0,<3",+ "pyjwt>=2.15.0,<3","pydantic>=2.11.9,<2.13","rich>=13.7.1","opentelemetry-api>=1.42,<2",diff --git a/lib/crewai-core/src/crewai_core/plus_api.py b/lib/crewai-core/src/crewai_core/plus_api.pyindex c1acbad4e..28c8968d6 100644--- a/lib/crewai-core/src/crewai_core/plus_api.py+++ b/lib/crewai-core/src/crewai_core/plus_api.py@@ -62,6 +62,9 @@ class CrewDeploymentSpec(TypedDict):class CreateCrewPayload(TypedDict):deploy: CrewDeploymentSpec+ # ``[tool.crewai].project_id``, at the top level as every other deploy+ # request sends it (AMP reads it there on every route); only when set.+ project_id: NotRequired[str]class _WithUserIdentifier(TypedDict):@@ -320,13 +323,29 @@ class PlusAPI:params=params or None,)- def deploy_by_name(self, project_name: str) -> httpx.Response:- return self._make_request(- "POST", f"{self.CREWS_RESOURCE}/by-name/{project_name}/deploy"- )+ def deploy_by_name(+ self, project_name: str, *, project_id: str | None = None+ ) -> httpx.Response:+ """Redeploy by name, with PROJECT_ID (`[tool.crewai].project_id`) when set.- def deploy_by_uuid(self, uuid: str) -> httpx.Response:- return self._make_request("POST", f"{self.CREWS_RESOURCE}/{uuid}/deploy")+ The body is sent only when there is a project id. An AMP that predates+ project ids ignores it: this route has read its body through Rails'+ ``params.permit`` since the v1 API (2024-08), which drops a key it does+ not name, and AMP never made unpermitted keys an error.+ """+ endpoint = f"{self.CREWS_RESOURCE}/by-name/{project_name}/deploy"+ if project_id:+ return self._make_request("POST", endpoint, json={"project_id": project_id})+ return self._make_request("POST", endpoint)++ def deploy_by_uuid(+ self, uuid: str, *, project_id: str | None = None+ ) -> httpx.Response:+ """Redeploy by uuid, with PROJECT_ID as for `deploy_by_name`."""+ endpoint = f"{self.CREWS_RESOURCE}/{uuid}/deploy"+ if project_id:+ return self._make_request("POST", endpoint, json={"project_id": project_id})+ return self._make_request("POST", endpoint)def crew_status_by_name(self, project_name: str) -> httpx.Response:return self._make_request(@@ -368,11 +387,18 @@ class PlusAPI:*,name: str | None = None,env: dict[str, str] | None = None,+ project_id: str | None = None,) -> httpx.Response:- """Create a crew deployment from a local project ZIP archive."""+ """Create a crew deployment from a local project ZIP archive.++ PROJECT_ID is `[tool.crewai].project_id`, so AMP knows which project the+ deployment runs; sent only when the project has one.+ """data: dict[str, str] = {}if name:data["name"] = name+ if project_id:+ data["project_id"] = project_idif env:data.update({f"env[{key}]": value for key, value in env.items()})return self._make_multipart_request(@@ -389,9 +415,15 @@ class PlusAPI:zip_file_path: str | Path,*,env: dict[str, str] | None = None,+ project_id: str | None = None,) -> httpx.Response:- """Update an existing crew deployment from a local project ZIP archive."""+ """Update an existing crew deployment from a local project ZIP archive.++ PROJECT_ID as for `create_crew_from_zip`.+ """data: dict[str, str] = {}+ if project_id:+ data["project_id"] = project_idif env:data.update({f"env[{key}]": value for key, value in env.items()})return self._make_multipart_request(diff --git a/lib/crewai-core/src/crewai_core/telemetry.py b/lib/crewai-core/src/crewai_core/telemetry.pyindex a5e4aca26..a8de2f025 100644--- a/lib/crewai-core/src/crewai_core/telemetry.py+++ b/lib/crewai-core/src/crewai_core/telemetry.py@@ -580,6 +580,11 @@ class Telemetry:# its use under somebody else's name.FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {"cli_usage:eval": frozenset({"authenticated"}),+ # The models compared: a model in crewAI's own catalog by name (anyone+ # can read it off a price list), any other — a fine-tune, a deployment+ # name, a self-hosted model — as `<provider>/other`; the CLI decides+ # (`telemetry_model_name`). Never a run, an output or an organization.+ "cli_usage:eval_models": frozenset({"authenticated", "models", "models_count"}),}def feature_usage_span(diff --git a/lib/crewai-core/tests/test_smoke.py b/lib/crewai-core/tests/test_smoke.pyindex 22097086a..4deeb99df 100644--- a/lib/crewai-core/tests/test_smoke.py+++ b/lib/crewai-core/tests/test_smoke.py@@ -311,6 +311,45 @@ def test_core_feature_span_keeps_only_what_the_feature_may_send(assert "authenticated" not in sent and sent["feature"] == "cli_usage:deploy"+def test_core_models_span_carries_the_models_compared_and_nothing_else(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """`cli_usage:eval_models` names the models and how many — provider/model+ names, not a run — and still drops an id or run content."""+ from crewai_core.telemetry import Telemetry++ Telemetry._instance = None+ monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)+ monkeypatch.delenv("CREWAI_DISABLE_TELEMETRY", raising=False)+ monkeypatch.delenv("CREWAI_DISABLE_TRACKING", raising=False)++ tracer = Mock()+ span = Mock()+ tracer.start_span.return_value = span+ monkeypatch.setattr(+ "crewai_core.telemetry.TracerProvider",+ lambda **_kwargs: Mock(get_tracer=Mock(return_value=tracer)),+ )++ Telemetry().feature_usage_span(+ "cli_usage:eval_models",+ {+ "authenticated": "true",+ "models": "openai/gpt-4o-mini,anthropic/claude-haiku-4-5",+ "models_count": "2",+ "evaluation_id": "ev-1",+ "organization_id": "org-1",+ },+ )++ sent = {call.args[0]: call.args[1] for call in span.set_attribute.call_args_list}+ assert sent["feature"] == "cli_usage:eval_models"+ assert sent["authenticated"] == "true"+ assert sent["models"] == "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"+ assert sent["models_count"] == "2"+ assert "evaluation_id" not in sent and "organization_id" not in sent++def test_core_telemetry_records_flow_creation_version(monkeypatch: pytest.MonkeyPatch,) -> None:diff --git a/lib/crewai/pyproject.toml b/lib/crewai/pyproject.tomlindex ffb6c8d0d..02ab9c30e 100644--- a/lib/crewai/pyproject.toml+++ b/lib/crewai/pyproject.toml@@ -29,7 +29,7 @@ dependencies = ["openpyxl~=3.1.5",# Authentication and Security"python-dotenv>=1.2.2,<2",- "pyjwt>=2.13.0,<3",+ "pyjwt>=2.15.0,<3",# Configuration and Utils"click>=8.1.7,<9","appdirs~=1.4.4",diff --git a/lib/crewai/src/crewai/agent/core.py b/lib/crewai/src/crewai/agent/core.pyindex 3a43cee93..1ce5f6392 100644--- a/lib/crewai/src/crewai/agent/core.py+++ b/lib/crewai/src/crewai/agent/core.py@@ -79,7 +79,7 @@ from crewai.hooks.dispatch import HookAbortedfrom crewai.knowledge.knowledge import Knowledgefrom crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSourcefrom crewai.lite_agent_output import LiteAgentOutput-from crewai.llm_overlay import active as overlay_active, overlay_model_for+from crewai.llm_overlay import active as overlay_activefrom crewai.llms.base_llm import BaseLLMfrom crewai.mcp.config import MCPServerConfigfrom crewai.rag.embeddings.types import EmbedderConfig@@ -114,7 +114,7 @@ from crewai.utilities.env import get_env_contextfrom crewai.utilities.guardrail import process_guardrail, serialize_guardrail_for_jsonfrom crewai.utilities.guardrail_types import GuardrailCallable, GuardrailTypefrom crewai.utilities.i18n import I18N_DEFAULT-from crewai.utilities.llm_utils import create_llm, create_llm_like+from crewai.utilities.llm_utils import create_llm, overlay_llm_forfrom crewai.utilities.prompts import Prompts, StandardPromptResult, SystemPromptResultfrom crewai.utilities.pydantic_schema_utils import generate_model_descriptionfrom crewai.utilities.string_utils import sanitize_tool_name@@ -420,13 +420,7 @@ class Agent(BaseAgent):else:declared = create_llm(self.llm)self._declared_llm = declared- overlay_model = overlay_model_for(self.role)- if overlay_model:- self.llm = self._overlay_built = create_llm_like(- overlay_model, declared- )- else:- self.llm = declared+ self.llm = self._resolved_overlay(declared)self._overlay_read = Trueif self.function_calling_llm and not isinstance(self.function_calling_llm, BaseLLM@@ -523,15 +517,22 @@ class Agent(BaseAgent):if self.role == role_before or overlay_active.get() is None:returnprevious = self.llm- declared = self._declared_now()- overlay_model = overlay_model_for(self.role)- if overlay_model:- self.llm = self._overlay_built = create_llm_like(overlay_model, declared)- else:- self.llm = declared+ self.llm = self._resolved_overlay(self._declared_now())if isinstance(previous, BaseLLM) and isinstance(self.llm, BaseLLM):self.llm.stream = previous.stream+ def _resolved_overlay(self, declared: BaseLLM | None) -> BaseLLM | None:+ """The llm ``llm_overlay`` puts this agent on, given its ``declared`` one.++ The role's key wins over the model keys; with neither, ``declared``.+ What the overlay built is remembered, so a later miss can tell it from+ an llm the caller assigned.+ """+ resolved = overlay_llm_for(self.role, declared)+ if resolved is not declared:+ self._overlay_built = resolved+ return resolved+def _declared_now(self) -> BaseLLM | None:"""The llm this agent is declared with, as of now.diff --git a/lib/crewai/src/crewai/lite_agent.py b/lib/crewai/src/crewai/lite_agent.pyindex 0fa370a9c..9699e2e5d 100644--- a/lib/crewai/src/crewai/lite_agent.py+++ b/lib/crewai/src/crewai/lite_agent.py@@ -69,7 +69,6 @@ from crewai.hooks.types import ()from crewai.lite_agent_output import LiteAgentOutputfrom crewai.llm import LLM-from crewai.llm_overlay import overlay_model_forfrom crewai.llms.base_llm import BaseLLMfrom crewai.tools.base_tool import BaseToolfrom crewai.tools.structured_tool import CrewStructuredTool@@ -102,7 +101,7 @@ from crewai.utilities.converter import (from crewai.utilities.guardrail import process_guardrail, serialize_guardrail_for_jsonfrom crewai.utilities.guardrail_types import GuardrailCallable, GuardrailTypefrom crewai.utilities.i18n import I18N_DEFAULT-from crewai.utilities.llm_utils import create_llm, create_llm_like+from crewai.utilities.llm_utils import create_llm, overlay_llm_forfrom crewai.utilities.pydantic_schema_utils import (generate_model_description,serialize_model_class,@@ -320,11 +319,7 @@ class LiteAgent(FlowTrackable, BaseModel):@model_validator(mode="after")def setup_llm(self) -> Self:"""Set up the LLM and other components after initialization."""- declared = create_llm(self.llm)- overlay_model = overlay_model_for(self.role)- self.llm = (- create_llm_like(overlay_model, declared) if overlay_model else declared- )+ self.llm = overlay_llm_for(self.role, create_llm(self.llm))if not isinstance(self.llm, BaseLLM):raise ValueError(f"Expected LLM instance of type BaseLLM, got {type(self.llm).__name__}"diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.pyindex 569296bb8..2ba1e6409 100644--- a/lib/crewai/src/crewai/llm.py+++ b/lib/crewai/src/crewai/llm.py@@ -2,6 +2,7 @@ from __future__ import annotationsfrom collections import defaultdictfrom collections.abc import Callable, Mapping+from contextvars import ContextVarfrom datetime import datetimeimport jsonimport logging@@ -33,6 +34,7 @@ from crewai.events.types.tool_usage_events import (ToolUsageStartedEvent,)from crewai.hooks.dispatch import HookAborted+from crewai.llm_overlay import overlay_model_for_modelfrom crewai.llms._finish_reason_utils import extract_choices_finish_reason_and_idfrom crewai.llms.base_llm import (BaseLLM,@@ -88,6 +90,10 @@ logger = logging.getLogger(__name__)_litellm_loaded = FalseLITELLM_AVAILABLE = False+# The instance ``LLM.__new__`` just built on a model ``llm_overlay`` mapped to,+# whose ``__init__`` call with the caller's arguments must be skipped.+_overlay_built: ContextVar[LLM | None] = ContextVar("llm_overlay_built", default=None)+if TYPE_CHECKING:import litellmfrom litellm.litellm_core_utils.get_supported_openai_params import (@@ -194,6 +200,30 @@ SUPPORTED_NATIVE_PROVIDERS: Final[list[str]] = []+# The provider a ``<prefix>/<model>`` string routes to, by prefix: the names+# crewAI accepts for each native provider. ``llm_overlay`` compares model+# names through the same table, so ``google/x`` and ``gemini/x`` are one model.+PROVIDER_ALIASES: Final[dict[str, str]] = {+ "openai": "openai",+ "anthropic": "anthropic",+ "claude": "anthropic",+ "azure": "azure",+ "azure_openai": "azure",+ "google": "gemini",+ "gemini": "gemini",+ "bedrock": "bedrock",+ "aws": "bedrock",+ "openrouter": "openrouter",+ "deepseek": "deepseek",+ "ollama": "ollama",+ "ollama_chat": "ollama_chat",+ "hosted_vllm": "hosted_vllm",+ "cerebras": "cerebras",+ "dashscope": "dashscope",+ "snowflake": "snowflake",+}++class Delta(TypedDict):content: str | Nonerole: str | None@@ -266,6 +296,11 @@ class LLM(BaseLLM):if not model or not isinstance(model, str):raise ValueError("Model must be a non-empty string")+ if cls is LLM:+ mapped = cls._overlay_mapped(model, is_litellm, kwargs)+ if mapped is not None:+ return mapped+route = cls._resolve_route(model, kwargs)if route.native_class is not None and not is_litellm:try:@@ -310,6 +345,39 @@ class LLM(BaseLLM):return object.__new__(cls)+ def __init__(self, model: str, is_litellm: bool = False, **kwargs: Any) -> None:+ # Python calls ``__init__`` on whatever ``__new__`` returns when it is an+ # ``LLM``; an instance ``llm_overlay`` built on a mapped model is already+ # initialized, and initializing it again with the caller's arguments+ # would put the declared model back.+ if _overlay_built.get() is self:+ _overlay_built.set(None)+ return+ super().__init__(model=model, is_litellm=is_litellm, **kwargs)++ @classmethod+ def _overlay_mapped(+ cls, model: str, is_litellm: bool, kwargs: dict[str, Any]+ ) -> LLM | None:+ """``LLM(model, **kwargs)`` built on the model an ``llm_overlay`` key maps it to.++ ``None`` when no model key of the active overlay matches ``model``. The+ provider a bare model routes to is put beside it, so a key written+ ``model:openai/gpt-4o`` matches ``LLM(model="gpt-4o")``.+ """+ provider = kwargs.get("provider")+ if not provider and "/" not in model:+ provider = cls._infer_provider_from_model(model)+ mapped_model = overlay_model_for_model(model, provider)+ if mapped_model is None:+ return None+ from crewai.utilities.llm_utils import create_llm_from_kwargs_like++ built = create_llm_from_kwargs_like(mapped_model, model, kwargs, is_litellm)+ if isinstance(built, LLM):+ _overlay_built.set(built)+ return cast(LLM, built)+@classmethoddef _matches_provider_pattern(cls, model: str, provider: str) -> bool:"""Check if a model name matches provider-specific patterns.@@ -416,27 +484,7 @@ class LLM(BaseLLM):elif "/" in model:prefix, _, model_part = model.partition("/")- provider_mapping = {- "openai": "openai",- "anthropic": "anthropic",- "claude": "anthropic",- "azure": "azure",- "azure_openai": "azure",- "google": "gemini",- "gemini": "gemini",- "bedrock": "bedrock",- "aws": "bedrock",- "openrouter": "openrouter",- "deepseek": "deepseek",- "ollama": "ollama",- "ollama_chat": "ollama_chat",- "hosted_vllm": "hosted_vllm",- "cerebras": "cerebras",- "dashscope": "dashscope",- "snowflake": "snowflake",- }-- canonical_provider = provider_mapping.get(prefix.lower())+ canonical_provider = PROVIDER_ALIASES.get(prefix.lower())valid_native_model = bool(canonical_providerdiff --git a/lib/crewai/src/crewai/llm_overlay.py b/lib/crewai/src/crewai/llm_overlay.pyindex 8cb520d6a..fde469432 100644--- a/lib/crewai/src/crewai/llm_overlay.py+++ b/lib/crewai/src/crewai/llm_overlay.py@@ -1,10 +1,37 @@-"""Process-context ``role -> model`` overlay for agents.+"""Process-context ``role -> model`` and ``model -> model`` overlay.-``llm_overlay`` lets a per-run caller make specific agents use a different-model without editing the code that builds them. While the block is active,-an ``Agent`` or ``LiteAgent`` whose ``role`` is a key of the mapping is built-with the mapped model instead of its declared ``llm``. Roles that are not-keys, and every agent built outside the block, keep their own model.+``llm_overlay`` lets a per-run caller make specific agents, or specific+models, use a different model without editing the code that builds them.+While the block is active, an ``Agent`` or ``LiteAgent`` whose ``role`` is a+key of the mapping is built with the mapped model instead of its declared+``llm``. Roles that are not keys, and every agent built outside the block,+keep their own model.++A key that starts with :data:`MODEL_KEY_PREFIX` names a model instead of a+role: ``"model:openai/gpt-4o"`` maps every LLM built from that model string+inside the block, and ``"model:*"`` every LLM built from any model string.+That is what reaches an LLM no role names — a flow step's own+``LLM(model="openai/gpt-4o").call(...)`` — and the simplest way to say "this+whole run on model X". A model key is read wherever an LLM is built from a+model string: ``LLM(model=...)`` (and so ``create_llm`` and an agent's+``llm="..."``, which go through it), and an agent's declared ``llm`` instance+when it was built outside the block. A model is compared as the caller wrote+it and without the provider prefix native providers strip, so+``"model:openai/gpt-4o"`` matches ``LLM(model="gpt-4o")`` and an instance+whose ``model`` reads ``"gpt-4o"``, and ``"model:gpt-4o"`` matches+``LLM(model="openai/gpt-4o")``; an exact key wins over a stripped one, and+both over ``"model:*"``. Only the model's own provider prefix is ever+stripped: an aggregator's route such as ``openrouter/openai/gpt-4o`` is that+aggregator's model and matches only a key naming it whole. The mapped model is built once, never looked up+again: ``{"model:a": "b", "model:b": "c"}`` puts an ``a`` on ``b``. For an+agent a role key wins over every model key, so ``{"Researcher": x,+"model:*": y}`` runs the Researcher on ``x`` and everything else on ``y`` —+and ``x`` is built like the llm the caller declared, even when a model key+already mapped that llm as it was built inside the block.+The settings the caller passed to ``LLM(...)`` are carried to the mapped model+by the same rule as a declared ``llm``'s (below). Only ``LLM`` itself is+mapped: a subclass of it, or a provider class built directly, is the caller's+own choice of class and keeps its model.Roles are matched exactly, by the text they have when the overlay is read;only whitespace around a role or a key is ignored, so a role a YAML file@@ -46,6 +73,9 @@ registers an agent when it first emits) does not read it again, so an agentbuilt outside a block keeps its model through a kickoff inside one.Example:+ >>> with llm_overlay({"model:*": "openai/gpt-4o-mini"}):+ ... flow.kickoff() # every LLM built in the run is on gpt-4o-mini+>>> with llm_overlay({"Researcher": "openai/gpt-4o"}):... crew = build_crew() # the Researcher agent is built on gpt-4o... crew.kickoff()@@ -60,24 +90,57 @@ from __future__ import annotationsfrom collections.abc import Iteratorfrom contextlib import contextmanagerfrom contextvars import ContextVar+from typing import Any, Final, TypeVar+import weakref+++MODEL_KEY_PREFIX: Final[str] = "model:"+"""A key starting with this names a model (``"model:openai/gpt-4o"``), not a role."""++_T = TypeVar("_T")++ANY_MODEL: Final[str] = "*"+"""The model part of the key that maps every LLM built from a model string."""active: ContextVar[dict[str, str] | None] = ContextVar("llm_overlay_active", default=None)+# Set while the overlay builds the model it mapped to, so that build is not+# mapped again (a chain of model keys, or ``model:*`` under a role's model).+_building_mapped: ContextVar[bool] = ContextVar(+ "llm_overlay_building_mapped", default=False+)++# The LLMs a model key already mapped, by id. An agent reading its declared+# ``llm`` skips one of these: its model is the mapped one, never a key again.+_mapped_llms: weakref.WeakValueDictionary[int, Any] = weakref.WeakValueDictionary()++# What the caller declared for an LLM the overlay built: the declared instance,+# or ``LLM(model, **kwargs)``'s ``(model, kwargs, is_litellm)``. Dropped with+# the LLM it describes.+Declared = Any+_declared_before: dict[int, Declared] = {}+@contextmanagerdef llm_overlay(mapping: dict[str, str] | None) -> Iterator[None]:- """Route agent roles to models for the duration of the block.+ """Route agent roles and models to models for the duration of the block.Args:- mapping: ``{role: model}`` for the block; ``None`` clears any active- overlay for the block. Whitespace around each role is dropped from- the copy the block uses, so a key matches a role that differs from- it only by its surrounding whitespace; the mapping passed in is- left as it is. The previous value is always restored on exit,- including when the block raises.+ mapping: ``{role: model}`` and ``{"model:<model>": model}`` (or+ ``{"model:*": model}``) for the block; ``None`` clears any active+ overlay for the block. Whitespace around each role, and around the+ model a model key names, is dropped from the copy the block uses,+ so a key matches a role that differs from it only by its+ surrounding whitespace; the mapping passed in is left as it is. The+ previous value is always restored on exit, including when the+ block raises.++ Raises:+ ValueError: Two keys name one role or one model with different+ models, or a model key names no model."""token = active.set(_stripped(mapping))try:@@ -90,7 +153,8 @@ def overlay_model_for(role: str | None) -> str | None:"""The model the active overlay assigns to ``role``.Whitespace around ``role`` is ignored, as it was around the keys when the- overlay was set. An empty or ``None`` role matches nothing.+ overlay was set. An empty or ``None`` role matches nothing, and a role is+ never matched against a model key.Returns:The mapped model string, or ``None`` when no overlay is active or@@ -100,22 +164,166 @@ def overlay_model_for(role: str | None) -> str | None:if not mapping or role is None:return Nonerole = role.strip()- return mapping.get(role) if role else None+ if not role or role.startswith(MODEL_KEY_PREFIX):+ return None+ return mapping.get(role)+++def overlay_model_for_model(+ model: str | None, provider: str | None = None+) -> str | None:+ """The model the active overlay's model keys assign to an LLM on ``model``.++ ``model`` is compared as written and, when a native provider strips its own+ prefix, without it: ``openai/gpt-4o`` also as ``gpt-4o``, and ``gpt-4o``+ routed to ``provider`` also as ``openai/gpt-4o``. A prefix is only ever the+ model's OWN provider: in an aggregator's route (``openrouter/openai/gpt-4o``)+ the rest is that aggregator's model id, never the model a key for+ ``openai/gpt-4o`` or ``gpt-4o`` names, so it is compared whole. An existing+ instance of such a route records ``model="openai/gpt-4o"`` under+ ``provider="openrouter"``, and is compared as ``openrouter/openai/gpt-4o``.+ ``model:*`` matches every model. Nothing matches while the overlay is+ building a model it mapped.++ Args:+ model: The model string an LLM is being built from, or the ``model``+ of an existing instance.+ provider: The provider the LLM routes to, when ``model`` does not say.++ Returns:+ The mapped model string, or ``None`` when no overlay is active, it has+ no model key for ``model``, or a mapped model is being built.+ """+ mapping = active.get()+ if not mapping or not model or _building_mapped.get():+ return None+ forms = _model_forms(model.strip(), provider)+ for form in forms:+ mapped = mapping.get(MODEL_KEY_PREFIX + form)+ if mapped is not None:+ return mapped+ # The same model under another of its provider's names: a key written+ # `google/x` for an llm that routed `gemini/x`, or the other way round.+ aliases = _aliases()+ canonical = {_canonical(form, aliases) for form in forms}+ for key, mapped in mapping.items():+ named = (+ key[len(MODEL_KEY_PREFIX) :] if key.startswith(MODEL_KEY_PREFIX) else None+ )+ if named and named != ANY_MODEL and _canonical(named, aliases) in canonical:+ return mapped+ return mapping.get(MODEL_KEY_PREFIX + ANY_MODEL)+++def _model_forms(model: str, provider: str | None) -> list[str]:+ """``model`` in the forms a key may name it by, the one as written first."""+ if provider and not model.startswith(f"{provider}/"):+ routed = f"{provider}/{model}"+ # A bare name is the native one with its prefix stripped; a name that+ # still has a slash is the provider's own route id and only means+ # something beside the provider.+ forms = [routed] if "/" in model else [model, routed]+ else:+ forms = [model]+ prefix, separator, rest = forms[-1].partition("/")+ if separator and "/" not in rest and prefix.lower() in _aliases():+ if rest not in forms:+ forms.append(rest)+ return forms+++def _aliases() -> dict[str, str]:+ """The router's provider aliases (``crewai.llm.PROVIDER_ALIASES``): every+ prefix ``LLM`` strips from a model string it routes natively, to the+ provider it names."""+ from crewai.llm import PROVIDER_ALIASES++ return PROVIDER_ALIASES+++def _canonical(form: str, aliases: dict[str, str]) -> str:+ """``form`` with its provider prefix under the provider's one name."""+ prefix, separator, rest = form.partition("/")+ provider = aliases.get(prefix.lower()) if separator else None+ return f"{provider}/{rest}" if provider else form+++def overlay_model_for_llm(llm: Any) -> str | None:+ """The model the active overlay's model keys assign to an existing ``llm``.++ An agent's declared ``llm`` built outside the block is looked up by its+ ``model`` and ``provider``; one a model key already mapped when it was+ built is not looked up again.+ """+ if llm is None or _mapped_llms.get(id(llm)) is llm:+ return None+ return overlay_model_for_model(+ getattr(llm, "model", None), getattr(llm, "provider", None)+ )+++def declared_before_overlay(llm: Any) -> Declared | None:+ """What the caller declared for ``llm`` before the overlay mapped it, or None."""+ if llm is None or _mapped_llms.get(id(llm)) is not llm:+ return None+ return _declared_before.get(id(llm))+++@contextmanager+def building_mapped_model() -> Iterator[None]:+ """Build the model the overlay mapped to, without mapping it again."""+ token = _building_mapped.set(True)+ try:+ yield+ finally:+ _building_mapped.reset(token)+++def mark_mapped(llm: _T, declared: Declared | None = None) -> _T:+ """Record ``llm`` as built by the overlay, so no model key maps it again.++ ``declared`` is what the caller declared before the overlay mapped it —+ the declared instance, or ``LLM(model, **kwargs)``'s arguments as+ ``(model, kwargs, is_litellm)`` — so a role key can still be built from+ the caller's own declaration (:func:`declared_before_overlay`).+ """+ try:+ _mapped_llms[id(llm)] = llm+ if declared is not None:+ _declared_before[id(llm)] = declared+ weakref.finalize(llm, _declared_before.pop, id(llm), None)+ except TypeError:+ # Only a weakly-referenceable object can be remembered (every+ # ``BaseLLM`` is). Anything else is returned as built: the one cost is+ # that an agent declared with it looks its model up again, as it would+ # for an llm built outside the block — never a failed build.+ pass+ return llmdef _stripped(mapping: dict[str, str] | None) -> dict[str, str] | None:"""A copy of ``mapping`` with the whitespace around each role dropped.- Two keys that differ only by whitespace name one role. When they name the- same model the copy holds it once; when they name different models the- mapping is ambiguous and is refused, so the model an agent runs on never- depends on dictionary order.+ Two keys that differ only by whitespace name one role (or, after the+ :data:`MODEL_KEY_PREFIX`, one model). When they name the same model the+ copy holds it once; when they name different models the mapping is+ ambiguous and is refused, so the model an agent runs on never depends on+ dictionary order."""if mapping is None:return Nonestripped: dict[str, str] = {}for role, model in mapping.items():key = role.strip()+ if key.startswith(MODEL_KEY_PREFIX):+ named = key[len(MODEL_KEY_PREFIX) :].strip()+ if not named:+ raise ValueError(+ f"llm_overlay: the key {role!r} names no model; write "+ f"'{MODEL_KEY_PREFIX}<provider/model>' or "+ f"'{MODEL_KEY_PREFIX}{ANY_MODEL}'"+ )+ key = MODEL_KEY_PREFIX + namedif key in stripped and stripped[key] != model:raise ValueError(f"llm_overlay: role {key!r} is mapped twice with different models "diff --git a/lib/crewai/src/crewai/telemetry/telemetry.py b/lib/crewai/src/crewai/telemetry/telemetry.pyindex a34fbe4ca..4de559bd9 100644--- a/lib/crewai/src/crewai/telemetry/telemetry.py+++ b/lib/crewai/src/crewai/telemetry/telemetry.py@@ -1293,6 +1293,11 @@ class Telemetry:# its use under somebody else's name.FEATURE_ATTRIBUTES: ClassVar[dict[str, frozenset[str]]] = {"cli_usage:eval": frozenset({"authenticated"}),+ # The models compared: a model in crewAI's own catalog by name (anyone+ # can read it off a price list), any other — a fine-tune, a deployment+ # name, a self-hosted model — as `<provider>/other`; the CLI decides+ # (`telemetry_model_name`). Never a run, an output or an organization.+ "cli_usage:eval_models": frozenset({"authenticated", "models", "models_count"}),}def feature_usage_span(diff --git a/lib/crewai/src/crewai/utilities/llm_utils.py b/lib/crewai/src/crewai/utilities/llm_utils.pyindex 57509bd67..a6c0bb376 100644--- a/lib/crewai/src/crewai/utilities/llm_utils.py+++ b/lib/crewai/src/crewai/utilities/llm_utils.py@@ -6,6 +6,13 @@ from pydantic import ValidationErrorfrom crewai.constants import DEFAULT_LLM_MODEL, ENV_VARS, LITELLM_PARAMSfrom crewai.llm import LLM+from crewai.llm_overlay import (+ building_mapped_model,+ declared_before_overlay,+ mark_mapped,+ overlay_model_for,+ overlay_model_for_llm,+)from crewai.llms.base_llm import BaseLLM@@ -208,22 +215,114 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:Returns:A new instance for ``model``; never ``base`` itself."""- if not isinstance(base, BaseLLM):- return LLM(model=model)-- # The declared endpoint decides where an unknown ``openai/`` model routes (a- # custom OpenAI-compatible endpoint), so the same-provider question is asked- # with it: a self-hosted model mapped to another model on the same endpoint- # must keep that endpoint.- route = LLM._resolve_route(model, _configured_settings(base, _ENDPOINT_NAMES))- carried = _configured_settings(base, GENERATION_SETTINGS)- if _same_provider(base, route):- carried.update(_configured_settings(base, PROVIDER_SETTINGS))- target = LLM._resolve_route(model, carried).native_class or LLM- if type(base) is LLM and base.is_litellm:+ with building_mapped_model():+ if not isinstance(base, BaseLLM):+ return mark_mapped(LLM(model=model))++ # The declared endpoint decides where an unknown ``openai/`` model routes (a+ # custom OpenAI-compatible endpoint), so the same-provider question is asked+ # with it: a self-hosted model mapped to another model on the same endpoint+ # must keep that endpoint.+ route = LLM._resolve_route(model, _configured_settings(base, _ENDPOINT_NAMES))+ carried = _configured_settings(base, GENERATION_SETTINGS)+ if _same_provider(base, route):+ carried.update(_configured_settings(base, PROVIDER_SETTINGS))# The declared llm runs through LiteLLM — by the caller's choice or# because no native class knew its model; either way that is the# environment its callbacks and extra kwargs were written for.+ through_litellm = type(base) is LLM and base.is_litellm+ return mark_mapped(+ _build_like(model, carried, type(base), through_litellm), base+ )+++def overlay_llm_for(role: str | None, declared: BaseLLM | None) -> BaseLLM | None:+ """The llm an agent with ``role`` and ``declared`` llm runs on under ``llm_overlay``.++ A role key wins, and its model is built from the caller's declaration even+ when a model key already mapped ``declared`` on its way in; else a model+ key matching ``declared``'s model (one a model key already mapped when it+ was built is not looked up again); else ``declared`` itself. A mapped model+ is built like ``declared`` (:func:`create_llm_like`). Outside any block+ this is ``declared``.+ """+ role_model = overlay_model_for(role)+ if role_model:+ # The role wins, built from what the caller declared: when a model key+ # already mapped the declared llm, its declaration, not the mapping.+ before = declared_before_overlay(declared)+ if isinstance(before, tuple):+ declared_model, kwargs, is_litellm = before+ return create_llm_from_kwargs_like(+ role_model, declared_model, kwargs, is_litellm+ )+ base = before if isinstance(before, BaseLLM) else declared+ return create_llm_like(role_model, base)+ model = overlay_model_for_llm(declared)+ return create_llm_like(model, declared) if model else declared+++def create_llm_from_kwargs_like(+ model: str, declared_model: str, kwargs: dict[str, Any], is_litellm: bool+) -> BaseLLM:+ """Build ``model`` configured like ``LLM(declared_model, **kwargs)`` would be.++ What ``llm_overlay``'s model keys build when ``LLM(model=...)`` is called+ inside the block: the settings the caller passed are carried by the rule of+ :func:`create_llm_like` — generation settings to any class that has them,+ credentials and endpoints only to the same provider — without building the+ declared model first (its provider's key may not be in this environment).+ Everything the caller passed is theirs, so nothing counts as derived.+ """+ declaration = (declared_model, dict(kwargs), is_litellm)+ with building_mapped_model():+ settings = {+ k: v+ for k, v in kwargs.items()+ if v is not None and not (isinstance(v, (list, dict)) and not v)+ }+ if isinstance(settings.get("endpoint"), str):+ settings["endpoint"] = settings["endpoint"].split(_AZURE_DEPLOYMENT_PATH)[0]+ endpoint = {k: settings[k] for k in _ENDPOINT_NAMES if k in settings}+ route = LLM._resolve_route(model, endpoint)+ carried = {k: settings[k] for k in GENERATION_SETTINGS if k in settings}+ try:+ declared = LLM._resolve_route(declared_model, kwargs)+ except ImportError:+ # The declared model's SDK is not installed here. Nothing is built+ # on it, so that is no reason to fail; its credentials, though,+ # cannot be matched to the new provider, so none are carried. A+ # caller who chose LiteLLM keeps it.+ return mark_mapped(+ _build_like(model, carried, None, is_litellm), declaration+ )+ declared_class = (+ LLM+ if is_litellm or declared.native_class is None+ else declared.native_class+ )+ if _same_route_provider(declared_class, declared.provider, route):+ carried.update({k: settings[k] for k in PROVIDER_SETTINGS if k in settings})+ return mark_mapped(+ _build_like(model, carried, declared_class, declared_class is LLM),+ declaration,+ )+++def _build_like(+ model: str,+ carried: dict[str, Any],+ base_class: type[BaseLLM] | None,+ through_litellm: bool,+) -> BaseLLM:+ """Build ``model`` with the ``carried`` settings its class accepts.++ ``base_class`` is the declared llm's class, ``None`` when it is not known;+ ``additional_params`` travel only within one class.+ """+ route = LLM._resolve_route(model, carried)+ target = route.native_class or LLM+ if through_litellm:target = LLMcarried["is_litellm"] = Trueaccepted = {@@ -231,7 +330,7 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:for k, v in carried.items()if k in target.model_fields or k == "is_litellm"}- if type(base) is not target:+ if base_class is not target:accepted.pop("additional_params", None)_carry_cap_under_the_targets_name(carried, accepted, target)if route.provider == "anthropic" and "temperature" in accepted:@@ -239,6 +338,34 @@ def create_llm_like(model: str, base: BaseLLM | None) -> BaseLLM:return _build(model, accepted)+def _same_route_provider(+ declared_class: type[BaseLLM], declared_provider: str, route: Any+) -> bool:+ """:func:`_same_provider` for a declared llm known by its route, not an instance.++ The same rule, case for case: a native declared class compares by class, a+ LiteLLM-routed ``LLM`` by its provider — to a native route through the class+ that provider names, so ``LLM(model="openai/gpt-4o", is_litellm=True)``+ mapped to a native OpenAI model keeps its key and endpoint.+ """+ if route.native_class is None:+ return declared_class is LLM and route.provider == declared_provider+ if declared_class is route.native_class:+ return not _serves_several_providers(route.native_class) or (+ route.provider == declared_provider+ )+ if declared_class is not LLM:+ return False+ try:+ named = LLM._get_native_provider(declared_provider or "")+ except ImportError:+ return False+ return named is route.native_class and (+ not _serves_several_providers(route.native_class)+ or route.provider == declared_provider+ )++def _same_provider(base: BaseLLM, route: Any) -> bool:"""Whether ``route`` lands where ``base``'s credentials and endpoint belong.diff --git a/lib/crewai/tests/llms/openai/test_openai.py b/lib/crewai/tests/llms/openai/test_openai.pyindex 5a06f4676..4ab77b962 100644--- a/lib/crewai/tests/llms/openai/test_openai.py+++ b/lib/crewai/tests/llms/openai/test_openai.py@@ -214,6 +214,15 @@ def test_openai_completion_module_is_imported(monkeypatch):"""module_name = "crewai.llms.providers.openai.completion"+ # Re-importing binds the fresh module on its package too, and monkeypatch+ # only restores sys.modules: without also restoring the attribute, every+ # later test in this worker sees two module objects for one name, and a+ # `patch("crewai.llms.providers.openai.completion.X")` patches the one that+ # `from ... import X` does not read.+ import crewai.llms.providers.openai as openai_package++ if hasattr(openai_package, "completion"):+ monkeypatch.setattr(openai_package, "completion", openai_package.completion)monkeypatch.delitem(sys.modules, module_name, raising=False)LLM(model="gpt-4o")diff --git a/lib/crewai/tests/telemetry/test_tracer_isolation.py b/lib/crewai/tests/telemetry/test_tracer_isolation.pyindex 3783988f7..645ea5645 100644--- a/lib/crewai/tests/telemetry/test_tracer_isolation.py+++ b/lib/crewai/tests/telemetry/test_tracer_isolation.py@@ -118,6 +118,32 @@ def test_a_feature_span_keeps_only_what_the_feature_may_send(telemetry_with_expoassert deploy_span.attributes["feature"] == "cli_usage:deploy"+def test_the_models_span_carries_the_models_compared_and_nothing_else(+ telemetry_with_exporter,+):+ telemetry, exporter = telemetry_with_exporter+ telemetry.set_tracer()++ telemetry.feature_usage_span(+ "cli_usage:eval_models",+ {+ "authenticated": "true",+ "models": "openai/gpt-4o-mini,anthropic/claude-haiku-4-5",+ "models_count": "2",+ "evaluation_id": "ev-1",+ "organization_id": "org-1",+ },+ )++ (span,) = exporter.get_finished_spans()+ assert span.attributes["feature"] == "cli_usage:eval_models"+ assert span.attributes["authenticated"] == "true"+ assert span.attributes["models"] == "openai/gpt-4o-mini,anthropic/claude-haiku-4-5"+ assert span.attributes["models_count"] == "2"+ assert "evaluation_id" not in span.attributes+ assert "organization_id" not in span.attributes++def test_our_spans_are_unaffected_by_an_application_provider(telemetry_with_exporter):"""An app that installs its own provider must not divert our telemetry.diff --git a/lib/crewai/tests/test_llm_overlay.py b/lib/crewai/tests/test_llm_overlay.pyindex 0e329bd3d..3642da3be 100644--- a/lib/crewai/tests/test_llm_overlay.py+++ b/lib/crewai/tests/test_llm_overlay.py@@ -1,10 +1,13 @@-"""`llm_overlay` swaps an agent's model by role, for the calling context only.+"""`llm_overlay` swaps an agent's model by role, and any LLM's by model, for+the calling context only.-The overlay is read in exactly three places: the validators where `Agent` and-`LiteAgent` resolve their `llm`, and `Agent.interpolate_inputs`, which looks the+The overlay is read in exactly four places: the validators where `Agent` and+`LiteAgent` resolve their `llm`, `Agent.interpolate_inputs`, which looks theinterpolated role up again because a templated role only becomes a key once a-kickoff fills its placeholders in. So these tests build agents, interpolate-them, and look at the model they end up with. No LLM is ever called.+kickoff fills its placeholders in, and `LLM.__new__`, where a `model:` key maps+an LLM built from a model string. So these tests build agents and LLMs,+interpolate them, and look at the model they end up with. No LLM is ever called+over the network.`create_llm("openai/gpt-4o")` returns the native OpenAI provider, which stripsthe `openai/` prefix, so the resolved model reads `"gpt-4o"`.@@ -19,8 +22,14 @@ from typing import Anyfrom crewai import Agent, Crew, Taskfrom crewai.lite_agent import LiteAgentfrom crewai.llm import LLM-from crewai.llm_overlay import active, llm_overlay, overlay_model_for+from crewai.llm_overlay import (+ MODEL_KEY_PREFIX,+ active,+ llm_overlay,+ overlay_model_for,+)from crewai.llms.base_llm import BaseLLM+from crewai.utilities.llm_utils import create_llmimport pytest@@ -567,3 +576,315 @@ def test_re_validation_keeps_the_llm_a_kickoff_time_swap_set() -> None:RuntimeState(root=[agent])assert agent.llm is swapped+++# ── model keys: `model:<provider/model>` and `model:*` ───────────────────────+++def test_the_model_key_prefix_is_a_public_constant() -> None:+ """Another package feature-detects the model-for-model form on it."""+ assert MODEL_KEY_PREFIX == "model:"+++def test_a_bare_llm_call_in_a_flow_step_runs_on_the_mapped_model(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The case roles cannot reach: a flow step's own `LLM(...).call()`."""+ from crewai.flow.flow import Flow, start++ openai_class = type(LLM(model="openai/gpt-4o-mini"))+ monkeypatch.setattr(+ openai_class,+ "call",+ lambda self, messages, *args, **kwargs: f"answered by {self.model}",+ )++ class Poem(Flow): # type: ignore[type-arg]+ @start()+ def write(self) -> str:+ return LLM(model="openai/gpt-5.6-sol").call("a poem")++ with llm_overlay({"model:openai/gpt-5.6-sol": "openai/gpt-4o-mini"}):+ assert Poem().kickoff() == "answered by gpt-4o-mini"++ assert Poem().kickoff() == "answered by gpt-5.6-sol"+++def test_a_model_key_matches_with_and_without_the_provider_prefix() -> None:+ """Native providers strip `openai/`; a key written either way matches both."""+ with llm_overlay({"model:openai/gpt-4o": "openai/gpt-4.1"}):+ assert LLM(model="gpt-4o").model == "gpt-4.1"+ assert LLM(model="openai/gpt-4o").model == "gpt-4.1"+ assert LLM(model="openai/gpt-4o-mini").model == "gpt-4o-mini"+ with llm_overlay({"model:gpt-4o": "openai/gpt-4.1"}):+ assert LLM(model="openai/gpt-4o").model == "gpt-4.1"+++def test_an_exact_model_key_wins_over_a_stripped_one_and_over_the_wildcard() -> (+ None+):+ with llm_overlay(+ {+ "model:gpt-4o": "openai/gpt-4.1-nano",+ "model:openai/gpt-4o": "openai/gpt-4.1",+ "model:*": "openai/gpt-4o-mini",+ }+ ):+ assert LLM(model="openai/gpt-4o").model == "gpt-4.1"+ assert LLM(model="gpt-4o").model == "gpt-4.1-nano"+ assert LLM(model="openai/o3-mini").model == "gpt-4o-mini"+++def test_the_wildcard_maps_every_llm_built_from_a_model_string() -> None:+ with llm_overlay({"model:*": "openai/gpt-4o-mini"}):+ built = [+ LLM(model="openai/gpt-4o"),+ create_llm("anthropic/claude-haiku-4-5"),+ LLM(model="gpt-5.6-sol"),+ ]++ assert [(type(b).__name__, b.model) for b in built] == [+ ("OpenAICompletion", "gpt-4o-mini")+ ] * 3+ assert LLM(model="openai/gpt-4o").model == "gpt-4o"+++def test_a_mapped_model_is_not_mapped_again() -> None:+ """A chain of keys is one step: the model a key maps to is built as it is."""+ with llm_overlay(+ {"model:openai/gpt-4o": "openai/gpt-4.1", "model:openai/gpt-4.1": "openai/o3"}+ ):+ assert LLM(model="openai/gpt-4o").model == "gpt-4.1"+++def test_the_caller_settings_follow_the_mapped_model_by_the_declared_llm_rule() -> (+ None+):+ """Generation settings go to any provider; a key and an endpoint only to+ the provider they were issued for."""+ with llm_overlay({"model:*": "openai/gpt-4o"}):+ same = LLM(model="openai/gpt-4o-mini", **CONFIGURATION)+ with llm_overlay({"model:*": "anthropic/claude-haiku-4-5"}):+ other = LLM(model="openai/gpt-4o-mini", **CONFIGURATION)++ assert type(same).__name__ == "OpenAICompletion" and same.model == "gpt-4o"+ assert _configuration_of(same) == CONFIGURATION+ assert type(other).__name__ == "AnthropicCompletion"+ assert other.model == "claude-haiku-4-5"+ assert other.timeout == 42 and other.temperature == 0.1+ assert other.max_tokens == 77+ assert other.api_key != "k" and other.base_url is None+++def test_a_model_mapped_onto_litellm_is_not_initialized_again() -> None:+ """`LLM.__new__` returning an `LLM` makes Python call `__init__` with the+ caller's arguments; the mapped instance must keep the mapped model."""+ pytest.importorskip("litellm")+ with llm_overlay({"model:*": "groq/llama-3.1-8b-instant"}):+ built = LLM(model="openai/gpt-4o-mini", temperature=0.3, api_key="k")++ assert type(built).__name__ == "LLM" and built.is_litellm+ assert built.model == "groq/llama-3.1-8b-instant"+ assert built.temperature == 0.3 and built.api_key != "k"+++def test_an_agent_declared_with_a_model_string_runs_on_the_mapped_model() -> None:+ with llm_overlay({"model:openai/gpt-4o-mini": "openai/gpt-4o"}):+ agent = _agent("Writer")++ assert agent.llm.model == "gpt-4o"+++def test_an_agent_whose_llm_was_built_outside_the_block_is_mapped_by_its_model() -> (+ None+):+ """The declared instance is looked up by its model, with the configuration+ carried like a role's swap."""+ declared = _configured_llm()+ with llm_overlay({"model:openai/gpt-4o-mini": "openai/gpt-4o"}):+ agent = Agent(role="Writer", goal="g", backstory="b", llm=declared)+ lite = LiteAgent(role="Writer", goal="g", backstory="b", llm=declared)++ for built in (agent.llm, lite.llm):+ assert built is not declared and built.model == "gpt-4o"+ assert _configuration_of(built) == CONFIGURATION+++def test_a_role_key_wins_over_model_keys_for_that_agent() -> None:+ with llm_overlay(+ {+ "Researcher": "openai/gpt-4o",+ "model:*": "openai/gpt-4.1-nano",+ }+ ):+ researcher = _agent("Researcher")+ writer = _agent("Writer")+ outside = Agent(+ role="Researcher", goal="g", backstory="b", llm=_configured_llm()+ )++ assert researcher.llm.model == "gpt-4o"+ assert writer.llm.model == "gpt-4.1-nano"+ assert outside.llm.model == "gpt-4o"+++def test_a_role_that_looks_like_a_model_key_is_never_one() -> None:+ with llm_overlay({"model:*": "openai/gpt-4o"}):+ assert overlay_model_for("model:*") is None+++def test_a_model_key_that_names_no_model_is_refused() -> None:+ with pytest.raises(ValueError, match="names no model"):+ with llm_overlay({"model: ": "openai/gpt-4o"}):+ pass+++def test_a_subclass_of_llm_keeps_its_model() -> None:+ """Only `LLM` itself routes; a subclass is the caller's own choice of class."""+ pytest.importorskip("litellm")++ class Mine(LLM):+ pass++ with llm_overlay({"model:*": "openai/gpt-4o"}):+ mine = Mine(model="groq/llama-3.1-8b-instant")++ assert type(mine) is Mine and mine.model == "groq/llama-3.1-8b-instant"+++def test_a_declared_model_whose_sdk_is_missing_still_maps(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Nothing is built on the declared model, so its SDK need not be here; its+ key cannot be matched to the new provider, so it stays behind."""+ native = LLM._get_native_provider.__func__ # type: ignore[attr-defined]++ def without_anthropic(cls: type[LLM], provider: str) -> Any:+ if provider in ("anthropic", "claude"):+ raise ImportError("Anthropic native provider not available")+ return native(cls, provider)++ monkeypatch.setattr(LLM, "_get_native_provider", classmethod(without_anthropic))+ with llm_overlay({"model:*": "openai/gpt-4o-mini"}):+ built = LLM(model="anthropic/claude-haiku-4-5", api_key="k", temperature=0.2)++ assert type(built).__name__ == "OpenAICompletion" and built.model == "gpt-4o-mini"+ assert built.temperature == 0.2 and built.api_key != "k"+++def test_a_litellm_routed_llm_keeps_its_key_on_the_same_provider_only() -> None:+ """Same provider is a question about the provider, not the class: an+ `LLM(...)` the caller routed through LiteLLM keeps its key and endpoint+ when mapped to another model of that provider, and another provider's+ model never gets them."""+ pytest.importorskip("litellm")+ declared = {"api_key": "k", "base_url": "http://localhost:9999/v1"}+ with llm_overlay({"model:*": "openai/gpt-4o-mini"}):+ same = LLM(model="openai/gpt-4o", is_litellm=True, **declared)+ with llm_overlay({"model:*": "anthropic/claude-haiku-4-5"}):+ other = LLM(model="openai/gpt-4o", is_litellm=True, **declared)++ assert "gpt-4o-mini" in same.model+ assert same.api_key == "k" and same.base_url == "http://localhost:9999/v1"+ assert "claude-haiku-4-5" in other.model+ assert other.api_key != "k" and other.base_url != "http://localhost:9999/v1"+++ ("built", "key"),+ [+ ("openrouter/openai/gpt-4o", "model:openai/gpt-4o"),+ ("openrouter/openai/gpt-4o", "model:gpt-4o"),+ ],+)+def test_an_aggregators_route_is_never_the_native_model_it_names(+ built: str, key: str+) -> None:+ """`openrouter/openai/gpt-4o` is OpenRouter's model, not OpenAI's: a key+ written for the native model leaves it alone."""+ with llm_overlay({key: "openai/gpt-4.1"}):+ llm = LLM(model=built)++ assert type(llm).__name__ == "OpenAICompatibleCompletion"+ assert llm.model == "openai/gpt-4o"+++def test_an_aggregators_route_is_matched_by_its_own_full_name() -> None:+ declared = LLM(model="openrouter/openai/gpt-4o")+ with llm_overlay({"model:openrouter/openai/gpt-4o": "openai/gpt-4.1"}):+ built = LLM(model="openrouter/openai/gpt-4o")+ agent = Agent(role="Writer", goal="g", backstory="b", llm=declared)+ with llm_overlay({"model:openai/gpt-4o": "openai/gpt-4.1"}):+ untouched = Agent(role="Writer", goal="g", backstory="b", llm=declared)++ assert built.model == "gpt-4.1" and agent.llm.model == "gpt-4.1"+ assert untouched.llm is declared+++def test_a_native_provider_prefix_is_still_its_own() -> None:+ with llm_overlay({"model:llama3": "ollama/qwen3"}):+ assert LLM(model="ollama/llama3").model == "qwen3"+++def test_a_role_key_is_built_from_the_declaration_a_model_key_mapped() -> None:+ """Role wins, with the caller's declared settings: the llm a model key+ swapped on the way in is not what the role's model is built like."""+ overlay = {"Researcher": "openai/gpt-4o", "model:*": "anthropic/claude-haiku-4-5"}+ with llm_overlay(overlay):+ declared = _configured_llm() # mapped to Anthropic, without the key+ researcher = Agent(role="Researcher", goal="g", backstory="b", llm=declared)+ by_string = Agent(+ role="Researcher", goal="g", backstory="b", llm="openai/gpt-4o-mini"+ )+ writer = Agent(role="Writer", goal="g", backstory="b", llm=declared)++ assert type(declared).__name__ == "AnthropicCompletion"+ assert type(researcher.llm).__name__ == "OpenAICompletion"+ assert researcher.llm.model == "gpt-4o"+ assert _configuration_of(researcher.llm) == CONFIGURATION+ assert by_string.llm.model == "gpt-4o"+ assert writer.llm is declared+++def test_a_providers_other_name_is_the_same_model_on_both_paths(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """`google/x` routes as `gemini/x`, so a key written either way matches an+ llm built in the block and one built before it, whichever name it used."""+ pytest.importorskip("google.genai")+ monkeypatch.setenv("GEMINI_API_KEY", "g")+ outside = LLM(model="google/gemini-2.5-pro") # records provider "gemini"+ for key in ("model:google/gemini-2.5-pro", "model:gemini/gemini-2.5-pro"):+ with llm_overlay({key: "openai/gpt-4o-mini"}):+ built = [LLM(model="google/gemini-2.5-pro"), LLM(model="gemini/gemini-2.5-pro")]+ agent = Agent(role="Writer", goal="g", backstory="b", llm=outside)++ assert [b.model for b in built] == ["gpt-4o-mini", "gpt-4o-mini"], key+ assert agent.llm.model == "gpt-4o-mini", key+++def test_an_aggregator_route_under_an_alias_still_is_not_the_native_model() -> None:+ with llm_overlay({"model:google/gemini-2.5-pro": "openai/gpt-4o-mini"}):+ llm = LLM(model="openrouter/google/gemini-2.5-pro")++ assert llm.model == "google/gemini-2.5-pro"+++def test_a_caller_who_chose_litellm_keeps_it_when_the_declared_sdk_is_missing(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ pytest.importorskip("litellm")+ native = LLM._get_native_provider.__func__ # type: ignore[attr-defined]++ def without_anthropic(cls: type[LLM], provider: str) -> Any:+ if provider in ("anthropic", "claude"):+ raise ImportError("Anthropic native provider not available")+ return native(cls, provider)++ monkeypatch.setattr(LLM, "_get_native_provider", classmethod(without_anthropic))+ with llm_overlay({"model:*": "openai/gpt-4o-mini"}):+ chosen = LLM(model="anthropic/claude-haiku-4-5", is_litellm=True)+ default = LLM(model="anthropic/claude-haiku-4-5")++ assert type(chosen).__name__ == "LLM" and chosen.is_litellm+ assert type(default).__name__ == "OpenAICompletion"diff --git a/pyproject.toml b/pyproject.tomlindex 5ad51f65e..97a0e38b6 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -209,6 +209,9 @@ exclude-newer-package = { msgpack = "2026-06-20T00:00:00Z", pydantic-settings =# global 3-day cutoff is now far later than that date, so the per-package pin only blocked the fix.# pyasn1 <0.6.4 has GHSA-8ppf-4f7h-5ppj and GHSA-hm4w-wwcw-mr6r; force 0.6.4+.# urllib3 <2.7.0 has GHSA-qccp-gfcp-xxvc (ProxyManager cross-origin redirect leaks Authorization/Cookie) and GHSA-mf9v-mfxr-j63j (streaming decompression-bomb bypass); force 2.7.0+.+# urllib3 <2.8.0 has GHSA-8988-9cw3-xx77 (HTTPS proxy TLS configuration ignored), GHSA-gh4c-6fx4-qh6g (chunked+# deflate infinite loop) and GHSA-vxq7-64xx-v4gw (unbounded chunk-size line); force 2.8.0+.+# virtualenv <21.7.13 has PYSEC-2026-4011..4014 (unchecked seed wheels, pyvenv.cfg and activator injection); force 21.7.13+.# langsmith <0.8.18 has GHSA-3644-q5cj-c5c7 (public prompt manifest deserialization, SSRF/secret disclosure)# and GHSA-f4xh-w4cj-qxq8; force 0.8.18+.# authlib <1.6.12 has GHSA-jj8c-mmj3-mmgv (CSRF bypass in cache-based state storage) and PYSEC-2026-188.@@ -269,7 +272,8 @@ override-dependencies = ["pillow>=12.3.0","langchain-core>=1.3.3,<2","langchain-text-splitters>=1.1.2,<2",- "urllib3>=2.7.0",+ "urllib3>=2.8.0",+ "virtualenv>=21.7.13","transformers>=5.4.0; python_version >= '3.10'","cryptography>=50.0.0","pypdf>=6.16.1,<7",diff --git a/uv.lock b/uv.lockindex b589d1790..ffb2d49cc 100644--- a/uv.lock+++ b/uv.lock@@ -64,8 +64,9 @@ overrides = [{ name = "starlette", specifier = ">=1.3.1" },{ name = "torch", specifier = ">=2.13.0" },{ name = "transformers", marker = "python_full_version >= '3.10'", specifier = ">=5.4.0" },- { name = "urllib3", specifier = ">=2.7.0" },+ { name = "urllib3", specifier = ">=2.8.0" },{ name = "uv", specifier = ">=0.11.15,<1" },+ { name = "virtualenv", specifier = ">=21.7.13" },][manifest.dependency-groups]@@ -1571,7 +1572,7 @@ requires-dist = [{ name = "portalocker", specifier = "~=2.7.0" },{ name = "pydantic", specifier = ">=2.11.9,<2.13" },{ name = "pydantic-settings", specifier = ">=2.14.2,<3" },- { name = "pyjwt", specifier = ">=2.13.0,<3" },+ { name = "pyjwt", specifier = ">=2.15.0,<3" },{ name = "python-dotenv", specifier = ">=1.2.2,<2" },{ name = "pyyaml", specifier = "~=6.0" },{ name = "qdrant-client", extras = ["fastembed"], marker = "extra == 'qdrant'", specifier = "~=1.14.3" },@@ -1618,7 +1619,7 @@ requires-dist = [{ name = "packaging", specifier = ">=23.0" },{ name = "pydantic", specifier = ">=2.11.9,<2.13" },{ name = "pydantic-settings", specifier = ">=2.14.2,<3" },- { name = "pyjwt", specifier = ">=2.13.0,<3" },+ { name = "pyjwt", specifier = ">=2.15.0,<3" },{ name = "python-dotenv", specifier = ">=1.2.2,<2" },{ name = "rich", specifier = ">=13.7.1" },{ name = "textual", specifier = ">=7.5.0" },@@ -1656,7 +1657,7 @@ requires-dist = [{ name = "packaging", specifier = ">=23.0" },{ name = "portalocker", specifier = "~=2.7.0" },{ name = "pydantic", specifier = ">=2.11.9,<2.13" },- { name = "pyjwt", specifier = ">=2.13.0,<3" },+ { name = "pyjwt", specifier = ">=2.15.0,<3" },{ name = "rich", specifier = ">=13.7.1" },{ name = "tomli", specifier = "~=2.0.2" },]@@ -7353,14 +7354,14 @@ wheels = [[[package]]name = "pyjwt"-version = "2.13.0"+version = "2.15.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "typing-extensions", marker = "python_full_version < '3.11'" },]-sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" }+sdist = { url = "https://files.pythonhosted.org/packages/02/a5/5197bfd06417837ac079921c66fa6393f1dea3557272a263cebfef69e432/pyjwt-2.15.0.tar.gz", hash = "sha256:b11c5f9791d7bf51c2b39a81ed669f6b2dbbd669df2942f6c60167e9e3d1abe4", size = 120513, upload-time = "2026-09-23T16:56:00.689Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" },+ { url = "https://files.pythonhosted.org/packages/e8/55/40e45bf052ee8ee12a4dfd785519660f8effa7b065442b91646ec6828619/pyjwt-2.15.0-py3-none-any.whl", hash = "sha256:7a3742debf6b879e912dbb9819ceec1594be812452b78c5f2e2dfc56564954f8", size = 33680, upload-time = "2026-09-23T16:55:59.241Z" },][package.optional-dependencies]@@ -7727,15 +7728,14 @@ wheels = [[[package]]name = "python-discovery"-version = "1.4.0"+version = "1.6.1"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "filelock" },- { name = "platformdirs" },]-sdist = { url = "https://files.pythonhosted.org/packages/a6/12/38c1a0b1e64806780c9563e3fc9f6e472251839662587cfbe9bfaf2ae10a/python_discovery-1.4.0.tar.gz", hash = "sha256:eb8bc7daad3c226c147e45bb4e970a1feb1bf4048ee178e6db59e197b8010ce3", size = 68455, upload-time = "2026-05-28T01:15:37.639Z" }+sdist = { url = "https://files.pythonhosted.org/packages/0c/57/250bd238b966cece44328235eb85290045d059265fdaf7527a3a958123db/python_discovery-1.6.1.tar.gz", hash = "sha256:cf87d3627dfb4412437fdd5b13eae402607722998d21567993aedbc59b23c15e", size = 84338, upload-time = "2026-09-18T01:31:53.971Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/c8/8d/3d316429f65029532bb1e28ff77b797d86b5ac3915bb44ca4e19aa283d43/python_discovery-1.4.0-py3-none-any.whl", hash = "sha256:26ed78d703e234879a66244c7d4114563fb13ec5cd30a2d1357e5fb4850782da", size = 33217, upload-time = "2026-05-28T01:15:36.573Z" },+ { url = "https://files.pythonhosted.org/packages/16/7d/e9ffbadfbf89c93848412d04594135c4ae8c1d37d9e053b9c3ed718fabc4/python_discovery-1.6.1-py3-none-any.whl", hash = "sha256:d43fcdef879fe795352bd13ccf8d185ba5a9f86f36cfcd00529f596e737442b3", size = 38664, upload-time = "2026-09-18T01:31:52.448Z" },][[package]]@@ -10152,11 +10152,11 @@ wheels = [[[package]]name = "urllib3"-version = "2.7.0"+version = "2.8.0"source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" }+sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" },+ { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" },][[package]]@@ -10351,7 +10351,7 @@ wheels = [[[package]]name = "virtualenv"-version = "21.4.2"+version = "21.13.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "distlib" },@@ -10360,9 +10360,9 @@ dependencies = [{ name = "python-discovery" },{ name = "typing-extensions", marker = "python_full_version < '3.11'" },]-sdist = { url = "https://files.pythonhosted.org/packages/e1/0d/4e93c8e6d1001a75763f87d8f5ecda8ebc7f4aa2153dddfaf4ae8892821a/virtualenv-21.4.2.tar.gz", hash = "sha256:38e6ee0a555615c0ea9da2ac7e9998fe8dc3b911dd33ad8eaad2020957653b0c", size = 7613326, upload-time = "2026-05-31T17:01:22.827Z" }+sdist = { url = "https://files.pythonhosted.org/packages/31/0b/825cbfd46beb2cc96c46403141081190c1afd1a348d0232a4ebcb6dcd362/virtualenv-21.13.0.tar.gz", hash = "sha256:e8aa144aabba43ffd9058e1fd1a02012b1915f8cb915acc16ad50aba9c13bfdc", size = 5399552, upload-time = "2026-09-26T16:42:09.321Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/bf/c4/557dc082be035381b85fdb2b74e21d3d21b57750b74f2b47a32f3a639ff9/virtualenv-21.4.2-py3-none-any.whl", hash = "sha256:854210ca524a1a4d0d744734f4acbc721c3ffe163b85bbf5d56d14d5ae2f0fae", size = 7594079, upload-time = "2026-05-31T17:01:20.735Z" },+ { url = "https://files.pythonhosted.org/packages/c5/2c/8b107f9e029da006c39af38a0a150dde57ecce48b495c79917d1ad963821/virtualenv-21.13.0-py3-none-any.whl", hash = "sha256:ca16ceba89e58d8e2f376769349c63f3ed66f79881ae68bc13e19a9d50e0e987", size = 5486498, upload-time = "2026-09-26T16:42:07.106Z" },][[package]]