Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
feat(llm): add llm_overlay contextvar to route agent roles to models (#7500)
João Mourac1beddf1a610
diff --git a/lib/crewai/src/crewai/agent/core.py b/lib/crewai/src/crewai/agent/core.pyindex 7f5bfcd5f..60a0dc2db 100644--- a/lib/crewai/src/crewai/agent/core.py+++ b/lib/crewai/src/crewai/agent/core.py@@ -79,6 +79,7 @@ from crewai.hooks.dispatch import HookAbortedfrom crewai.knowledge.knowledge import Knowledgefrom crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSourcefrom crewai.lite_agent_output import LiteAgentOutput+from crewai.llm_overlay import overlay_model_forfrom crewai.llms.base_llm import BaseLLMfrom crewai.mcp.config import MCPServerConfigfrom crewai.rag.embeddings.types import EmbedderConfig@@ -400,7 +401,7 @@ class Agent(BaseAgent):@model_validator(mode="after")def post_init_setup(self) -> Self:"""Initialize LLM, executor, code tools, and skills after model creation."""- self.llm = create_llm(self.llm)+ self.llm = create_llm(overlay_model_for(self.role) or self.llm)if self.function_calling_llm and not isinstance(self.function_calling_llm, BaseLLM):diff --git a/lib/crewai/src/crewai/lite_agent.py b/lib/crewai/src/crewai/lite_agent.pyindex fe393478b..3f3d40266 100644--- a/lib/crewai/src/crewai/lite_agent.py+++ b/lib/crewai/src/crewai/lite_agent.py@@ -69,6 +69,7 @@ from crewai.hooks.types import ()from crewai.lite_agent_output import LiteAgentOutputfrom crewai.llm import LLM+from crewai.llm_overlay import overlay_model_forfrom crewai.llms.base_llm import BaseLLMfrom crewai.tools.base_tool import BaseToolfrom crewai.tools.structured_tool import CrewStructuredTool@@ -319,7 +320,7 @@ class LiteAgent(FlowTrackable, BaseModel):@model_validator(mode="after")def setup_llm(self) -> Self:"""Set up the LLM and other components after initialization."""- self.llm = create_llm(self.llm)+ self.llm = create_llm(overlay_model_for(self.role) or self.llm)if not isinstance(self.llm, BaseLLM):raise ValueError(f"Expected LLM instance of type BaseLLM, got {type(self.llm).__name__}"diff --git a/lib/crewai/src/crewai/llm_overlay.py b/lib/crewai/src/crewai/llm_overlay.pynew file mode 100644index 000000000..f077f23cf--- /dev/null+++ b/lib/crewai/src/crewai/llm_overlay.py@@ -0,0 +1,57 @@+"""Process-context ``role -> model`` overlay for agents.++``llm_overlay`` lets a per-run caller make specific agents use a different+model without editing the code that builds them. While the block is active,+an ``Agent`` or ``LiteAgent`` whose ``role`` is a key of the mapping is built+with the mapped model instead of its declared ``llm``. Roles that are not+keys, and every agent built outside the block, keep their own model.++The overlay is a :class:`contextvars.ContextVar`, so it follows the calling+context, not the process. A plain ``threading.Thread`` started inside the+block does not see it: callers that run agents in their own threads must+propagate the context themselves, e.g.+``contextvars.copy_context().run(build_and_run_agents)``.++Example:+ >>> with llm_overlay({"Researcher": "openai/gpt-4o"}):+ ... crew = build_crew() # the Researcher agent is built on gpt-4o+ ... crew.kickoff()+"""++from __future__ import annotations++from collections.abc import Iterator+from contextlib import contextmanager+from contextvars import ContextVar+++active: ContextVar[dict[str, str] | None] = ContextVar(+ "llm_overlay_active", default=None+)+++@contextmanager+def llm_overlay(mapping: dict[str, str] | None) -> Iterator[None]:+ """Route agent roles to models for the duration of the block.++ Args:+ mapping: ``{role: model}`` for the block; ``None`` clears any active+ overlay for the block. The previous value is always restored on+ exit, including when the block raises.+ """+ token = active.set(mapping)+ try:+ yield+ finally:+ active.reset(token)+++def overlay_model_for(role: str) -> str | None:+ """The model the active overlay assigns to ``role``.++ Returns:+ The mapped model string, or ``None`` when no overlay is active or+ ``role`` is not one of its keys.+ """+ mapping = active.get()+ return mapping.get(role) if mapping else Nonediff --git a/lib/crewai/tests/test_llm_overlay.py b/lib/crewai/tests/test_llm_overlay.pynew file mode 100644index 000000000..708ed89af--- /dev/null+++ b/lib/crewai/tests/test_llm_overlay.py@@ -0,0 +1,89 @@+"""`llm_overlay` swaps an agent's model by role, for the calling context only.++The overlay is read in exactly two places, the validators where `Agent` and+`LiteAgent` resolve their `llm`, so these tests build agents and look at the+model they end up with. No LLM is ever called.++`create_llm("openai/gpt-4o")` returns the native OpenAI provider, which strips+the `openai/` prefix, so the resolved model reads `"gpt-4o"`.+"""++from __future__ import annotations++import contextvars+import threading++from crewai import Agent+from crewai.lite_agent import LiteAgent+from crewai.llm_overlay import active, llm_overlay, overlay_model_for+import pytest+++OVERLAY = {"Researcher": "openai/gpt-4o"}+++def _agent(role: str) -> Agent:+ return Agent(role=role, goal="g", backstory="b", llm="openai/gpt-4o-mini")+++def test_matching_role_gets_the_overlay_model_others_keep_their_own() -> None:+ with llm_overlay(OVERLAY):+ researcher = _agent("Researcher")+ writer = _agent("Writer")++ assert researcher.llm.model == "gpt-4o"+ assert writer.llm.model == "gpt-4o-mini"+++def test_overlay_does_not_leak_past_the_block() -> None:+ with llm_overlay(OVERLAY):+ assert overlay_model_for("Researcher") == "openai/gpt-4o"++ assert active.get() is None+ assert overlay_model_for("Researcher") is None+ assert _agent("Researcher").llm.model == "gpt-4o-mini"+++def test_overlay_is_reset_when_the_block_raises() -> None:+ with pytest.raises(RuntimeError), llm_overlay(OVERLAY):+ raise RuntimeError("boom")++ assert active.get() is None+++def test_nested_overlay_restores_the_outer_one() -> None:+ with llm_overlay(OVERLAY):+ with llm_overlay(None):+ assert overlay_model_for("Researcher") is None+ assert overlay_model_for("Researcher") == "openai/gpt-4o"++[email protected]("ignore:LiteAgent is deprecated")+def test_lite_agent_gets_the_overlay_model() -> None:+ with llm_overlay(OVERLAY):+ agent = LiteAgent(+ role="Researcher", goal="g", backstory="b", llm="openai/gpt-4o-mini"+ )++ assert agent.llm.model == "gpt-4o"+++def test_overlay_does_not_cross_plain_threads_unless_context_is_copied() -> None:+ """Pins contextvar semantics; callers threading agents must copy the context."""+ seen: dict[str, dict[str, str] | None] = {}++ def record(key: str) -> None:+ seen[key] = active.get()++ with llm_overlay(OVERLAY):+ plain = threading.Thread(target=record, args=("plain",))+ plain.start()+ plain.join()++ ctx = contextvars.copy_context()+ copied = threading.Thread(target=ctx.run, args=(record, "copied"))+ copied.start()+ copied.join()++ assert seen["plain"] is None+ assert seen["copied"] == OVERLAY