Skip to content

Self-improving AI Employees for everyone

fix(llm): default max_tokens to the model's max output, not the 8192 provider floor

trohitgee8bd2467b32
diff --git a/server/services/ai.py b/server/services/ai.py
index b3c976cf..1992c2b5 100644
--- a/server/services/ai.py
+++ b/server/services/ai.py
@@ -354,27 +354,16 @@ def _get_default_model(provider: str, fallback: str) -> str:
def _resolve_max_tokens(flattened: dict, model: str, provider: str) -> int:
- """Resolve max_tokens: user param (clamped to model max) -> provider default -> 4096.
-
- When user sets max_tokens explicitly, clamp to model's actual maximum.
- When user doesn't set it, use the provider's default from llm_defaults.json.
+ """Resolve max_tokens: user param (clamped to model max) -> model max.
+
+ Delegates to ``services/llm/config.py::resolve_max_tokens`` so the
+ agent paths (execute_agent / execute_chat_agent / F4.B
+ prepare_agent_payload) and the native chat path agree: when the user
+ doesn't set max_tokens, the model's full supported output budget is
+ used (registry -> llm_defaults fallback), never an artificial
+ provider-wide floor.
"""
- from services.model_registry import get_model_registry
-
- registry = get_model_registry()
- model_max = registry.get_max_output_tokens(model, provider)
-
- user_val = flattened.get("max_tokens")
- if user_val:
- user_int = int(user_val)
- if user_int > model_max:
- logger.info(f"[AI] Clamping max_tokens from {user_int} to {model_max} for {provider}/{model}")
- return model_max
- return user_int
-
- # No user value: use provider default from llm_defaults.json, not model's max capacity
- provider_default = registry._get_default_max_output_tokens(provider, "_no_match_")
- return min(provider_default, model_max)
+ return native_resolve_max_tokens(flattened, model, provider)
def _resolve_temperature(flattened: dict, model: str, provider: str, thinking_enabled: bool) -> float:
diff --git a/server/services/model_registry.py b/server/services/model_registry.py
index 1fb96f13..c6c85f94 100644
--- a/server/services/model_registry.py
+++ b/server/services/model_registry.py
@@ -438,6 +438,11 @@ class ModelRegistryService:
return None
or_provider, local_id = model_id.split("/", 1)
+ # OpenRouter "~provider" alias rows (e.g. ~google/gemini-flash-latest)
+ # carry the same metadata as canonical rows — strip the tilde so they
+ # key under the canonical provider and local lookups
+ # (get_model_info("gemini-flash-latest", "gemini")) match.
+ or_provider = or_provider.lstrip("~")
provider = PROVIDER_MAP.get(or_provider, or_provider)
# Extract top_provider data
@@ -521,9 +526,15 @@ class ModelRegistryService:
raw_models = data.get("models", {})
for key, model_data in raw_models.items():
try:
- self._models[key] = ModelInfo.from_dict(model_data)
+ info = ModelInfo.from_dict(model_data)
except (TypeError, KeyError) as e:
logger.debug(f"Skipping invalid model cache entry {key}: {e}")
+ continue
+ # Same "~provider" alias normalization as _parse_openrouter_model,
+ # for caches written before it landed.
+ stripped = info.provider.lstrip("~")
+ info.provider = PROVIDER_MAP.get(stripped, stripped)
+ self._models[f"{info.provider}/{info.local_id}"] = info
except Exception as e:
logger.warning(f"Failed to load model registry cache: {e}")
diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.py
index 953157b4..3fa55009 100644
--- a/server/services/temporal/worker.py
+++ b/server/services/temporal/worker.py
@@ -363,6 +363,13 @@ async def run_standalone_worker(
pool_size=pool_size,
)
+ # Embedded workers get this from main.py's lifespan; standalone
+ # workers must load the model registry themselves or max_tokens /
+ # context_length lookups degrade to hard fallbacks (4096).
+ from services.model_registry import get_model_registry
+
+ get_model_registry().startup()
+
# Use custom runtime with heartbeating disabled to avoid warning on older servers
runtime = create_runtime()
diff --git a/server/tests/llm/test_max_tokens_resolution.py b/server/tests/llm/test_max_tokens_resolution.py
new file mode 100644
index 00000000..5b368e18
--- /dev/null
+++ b/server/tests/llm/test_max_tokens_resolution.py
@@ -0,0 +1,136 @@
+"""max_tokens resolution contract — default is the MODEL's max output.
+
+Two resolvers exist (native chat path in ``services/llm/config.py`` and
+the agent path in ``services/ai.py``); the agent one must delegate to
+the native one so all four call sites agree (execute_chat,
+execute_agent, execute_chat_agent, and the Temporal F4.B
+``prepare_agent_payload``):
+
+- user value -> clamped to the model's max output tokens
+- no user value -> the model's max output tokens (registry ->
+ llm_defaults fallback), never an artificial provider-wide floor
+ (the old behaviour capped agents at the 8192 ``_default`` from
+ llm_defaults.json regardless of model capability).
+"""
+
+from __future__ import annotations
+
+import inspect
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+pytestmark = pytest.mark.unit
+
+MODEL_MAX = 65536
+
+
+def _registry_mock():
+ registry = MagicMock(name="ModelRegistry")
+ registry.get_max_output_tokens.return_value = MODEL_MAX
+ return registry
+
+
+class TestNativeResolver:
+ def test_no_user_value_defaults_to_model_max(self):
+ from services.llm.config import resolve_max_tokens
+
+ with patch("services.model_registry.get_model_registry", return_value=_registry_mock()):
+ assert resolve_max_tokens({}, "gemini-flash-latest", "gemini") == MODEL_MAX
+
+ def test_user_value_clamped_to_model_max(self):
+ from services.llm.config import resolve_max_tokens
+
+ with patch("services.model_registry.get_model_registry", return_value=_registry_mock()):
+ assert resolve_max_tokens({"max_tokens": 100_000}, "m", "p") == MODEL_MAX
+
+ def test_user_value_below_max_respected(self):
+ from services.llm.config import resolve_max_tokens
+
+ with patch("services.model_registry.get_model_registry", return_value=_registry_mock()):
+ assert resolve_max_tokens({"max_tokens": 2048}, "m", "p") == 2048
+
+
+class TestAgentResolverDelegates:
+ def test_agent_resolver_matches_native_default(self):
+ from services.ai import _resolve_max_tokens
+
+ with patch("services.model_registry.get_model_registry", return_value=_registry_mock()):
+ assert _resolve_max_tokens({}, "gemini-flash-latest", "gemini") == MODEL_MAX
+
+ def test_agent_resolver_delegates_to_native(self):
+ # Source invariant: no duplicated resolution logic — the agent
+ # path must call the native resolver so the two can't drift.
+ from services import ai
+
+ src = inspect.getsource(ai._resolve_max_tokens)
+ assert "native_resolve_max_tokens" in src
+
+ def test_temporal_prepare_payload_uses_agent_resolver(self):
+ # The F4.B path must resolve max_tokens through the same helper.
+ from services.temporal.agent_activities import prepare_agent_payload
+
+ src = inspect.getsource(prepare_agent_payload)
+ assert "_resolve_max_tokens" in src
+
+
+class TestRegistryAliasNormalization:
+ """OpenRouter "~provider" alias rows (~google/gemini-flash-latest)
+ must key under the canonical provider, or get_model_info misses the
+ registry and max_tokens/context_length degrade to llm_defaults
+ fallbacks."""
+
+ def test_parse_normalizes_tilde_provider(self):
+ from services.model_registry import ModelRegistryService
+
+ svc = ModelRegistryService()
+ info = svc._parse_openrouter_model(
+ {
+ "id": "~google/gemini-test-latest",
+ "name": "Google: Gemini Test",
+ "context_length": 1_000_000,
+ "top_provider": {"max_completion_tokens": 65536},
+ }
+ )
+ assert info is not None
+ assert info.provider == "gemini"
+ assert info.local_id == "gemini-test-latest"
+
+ def test_load_cache_normalizes_tilde_keys(self, tmp_path, monkeypatch):
+ import services.model_registry as mr
+
+ cache = tmp_path / "model_registry.json"
+ cache.write_text(
+ __import__("json").dumps(
+ {
+ "models": {
+ "~google/gemini-test-latest": {
+ "id": "~google/gemini-test-latest",
+ "name": "Google: Gemini Test",
+ "provider": "~google",
+ "local_id": "gemini-test-latest",
+ "context_length": 1_000_000,
+ "max_output_tokens": 65536,
+ }
+ }
+ }
+ ),
+ encoding="utf-8",
+ )
+ monkeypatch.setattr(mr, "CACHE_FILE", cache)
+ svc = mr.ModelRegistryService()
+ svc._load_cache()
+ info = svc.get_model_info("gemini-test-latest", "gemini")
+ assert info is not None
+ assert info.max_output_tokens == 65536
+
+
+class TestStandaloneWorkerRegistryStartup:
+ def test_standalone_worker_loads_model_registry(self):
+ # Without startup(), a standalone worker's registry is empty and
+ # every agent resolves max_tokens to the hard 4096 fallback.
+ from services.temporal.worker import run_standalone_worker
+
+ src = inspect.getsource(run_standalone_worker)
+ assert "get_model_registry().startup()" in src