Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(llm): default max_tokens to the model's max output, not the 8192 provider floor
trohitgee8bd2467b32
diff --git a/server/services/ai.py b/server/services/ai.pyindex b3c976cf..1992c2b5 100644--- a/server/services/ai.py+++ b/server/services/ai.py@@ -354,27 +354,16 @@ def _get_default_model(provider: str, fallback: str) -> str:def _resolve_max_tokens(flattened: dict, model: str, provider: str) -> int:- """Resolve max_tokens: user param (clamped to model max) -> provider default -> 4096.-- When user sets max_tokens explicitly, clamp to model's actual maximum.- When user doesn't set it, use the provider's default from llm_defaults.json.+ """Resolve max_tokens: user param (clamped to model max) -> model max.++ Delegates to ``services/llm/config.py::resolve_max_tokens`` so the+ agent paths (execute_agent / execute_chat_agent / F4.B+ prepare_agent_payload) and the native chat path agree: when the user+ doesn't set max_tokens, the model's full supported output budget is+ used (registry -> llm_defaults fallback), never an artificial+ provider-wide floor."""- from services.model_registry import get_model_registry-- registry = get_model_registry()- model_max = registry.get_max_output_tokens(model, provider)-- user_val = flattened.get("max_tokens")- if user_val:- user_int = int(user_val)- if user_int > model_max:- logger.info(f"[AI] Clamping max_tokens from {user_int} to {model_max} for {provider}/{model}")- return model_max- return user_int-- # No user value: use provider default from llm_defaults.json, not model's max capacity- provider_default = registry._get_default_max_output_tokens(provider, "_no_match_")- return min(provider_default, model_max)+ return native_resolve_max_tokens(flattened, model, provider)def _resolve_temperature(flattened: dict, model: str, provider: str, thinking_enabled: bool) -> float:diff --git a/server/services/model_registry.py b/server/services/model_registry.pyindex 1fb96f13..c6c85f94 100644--- a/server/services/model_registry.py+++ b/server/services/model_registry.py@@ -438,6 +438,11 @@ class ModelRegistryService:return Noneor_provider, local_id = model_id.split("/", 1)+ # OpenRouter "~provider" alias rows (e.g. ~google/gemini-flash-latest)+ # carry the same metadata as canonical rows — strip the tilde so they+ # key under the canonical provider and local lookups+ # (get_model_info("gemini-flash-latest", "gemini")) match.+ or_provider = or_provider.lstrip("~")provider = PROVIDER_MAP.get(or_provider, or_provider)# Extract top_provider data@@ -521,9 +526,15 @@ class ModelRegistryService:raw_models = data.get("models", {})for key, model_data in raw_models.items():try:- self._models[key] = ModelInfo.from_dict(model_data)+ info = ModelInfo.from_dict(model_data)except (TypeError, KeyError) as e:logger.debug(f"Skipping invalid model cache entry {key}: {e}")+ continue+ # Same "~provider" alias normalization as _parse_openrouter_model,+ # for caches written before it landed.+ stripped = info.provider.lstrip("~")+ info.provider = PROVIDER_MAP.get(stripped, stripped)+ self._models[f"{info.provider}/{info.local_id}"] = infoexcept Exception as e:logger.warning(f"Failed to load model registry cache: {e}")diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex 953157b4..3fa55009 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -363,6 +363,13 @@ async def run_standalone_worker(pool_size=pool_size,)+ # Embedded workers get this from main.py's lifespan; standalone+ # workers must load the model registry themselves or max_tokens /+ # context_length lookups degrade to hard fallbacks (4096).+ from services.model_registry import get_model_registry++ get_model_registry().startup()+# Use custom runtime with heartbeating disabled to avoid warning on older serversruntime = create_runtime()diff --git a/server/tests/llm/test_max_tokens_resolution.py b/server/tests/llm/test_max_tokens_resolution.pynew file mode 100644index 00000000..5b368e18--- /dev/null+++ b/server/tests/llm/test_max_tokens_resolution.py@@ -0,0 +1,136 @@+"""max_tokens resolution contract — default is the MODEL's max output.++Two resolvers exist (native chat path in ``services/llm/config.py`` and+the agent path in ``services/ai.py``); the agent one must delegate to+the native one so all four call sites agree (execute_chat,+execute_agent, execute_chat_agent, and the Temporal F4.B+``prepare_agent_payload``):++- user value -> clamped to the model's max output tokens+- no user value -> the model's max output tokens (registry ->+ llm_defaults fallback), never an artificial provider-wide floor+ (the old behaviour capped agents at the 8192 ``_default`` from+ llm_defaults.json regardless of model capability).+"""++from __future__ import annotations++import inspect+from unittest.mock import MagicMock, patch++import pytest+++pytestmark = pytest.mark.unit++MODEL_MAX = 65536+++def _registry_mock():+ registry = MagicMock(name="ModelRegistry")+ registry.get_max_output_tokens.return_value = MODEL_MAX+ return registry+++class TestNativeResolver:+ def test_no_user_value_defaults_to_model_max(self):+ from services.llm.config import resolve_max_tokens++ with patch("services.model_registry.get_model_registry", return_value=_registry_mock()):+ assert resolve_max_tokens({}, "gemini-flash-latest", "gemini") == MODEL_MAX++ def test_user_value_clamped_to_model_max(self):+ from services.llm.config import resolve_max_tokens++ with patch("services.model_registry.get_model_registry", return_value=_registry_mock()):+ assert resolve_max_tokens({"max_tokens": 100_000}, "m", "p") == MODEL_MAX++ def test_user_value_below_max_respected(self):+ from services.llm.config import resolve_max_tokens++ with patch("services.model_registry.get_model_registry", return_value=_registry_mock()):+ assert resolve_max_tokens({"max_tokens": 2048}, "m", "p") == 2048+++class TestAgentResolverDelegates:+ def test_agent_resolver_matches_native_default(self):+ from services.ai import _resolve_max_tokens++ with patch("services.model_registry.get_model_registry", return_value=_registry_mock()):+ assert _resolve_max_tokens({}, "gemini-flash-latest", "gemini") == MODEL_MAX++ def test_agent_resolver_delegates_to_native(self):+ # Source invariant: no duplicated resolution logic — the agent+ # path must call the native resolver so the two can't drift.+ from services import ai++ src = inspect.getsource(ai._resolve_max_tokens)+ assert "native_resolve_max_tokens" in src++ def test_temporal_prepare_payload_uses_agent_resolver(self):+ # The F4.B path must resolve max_tokens through the same helper.+ from services.temporal.agent_activities import prepare_agent_payload++ src = inspect.getsource(prepare_agent_payload)+ assert "_resolve_max_tokens" in src+++class TestRegistryAliasNormalization:+ """OpenRouter "~provider" alias rows (~google/gemini-flash-latest)+ must key under the canonical provider, or get_model_info misses the+ registry and max_tokens/context_length degrade to llm_defaults+ fallbacks."""++ def test_parse_normalizes_tilde_provider(self):+ from services.model_registry import ModelRegistryService++ svc = ModelRegistryService()+ info = svc._parse_openrouter_model(+ {+ "id": "~google/gemini-test-latest",+ "name": "Google: Gemini Test",+ "context_length": 1_000_000,+ "top_provider": {"max_completion_tokens": 65536},+ }+ )+ assert info is not None+ assert info.provider == "gemini"+ assert info.local_id == "gemini-test-latest"++ def test_load_cache_normalizes_tilde_keys(self, tmp_path, monkeypatch):+ import services.model_registry as mr++ cache = tmp_path / "model_registry.json"+ cache.write_text(+ __import__("json").dumps(+ {+ "models": {+ "~google/gemini-test-latest": {+ "id": "~google/gemini-test-latest",+ "name": "Google: Gemini Test",+ "provider": "~google",+ "local_id": "gemini-test-latest",+ "context_length": 1_000_000,+ "max_output_tokens": 65536,+ }+ }+ }+ ),+ encoding="utf-8",+ )+ monkeypatch.setattr(mr, "CACHE_FILE", cache)+ svc = mr.ModelRegistryService()+ svc._load_cache()+ info = svc.get_model_info("gemini-test-latest", "gemini")+ assert info is not None+ assert info.max_output_tokens == 65536+++class TestStandaloneWorkerRegistryStartup:+ def test_standalone_worker_loads_model_registry(self):+ # Without startup(), a standalone worker's registry is empty and+ # every agent resolves max_tokens to the hard 4096 fallback.+ from services.temporal.worker import run_standalone_worker++ src = inspect.getsource(run_standalone_worker)+ assert "get_model_registry().startup()" in src