~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Integration tests for get_llm() dispatch through the provider registry.
PR #3984 removed the procedural if/elif provider chain in get_llm(); the
registry path (auto-discovery -> provider class create_llm) is now the only
construction path. These tests pin the full dispatch chain end to end:
get_llm(provider=..., settings_snapshot=...)
-> _llm_registry (populated by discover_providers() at import)
-> <Provider>.create_llm
-> langchain client constructor (patched at the provider module)
Only the langchain client class itself is patched — settings resolution,
registry dispatch, URL handling, key resolution, and the max_tokens cap all
execute real production code.
"""
import pytest
from unittest.mock import patch
from langchain_core.language_models.fake_chat_models import FakeListChatModel
from local_deep_research.config.llm_config import get_llm
from local_deep_research.llm import is_llm_registered
from local_deep_research.llm.providers import get_discovered_provider_options
from local_deep_research.llm.providers.base import normalize_provider
OPENAI_CHAT = (
"local_deep_research.llm.providers.implementations.openai.ChatOpenAI"
)
ANTHROPIC_CHAT = (
"local_deep_research.llm.providers.implementations.anthropic.ChatAnthropic"
)
OLLAMA_CHAT = (
"local_deep_research.llm.providers.implementations.ollama.ChatOllama"
)
# LM Studio and llama.cpp construct their client in the shared parent
# (OpenAICompatibleProvider._create_llm_instance), so the patch point is
# openai_base, not the implementation module.
OPENAI_BASE_CHAT = "local_deep_research.llm.providers.openai_base.ChatOpenAI"
@pytest.fixture(autouse=True, scope="module")
def _ensure_providers_registered():
"""Re-run auto-discovery in case an earlier test cleared the registry."""
from local_deep_research.llm.providers import discover_providers
discover_providers(force_refresh=True)
def _snapshot(**overrides):
"""Minimal permissive settings snapshot for the live dispatch path."""
snap = {
"llm.supports_max_tokens": True,
"llm.max_tokens": 4096,
"rate_limiting.llm_enabled": False,
"search.tool": "searxng",
}
snap.update(overrides)
return snap
def _fake_llm():
return FakeListChatModel(responses=["ok"])
class TestRegistryPopulation:
"""Every built-in provider must be reachable through the registry,
since get_llm() no longer has any per-provider construction code."""
def test_all_discovered_providers_are_registered(self):
discovered = [
normalize_provider(option["value"])
for option in get_discovered_provider_options()
]
unregistered = [p for p in discovered if not is_llm_registered(p)]
assert unregistered == [], (
f"Providers missing from the LLM registry: {unregistered}. "
"get_llm() has no fallback construction path for these anymore."
)
class TestCloudProviderDispatch:
def test_openai_dispatches_through_provider_class(self):
snapshot = _snapshot(**{"llm.openai.api_key": "sk-test-key"})
fake = _fake_llm()
with patch(OPENAI_CHAT, return_value=fake) as mock_chat:
result = get_llm(
provider="openai",
model_name="gpt-4o-mini",
temperature=0.5,
settings_snapshot=snapshot,
)
kwargs = mock_chat.call_args.kwargs
assert kwargs["model"] == "gpt-4o-mini"
assert kwargs["api_key"] == "sk-test-key"
assert kwargs["temperature"] == 0.5
assert result.base_llm is fake
def test_anthropic_dispatches_through_provider_class(self):
snapshot = _snapshot(**{"llm.anthropic.api_key": "sk-ant-test"})
fake = _fake_llm()
with patch(ANTHROPIC_CHAT, return_value=fake) as mock_chat:
result = get_llm(
provider="anthropic",
model_name="claude-sonnet-4-5",
settings_snapshot=snapshot,
)
kwargs = mock_chat.call_args.kwargs
assert kwargs["model"] == "claude-sonnet-4-5"
assert kwargs["anthropic_api_key"] == "sk-ant-test"
assert result.base_llm is fake
def test_anthropic_missing_key_raises_value_error(self):
"""Required-key semantics survive on the live path: no placeholder."""
with pytest.raises(ValueError, match="API key not configured"):
get_llm(
provider="anthropic",
model_name="claude-sonnet-4-5",
settings_snapshot=_snapshot(),
)
class TestOpenAIEndpointOverride:
def test_explicit_url_overrides_snapshot_without_mutating_it(self):
snapshot = _snapshot(
**{
"llm.openai_endpoint.url": "http://localhost:9000/v1",
}
)
with patch(OPENAI_BASE_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="openai_endpoint",
model_name="custom-model",
openai_endpoint_url="http://localhost:8000/v1",
settings_snapshot=snapshot,
)
assert (
mock_chat.call_args.kwargs["base_url"] == "http://localhost:8000/v1"
)
assert snapshot["llm.openai_endpoint.url"] == "http://localhost:9000/v1"
def test_explicit_local_url_works_without_snapshot(self):
with patch(OPENAI_BASE_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="openai_endpoint",
model_name="custom-model",
openai_endpoint_url="http://localhost:8000/v1",
settings_snapshot=None,
)
assert (
mock_chat.call_args.kwargs["base_url"] == "http://localhost:8000/v1"
)
def test_snapshot_url_is_used_when_override_is_omitted(self):
snapshot = _snapshot(
**{
"llm.openai_endpoint.url": "http://localhost:9000/v1",
}
)
with patch(OPENAI_BASE_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="openai_endpoint",
model_name="custom-model",
settings_snapshot=snapshot,
)
assert (
mock_chat.call_args.kwargs["base_url"] == "http://localhost:9000/v1"
)
def test_empty_explicit_url_is_rejected(self):
with pytest.raises(
ValueError, match="openai_endpoint_url must be a non-empty string"
):
get_llm(
provider="openai_endpoint",
model_name="custom-model",
openai_endpoint_url=" ",
settings_snapshot=_snapshot(),
)
def test_non_string_explicit_url_is_rejected(self):
with pytest.raises(
ValueError, match="openai_endpoint_url must be a non-empty string"
):
get_llm(
provider="openai_endpoint",
model_name="custom-model",
openai_endpoint_url=123,
settings_snapshot=_snapshot(),
)
def test_explicit_url_is_ignored_for_non_endpoint_provider(self):
snapshot = _snapshot(**{"llm.ollama.url": "http://localhost:11434"})
with patch(OLLAMA_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="ollama",
model_name="llama3.1:8b",
openai_endpoint_url=123,
settings_snapshot=snapshot,
)
assert mock_chat.call_args.kwargs["base_url"] == (
"http://localhost:11434"
)
class TestLocalProviderDispatch:
def test_lmstudio_appends_v1_suffix(self):
"""#4532: a URL without /v1 gets the suffix on the live path."""
snapshot = _snapshot(**{"llm.lmstudio.url": "http://localhost:1234"})
with patch(OPENAI_BASE_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="lmstudio",
model_name="qwen2.5-7b",
settings_snapshot=snapshot,
)
kwargs = mock_chat.call_args.kwargs
assert kwargs["base_url"] == "http://localhost:1234/v1"
assert kwargs["api_key"] == "not-required"
def test_lmstudio_does_not_double_v1_suffix(self):
snapshot = _snapshot(**{"llm.lmstudio.url": "http://localhost:1234/v1"})
with patch(OPENAI_BASE_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="lmstudio",
model_name="qwen2.5-7b",
settings_snapshot=snapshot,
)
assert (
mock_chat.call_args.kwargs["base_url"] == "http://localhost:1234/v1"
)
def test_lmstudio_real_api_key_passed_through(self):
snapshot = _snapshot(
**{
"llm.lmstudio.url": "http://localhost:1234/v1",
"llm.lmstudio.api_key": "lms-real-key",
}
)
with patch(OPENAI_BASE_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="lmstudio",
model_name="qwen2.5-7b",
settings_snapshot=snapshot,
)
assert mock_chat.call_args.kwargs["api_key"] == "lms-real-key"
def test_llamacpp_uses_placeholder_key_and_verbatim_url(self):
"""llama.cpp does NOT force /v1 — the user-provided URL is used as-is."""
snapshot = _snapshot(**{"llm.llamacpp.url": "http://localhost:8080"})
with patch(OPENAI_BASE_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="llamacpp",
model_name="llama-3.1-8b",
settings_snapshot=snapshot,
)
kwargs = mock_chat.call_args.kwargs
assert kwargs["base_url"] == "http://localhost:8080"
assert kwargs["api_key"] == "not-required"
def test_ollama_enable_thinking_false_reaches_chatollama(self):
"""#3984's headline bug fix: llm.ollama.enable_thinking was silently
ignored on the live path before this PR."""
snapshot = _snapshot(
**{
"llm.ollama.url": "http://localhost:11434",
"llm.ollama.enable_thinking": False,
}
)
fake = _fake_llm()
with patch(OLLAMA_CHAT, return_value=fake) as mock_chat:
result = get_llm(
provider="ollama",
model_name="deepseek-r1:14b",
settings_snapshot=snapshot,
)
kwargs = mock_chat.call_args.kwargs
assert kwargs["reasoning"] is False
assert kwargs["model"] == "deepseek-r1:14b"
assert result.base_llm is fake
def test_ollama_enable_thinking_default_true(self):
snapshot = _snapshot(**{"llm.ollama.url": "http://localhost:11434"})
with patch(OLLAMA_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="ollama",
model_name="deepseek-r1:14b",
settings_snapshot=snapshot,
)
assert mock_chat.call_args.kwargs["reasoning"] is True
class TestMaxTokensCap:
"""The 80%-of-context-window cap previously lived only in dead code;
these pin it on the live path end to end through get_llm()."""
def test_cloud_max_tokens_capped_at_80_percent_of_window(self):
snapshot = _snapshot(
**{
"llm.openai.api_key": "sk-test-key",
"llm.context_window_unrestricted": False,
"llm.context_window_size": 10000,
"llm.max_tokens": 200000,
}
)
with patch(OPENAI_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="openai",
model_name="gpt-4o-mini",
settings_snapshot=snapshot,
)
assert mock_chat.call_args.kwargs["max_tokens"] == 8000
def test_cloud_unrestricted_window_passes_max_tokens_uncapped(self):
snapshot = _snapshot(
**{
"llm.openai.api_key": "sk-test-key",
"llm.context_window_unrestricted": True,
"llm.max_tokens": 200000,
}
)
with patch(OPENAI_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="openai",
model_name="gpt-4o-mini",
settings_snapshot=snapshot,
)
assert mock_chat.call_args.kwargs["max_tokens"] == 200000
def test_ollama_num_ctx_from_shared_helper_no_max_tokens(self):
"""Ollama's window resolves through the shared helper (same source
as context_limit bookkeeping); no max_tokens kwarg is passed since
ChatOllama silently ignores it."""
snapshot = _snapshot(
**{
"llm.ollama.url": "http://localhost:11434",
"llm.local_context_window_size": 4096,
"llm.max_tokens": 200000,
}
)
with patch(OLLAMA_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="ollama",
model_name="llama3.1:8b",
settings_snapshot=snapshot,
)
kwargs = mock_chat.call_args.kwargs
assert kwargs["num_ctx"] == 4096
assert "max_tokens" not in kwargs
def test_lmstudio_capped_by_local_window_not_cloud_window(self):
"""LM Studio resolves its window via provider_key through the
LOCAL_PROVIDERS branch — the cloud llm.context_window_size must
play no role."""
snapshot = _snapshot(
**{
"llm.lmstudio.url": "http://localhost:1234/v1",
"llm.context_window_unrestricted": False,
"llm.context_window_size": 128000,
"llm.local_context_window_size": 4096,
"llm.max_tokens": 200000,
}
)
with patch(OPENAI_BASE_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="lmstudio",
model_name="qwen2.5-7b",
settings_snapshot=snapshot,
)
assert mock_chat.call_args.kwargs["max_tokens"] == int(4096 * 0.8)
def test_unset_max_tokens_omits_kwarg(self):
"""Partial snapshots without llm.max_tokens must not inject a
hardcoded default — the provider SDK's own default applies."""
snapshot = _snapshot(**{"llm.openai.api_key": "sk-test-key"})
del snapshot["llm.max_tokens"]
with patch(OPENAI_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="openai",
model_name="gpt-4o-mini",
settings_snapshot=snapshot,
)
assert "max_tokens" not in mock_chat.call_args.kwargs
def test_supports_max_tokens_false_omits_kwarg(self):
snapshot = _snapshot(
**{
"llm.openai.api_key": "sk-test-key",
"llm.supports_max_tokens": False,
}
)
with patch(OPENAI_CHAT, return_value=_fake_llm()) as mock_chat:
get_llm(
provider="openai",
model_name="gpt-4o-mini",
settings_snapshot=snapshot,
)
assert "max_tokens" not in mock_chat.call_args.kwargs
class TestContextLimitBookkeeping:
def test_context_limit_set_in_research_context(self):
"""Overflow detection relies on context_limit being populated even
though the registered path returns before get_llm's own bookkeeping."""
snapshot = _snapshot(
**{
"llm.openai.api_key": "sk-test-key",
"llm.context_window_unrestricted": False,
"llm.context_window_size": 10000,
}
)
research_context = {}
with patch(OPENAI_CHAT, return_value=_fake_llm()):
get_llm(
provider="openai",
model_name="gpt-4o-mini",
settings_snapshot=snapshot,
research_context=research_context,
)
assert research_context["context_limit"] == 10000