Agent Zero AI framework
Add llama.cpp and vLLM local providers
Alessandro9bcef39028a8
diff --git a/conf/model_providers.yaml b/conf/model_providers.yamlindex 0743a6512..541a4888b 100644--- a/conf/model_providers.yaml+++ b/conf/model_providers.yaml@@ -86,6 +86,15 @@ chat:kwargs:api_base: "http://host.docker.internal:1234/v1"api_key: "lm-studio"+ llama_cpp:+ name: llama.cpp+ litellm_provider: hosted_vllm+ models_list:+ endpoint_url: "/v1/models"+ default_base: "http://host.docker.internal:8080"+ kwargs:+ api_base: "http://host.docker.internal:8080/v1"+ api_key: "llama-cpp"mistral:name: Mistral AIlitellm_provider: mistral@@ -167,6 +176,15 @@ chat:api_base: https://api.venice.ai/api/v1venice_parameters:include_venice_system_prompt: false+ vllm:+ name: vLLM+ litellm_provider: hosted_vllm+ models_list:+ endpoint_url: "/v1/models"+ default_base: "http://host.docker.internal:8000"+ kwargs:+ api_base: "http://host.docker.internal:8000/v1"+ api_key: "vllm"xai:name: xAIlitellm_provider: xai@@ -203,6 +221,12 @@ embedding:kwargs:api_base: "http://host.docker.internal:1234/v1"api_key: "lm-studio"+ llama_cpp:+ name: llama.cpp+ litellm_provider: hosted_vllm+ kwargs:+ api_base: "http://host.docker.internal:8080/v1"+ api_key: "llama-cpp"mistral:name: Mistral AIlitellm_provider: mistral@@ -250,6 +274,12 @@ embedding:endpoint_url: "https://api.venice.ai/api/v1/models"kwargs:api_base: https://api.venice.ai/api/v1+ vllm:+ name: vLLM+ litellm_provider: hosted_vllm+ kwargs:+ api_base: "http://host.docker.internal:8000/v1"+ api_key: "vllm"other:name: Other OpenAI compatiblelitellm_provider: openaidiff --git a/docs/setup/installation.md b/docs/setup/installation.mdindex 57c0b76b9..f4836a980 100644--- a/docs/setup/installation.md+++ b/docs/setup/installation.md@@ -439,6 +439,8 @@ Use the naming format required by your selected provider:| OpenRouter | Provider prefix mostly required | `anthropic/claude-sonnet-4-5` || Ollama | Model name only | `gpt-oss:20b` || oMLX | API-visible model name from `/v1/models` | `Qwen3-0.6B-4bit` |+| llama.cpp | API-visible model name from `/v1/models` or `--alias` | `local-gguf` |+| vLLM | Hugging Face model ID or served model alias | `Qwen/Qwen2.5-1.5B-Instruct` |> [!TIP]> If you see "Invalid model ID," verify the provider and naming format on the provider website, or search the web for "<name-of-ai-model> model naming".@@ -504,6 +506,71 @@ omlx serve --model-dir ~/.omlx/models --paged-ssd-cache-dir ~/.omlx/cache---+## Installing and Using llama.cpp (GGUF Local Models)++llama.cpp provides `llama-server`, a lightweight OpenAI-compatible HTTP server for GGUF models. Agent Zero talks to it through the same `/v1` API used by OpenAI-compatible clients.++### macOS llama.cpp Installation++**Using Homebrew:**++```bash+brew install llama.cpp+```++Start a server with a downloaded GGUF model:++```bash+llama-server -m ~/models/model.gguf --port 8080 --alias local-gguf+```++By default, Agent Zero expects llama.cpp at `http://host.docker.internal:8080/v1`. The model name can be the model path returned by `/v1/models`, but using `--alias` gives you a short stable name such as `local-gguf`.++### Configuring llama.cpp in Agent Zero++1. Start `llama-server` and confirm `http://localhost:8080/v1/models` returns your model.+2. In Agent Zero Settings, choose **llama.cpp** as the Chat model, Utility model, or Embedding model provider.+3. Use the model ID shown by `/v1/models`, or the alias you passed with `--alias`.+4. Override the API base URL only if you started `llama-server` on another host or port.+5. Click `Save` to confirm your settings.++> [!NOTE]+> If Agent Zero runs in Docker and cannot reach a host-side `llama-server`, start the server on an address Docker can reach, for example `--host 0.0.0.0`, and keep the port firewalled to trusted clients.++---++## Installing and Using vLLM (Local OpenAI-Compatible Serving)++vLLM is a high-throughput local inference server with an OpenAI-compatible API. It is most common on Linux GPU hosts, and can also run on Apple Silicon through the vLLM Apple Silicon path or vLLM-Metal.++For Apple Silicon Macs, install and activate vLLM-Metal:++```bash+curl -fsSL https://raw.githubusercontent.com/vllm-project/vllm-metal/main/install.sh | bash+source ~/.venv-vllm-metal/bin/activate+```++Start a basic OpenAI-compatible server:++```bash+vllm serve Qwen/Qwen2.5-1.5B-Instruct --host 0.0.0.0 --port 8000+```++By default, Agent Zero expects vLLM at `http://host.docker.internal:8000/v1`, matching vLLM's default HTTP port. If another local provider already uses port 8000, start vLLM on another port and update Agent Zero's API base, for example `http://host.docker.internal:8001/v1`.++### Configuring vLLM in Agent Zero++1. Start vLLM and confirm `http://localhost:8000/v1/models` returns the served model.+2. In Agent Zero Settings, choose **vLLM** as the Chat model, Utility model, or Embedding model provider.+3. Use the model ID returned by vLLM's model list endpoint.+4. If you started vLLM with `--api-key`, enter the same key in the advanced provider settings or environment.+5. Click `Save` to confirm your settings.++> [!NOTE]+> vLLM serves one model at a time by default. Use a generation model for Chat and Utility slots, and a separate embedding-capable vLLM server if you want vLLM embeddings.++---+## Installing and Using Ollama (Local Models)Ollama is a powerful tool that allows you to run various large language models locally.diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.pyindex de5a2b884..05b9c6b2a 100644--- a/helpers/litellm_transport.py+++ b/helpers/litellm_transport.py@@ -459,6 +459,7 @@ class ChatCompletionsTransport:chat_kwargs = dict(kwargs)_drop_internal_transport_kwargs(chat_kwargs)if not _has_tools(chat_kwargs.get("tools")):+ chat_kwargs.pop("tools", None)chat_kwargs.pop("tool_choice", None)chat_kwargs.pop("parallel_tool_calls", None)if _is_openai_prompt_cache_provider(model, chat_kwargs):@@ -1538,6 +1539,8 @@ def _is_responses_not_supported_error(exc: Exception) -> bool:"unsupportedparamserror","does not support parameters","no 'tools' defined while 'tool_choice' is specified",+ "tools` must not be an empty array",+ "tools must not be an empty array",))diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.mdnew file mode 100644index 000000000..4105d7ba0--- /dev/null+++ b/helpers/litellm_transport.py.dox.md@@ -0,0 +1,45 @@+# litellm_transport.py DOX++## Purpose++- Own Agent Zero's LiteLLM transport adapter for Chat Completions and Responses API calls.+- Normalize Agent Zero model-call kwargs into provider-safe LiteLLM requests.+- Preserve canonical response metadata for history, provider-state continuation, and fallback decisions.++## Ownership++- `litellm_transport.py` owns the runtime implementation.+- `litellm_transport.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation.+- Classes:+- `TransportMode`+- `TransportRecovery`+- `TransportPolicy`+- `LiteLLMTransport`+- `ChatCompletionsTransport`+- `ResponsesTransport`+- `ResponsesEventParser`+- Top-level functions include transport cache reset, request normalization, parsing, prompt-cache preparation, and response/error classifiers.++## Runtime Contracts++- Keep provider selection and provider-specific defaults outside this helper; callers pass a resolved LiteLLM model name and kwargs.+- Strip Agent Zero internal kwargs before sending requests to LiteLLM.+- Do not send orphan tool controls when no tools are present; strict OpenAI-compatible servers can reject empty `tools` arrays.+- Prefer Responses API when configured, but fallback to Chat Completions when the provider does not support Responses.+- Preserve provider-state metadata when Responses API calls succeed, and fall back to local replay when provider state is unsupported.+- Keep prompt-cache markers only for providers that accept them.++## Work Guidance++- Add provider-agnostic request cleanup here when multiple OpenAI-compatible providers can benefit.+- Treat fallback behavior as a shared transport contract, not a provider registry.+- Keep tool conversion symmetric between Chat Completions and Responses requests.++## Verification++- Run `pytest tests/test_stream_tool_early_stop.py tests/test_responses_architecture.py -q` after changing transport normalization or fallback behavior.+- Run local-provider smoke checks when changing OpenAI-compatible request cleanup.++## Child DOX Index++No child DOX files.diff --git a/helpers/llm_result.py.dox.md b/helpers/llm_result.py.dox.mdnew file mode 100644index 000000000..b48cea191--- /dev/null+++ b/helpers/llm_result.py.dox.md@@ -0,0 +1,39 @@+# llm_result.py DOX++## Purpose++- Own canonical LLM result metadata shared by model transports, history, and tool-result processing.+- Preserve Responses API output items, provider response IDs, reasoning text, usage, and capability metadata in a serializable form.++## Ownership++- `llm_result.py` owns the runtime implementation.+- `llm_result.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation.+- Classes:+- `ResponseItem`+- `ResponseFunctionCall`+- `LLMResult`+- Top-level functions include metadata conversion, function-call output item construction, object normalization, output-text extraction, reasoning extraction, and function-call argument parsing.++## Runtime Contracts++- `LLMResult.metadata()` stores data under `RESPONSE_METADATA_KEY` so history can round-trip provider state.+- `from_response(...)` must preserve provider `response_id`, `previous_response_id`, raw output items, usage, and capability metadata.+- `from_chat(...)` must produce an equivalent chat-completions result with `mode="chat_completions"` and `state="off"`.+- Function-call output items must preserve `call_id` and optional acknowledged safety checks.+- Argument parsing must tolerate JSON strings, dictionaries, and malformed values without throwing.++## Work Guidance++- Keep metadata backward-compatible with existing serialized chat history.+- Treat unknown response item types as preserved built-in items unless they are local function calls, message text, or reasoning.+- Avoid provider-specific assumptions in result parsing.++## Verification++- Run `pytest tests/test_responses_architecture.py -q` after changing result metadata behavior.+- Run focused history/tool-processing tests when changing function-call serialization.++## Child DOX Index++No child DOX files.diff --git a/helpers/tunnel_origins.py.dox.md b/helpers/tunnel_origins.py.dox.mdnew file mode 100644index 000000000..83c829d8b--- /dev/null+++ b/helpers/tunnel_origins.py.dox.md@@ -0,0 +1,37 @@+# tunnel_origins.py DOX++## Purpose++- Own origin normalization for Remote Control tunnel URLs and CSRF/WebSocket same-origin checks.+- Provide a small helper boundary between tunnel discovery and security enforcement.++## Ownership++- `tunnel_origins.py` owns the runtime implementation.+- `tunnel_origins.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation.+- Top-level functions:+- `origin_from_url(value)`+- `origin_key(value)`+- `get_active_tunnel_origins()`++## Runtime Contracts++- Normalize URL and Origin header values to `scheme://host[:port]`, omitting default ports.+- Return comparable origin keys with default ports restored for same-origin checks.+- Treat invalid, missing, or malformed origins as `None`.+- Discover active tunnel origins from `TunnelManager` and the Docker tunnel API without raising if either source is unavailable.+- Keep tunnel service lookups short-timeout and local-only.++## Work Guidance++- Keep parsing based on `urllib.parse` rather than hand-rolled string checks.+- Preserve defensive exception handling because tunnel services are optional and may not be running.+- Coordinate security-sensitive changes with CSRF and WebSocket tests.++## Verification++- Run `pytest tests/test_csrf_tunnel_origins.py tests/test_ws_csrf.py -q` after changing tunnel origin behavior.++## Child DOX Index++No child DOX files.diff --git a/plugins/_browser/helpers/connector_runtime.py b/plugins/_browser/helpers/connector_runtime.pyindex 28c54b5fe..47b31512b 100644--- a/plugins/_browser/helpers/connector_runtime.py+++ b/plugins/_browser/helpers/connector_runtime.py@@ -58,7 +58,7 @@ HOST_BROWSER_PROFILE_MODE_KEY = getattr("host_browser_profile_mode",)get_browser_config = browser_config.get_browser_config-_LOCAL_PROVIDERS = {"ollama", "lm_studio", "omlx"}+_LOCAL_PROVIDERS = {"ollama", "lm_studio", "llama_cpp", "omlx", "vllm"}_LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1", "host.docker.internal"}_SENSITIVE_ACTIONS = {"content", "detail", "evaluate", "screenshot", "screenshot_file"}_KEY_ALIASES = {diff --git a/plugins/_model_config/api/model_search.py b/plugins/_model_config/api/model_search.pyindex ec5f50d24..a83ff68f5 100644--- a/plugins/_model_config/api/model_search.py+++ b/plugins/_model_config/api/model_search.py@@ -26,6 +26,12 @@ _NON_CHAT_EXCLUDE = frozenset({"omni-moderation","vision-preview",})+_LOCAL_PLACEHOLDER_KEYS = {+ "lm_studio": {"lm-studio"},+ "llama_cpp": {"llama-cpp"},+ "omlx": {"omlx"},+ "vllm": {"vllm"},+}class ModelSearch(ApiHandler):@@ -179,8 +185,8 @@ class ModelSearch(ApiHandler):elif provider == "azure":if has_key:headers["api-key"] = api_key- elif provider not in ("ollama", "lm_studio", "omlx"):- if has_key:+ elif provider != "ollama":+ if has_key and api_key not in _LOCAL_PLACEHOLDER_KEYS.get(provider, set()):headers["Authorization"] = f"Bearer {api_key}"extra = (cfg or {}).get("kwargs", {}).get("extra_headers", {})diff --git a/plugins/_model_config/extensions/python/banners/_20_missing_api_key.py b/plugins/_model_config/extensions/python/banners/_20_missing_api_key.pyindex a3fc48301..c5722435c 100644--- a/plugins/_model_config/extensions/python/banners/_20_missing_api_key.py+++ b/plugins/_model_config/extensions/python/banners/_20_missing_api_key.py@@ -6,7 +6,7 @@ from plugins._model_config.helpers import model_configclass MissingApiKeyCheck(Extension):"""Check if API keys are configured for selected model providers."""- LOCAL_PROVIDERS = {"ollama", "lm_studio", "omlx"}+ LOCAL_PROVIDERS = {"ollama", "lm_studio", "llama_cpp", "omlx", "vllm"}CONFIGURE_MODEL_SETTINGS_LINK = ("""<div class="onboarding-banner-btn-container" style="margin-top: 12px;">""""""<button class="btn btn-ok" onclick="window.openModal('/plugins/_onboarding/webui/onboarding.html');return false;">"""diff --git a/plugins/_model_config/helpers/model_config.py b/plugins/_model_config/helpers/model_config.pyindex 08b834ce2..175378d58 100644--- a/plugins/_model_config/helpers/model_config.py+++ b/plugins/_model_config/helpers/model_config.py@@ -32,7 +32,7 @@ IMPLICIT_PRESET_SLOT_DEFAULTS = {"kwargs": {},},}-LOCAL_PROVIDERS = {"ollama", "lm_studio", "omlx"}+LOCAL_PROVIDERS = {"ollama", "lm_studio", "llama_cpp", "omlx", "vllm"}LOCAL_EMBEDDING = {"huggingface"}_PROVIDER_METADATA_CACHE: dict | None = Nonediff --git a/plugins/_model_config/provider_metadata.yaml b/plugins/_model_config/provider_metadata.yamlindex d6dc5bfc3..e3a52596f 100644--- a/plugins/_model_config/provider_metadata.yaml+++ b/plugins/_model_config/provider_metadata.yaml@@ -1,10 +1,14 @@chat:lm_studio:api_key_mode: none+ llama_cpp:+ api_key_mode: noneollama:api_key_mode: noneomlx:api_key_mode: none+ vllm:+ api_key_mode: noneother:api_key_mode: optional@@ -13,9 +17,13 @@ embedding:api_key_mode: nonelm_studio:api_key_mode: none+ llama_cpp:+ api_key_mode: noneollama:api_key_mode: noneomlx:api_key_mode: none+ vllm:+ api_key_mode: noneother:api_key_mode: optionaldiff --git a/plugins/_onboarding/webui/assets/provider-logos/llama-cpp.svg b/plugins/_onboarding/webui/assets/provider-logos/llama-cpp.svgnew file mode 100644index 000000000..a0e658e6e--- /dev/null+++ b/plugins/_onboarding/webui/assets/provider-logos/llama-cpp.svg@@ -0,0 +1,17 @@+<svg xmlns="http://www.w3.org/2000/svg" width="160" height="160" viewBox="0 0 160 160" role="img" aria-labelledby="llama-cpp-title">+ <title id="llama-cpp-title">llama.cpp</title>+ <defs>+ <linearGradient id="llama-cpp-bg" x1="0" y1="0" x2="1" y2="1">+ <stop offset="0" stop-color="#242424"/>+ <stop offset="1" stop-color="#111111"/>+ </linearGradient>+ </defs>+ <rect x="10" y="10" width="140" height="140" rx="28" fill="url(#llama-cpp-bg)"/>+ <path d="M47 111V65c0-18 14-32 32-32h9c18 0 32 14 32 32v46" fill="none" stroke="#f1e4c8" stroke-width="11" stroke-linecap="round" stroke-linejoin="round"/>+ <path d="M58 42 47 24M103 42l12-18" fill="none" stroke="#f1e4c8" stroke-width="10" stroke-linecap="round"/>+ <path d="M58 72h50" stroke="#79c7b4" stroke-width="9" stroke-linecap="round"/>+ <circle cx="67" cy="88" r="5" fill="#f1e4c8"/>+ <circle cx="96" cy="88" r="5" fill="#f1e4c8"/>+ <path d="M72 106h22" stroke="#f1e4c8" stroke-width="8" stroke-linecap="round"/>+ <text x="80" y="133" text-anchor="middle" font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace" font-size="23" font-weight="800" fill="#79c7b4">.cpp</text>+</svg>diff --git a/plugins/_onboarding/webui/assets/provider-logos/vllm.svg b/plugins/_onboarding/webui/assets/provider-logos/vllm.svgnew file mode 100644index 000000000..dbf4f7d7e--- /dev/null+++ b/plugins/_onboarding/webui/assets/provider-logos/vllm.svg@@ -0,0 +1,18 @@+<svg xmlns="http://www.w3.org/2000/svg" width="160" height="160" viewBox="0 0 160 160" role="img" aria-labelledby="vllm-title">+ <title id="vllm-title">vLLM</title>+ <defs>+ <linearGradient id="vllm-bg" x1="0" y1="0" x2="1" y2="1">+ <stop offset="0" stop-color="#12303a"/>+ <stop offset="1" stop-color="#101820"/>+ </linearGradient>+ <linearGradient id="vllm-mark" x1="0" y1="0" x2="1" y2="1">+ <stop offset="0" stop-color="#8ce9dc"/>+ <stop offset="1" stop-color="#4da3ff"/>+ </linearGradient>+ </defs>+ <rect x="10" y="10" width="140" height="140" rx="28" fill="url(#vllm-bg)"/>+ <path d="M37 43 65 116 93 43" fill="none" stroke="url(#vllm-mark)" stroke-width="15" stroke-linecap="round" stroke-linejoin="round"/>+ <path d="M99 68v48M120 68v48" fill="none" stroke="#efffff" stroke-width="12" stroke-linecap="round"/>+ <path d="M99 92h21" stroke="#efffff" stroke-width="12" stroke-linecap="round"/>+ <text x="80" y="135" text-anchor="middle" font-family="Inter, ui-sans-serif, system-ui, sans-serif" font-size="21" font-weight="900" fill="#8ce9dc">vLLM</text>+</svg>diff --git a/plugins/_onboarding/webui/onboarding-providers.js b/plugins/_onboarding/webui/onboarding-providers.jsindex 7680db5f2..824bdde22 100644--- a/plugins/_onboarding/webui/onboarding-providers.js+++ b/plugins/_onboarding/webui/onboarding-providers.js@@ -25,7 +25,7 @@ export const MORE_CLOUD_PROVIDER_IDS = ["other",];-export const LOCAL_PROVIDER_IDS = ["ollama", "lm_studio", "omlx", "other"];+export const LOCAL_PROVIDER_IDS = ["ollama", "lm_studio", "omlx", "llama_cpp", "vllm", "other"];export const ONBOARDING_PROVIDER_OVERRIDES = {a0_venice: {@@ -126,6 +126,15 @@ export const ONBOARDING_PROVIDER_OVERRIDES = {model_list_autoload: true,short_description: "Run local models through LM Studio.",},+ llama_cpp: {+ logo: "/plugins/_onboarding/webui/assets/provider-logos/llama-cpp.svg",+ setup_url: "https://github.com/ggml-org/llama.cpp",+ docs_url: "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md",+ default_api_base: "http://host.docker.internal:8080/v1",+ api_key_mode: "none",+ model_list_autoload: true,+ short_description: "Run GGUF models with llama-server.",+ },mistral: {logo: "https://mistral.ai/favicon.ico",setup_url: "https://console.mistral.ai/",@@ -181,6 +190,15 @@ export const ONBOARDING_PROVIDER_OVERRIDES = {model_list_autoload: true,short_description: "Apple Silicon local inference with MLX.",},+ vllm: {+ logo: "/plugins/_onboarding/webui/assets/provider-logos/vllm.svg",+ setup_url: "https://docs.vllm.ai/",+ docs_url: "https://docs.vllm.ai/en/stable/serving/online_serving/",+ default_api_base: "http://host.docker.internal:8000/v1",+ api_key_mode: "none",+ model_list_autoload: true,+ short_description: "High-throughput local OpenAI-compatible serving.",+ },openai: {logo: "https://openai.com/favicon.ico",setup_url: "https://platform.openai.com/",diff --git a/tests/test_model_config_api_keys.py b/tests/test_model_config_api_keys.pyindex 0ec597de8..bb9b6080b 100644--- a/tests/test_model_config_api_keys.py+++ b/tests/test_model_config_api_keys.py@@ -274,10 +274,14 @@ def test_provider_key_modes_for_local_and_ollama_cloud():assert model_config.provider_requires_api_key("ollama") is Falseassert model_config.provider_requires_api_key("lm_studio") is False+ assert model_config.provider_requires_api_key("llama_cpp") is Falseassert model_config.provider_requires_api_key("omlx") is False+ assert model_config.provider_requires_api_key("vllm") is Falseassert model_config.provider_requires_api_key("other") is Falseassert model_config.provider_requires_api_key("ollama_cloud") is True+ assert "llama_cpp" in missing_key_banner.MissingApiKeyCheck.LOCAL_PROVIDERSassert "omlx" in missing_key_banner.MissingApiKeyCheck.LOCAL_PROVIDERS+ assert "vllm" in missing_key_banner.MissingApiKeyCheck.LOCAL_PROVIDERSdef test_local_provider_defaults_are_docker_friendly():@@ -293,6 +297,15 @@ def test_local_provider_defaults_are_docker_friendly():assert provider_config["chat"]["lm_studio"]["models_list"]["default_base"] == ("http://host.docker.internal:1234")+ assert provider_config["chat"]["llama_cpp"]["litellm_provider"] == "hosted_vllm"+ assert provider_config["chat"]["llama_cpp"]["kwargs"]["api_base"] == (+ "http://host.docker.internal:8080/v1"+ )+ assert provider_config["chat"]["llama_cpp"]["kwargs"]["api_key"] == "llama-cpp"+ assert provider_config["chat"]["llama_cpp"]["models_list"]["default_base"] == (+ "http://host.docker.internal:8080"+ )+ assert provider_config["chat"]["llama_cpp"]["models_list"]["endpoint_url"] == "/v1/models"assert provider_config["chat"]["ollama"]["kwargs"]["api_base"] == ("http://host.docker.internal:11434")@@ -308,10 +321,24 @@ def test_local_provider_defaults_are_docker_friendly():"http://host.docker.internal:8000")assert provider_config["chat"]["omlx"]["models_list"]["endpoint_url"] == "/v1/models"+ assert provider_config["chat"]["vllm"]["litellm_provider"] == "hosted_vllm"+ assert provider_config["chat"]["vllm"]["kwargs"]["api_base"] == (+ "http://host.docker.internal:8000/v1"+ )+ assert provider_config["chat"]["vllm"]["kwargs"]["api_key"] == "vllm"+ assert provider_config["chat"]["vllm"]["models_list"]["default_base"] == (+ "http://host.docker.internal:8000"+ )+ assert provider_config["chat"]["vllm"]["models_list"]["endpoint_url"] == "/v1/models"assert provider_config["embedding"]["lm_studio"]["kwargs"]["api_base"] == ("http://host.docker.internal:1234/v1")assert provider_config["embedding"]["lm_studio"]["kwargs"]["api_key"] == "lm-studio"+ assert provider_config["embedding"]["llama_cpp"]["litellm_provider"] == "hosted_vllm"+ assert provider_config["embedding"]["llama_cpp"]["kwargs"]["api_base"] == (+ "http://host.docker.internal:8080/v1"+ )+ assert provider_config["embedding"]["llama_cpp"]["kwargs"]["api_key"] == "llama-cpp"assert provider_config["embedding"]["ollama"]["kwargs"]["api_base"] == ("http://host.docker.internal:11434")@@ -320,6 +347,11 @@ def test_local_provider_defaults_are_docker_friendly():"http://host.docker.internal:8000/v1")assert provider_config["embedding"]["omlx"]["kwargs"]["api_key"] == "omlx"+ assert provider_config["embedding"]["vllm"]["litellm_provider"] == "hosted_vllm"+ assert provider_config["embedding"]["vllm"]["kwargs"]["api_base"] == (+ "http://host.docker.internal:8000/v1"+ )+ assert provider_config["embedding"]["vllm"]["kwargs"]["api_key"] == "vllm"def test_local_provider_runtime_defaults_and_overrides(monkeypatch):@@ -344,6 +376,16 @@ def test_local_provider_runtime_defaults_and_overrides(monkeypatch):assert custom_lm_embedding.kwargs["api_base"] == "http://127.0.0.1:1234/v1"assert custom_lm_embedding.kwargs["api_key"] == "real-local-key"+ llama_cpp_chat = models.get_chat_model("llama_cpp", "local-chat-model")+ assert llama_cpp_chat.model_name == "hosted_vllm/local-chat-model"+ assert llama_cpp_chat.kwargs["api_base"] == "http://host.docker.internal:8080/v1"+ assert llama_cpp_chat.kwargs["api_key"] == "llama-cpp"++ llama_cpp_embedding = models.get_embedding_model("llama_cpp", "local-embedding-model")+ assert llama_cpp_embedding.model_name == "hosted_vllm/local-embedding-model"+ assert llama_cpp_embedding.kwargs["api_base"] == "http://host.docker.internal:8080/v1"+ assert llama_cpp_embedding.kwargs["api_key"] == "llama-cpp"+ollama_embedding = models.get_embedding_model("ollama", "nomic-embed-text")assert ollama_embedding.model_name == "ollama/nomic-embed-text"assert ollama_embedding.kwargs["api_base"] == "http://host.docker.internal:11434"@@ -368,6 +410,25 @@ def test_local_provider_runtime_defaults_and_overrides(monkeypatch):assert custom_omlx_chat.kwargs["api_base"] == "http://127.0.0.1:8000/v1"assert custom_omlx_chat.kwargs["api_key"] == "real-local-key"+ vllm_chat = models.get_chat_model("vllm", "local-chat-model")+ assert vllm_chat.model_name == "hosted_vllm/local-chat-model"+ assert vllm_chat.kwargs["api_base"] == "http://host.docker.internal:8000/v1"+ assert vllm_chat.kwargs["api_key"] == "vllm"++ vllm_embedding = models.get_embedding_model("vllm", "local-embedding-model")+ assert vllm_embedding.model_name == "hosted_vllm/local-embedding-model"+ assert vllm_embedding.kwargs["api_base"] == "http://host.docker.internal:8000/v1"+ assert vllm_embedding.kwargs["api_key"] == "vllm"++ custom_vllm_chat = models.get_chat_model(+ "vllm",+ "local-chat-model",+ api_base="http://127.0.0.1:8001/v1",+ api_key="real-local-key",+ )+ assert custom_vllm_chat.kwargs["api_base"] == "http://127.0.0.1:8001/v1"+ assert custom_vllm_chat.kwargs["api_key"] == "real-local-key"+def test_docker_compose_maps_host_docker_internal_for_local_models():import yamldiff --git a/tests/test_model_search.py b/tests/test_model_search.pyindex a90c12ce5..2e97780c5 100644--- a/tests/test_model_search.py+++ b/tests/test_model_search.py@@ -72,6 +72,16 @@ def test_model_search_omits_auth_header_for_omlx_placeholder_key():assert handler._build_headers("omlx", "omlx", {}) == {}+def test_model_search_omits_auth_header_for_local_placeholder_keys():+ handler = _handler()++ assert handler._build_headers("llama_cpp", "llama-cpp", {}) == {}+ assert handler._build_headers("vllm", "vllm", {}) == {}+ assert handler._build_headers("llama_cpp", "real-local-key", {}) == {+ "Authorization": "Bearer real-local-key"+ }++def test_model_search_filters_non_chat_models():handler = _handler()diff --git a/tests/test_onboarding_static.py b/tests/test_onboarding_static.pyindex 41abe7341..6666363a2 100644--- a/tests/test_onboarding_static.py+++ b/tests/test_onboarding_static.py@@ -65,16 +65,24 @@ def test_onboarding_provider_grid_names_are_present_in_metadata():assert 'docs_url: "https://docs.venice.ai/guides/getting-started/generating-api-key"' in provider_uiassert 'docs_url: "https://docs.tokenfactory.nebius.com/api-reference/introduction"' in provider_uiassert 'docs_url: "https://lmstudio.ai/docs/developer/core/authentication"' in provider_ui+ assert 'logo: "/plugins/_onboarding/webui/assets/provider-logos/llama-cpp.svg"' in provider_ui+ assert 'docs_url: "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md"' in provider_ui+ assert 'default_api_base: "http://host.docker.internal:8080/v1"' in provider_uiassert 'logo: "/plugins/_onboarding/webui/assets/provider-logos/omlx.svg"' in provider_uiassert 'docs_url: "https://github.com/jundot/omlx#readme"' in provider_uiassert 'default_api_base: "http://host.docker.internal:8000/v1"' in provider_ui+ assert 'logo: "/plugins/_onboarding/webui/assets/provider-logos/vllm.svg"' in provider_ui+ assert 'docs_url: "https://docs.vllm.ai/en/stable/serving/online_serving/"' in provider_uiassert 'docs_url: ""' in provider_uiassert "api_key_mode: none" in model_metadataassert "api_key_mode: optional" in model_metadataassert "Ollama Cloud" in provider_yamlassert "https://ollama.com/v1" in provider_yaml+ assert "llama.cpp" in provider_yaml+ assert "http://host.docker.internal:8080/v1" in provider_yamlassert "oMLX" in provider_yamlassert "http://host.docker.internal:8000/v1" in provider_yaml+ assert "vLLM" in provider_yamlassert "Nebius Token Factory" in provider_yamlassert "https://api.tokenfactory.nebius.com/v1" in provider_yamlassert not (PROJECT_ROOT / "plugins/_model_config/conf/model_providers.yaml").exists()@@ -92,7 +100,9 @@ def test_onboarding_provider_grid_names_are_present_in_metadata():"Z.AI","Mistral AI","Azure OpenAI",+ "llama.cpp","oMLX",+ "vLLM",]:assert name in provider_yaml + provider_ui@@ -122,12 +132,16 @@ def test_onboarding_provider_grid_names_are_present_in_metadata():"sambanova.png","cometapi.ico","github-copilot.svg",+ "llama-cpp.svg","zai-logo.svg","omlx.svg",+ "vllm.svg",]:assert logo in provider_ui+ assert (PROJECT_ROOT / "plugins/_onboarding/webui/assets/provider-logos/llama-cpp.svg").exists()assert (PROJECT_ROOT / "plugins/_onboarding/webui/assets/provider-logos/omlx.svg").exists()+ assert (PROJECT_ROOT / "plugins/_onboarding/webui/assets/provider-logos/vllm.svg").exists()def test_nebius_provider_config_uses_openai_compatible_token_factory_endpoint():diff --git a/tests/test_stream_tool_early_stop.py b/tests/test_stream_tool_early_stop.pyindex a1d0ac63e..759079ae6 100644--- a/tests/test_stream_tool_early_stop.py+++ b/tests/test_stream_tool_early_stop.py@@ -643,6 +643,77 @@ def test_responses_request_normalizes_reasoning_and_orphan_tool_choice():assert "reasoning" not in request+def test_chat_completions_kwargs_omit_empty_tools():+ kwargs = litellm_transport.ChatCompletionsTransport.prepare_kwargs(+ {+ "tools": [],+ "tool_choice": "auto",+ "parallel_tool_calls": True,+ "max_tokens": 8,+ }+ )++ assert kwargs == {"max_tokens": 8}++ kwargs = litellm_transport.ChatCompletionsTransport.prepare_kwargs(+ {+ "tools": [+ {+ "type": "function",+ "function": {+ "name": "lookup",+ "parameters": {"type": "object"},+ },+ }+ ],+ "tool_choice": "auto",+ }+ )++ assert kwargs["tools"][0]["function"]["name"] == "lookup"+ assert kwargs["tool_choice"] == "auto"+++def test_complete_falls_back_to_chat_when_responses_shim_sends_empty_tools(+ monkeypatch,+):+ calls: list[str] = []++ def fake_responses(*args, **kwargs):+ calls.append("responses")+ raise RuntimeError(+ "Value error, `tools` must not be an empty array. "+ "Either provide at least one tool or omit the field entirely."+ )++ def fake_completion(*args, **kwargs):+ calls.append("chat")+ assert kwargs["drop_params"] is True+ assert "tools" not in kwargs+ assert "tool_choice" not in kwargs+ assert "parallel_tool_calls" not in kwargs+ return {"choices": [{"message": {"content": "ok"}}]}++ monkeypatch.setattr(litellm_transport, "responses", fake_responses)+ monkeypatch.setattr(litellm_transport, "completion", fake_completion)++ transport = litellm_transport.LiteLLMTransport(+ model="hosted_vllm/qwen",+ messages=[{"role": "user", "content": "hi"}],+ kwargs={+ "tools": [],+ "tool_choice": "auto",+ "parallel_tool_calls": True,+ "max_tokens": 8,+ },+ )++ parsed = transport.complete()++ assert parsed["response_delta"] == "ok"+ assert calls == ["responses", "chat"]++def test_responses_request_adds_openai_prompt_cache_key_for_static_prefix():request = litellm_transport.ResponsesTransport.from_chat([