Skip to content

Agent Zero AI framework

fix(llm): enable Venice cache routing and custom provider overrides

Alessandrof72945e70cf4
diff --git a/conf/AGENTS.md b/conf/AGENTS.md
index 285b67d63..72ebd13d1 100644
--- a/conf/AGENTS.md
+++ b/conf/AGENTS.md
@@ -20,6 +20,7 @@
- Local provider defaults omit `api_key`: LiteLLM handles unauthenticated endpoints, while saved or explicit keys remain available for authenticated servers. Static dummy keys would override saved keys. OAuth compatibility keys belong to the OAuth plugin's connected-account hook.
- Prefer native LiteLLM routes for shipped providers; keep endpoint overrides for Nebius Token Factory and Z.AI Coding. Model-list `default_base` is independent of LiteLLM's inference defaults.
- Venice chat uses `veniceai` with `venice_parameters` in `extra_body`; Venice embeddings and the Agent Zero proxy retain their OpenAI-compatible routes. OpenRouter embeddings use the native route and retain Agent Zero attribution headers.
+- Venice and Agent Zero API chat defaults select `a0_prompt_cache_mode: prompt_cache_key` for cache routing. Explicit model kwargs may override the mode; embeddings must not inherit it.
- Templates must avoid accidentally unignoring private runtime content.
## Work Guidance
diff --git a/conf/model_providers.yaml b/conf/model_providers.yaml
index 56203c418..6af6892fe 100644
--- a/conf/model_providers.yaml
+++ b/conf/model_providers.yaml
@@ -33,6 +33,7 @@ chat:
endpoint_url: "https://api.venice.ai/api/v1/models"
kwargs:
a0_api_mode: chat
+ a0_prompt_cache_mode: prompt_cache_key
api_base: https://llm.agent-zero.ai/v1
venice_parameters:
include_venice_system_prompt: false
@@ -215,6 +216,7 @@ chat:
endpoint_url: "https://api.venice.ai/api/v1/models"
kwargs:
a0_api_mode: chat
+ a0_prompt_cache_mode: prompt_cache_key
extra_body:
venice_parameters:
include_venice_system_prompt: false
diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.py
index 8763de26e..7c38957bd 100644
--- a/helpers/litellm_transport.py
+++ b/helpers/litellm_transport.py
@@ -384,7 +384,7 @@ class LiteLLMTransport:
"messages": ChatCompletionsTransport.prepare_messages(
self.messages,
model=self.model,
- kwargs=chat_kwargs,
+ kwargs=self.kwargs,
),
"stream": stream,
**chat_kwargs,
@@ -543,7 +543,7 @@ class ChatCompletionsTransport:
model: str = "",
kwargs: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
- if _is_openai_prompt_cache_provider(model, kwargs or {}):
+ if _prompt_cache_mode(model, kwargs or {}) == "prompt_cache_key":
stripped = _without_cache_control(messages)
return stripped if isinstance(stripped, list) else messages
return messages
@@ -558,18 +558,19 @@ class ChatCompletionsTransport:
explicit_prompt_caching: bool = False,
) -> dict[str, Any]:
chat_kwargs = dict(kwargs)
+ cache_mode = _prompt_cache_mode(model, chat_kwargs)
_drop_internal_transport_kwargs(chat_kwargs)
if not _has_tools(chat_kwargs.get("tools")):
chat_kwargs.pop("tools", None)
chat_kwargs.pop("tool_choice", None)
chat_kwargs.pop("parallel_tool_calls", None)
- if _is_openai_prompt_cache_provider(model, chat_kwargs):
+ if cache_mode == "prompt_cache_key":
_prepare_openai_prompt_cache_params(
chat_kwargs,
messages or [],
model=model,
)
- elif _supports_cache_control_markers(model, chat_kwargs) and (
+ elif cache_mode == "cache_control" and (
explicit_prompt_caching or _has_cache_control(messages or [])
):
_apply_tool_cache_control(chat_kwargs)
@@ -872,6 +873,7 @@ class ResponsesTransport:
request.pop("parallel_tool_calls", None)
cls.prepare_prompt_caching(request, messages or [], model=model)
+ request.pop("a0_prompt_cache_mode", None)
_ = stop
return {key: value for key, value in request.items() if value is not None}
@@ -954,11 +956,11 @@ class ResponsesTransport:
messages: list[dict[str, Any]],
model: str = "",
) -> None:
- if not _is_openai_prompt_cache_provider(model, request):
+ if _prompt_cache_mode(model, request) == "prompt_cache_key":
+ _prepare_openai_prompt_cache_params(request, messages, model=model)
+ elif not _is_openai_prompt_cache_provider(model, request):
return
- _prepare_openai_prompt_cache_params(request, messages, model=model)
-
for key in OPENAI_RESPONSES_EXTRA_BODY_PARAMS:
if key not in request:
continue
@@ -1558,6 +1560,7 @@ def _drop_responses_only_kwargs(kwargs: dict[str, Any]) -> None:
def _drop_internal_transport_kwargs(kwargs: dict[str, Any]) -> None:
_drop_legacy_transport_kwargs(kwargs)
+ kwargs.pop("a0_prompt_cache_mode", None)
kwargs.pop("a0_explicit_prompt_caching", None)
kwargs.pop("a0_responses_function_tools", None)
kwargs.pop("responses_builtin_tools", None)
@@ -1628,7 +1631,7 @@ def apply_chat_prompt_cache_markers(
model: str = "",
kwargs: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
- if not _supports_cache_control_markers(model, kwargs or {}):
+ if _prompt_cache_mode(model, kwargs or {}) != "cache_control":
return [dict(message) for message in messages]
prepared = [_strip_message_cache_control(message) for message in messages]
@@ -1750,7 +1753,26 @@ def _should_preserve_cache_control_on_chat(
) -> bool:
if not (_has_cache_control(messages) or _has_cache_control(kwargs.get("tools"))):
return False
- return not _is_native_responses_provider(model, kwargs)
+ return (
+ _prompt_cache_mode(model, kwargs) == "cache_control"
+ or not _is_native_responses_provider(model, kwargs)
+ )
+
+
+def _prompt_cache_mode(model: str, kwargs: dict[str, Any]) -> str:
+ value = kwargs.get("a0_prompt_cache_mode")
+ mode = "auto" if value is None else str(value).strip().lower()
+ if mode not in {"auto", "prompt_cache_key", "cache_control", "off"}:
+ raise ValueError(
+ "a0_prompt_cache_mode must be auto, prompt_cache_key, cache_control, or off"
+ )
+ if mode != "auto":
+ return mode
+ if _is_openai_prompt_cache_provider(model, kwargs):
+ return "prompt_cache_key"
+ if _supports_cache_control_markers(model, kwargs):
+ return "cache_control"
+ return "off"
def _is_native_responses_provider(model: str, kwargs: dict[str, Any]) -> bool:
@@ -1811,7 +1833,10 @@ def _prepare_openai_prompt_cache_params(
*,
model: str = "",
) -> None:
- if "prompt_cache_key" in request:
+ extra_body = request.get("extra_body")
+ if "prompt_cache_key" in request or (
+ isinstance(extra_body, dict) and "prompt_cache_key" in extra_body
+ ):
return
sanitized_messages = _without_cache_control(messages)
diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.md
index 936065a99..9707f56b0 100644
--- a/helpers/litellm_transport.py.dox.md
+++ b/helpers/litellm_transport.py.dox.md
@@ -47,7 +47,9 @@
- Preserve provider-state metadata when Responses API calls succeed, and fall back to local replay when provider state is unsupported.
- Keep prompt-cache markers only for providers that accept them.
- Chat Completions parsing prefers `reasoning_content` and falls back to `reasoning` for both streaming deltas and complete messages, preserving response content and applying the same encrypted-reasoning filter. Verify with `test_chat_reasoning_field_fallback` in `tests/test_stream_tool_early_stop.py`.
-- Automatic `cache_control` marker placement is provider-gated; the OpenAI/Codex Responses configuration bypasses it. Skipping a marker target never removes its message, tool call, or reasoning content. Responses `phase: commentary`, encrypted reasoning, native replay, and request flags remain owned by their Responses paths.
+- `a0_prompt_cache_mode` selects `auto` (existing provider detection), `prompt_cache_key`, `cache_control`, or `off` (no automatic hints). Reject invalid values before provider calls and strip the internal control from both transports. Provider defaults and model Additional parameters may set it; embeddings do not acquire chat-provider defaults.
+- Cache-key mode hashes the stable leading context and tool definitions, excluding changing conversation tails. Preserve caller keys at the request level or in `extra_body`; explicit retention still passes through when automatic hints are off. Strip message markers in cache-key mode without mutating callers.
+- Automatic `cache_control` marker placement follows the selected cache mode; default OpenAI/Codex Responses configuration bypasses it. An explicit `cache_control` override keeps marked requests on Chat Completions, including custom OpenAI-compatible endpoints. Skipping a marker target never removes its message, tool call, or reasoning content. Responses `phase: commentary`, encrypted reasoning, native replay, and request flags remain owned by their Responses paths.
- For supported Chat providers, mark the last leading system/developer message and the last two nonempty assistant messages. Retain the previous assistant boundary as the conversation grows; leave user/tool tails containing rebuilt extras unmarked. Before the first assistant message, cache only the leading context. Do not add markers to empty assistant content or thinking/redacted-thinking blocks. Automatic placement uses at most three message breakpoints, leaving one for tool definitions; this is not a limit on conversation messages or native tool calls. Do not mutate caller messages or stored history.
## Work Guidance
@@ -65,6 +67,7 @@
- Run `pytest tests/test_stream_tool_early_stop.py tests/test_responses_architecture.py -q` after changing transport normalization or fallback behavior.
- Run `pytest tests/test_prompt_protocol.py -q` for cache-boundary changes; the multi-turn check covers changing extras, preserved prior boundaries, and deliberate invalidation after a protocol edit.
- Run local-provider smoke checks when changing OpenAI-compatible request cleanup.
+- Run `pytest tests/test_litellm_sdk.py -q` for outgoing cache-routing bodies on Venice, Agent Zero API, and custom providers across sync/async and streaming/non-streaming calls.
## Child DOX Index
diff --git a/plugins/_model_config/AGENTS.md b/plugins/_model_config/AGENTS.md
index 020c73f21..e69246848 100644
--- a/plugins/_model_config/AGENTS.md
+++ b/plugins/_model_config/AGENTS.md
@@ -33,6 +33,7 @@
- Show `Use separate Vision Model` immediately below `Supports Vision` while Main vision is enabled, not inside Advanced Settings; describe the disabled state as using Main's native vision.
- In model overviews, render the effective Vision Model as a text-only `Vision override / Provider / Model` child aligned with Main's provider column, not as an icon-bearing peer row.
- Changing a model provider in the settings UI must clear `api_base` and `kwargs` because both may be provider-specific.
+- Chat, Vision, and Utility Additional parameters expose `a0_prompt_cache_mode` overrides through existing kwargs; keep cache-mode help out of embedding fields. The shared transport validates and strips this control.
- Repair provider-specific model-config aliases at the model-config read/build boundary; keep provider-specific repairs out of provider-agnostic core wrappers such as `models.py`.
- The `Agent.read_prompt/end` Anthropic hook adapts only the two bundled thoughts instructions in main/communication/solving prompt results when the effective main provider or model name contains `anthropic`, or the model name contains `claude` (case-insensitive, regardless of provider). Keep prompt files, JSON fields, history, custom wording, and other models unchanged; standalone communication reads must match main-prompt rendering for native protocol projection. Verify with `tests/test_anthropic_thoughts_prompt.py`.
- `modelConfig.createPresetEditor()` owns local preset drafts, row actions, and stable UI-only row keys so deletion or renaming cannot rebind nested model fields.
diff --git a/plugins/_model_config/README.md b/plugins/_model_config/README.md
index a9c21f114..fc9857f6b 100644
--- a/plugins/_model_config/README.md
+++ b/plugins/_model_config/README.md
@@ -36,6 +36,12 @@ The normal plugin resolution order selects the most specific available reference
- The preset editor exposes the shared API key for each selected provider and saves key changes separately from secret-free preset definitions.
- Project Settings selects from the same global preset definitions.
+## Cache Routing
+
+Venice and Agent Zero API send a stable `prompt_cache_key` derived from the model, leading instructions, and tool definitions. This is a routing hint; cache availability and retention remain provider-controlled. Venice documents it in its [Chat Completions API](https://docs.venice.ai/api-reference/endpoint/chat/completions).
+
+In a chat model's **Advanced Settings → Additional parameters**, set `a0_prompt_cache_mode=prompt_cache_key` or `a0_prompt_cache_mode=cache_control` to match a custom provider's supported format. `auto` keeps the existing provider detection; `off` disables automatic hints. Explicit keys, including `extra_body` keys, remain unchanged. Marker mode keeps marked requests on Chat Completions and uses the existing stable message/tool boundaries.
+
## Initial Presets And Migration
The startup migration converts the prior global full model configuration into `Default`, preserves existing global presets, promotes distinct scoped full configurations and legacy project presets into uniquely named global presets, and rewrites scoped config files to selection-only JSON. Original files receive a `.pre-unified-presets.bak` backup before replacement; the migration is idempotent.
diff --git a/plugins/_model_config/webui/model-field.html b/plugins/_model_config/webui/model-field.html
index d38f5ae9a..f7983b80e 100644
--- a/plugins/_model_config/webui/model-field.html
+++ b/plugins/_model_config/webui/model-field.html
@@ -325,6 +325,7 @@
<div class="field-title">Additional parameters</div>
<div class="field-description">
Any other parameters supported by <a href='https://docs.litellm.ai/docs/set_keys' target='_blank'>LiteLLM</a>. Format is KEY=VALUE on individual lines. Values can be JSON objects, arrays, numbers, lowercase true/false/null, or text. Quote strings to keep values such as "true" as text.
+ <div x-show="modelType !== 'embedding'">For custom providers, set <code>a0_prompt_cache_mode=prompt_cache_key</code> or <code>a0_prompt_cache_mode=cache_control</code> to match the provider's cache support. The default is <code>auto</code>; <code>off</code> disables automatic cache hints.</div>
</div>
</div>
<div class="field-control">
diff --git a/tests/test_litellm_sdk.py b/tests/test_litellm_sdk.py
index 3336d9b97..e4ece19f7 100644
--- a/tests/test_litellm_sdk.py
+++ b/tests/test_litellm_sdk.py
@@ -203,6 +203,72 @@ def test_sdk_anthropic_cache_markers_over_http(provider):
assert body["messages"][1]["content"][-1]["cache_control"] == {"type": "ephemeral"}
[email protected]("async_call", [False, True])
[email protected]("stream", [False, True])
[email protected]("provider_id,mode,expected", [
+ ("venice", None, "prompt_cache_key"),
+ ("a0_venice", None, "prompt_cache_key"),
+ ("other", "prompt_cache_key", "prompt_cache_key"),
+ ("other", "cache_control", "cache_control"),
+ ("other", "auto", "off"),
+ ("venice", "off", "off"),
+])
+async def test_sdk_provider_cache_routing_over_http(provider, monkeypatch, provider_id, mode, expected, stream, async_call):
+ from copy import deepcopy
+ from pathlib import Path
+ import yaml
+ from helpers.litellm_transport import _has_cache_control
+
+ definitions = yaml.safe_load((Path(__file__).resolve().parents[1] / "conf/model_providers.yaml").read_text())
+ monkeypatch.setattr(models, "get_provider_config", lambda kind, pid: definitions[kind].get(pid))
+ monkeypatch.setattr(models, "get_api_key", lambda _pid: "test-key")
+ wrapper = models.get_chat_model(provider_id, "llama-3.3-70b", **(
+ {"a0_prompt_cache_mode": mode} if mode is not None else {}
+ ))
+ messages = [{"role": "system", "content": "Stable instructions"},
+ {"role": "user", "content": "Hello"}]
+ kwargs = {**wrapper.kwargs, **provider.kwargs, "a0_explicit_prompt_caching": True,
+ "tools": [{"type": "function", "function": {k: v for k, v in TOOL.items() if k != "type"}}]}
+ original = deepcopy((messages, kwargs))
+ provider.replies.append((200, chat_reply(stream)))
+ transport = LiteLLMTransport(model=wrapper.model_name, messages=messages, kwargs=kwargs)
+ if stream:
+ chunks = [chunk async for chunk in transport.astream()] if async_call else list(transport.stream())
+ assert chunks
+ elif async_call:
+ await transport.acomplete()
+ else:
+ transport.complete()
+
+ _, path, body = provider.requests[0]
+ assert path == "/v1/chat/completions"
+ assert (messages, kwargs) == original
+ assert not any(key.startswith("a0_") for key in body)
+ assert ("prompt_cache_key" in body) == (expected == "prompt_cache_key")
+ assert _has_cache_control(body) == (expected == "cache_control")
+ if expected == "prompt_cache_key":
+ assert body["prompt_cache_key"].startswith("a0-")
+ elif expected == "cache_control":
+ assert body["messages"][0]["content"][-1]["cache_control"] == {"type": "ephemeral"}
+ assert body["tools"][0]["function"]["cache_control"] == {"type": "ephemeral"}
+
+
[email protected]("mode", ["prompt_cache_key", "off"])
+async def test_sdk_custom_responses_cache_mode_over_http(provider, mode):
+ provider.replies.append((200, response_reply(False)))
+ transport = LiteLLMTransport(
+ model="openai/gpt-4o", messages=[{"role": "system", "content": "Stable instructions"}],
+ kwargs={**provider.kwargs, "a0_api_mode": "responses", "a0_prompt_cache_mode": mode},
+ )
+ await transport.acomplete()
+ _, path, body = provider.requests[0]
+ assert path == "/v1/responses"
+ assert ("prompt_cache_key" in body) == (mode == "prompt_cache_key")
+ assert "a0_prompt_cache_mode" not in body
+
+
@pytest.mark.parametrize("provider_id,name,route,base", [
("cerebras", "gpt-oss-120b", "cerebras", "https://api.cerebras.ai/v1"),
("nebius", "meta-llama/Llama-3.3-70B-Instruct", "nebius", "https://api.tokenfactory.nebius.com/v1"),
diff --git a/tests/test_stream_tool_early_stop.py b/tests/test_stream_tool_early_stop.py
index 988962009..ed03e82dd 100644
--- a/tests/test_stream_tool_early_stop.py
+++ b/tests/test_stream_tool_early_stop.py
@@ -1283,6 +1283,81 @@ def test_chat_kwargs_add_openai_prompt_cache_key_for_chat_completions():
assert kwargs["max_tokens"] == 10
[email protected]("model,mode,expected", [
+ ("openai/gpt-5.4", "auto", "prompt_cache_key"),
+ ("anthropic/claude-sonnet-4-5", "auto", "cache_control"),
+ ("openai/custom", "auto", "off"),
+ ("openai/custom", "prompt_cache_key", "prompt_cache_key"),
+ ("openai/custom", "cache_control", "cache_control"),
+ ("openai/gpt-5.4", "off", "off"),
+ ("anthropic/claude-sonnet-4-5", "off", "off"),
+])
+def test_prompt_cache_mode_controls_hints_without_mutating_callers(model, mode, expected):
+ from copy import deepcopy
+
+ messages = [{"role": "system", "content": "Stable instructions"},
+ {"role": "user", "content": "Question"}]
+ kwargs = {"a0_prompt_cache_mode": mode, "a0_explicit_prompt_caching": True}
+ if model == "openai/custom":
+ kwargs["api_base"] = "https://custom.example/v1"
+ original = deepcopy((messages, kwargs))
+ request = litellm_transport.LiteLLMTransport(
+ model=model, messages=messages, kwargs=kwargs
+ )._chat_request(stream=False)
+
+ assert (messages, kwargs) == original
+ assert ("prompt_cache_key" in request) == (expected == "prompt_cache_key")
+ assert litellm_transport._has_cache_control(request["messages"]) == (expected == "cache_control")
+ assert "a0_prompt_cache_mode" not in request
+
+
[email protected]("value", [False, "", "typo"])
+def test_invalid_prompt_cache_mode_fails_before_provider_call(value):
+ with pytest.raises(ValueError, match="a0_prompt_cache_mode must be"):
+ litellm_transport.ChatCompletionsTransport.prepare_kwargs(
+ {"a0_prompt_cache_mode": value}, model="openai/custom"
+ )
+
+
+def test_custom_cache_control_override_keeps_markers_on_chat():
+ transport = litellm_transport.LiteLLMTransport(
+ model="openai/custom",
+ messages=[{"role": "system", "content": "Stable instructions"}],
+ kwargs={"api_base": "https://custom.example/v1", "a0_api_mode": "responses",
+ "a0_prompt_cache_mode": "cache_control", "a0_explicit_prompt_caching": True},
+ )
+ assert transport.policy.mode is litellm_transport.TransportMode.CHAT_COMPLETIONS
+ assert litellm_transport._has_cache_control(transport._chat_request(stream=False)["messages"])
+
+
[email protected]("mode", ["prompt_cache_key", "off"])
+def test_responses_cache_mode_preserves_explicit_retention(mode):
+ request = litellm_transport.ResponsesTransport.from_chat(
+ [{"role": "system", "content": "Stable instructions"}],
+ {"a0_prompt_cache_mode": mode, "prompt_cache_retention": "24h"},
+ model="openai/gpt-5.4",
+ )
+ assert ("prompt_cache_key" in request) == (mode == "prompt_cache_key")
+ assert request["extra_body"]["prompt_cache_retention"] == "24h"
+ assert "a0_prompt_cache_mode" not in request
+
+
[email protected]("key_kwargs", [
+ {"prompt_cache_key": "caller-key"},
+ {"extra_body": {"prompt_cache_key": "caller-key"}},
+])
+def test_prompt_cache_mode_preserves_explicit_keys_on_both_transports(key_kwargs):
+ messages = [{"role": "system", "content": "Stable instructions"}]
+ kwargs = {"api_base": "https://custom.example/v1",
+ "a0_prompt_cache_mode": "prompt_cache_key", **key_kwargs}
+ for prepare in (litellm_transport.ChatCompletionsTransport.prepare_kwargs,
+ litellm_transport.ResponsesTransport.prepare_kwargs):
+ request = prepare(kwargs, model="openai/custom", messages=messages)
+ assert request.get("prompt_cache_key") == key_kwargs.get("prompt_cache_key")
+ assert request.get("extra_body", {}).get("prompt_cache_key") == key_kwargs.get("extra_body", {}).get("prompt_cache_key")
+ assert "a0_prompt_cache_mode" not in request
+
+
def test_chat_messages_strip_cache_control_for_openai_prompt_cache():
messages = [
{