Skip to content

Agent Zero AI framework

fix(llm): stabilize Chat Completions cache boundaries

Alessandro00abf6bd09bb
diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.py
index aab6399f0..c65cbf63b 100644
--- a/helpers/litellm_transport.py
+++ b/helpers/litellm_transport.py
@@ -1628,18 +1628,13 @@ def _prompt_cache_message_indexes(messages: list[dict[str, Any]]) -> list[int]:
if leading_context:
indexes.append(leading_context[-1])
- user_indexes = [
+ # The trailing user/tool result includes extras rebuilt on every turn.
+ assistant_indexes = [
index
for index, message in enumerate(messages)
- if str(message.get("role") or "") == "user"
+ if str(message.get("role") or "") == "assistant" and message.get("content")
]
- indexes.extend(user_indexes[-2:])
-
- deduplicated: list[int] = []
- for index in indexes:
- if index not in deduplicated:
- deduplicated.append(index)
- return deduplicated[:3]
+ return indexes + assistant_indexes[-2:]
def _strip_message_cache_control(message: dict[str, Any]) -> dict[str, Any]:
@@ -1663,6 +1658,8 @@ def _content_with_cache_control(content: Any) -> Any:
for index in range(len(blocks) - 1, -1, -1):
block = blocks[index]
if isinstance(block, dict):
+ if block.get("type") in {"thinking", "redacted_thinking"}:
+ continue
block["cache_control"] = marker
return blocks
if isinstance(block, str):
diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.md
index 42bbf5e83..27707906f 100644
--- a/helpers/litellm_transport.py.dox.md
+++ b/helpers/litellm_transport.py.dox.md
@@ -44,6 +44,8 @@
- Serialize synthesized Responses function-call JSON with literal Unicode so streamed raw-response logs preserve tool arguments.
- Preserve provider-state metadata when Responses API calls succeed, and fall back to local replay when provider state is unsupported.
- Keep prompt-cache markers only for providers that accept them.
+- Automatic `cache_control` marker placement is provider-gated; the OpenAI/Codex Responses configuration bypasses it. Skipping a marker target never removes its message, tool call, or reasoning content. Responses `phase: commentary`, encrypted reasoning, native replay, and request flags remain owned by their Responses paths.
+- For supported Chat providers, mark the last leading system/developer message and the last two nonempty assistant messages. Retain the previous assistant boundary as the conversation grows; leave user/tool tails containing rebuilt extras unmarked. Before the first assistant message, cache only the leading context. Do not add markers to empty assistant content or thinking/redacted-thinking blocks. Automatic placement uses at most three message breakpoints, leaving one for tool definitions; this is not a limit on conversation messages or native tool calls. Do not mutate caller messages or stored history.
## Work Guidance
@@ -58,6 +60,7 @@
## Verification
- Run `pytest tests/test_stream_tool_early_stop.py tests/test_responses_architecture.py -q` after changing transport normalization or fallback behavior.
+- Run `pytest tests/test_prompt_protocol.py -q` for cache-boundary changes; the multi-turn check covers changing extras, preserved prior boundaries, and deliberate invalidation after a protocol edit.
- Run local-provider smoke checks when changing OpenAI-compatible request cleanup.
## Child DOX Index
diff --git a/tests/test_prompt_protocol.py b/tests/test_prompt_protocol.py
index feac950ed..bbb6b8ce3 100644
--- a/tests/test_prompt_protocol.py
+++ b/tests/test_prompt_protocol.py
@@ -13,6 +13,91 @@ class _DummyLog:
return None
+async def test_cache_boundaries_survive_tool_turns_with_changing_extras(monkeypatch):
+ from copy import deepcopy
+
+ import models
+ from helpers import litellm_transport
+
+ project_rule = "Project rule."
+ turn = 0
+
+ async def fake_extensions(point, agent=None, **kwargs):
+ if point == "message_loop_prompts_after":
+ loop = kwargs["loop_data"]
+ loop.protocol_persistent["project_instructions"] = project_rule
+ loop.extras_temporary["current_datetime"] = f"time-{turn}"
+
+ monkeypatch.setattr(extension, "call_extensions_async", fake_extensions)
+ monkeypatch.setattr(history.History, "_get_max_embeds", lambda self: 0)
+ agent = object.__new__(Agent)
+ agent.loop_data = LoopData()
+ agent.context = SimpleNamespace(log=_DummyLog())
+ agent.history = history.History(agent)
+ agent.data = {}
+ agent.read_prompt = lambda name, **values: (
+ "[PROTOCOL]\n" + values["protocol"]
+ if name == "agent.context.protocol.md"
+ else "[EXTRAS]\n" + values["extras"]
+ )
+
+ async def system_prompt(_loop):
+ return ["Stable system instructions."]
+
+ agent.get_system_prompt = system_prompt
+ agent.history.add_message(False, "User task.")
+ wrapper = models.LiteLLMChatWrapper(
+ model="claude-sonnet-4-5", provider="anthropic", model_config=None
+ )
+ requests = []
+
+ def completion(**request):
+ requests.append(request)
+ return {"choices": [{"message": {"content": "ok"}}]}
+
+ monkeypatch.setattr(litellm_transport, "completion", completion)
+
+ async def cache_boundaries():
+ stored_history = agent.history.serialize()
+ prompt = await Agent.prepare_prompt(agent, agent.loop_data)
+ messages = wrapper._convert_messages(prompt)
+ original = deepcopy(messages)
+ transport = litellm_transport.LiteLLMTransport(
+ model=wrapper.model_name,
+ messages=messages,
+ kwargs={"a0_explicit_prompt_caching": True},
+ )
+ assert transport.complete()["response_delta"] == "ok"
+ assert messages == original
+ assert agent.history.serialize() == stored_history
+
+ prefix = []
+ boundaries = {}
+ for index, message in enumerate(requests[-1]["messages"]):
+ prefix.append((message["role"], litellm_transport._content_to_text(message["content"])))
+ if litellm_transport._has_cache_control(message):
+ boundaries[index] = tuple(prefix)
+ assert len(boundaries) <= 3 # Reserve the fourth provider breakpoint for tools.
+ assert all("[EXTRAS]" not in text for prefix in boundaries.values() for _, text in prefix)
+ return boundaries
+
+ previous = await cache_boundaries()
+ assert list(previous) == [0]
+ for turn in range(1, 5):
+ agent.history.add_message(True, f"Assistant tool call {turn}.")
+ agent.history.add_message(False, f"Tool result {turn}.")
+ current = await cache_boundaries()
+ assert current[max(previous)] == previous[max(previous)]
+ assert current[max(current)][-1] == ("assistant", f"Assistant tool call {turn}.")
+ previous = current
+
+ project_rule = "Changed project rule."
+ changed = await cache_boundaries()
+ assert changed[0] == previous[0]
+ assert changed[max(changed)] != previous[max(previous)]
+
+
@pytest.mark.asyncio
async def test_prepare_prompt_places_protocol_before_history_and_extras(monkeypatch):
async def fake_call_extensions(extension_point: str, agent=None, **kwargs):
diff --git a/tests/test_stream_tool_early_stop.py b/tests/test_stream_tool_early_stop.py
index 704cd9275..65d9310c6 100644
--- a/tests/test_stream_tool_early_stop.py
+++ b/tests/test_stream_tool_early_stop.py
@@ -866,13 +866,11 @@ async def test_unified_call_preserves_cache_control_with_chat_for_non_native_res
assert messages[0]["content"][-1]["cache_control"] == {
"type": "ephemeral"
}
- assert messages[1]["content"][-1]["cache_control"] == {
- "type": "ephemeral"
- }
- assert "cache_control" not in messages[2]
- assert messages[3]["content"][-1]["cache_control"] == {
+ assert messages[1]["content"] == "question"
+ assert messages[2]["content"][-1]["cache_control"] == {
"type": "ephemeral"
}
+ assert messages[3]["content"] == "follow up"
return _AsyncChunkStream([_chunk("cached")])
async def fake_rate_limiter(*args, **kwargs):
@@ -1346,6 +1344,36 @@ def test_chat_kwargs_mark_cached_tools_for_cache_control_providers():
}
[email protected]("content", [
+ None, "", [],
+ [{"type": "thinking", "thinking": "Plan", "signature": "signature"}],
+ [{"type": "redacted_thinking", "data": "opaque"}],
+])
+def test_cache_markers_preserve_assistant_tool_calls_without_cacheable_text(content):
+ messages = [
+ {"role": "system", "content": "Instructions"},
+ {"role": "user", "content": "Question"},
+ {"role": "assistant", "content": "Previous answer"},
+ {
+ "role": "assistant", "content": content,
+ "tool_calls": [{"id": "call_1", "type": "function", "function": {
+ "name": "lookup", "arguments": "{}",
+ }}],
+ },
+ {"role": "tool", "tool_call_id": "call_1", "content": "Result"},
+ {"role": "user", "content": "Current extras"},
+ ]
+
+ prepared = litellm_transport.apply_chat_prompt_cache_markers(
+ messages, model="anthropic/claude-sonnet-4-5"
+ )
+
+ assert prepared[3] == messages[3]
+ assert prepared[4:] == messages[4:]
+ assert [i for i, message in enumerate(prepared)
+ if litellm_transport._has_cache_control(message)] == [0, 2]
+
+
def test_chat_kwargs_strip_orphan_tool_choice_and_enable_fallback_drop_params():
kwargs = litellm_transport.ChatCompletionsTransport.prepare_kwargs(
{