Agent Zero AI framework
fix(llm): stabilize Chat Completions cache boundaries
Alessandro00abf6bd09bb
diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.pyindex aab6399f0..c65cbf63b 100644--- a/helpers/litellm_transport.py+++ b/helpers/litellm_transport.py@@ -1628,18 +1628,13 @@ def _prompt_cache_message_indexes(messages: list[dict[str, Any]]) -> list[int]:if leading_context:indexes.append(leading_context[-1])- user_indexes = [+ # The trailing user/tool result includes extras rebuilt on every turn.+ assistant_indexes = [indexfor index, message in enumerate(messages)- if str(message.get("role") or "") == "user"+ if str(message.get("role") or "") == "assistant" and message.get("content")]- indexes.extend(user_indexes[-2:])-- deduplicated: list[int] = []- for index in indexes:- if index not in deduplicated:- deduplicated.append(index)- return deduplicated[:3]+ return indexes + assistant_indexes[-2:]def _strip_message_cache_control(message: dict[str, Any]) -> dict[str, Any]:@@ -1663,6 +1658,8 @@ def _content_with_cache_control(content: Any) -> Any:for index in range(len(blocks) - 1, -1, -1):block = blocks[index]if isinstance(block, dict):+ if block.get("type") in {"thinking", "redacted_thinking"}:+ continueblock["cache_control"] = markerreturn blocksif isinstance(block, str):diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.mdindex 42bbf5e83..27707906f 100644--- a/helpers/litellm_transport.py.dox.md+++ b/helpers/litellm_transport.py.dox.md@@ -44,6 +44,8 @@- Serialize synthesized Responses function-call JSON with literal Unicode so streamed raw-response logs preserve tool arguments.- Preserve provider-state metadata when Responses API calls succeed, and fall back to local replay when provider state is unsupported.- Keep prompt-cache markers only for providers that accept them.+- Automatic `cache_control` marker placement is provider-gated; the OpenAI/Codex Responses configuration bypasses it. Skipping a marker target never removes its message, tool call, or reasoning content. Responses `phase: commentary`, encrypted reasoning, native replay, and request flags remain owned by their Responses paths.+- For supported Chat providers, mark the last leading system/developer message and the last two nonempty assistant messages. Retain the previous assistant boundary as the conversation grows; leave user/tool tails containing rebuilt extras unmarked. Before the first assistant message, cache only the leading context. Do not add markers to empty assistant content or thinking/redacted-thinking blocks. Automatic placement uses at most three message breakpoints, leaving one for tool definitions; this is not a limit on conversation messages or native tool calls. Do not mutate caller messages or stored history.## Work Guidance@@ -58,6 +60,7 @@## Verification- Run `pytest tests/test_stream_tool_early_stop.py tests/test_responses_architecture.py -q` after changing transport normalization or fallback behavior.+- Run `pytest tests/test_prompt_protocol.py -q` for cache-boundary changes; the multi-turn check covers changing extras, preserved prior boundaries, and deliberate invalidation after a protocol edit.- Run local-provider smoke checks when changing OpenAI-compatible request cleanup.## Child DOX Indexdiff --git a/tests/test_prompt_protocol.py b/tests/test_prompt_protocol.pyindex feac950ed..bbb6b8ce3 100644--- a/tests/test_prompt_protocol.py+++ b/tests/test_prompt_protocol.py@@ -13,6 +13,91 @@ class _DummyLog:return None+async def test_cache_boundaries_survive_tool_turns_with_changing_extras(monkeypatch):+ from copy import deepcopy++ import models+ from helpers import litellm_transport++ project_rule = "Project rule."+ turn = 0++ async def fake_extensions(point, agent=None, **kwargs):+ if point == "message_loop_prompts_after":+ loop = kwargs["loop_data"]+ loop.protocol_persistent["project_instructions"] = project_rule+ loop.extras_temporary["current_datetime"] = f"time-{turn}"++ monkeypatch.setattr(extension, "call_extensions_async", fake_extensions)+ monkeypatch.setattr(history.History, "_get_max_embeds", lambda self: 0)+ agent = object.__new__(Agent)+ agent.loop_data = LoopData()+ agent.context = SimpleNamespace(log=_DummyLog())+ agent.history = history.History(agent)+ agent.data = {}+ agent.read_prompt = lambda name, **values: (+ "[PROTOCOL]\n" + values["protocol"]+ if name == "agent.context.protocol.md"+ else "[EXTRAS]\n" + values["extras"]+ )++ async def system_prompt(_loop):+ return ["Stable system instructions."]++ agent.get_system_prompt = system_prompt+ agent.history.add_message(False, "User task.")+ wrapper = models.LiteLLMChatWrapper(+ model="claude-sonnet-4-5", provider="anthropic", model_config=None+ )+ requests = []++ def completion(**request):+ requests.append(request)+ return {"choices": [{"message": {"content": "ok"}}]}++ monkeypatch.setattr(litellm_transport, "completion", completion)++ async def cache_boundaries():+ stored_history = agent.history.serialize()+ prompt = await Agent.prepare_prompt(agent, agent.loop_data)+ messages = wrapper._convert_messages(prompt)+ original = deepcopy(messages)+ transport = litellm_transport.LiteLLMTransport(+ model=wrapper.model_name,+ messages=messages,+ kwargs={"a0_explicit_prompt_caching": True},+ )+ assert transport.complete()["response_delta"] == "ok"+ assert messages == original+ assert agent.history.serialize() == stored_history++ prefix = []+ boundaries = {}+ for index, message in enumerate(requests[-1]["messages"]):+ prefix.append((message["role"], litellm_transport._content_to_text(message["content"])))+ if litellm_transport._has_cache_control(message):+ boundaries[index] = tuple(prefix)+ assert len(boundaries) <= 3 # Reserve the fourth provider breakpoint for tools.+ assert all("[EXTRAS]" not in text for prefix in boundaries.values() for _, text in prefix)+ return boundaries++ previous = await cache_boundaries()+ assert list(previous) == [0]+ for turn in range(1, 5):+ agent.history.add_message(True, f"Assistant tool call {turn}.")+ agent.history.add_message(False, f"Tool result {turn}.")+ current = await cache_boundaries()+ assert current[max(previous)] == previous[max(previous)]+ assert current[max(current)][-1] == ("assistant", f"Assistant tool call {turn}.")+ previous = current++ project_rule = "Changed project rule."+ changed = await cache_boundaries()+ assert changed[0] == previous[0]+ assert changed[max(changed)] != previous[max(previous)]++@pytest.mark.asyncioasync def test_prepare_prompt_places_protocol_before_history_and_extras(monkeypatch):async def fake_call_extensions(extension_point: str, agent=None, **kwargs):diff --git a/tests/test_stream_tool_early_stop.py b/tests/test_stream_tool_early_stop.pyindex 704cd9275..65d9310c6 100644--- a/tests/test_stream_tool_early_stop.py+++ b/tests/test_stream_tool_early_stop.py@@ -866,13 +866,11 @@ async def test_unified_call_preserves_cache_control_with_chat_for_non_native_resassert messages[0]["content"][-1]["cache_control"] == {"type": "ephemeral"}- assert messages[1]["content"][-1]["cache_control"] == {- "type": "ephemeral"- }- assert "cache_control" not in messages[2]- assert messages[3]["content"][-1]["cache_control"] == {+ assert messages[1]["content"] == "question"+ assert messages[2]["content"][-1]["cache_control"] == {"type": "ephemeral"}+ assert messages[3]["content"] == "follow up"return _AsyncChunkStream([_chunk("cached")])async def fake_rate_limiter(*args, **kwargs):@@ -1346,6 +1344,36 @@ def test_chat_kwargs_mark_cached_tools_for_cache_control_providers():}[email protected]("content", [+ None, "", [],+ [{"type": "thinking", "thinking": "Plan", "signature": "signature"}],+ [{"type": "redacted_thinking", "data": "opaque"}],+])+def test_cache_markers_preserve_assistant_tool_calls_without_cacheable_text(content):+ messages = [+ {"role": "system", "content": "Instructions"},+ {"role": "user", "content": "Question"},+ {"role": "assistant", "content": "Previous answer"},+ {+ "role": "assistant", "content": content,+ "tool_calls": [{"id": "call_1", "type": "function", "function": {+ "name": "lookup", "arguments": "{}",+ }}],+ },+ {"role": "tool", "tool_call_id": "call_1", "content": "Result"},+ {"role": "user", "content": "Current extras"},+ ]++ prepared = litellm_transport.apply_chat_prompt_cache_markers(+ messages, model="anthropic/claude-sonnet-4-5"+ )++ assert prepared[3] == messages[3]+ assert prepared[4:] == messages[4:]+ assert [i for i, message in enumerate(prepared)+ if litellm_transport._has_cache_control(message)] == [0, 2]++def test_chat_kwargs_strip_orphan_tool_choice_and_enable_fallback_drop_params():kwargs = litellm_transport.ChatCompletionsTransport.prepare_kwargs({