Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(session): stop the rolling summary from breaking the prompt cache

zane-i00b92f20c6fc
diff --git a/deeptutor/agents/loop/pipeline.py b/deeptutor/agents/loop/pipeline.py
index fe4fb6c62..79b2d05d9 100644
--- a/deeptutor/agents/loop/pipeline.py
+++ b/deeptutor/agents/loop/pipeline.py
@@ -518,6 +518,22 @@ class AgenticLoopPipeline:
if isinstance(item.get("_context_snapshot"), str)
}
snapshots = dict(self._runtime_snapshots)
+ # The rolling summary is a runtime fact like any other: it is rewritten
+ # when compaction runs, so it must reach the model as a *trailing*
+ # snapshot rather than a leading system message. Appended last, a
+ # rewrite (and the compaction that causes it) only ever adds bytes after
+ # everything the previous request already sent, keeping that request a
+ # prompt-cache hit; the snapshot tag makes the unchanged case a no-op on
+ # later turns instead of re-appending a fresh copy every turn.
+ from deeptutor.services.session.model_history import (
+ SUMMARY_SNAPSHOT_NAME,
+ summary_snapshot,
+ )
+
+ metadata = context.metadata if isinstance(context.metadata, dict) else {}
+ summary_row = summary_snapshot(str(metadata.get("conversation_summary") or ""))
+ if summary_row is not None:
+ snapshots[SUMMARY_SNAPSHOT_NAME] = summary_row["content"]
for name in sorted(previous_snapshots.keys() - snapshots.keys()):
snapshots[name] = (
f"[Runtime context: {name}]\nThis section is now empty; earlier values no longer apply."
diff --git a/deeptutor/services/session/context_builder.py b/deeptutor/services/session/context_builder.py
index 74479b1f0..79625e113 100644
--- a/deeptutor/services/session/context_builder.py
+++ b/deeptutor/services/session/context_builder.py
@@ -4,6 +4,7 @@ from __future__ import annotations
from dataclasses import dataclass
import json
+import re
from typing import Any, Awaitable, Callable
from deeptutor.agents.base_agent import BaseAgent
@@ -20,7 +21,13 @@ from .ask_user_trace import (
extract_ask_user_clarification_blocks,
extract_ask_user_clarifications,
)
-from .model_history import history_groups, model_turn, replay_history
+from .model_history import (
+ SUMMARY_SNAPSHOT_NAME,
+ history_groups,
+ model_turn,
+ replay_history,
+ summary_snapshot,
+)
from .protocol import SessionStoreProtocol
from .provider_response_state import normalize_provider_response_state
@@ -46,6 +53,45 @@ MAX_RAW_REBUILD_TOKENS = 131_072
IMAGE_CONTEXT_TOKEN_ESTIMATE = 4096
+# The summarizer's scratchpad has to be removed defensively: a custom
+# OpenAI-compatible route can emit the model's reasoning inline instead of on a
+# separate reasoning channel, and the observed output *began* with the drafting
+# monologue under an unclosed tag. ``clean_thinking_tags`` removes an unclosed
+# block through end of text, which here would have discarded the entire
+# summary, so the tag is only trusted when it is closed.
+_CLOSED_THINK_RE = re.compile(
+ r"<\s*think(?:ing)?\b[^>]*>.*?<\s*/\s*think(?:ing)?\s*>", re.DOTALL | re.IGNORECASE
+)
+_OPEN_THINK_RE = re.compile(r"^<\s*think(?:ing)?\b[^>]*>", re.IGNORECASE | re.DOTALL)
+_SUMMARY_SECTION_RE = re.compile(r"(?m)^[ \t]{0,3}#{1,6}[ \t]")
+_MIN_SALVAGED_CHARS = 200
+
+
+def sanitize_summary(text: str) -> str:
+ """Drop a leaked reasoning scratchpad without ever losing the summary.
+
+ A summary is replayed on every later turn until the next compaction, so
+ scratchpad that leaks into it is paid for on every request — and, because
+ the scratchpad is written first, it also eats the summarizer's generation
+ budget and truncates the real content.
+ """
+ raw = (text or "").strip()
+ if not raw:
+ return ""
+ cleaned = _CLOSED_THINK_RE.sub("", raw).strip()
+ match = _OPEN_THINK_RE.match(cleaned)
+ if match is None:
+ return cleaned or raw
+ body = cleaned[match.end() :]
+ # The tag was never closed: cut at the first heading the summarizer was
+ # told to emit, and accept the cut only when a real body survives it.
+ section = _SUMMARY_SECTION_RE.search(body)
+ if section is not None and len(body) - section.start() >= _MIN_SALVAGED_CHARS:
+ return body[section.start() :].strip()
+ # Unstructured output: drop the marker itself rather than the content.
+ return _OPEN_THINK_RE.sub("", cleaned).replace("</think>", "").strip() or raw
+
+
def count_tokens(text: str) -> int:
"""Estimate token count with tiktoken when available."""
if not text:
@@ -308,7 +354,19 @@ class ContextBuilder:
def _model_tokens(self, messages: list[dict[str, Any]], summary: str = "") -> int:
if any(model_turn(row) is not None for row in messages):
- return _count_model_context_tokens(replay_history(messages, summary))
+ replayed = replay_history(messages)
+ tokens = _count_model_context_tokens(replayed)
+ # The answer loop appends the summary as a trailing snapshot, but
+ # only when it differs from the revision already replayed with the
+ # history. Count what the request will actually carry.
+ pending = summary_snapshot(summary)
+ if pending is not None and not any(
+ item.get("_context_snapshot") == SUMMARY_SNAPSHOT_NAME
+ and item.get("content") == pending["content"]
+ for item in replayed
+ ):
+ tokens += _count_model_context_tokens(pending)
+ return tokens
return count_tokens(build_history_text(self._build_history(summary, messages))) + sum(
_provider_response_state_tokens(row) for row in messages
)
@@ -439,6 +497,9 @@ class ContextBuilder:
"unless new information contradicts them; drop only what is obsolete. "
"Prefer concrete details (numbers, identifiers, exact terms) over "
"abstract restatement. Never invent information. "
+ "Output the summary body only: no preamble, no drafting notes, no "
+ "step-by-step reasoning, and no <think> blocks — start directly at "
+ "the first heading. "
f"Write the summary itself in {summary_language}."
)
if language.startswith("zh"):
@@ -452,6 +513,8 @@ class ContextBuilder:
"- 待办事项:未回答的问题、未完成的任务、已知阻塞\n"
"已有摘要中仍然有效的条目应原样保留,仅在新信息与之矛盾时修改,只删除确已过时"
"的内容。优先保留具体细节(数字、标识符、确切措辞),不要抽象转述,绝不虚构。"
+ "只输出摘要正文:不要输出任何思考过程或 <think> 标记,不要写草稿、不要写"
+ "「Let me…」「Draft:」之类的推演,直接从第一个小节标题开始。"
f"摘要正文本身请使用{summary_language}撰写。"
)
user_prompt = (
@@ -488,7 +551,7 @@ class ContextBuilder:
**replay_kwargs,
):
_chunks.append(_c)
- summary = "".join(_chunks).strip()
+ summary = sanitize_summary("".join(_chunks))
if count_tokens(summary) >= int(summary_budget * TRUNCATION_GUARD_RATIO):
summary = trim_incomplete_tail(summary)
return summary, events
diff --git a/deeptutor/services/session/model_history.py b/deeptutor/services/session/model_history.py
index fe9b4ada2..d36ed5240 100644
--- a/deeptutor/services/session/model_history.py
+++ b/deeptutor/services/session/model_history.py
@@ -154,14 +154,38 @@ def replay_group(
return messages
+#: Snapshot name the rolling summary rides under when it is replayed.
+SUMMARY_SNAPSHOT_NAME = "conversation_summary"
+
+
+def summary_snapshot(summary: str) -> dict[str, Any] | None:
+ """Render the rolling summary as a replayable trailing context snapshot."""
+ cleaned = (summary or "").strip()
+ if not cleaned:
+ return None
+ return {
+ "role": "user",
+ "content": f"[Conversation summary]\n{cleaned}",
+ "_context_snapshot": SUMMARY_SNAPSHOT_NAME,
+ }
+
+
def replay_history(
rows: list[dict[str, Any]],
summary: str = "",
route: dict[str, str] | None = None,
) -> list[dict[str, Any]]:
- messages = (
- [{"role": "system", "content": f"[Conversation summary]\n{summary}"}] if summary else []
- )
+ """Replay the retained model turns. ``summary`` is *not* injected here.
+
+ The rolling summary is delivered by the answer loop as a trailing runtime
+ snapshot instead (see :data:`SUMMARY_SNAPSHOT_NAME`, emitted from
+ ``context.metadata["conversation_summary"]``). Prepending it — or even
+ appending it to this list — would still place it ahead of the newest turn's
+ own messages, so a later rewrite would invalidate everything behind it.
+ Emitted as a snapshot after those bytes, a rewrite is a plain append and
+ the previous request stays a cache hit.
+ """
+ messages: list[dict[str, Any]] = []
for group in history_groups(rows):
messages.extend(replay_group(group, route))
return messages
diff --git a/tests/services/session/test_image_context_budget.py b/tests/services/session/test_image_context_budget.py
index f470c4e22..cc65abc3d 100644
--- a/tests/services/session/test_image_context_budget.py
+++ b/tests/services/session/test_image_context_budget.py
@@ -9,7 +9,7 @@ import unittest
from unittest.mock import AsyncMock
from deeptutor.services.session.context_builder import ContextBuilder, count_tokens
-from deeptutor.services.session.model_history import replay_history
+from deeptutor.services.session.model_history import replay_history, summary_snapshot
CONFIG = SimpleNamespace(model="test", context_window=272000, max_tokens=4096, binding="openai")
IMAGE_ALLOWANCE = 4096
@@ -100,8 +100,14 @@ class ImageTokenTests(unittest.TestCase):
record["messages"][-1]["thinking_blocks"] = [
{"type": "thinking", "thinking": "reasoning", "signature": "sig"}
]
- expected = count_tokens(json.dumps(replay_history(rows, "summary"), ensure_ascii=False))
- self.assertEqual(self.builder._model_tokens(rows, "summary"), expected)
+ expected = count_tokens(json.dumps(replay_history(rows), ensure_ascii=False))
+ self.assertEqual(self.builder._model_tokens(rows), expected)
+ # The summary is delivered as a trailing snapshot instead of a leading
+ # row inside the replay, so it is counted on top of the history.
+ self.assertEqual(
+ self.builder._model_tokens(rows, "summary"),
+ expected + count_tokens(json.dumps(summary_snapshot("summary"), ensure_ascii=False)),
+ )
def test_encoding_in_user_text_is_still_text(self):
rows = rows_with_parts([{"type": "text", "text": RANDOM_IMAGE}])
@@ -150,7 +156,8 @@ class BuilderTests(unittest.IsolatedAsyncioTestCase):
result = await builder.build(session_id="test", llm_config=CONFIG)
builder._summarize.assert_awaited_once()
store.update_summary.assert_awaited_once_with("test", "compact summary", 2)
- self.assertEqual(result.model_history[1:], replay_history(latest))
+ # The summary rides separately from the replayed history now.
+ self.assertEqual(result.model_history, replay_history(latest))
async def test_many_images_still_consume_finite_budget(self):
rows = []
diff --git a/tests/services/session/test_model_history.py b/tests/services/session/test_model_history.py
index 8448091f8..afe0a30bb 100644
--- a/tests/services/session/test_model_history.py
+++ b/tests/services/session/test_model_history.py
@@ -142,7 +142,11 @@ async def test_compaction_preserves_whole_tool_turns_and_reuses_the_old_prefix()
]
assert request["tools"] == recent["tools"]
store.update_summary.assert_awaited_once_with("s", "Summary preserving the evidence", 2)
- assert result.model_history[1:] == recent["messages"]
+ # The summary no longer occupies model_history[0]: the answer loop delivers
+ # it as a trailing runtime snapshot, so the replayed history starts at the
+ # oldest retained turn while the summary still reaches the model.
+ assert result.model_history == recent["messages"]
+ assert result.conversation_summary == "Summary preserving the evidence"
assert result.conversation_history[-1]["content"] == "Displayed answer"