DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(session): stop the rolling summary from breaking the prompt cache
zane-i00b92f20c6fc
diff --git a/deeptutor/agents/loop/pipeline.py b/deeptutor/agents/loop/pipeline.pyindex fe4fb6c62..79b2d05d9 100644--- a/deeptutor/agents/loop/pipeline.py+++ b/deeptutor/agents/loop/pipeline.py@@ -518,6 +518,22 @@ class AgenticLoopPipeline:if isinstance(item.get("_context_snapshot"), str)}snapshots = dict(self._runtime_snapshots)+ # The rolling summary is a runtime fact like any other: it is rewritten+ # when compaction runs, so it must reach the model as a *trailing*+ # snapshot rather than a leading system message. Appended last, a+ # rewrite (and the compaction that causes it) only ever adds bytes after+ # everything the previous request already sent, keeping that request a+ # prompt-cache hit; the snapshot tag makes the unchanged case a no-op on+ # later turns instead of re-appending a fresh copy every turn.+ from deeptutor.services.session.model_history import (+ SUMMARY_SNAPSHOT_NAME,+ summary_snapshot,+ )++ metadata = context.metadata if isinstance(context.metadata, dict) else {}+ summary_row = summary_snapshot(str(metadata.get("conversation_summary") or ""))+ if summary_row is not None:+ snapshots[SUMMARY_SNAPSHOT_NAME] = summary_row["content"]for name in sorted(previous_snapshots.keys() - snapshots.keys()):snapshots[name] = (f"[Runtime context: {name}]\nThis section is now empty; earlier values no longer apply."diff --git a/deeptutor/services/session/context_builder.py b/deeptutor/services/session/context_builder.pyindex 74479b1f0..79625e113 100644--- a/deeptutor/services/session/context_builder.py+++ b/deeptutor/services/session/context_builder.py@@ -4,6 +4,7 @@ from __future__ import annotationsfrom dataclasses import dataclassimport json+import refrom typing import Any, Awaitable, Callablefrom deeptutor.agents.base_agent import BaseAgent@@ -20,7 +21,13 @@ from .ask_user_trace import (extract_ask_user_clarification_blocks,extract_ask_user_clarifications,)-from .model_history import history_groups, model_turn, replay_history+from .model_history import (+ SUMMARY_SNAPSHOT_NAME,+ history_groups,+ model_turn,+ replay_history,+ summary_snapshot,+)from .protocol import SessionStoreProtocolfrom .provider_response_state import normalize_provider_response_state@@ -46,6 +53,45 @@ MAX_RAW_REBUILD_TOKENS = 131_072IMAGE_CONTEXT_TOKEN_ESTIMATE = 4096+# The summarizer's scratchpad has to be removed defensively: a custom+# OpenAI-compatible route can emit the model's reasoning inline instead of on a+# separate reasoning channel, and the observed output *began* with the drafting+# monologue under an unclosed tag. ``clean_thinking_tags`` removes an unclosed+# block through end of text, which here would have discarded the entire+# summary, so the tag is only trusted when it is closed.+_CLOSED_THINK_RE = re.compile(+ r"<\s*think(?:ing)?\b[^>]*>.*?<\s*/\s*think(?:ing)?\s*>", re.DOTALL | re.IGNORECASE+)+_OPEN_THINK_RE = re.compile(r"^<\s*think(?:ing)?\b[^>]*>", re.IGNORECASE | re.DOTALL)+_SUMMARY_SECTION_RE = re.compile(r"(?m)^[ \t]{0,3}#{1,6}[ \t]")+_MIN_SALVAGED_CHARS = 200+++def sanitize_summary(text: str) -> str:+ """Drop a leaked reasoning scratchpad without ever losing the summary.++ A summary is replayed on every later turn until the next compaction, so+ scratchpad that leaks into it is paid for on every request — and, because+ the scratchpad is written first, it also eats the summarizer's generation+ budget and truncates the real content.+ """+ raw = (text or "").strip()+ if not raw:+ return ""+ cleaned = _CLOSED_THINK_RE.sub("", raw).strip()+ match = _OPEN_THINK_RE.match(cleaned)+ if match is None:+ return cleaned or raw+ body = cleaned[match.end() :]+ # The tag was never closed: cut at the first heading the summarizer was+ # told to emit, and accept the cut only when a real body survives it.+ section = _SUMMARY_SECTION_RE.search(body)+ if section is not None and len(body) - section.start() >= _MIN_SALVAGED_CHARS:+ return body[section.start() :].strip()+ # Unstructured output: drop the marker itself rather than the content.+ return _OPEN_THINK_RE.sub("", cleaned).replace("</think>", "").strip() or raw++def count_tokens(text: str) -> int:"""Estimate token count with tiktoken when available."""if not text:@@ -308,7 +354,19 @@ class ContextBuilder:def _model_tokens(self, messages: list[dict[str, Any]], summary: str = "") -> int:if any(model_turn(row) is not None for row in messages):- return _count_model_context_tokens(replay_history(messages, summary))+ replayed = replay_history(messages)+ tokens = _count_model_context_tokens(replayed)+ # The answer loop appends the summary as a trailing snapshot, but+ # only when it differs from the revision already replayed with the+ # history. Count what the request will actually carry.+ pending = summary_snapshot(summary)+ if pending is not None and not any(+ item.get("_context_snapshot") == SUMMARY_SNAPSHOT_NAME+ and item.get("content") == pending["content"]+ for item in replayed+ ):+ tokens += _count_model_context_tokens(pending)+ return tokensreturn count_tokens(build_history_text(self._build_history(summary, messages))) + sum(_provider_response_state_tokens(row) for row in messages)@@ -439,6 +497,9 @@ class ContextBuilder:"unless new information contradicts them; drop only what is obsolete. ""Prefer concrete details (numbers, identifiers, exact terms) over ""abstract restatement. Never invent information. "+ "Output the summary body only: no preamble, no drafting notes, no "+ "step-by-step reasoning, and no <think> blocks — start directly at "+ "the first heading. "f"Write the summary itself in {summary_language}.")if language.startswith("zh"):@@ -452,6 +513,8 @@ class ContextBuilder:"- 待办事项:未回答的问题、未完成的任务、已知阻塞\n""已有摘要中仍然有效的条目应原样保留,仅在新信息与之矛盾时修改,只删除确已过时""的内容。优先保留具体细节(数字、标识符、确切措辞),不要抽象转述,绝不虚构。"+ "只输出摘要正文:不要输出任何思考过程或 <think> 标记,不要写草稿、不要写"+ "「Let me…」「Draft:」之类的推演,直接从第一个小节标题开始。"f"摘要正文本身请使用{summary_language}撰写。")user_prompt = (@@ -488,7 +551,7 @@ class ContextBuilder:**replay_kwargs,):_chunks.append(_c)- summary = "".join(_chunks).strip()+ summary = sanitize_summary("".join(_chunks))if count_tokens(summary) >= int(summary_budget * TRUNCATION_GUARD_RATIO):summary = trim_incomplete_tail(summary)return summary, eventsdiff --git a/deeptutor/services/session/model_history.py b/deeptutor/services/session/model_history.pyindex fe9b4ada2..d36ed5240 100644--- a/deeptutor/services/session/model_history.py+++ b/deeptutor/services/session/model_history.py@@ -154,14 +154,38 @@ def replay_group(return messages+#: Snapshot name the rolling summary rides under when it is replayed.+SUMMARY_SNAPSHOT_NAME = "conversation_summary"+++def summary_snapshot(summary: str) -> dict[str, Any] | None:+ """Render the rolling summary as a replayable trailing context snapshot."""+ cleaned = (summary or "").strip()+ if not cleaned:+ return None+ return {+ "role": "user",+ "content": f"[Conversation summary]\n{cleaned}",+ "_context_snapshot": SUMMARY_SNAPSHOT_NAME,+ }++def replay_history(rows: list[dict[str, Any]],summary: str = "",route: dict[str, str] | None = None,) -> list[dict[str, Any]]:- messages = (- [{"role": "system", "content": f"[Conversation summary]\n{summary}"}] if summary else []- )+ """Replay the retained model turns. ``summary`` is *not* injected here.++ The rolling summary is delivered by the answer loop as a trailing runtime+ snapshot instead (see :data:`SUMMARY_SNAPSHOT_NAME`, emitted from+ ``context.metadata["conversation_summary"]``). Prepending it — or even+ appending it to this list — would still place it ahead of the newest turn's+ own messages, so a later rewrite would invalidate everything behind it.+ Emitted as a snapshot after those bytes, a rewrite is a plain append and+ the previous request stays a cache hit.+ """+ messages: list[dict[str, Any]] = []for group in history_groups(rows):messages.extend(replay_group(group, route))return messagesdiff --git a/tests/services/session/test_image_context_budget.py b/tests/services/session/test_image_context_budget.pyindex f470c4e22..cc65abc3d 100644--- a/tests/services/session/test_image_context_budget.py+++ b/tests/services/session/test_image_context_budget.py@@ -9,7 +9,7 @@ import unittestfrom unittest.mock import AsyncMockfrom deeptutor.services.session.context_builder import ContextBuilder, count_tokens-from deeptutor.services.session.model_history import replay_history+from deeptutor.services.session.model_history import replay_history, summary_snapshotCONFIG = SimpleNamespace(model="test", context_window=272000, max_tokens=4096, binding="openai")IMAGE_ALLOWANCE = 4096@@ -100,8 +100,14 @@ class ImageTokenTests(unittest.TestCase):record["messages"][-1]["thinking_blocks"] = [{"type": "thinking", "thinking": "reasoning", "signature": "sig"}]- expected = count_tokens(json.dumps(replay_history(rows, "summary"), ensure_ascii=False))- self.assertEqual(self.builder._model_tokens(rows, "summary"), expected)+ expected = count_tokens(json.dumps(replay_history(rows), ensure_ascii=False))+ self.assertEqual(self.builder._model_tokens(rows), expected)+ # The summary is delivered as a trailing snapshot instead of a leading+ # row inside the replay, so it is counted on top of the history.+ self.assertEqual(+ self.builder._model_tokens(rows, "summary"),+ expected + count_tokens(json.dumps(summary_snapshot("summary"), ensure_ascii=False)),+ )def test_encoding_in_user_text_is_still_text(self):rows = rows_with_parts([{"type": "text", "text": RANDOM_IMAGE}])@@ -150,7 +156,8 @@ class BuilderTests(unittest.IsolatedAsyncioTestCase):result = await builder.build(session_id="test", llm_config=CONFIG)builder._summarize.assert_awaited_once()store.update_summary.assert_awaited_once_with("test", "compact summary", 2)- self.assertEqual(result.model_history[1:], replay_history(latest))+ # The summary rides separately from the replayed history now.+ self.assertEqual(result.model_history, replay_history(latest))async def test_many_images_still_consume_finite_budget(self):rows = []diff --git a/tests/services/session/test_model_history.py b/tests/services/session/test_model_history.pyindex 8448091f8..afe0a30bb 100644--- a/tests/services/session/test_model_history.py+++ b/tests/services/session/test_model_history.py@@ -142,7 +142,11 @@ async def test_compaction_preserves_whole_tool_turns_and_reuses_the_old_prefix()]assert request["tools"] == recent["tools"]store.update_summary.assert_awaited_once_with("s", "Summary preserving the evidence", 2)- assert result.model_history[1:] == recent["messages"]+ # The summary no longer occupies model_history[0]: the answer loop delivers+ # it as a trailing runtime snapshot, so the replayed history starts at the+ # oldest retained turn while the summary still reaches the model.+ assert result.model_history == recent["messages"]+ assert result.conversation_summary == "Summary preserving the evidence"assert result.conversation_history[-1]["content"] == "Displayed answer"