Your lightweight open-source personal agent.
Python agent-frameworkai-agentai-agentschatbotchatopsdiscord-botllm-agentsllmslocal-llmmcpmodel-context-protocolmulti-agentopenai-compatibleopenclawpersonal-ai-assistantpythonself-hostedtelegram-bot-ai-assistantwebuiworkflow-automation
fix(memory): sanitize raw history before lossless chunking
Xubin Ren122a0f5d6f92
diff --git a/nanobot/agent/memory.py b/nanobot/agent/memory.pyindex a381d13f7..924eb5c09 100644--- a/nanobot/agent/memory.py+++ b/nanobot/agent/memory.py@@ -300,19 +300,21 @@ class MemoryStore:content more tightly; this default only exists to catch unintentionallarge writes (e.g. an LLM echoing its input back as a "summary")."""- ts = datetime.now().strftime("%Y-%m-%d %H:%M")- raw = entry.rstrip()content = self._normalize_history_entry(entry, max_chars=max_chars)+ if entry.rstrip() and not content:+ logger.debug(+ "history entry stripped to empty (likely template leak); "+ "persisting empty content to avoid re-polluting Dream input",+ )+ return self._append_history_record(content, session_key=session_key)++ def _append_history_record(self, content: str, *, session_key: str | None = None) -> int:+ """Persist already sanitized, bounded content without rewriting chunk edges."""+ ts = datetime.now().strftime("%Y-%m-%d %H:%M")# Cursor allocation and the append must be atomic: concurrent writers# could otherwise read the same current cursor and emit duplicates.with self._append_lock:cursor = self._next_cursor()- if raw and not content:- logger.debug(- "history entry {} stripped to empty (likely template leak); "- "persisting empty content to avoid re-polluting Dream input",- cursor,- )record = {"cursor": cursor, "timestamp": ts, "content": content}if session_key:record["session_key"] = session_key@@ -672,14 +674,17 @@ class MemoryStore:"""Persist raw messages in bounded chunks and return a checkpoint."""limit = max_chars if max_chars is not None else _RAW_ARCHIVE_MAX_CHARSchunk_size = min(max(1, limit), _HISTORY_ENTRY_HARD_CAP - 1_000)- formatted = self._format_messages(public_history_messages(messages))+ # Clean the complete text first: a thinking block may span chunk boundaries.+ formatted = strip_think(self._format_messages(public_history_messages(messages)))chunks = [formatted[start:start + chunk_size]for start in range(0, len(formatted), chunk_size)] or [""]for part, chunk in enumerate(chunks, start=1):suffix = f" (part {part}/{len(chunks)})" if len(chunks) > 1 else ""- self.append_history(+ # The whole text is sanitized and each record fits the hard cap.+ # Normalizing again would trim meaningful whitespace at chunk edges.+ self._append_history_record(f"[RAW] {len(messages)} messages{suffix}\n{chunk}",session_key=session_key,)diff --git a/tests/agent/test_consolidator.py b/tests/agent/test_consolidator.pyindex 05a217b85..8ea3fe0e4 100644--- a/tests/agent/test_consolidator.py+++ b/tests/agent/test_consolidator.py@@ -572,6 +572,45 @@ class TestCompactIdleSession:get_tool_definitions=MagicMock(return_value=[]),)+ async def test_partial_raw_write_does_not_advance_checkpoint(+ self, real_consolidator, store, runtime, mock_provider, monkeypatch+ ):+ runtime = replace(runtime, context_window_tokens=1_000)+ sessions = real_consolidator.sessions+ session = sessions.get_or_create("cli:partial-raw")+ session.add_message("user", "x " * 20_000 + "TAIL_MARKER")+ expected = store._format_messages(session.messages)+ sessions.save(session)+ append = store._append_history_record+ writes = 0++ def fail_second(content, *, session_key=None):+ nonlocal writes+ writes += 1+ if writes == 2:+ raise OSError("disk full")+ return append(content, session_key=session_key)++ with monkeypatch.context() as patcher:+ patcher.setattr(store, "_append_history_record", fail_second)+ with pytest.raises(OSError, match="disk full"):+ await real_consolidator.compact_idle_session(session.key, runtime=runtime)++ sessions.invalidate(session.key)+ unchanged = sessions.get_or_create(session.key)+ assert unchanged.last_archived == 0+ assert unchanged.messages == session.messages+ partial_cursor = store.get_latest_cursor()+ assert partial_cursor == 1++ await real_consolidator.compact_idle_session(session.key, runtime=runtime)++ entries = MemoryStore(store.workspace).read_unprocessed_history(partial_cursor)+ assert "".join(entry["content"].split("\n", 1)[1] for entry in entries) == expected+ sessions.invalidate(session.key)+ assert sessions.get_or_create(session.key).last_archived == 1+ mock_provider.chat_stream_with_retry.assert_not_awaited()+@pytest.mark.asyncioasync def test_archives_full_tail_preserves_messages_and_replays_checkpoint(self, real_consolidator, mock_provider, runtime@@ -1515,7 +1554,25 @@ class TestCompactIdleSession:class TestRawArchiveTruncation:- """raw_archive() must cap entry size to avoid bloating history.jsonl."""+ """raw_archive() keeps complete journal content with bounded individual entries."""++ @pytest.mark.parametrize("boundary", ["A ", "\n\n", "<think>PRIVATE</think>"])+ def test_raw_chunks_preserve_sanitized_boundaries(self, store, boundary):+ from nanobot.utils.helpers import strip_think++ message = {"role": "assistant", "content": ""}+ prefix_len = len(store._format_messages([{**message, "content": "x"}])) - 1+ message["content"] = "x" * (16_000 - prefix_len - 2) + boundary + "PUBLIC_TAIL"+ expected = strip_think(store._format_messages([message]))++ store.raw_archive([message])++ # Reload the real journal, rather than observing append call arguments.+ entries = MemoryStore(store.workspace).read_unprocessed_history(since_cursor=0)+ joined = "".join(entry["content"].split("\n", 1)[1] for entry in entries)+ assert joined == expected+ assert "PRIVATE" not in joined+ assert "PUBLIC_TAIL" in joineddef test_raw_archive_truncates_large_content(self, store):"""Large messages should be truncated to _RAW_ARCHIVE_MAX_CHARS."""@@ -1579,6 +1636,19 @@ class TestRawArchiveTruncation:entries = store.read_unprocessed_history(since_cursor=0)assert len(entries[0]["content"]) < 200+ @pytest.mark.parametrize("limit", [100, 100_000])+ def test_raw_chunks_keep_tail_with_custom_limit(self, store, limit):+ messages = [{"role": "user", "content": "x " * 40_000 + "TAIL_MARKER"}]+ expected = store._format_messages(messages)++ checkpoint = store.raw_archive(messages, max_chars=limit, session_key="cli:chunks")++ entries = MemoryStore(store.workspace).read_unprocessed_history(since_cursor=0)+ assert "".join(entry["content"].split("\n", 1)[1] for entry in entries) == expected+ assert all(len(entry["content"]) <= _HISTORY_ENTRY_HARD_CAP for entry in entries)+ assert all(entry["session_key"] == "cli:chunks" for entry in entries)+ assert checkpoint == store._build_raw_checkpoint(messages, max_chars=limit)+class TestArchivePersistence:async def test_archive_returns_the_sanitized_persisted_summary(