Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(ingest): per-file cursor for forked codex rollouts, and no commit on server pending tokens (#5048)
zhangxine56f086e9006
diff --git a/openviking/ingest/replay.py b/openviking/ingest/replay.pyindex daf1c8c5f..03a6447e3 100644--- a/openviking/ingest/replay.py+++ b/openviking/ingest/replay.py@@ -181,13 +181,22 @@ class SessionReplayer:async def commit_if_needed(self, harness: str, ref: SessionRef, keep_recent_count: int = 0) -> bool:- """Commit when the session has un-committed appends or live pending tokens."""+ """Commit when this session has appended-but-uncommitted messages.++ Deliberately does **not** fall back to the server's ``pending_tokens``: the+ periodic sweep calls this for every session on every tick (and ``backfill``+ calls it once per session even when nothing was appended), while the server+ keeps reporting pending tokens until the queued commit is actually processed.+ That fallback therefore re-enqueues the same session on every tick whenever the+ commit queue is behind, which grows the queue without bound. ``needs_commit``+ is persisted by ``confirm_append``, so crash recovery (appended but not yet+ committed) is still covered.+ """sid = self.ov_session_id(harness, ref.native_session_id)rec = self.store.get(harness, ref.native_session_id)- need = bool(rec and rec.needs_commit)- pending = await self.client.pending_tokens(sid)- if not need and pending <= 0:+ if not (rec and rec.needs_commit):return False+ pending = await self.client.pending_tokens(sid)if pending <= 0:# Nothing live to archive (already committed elsewhere); clear the stale flag.self.store.mark_committed(harness, ref.native_session_id)diff --git a/openviking/ingest/sources/codex.py b/openviking/ingest/sources/codex.pyindex db76cb937..472b1a634 100644--- a/openviking/ingest/sources/codex.py+++ b/openviking/ingest/sources/codex.py@@ -13,13 +13,36 @@ Records: ``{timestamp, type, payload}``. Conversation turns arefrom __future__ import annotationsimport json+import refrom pathlib import Path-from typing import Any, Dict, List+from typing import Any, Dict, List, Optionalfrom openviking.ingest.models import NormalizedMessage, SessionReffrom openviking.ingest.registry import register_sourcefrom openviking.ingest.sources.base import JsonlLogSource+_UUID_RE = re.compile(+ r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"+)+++def _session_id_for_file(meta_id: Optional[str], path: Path) -> str:+ """Session id for a single rollout file.++ Codex keeps ``session_meta.id`` unchanged when a session is forked/continued, but+ writes the continuation to a *new* file whose name carries an extra UUID+ (``rollout-<ts>-<base-uuid>_<fork-uuid>.jsonl``). Using the shared id as the cursor+ key makes two files fight over one byte offset, and because the cursor also stores+ the inode, every poll then looks like log rotation and re-reads the file from the+ top. Append the extra UUID so each file owns its own cursor.+ """+ base = meta_id or path.stem+ uuids = _UUID_RE.findall(path.stem)+ extras = [u for u in uuids if u != base]+ if len(uuids) >= 2 and extras:+ return f"{base}_{extras[-1]}"+ return base+def _join_content(content: Any) -> str:if not isinstance(content, list):@@ -44,7 +67,7 @@ class CodexSource(JsonlLogSource):meta = self._peek_session_meta(path)return SessionRef(harness=self.name,- native_session_id=meta.get("id") or path.stem,+ native_session_id=_session_id_for_file(meta.get("id"), path),locator=str(path),started_at=meta.get("timestamp"),meta={"model": meta.get("model_provider"), "cwd": meta.get("cwd")},diff --git a/tests/ingest/test_parsers.py b/tests/ingest/test_parsers.pyindex aa9ba6803..acf984f99 100644--- a/tests/ingest/test_parsers.py+++ b/tests/ingest/test_parsers.py@@ -118,6 +118,60 @@ def test_codex(tmp_path):assert msgs[1].peer_id == "codex__openai"+def test_codex_forked_rollout_file_gets_its_own_session_id(tmp_path):+ """A forked/continued session writes a second file that reuses ``session_meta.id``.++ Sharing one native_session_id makes the two files share one byte-offset cursor, and+ since the cursor also stores the inode, every poll looks like log rotation and+ re-reads the file from the top. Each file must own its cursor instead.+ """+ root = tmp_path / "sessions"+ day = root / "2026" / "09" / "02"+ base = "01a0600e-e891-7b42-8da9-b73b83d46acd"+ fork = "01a06126-7ca2-7eb1-b795-40499a2b3ce6"++ def _records(text):+ return [+ {+ "type": "session_meta",+ "timestamp": "2026-09-02T00:00:00Z",+ "payload": {"id": base, "model_provider": "openai", "cwd": str(tmp_path)},+ },+ {+ "type": "response_item",+ "timestamp": "2026-09-02T00:00:01Z",+ "payload": {+ "type": "message",+ "role": "user",+ "content": [{"type": "input_text", "text": text}],+ },+ },+ ]++ parent = day / f"rollout-2026-09-02T10-59-44-{base}.jsonl"+ child = day / f"rollout-2026-09-02T16-05-07-{base}_{fork}.jsonl"+ _write_jsonl(parent, _records("parent turn"))+ _write_jsonl(child, _records("child turn"))++ src = CodexSource(_cfg(root), fallback_user="tester")+ refs = {r.native_session_id: r for r in src.discover_sessions()}+ # Two files -> two cursors. Before the fix both collapsed onto ``base``.+ assert set(refs) == {base, f"{base}_{fork}"}++ # Simulate two ingest sweeps, keying the cursor by native_session_id like the store does.+ first = {}+ for sid, ref in refs.items():+ msgs, cursor = src.read_messages(ref, None)+ first[sid] = cursor+ assert len(msgs) == 1+ assert {src.read_messages(refs[sid], None)[0][0].text for sid in refs} == {+ "parent turn",+ "child turn",+ }++ for sid, ref in refs.items():+ msgs, _ = src.read_messages(ref, first[sid])+ assert msgs == [], f"{sid} re-read its rollout file on the second sweep"def _workbuddy_turn(role, text, ts_ms, model=None):record = {"id": f"m-{ts_ms}",diff --git a/tests/ingest/test_replay.py b/tests/ingest/test_replay.pyindex 8c9fcd50f..7d787ee5e 100644--- a/tests/ingest/test_replay.py+++ b/tests/ingest/test_replay.py@@ -195,3 +195,46 @@ async def test_commit_if_needed_commits_when_needs_commit_and_pending(tmp_path):assert await replayer.commit_if_needed("claude_code", _ref()) is Trueassert replayer.client.committed == ["import__claude_code__s1"]assert store.get("claude_code", "s1").needs_commit is False+++async def test_commit_if_needed_ignores_server_pending_without_local_appends(tmp_path):+ """Regression: server-side pending tokens alone must not trigger a commit.++ The watch sweep calls ``commit_if_needed`` for every session on every tick, and the+ server keeps reporting pending tokens until the queued commit is processed. Treating+ that as "needs commit" re-enqueues the same session on every tick and the queue can+ never catch up.+ """+ store = CursorStore(tmp_path)+ ref = _ref("s1")+ # Known session with live pending tokens server-side, but nothing appended locally.+ store.ensure_row(+ "claude_code", "s1", "import__claude_code__s1", Cursor(BYTE_OFFSET, {"offset": 10})+ )+ fake = _FakeReplay(pending=9000)+ replayer = SessionReplayer(fake, store)++ for _ in range(3): # three consecutive sweeps+ assert await replayer.commit_if_needed("claude_code", ref) is False+ assert fake.committed == []+++async def test_commit_if_needed_still_recovers_after_crash(tmp_path):+ """``needs_commit`` persists, so appended-but-uncommitted work is still flushed."""+ store = CursorStore(tmp_path)+ ref = _ref("s1")+ store.set_pending(+ "claude_code",+ "s1",+ "import__claude_code__s1",+ Cursor(BYTE_OFFSET, {"offset": 0}),+ Cursor(BYTE_OFFSET, {"offset": 10}),+ 1,+ 0,+ )+ store.confirm_append("claude_code", "s1", Cursor(BYTE_OFFSET, {"offset": 10}), 1)+ fake = _FakeReplay(pending=500)+ # Simulates a restart: a fresh replayer with no in-memory state still commits.+ replayer = SessionReplayer(fake, store)+ assert await replayer.commit_if_needed("claude_code", ref) is True+ assert fake.committed == ["import__claude_code__s1"]