Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(ingest): per-file cursor for forked codex rollouts, and no commit on server pending tokens (#5048)

zhangxine56f086e9006
diff --git a/openviking/ingest/replay.py b/openviking/ingest/replay.py
index daf1c8c5f..03a6447e3 100644
--- a/openviking/ingest/replay.py
+++ b/openviking/ingest/replay.py
@@ -181,13 +181,22 @@ class SessionReplayer:
async def commit_if_needed(
self, harness: str, ref: SessionRef, keep_recent_count: int = 0
) -> bool:
- """Commit when the session has un-committed appends or live pending tokens."""
+ """Commit when this session has appended-but-uncommitted messages.
+
+ Deliberately does **not** fall back to the server's ``pending_tokens``: the
+ periodic sweep calls this for every session on every tick (and ``backfill``
+ calls it once per session even when nothing was appended), while the server
+ keeps reporting pending tokens until the queued commit is actually processed.
+ That fallback therefore re-enqueues the same session on every tick whenever the
+ commit queue is behind, which grows the queue without bound. ``needs_commit``
+ is persisted by ``confirm_append``, so crash recovery (appended but not yet
+ committed) is still covered.
+ """
sid = self.ov_session_id(harness, ref.native_session_id)
rec = self.store.get(harness, ref.native_session_id)
- need = bool(rec and rec.needs_commit)
- pending = await self.client.pending_tokens(sid)
- if not need and pending <= 0:
+ if not (rec and rec.needs_commit):
return False
+ pending = await self.client.pending_tokens(sid)
if pending <= 0:
# Nothing live to archive (already committed elsewhere); clear the stale flag.
self.store.mark_committed(harness, ref.native_session_id)
diff --git a/openviking/ingest/sources/codex.py b/openviking/ingest/sources/codex.py
index db76cb937..472b1a634 100644
--- a/openviking/ingest/sources/codex.py
+++ b/openviking/ingest/sources/codex.py
@@ -13,13 +13,36 @@ Records: ``{timestamp, type, payload}``. Conversation turns are
from __future__ import annotations
import json
+import re
from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
from openviking.ingest.models import NormalizedMessage, SessionRef
from openviking.ingest.registry import register_source
from openviking.ingest.sources.base import JsonlLogSource
+_UUID_RE = re.compile(
+ r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+)
+
+
+def _session_id_for_file(meta_id: Optional[str], path: Path) -> str:
+ """Session id for a single rollout file.
+
+ Codex keeps ``session_meta.id`` unchanged when a session is forked/continued, but
+ writes the continuation to a *new* file whose name carries an extra UUID
+ (``rollout-<ts>-<base-uuid>_<fork-uuid>.jsonl``). Using the shared id as the cursor
+ key makes two files fight over one byte offset, and because the cursor also stores
+ the inode, every poll then looks like log rotation and re-reads the file from the
+ top. Append the extra UUID so each file owns its own cursor.
+ """
+ base = meta_id or path.stem
+ uuids = _UUID_RE.findall(path.stem)
+ extras = [u for u in uuids if u != base]
+ if len(uuids) >= 2 and extras:
+ return f"{base}_{extras[-1]}"
+ return base
+
def _join_content(content: Any) -> str:
if not isinstance(content, list):
@@ -44,7 +67,7 @@ class CodexSource(JsonlLogSource):
meta = self._peek_session_meta(path)
return SessionRef(
harness=self.name,
- native_session_id=meta.get("id") or path.stem,
+ native_session_id=_session_id_for_file(meta.get("id"), path),
locator=str(path),
started_at=meta.get("timestamp"),
meta={"model": meta.get("model_provider"), "cwd": meta.get("cwd")},
diff --git a/tests/ingest/test_parsers.py b/tests/ingest/test_parsers.py
index aa9ba6803..acf984f99 100644
--- a/tests/ingest/test_parsers.py
+++ b/tests/ingest/test_parsers.py
@@ -118,6 +118,60 @@ def test_codex(tmp_path):
assert msgs[1].peer_id == "codex__openai"
+def test_codex_forked_rollout_file_gets_its_own_session_id(tmp_path):
+ """A forked/continued session writes a second file that reuses ``session_meta.id``.
+
+ Sharing one native_session_id makes the two files share one byte-offset cursor, and
+ since the cursor also stores the inode, every poll looks like log rotation and
+ re-reads the file from the top. Each file must own its cursor instead.
+ """
+ root = tmp_path / "sessions"
+ day = root / "2026" / "09" / "02"
+ base = "01a0600e-e891-7b42-8da9-b73b83d46acd"
+ fork = "01a06126-7ca2-7eb1-b795-40499a2b3ce6"
+
+ def _records(text):
+ return [
+ {
+ "type": "session_meta",
+ "timestamp": "2026-09-02T00:00:00Z",
+ "payload": {"id": base, "model_provider": "openai", "cwd": str(tmp_path)},
+ },
+ {
+ "type": "response_item",
+ "timestamp": "2026-09-02T00:00:01Z",
+ "payload": {
+ "type": "message",
+ "role": "user",
+ "content": [{"type": "input_text", "text": text}],
+ },
+ },
+ ]
+
+ parent = day / f"rollout-2026-09-02T10-59-44-{base}.jsonl"
+ child = day / f"rollout-2026-09-02T16-05-07-{base}_{fork}.jsonl"
+ _write_jsonl(parent, _records("parent turn"))
+ _write_jsonl(child, _records("child turn"))
+
+ src = CodexSource(_cfg(root), fallback_user="tester")
+ refs = {r.native_session_id: r for r in src.discover_sessions()}
+ # Two files -> two cursors. Before the fix both collapsed onto ``base``.
+ assert set(refs) == {base, f"{base}_{fork}"}
+
+ # Simulate two ingest sweeps, keying the cursor by native_session_id like the store does.
+ first = {}
+ for sid, ref in refs.items():
+ msgs, cursor = src.read_messages(ref, None)
+ first[sid] = cursor
+ assert len(msgs) == 1
+ assert {src.read_messages(refs[sid], None)[0][0].text for sid in refs} == {
+ "parent turn",
+ "child turn",
+ }
+
+ for sid, ref in refs.items():
+ msgs, _ = src.read_messages(ref, first[sid])
+ assert msgs == [], f"{sid} re-read its rollout file on the second sweep"
def _workbuddy_turn(role, text, ts_ms, model=None):
record = {
"id": f"m-{ts_ms}",
diff --git a/tests/ingest/test_replay.py b/tests/ingest/test_replay.py
index 8c9fcd50f..7d787ee5e 100644
--- a/tests/ingest/test_replay.py
+++ b/tests/ingest/test_replay.py
@@ -195,3 +195,46 @@ async def test_commit_if_needed_commits_when_needs_commit_and_pending(tmp_path):
assert await replayer.commit_if_needed("claude_code", _ref()) is True
assert replayer.client.committed == ["import__claude_code__s1"]
assert store.get("claude_code", "s1").needs_commit is False
+
+
+async def test_commit_if_needed_ignores_server_pending_without_local_appends(tmp_path):
+ """Regression: server-side pending tokens alone must not trigger a commit.
+
+ The watch sweep calls ``commit_if_needed`` for every session on every tick, and the
+ server keeps reporting pending tokens until the queued commit is processed. Treating
+ that as "needs commit" re-enqueues the same session on every tick and the queue can
+ never catch up.
+ """
+ store = CursorStore(tmp_path)
+ ref = _ref("s1")
+ # Known session with live pending tokens server-side, but nothing appended locally.
+ store.ensure_row(
+ "claude_code", "s1", "import__claude_code__s1", Cursor(BYTE_OFFSET, {"offset": 10})
+ )
+ fake = _FakeReplay(pending=9000)
+ replayer = SessionReplayer(fake, store)
+
+ for _ in range(3): # three consecutive sweeps
+ assert await replayer.commit_if_needed("claude_code", ref) is False
+ assert fake.committed == []
+
+
+async def test_commit_if_needed_still_recovers_after_crash(tmp_path):
+ """``needs_commit`` persists, so appended-but-uncommitted work is still flushed."""
+ store = CursorStore(tmp_path)
+ ref = _ref("s1")
+ store.set_pending(
+ "claude_code",
+ "s1",
+ "import__claude_code__s1",
+ Cursor(BYTE_OFFSET, {"offset": 0}),
+ Cursor(BYTE_OFFSET, {"offset": 10}),
+ 1,
+ 0,
+ )
+ store.confirm_append("claude_code", "s1", Cursor(BYTE_OFFSET, {"offset": 10}), 1)
+ fake = _FakeReplay(pending=500)
+ # Simulates a restart: a fresh replayer with no in-memory state still commits.
+ replayer = SessionReplayer(fake, store)
+ assert await replayer.commit_if_needed("claude_code", ref) is True
+ assert fake.committed == ["import__claude_code__s1"]