Personal AI, On Personal Devices
Python
fix(connectors): rewrite edited documents on re-ingest instead of keeping stale chunks (#1017)
K Karthikd0cef3e630fa
diff --git a/src/openjarvis/connectors/pipeline.py b/src/openjarvis/connectors/pipeline.pyindex 9c2d5e77..b1c9c138 100644--- a/src/openjarvis/connectors/pipeline.py+++ b/src/openjarvis/connectors/pipeline.py@@ -94,20 +94,11 @@ class IngestionPipeline:self._chunker = SemanticChunker(max_tokens=max_tokens)self._attachment_store = attachment_storeself._embedder = embedder- self._seen_doc_ids: set[str] = set()- self._load_existing_doc_ids()# ------------------------------------------------------------------# Internal helpers# ------------------------------------------------------------------- def _load_existing_doc_ids(self) -> None:- """Populate ``_seen_doc_ids`` from rows already in the store."""- rows = self._store._conn.execute(- "SELECT DISTINCT doc_id FROM knowledge_chunks"- ).fetchall()- self._seen_doc_ids = {r[0] for r in rows}-def _embed_chunk(self, content: str) -> tuple[Optional[bytes], str]:"""Return ``(embedding_bytes, model_version)`` for a chunk.@@ -149,8 +140,9 @@ class IngestionPipeline:def ingest(self, documents: Iterable[Document]) -> int:"""Ingest an iterable of documents into the knowledge store.- Duplicate ``doc_id`` values are silently skipped (both across- calls and within a single batch).+ Duplicate ``doc_id`` values within a call are skipped. Documents+ already in the store are compared by chunk hash: unchanged documents+ are skipped, while edited documents are rewritten.Parameters----------@@ -164,9 +156,13 @@ class IngestionPipeline:The total number of chunks written to the store in this call."""chunks_stored = 0+ # Keep dedup local to this call. A long-lived pipeline is reused by+ # SyncEngine/SyncScheduler across syncs, where a previously seen+ # document may now contain edits that need hash comparison below.+ seen_doc_ids: set[str] = set()for doc in documents:- if doc.doc_id in self._seen_doc_ids:+ if doc.doc_id in seen_doc_ids:continue# Compute v1 provenance fields once per document.@@ -203,6 +199,21 @@ class IngestionPipeline:metadata=parent_meta,)+ # Change detection against what is already stored. Unchanged+ # documents are skipped outright; changed ones are deleted+ # wholesale (body and attachment chunks) and rewritten so a+ # document that shrank does not leave orphaned trailing chunks+ # behind. Compared before the chunks are embedded so an unchanged+ # vault costs no embedder calls on re-sync. Only body chunks are+ # compared: attachments never change without their parent.+ new_hashes = {c.index: _content_hash(c.content) for c in chunks}+ existing_hashes = self._store.chunk_hashes(doc.doc_id, source_id)+ if existing_hashes:+ if existing_hashes == new_hashes:+ seen_doc_ids.add(doc.doc_id)+ continue+ self._store.delete(doc.doc_id)+for chunk in chunks:embedding_bytes, embedding_version = self._embed_chunk(chunk.content)self._store.store(@@ -286,7 +297,7 @@ class IngestionPipeline:)chunks_stored += 1- self._seen_doc_ids.add(doc.doc_id)+ seen_doc_ids.add(doc.doc_id)return chunks_storeddiff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.pyindex 3e6300d2..712cb7e4 100644--- a/src/openjarvis/connectors/store.py+++ b/src/openjarvis/connectors/store.py@@ -446,6 +446,22 @@ class KnowledgeStore(MemoryBackend):)return results+ def chunk_hashes(self, doc_id: str, source_id: str) -> Dict[int, str]:+ """Return ``{chunk_index: content_hash}`` for one document's body chunks.++ Scoped by *source_id* as well as *doc_id* because attachment chunks+ share the parent ``doc_id`` under a synthetic ``source_id`` and+ restart ``chunk_index`` at 0. Empty when nothing is stored. The+ ingestion pipeline compares this against a freshly chunked document+ to decide whether a re-sync needs to rewrite it.+ """+ rows = self._conn.execute(+ "SELECT chunk_index, content_hash FROM knowledge_chunks "+ "WHERE doc_id = ? AND source_id = ?",+ (doc_id, source_id),+ ).fetchall()+ return {row[0]: row[1] or "" for row in rows}+def delete(self, doc_id: str) -> bool:"""Delete all chunks with the given *doc_id*. Returns True if any existed."""cur = self._conn.execute(diff --git a/tests/connectors/test_pipeline.py b/tests/connectors/test_pipeline.pyindex 1f759899..a86617b2 100644--- a/tests/connectors/test_pipeline.py+++ b/tests/connectors/test_pipeline.py@@ -455,3 +455,69 @@ def test_pipeline_skips_embedding_when_no_embedder(assert len(rows) == 1assert rows[0][0] is Noneassert rows[0][1] == ""+++# ---------------------------------------------------------------------------+# Re-sync change detection: edited documents are rewritten, unchanged skipped+# ---------------------------------------------------------------------------+++def test_reingest_edited_document_replaces_stale_chunks(+ store: KnowledgeStore,+) -> None:+ """An edited file re-synced under the same doc_id must replace its chunks.++ Regression: previously the pipeline preloaded every stored doc_id and+ skipped them unconditionally, so a vault note edited after the first+ ingest kept its stale text forever.+ """+ doc_id = "obsidian:notes/setup.md"+ IngestionPipeline(store).ingest(+ [_make_doc(doc_id=doc_id, content="Use jarvis connect obsidian --path.")]+ )+ assert store.count() == 1++ n = IngestionPipeline(store).ingest(+ [_make_doc(doc_id=doc_id, content="Use jarvis deep-research-setup instead.")]+ )++ assert n == 1+ assert store.count() == 1+ row = store._conn.execute("SELECT content FROM knowledge_chunks").fetchone()+ assert "deep-research-setup" in row[0]+ assert "connect obsidian" not in row[0]+++def test_reingest_shrunk_document_drops_trailing_chunks(+ store: KnowledgeStore,+) -> None:+ """A document that now produces fewer chunks leaves no orphans behind."""+ doc_id = "obsidian:notes/long.md"+ long_body = "\n\n".join(f"Paragraph {i}. " + ("word " * 200) for i in range(6))+ IngestionPipeline(store, max_tokens=128).ingest(+ [_make_doc(doc_id=doc_id, content=long_body)]+ )+ assert store.count() > 1++ IngestionPipeline(store, max_tokens=128).ingest(+ [_make_doc(doc_id=doc_id, content="Now it is one short paragraph.")]+ )++ assert store.count() == 1+++def test_reingest_unchanged_document_skips_embedding(+ store: KnowledgeStore,+) -> None:+ """Unchanged documents cost no embedder calls on re-sync."""+ doc = _make_doc(doc_id="doc:unchanged", content="Stable content, never edited.")+ first = _StubEmbedder()+ IngestionPipeline(store, embedder=first).ingest([doc]) # type: ignore[arg-type]+ assert first.calls == 1++ second = _StubEmbedder()+ n = IngestionPipeline(store, embedder=second).ingest([doc]) # type: ignore[arg-type]++ assert n == 0+ assert second.calls == 0+ assert store.count() == 1diff --git a/tests/connectors/test_sync_engine.py b/tests/connectors/test_sync_engine.pyindex eb1f7b54..dd5b3a3c 100644--- a/tests/connectors/test_sync_engine.py+++ b/tests/connectors/test_sync_engine.py@@ -149,6 +149,23 @@ def test_sync_connector(engine: SyncEngine, store: KnowledgeStore) -> None:assert store.count() == 5+def test_repeat_sync_rewrites_edited_document_with_same_pipeline(+ engine: SyncEngine, store: KnowledgeStore+) -> None:+ """A long-lived SyncEngine must compare an edited document on each run."""+ doc = _make_doc("doc:edited", content="Original note content.")+ connector = StubConnector([doc])++ assert engine.sync(connector) == 1+ doc.content = "Updated note content."+ assert engine.sync(connector) == 1++ rows = store._conn.execute(+ "SELECT content FROM knowledge_chunks WHERE doc_id = ?", (doc.doc_id,)+ ).fetchall()+ assert [row[0] for row in rows] == ["Updated note content."]++# ---------------------------------------------------------------------------# Test 2: sync_saves_checkpoint — checkpoint items_synced is correct# ---------------------------------------------------------------------------