Skip to content

Personal AI, On Personal Devices

Python

fix(connectors): rewrite edited documents on re-ingest instead of keeping stale chunks (#1017)

K Karthikd0cef3e630fa
diff --git a/src/openjarvis/connectors/pipeline.py b/src/openjarvis/connectors/pipeline.py
index 9c2d5e77..b1c9c138 100644
--- a/src/openjarvis/connectors/pipeline.py
+++ b/src/openjarvis/connectors/pipeline.py
@@ -94,20 +94,11 @@ class IngestionPipeline:
self._chunker = SemanticChunker(max_tokens=max_tokens)
self._attachment_store = attachment_store
self._embedder = embedder
- self._seen_doc_ids: set[str] = set()
- self._load_existing_doc_ids()
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
- def _load_existing_doc_ids(self) -> None:
- """Populate ``_seen_doc_ids`` from rows already in the store."""
- rows = self._store._conn.execute(
- "SELECT DISTINCT doc_id FROM knowledge_chunks"
- ).fetchall()
- self._seen_doc_ids = {r[0] for r in rows}
-
def _embed_chunk(self, content: str) -> tuple[Optional[bytes], str]:
"""Return ``(embedding_bytes, model_version)`` for a chunk.
@@ -149,8 +140,9 @@ class IngestionPipeline:
def ingest(self, documents: Iterable[Document]) -> int:
"""Ingest an iterable of documents into the knowledge store.
- Duplicate ``doc_id`` values are silently skipped (both across
- calls and within a single batch).
+ Duplicate ``doc_id`` values within a call are skipped. Documents
+ already in the store are compared by chunk hash: unchanged documents
+ are skipped, while edited documents are rewritten.
Parameters
----------
@@ -164,9 +156,13 @@ class IngestionPipeline:
The total number of chunks written to the store in this call.
"""
chunks_stored = 0
+ # Keep dedup local to this call. A long-lived pipeline is reused by
+ # SyncEngine/SyncScheduler across syncs, where a previously seen
+ # document may now contain edits that need hash comparison below.
+ seen_doc_ids: set[str] = set()
for doc in documents:
- if doc.doc_id in self._seen_doc_ids:
+ if doc.doc_id in seen_doc_ids:
continue
# Compute v1 provenance fields once per document.
@@ -203,6 +199,21 @@ class IngestionPipeline:
metadata=parent_meta,
)
+ # Change detection against what is already stored. Unchanged
+ # documents are skipped outright; changed ones are deleted
+ # wholesale (body and attachment chunks) and rewritten so a
+ # document that shrank does not leave orphaned trailing chunks
+ # behind. Compared before the chunks are embedded so an unchanged
+ # vault costs no embedder calls on re-sync. Only body chunks are
+ # compared: attachments never change without their parent.
+ new_hashes = {c.index: _content_hash(c.content) for c in chunks}
+ existing_hashes = self._store.chunk_hashes(doc.doc_id, source_id)
+ if existing_hashes:
+ if existing_hashes == new_hashes:
+ seen_doc_ids.add(doc.doc_id)
+ continue
+ self._store.delete(doc.doc_id)
+
for chunk in chunks:
embedding_bytes, embedding_version = self._embed_chunk(chunk.content)
self._store.store(
@@ -286,7 +297,7 @@ class IngestionPipeline:
)
chunks_stored += 1
- self._seen_doc_ids.add(doc.doc_id)
+ seen_doc_ids.add(doc.doc_id)
return chunks_stored
diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.py
index 3e6300d2..712cb7e4 100644
--- a/src/openjarvis/connectors/store.py
+++ b/src/openjarvis/connectors/store.py
@@ -446,6 +446,22 @@ class KnowledgeStore(MemoryBackend):
)
return results
+ def chunk_hashes(self, doc_id: str, source_id: str) -> Dict[int, str]:
+ """Return ``{chunk_index: content_hash}`` for one document's body chunks.
+
+ Scoped by *source_id* as well as *doc_id* because attachment chunks
+ share the parent ``doc_id`` under a synthetic ``source_id`` and
+ restart ``chunk_index`` at 0. Empty when nothing is stored. The
+ ingestion pipeline compares this against a freshly chunked document
+ to decide whether a re-sync needs to rewrite it.
+ """
+ rows = self._conn.execute(
+ "SELECT chunk_index, content_hash FROM knowledge_chunks "
+ "WHERE doc_id = ? AND source_id = ?",
+ (doc_id, source_id),
+ ).fetchall()
+ return {row[0]: row[1] or "" for row in rows}
+
def delete(self, doc_id: str) -> bool:
"""Delete all chunks with the given *doc_id*. Returns True if any existed."""
cur = self._conn.execute(
diff --git a/tests/connectors/test_pipeline.py b/tests/connectors/test_pipeline.py
index 1f759899..a86617b2 100644
--- a/tests/connectors/test_pipeline.py
+++ b/tests/connectors/test_pipeline.py
@@ -455,3 +455,69 @@ def test_pipeline_skips_embedding_when_no_embedder(
assert len(rows) == 1
assert rows[0][0] is None
assert rows[0][1] == ""
+
+
+# ---------------------------------------------------------------------------
+# Re-sync change detection: edited documents are rewritten, unchanged skipped
+# ---------------------------------------------------------------------------
+
+
+def test_reingest_edited_document_replaces_stale_chunks(
+ store: KnowledgeStore,
+) -> None:
+ """An edited file re-synced under the same doc_id must replace its chunks.
+
+ Regression: previously the pipeline preloaded every stored doc_id and
+ skipped them unconditionally, so a vault note edited after the first
+ ingest kept its stale text forever.
+ """
+ doc_id = "obsidian:notes/setup.md"
+ IngestionPipeline(store).ingest(
+ [_make_doc(doc_id=doc_id, content="Use jarvis connect obsidian --path.")]
+ )
+ assert store.count() == 1
+
+ n = IngestionPipeline(store).ingest(
+ [_make_doc(doc_id=doc_id, content="Use jarvis deep-research-setup instead.")]
+ )
+
+ assert n == 1
+ assert store.count() == 1
+ row = store._conn.execute("SELECT content FROM knowledge_chunks").fetchone()
+ assert "deep-research-setup" in row[0]
+ assert "connect obsidian" not in row[0]
+
+
+def test_reingest_shrunk_document_drops_trailing_chunks(
+ store: KnowledgeStore,
+) -> None:
+ """A document that now produces fewer chunks leaves no orphans behind."""
+ doc_id = "obsidian:notes/long.md"
+ long_body = "\n\n".join(f"Paragraph {i}. " + ("word " * 200) for i in range(6))
+ IngestionPipeline(store, max_tokens=128).ingest(
+ [_make_doc(doc_id=doc_id, content=long_body)]
+ )
+ assert store.count() > 1
+
+ IngestionPipeline(store, max_tokens=128).ingest(
+ [_make_doc(doc_id=doc_id, content="Now it is one short paragraph.")]
+ )
+
+ assert store.count() == 1
+
+
+def test_reingest_unchanged_document_skips_embedding(
+ store: KnowledgeStore,
+) -> None:
+ """Unchanged documents cost no embedder calls on re-sync."""
+ doc = _make_doc(doc_id="doc:unchanged", content="Stable content, never edited.")
+ first = _StubEmbedder()
+ IngestionPipeline(store, embedder=first).ingest([doc]) # type: ignore[arg-type]
+ assert first.calls == 1
+
+ second = _StubEmbedder()
+ n = IngestionPipeline(store, embedder=second).ingest([doc]) # type: ignore[arg-type]
+
+ assert n == 0
+ assert second.calls == 0
+ assert store.count() == 1
diff --git a/tests/connectors/test_sync_engine.py b/tests/connectors/test_sync_engine.py
index eb1f7b54..dd5b3a3c 100644
--- a/tests/connectors/test_sync_engine.py
+++ b/tests/connectors/test_sync_engine.py
@@ -149,6 +149,23 @@ def test_sync_connector(engine: SyncEngine, store: KnowledgeStore) -> None:
assert store.count() == 5
+def test_repeat_sync_rewrites_edited_document_with_same_pipeline(
+ engine: SyncEngine, store: KnowledgeStore
+) -> None:
+ """A long-lived SyncEngine must compare an edited document on each run."""
+ doc = _make_doc("doc:edited", content="Original note content.")
+ connector = StubConnector([doc])
+
+ assert engine.sync(connector) == 1
+ doc.content = "Updated note content."
+ assert engine.sync(connector) == 1
+
+ rows = store._conn.execute(
+ "SELECT content FROM knowledge_chunks WHERE doc_id = ?", (doc.doc_id,)
+ ).fetchall()
+ assert [row[0] for row in rows] == ["Updated note content."]
+
+
# ---------------------------------------------------------------------------
# Test 2: sync_saves_checkpoint — checkpoint items_synced is correct
# ---------------------------------------------------------------------------