Personal AI, On Personal Devices
Python
fix(connectors): embed chunks at ingest so hybrid search has vectors to score (#1018)
K Karthik0c391fdd1af3
diff --git a/src/openjarvis/cli/deep_research_setup_cmd.py b/src/openjarvis/cli/deep_research_setup_cmd.pyindex 00b61075..867d39ab 100644--- a/src/openjarvis/cli/deep_research_setup_cmd.py+++ b/src/openjarvis/cli/deep_research_setup_cmd.py@@ -16,6 +16,7 @@ import clickfrom rich.console import Consolefrom rich.table import Table+from openjarvis.connectors.embeddings import default_embedderfrom openjarvis.connectors.pipeline import IngestionPipelinefrom openjarvis.connectors.store import KnowledgeStorefrom openjarvis.connectors.sync_engine import SyncEngine@@ -260,7 +261,7 @@ def ingest_sources(Returns total chunks indexed across all sources."""- pipeline = IngestionPipeline(store)+ pipeline = IngestionPipeline(store, embedder=default_embedder())engine = SyncEngine(pipeline, state_db=state_db)total = 0for src in sources:@@ -423,6 +424,14 @@ def deep_research_setup(obsidian_vault: Optional[str], skip_chat: bool) -> None:db_path.parent.mkdir(parents=True, exist_ok=True)store = KnowledgeStore(str(db_path))+ embedder = default_embedder()+ if embedder is None:+ console.print(+ "[yellow]No embedding model reachable — indexing lexical-only "+ "(BM25). Run `ollama pull nomic-embed-text` and re-run to enable "+ "hybrid search.[/yellow]"+ )+console.print("\n[bold]Ingesting...[/bold]")for src in all_sources:try:@@ -430,7 +439,7 @@ def deep_research_setup(obsidian_vault: Optional[str], skip_chat: bool) -> None:src["connector_id"],src["config"],)- pipeline = IngestionPipeline(store)+ pipeline = IngestionPipeline(store, embedder=embedder)engine = SyncEngine(pipeline)chunks = engine.sync(connector)console.print(f" {src['display_name']}: [green]{chunks} chunks[/green]")diff --git a/src/openjarvis/connectors/embeddings.py b/src/openjarvis/connectors/embeddings.pyindex 09fb0c27..c4a31270 100644--- a/src/openjarvis/connectors/embeddings.py+++ b/src/openjarvis/connectors/embeddings.py@@ -13,6 +13,7 @@ so ingestion never fails because a sidecar service is down.from __future__ import annotationsimport logging+import osfrom typing import TYPE_CHECKING, List, Optionalimport requests@@ -39,7 +40,9 @@ class OllamaEmbedder:model:Ollama model tag (e.g. ``nomic-embed-text``, ``mxbai-embed-large``).host:- Base URL for the Ollama HTTP API. Defaults to ``http://localhost:11434``.+ Base URL for the Ollama HTTP API. When ``None`` (default) the+ ``OLLAMA_HOST`` environment variable is used, then+ ``http://localhost:11434`` -- the same precedence as ``OllamaEngine``.timeout:Per-request timeout in seconds."""@@ -48,10 +51,12 @@ class OllamaEmbedder:self,*,model: str = DEFAULT_EMBED_MODEL,- host: str = DEFAULT_OLLAMA_HOST,+ host: Optional[str] = None,timeout: float = 30.0,) -> None:self._model = model+ if host is None:+ host = os.environ.get("OLLAMA_HOST") or DEFAULT_OLLAMA_HOSTself._host = host.rstrip("/")self._timeout = timeoutself._dim: Optional[int] = None@@ -139,6 +144,19 @@ class OllamaEmbedder:return [self.embed(t) for t in texts]+def default_embedder() -> Optional[OllamaEmbedder]:+ """Return an ``OllamaEmbedder`` if the daemon and default model are up.++ Ingestion call sites use this so chunks get embedded whenever an embedder+ is actually reachable, and fall back to lexical-only rows (``None``)+ otherwise. Query-side hybrid search does the same probe, so the two ends+ stay in step: if this returns ``None`` at ingest, ``_vector_recall`` finds+ no rows to score either way.+ """+ embedder = OllamaEmbedder()+ return embedder if embedder.is_available() else None++# ---------------------------------------------------------------------------# Deserialisation helper (used by verification + future retrieval code)# ---------------------------------------------------------------------------@@ -162,6 +180,7 @@ def decode_embedding(blob: Optional[bytes], *, dtype=None) -> Optional[np.ndarra__all__ = ["OllamaEmbedder","decode_embedding",+ "default_embedder","DEFAULT_EMBED_MODEL","DEFAULT_OLLAMA_HOST",]diff --git a/src/openjarvis/connectors/pipeline.py b/src/openjarvis/connectors/pipeline.pyindex b1c9c138..8afbb7fa 100644--- a/src/openjarvis/connectors/pipeline.py+++ b/src/openjarvis/connectors/pipeline.py@@ -206,10 +206,23 @@ class IngestionPipeline:# behind. Compared before the chunks are embedded so an unchanged# vault costs no embedder calls on re-sync. Only body chunks are# compared: attachments never change without their parent.+ #+ # An unchanged document is still rewritten when an embedder is+ # configured and its stored rows carry a different+ # ``embedding_model_version`` -- including "" for rows ingested+ # before any embedder was available -- so pulling an embedding+ # model after the first sync backfills vectors instead of leaving+ # hybrid search with nothing to score. With no embedder configured+ # (daemon down, model not pulled) existing vectors are left alone.new_hashes = {c.index: _content_hash(c.content) for c in chunks}existing_hashes = self._store.chunk_hashes(doc.doc_id, source_id)if existing_hashes:- if existing_hashes == new_hashes:+ stale_vectors = (+ self._embedder is not None+ and self._store.embedding_versions(doc.doc_id, source_id)+ != {self._embedder.model_version}+ )+ if existing_hashes == new_hashes and not stale_vectors:seen_doc_ids.add(doc.doc_id)continueself._store.delete(doc.doc_id)diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.pyindex 712cb7e4..bf68a9d5 100644--- a/src/openjarvis/connectors/store.py+++ b/src/openjarvis/connectors/store.py@@ -462,6 +462,21 @@ class KnowledgeStore(MemoryBackend):).fetchall()return {row[0]: row[1] or "" for row in rows}+ def embedding_versions(self, doc_id: str, source_id: str) -> set[str]:+ """Return the distinct ``embedding_model_version`` values on a document's body.++ Rows that were ingested without an embedder carry ``""``. The+ ingestion pipeline uses this to notice that an embedder has since+ become available (or changed model) and re-embed an otherwise+ unchanged document.+ """+ rows = self._conn.execute(+ "SELECT DISTINCT embedding_model_version FROM knowledge_chunks "+ "WHERE doc_id = ? AND source_id = ?",+ (doc_id, source_id),+ ).fetchall()+ return {row[0] or "" for row in rows}+def delete(self, doc_id: str) -> bool:"""Delete all chunks with the given *doc_id*. Returns True if any existed."""cur = self._conn.execute(diff --git a/src/openjarvis/server/connectors_router.py b/src/openjarvis/server/connectors_router.pyindex 0418d751..435d0a4e 100644--- a/src/openjarvis/server/connectors_router.py+++ b/src/openjarvis/server/connectors_router.py@@ -71,6 +71,9 @@ def _ensure_connectors_registered() -> None:and not mod_name.endswith("retriever")and not mod_name.endswith("sync_engine")and not mod_name.endswith("oauth")+ # Registers nothing; reloading it would only swap out the+ # OllamaEmbedder class under anyone holding (or patching) it.+ and not mod_name.endswith("embeddings")):try:importlib.reload(sys.modules[mod_name])@@ -358,13 +361,16 @@ def create_connectors_router():def _run_sync() -> None:try:+ from openjarvis.connectors.embeddings import default_embedderfrom openjarvis.connectors.pipeline import IngestionPipelinefrom openjarvis.connectors.store import KnowledgeStorefrom openjarvis.connectors.sync_engine import SyncEnginewith KnowledgeStore() as store:with SyncEngine(- pipeline=IngestionPipeline(store=store),+ pipeline=IngestionPipeline(+ store=store, embedder=default_embedder()+ ),) as engine:engine.sync(instance, cancel_event=cancel_event)final_state = "cancelled" if cancel_event.is_set() else "complete"diff --git a/tests/connectors/test_embeddings.py b/tests/connectors/test_embeddings.pynew file mode 100644index 00000000..3967fa9d--- /dev/null+++ b/tests/connectors/test_embeddings.py@@ -0,0 +1,35 @@+"""Tests for openjarvis.connectors.embeddings.default_embedder."""++from __future__ import annotations++import pytest++from openjarvis.connectors import embeddings+++def test_default_embedder_returns_instance_when_available(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(embeddings.OllamaEmbedder, "is_available", lambda self: True)+ emb = embeddings.default_embedder()+ assert isinstance(emb, embeddings.OllamaEmbedder)+ assert emb.model_version == f"ollama:{embeddings.DEFAULT_EMBED_MODEL}"+++def test_default_embedder_returns_none_when_unreachable(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(embeddings.OllamaEmbedder, "is_available", lambda self: False)+ assert embeddings.default_embedder() is None+++def test_embedder_host_precedence(monkeypatch: pytest.MonkeyPatch) -> None:+ """Explicit host > OLLAMA_HOST env > built-in default, like OllamaEngine."""+ monkeypatch.delenv("OLLAMA_HOST", raising=False)+ assert embeddings.OllamaEmbedder()._host == embeddings.DEFAULT_OLLAMA_HOST++ monkeypatch.setenv("OLLAMA_HOST", "http://127.0.0.1:11434/")+ assert embeddings.OllamaEmbedder()._host == "http://127.0.0.1:11434"++ explicit = embeddings.OllamaEmbedder(host="http://gpu-box:11434")+ assert explicit._host == "http://gpu-box:11434"diff --git a/tests/connectors/test_pipeline.py b/tests/connectors/test_pipeline.pyindex a86617b2..7dda4b79 100644--- a/tests/connectors/test_pipeline.py+++ b/tests/connectors/test_pipeline.py@@ -521,3 +521,44 @@ def test_reingest_unchanged_document_skips_embedding(assert n == 0assert second.calls == 0assert store.count() == 1+++def test_reingest_backfills_embeddings_when_embedder_appears(+ store: KnowledgeStore,+) -> None:+ """Rows ingested without an embedder are re-embedded once one is available."""+ doc = _make_doc(doc_id="doc:backfill", content="Indexed before nomic was pulled.")+ IngestionPipeline(store).ingest([doc])+ row = store._conn.execute(+ "SELECT embedding, embedding_model_version FROM knowledge_chunks"+ ).fetchone()+ assert row[0] is None and row[1] == ""++ embedder = _StubEmbedder()+ n = IngestionPipeline(store, embedder=embedder).ingest([doc]) # type: ignore[arg-type]++ assert n == 1+ assert embedder.calls == 1+ assert store.count() == 1+ row = store._conn.execute(+ "SELECT embedding, embedding_model_version FROM knowledge_chunks"+ ).fetchone()+ assert row[0] is not None+ assert row[1] == "stub:test-embedder"+++def test_reingest_without_embedder_keeps_existing_vectors(+ store: KnowledgeStore,+) -> None:+ """A sync while the embedder is down must not strip vectors already stored."""+ doc = _make_doc(doc_id="doc:keep-vec", content="Embedded on the first pass.")+ IngestionPipeline(store, embedder=_StubEmbedder()).ingest([doc]) # type: ignore[arg-type]++ n = IngestionPipeline(store).ingest([doc])++ assert n == 0+ row = store._conn.execute(+ "SELECT embedding, embedding_model_version FROM knowledge_chunks"+ ).fetchone()+ assert row[0] is not None+ assert row[1] == "stub:test-embedder"diff --git a/tests/server/test_connectors_router.py b/tests/server/test_connectors_router.pyindex 95b4c801..644c42ea 100644--- a/tests/server/test_connectors_router.py+++ b/tests/server/test_connectors_router.py@@ -12,6 +12,24 @@ from pathlib import Pathimport pytest[email protected](scope="module", autouse=True)+def _no_embedder():+ """Keep sync threads off the network for the whole module.++ Sync runs probe for an embedder before ingesting. Several tests here+ kick off a background sync and return immediately, so a function-scoped+ patch would be undone while that thread is still running and it would+ fall through to a real daemon lookup -- slow enough on some hosts to+ leave the connector "syncing" into the next test.+ """+ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(+ "openjarvis.connectors.embeddings.OllamaEmbedder.is_available",+ lambda self: False,+ )+ yield++@pytest.fixturedef app():try: