Skip to content

Personal AI, On Personal Devices

Python

fix(connectors): embed chunks at ingest so hybrid search has vectors to score (#1018)

K Karthik0c391fdd1af3
diff --git a/src/openjarvis/cli/deep_research_setup_cmd.py b/src/openjarvis/cli/deep_research_setup_cmd.py
index 00b61075..867d39ab 100644
--- a/src/openjarvis/cli/deep_research_setup_cmd.py
+++ b/src/openjarvis/cli/deep_research_setup_cmd.py
@@ -16,6 +16,7 @@ import click
from rich.console import Console
from rich.table import Table
+from openjarvis.connectors.embeddings import default_embedder
from openjarvis.connectors.pipeline import IngestionPipeline
from openjarvis.connectors.store import KnowledgeStore
from openjarvis.connectors.sync_engine import SyncEngine
@@ -260,7 +261,7 @@ def ingest_sources(
Returns total chunks indexed across all sources.
"""
- pipeline = IngestionPipeline(store)
+ pipeline = IngestionPipeline(store, embedder=default_embedder())
engine = SyncEngine(pipeline, state_db=state_db)
total = 0
for src in sources:
@@ -423,6 +424,14 @@ def deep_research_setup(obsidian_vault: Optional[str], skip_chat: bool) -> None:
db_path.parent.mkdir(parents=True, exist_ok=True)
store = KnowledgeStore(str(db_path))
+ embedder = default_embedder()
+ if embedder is None:
+ console.print(
+ "[yellow]No embedding model reachable — indexing lexical-only "
+ "(BM25). Run `ollama pull nomic-embed-text` and re-run to enable "
+ "hybrid search.[/yellow]"
+ )
+
console.print("\n[bold]Ingesting...[/bold]")
for src in all_sources:
try:
@@ -430,7 +439,7 @@ def deep_research_setup(obsidian_vault: Optional[str], skip_chat: bool) -> None:
src["connector_id"],
src["config"],
)
- pipeline = IngestionPipeline(store)
+ pipeline = IngestionPipeline(store, embedder=embedder)
engine = SyncEngine(pipeline)
chunks = engine.sync(connector)
console.print(f" {src['display_name']}: [green]{chunks} chunks[/green]")
diff --git a/src/openjarvis/connectors/embeddings.py b/src/openjarvis/connectors/embeddings.py
index 09fb0c27..c4a31270 100644
--- a/src/openjarvis/connectors/embeddings.py
+++ b/src/openjarvis/connectors/embeddings.py
@@ -13,6 +13,7 @@ so ingestion never fails because a sidecar service is down.
from __future__ import annotations
import logging
+import os
from typing import TYPE_CHECKING, List, Optional
import requests
@@ -39,7 +40,9 @@ class OllamaEmbedder:
model:
Ollama model tag (e.g. ``nomic-embed-text``, ``mxbai-embed-large``).
host:
- Base URL for the Ollama HTTP API. Defaults to ``http://localhost:11434``.
+ Base URL for the Ollama HTTP API. When ``None`` (default) the
+ ``OLLAMA_HOST`` environment variable is used, then
+ ``http://localhost:11434`` -- the same precedence as ``OllamaEngine``.
timeout:
Per-request timeout in seconds.
"""
@@ -48,10 +51,12 @@ class OllamaEmbedder:
self,
*,
model: str = DEFAULT_EMBED_MODEL,
- host: str = DEFAULT_OLLAMA_HOST,
+ host: Optional[str] = None,
timeout: float = 30.0,
) -> None:
self._model = model
+ if host is None:
+ host = os.environ.get("OLLAMA_HOST") or DEFAULT_OLLAMA_HOST
self._host = host.rstrip("/")
self._timeout = timeout
self._dim: Optional[int] = None
@@ -139,6 +144,19 @@ class OllamaEmbedder:
return [self.embed(t) for t in texts]
+def default_embedder() -> Optional[OllamaEmbedder]:
+ """Return an ``OllamaEmbedder`` if the daemon and default model are up.
+
+ Ingestion call sites use this so chunks get embedded whenever an embedder
+ is actually reachable, and fall back to lexical-only rows (``None``)
+ otherwise. Query-side hybrid search does the same probe, so the two ends
+ stay in step: if this returns ``None`` at ingest, ``_vector_recall`` finds
+ no rows to score either way.
+ """
+ embedder = OllamaEmbedder()
+ return embedder if embedder.is_available() else None
+
+
# ---------------------------------------------------------------------------
# Deserialisation helper (used by verification + future retrieval code)
# ---------------------------------------------------------------------------
@@ -162,6 +180,7 @@ def decode_embedding(blob: Optional[bytes], *, dtype=None) -> Optional[np.ndarra
__all__ = [
"OllamaEmbedder",
"decode_embedding",
+ "default_embedder",
"DEFAULT_EMBED_MODEL",
"DEFAULT_OLLAMA_HOST",
]
diff --git a/src/openjarvis/connectors/pipeline.py b/src/openjarvis/connectors/pipeline.py
index b1c9c138..8afbb7fa 100644
--- a/src/openjarvis/connectors/pipeline.py
+++ b/src/openjarvis/connectors/pipeline.py
@@ -206,10 +206,23 @@ class IngestionPipeline:
# behind. Compared before the chunks are embedded so an unchanged
# vault costs no embedder calls on re-sync. Only body chunks are
# compared: attachments never change without their parent.
+ #
+ # An unchanged document is still rewritten when an embedder is
+ # configured and its stored rows carry a different
+ # ``embedding_model_version`` -- including "" for rows ingested
+ # before any embedder was available -- so pulling an embedding
+ # model after the first sync backfills vectors instead of leaving
+ # hybrid search with nothing to score. With no embedder configured
+ # (daemon down, model not pulled) existing vectors are left alone.
new_hashes = {c.index: _content_hash(c.content) for c in chunks}
existing_hashes = self._store.chunk_hashes(doc.doc_id, source_id)
if existing_hashes:
- if existing_hashes == new_hashes:
+ stale_vectors = (
+ self._embedder is not None
+ and self._store.embedding_versions(doc.doc_id, source_id)
+ != {self._embedder.model_version}
+ )
+ if existing_hashes == new_hashes and not stale_vectors:
seen_doc_ids.add(doc.doc_id)
continue
self._store.delete(doc.doc_id)
diff --git a/src/openjarvis/connectors/store.py b/src/openjarvis/connectors/store.py
index 712cb7e4..bf68a9d5 100644
--- a/src/openjarvis/connectors/store.py
+++ b/src/openjarvis/connectors/store.py
@@ -462,6 +462,21 @@ class KnowledgeStore(MemoryBackend):
).fetchall()
return {row[0]: row[1] or "" for row in rows}
+ def embedding_versions(self, doc_id: str, source_id: str) -> set[str]:
+ """Return the distinct ``embedding_model_version`` values on a document's body.
+
+ Rows that were ingested without an embedder carry ``""``. The
+ ingestion pipeline uses this to notice that an embedder has since
+ become available (or changed model) and re-embed an otherwise
+ unchanged document.
+ """
+ rows = self._conn.execute(
+ "SELECT DISTINCT embedding_model_version FROM knowledge_chunks "
+ "WHERE doc_id = ? AND source_id = ?",
+ (doc_id, source_id),
+ ).fetchall()
+ return {row[0] or "" for row in rows}
+
def delete(self, doc_id: str) -> bool:
"""Delete all chunks with the given *doc_id*. Returns True if any existed."""
cur = self._conn.execute(
diff --git a/src/openjarvis/server/connectors_router.py b/src/openjarvis/server/connectors_router.py
index 0418d751..435d0a4e 100644
--- a/src/openjarvis/server/connectors_router.py
+++ b/src/openjarvis/server/connectors_router.py
@@ -71,6 +71,9 @@ def _ensure_connectors_registered() -> None:
and not mod_name.endswith("retriever")
and not mod_name.endswith("sync_engine")
and not mod_name.endswith("oauth")
+ # Registers nothing; reloading it would only swap out the
+ # OllamaEmbedder class under anyone holding (or patching) it.
+ and not mod_name.endswith("embeddings")
):
try:
importlib.reload(sys.modules[mod_name])
@@ -358,13 +361,16 @@ def create_connectors_router():
def _run_sync() -> None:
try:
+ from openjarvis.connectors.embeddings import default_embedder
from openjarvis.connectors.pipeline import IngestionPipeline
from openjarvis.connectors.store import KnowledgeStore
from openjarvis.connectors.sync_engine import SyncEngine
with KnowledgeStore() as store:
with SyncEngine(
- pipeline=IngestionPipeline(store=store),
+ pipeline=IngestionPipeline(
+ store=store, embedder=default_embedder()
+ ),
) as engine:
engine.sync(instance, cancel_event=cancel_event)
final_state = "cancelled" if cancel_event.is_set() else "complete"
diff --git a/tests/connectors/test_embeddings.py b/tests/connectors/test_embeddings.py
new file mode 100644
index 00000000..3967fa9d
--- /dev/null
+++ b/tests/connectors/test_embeddings.py
@@ -0,0 +1,35 @@
+"""Tests for openjarvis.connectors.embeddings.default_embedder."""
+
+from __future__ import annotations
+
+import pytest
+
+from openjarvis.connectors import embeddings
+
+
+def test_default_embedder_returns_instance_when_available(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(embeddings.OllamaEmbedder, "is_available", lambda self: True)
+ emb = embeddings.default_embedder()
+ assert isinstance(emb, embeddings.OllamaEmbedder)
+ assert emb.model_version == f"ollama:{embeddings.DEFAULT_EMBED_MODEL}"
+
+
+def test_default_embedder_returns_none_when_unreachable(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(embeddings.OllamaEmbedder, "is_available", lambda self: False)
+ assert embeddings.default_embedder() is None
+
+
+def test_embedder_host_precedence(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Explicit host > OLLAMA_HOST env > built-in default, like OllamaEngine."""
+ monkeypatch.delenv("OLLAMA_HOST", raising=False)
+ assert embeddings.OllamaEmbedder()._host == embeddings.DEFAULT_OLLAMA_HOST
+
+ monkeypatch.setenv("OLLAMA_HOST", "http://127.0.0.1:11434/")
+ assert embeddings.OllamaEmbedder()._host == "http://127.0.0.1:11434"
+
+ explicit = embeddings.OllamaEmbedder(host="http://gpu-box:11434")
+ assert explicit._host == "http://gpu-box:11434"
diff --git a/tests/connectors/test_pipeline.py b/tests/connectors/test_pipeline.py
index a86617b2..7dda4b79 100644
--- a/tests/connectors/test_pipeline.py
+++ b/tests/connectors/test_pipeline.py
@@ -521,3 +521,44 @@ def test_reingest_unchanged_document_skips_embedding(
assert n == 0
assert second.calls == 0
assert store.count() == 1
+
+
+def test_reingest_backfills_embeddings_when_embedder_appears(
+ store: KnowledgeStore,
+) -> None:
+ """Rows ingested without an embedder are re-embedded once one is available."""
+ doc = _make_doc(doc_id="doc:backfill", content="Indexed before nomic was pulled.")
+ IngestionPipeline(store).ingest([doc])
+ row = store._conn.execute(
+ "SELECT embedding, embedding_model_version FROM knowledge_chunks"
+ ).fetchone()
+ assert row[0] is None and row[1] == ""
+
+ embedder = _StubEmbedder()
+ n = IngestionPipeline(store, embedder=embedder).ingest([doc]) # type: ignore[arg-type]
+
+ assert n == 1
+ assert embedder.calls == 1
+ assert store.count() == 1
+ row = store._conn.execute(
+ "SELECT embedding, embedding_model_version FROM knowledge_chunks"
+ ).fetchone()
+ assert row[0] is not None
+ assert row[1] == "stub:test-embedder"
+
+
+def test_reingest_without_embedder_keeps_existing_vectors(
+ store: KnowledgeStore,
+) -> None:
+ """A sync while the embedder is down must not strip vectors already stored."""
+ doc = _make_doc(doc_id="doc:keep-vec", content="Embedded on the first pass.")
+ IngestionPipeline(store, embedder=_StubEmbedder()).ingest([doc]) # type: ignore[arg-type]
+
+ n = IngestionPipeline(store).ingest([doc])
+
+ assert n == 0
+ row = store._conn.execute(
+ "SELECT embedding, embedding_model_version FROM knowledge_chunks"
+ ).fetchone()
+ assert row[0] is not None
+ assert row[1] == "stub:test-embedder"
diff --git a/tests/server/test_connectors_router.py b/tests/server/test_connectors_router.py
index 95b4c801..644c42ea 100644
--- a/tests/server/test_connectors_router.py
+++ b/tests/server/test_connectors_router.py
@@ -12,6 +12,24 @@ from pathlib import Path
import pytest
[email protected](scope="module", autouse=True)
+def _no_embedder():
+ """Keep sync threads off the network for the whole module.
+
+ Sync runs probe for an embedder before ingesting. Several tests here
+ kick off a background sync and return immediately, so a function-scoped
+ patch would be undone while that thread is still running and it would
+ fall through to a real daemon lookup -- slow enough on some hosts to
+ leave the connector "syncing" into the next test.
+ """
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr(
+ "openjarvis.connectors.embeddings.OllamaEmbedder.is_available",
+ lambda self: False,
+ )
+ yield
+
+
@pytest.fixture
def app():
try: