Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(rag): make H3 embedding_dimension probe real + extend dedup to index_all (#4502 follow-up) (#4593)

LearningCircuit954825ab8f75
diff --git a/changelog.d/4593.bugfix.md b/changelog.d/4593.bugfix.md
new file mode 100644
index 0000000000..3c107d8f72
--- /dev/null
+++ b/changelog.d/4593.bugfix.md
@@ -0,0 +1 @@
+Fixed the RAG collection `embedding_dimension`, which was always stored as NULL (the metadata helper probed a non-existent `provider.embedding_dimension` attribute); it is now derived from the actual embedding model. Also extended the collection-indexing dedup to the bulk `index-all` route, which previously stored no embedding metadata and never cleaned up stale chunks/indices on force-reindex.
diff --git a/src/local_deep_research/research_library/routes/rag_routes.py b/src/local_deep_research/research_library/routes/rag_routes.py
index cbb5a40eda..4cd470631e 100644
--- a/src/local_deep_research/research_library/routes/rag_routes.py
+++ b/src/local_deep_research/research_library/routes/rag_routes.py
@@ -873,8 +873,10 @@ def index_all():
f"Starting index-all for collection_id: {collection_id}, force_reindex: {force_reindex}"
)
- # Create RAG service in request context before generator runs
- rag_service = get_rag_service(collection_id)
+ # Create RAG service in request context before generator runs.
+ # use_defaults=force_reindex mirrors index_collection / the background
+ # worker so a force-reindex resolves the same embedding config.
+ rag_service = get_rag_service(collection_id, use_defaults=force_reindex)
def generate():
"""Generator function for SSE progress updates."""
@@ -884,21 +886,33 @@ def index_all():
# Get document IDs to index from DocumentCollection
with get_user_db_session(username) as db_session:
- # Query Document joined with DocumentCollection for the collection
- query = (
- db_session.query(Document.id, Document.title)
- .join(
- DocumentCollection,
- Document.id == DocumentCollection.document_id,
- )
- .filter(DocumentCollection.collection_id == collection_id)
+ # Persist embedding metadata + clean up on force-reindex via the
+ # shared helpers, so this bulk route behaves like the
+ # single-collection SSE route and the background worker
+ # (previously it stored no embedding_dimension and never reset
+ # stale chunks/indices on force-reindex).
+ collection = (
+ db_session.query(Collection)
+ .filter_by(id=collection_id)
+ .first()
)
+ if collection is not None and (
+ collection.embedding_model is None or force_reindex
+ ):
+ _store_collection_embedding_metadata(
+ collection, rag_service
+ )
+ db_session.commit()
+ if force_reindex:
+ _reset_collection_for_reindex(db_session, collection_id)
+ db_session.commit()
- if not force_reindex:
- # Only index documents that haven't been indexed yet
- query = query.filter(DocumentCollection.indexed.is_(False))
-
- doc_info = [(doc_id, title) for doc_id, title in query.all()]
+ doc_info = [
+ (doc.id, doc.title)
+ for _dc, doc in _query_documents_to_index(
+ db_session, collection_id, force_reindex
+ )
+ ]
if not doc_info:
yield f"data: {json.dumps({'type': 'complete', 'results': {'successful': 0, 'skipped': 0, 'failed': 0, 'message': 'No documents to index'}})}\n\n"
@@ -2080,12 +2094,13 @@ def get_collection_documents(collection_id):
# =============================================================================
# Shared collection-indexing helpers
#
-# Both the SSE route (index_collection) and the background worker
-# (_background_index_worker) index a collection. These helpers hold the logic
-# that must stay identical between the two paths so it cannot drift again — the
-# embedding metadata to persist, the force-reindex cleanup, and the query for
-# documents to index. The two callers differ only in how they report progress
-# (SSE stream vs TaskMetadata) and so keep their own loops.
+# Three routes index a collection: the single-collection SSE route
+# (index_collection), the bulk SSE route (index_all), and the background worker
+# (_background_index_worker). These helpers hold the logic that must stay
+# identical across all three so it cannot drift again — the embedding metadata
+# to persist, the force-reindex cleanup, and the query for documents to index.
+# The callers differ only in how they report progress (SSE stream vs
+# TaskMetadata) and so keep their own loops.
# =============================================================================
@@ -2093,19 +2108,24 @@ def _store_collection_embedding_metadata(collection, rag_service):
"""Persist the embedding/index configuration used to index a collection.
Run on first index or force-reindex. Includes the embedding dimension,
- probed from the provider; previously the background worker omitted it, so
- collections indexed that way lost ``embedding_dimension``. Does not commit —
- the caller owns the transaction.
+ probed by embedding a test string (the same way LibraryRAGService derives
+ it for the RAGIndex row). Previously this read
+ ``embedding_manager.provider.embedding_dimension``, an attribute the real
+ LocalEmbeddingManager does not have, so the probe always failed and
+ ``embedding_dimension`` was persisted as NULL via both indexing paths. Does
+ not commit — the caller owns the transaction.
"""
embedding_dim = None
try:
- # Try to get dimension from the embedding manager's provider
- if hasattr(rag_service.embedding_manager, "provider"):
- provider = rag_service.embedding_manager.provider
- if hasattr(provider, "embedding_dimension"):
- embedding_dim = provider.embedding_dimension
+ # Derive the dimension by embedding a test string — mirrors
+ # LibraryRAGService.* which computes len(embed_query("test")) for the
+ # RAGIndex row. The embedding manager has no usable dimension attribute.
+ test_embedding = rag_service.embedding_manager.embeddings.embed_query(
+ "test"
+ )
+ embedding_dim = len(test_embedding)
except Exception:
- logger.warning("Could not get embedding dimension")
+ logger.debug("Could not determine embedding dimension", exc_info=True)
collection.embedding_model = rag_service.embedding_model
collection.embedding_model_type = EmbeddingProvider(
diff --git a/tests/research_library/routes/test_rag_indexing_helpers.py b/tests/research_library/routes/test_rag_indexing_helpers.py
index 68155d0acb..83b67ebcab 100644
--- a/tests/research_library/routes/test_rag_indexing_helpers.py
+++ b/tests/research_library/routes/test_rag_indexing_helpers.py
@@ -10,7 +10,10 @@ collections indexed that way lost it.
from types import SimpleNamespace
from unittest.mock import Mock, patch
-from local_deep_research.database.models.library import EmbeddingProvider
+from local_deep_research.database.models.library import (
+ DocumentCollection,
+ EmbeddingProvider,
+)
from local_deep_research.research_library.routes.rag_routes import (
_store_collection_embedding_metadata,
_reset_collection_for_reindex,
@@ -20,13 +23,22 @@ from local_deep_research.research_library.routes.rag_routes import (
MODULE = "local_deep_research.research_library.routes.rag_routes"
-def _mock_rag_service(embedding_dimension=768, provider_has_dim=True):
- if provider_has_dim:
- provider = SimpleNamespace(embedding_dimension=embedding_dimension)
+def _mock_rag_service(embedding_dimension=768, embed_raises=False):
+ """Mock a LibraryRAGService.
+
+ The dimension is derived by ``embedding_manager.embeddings.embed_query``
+ (the real path). The mock deliberately has NO ``embedding_manager.provider``
+ attribute — the original code probed ``provider.embedding_dimension``, which
+ never exists on the real LocalEmbeddingManager, so the dimension was always
+ NULL. Reverting to that probe must fail these tests.
+ """
+ embeddings = Mock()
+ if embed_raises:
+ embeddings.embed_query.side_effect = RuntimeError("no embedder")
else:
- provider = SimpleNamespace() # no embedding_dimension attribute
+ embeddings.embed_query.return_value = [0.0] * embedding_dimension
return SimpleNamespace(
- embedding_manager=SimpleNamespace(provider=provider),
+ embedding_manager=SimpleNamespace(embeddings=embeddings),
embedding_model="nomic-embed-text",
embedding_provider="ollama",
chunk_size=512,
@@ -46,7 +58,11 @@ class TestStoreCollectionEmbeddingMetadata:
_store_collection_embedding_metadata(collection, rag)
- # The regression this fixes: dimension is persisted.
+ # The regression this fixes: dimension is persisted, derived from the
+ # real embed_query path (not a non-existent provider attribute).
+ rag.embedding_manager.embeddings.embed_query.assert_called_once_with(
+ "test"
+ )
assert collection.embedding_dimension == 768
assert collection.embedding_model == "nomic-embed-text"
assert collection.embedding_model_type == EmbeddingProvider.OLLAMA
@@ -63,12 +79,12 @@ class TestStoreCollectionEmbeddingMetadata:
_store_collection_embedding_metadata(collection, rag)
assert collection.normalize_vectors is True
- def test_missing_provider_dimension_yields_none(self):
+ def test_embed_failure_yields_none_dimension(self):
collection = Mock()
- rag = _mock_rag_service(provider_has_dim=False)
+ rag = _mock_rag_service(embed_raises=True)
_store_collection_embedding_metadata(collection, rag)
assert collection.embedding_dimension is None
- # Other fields are still stored.
+ # Other fields are still stored despite the dimension probe failing.
assert collection.embedding_model == "nomic-embed-text"
# Note: "does not commit" needs no test here — the helper takes no session
@@ -87,15 +103,41 @@ class TestQueryDocumentsToIndex:
db, chain = self._chainable_session(["doc"])
result = _query_documents_to_index(db, "cid", force_reindex=False)
assert result == ["doc"]
- # collection_id filter + indexed==False filter
- assert chain.filter.call_count == 2
+ # collection_id filter + the load-bearing indexed == False filter.
+ calls = chain.filter.call_args_list
+ assert len(calls) == 2
+ assert (
+ calls[0].args[0].compare(DocumentCollection.collection_id == "cid")
+ )
+ # Assert the SECOND filter is exactly `indexed == False` — not `== True`,
+ # not a `not`/`is_(False)` variant. This is the invariant a collapsing
+ # call-count check could never catch.
+ assert (
+ calls[1]
+ .args[0]
+ .compare(
+ DocumentCollection.indexed == False # noqa: E712
+ )
+ )
def test_force_reindex_skips_indexed_filter(self):
db, chain = self._chainable_session(["a", "b"])
result = _query_documents_to_index(db, "cid", force_reindex=True)
assert result == ["a", "b"]
- # only the collection_id filter
- assert chain.filter.call_count == 1
+ # Only the collection_id filter — and crucially NO indexed predicate,
+ # so a force-reindex returns every document.
+ calls = chain.filter.call_args_list
+ assert len(calls) == 1
+ assert (
+ calls[0].args[0].compare(DocumentCollection.collection_id == "cid")
+ )
+ assert (
+ not calls[0]
+ .args[0]
+ .compare(
+ DocumentCollection.indexed == False # noqa: E712
+ )
+ )
class TestResetCollectionForReindex:
diff --git a/tests/research_library/routes/test_rag_routes_gaps_coverage.py b/tests/research_library/routes/test_rag_routes_gaps_coverage.py
index d4e6790e70..b58bbb4dd4 100644
--- a/tests/research_library/routes/test_rag_routes_gaps_coverage.py
+++ b/tests/research_library/routes/test_rag_routes_gaps_coverage.py
@@ -753,6 +753,77 @@ class TestIndexCollection:
assert resp.headers.get("X-Accel-Buffering") == "no"
+class TestIndexAll:
+ """The bulk index-all SSE route must share the indexing helpers.
+
+ Regression for H3 incompleteness: index_all previously stored no embedding
+ metadata and never reset stale chunks/indices on force-reindex (the two
+ drift bugs the dedup was meant to eliminate everywhere).
+ """
+
+ def test_force_reindex_stores_metadata_and_resets(self, app):
+ mock_coll = Mock()
+ mock_coll.id = "coll-1"
+ mock_coll.embedding_model = None
+ db_session = _make_db_session()
+ db_session.query = Mock(
+ return_value=_build_mock_query(first_result=mock_coll)
+ )
+ mock_rag = Mock()
+
+ with (
+ patch(
+ f"{_ROUTES}._store_collection_embedding_metadata"
+ ) as mock_store,
+ patch(f"{_ROUTES}._reset_collection_for_reindex") as mock_reset,
+ patch(f"{_ROUTES}._query_documents_to_index", return_value=[]),
+ ):
+ with _auth_client(
+ app,
+ mock_db_session=db_session,
+ extra_patches=[
+ patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
+ ],
+ ) as (client, ctx):
+ resp = client.get(
+ "/library/api/rag/index-all"
+ "?collection_id=coll-1&force_reindex=true"
+ )
+ resp.get_data() # drain the SSE generator to completion
+
+ mock_store.assert_called_once_with(mock_coll, mock_rag)
+ mock_reset.assert_called_once_with(db_session, "coll-1")
+
+ def test_incremental_does_not_reset(self, app):
+ """A non-force index-all must NOT wipe existing chunks/indices."""
+ mock_coll = Mock()
+ mock_coll.id = "coll-1"
+ mock_coll.embedding_model = "already-set"
+ db_session = _make_db_session()
+ db_session.query = Mock(
+ return_value=_build_mock_query(first_result=mock_coll)
+ )
+
+ with (
+ patch(f"{_ROUTES}._store_collection_embedding_metadata"),
+ patch(f"{_ROUTES}._reset_collection_for_reindex") as mock_reset,
+ patch(f"{_ROUTES}._query_documents_to_index", return_value=[]),
+ ):
+ with _auth_client(
+ app,
+ mock_db_session=db_session,
+ extra_patches=[
+ patch(f"{_ROUTES}.get_rag_service", return_value=Mock()),
+ ],
+ ) as (client, ctx):
+ resp = client.get(
+ "/library/api/rag/index-all?collection_id=coll-1"
+ )
+ resp.get_data()
+
+ mock_reset.assert_not_called()
+
+
class TestRagServiceCloseLifecycle:
"""Regression coverage for the RAG-service close-on-exit guarantee.