~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(rag): make H3 embedding_dimension probe real + extend dedup to index_all (#4502 follow-up) (#4593)
LearningCircuit954825ab8f75
diff --git a/changelog.d/4593.bugfix.md b/changelog.d/4593.bugfix.mdnew file mode 100644index 0000000000..3c107d8f72--- /dev/null+++ b/changelog.d/4593.bugfix.md@@ -0,0 +1 @@+Fixed the RAG collection `embedding_dimension`, which was always stored as NULL (the metadata helper probed a non-existent `provider.embedding_dimension` attribute); it is now derived from the actual embedding model. Also extended the collection-indexing dedup to the bulk `index-all` route, which previously stored no embedding metadata and never cleaned up stale chunks/indices on force-reindex.diff --git a/src/local_deep_research/research_library/routes/rag_routes.py b/src/local_deep_research/research_library/routes/rag_routes.pyindex cbb5a40eda..4cd470631e 100644--- a/src/local_deep_research/research_library/routes/rag_routes.py+++ b/src/local_deep_research/research_library/routes/rag_routes.py@@ -873,8 +873,10 @@ def index_all():f"Starting index-all for collection_id: {collection_id}, force_reindex: {force_reindex}")- # Create RAG service in request context before generator runs- rag_service = get_rag_service(collection_id)+ # Create RAG service in request context before generator runs.+ # use_defaults=force_reindex mirrors index_collection / the background+ # worker so a force-reindex resolves the same embedding config.+ rag_service = get_rag_service(collection_id, use_defaults=force_reindex)def generate():"""Generator function for SSE progress updates."""@@ -884,21 +886,33 @@ def index_all():# Get document IDs to index from DocumentCollectionwith get_user_db_session(username) as db_session:- # Query Document joined with DocumentCollection for the collection- query = (- db_session.query(Document.id, Document.title)- .join(- DocumentCollection,- Document.id == DocumentCollection.document_id,- )- .filter(DocumentCollection.collection_id == collection_id)+ # Persist embedding metadata + clean up on force-reindex via the+ # shared helpers, so this bulk route behaves like the+ # single-collection SSE route and the background worker+ # (previously it stored no embedding_dimension and never reset+ # stale chunks/indices on force-reindex).+ collection = (+ db_session.query(Collection)+ .filter_by(id=collection_id)+ .first())+ if collection is not None and (+ collection.embedding_model is None or force_reindex+ ):+ _store_collection_embedding_metadata(+ collection, rag_service+ )+ db_session.commit()+ if force_reindex:+ _reset_collection_for_reindex(db_session, collection_id)+ db_session.commit()- if not force_reindex:- # Only index documents that haven't been indexed yet- query = query.filter(DocumentCollection.indexed.is_(False))-- doc_info = [(doc_id, title) for doc_id, title in query.all()]+ doc_info = [+ (doc.id, doc.title)+ for _dc, doc in _query_documents_to_index(+ db_session, collection_id, force_reindex+ )+ ]if not doc_info:yield f"data: {json.dumps({'type': 'complete', 'results': {'successful': 0, 'skipped': 0, 'failed': 0, 'message': 'No documents to index'}})}\n\n"@@ -2080,12 +2094,13 @@ def get_collection_documents(collection_id):# =============================================================================# Shared collection-indexing helpers#-# Both the SSE route (index_collection) and the background worker-# (_background_index_worker) index a collection. These helpers hold the logic-# that must stay identical between the two paths so it cannot drift again — the-# embedding metadata to persist, the force-reindex cleanup, and the query for-# documents to index. The two callers differ only in how they report progress-# (SSE stream vs TaskMetadata) and so keep their own loops.+# Three routes index a collection: the single-collection SSE route+# (index_collection), the bulk SSE route (index_all), and the background worker+# (_background_index_worker). These helpers hold the logic that must stay+# identical across all three so it cannot drift again — the embedding metadata+# to persist, the force-reindex cleanup, and the query for documents to index.+# The callers differ only in how they report progress (SSE stream vs+# TaskMetadata) and so keep their own loops.# =============================================================================@@ -2093,19 +2108,24 @@ def _store_collection_embedding_metadata(collection, rag_service):"""Persist the embedding/index configuration used to index a collection.Run on first index or force-reindex. Includes the embedding dimension,- probed from the provider; previously the background worker omitted it, so- collections indexed that way lost ``embedding_dimension``. Does not commit —- the caller owns the transaction.+ probed by embedding a test string (the same way LibraryRAGService derives+ it for the RAGIndex row). Previously this read+ ``embedding_manager.provider.embedding_dimension``, an attribute the real+ LocalEmbeddingManager does not have, so the probe always failed and+ ``embedding_dimension`` was persisted as NULL via both indexing paths. Does+ not commit — the caller owns the transaction."""embedding_dim = Nonetry:- # Try to get dimension from the embedding manager's provider- if hasattr(rag_service.embedding_manager, "provider"):- provider = rag_service.embedding_manager.provider- if hasattr(provider, "embedding_dimension"):- embedding_dim = provider.embedding_dimension+ # Derive the dimension by embedding a test string — mirrors+ # LibraryRAGService.* which computes len(embed_query("test")) for the+ # RAGIndex row. The embedding manager has no usable dimension attribute.+ test_embedding = rag_service.embedding_manager.embeddings.embed_query(+ "test"+ )+ embedding_dim = len(test_embedding)except Exception:- logger.warning("Could not get embedding dimension")+ logger.debug("Could not determine embedding dimension", exc_info=True)collection.embedding_model = rag_service.embedding_modelcollection.embedding_model_type = EmbeddingProvider(diff --git a/tests/research_library/routes/test_rag_indexing_helpers.py b/tests/research_library/routes/test_rag_indexing_helpers.pyindex 68155d0acb..83b67ebcab 100644--- a/tests/research_library/routes/test_rag_indexing_helpers.py+++ b/tests/research_library/routes/test_rag_indexing_helpers.py@@ -10,7 +10,10 @@ collections indexed that way lost it.from types import SimpleNamespacefrom unittest.mock import Mock, patch-from local_deep_research.database.models.library import EmbeddingProvider+from local_deep_research.database.models.library import (+ DocumentCollection,+ EmbeddingProvider,+)from local_deep_research.research_library.routes.rag_routes import (_store_collection_embedding_metadata,_reset_collection_for_reindex,@@ -20,13 +23,22 @@ from local_deep_research.research_library.routes.rag_routes import (MODULE = "local_deep_research.research_library.routes.rag_routes"-def _mock_rag_service(embedding_dimension=768, provider_has_dim=True):- if provider_has_dim:- provider = SimpleNamespace(embedding_dimension=embedding_dimension)+def _mock_rag_service(embedding_dimension=768, embed_raises=False):+ """Mock a LibraryRAGService.++ The dimension is derived by ``embedding_manager.embeddings.embed_query``+ (the real path). The mock deliberately has NO ``embedding_manager.provider``+ attribute — the original code probed ``provider.embedding_dimension``, which+ never exists on the real LocalEmbeddingManager, so the dimension was always+ NULL. Reverting to that probe must fail these tests.+ """+ embeddings = Mock()+ if embed_raises:+ embeddings.embed_query.side_effect = RuntimeError("no embedder")else:- provider = SimpleNamespace() # no embedding_dimension attribute+ embeddings.embed_query.return_value = [0.0] * embedding_dimensionreturn SimpleNamespace(- embedding_manager=SimpleNamespace(provider=provider),+ embedding_manager=SimpleNamespace(embeddings=embeddings),embedding_model="nomic-embed-text",embedding_provider="ollama",chunk_size=512,@@ -46,7 +58,11 @@ class TestStoreCollectionEmbeddingMetadata:_store_collection_embedding_metadata(collection, rag)- # The regression this fixes: dimension is persisted.+ # The regression this fixes: dimension is persisted, derived from the+ # real embed_query path (not a non-existent provider attribute).+ rag.embedding_manager.embeddings.embed_query.assert_called_once_with(+ "test"+ )assert collection.embedding_dimension == 768assert collection.embedding_model == "nomic-embed-text"assert collection.embedding_model_type == EmbeddingProvider.OLLAMA@@ -63,12 +79,12 @@ class TestStoreCollectionEmbeddingMetadata:_store_collection_embedding_metadata(collection, rag)assert collection.normalize_vectors is True- def test_missing_provider_dimension_yields_none(self):+ def test_embed_failure_yields_none_dimension(self):collection = Mock()- rag = _mock_rag_service(provider_has_dim=False)+ rag = _mock_rag_service(embed_raises=True)_store_collection_embedding_metadata(collection, rag)assert collection.embedding_dimension is None- # Other fields are still stored.+ # Other fields are still stored despite the dimension probe failing.assert collection.embedding_model == "nomic-embed-text"# Note: "does not commit" needs no test here — the helper takes no session@@ -87,15 +103,41 @@ class TestQueryDocumentsToIndex:db, chain = self._chainable_session(["doc"])result = _query_documents_to_index(db, "cid", force_reindex=False)assert result == ["doc"]- # collection_id filter + indexed==False filter- assert chain.filter.call_count == 2+ # collection_id filter + the load-bearing indexed == False filter.+ calls = chain.filter.call_args_list+ assert len(calls) == 2+ assert (+ calls[0].args[0].compare(DocumentCollection.collection_id == "cid")+ )+ # Assert the SECOND filter is exactly `indexed == False` — not `== True`,+ # not a `not`/`is_(False)` variant. This is the invariant a collapsing+ # call-count check could never catch.+ assert (+ calls[1]+ .args[0]+ .compare(+ DocumentCollection.indexed == False # noqa: E712+ )+ )def test_force_reindex_skips_indexed_filter(self):db, chain = self._chainable_session(["a", "b"])result = _query_documents_to_index(db, "cid", force_reindex=True)assert result == ["a", "b"]- # only the collection_id filter- assert chain.filter.call_count == 1+ # Only the collection_id filter — and crucially NO indexed predicate,+ # so a force-reindex returns every document.+ calls = chain.filter.call_args_list+ assert len(calls) == 1+ assert (+ calls[0].args[0].compare(DocumentCollection.collection_id == "cid")+ )+ assert (+ not calls[0]+ .args[0]+ .compare(+ DocumentCollection.indexed == False # noqa: E712+ )+ )class TestResetCollectionForReindex:diff --git a/tests/research_library/routes/test_rag_routes_gaps_coverage.py b/tests/research_library/routes/test_rag_routes_gaps_coverage.pyindex d4e6790e70..b58bbb4dd4 100644--- a/tests/research_library/routes/test_rag_routes_gaps_coverage.py+++ b/tests/research_library/routes/test_rag_routes_gaps_coverage.py@@ -753,6 +753,77 @@ class TestIndexCollection:assert resp.headers.get("X-Accel-Buffering") == "no"+class TestIndexAll:+ """The bulk index-all SSE route must share the indexing helpers.++ Regression for H3 incompleteness: index_all previously stored no embedding+ metadata and never reset stale chunks/indices on force-reindex (the two+ drift bugs the dedup was meant to eliminate everywhere).+ """++ def test_force_reindex_stores_metadata_and_resets(self, app):+ mock_coll = Mock()+ mock_coll.id = "coll-1"+ mock_coll.embedding_model = None+ db_session = _make_db_session()+ db_session.query = Mock(+ return_value=_build_mock_query(first_result=mock_coll)+ )+ mock_rag = Mock()++ with (+ patch(+ f"{_ROUTES}._store_collection_embedding_metadata"+ ) as mock_store,+ patch(f"{_ROUTES}._reset_collection_for_reindex") as mock_reset,+ patch(f"{_ROUTES}._query_documents_to_index", return_value=[]),+ ):+ with _auth_client(+ app,+ mock_db_session=db_session,+ extra_patches=[+ patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),+ ],+ ) as (client, ctx):+ resp = client.get(+ "/library/api/rag/index-all"+ "?collection_id=coll-1&force_reindex=true"+ )+ resp.get_data() # drain the SSE generator to completion++ mock_store.assert_called_once_with(mock_coll, mock_rag)+ mock_reset.assert_called_once_with(db_session, "coll-1")++ def test_incremental_does_not_reset(self, app):+ """A non-force index-all must NOT wipe existing chunks/indices."""+ mock_coll = Mock()+ mock_coll.id = "coll-1"+ mock_coll.embedding_model = "already-set"+ db_session = _make_db_session()+ db_session.query = Mock(+ return_value=_build_mock_query(first_result=mock_coll)+ )++ with (+ patch(f"{_ROUTES}._store_collection_embedding_metadata"),+ patch(f"{_ROUTES}._reset_collection_for_reindex") as mock_reset,+ patch(f"{_ROUTES}._query_documents_to_index", return_value=[]),+ ):+ with _auth_client(+ app,+ mock_db_session=db_session,+ extra_patches=[+ patch(f"{_ROUTES}.get_rag_service", return_value=Mock()),+ ],+ ) as (client, ctx):+ resp = client.get(+ "/library/api/rag/index-all?collection_id=coll-1"+ )+ resp.get_data()++ mock_reset.assert_not_called()++class TestRagServiceCloseLifecycle:"""Regression coverage for the RAG-service close-on-exit guarantee.