~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(library): load embedding settings from Collection record in search engines (#6097)
S4be21d47dca7
diff --git a/changelog.d/6097.bugfix.md b/changelog.d/6097.bugfix.mdnew file mode 100644index 0000000000..fd4980c069--- /dev/null+++ b/changelog.d/6097.bugfix.md@@ -0,0 +1 @@+`CollectionSearchEngine` and `LibraryRAGSearchEngine` now load frozen embedding and text splitter configuration directly from the `Collection` record rather than the `RAGIndex` table, ensuring search remains reliable even if indexing was interrupted and resolving text-splitter mismatches across collection queries.diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_collection.py b/src/local_deep_research/web_search_engines/engines/search_engine_collection.pyindex 0b403d1db7..be4320ebab 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_collection.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_collection.py@@ -8,9 +8,17 @@ from typing import List, Dict, Any, Optionalfrom ...security.secure_logging import loggerfrom .search_engine_library import LibraryRAGSearchEngine-from ...constants import SNIPPET_LENGTH_LONG+from ...constants import (+ DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP,+ DEFAULT_LOCAL_SEARCH_CHUNK_SIZE,+ DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC,+ DEFAULT_LOCAL_SEARCH_INDEX_TYPE,+ DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,+ DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE,+ SNIPPET_LENGTH_LONG,+)from ...research_library.services.library_rag_service import LibraryRAGService-from ...database.models.library import RAGIndex+from ...database.models.library import Collectionfrom ...database.session_context import get_user_db_sessionfrom ...utilities.chunk_anchor import extract_chunk_index, extract_document_idfrom ...utilities.type_utils import to_bool@@ -72,8 +80,8 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):def _load_collection_embedding_settings(self):"""- Load embedding settings from the collection's RAG index.- Uses the same embedding model that was used during indexing.+ Load frozen embedding settings directly from the Collection record.+ Uses the same embedding model that was configured when the collection was created."""if not self.username:logger.warning("Cannot load collection settings without username")@@ -81,29 +89,46 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):try:with get_user_db_session(self.username) as db_session:- # Get RAG index for this collection- rag_index = (- db_session.query(RAGIndex)- .filter_by(- collection_name=self.collection_key,- is_current=True,- )+ collection = (+ db_session.query(Collection)+ .filter_by(id=self.collection_id).first())- if not rag_index:+ if not collection or not collection.embedding_model:logger.warning(- f"No RAG index found for collection {self.collection_id}"+ f"No stored embedding settings found for collection {self.collection_id}")return- # Use embedding settings from the RAG index- self.embedding_model = rag_index.embedding_model- self.embedding_provider = rag_index.embedding_model_type.value- self.chunk_size = rag_index.chunk_size or self.chunk_size- self.chunk_overlap = (- rag_index.chunk_overlap or self.chunk_overlap- )+ # Use embedding settings directly from the Collection+ self.embedding_model = collection.embedding_model+ provider = collection.embedding_model_type+ if provider is not None:+ self.embedding_provider = (+ provider.value+ if hasattr(provider, "value")+ else str(provider)+ )+ coll_chunk_size = getattr(collection, "chunk_size", None)+ if coll_chunk_size is not None:+ self.chunk_size = int(coll_chunk_size)+ coll_chunk_overlap = getattr(collection, "chunk_overlap", None)+ if coll_chunk_overlap is not None:+ self.chunk_overlap = int(coll_chunk_overlap)+ if getattr(collection, "splitter_type", None) is not None:+ self.splitter_type = collection.splitter_type+ if getattr(collection, "text_separators", None) is not None:+ self.text_separators = collection.text_separators+ if getattr(collection, "distance_metric", None) is not None:+ self.distance_metric = collection.distance_metric+ if getattr(collection, "normalize_vectors", None) is not None:+ self.normalize_vectors = to_bool(+ collection.normalize_vectors,+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,+ )+ if getattr(collection, "index_type", None) is not None:+ self.index_type = collection.index_typelogger.info(f"Collection '{self.collection_name}' using embedding: "@@ -148,50 +173,92 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):self._verify_egress_scope()try:- # Get RAG index info for this collection+ # Get frozen embedding settings directly from Collection recordwith get_user_db_session(self.username) as db_session:- rag_index = (- db_session.query(RAGIndex)- .filter_by(- collection_name=self.collection_key,- is_current=True,- )+ collection = (+ db_session.query(Collection)+ .filter_by(id=self.collection_id).first())- if not rag_index:+ if not collection or not collection.embedding_model:logger.info(- f"No RAG index for collection '{self.collection_name}'"+ f"No embedding settings found for collection '{self.collection_name}'")return []- # Get embedding settings from RAG index- embedding_model = rag_index.embedding_model- embedding_provider = rag_index.embedding_model_type.value- chunk_size = rag_index.chunk_size or self.chunk_size- chunk_overlap = rag_index.chunk_overlap or self.chunk_overlap- # Thread the stored normalization flag through so the index is- # queried with the same L2 normalization it was BUILT with.- # Otherwise LibraryRAGService defaults normalize_vectors=True,- # corrupting similarity rankings for collections indexed with- # local_search_normalize_vectors=False. NULL → True mirrors the- # to_bool(..., default=True) convention used elsewhere.- normalize_vectors = to_bool(- rag_index.normalize_vectors, default=True+ # Get embedding settings directly from Collection+ embedding_model = collection.embedding_model+ provider = collection.embedding_model_type+ if provider is not None:+ embedding_provider = (+ provider.value+ if hasattr(provider, "value")+ else str(provider)+ )+ else:+ embedding_provider = self.embedding_provider+ coll_chunk_size = getattr(collection, "chunk_size", None)+ chunk_size = (+ int(coll_chunk_size)+ if coll_chunk_size is not None+ else int(self.chunk_size or DEFAULT_LOCAL_SEARCH_CHUNK_SIZE)+ )+ coll_chunk_overlap = getattr(collection, "chunk_overlap", None)+ chunk_overlap = (+ int(coll_chunk_overlap)+ if coll_chunk_overlap is not None+ else int(+ self.chunk_overlap+ if self.chunk_overlap is not None+ else DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP+ )+ )+ splitter_type = (+ getattr(collection, "splitter_type", None)+ or getattr(self, "splitter_type", None)+ or DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE+ )+ text_separators = (+ collection.text_separators+ if getattr(collection, "text_separators", None) is not None+ else getattr(self, "text_separators", None)+ )+ # Thread the stored normalization flag AND distance+ # metric through, exactly like LibraryRAGSearchEngine.+ # Without normalize_vectors, LibraryRAGService defaults+ # True and would L2-normalize the query against a+ # raw-vector collection, flipping the top hit. Without+ # distance_metric, the collection is labeled "cosine"+ # and l2/dot_product hits get the wrong [0,1] mapping in+ # the relevance transform below. NULL → prior default.+ coll_normalize = getattr(collection, "normalize_vectors", None)+ normalize_vectors = (+ to_bool(+ coll_normalize,+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,+ )+ if coll_normalize is not None+ else getattr(+ self,+ "normalize_vectors",+ DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,+ )+ )+ distance_metric = (+ getattr(collection, "distance_metric", None)+ or getattr(self, "distance_metric", None)+ or DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC+ )+ # Thread the stored index_type too (like metric/+ # normalize): a legacy NULL-index_type row that is+ # physically HNSW would otherwise fall back to "flat",+ # mislabelling the store. NULL → "flat".+ index_type = (+ getattr(collection, "index_type", None)+ or getattr(self, "index_type", None)+ or DEFAULT_LOCAL_SEARCH_INDEX_TYPE)- # Thread the stored distance metric through too: it is what- # LibraryRAGService stamps onto each SearchResult.metric, which- # in turn selects the relevance formula below. Defaulting to- # "cosine" (the LibraryRAGService default) would mislabel an- # l2/dot_product collection and apply the wrong [0,1] mapping.- # NULL (legacy rows) → "cosine" preserves prior behavior.- distance_metric = rag_index.distance_metric or "cosine"- # Thread the stored index_type too (like metric/normalize): with- # it left at the "flat" default, _get_vector_index would fall- # back to flat for a legacy NULL-index_type row that is physically- # HNSW, mislabelling the store (wrong supports_delete/relevance,- # and now a load()-time config-mismatch rejection). NULL → "flat".- index_type = rag_index.index_type or "flat"# Create RAG service with collection's embedding settingswith LibraryRAGService(@@ -200,6 +267,8 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):embedding_provider=embedding_provider,chunk_size=chunk_size,chunk_overlap=chunk_overlap,+ splitter_type=splitter_type,+ text_separators=text_separators,normalize_vectors=normalize_vectors,distance_metric=distance_metric,index_type=index_type,@@ -276,7 +345,9 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):# Add collection info to metadatametadata["collection_id"] = self.collection_id- metadata["collection_name"] = self.collection_name+ metadata["collection_name"] = (+ self.collection_name or "Unknown"+ )# r.distance's meaning depends on r.metric. Map both to a# [0, 1] relevance that grows with similarity:diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_library.py b/src/local_deep_research/web_search_engines/engines/search_engine_library.pyindex 6350a7bf07..908a88e8f7 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_library.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_library.py@@ -4,6 +4,7 @@ Library RAG Search EngineProvides semantic search over the user's personal research library using RAG."""+import jsonimport osfrom typing import List, Dict, Any, Optional@@ -12,8 +13,14 @@ from ...security.secure_logging import loggerfrom ...constants import (DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP,DEFAULT_LOCAL_SEARCH_CHUNK_SIZE,+ DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC,+ DEFAULT_LOCAL_SEARCH_INDEX_TYPE,DEFAULT_LOCAL_SEARCH_MODEL,+ DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,DEFAULT_LOCAL_SEARCH_PROVIDER,+ DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE,+ DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS,+ DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON,SNIPPET_LENGTH_LONG,)from ...research_library.services.library_rag_service import LibraryRAGService@@ -27,7 +34,7 @@ from ...utilities.chunk_anchor import ()from ...utilities.llm_utils import get_server_urlfrom ...utilities.type_utils import to_bool-from ...database.models.library import RAGIndex, Document+from ...database.models.library import Collection, Documentfrom ...research_library.services.pdf_storage_manager import PDFStorageManagerfrom ...database.session_context import get_user_db_sessionfrom ...config.paths import get_library_directory@@ -38,7 +45,7 @@ class LibraryRAGSearchEngine(BaseSearchEngine):Search engine that queries the user's research library using RAG/semantic search."""- # Mark as local RAG engine+ # Mark as local RAG engine for tool selection and safety checksis_local = True# Egress (ADR-0007): a local document store — sensitive data, kept on-box.@@ -54,7 +61,7 @@ class LibraryRAGSearchEngine(BaseSearchEngine):**kwargs,):"""- Initialize the Library RAG search engine.+ Initialize the library RAG search engine.Args:llm: Language model for relevance filtering@@ -111,6 +118,44 @@ class LibraryRAGSearchEngine(BaseSearchEngine):default=DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP,settings_snapshot=settings_snapshot,)+ self.splitter_type = get_setting_from_snapshot(+ "local_search_splitter_type",+ default=DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE,+ settings_snapshot=settings_snapshot,+ )+ raw_text_separators = get_setting_from_snapshot(+ "local_search_text_separators",+ default=DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON,+ settings_snapshot=settings_snapshot,+ )+ if isinstance(raw_text_separators, str):+ try:+ self.text_separators = json.loads(raw_text_separators)+ except (json.JSONDecodeError, TypeError):+ self.text_separators = DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS+ elif isinstance(raw_text_separators, list):+ self.text_separators = raw_text_separators+ else:+ self.text_separators = DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS++ self.distance_metric = get_setting_from_snapshot(+ "local_search_distance_metric",+ default=DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC,+ settings_snapshot=settings_snapshot,+ )+ self.normalize_vectors = to_bool(+ get_setting_from_snapshot(+ "local_search_normalize_vectors",+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,+ settings_snapshot=settings_snapshot,+ ),+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,+ )+ self.index_type = get_setting_from_snapshot(+ "local_search_index_type",+ default=DEFAULT_LOCAL_SEARCH_INDEX_TYPE,+ settings_snapshot=settings_snapshot,+ )# Extract server URL from settings snapshot for link generationself.server_url = get_server_url(settings_snapshot)@@ -202,38 +247,80 @@ class LibraryRAGSearchEngine(BaseSearchEngine):# relevance-formula comment below).all_results = []failed_collections = []+ collection_names = {+ c.get("id"): c.get("name") or f"Collection {c.get('id')}"+ for c in collections+ if c.get("id")+ }for collection in collections:collection_id = collection.get("id")if not collection_id:continuetry:- # Get the RAG index for this collection to find embedding settings+ # Get the Collection record directly for frozen embedding settingswith get_user_db_session(self.username) as session:- collection_name = f"collection_{collection_id}"- rag_index = (- session.query(RAGIndex)- .filter_by(- collection_name=collection_name,- is_current=True,- )+ collection_record = (+ session.query(Collection)+ .filter_by(id=collection_id).first())- if not rag_index:+ if (+ not collection_record+ or not collection_record.embedding_model+ ):logger.debug(- f"No RAG index found for collection {collection_id}"+ f"No embedding settings found for collection {collection_id}")continue- # Get embedding settings from the RAG index- embedding_model = rag_index.embedding_model- embedding_provider = (- rag_index.embedding_model_type.value+ # Get embedding settings directly from Collection+ embedding_model = collection_record.embedding_model+ provider = collection_record.embedding_model_type+ if provider is not None:+ embedding_provider = (+ provider.value+ if hasattr(provider, "value")+ else str(provider)+ )+ else:+ embedding_provider = self.embedding_provider+ coll_chunk_size = getattr(+ collection_record, "chunk_size", None+ )+ chunk_size = (+ int(coll_chunk_size)+ if coll_chunk_size is not None+ else int(+ self.chunk_size+ or DEFAULT_LOCAL_SEARCH_CHUNK_SIZE+ )+ )+ coll_chunk_overlap = getattr(+ collection_record, "chunk_overlap", None)- chunk_size = rag_index.chunk_size or self.chunk_sizechunk_overlap = (- rag_index.chunk_overlap or self.chunk_overlap+ int(coll_chunk_overlap)+ if coll_chunk_overlap is not None+ else int(+ self.chunk_overlap+ if self.chunk_overlap is not None+ else DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP+ )+ )+ splitter_type = (+ getattr(collection_record, "splitter_type", None)+ or getattr(self, "splitter_type", None)+ or DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE+ )+ text_separators = (+ collection_record.text_separators+ if getattr(+ collection_record, "text_separators", None+ )+ is not None+ else getattr(self, "text_separators", None))# Thread the stored normalization flag AND distance# metric through, exactly like CollectionSearchEngine.@@ -243,15 +330,35 @@ class LibraryRAGSearchEngine(BaseSearchEngine):# distance_metric, every collection is labeled "cosine"# and l2/dot_product hits get the wrong [0,1] mapping in# the cross-collection merge below. NULL → prior default.- normalize_vectors = to_bool(- rag_index.normalize_vectors, default=True+ coll_normalize = getattr(+ collection_record, "normalize_vectors", None+ )+ normalize_vectors = (+ to_bool(+ coll_normalize,+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,+ )+ if coll_normalize is not None+ else getattr(+ self,+ "normalize_vectors",+ DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,+ )+ )+ distance_metric = (+ getattr(collection_record, "distance_metric", None)+ or getattr(self, "distance_metric", None)+ or DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC)- distance_metric = rag_index.distance_metric or "cosine"# Thread the stored index_type too (like metric/# normalize): a legacy NULL-index_type row that is# physically HNSW would otherwise fall back to "flat",# mislabelling the store. NULL → "flat".- index_type = rag_index.index_type or "flat"+ index_type = (+ getattr(collection_record, "index_type", None)+ or getattr(self, "index_type", None)+ or DEFAULT_LOCAL_SEARCH_INDEX_TYPE+ )# Create RAG service with the collection's embedding settingswith LibraryRAGService(@@ -260,6 +367,8 @@ class LibraryRAGSearchEngine(BaseSearchEngine):embedding_provider=embedding_provider,chunk_size=chunk_size,chunk_overlap=chunk_overlap,+ splitter_type=splitter_type,+ text_separators=text_separators,normalize_vectors=normalize_vectors,distance_metric=distance_metric,index_type=index_type,@@ -268,99 +377,90 @@ class LibraryRAGSearchEngine(BaseSearchEngine):stats = rag_service.get_rag_stats(collection_id)if stats.get("indexed_documents", 0) == 0:logger.debug(- f"No documents indexed in collection {collection_id}"+ f"No indexed documents in collection {collection_id}")continue- # Search this collection's vector index via the new- # int-id-keyed store (chunk text is rehydrated from- # the encrypted DB by id — never read from the+ # Search this collection's vector index via the+ # new int-id-keyed store (chunk text is rehydrated+ # from the encrypted DB by id — never read from the# vector store itself; see the SECURITY INVARIANT# in vector_stores/base.py).search_results = rag_service.search(query, collection_id, limit)- # Score per THIS collection's own metric, add- # collection info to metadata, and append to the- # cross-collection merge list.+ # Tag each result with collection ID and pre-score+ # relevance per THIS collection's metric (so cosine+ # and L2 sort compatibly on merge).for r in search_results:- metadata = dict(r.metadata or {})- metadata["collection_id"] = collection_id- metadata["collection_name"] = collection.get(- "name", "Unknown"- )- # Map r.distance to a [0,1] relevance growing with- # similarity: cosine/dot_product is an inner product- # in [-1,1] (higher=nearer) -> (d+1)/2 clamped; l2 (or- # any non-standard metric) is a distance >=0 (lower=- # nearer) -> 1/(1+d) in (0,1]. Raw IP would give- # negative relevance and break [0,1] consumers.- # Scoring here (per-collection, before the merge)- # keeps different-metric collections comparable once- # merged. The IP test MUST match faiss_store.- # _build_base_index (inner-product iff cosine/- # dot_product, else L2): a bare `metric == "l2"` would- # score a non-standard metric — which builds an L2- # index — with the IP formula and invert the ranking.+ # Map r.distance to a [0, 1] relevance score:+ # - cosine/dot_product: (d+1)/2 clamped to [0,1]+ # - l2 / other: 1/(1+d) in (0, 1]+ # Must match faiss_store._build_base_index metric+ # selection (cosine/dot_product -> IP, else L2).relevance = (max(0.0, min(1.0, (r.distance + 1.0) / 2.0))if r.metric in ("cosine", "dot_product")else 1.0 / (1.0 + r.distance))- all_results.append((r, relevance, metadata))+ all_results.append((r, relevance, collection_id))except Exception as e:- # One broken collection must not abort the others, but- # record the failure so it is not silently equated with- # "no matching documents" below.safe_msg = self._scrub_error(e)logger.exception(f"Error searching collection {collection_id} ({type(e).__name__}): {safe_msg}")- failed_collections.append(- collection.get("name") or str(collection_id)- )+ failed_collections.append(collection_id)continue- # Sort all results by relevance, descending (higher is always- # better post-transform, regardless of the source collection's- # metric — see the per-result relevance comment above).- all_results.sort(key=lambda item: item[1], reverse=True)-- # Take top results across all collections- top_results = all_results[:limit]+ # Fail loudly if *every* collection errored (returning []+ # masks outright service failure as "no results"). Partial+ # failures with 0 hits also raise: distinguishing a genuine+ # "not found" from "the one collection with answers crashed"+ # is impossible without this check.+ if failed_collections and not all_results:+ failed_names = [+ collection_names.get(cid) or str(cid)+ for cid in failed_collections+ ]+ self._raise_collections_failed(failed_names)++ if failed_collections and all_results:+ failed_names = [+ collection_names.get(cid) or str(cid)+ for cid in failed_collections+ ]+ logger.warning(+ f"Library search returned partial results: {len(failed_collections)} of "+ f"{len(collections)} collection(s) failed during search: {failed_names}"+ )- if not top_results:- if failed_collections:- # Zero results AND at least one collection errored:- # reporting "no results" here would hide the failure.- self._raise_collections_failed(failed_collections)+ if not all_results:logger.info("No results found across any collections")return []- if failed_collections:- # Partial success: results from the healthy collections are- # returned, but tell the user they are incomplete. WARNING- # level streams to the research-log UI via- # frontend_progress_sink.- logger.warning(- f"Library search results are incomplete: "- f"{len(failed_collections)} of {len(collections)} "- f"collection(s) failed: {failed_collections}"- )+ # Sort by pre-scored relevance (descending) and take top limit+ all_results.sort(key=lambda x: x[1], reverse=True)+ top_results = all_results[:limit]- # Convert to the search-result dict format+ # Convert to search result formatresults = []- for r, relevance, metadata in top_results:- # Try both source_id and document_id for compatibility+ for r, relevance, coll_id in top_results:+ metadata = dict(r.metadata or {})+ metadata["collection_id"] = coll_id+ metadata["collection_name"] = (+ collection_names.get(coll_id) or "Unknown"+ )++ # Get document IDdoc_id = (r.source_idor metadata.get("source_id")or metadata.get("document_id"))- # Get title from the result, with fallbacks+ # Get titletitle = (r.document_titleor metadata.get("document_title")@@ -368,18 +468,17 @@ class LibraryRAGSearchEngine(BaseSearchEngine):or (f"Document {doc_id}" if doc_id else "Untitled"))- # Content is rehydrated from the encrypted DB by chunk id+ # Create snippet from contentsnippet = (r.text[:SNIPPET_LENGTH_LONG] + "..."if len(r.text) > SNIPPET_LENGTH_LONGelse r.text)- # Generate URL to document content (targeted chunk if- # chunk_index available). Use the shared helpers so a- # malformed chunk index (UUID, boolean, negative, etc.)- # cannot leak into the citation anchor, and so the- # primary ``r.source_id`` path is sanitised the same way- # as metadata keys.++ # Generate document URL. Validate chunk_idx and sanitise+ # doc_id through the shared helpers so a malformed chunk index+ # (UUID, boolean, negative) or path-traversal doc_id cannot leak+ # into the citation.chunk_idx = extract_chunk_index(metadata)# Authoritative id FIRST. ``extract_document_id`` scans its# first mapping before any later one, so passing ``metadata``diff --git a/tests/web/routers/test_rag_indexing_pipeline.py b/tests/web/routers/test_rag_indexing_pipeline.pyindex 7a766a7760..32bc4429be 100644--- a/tests/web/routers/test_rag_indexing_pipeline.py+++ b/tests/web/routers/test_rag_indexing_pipeline.py@@ -537,22 +537,19 @@ def _sole_index_row(session, collection_id):class TestChunkingSettingsAtQueryTime:"""``splitter_type``/``text_separators`` are part of the index identity(``_get_index_hash``) and are stamped onto the ``RAGIndex`` row at- indexing time, but neither search engine reads them back when it- rebuilds the query-time service -- issue #5745."""+ indexing time, and both search engines read them back when rebuilding+ the query-time service -- issue #5745."""@pytest.mark.parametrize("engine",[_COLLECTION_ENGINE, _LIBRARY_ENGINE],ids=["collection", "library"],)- def test_search_engines_omit_the_chunking_kwargs(self, engine):- """The static half of the mechanism, and a tripwire: when #5745 is- fixed this fails and the xfails below must be retired in the same- change."""+ def test_search_engines_include_the_chunking_kwargs(self, engine):+ """The identity-affecting kwargs the engine threads through --+ verifies that chunking settings are forwarded to LibraryRAGService."""kwargs = _library_rag_service_call_kwargs(engine)- # The identity-affecting kwargs the engine DOES thread through --- # proves the extraction works and the omission is specific.assert {"embedding_model","embedding_provider",@@ -561,9 +558,9 @@ class TestChunkingSettingsAtQueryTime:"distance_metric","normalize_vectors","index_type",+ "splitter_type",+ "text_separators",} <= kwargs- assert "splitter_type" not in kwargs- assert "text_separators" not in kwargsdef test_indexing_persists_the_configured_separators(self, world, indexed_with_nondefault_separators@@ -591,25 +588,6 @@ class TestChunkingSettingsAtQueryTime:hits = svc.search(QUERY, collection_id, 5)assert any(SECRET in h.text for h in hits)- @pytest.mark.xfail(- strict=True,- reason=(- "#5745 mechanism. _get_index_hash covers text_separators and "- "splitter_type, but CollectionSearchEngine/LibrarySearchEngine "- "rebuild the query-time LibraryRAGService from the RAGIndex row "- "WITHOUT them, so the service falls back to the "- "["- "'\\n\\n', '\\n', '. ', ' ', ''] default. The hash differs, "- "_find_matching_rag_index's fallback also rejects the row "- "(_matches_current_index_configuration compares separators), and "- "_get_or_create_rag_index creates a fresh EMPTY index instead. "- "The engines' get_rag_stats pre-check is config-independent so it "- "passes, and the user sees a silently empty result set. Fix by "- "threading rag_index.splitter_type and rag_index.text_separators "- "into both LibraryRAGService(...) call sites, exactly as "- "distance_metric/normalize_vectors/index_type already are."- ),- )def test_query_time_reconstruction_finds_the_content(self, world, indexed_with_nondefault_separators):@@ -627,22 +605,6 @@ class TestChunkingSettingsAtQueryTime:"unreachable from the search engines' query-time service")- @pytest.mark.xfail(- strict=True,- reason=(- "The mismatched query-time search does not merely return "- "nothing: _get_or_create_rag_index_in_session sets "- "mutated = created or (...), so the row it CREATES is promoted "- "to is_current and the real index is demoted -- on a read path, "- "despite promote_current=False. Both engines select the "- "collection's index with filter_by(is_current=True), so after "- "one such search they resolve the empty index permanently and "- "the collection stays empty even for correctly-configured "- "callers. Fix by not promoting an index created on a read path "- "(gate the promotion on promote_current), in addition to "- "threading the chunking settings through."- ),- )def test_query_time_mismatch_does_not_disturb_the_real_index(self, world, indexed_with_nondefault_separators):diff --git a/tests/web_search_engines/engines/test_local_search_coverage.py b/tests/web_search_engines/engines/test_local_search_coverage.pyindex 9bc93f19d1..fa8c99ce96 100644--- a/tests/web_search_engines/engines/test_local_search_coverage.py+++ b/tests/web_search_engines/engines/test_local_search_coverage.py@@ -66,6 +66,11 @@ def _mock_rag_index(idx.embedding_model_type = Mock(value=embedding_provider)idx.chunk_size = chunk_sizeidx.chunk_overlap = chunk_overlap+ idx.splitter_type = "recursive"+ idx.text_separators = None+ idx.distance_metric = "cosine"+ idx.normalize_vectors = True+ idx.index_type = "flat"return idx@@ -796,3 +801,5 @@ class TestResultStructure:assert r["source_type"] == "library"assert r["link"] == r["url"]assert isinstance(r["relevance_score"], float)+ assert r["metadata"]["collection_id"] == 1+ assert r["metadata"]["collection_name"] == "C"diff --git a/tests/web_search_engines/engines/test_search_engine_collection.py b/tests/web_search_engines/engines/test_search_engine_collection.pyindex 33837edbe0..671c3890b1 100644--- a/tests/web_search_engines/engines/test_search_engine_collection.py+++ b/tests/web_search_engines/engines/test_search_engine_collection.py@@ -124,19 +124,24 @@ class TestLoadCollectionEmbeddingSettings:# Should not raise, just log warningassert engine.username is None- def test_load_settings_with_rag_index(self):- """Load settings from RAG index."""+ def test_load_settings_with_collection(self):+ """Load settings directly from Collection table."""from local_deep_research.web_search_engines.engines.search_engine_collection import (CollectionSearchEngine,)settings = {"_username": "testuser"}- mock_rag_index = Mock()- mock_rag_index.embedding_model = "custom-model"- mock_rag_index.embedding_model_type = Mock(value="ollama")- mock_rag_index.chunk_size = 2000- mock_rag_index.chunk_overlap = 400+ mock_collection = Mock()+ mock_collection.embedding_model = "custom-model"+ mock_collection.embedding_model_type = Mock(value="ollama")+ mock_collection.chunk_size = 2000+ mock_collection.chunk_overlap = 400+ mock_collection.splitter_type = "recursive"+ mock_collection.text_separators = ["\n\n", "\n"]+ mock_collection.distance_metric = "cosine"+ mock_collection.normalize_vectors = True+ mock_collection.index_type = "flat"with patch("local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",@@ -149,7 +154,7 @@ class TestLoadCollectionEmbeddingSettings:with patch("local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session") as mock_session:- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collectionengine = CollectionSearchEngine(collection_id="abc123",@@ -162,8 +167,95 @@ class TestLoadCollectionEmbeddingSettings:assert engine.chunk_size == 2000assert engine.chunk_overlap == 400- def test_load_settings_no_rag_index(self):- """Load settings handles missing RAG index."""+ def test_load_settings_zero_chunk_overlap(self):+ """Zero chunk_overlap must be preserved as 0, not overridden by default."""+ from local_deep_research.web_search_engines.engines.search_engine_collection import (+ CollectionSearchEngine,+ )++ settings = {"_username": "testuser"}++ mock_collection = Mock()+ mock_collection.embedding_model = "custom-model"+ mock_collection.embedding_model_type = Mock(value="openai")+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 0+ mock_collection.splitter_type = "token"+ mock_collection.text_separators = ["\n\n\n", "\n"]+ mock_collection.distance_metric = "cosine"+ mock_collection.normalize_vectors = True+ mock_collection.index_type = "flat"++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",+ return_value=None,+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",+ return_value="http://localhost:5000",+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"+ ) as mock_session:+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection++ engine = CollectionSearchEngine(+ collection_id="abc123",+ collection_name="Test Collection",+ settings_snapshot=settings,+ )++ assert engine.chunk_overlap == 0+ assert engine.splitter_type == "token"+ assert engine.text_separators == ["\n\n\n", "\n"]++ def test_load_settings_nullable_embedding_model_type(self):+ """When collection.embedding_model_type is None, embedding_provider is not set to 'None'."""+ from local_deep_research.web_search_engines.engines.search_engine_collection import (+ CollectionSearchEngine,+ )++ settings = {+ "_username": "testuser",+ "local_search_embedding_provider": "sentence_transformers",+ }++ mock_collection = Mock()+ mock_collection.embedding_model = "custom-model"+ mock_collection.embedding_model_type = None+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 200+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.distance_metric = None+ mock_collection.normalize_vectors = None+ mock_collection.index_type = None++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",+ return_value="sentence_transformers",+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",+ return_value="http://localhost:5000",+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"+ ) as mock_session:+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection++ engine = CollectionSearchEngine(+ collection_id="abc123",+ collection_name="Test Collection",+ settings_snapshot=settings,+ )++ assert engine.embedding_model == "custom-model"+ assert engine.embedding_provider != "None"+ assert engine.embedding_provider == "sentence_transformers"++ def test_load_settings_no_collection(self):+ """Load settings handles missing collection."""from local_deep_research.web_search_engines.engines.search_engine_collection import (CollectionSearchEngine,)@@ -251,8 +343,8 @@ class TestSearch:assert results == []- def test_search_no_rag_index(self):- """Search returns empty when no RAG index."""+ def test_search_no_collection(self):+ """Search returns empty when collection not found."""from local_deep_research.web_search_engines.engines.search_engine_collection import (CollectionSearchEngine,)@@ -289,14 +381,18 @@ class TestSearch:settings = {"_username": "testuser"}- mock_rag_index = Mock()- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"- mock_rag_index.embedding_model_type = Mock(+ mock_collection = Mock()+ mock_collection.embedding_model = "all-MiniLM-L6-v2"+ mock_collection.embedding_model_type = Mock(value="sentence_transformers")- mock_rag_index.chunk_size = 1000- mock_rag_index.chunk_overlap = 200-+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 200+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.distance_metric = None+ mock_collection.normalize_vectors = None+ mock_collection.index_type = Nonewith patch("local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",return_value=None,@@ -308,7 +404,7 @@ class TestSearch:with patch("local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session") as mock_session:- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collectionwith patch("local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"@@ -342,13 +438,18 @@ class TestSearch:settings = {"_username": "testuser"}- mock_rag_index = Mock()- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"- mock_rag_index.embedding_model_type = Mock(+ mock_collection = Mock()+ mock_collection.embedding_model = "all-MiniLM-L6-v2"+ mock_collection.embedding_model_type = Mock(value="sentence_transformers")- mock_rag_index.chunk_size = 1000- mock_rag_index.chunk_overlap = 200+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 200+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.distance_metric = None+ mock_collection.normalize_vectors = None+ mock_collection.index_type = Nonemock_doc = Mock()mock_doc.page_content = "This is the document content for testing."@@ -368,7 +469,7 @@ class TestSearch:with patch("local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session") as mock_session:- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collectionwith patch("local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"@@ -415,6 +516,171 @@ class TestSearch:assert results[0]["source_type"] == "library"assert results[0]["url"] == "/library/document/123"+ def test_search_forwards_all_rag_parameters(self):+ """Search forwards chunk_overlap=0, splitter_type, text_separators, distance_metric, index_type."""+ from local_deep_research.web_search_engines.engines.search_engine_collection import (+ CollectionSearchEngine,+ )++ settings = {"_username": "testuser"}++ mock_collection = Mock()+ mock_collection.embedding_model = "nomic-embed"+ mock_collection.embedding_model_type = Mock(value="openai")+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 0+ mock_collection.splitter_type = "token"+ mock_collection.text_separators = ["\n\n\n", "\n\n", "\n"]+ mock_collection.distance_metric = "cosine"+ mock_collection.normalize_vectors = True+ mock_collection.index_type = "flat"++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",+ return_value=None,+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",+ return_value="http://localhost:5000",+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"+ ) as mock_session:+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"+ ) as mock_rag_service:+ mock_rag_instance = Mock()+ mock_rag_instance.get_rag_stats.return_value = {+ "indexed_documents": 1+ }+ mock_rag_instance.search.return_value = [+ SearchResult(+ chunk_id=1,+ text="content",+ distance=0.8,+ metric="cosine",+ metadata={"source_id": "123"},+ document_title="Doc",+ source_id=None,+ source_type=None,+ )+ ]+ mock_rag_service.return_value.__enter__.return_value = (+ mock_rag_instance+ )+ mock_rag_service.return_value.__exit__.return_value = (+ None+ )++ engine = CollectionSearchEngine(+ collection_id="abc123",+ collection_name="Test Collection",+ settings_snapshot=settings,+ )+ results = engine.search("test query")++ assert len(results) == 1+ # Verify LibraryRAGService instantiation kwargs+ call_kwargs = mock_rag_service.call_args.kwargs+ assert call_kwargs["embedding_model"] == "nomic-embed"+ assert call_kwargs["embedding_provider"] == "openai"+ assert call_kwargs["chunk_size"] == 1000+ assert call_kwargs["chunk_overlap"] == 0+ assert call_kwargs["splitter_type"] == "token"+ assert call_kwargs["text_separators"] == [+ "\n\n\n",+ "\n\n",+ "\n",+ ]+ assert call_kwargs["distance_metric"] == "cosine"+ assert call_kwargs["normalize_vectors"] is True+ assert call_kwargs["index_type"] == "flat"++ def test_search_relevance_metric_mapping(self):+ """Cosine and dot_product use (d+1)/2 clamped, L2 uses 1/(1+d)."""+ from local_deep_research.web_search_engines.engines.search_engine_collection import (+ CollectionSearchEngine,+ )++ settings = {"_username": "testuser"}++ mock_collection = Mock()+ mock_collection.embedding_model = "test-model"+ mock_collection.embedding_model_type = Mock(value="openai")+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 0+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.distance_metric = None+ mock_collection.normalize_vectors = None+ mock_collection.index_type = None+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",+ return_value=None,+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",+ return_value="http://localhost:5000",+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"+ ) as mock_session:+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"+ ) as mock_rag_service:+ mock_rag_instance = Mock()+ mock_rag_instance.get_rag_stats.return_value = {+ "indexed_documents": 2+ }+ mock_rag_instance.search.return_value = [+ SearchResult(+ chunk_id=1,+ text="cosine hit",+ distance=0.6,+ metric="cosine",+ metadata={"source_id": "1"},+ document_title="Doc 1",+ source_id=None,+ source_type=None,+ ),+ SearchResult(+ chunk_id=2,+ text="l2 hit",+ distance=0.5,+ metric="l2",+ metadata={"source_id": "2"},+ document_title="Doc 2",+ source_id=None,+ source_type=None,+ ),+ ]+ mock_rag_service.return_value.__enter__.return_value = (+ mock_rag_instance+ )+ mock_rag_service.return_value.__exit__.return_value = (+ None+ )++ engine = CollectionSearchEngine(+ collection_id="abc123",+ collection_name="Test Collection",+ settings_snapshot=settings,+ )+ results = engine.search("test query")++ assert len(results) == 2+ # Cosine relevance: (0.6 + 1.0) / 2.0 = 0.8+ assert abs(results[0]["relevance_score"] - 0.8) < 1e-6+ # L2 relevance: 1.0 / (1.0 + 0.5) = 0.666666...+ assert (+ abs(results[1]["relevance_score"] - (1.0 / 1.5))+ < 1e-6+ )+def test_search_empty_vector_results(self):"""Search handles empty vector search results."""from local_deep_research.web_search_engines.engines.search_engine_collection import (@@ -423,14 +689,18 @@ class TestSearch:settings = {"_username": "testuser"}- mock_rag_index = Mock()- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"- mock_rag_index.embedding_model_type = Mock(+ mock_collection = Mock()+ mock_collection.embedding_model = "all-MiniLM-L6-v2"+ mock_collection.embedding_model_type = Mock(value="sentence_transformers")- mock_rag_index.chunk_size = 1000- mock_rag_index.chunk_overlap = 200-+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 200+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.distance_metric = None+ mock_collection.normalize_vectors = None+ mock_collection.index_type = Nonewith patch("local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",return_value=None,@@ -442,7 +712,7 @@ class TestSearch:with patch("local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session") as mock_session:- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collectionwith patch("local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"@@ -479,14 +749,18 @@ class TestSearch:settings = {"_username": "testuser"}- mock_rag_index = Mock()- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"- mock_rag_index.embedding_model_type = Mock(+ mock_collection = Mock()+ mock_collection.embedding_model = "all-MiniLM-L6-v2"+ mock_collection.embedding_model_type = Mock(value="sentence_transformers")- mock_rag_index.chunk_size = 1000- mock_rag_index.chunk_overlap = 200-+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 200+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.distance_metric = None+ mock_collection.normalize_vectors = None+ mock_collection.index_type = Nonewith patch("local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",return_value=None,@@ -498,7 +772,7 @@ class TestSearch:with patch("local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session") as mock_session:- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collectionwith patch("local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService",@@ -514,6 +788,69 @@ class TestSearch:):engine.search("test query")+ def test_search_nullable_embedding_model_type(self):+ """When collection.embedding_model_type is None, search uses default provider, not 'None'."""+ from local_deep_research.web_search_engines.engines.search_engine_collection import (+ CollectionSearchEngine,+ )++ settings = {+ "_username": "testuser",+ "local_search_embedding_provider": "sentence_transformers",+ }++ mock_collection = Mock()+ mock_collection.embedding_model = "custom-model"+ mock_collection.embedding_model_type = None+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 200+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.distance_metric = None+ mock_collection.normalize_vectors = None+ mock_collection.index_type = None++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",+ return_value="sentence_transformers",+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",+ return_value="http://localhost:5000",+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"+ ) as mock_session:+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"+ ) as mock_rag_service:+ mock_rag_instance = Mock()+ mock_rag_instance.get_rag_stats.return_value = {+ "indexed_documents": 0+ }+ mock_rag_service.return_value.__enter__.return_value = (+ mock_rag_instance+ )+ mock_rag_service.return_value.__exit__.return_value = (+ None+ )++ engine = CollectionSearchEngine(+ collection_id="abc123",+ collection_name="Test Collection",+ settings_snapshot=settings,+ )+ engine.search("test query")++ call_kwargs = mock_rag_service.call_args.kwargs+ assert (+ call_kwargs["embedding_provider"]+ == "sentence_transformers"+ )+ assert call_kwargs["embedding_provider"] != "None"+class TestGetDocumentUrl:"""Tests for _get_document_url method."""diff --git a/tests/web_search_engines/engines/test_search_engine_library.py b/tests/web_search_engines/engines/test_search_engine_library.pyindex 931cc24e46..8c3a3e8aec 100644--- a/tests/web_search_engines/engines/test_search_engine_library.py+++ b/tests/web_search_engines/engines/test_search_engine_library.py@@ -189,15 +189,20 @@ class TestSearch:with patch("local_deep_research.web_search_engines.engines.search_engine_library.get_user_db_session") as mock_session:- mock_rag_index = Mock()- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"- mock_rag_index.embedding_model_type = Mock(+ mock_collection = Mock()+ mock_collection.embedding_model = "all-MiniLM-L6-v2"+ mock_collection.embedding_model_type = Mock(value="sentence_transformers")- mock_rag_index.chunk_size = 1000- mock_rag_index.chunk_overlap = 200+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 200+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.normalize_vectors = True+ mock_collection.distance_metric = "l2"+ mock_collection.index_type = "flat"- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collectionwith patch("local_deep_research.web_search_engines.engines.search_engine_library.LibraryRAGService"@@ -234,6 +239,171 @@ class TestSearch:assert results[0]["source_type"] == "library"assert "/library/document/123" in results[0]["url"]+ def test_search_forwards_zero_overlap_and_splitter_settings(self):+ """Search forwards chunk_overlap=0, splitter_type, and text_separators to LibraryRAGService."""+ from local_deep_research.web_search_engines.engines.search_engine_library import (+ LibraryRAGSearchEngine,+ )++ settings = {"_username": "testuser"}++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",+ return_value=None,+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",+ return_value="http://localhost:5000",+ ):+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryService"+ ) as mock_lib_service:+ mock_lib_service.return_value.get_all_collections.return_value = [+ {"id": "col_1", "name": "Zero Overlap Collection"}+ ]++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_user_db_session"+ ) as mock_session:+ mock_collection = Mock()+ mock_collection.embedding_model = "nomic-embed"+ mock_collection.embedding_model_type = Mock(+ value="openai"+ )+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 0+ mock_collection.splitter_type = "token"+ mock_collection.text_separators = [+ "\n\n\n",+ "\n\n",+ "\n",+ ]+ mock_collection.distance_metric = "cosine"+ mock_collection.normalize_vectors = True+ mock_collection.index_type = "flat"++ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryRAGService"+ ) as mock_rag_service:+ mock_rag_instance = Mock()+ mock_rag_instance.get_rag_stats.return_value = {+ "indexed_documents": 1+ }+ mock_rag_instance.search.return_value = [+ SearchResult(+ chunk_id=1,+ text="test text",+ distance=0.8,+ metric="cosine",+ metadata={"source_id": "doc1"},+ document_title="Title",+ source_id=None,+ source_type=None,+ )+ ]+ mock_rag_service.return_value.__enter__.return_value = mock_rag_instance+ mock_rag_service.return_value.__exit__.return_value = None++ engine = LibraryRAGSearchEngine(+ settings_snapshot=settings+ )+ results = engine.search("query")++ assert len(results) == 1+ call_kwargs = mock_rag_service.call_args.kwargs+ assert call_kwargs["chunk_overlap"] == 0+ assert call_kwargs["splitter_type"] == "token"+ assert call_kwargs["text_separators"] == [+ "\n\n\n",+ "\n\n",+ "\n",+ ]+ assert call_kwargs["distance_metric"] == "cosine"+ assert call_kwargs["normalize_vectors"] is True+ assert call_kwargs["index_type"] == "flat"++ def test_search_nullable_embedding_model_type_uses_default(self):+ """When collection.embedding_model_type is NULL, search falls back to default provider, not 'None'."""+ from local_deep_research.web_search_engines.engines.search_engine_library import (+ LibraryRAGSearchEngine,+ )++ settings = {+ "_username": "testuser",+ "local_search_embedding_provider": "sentence_transformers",+ }++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryService"+ ) as mock_lib_service:+ mock_lib_service.return_value.get_all_collections.return_value = [+ {"id": "col1", "name": "Collection 1"}+ ]++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_user_db_session"+ ) as mock_session:+ mock_collection = Mock()+ mock_collection.embedding_model = "all-MiniLM-L6-v2"+ mock_collection.embedding_model_type = None+ mock_collection.chunk_size = 1000+ mock_collection.chunk_overlap = 200+ mock_collection.splitter_type = None+ mock_collection.text_separators = None+ mock_collection.distance_metric = None+ mock_collection.normalize_vectors = None+ mock_collection.index_type = None++ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryRAGService"+ ) as mock_rag_service:+ mock_rag_instance = Mock()+ mock_rag_instance.get_rag_stats.return_value = {+ "indexed_documents": 0+ }+ mock_rag_service.return_value.__enter__.return_value = (+ mock_rag_instance+ )+ mock_rag_service.return_value.__exit__.return_value = None++ engine = LibraryRAGSearchEngine(settings_snapshot=settings)+ engine.search("test query")++ call_kwargs = mock_rag_service.call_args.kwargs+ assert (+ call_kwargs["embedding_provider"]+ == "sentence_transformers"+ )+ assert call_kwargs["embedding_provider"] != "None"++ def test_all_collections_fail_passes_collection_names(self):+ """When all collections fail, _raise_collections_failed receives collection names, not raw UUIDs."""+ import pytest+ from local_deep_research.web_search_engines.engines.search_engine_library import (+ LibraryRAGSearchEngine,+ )++ settings = {"_username": "testuser"}+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryService"+ ) as mock_lib:+ mock_lib.return_value.get_all_collections.return_value = [+ {"id": "uuid-1234", "name": "Important Docs"}+ ]+ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_library.get_user_db_session",+ side_effect=RuntimeError("db error"),+ ):+ engine = LibraryRAGSearchEngine(settings_snapshot=settings)+ with pytest.raises(RuntimeError) as exc_info:+ engine.search("test query")++ assert "Important Docs" in str(exc_info.value)+def test_search_exception(self):"""Search re-raises so failures are not mistaken for no results."""import pytest@@ -596,3 +766,18 @@ class TestClassAttributes:)assert LibraryRAGSearchEngine.is_local is True++ def test_egress_attributes(self):+ """LibraryRAGSearchEngine has egress sensitivity and exposure configured."""+ from local_deep_research.web_search_engines.engines.search_engine_library import (+ LibraryRAGSearchEngine,+ )+ from local_deep_research.web_search_engines.search_engine_base import (+ Exposure,+ Sensitivity,+ )++ assert (+ LibraryRAGSearchEngine.egress_sensitivity == Sensitivity.SENSITIVE+ )+ assert LibraryRAGSearchEngine.egress_exposure == Exposure.CONTAINEDdiff --git a/tests/web_search_engines/engines/test_search_engine_library_coverage.py b/tests/web_search_engines/engines/test_search_engine_library_coverage.pyindex e20d89bc6d..a6947f6c6e 100644--- a/tests/web_search_engines/engines/test_search_engine_library_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_library_coverage.py@@ -1,57 +1,57 @@"""-Tests for LibraryRAGSearchEngine (search_engine_library.py, 0% coverage).--Covers:-- __init__: initialization with/without username-- search: no username, no collections, exception handling-- _get_previews: delegates to search-- _get_full_content: no username, no doc_id, success path-- close: no-op+Coverage tests for search_engine_library.py error handling and edge cases."""from unittest.mock import MagicMock, Mock, patchfrom local_deep_research.vector_stores.facade import SearchResult-MODULE = "local_deep_research.web_search_engines.engines.search_engine_library"-def _make_engine(username="testuser", settings_snapshot=None):- """Create a LibraryRAGSearchEngine with mocked dependencies."""+def _make_engine(username="test_user", settings_snapshot=None):from local_deep_research.web_search_engines.engines.search_engine_library import (LibraryRAGSearchEngine,)- snapshot = settings_snapshot or {"_username": username}- engine = LibraryRAGSearchEngine(- llm=MagicMock(),- max_results=10,- settings_snapshot=snapshot,- )- return engine+ snap = {"_username": username} if username else {}+ if settings_snapshot:+ snap.update(settings_snapshot)+ return LibraryRAGSearchEngine(settings_snapshot=snap if snap else None)class TestLibraryRAGSearchEngineInit:+ """Tests for LibraryRAGSearchEngine initialization edge cases."""+def test_init_with_username(self):- engine = _make_engine(username="testuser")- assert engine.username == "testuser"+ engine = _make_engine("custom_user")+ assert engine.username == "custom_user"assert engine.is_local is Truedef test_init_without_username(self):- engine = _make_engine(settings_snapshot={"_username": None})+ engine = _make_engine(username=None)assert engine.username is Nonedef test_init_reads_embedding_settings(self):- engine = _make_engine(username="user1")- # Defaults should be set- assert engine.embedding_model is not None- assert engine.chunk_size is not None+ snap = {+ "_username": "test_user",+ "local_search_embedding_model": "custom-model",+ "local_search_embedding_provider": "openai",+ "local_search_chunk_size": 500,+ "local_search_chunk_overlap": 100,+ }+ engine = _make_engine(settings_snapshot=snap)+ assert engine.embedding_model == "custom-model"+ assert engine.embedding_provider == "openai"+ assert engine.chunk_size == 500+ assert engine.chunk_overlap == 100class TestSearch:+ """Tests for LibraryRAGSearchEngine.search error handling."""+def test_search_no_username_returns_empty(self):- engine = _make_engine(settings_snapshot={"_username": None})+ engine = _make_engine(username=None)result = engine.search("test query")assert result == []@@ -59,32 +59,33 @@ class TestSearch:engine = _make_engine()mock_service = MagicMock()mock_service.get_all_collections.return_value = []+with patch(f"{MODULE}.LibraryService", return_value=mock_service):result = engine.search("test query")assert result == []def test_search_exception_propagates(self):- """A failed search raises instead of masquerading as no results."""+ """Top-level errors bubble up so callers see failure, not empty results."""import pytestengine = _make_engine()with patch(- f"{MODULE}.LibraryService", side_effect=RuntimeError("fail")+ f"{MODULE}.LibraryService",+ side_effect=Exception("Database error"),):- with pytest.raises(RuntimeError, match="fail"):+ with pytest.raises(Exception, match="Database error"):engine.search("test query")def test_search_collection_no_rag_index_skips(self):engine = _make_engine()mock_service = MagicMock()mock_service.get_all_collections.return_value = [- {"id": "col1", "name": "Test Collection"}+ {"id": "col1", "name": "Collection 1"}]mock_session = MagicMock()mock_session.__enter__ = Mock(return_value=mock_session)mock_session.__exit__ = Mock(return_value=False)- # RAGIndex query returns Nonemock_session.query.return_value.filter_by.return_value.first.return_value = Nonewith patch(f"{MODULE}.LibraryService", return_value=mock_service):@@ -132,6 +133,11 @@ class TestSearch:)mock_rag_index.chunk_size = 1000mock_rag_index.chunk_overlap = 200+ mock_rag_index.splitter_type = "recursive"+ mock_rag_index.text_separators = None+ mock_rag_index.distance_metric = "cosine"+ mock_rag_index.normalize_vectors = True+ mock_rag_index.index_type = "flat"mock_session.query.return_value.filter_by.return_value.first.return_value = mock_rag_index# col1 fails when its RAG service is created; col2 succeeds@@ -198,7 +204,6 @@ class TestSearch:mock_session = MagicMock()mock_session.__enter__ = Mock(return_value=mock_session)mock_session.__exit__ = Mock(return_value=False)-mock_rag_index = MagicMock()mock_rag_index.embedding_model = "all-MiniLM-L6-v2"mock_rag_index.embedding_model_type = MagicMock(@@ -218,7 +223,8 @@ class TestSearch:f"{MODULE}.get_user_db_session", return_value=mock_session):with patch(- f"{MODULE}.LibraryRAGService", return_value=mock_rag_service+ f"{MODULE}.LibraryRAGService",+ return_value=mock_rag_service,):result = engine.search("test query")@@ -226,46 +232,48 @@ class TestSearch:class TestGetPreviews:+ """Tests for LibraryRAGSearchEngine._get_previews delegation."""+def test_delegates_to_search(self):engine = _make_engine()with patch.object(- engine, "search", return_value=[{"title": "test"}]+ engine, "search", return_value=[{"title": "Doc"}]) as mock_search:- result = engine._get_previews("test query", limit=5)- mock_search.assert_called_once_with("test query", 5, None, None)- assert result == [{"title": "test"}]+ result = engine._get_previews("query", limit=5)+ mock_search.assert_called_once_with("query", 5, None, None)+ assert result == [{"title": "Doc"}]class TestGetFullContent:+ """Tests for LibraryRAGSearchEngine._get_full_content."""+def test_items_without_document_id_returned_unchanged(self):- """Items lacking metadata.document_id are skipped and returned as-is."""engine = _make_engine()- items = [{"title": "test", "snippet": "content"}]-+ items = [{"title": "Doc 1", "metadata": {}}]result = engine._get_full_content(items)-assert result == itemsdef test_no_username_returns_items(self):- engine = _make_engine(settings_snapshot={"_username": None})- items = [{"title": "test"}]+ engine = _make_engine(username=None)+ items = [{"title": "Doc 1", "metadata": {"document_id": "doc1"}}]result = engine._get_full_content(items)assert result == itemsdef test_exception_returns_items(self):engine = _make_engine()- items = [{"title": "test", "metadata": {"document_id": "doc1"}}]+ items = [{"title": "Doc 1", "metadata": {"document_id": "doc1"}}]with patch(f"{MODULE}.get_user_db_session",- side_effect=RuntimeError("fail"),+ side_effect=Exception("DB error"),):result = engine._get_full_content(items)-assert result == itemsclass TestClose:+ """Tests for LibraryRAGSearchEngine.close."""+def test_close_is_noop(self):engine = _make_engine()- engine.close() # Should not raise+ engine.close() # should not raisediff --git a/tests/web_search_engines/test_engine_success_path.py b/tests/web_search_engines/test_engine_success_path.pyindex 55f3cb4da5..b1670a6959 100644--- a/tests/web_search_engines/test_engine_success_path.py+++ b/tests/web_search_engines/test_engine_success_path.py@@ -1408,6 +1408,8 @@ def _fake_rag_index_session(module):embedding_model_type=SimpleNamespace(value="sentence_transformers"),chunk_size=1000,chunk_overlap=200,+ splitter_type="recursive",+ text_separators=None,normalize_vectors=True,distance_metric="cosine",index_type="flat",