Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(library): load embedding settings from Collection record in search engines (#6097)

S4be21d47dca7
diff --git a/changelog.d/6097.bugfix.md b/changelog.d/6097.bugfix.md
new file mode 100644
index 0000000000..fd4980c069
--- /dev/null
+++ b/changelog.d/6097.bugfix.md
@@ -0,0 +1 @@
+`CollectionSearchEngine` and `LibraryRAGSearchEngine` now load frozen embedding and text splitter configuration directly from the `Collection` record rather than the `RAGIndex` table, ensuring search remains reliable even if indexing was interrupted and resolving text-splitter mismatches across collection queries.
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_collection.py b/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
index 0b403d1db7..be4320ebab 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_collection.py
@@ -8,9 +8,17 @@ from typing import List, Dict, Any, Optional
from ...security.secure_logging import logger
from .search_engine_library import LibraryRAGSearchEngine
-from ...constants import SNIPPET_LENGTH_LONG
+from ...constants import (
+ DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP,
+ DEFAULT_LOCAL_SEARCH_CHUNK_SIZE,
+ DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC,
+ DEFAULT_LOCAL_SEARCH_INDEX_TYPE,
+ DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
+ DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE,
+ SNIPPET_LENGTH_LONG,
+)
from ...research_library.services.library_rag_service import LibraryRAGService
-from ...database.models.library import RAGIndex
+from ...database.models.library import Collection
from ...database.session_context import get_user_db_session
from ...utilities.chunk_anchor import extract_chunk_index, extract_document_id
from ...utilities.type_utils import to_bool
@@ -72,8 +80,8 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):
def _load_collection_embedding_settings(self):
"""
- Load embedding settings from the collection's RAG index.
- Uses the same embedding model that was used during indexing.
+ Load frozen embedding settings directly from the Collection record.
+ Uses the same embedding model that was configured when the collection was created.
"""
if not self.username:
logger.warning("Cannot load collection settings without username")
@@ -81,29 +89,46 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):
try:
with get_user_db_session(self.username) as db_session:
- # Get RAG index for this collection
- rag_index = (
- db_session.query(RAGIndex)
- .filter_by(
- collection_name=self.collection_key,
- is_current=True,
- )
+ collection = (
+ db_session.query(Collection)
+ .filter_by(id=self.collection_id)
.first()
)
- if not rag_index:
+ if not collection or not collection.embedding_model:
logger.warning(
- f"No RAG index found for collection {self.collection_id}"
+ f"No stored embedding settings found for collection {self.collection_id}"
)
return
- # Use embedding settings from the RAG index
- self.embedding_model = rag_index.embedding_model
- self.embedding_provider = rag_index.embedding_model_type.value
- self.chunk_size = rag_index.chunk_size or self.chunk_size
- self.chunk_overlap = (
- rag_index.chunk_overlap or self.chunk_overlap
- )
+ # Use embedding settings directly from the Collection
+ self.embedding_model = collection.embedding_model
+ provider = collection.embedding_model_type
+ if provider is not None:
+ self.embedding_provider = (
+ provider.value
+ if hasattr(provider, "value")
+ else str(provider)
+ )
+ coll_chunk_size = getattr(collection, "chunk_size", None)
+ if coll_chunk_size is not None:
+ self.chunk_size = int(coll_chunk_size)
+ coll_chunk_overlap = getattr(collection, "chunk_overlap", None)
+ if coll_chunk_overlap is not None:
+ self.chunk_overlap = int(coll_chunk_overlap)
+ if getattr(collection, "splitter_type", None) is not None:
+ self.splitter_type = collection.splitter_type
+ if getattr(collection, "text_separators", None) is not None:
+ self.text_separators = collection.text_separators
+ if getattr(collection, "distance_metric", None) is not None:
+ self.distance_metric = collection.distance_metric
+ if getattr(collection, "normalize_vectors", None) is not None:
+ self.normalize_vectors = to_bool(
+ collection.normalize_vectors,
+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
+ )
+ if getattr(collection, "index_type", None) is not None:
+ self.index_type = collection.index_type
logger.info(
f"Collection '{self.collection_name}' using embedding: "
@@ -148,50 +173,92 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):
self._verify_egress_scope()
try:
- # Get RAG index info for this collection
+ # Get frozen embedding settings directly from Collection record
with get_user_db_session(self.username) as db_session:
- rag_index = (
- db_session.query(RAGIndex)
- .filter_by(
- collection_name=self.collection_key,
- is_current=True,
- )
+ collection = (
+ db_session.query(Collection)
+ .filter_by(id=self.collection_id)
.first()
)
- if not rag_index:
+ if not collection or not collection.embedding_model:
logger.info(
- f"No RAG index for collection '{self.collection_name}'"
+ f"No embedding settings found for collection '{self.collection_name}'"
)
return []
- # Get embedding settings from RAG index
- embedding_model = rag_index.embedding_model
- embedding_provider = rag_index.embedding_model_type.value
- chunk_size = rag_index.chunk_size or self.chunk_size
- chunk_overlap = rag_index.chunk_overlap or self.chunk_overlap
- # Thread the stored normalization flag through so the index is
- # queried with the same L2 normalization it was BUILT with.
- # Otherwise LibraryRAGService defaults normalize_vectors=True,
- # corrupting similarity rankings for collections indexed with
- # local_search_normalize_vectors=False. NULL → True mirrors the
- # to_bool(..., default=True) convention used elsewhere.
- normalize_vectors = to_bool(
- rag_index.normalize_vectors, default=True
+ # Get embedding settings directly from Collection
+ embedding_model = collection.embedding_model
+ provider = collection.embedding_model_type
+ if provider is not None:
+ embedding_provider = (
+ provider.value
+ if hasattr(provider, "value")
+ else str(provider)
+ )
+ else:
+ embedding_provider = self.embedding_provider
+ coll_chunk_size = getattr(collection, "chunk_size", None)
+ chunk_size = (
+ int(coll_chunk_size)
+ if coll_chunk_size is not None
+ else int(self.chunk_size or DEFAULT_LOCAL_SEARCH_CHUNK_SIZE)
+ )
+ coll_chunk_overlap = getattr(collection, "chunk_overlap", None)
+ chunk_overlap = (
+ int(coll_chunk_overlap)
+ if coll_chunk_overlap is not None
+ else int(
+ self.chunk_overlap
+ if self.chunk_overlap is not None
+ else DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP
+ )
+ )
+ splitter_type = (
+ getattr(collection, "splitter_type", None)
+ or getattr(self, "splitter_type", None)
+ or DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE
+ )
+ text_separators = (
+ collection.text_separators
+ if getattr(collection, "text_separators", None) is not None
+ else getattr(self, "text_separators", None)
+ )
+ # Thread the stored normalization flag AND distance
+ # metric through, exactly like LibraryRAGSearchEngine.
+ # Without normalize_vectors, LibraryRAGService defaults
+ # True and would L2-normalize the query against a
+ # raw-vector collection, flipping the top hit. Without
+ # distance_metric, the collection is labeled "cosine"
+ # and l2/dot_product hits get the wrong [0,1] mapping in
+ # the relevance transform below. NULL → prior default.
+ coll_normalize = getattr(collection, "normalize_vectors", None)
+ normalize_vectors = (
+ to_bool(
+ coll_normalize,
+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
+ )
+ if coll_normalize is not None
+ else getattr(
+ self,
+ "normalize_vectors",
+ DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
+ )
+ )
+ distance_metric = (
+ getattr(collection, "distance_metric", None)
+ or getattr(self, "distance_metric", None)
+ or DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC
+ )
+ # Thread the stored index_type too (like metric/
+ # normalize): a legacy NULL-index_type row that is
+ # physically HNSW would otherwise fall back to "flat",
+ # mislabelling the store. NULL → "flat".
+ index_type = (
+ getattr(collection, "index_type", None)
+ or getattr(self, "index_type", None)
+ or DEFAULT_LOCAL_SEARCH_INDEX_TYPE
)
- # Thread the stored distance metric through too: it is what
- # LibraryRAGService stamps onto each SearchResult.metric, which
- # in turn selects the relevance formula below. Defaulting to
- # "cosine" (the LibraryRAGService default) would mislabel an
- # l2/dot_product collection and apply the wrong [0,1] mapping.
- # NULL (legacy rows) → "cosine" preserves prior behavior.
- distance_metric = rag_index.distance_metric or "cosine"
- # Thread the stored index_type too (like metric/normalize): with
- # it left at the "flat" default, _get_vector_index would fall
- # back to flat for a legacy NULL-index_type row that is physically
- # HNSW, mislabelling the store (wrong supports_delete/relevance,
- # and now a load()-time config-mismatch rejection). NULL → "flat".
- index_type = rag_index.index_type or "flat"
# Create RAG service with collection's embedding settings
with LibraryRAGService(
@@ -200,6 +267,8 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):
embedding_provider=embedding_provider,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
+ splitter_type=splitter_type,
+ text_separators=text_separators,
normalize_vectors=normalize_vectors,
distance_metric=distance_metric,
index_type=index_type,
@@ -276,7 +345,9 @@ class CollectionSearchEngine(LibraryRAGSearchEngine):
# Add collection info to metadata
metadata["collection_id"] = self.collection_id
- metadata["collection_name"] = self.collection_name
+ metadata["collection_name"] = (
+ self.collection_name or "Unknown"
+ )
# r.distance's meaning depends on r.metric. Map both to a
# [0, 1] relevance that grows with similarity:
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_library.py b/src/local_deep_research/web_search_engines/engines/search_engine_library.py
index 6350a7bf07..908a88e8f7 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_library.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_library.py
@@ -4,6 +4,7 @@ Library RAG Search Engine
Provides semantic search over the user's personal research library using RAG.
"""
+import json
import os
from typing import List, Dict, Any, Optional
@@ -12,8 +13,14 @@ from ...security.secure_logging import logger
from ...constants import (
DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP,
DEFAULT_LOCAL_SEARCH_CHUNK_SIZE,
+ DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC,
+ DEFAULT_LOCAL_SEARCH_INDEX_TYPE,
DEFAULT_LOCAL_SEARCH_MODEL,
+ DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
DEFAULT_LOCAL_SEARCH_PROVIDER,
+ DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE,
+ DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS,
+ DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON,
SNIPPET_LENGTH_LONG,
)
from ...research_library.services.library_rag_service import LibraryRAGService
@@ -27,7 +34,7 @@ from ...utilities.chunk_anchor import (
)
from ...utilities.llm_utils import get_server_url
from ...utilities.type_utils import to_bool
-from ...database.models.library import RAGIndex, Document
+from ...database.models.library import Collection, Document
from ...research_library.services.pdf_storage_manager import PDFStorageManager
from ...database.session_context import get_user_db_session
from ...config.paths import get_library_directory
@@ -38,7 +45,7 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
Search engine that queries the user's research library using RAG/semantic search.
"""
- # Mark as local RAG engine
+ # Mark as local RAG engine for tool selection and safety checks
is_local = True
# Egress (ADR-0007): a local document store — sensitive data, kept on-box.
@@ -54,7 +61,7 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
**kwargs,
):
"""
- Initialize the Library RAG search engine.
+ Initialize the library RAG search engine.
Args:
llm: Language model for relevance filtering
@@ -111,6 +118,44 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
default=DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP,
settings_snapshot=settings_snapshot,
)
+ self.splitter_type = get_setting_from_snapshot(
+ "local_search_splitter_type",
+ default=DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE,
+ settings_snapshot=settings_snapshot,
+ )
+ raw_text_separators = get_setting_from_snapshot(
+ "local_search_text_separators",
+ default=DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS_JSON,
+ settings_snapshot=settings_snapshot,
+ )
+ if isinstance(raw_text_separators, str):
+ try:
+ self.text_separators = json.loads(raw_text_separators)
+ except (json.JSONDecodeError, TypeError):
+ self.text_separators = DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS
+ elif isinstance(raw_text_separators, list):
+ self.text_separators = raw_text_separators
+ else:
+ self.text_separators = DEFAULT_LOCAL_SEARCH_TEXT_SEPARATORS
+
+ self.distance_metric = get_setting_from_snapshot(
+ "local_search_distance_metric",
+ default=DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC,
+ settings_snapshot=settings_snapshot,
+ )
+ self.normalize_vectors = to_bool(
+ get_setting_from_snapshot(
+ "local_search_normalize_vectors",
+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
+ settings_snapshot=settings_snapshot,
+ ),
+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
+ )
+ self.index_type = get_setting_from_snapshot(
+ "local_search_index_type",
+ default=DEFAULT_LOCAL_SEARCH_INDEX_TYPE,
+ settings_snapshot=settings_snapshot,
+ )
# Extract server URL from settings snapshot for link generation
self.server_url = get_server_url(settings_snapshot)
@@ -202,38 +247,80 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
# relevance-formula comment below).
all_results = []
failed_collections = []
+ collection_names = {
+ c.get("id"): c.get("name") or f"Collection {c.get('id')}"
+ for c in collections
+ if c.get("id")
+ }
for collection in collections:
collection_id = collection.get("id")
if not collection_id:
continue
try:
- # Get the RAG index for this collection to find embedding settings
+ # Get the Collection record directly for frozen embedding settings
with get_user_db_session(self.username) as session:
- collection_name = f"collection_{collection_id}"
- rag_index = (
- session.query(RAGIndex)
- .filter_by(
- collection_name=collection_name,
- is_current=True,
- )
+ collection_record = (
+ session.query(Collection)
+ .filter_by(id=collection_id)
.first()
)
- if not rag_index:
+ if (
+ not collection_record
+ or not collection_record.embedding_model
+ ):
logger.debug(
- f"No RAG index found for collection {collection_id}"
+ f"No embedding settings found for collection {collection_id}"
)
continue
- # Get embedding settings from the RAG index
- embedding_model = rag_index.embedding_model
- embedding_provider = (
- rag_index.embedding_model_type.value
+ # Get embedding settings directly from Collection
+ embedding_model = collection_record.embedding_model
+ provider = collection_record.embedding_model_type
+ if provider is not None:
+ embedding_provider = (
+ provider.value
+ if hasattr(provider, "value")
+ else str(provider)
+ )
+ else:
+ embedding_provider = self.embedding_provider
+ coll_chunk_size = getattr(
+ collection_record, "chunk_size", None
+ )
+ chunk_size = (
+ int(coll_chunk_size)
+ if coll_chunk_size is not None
+ else int(
+ self.chunk_size
+ or DEFAULT_LOCAL_SEARCH_CHUNK_SIZE
+ )
+ )
+ coll_chunk_overlap = getattr(
+ collection_record, "chunk_overlap", None
)
- chunk_size = rag_index.chunk_size or self.chunk_size
chunk_overlap = (
- rag_index.chunk_overlap or self.chunk_overlap
+ int(coll_chunk_overlap)
+ if coll_chunk_overlap is not None
+ else int(
+ self.chunk_overlap
+ if self.chunk_overlap is not None
+ else DEFAULT_LOCAL_SEARCH_CHUNK_OVERLAP
+ )
+ )
+ splitter_type = (
+ getattr(collection_record, "splitter_type", None)
+ or getattr(self, "splitter_type", None)
+ or DEFAULT_LOCAL_SEARCH_SPLITTER_TYPE
+ )
+ text_separators = (
+ collection_record.text_separators
+ if getattr(
+ collection_record, "text_separators", None
+ )
+ is not None
+ else getattr(self, "text_separators", None)
)
# Thread the stored normalization flag AND distance
# metric through, exactly like CollectionSearchEngine.
@@ -243,15 +330,35 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
# distance_metric, every collection is labeled "cosine"
# and l2/dot_product hits get the wrong [0,1] mapping in
# the cross-collection merge below. NULL → prior default.
- normalize_vectors = to_bool(
- rag_index.normalize_vectors, default=True
+ coll_normalize = getattr(
+ collection_record, "normalize_vectors", None
+ )
+ normalize_vectors = (
+ to_bool(
+ coll_normalize,
+ default=DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
+ )
+ if coll_normalize is not None
+ else getattr(
+ self,
+ "normalize_vectors",
+ DEFAULT_LOCAL_SEARCH_NORMALIZE_VECTORS,
+ )
+ )
+ distance_metric = (
+ getattr(collection_record, "distance_metric", None)
+ or getattr(self, "distance_metric", None)
+ or DEFAULT_LOCAL_SEARCH_DISTANCE_METRIC
)
- distance_metric = rag_index.distance_metric or "cosine"
# Thread the stored index_type too (like metric/
# normalize): a legacy NULL-index_type row that is
# physically HNSW would otherwise fall back to "flat",
# mislabelling the store. NULL → "flat".
- index_type = rag_index.index_type or "flat"
+ index_type = (
+ getattr(collection_record, "index_type", None)
+ or getattr(self, "index_type", None)
+ or DEFAULT_LOCAL_SEARCH_INDEX_TYPE
+ )
# Create RAG service with the collection's embedding settings
with LibraryRAGService(
@@ -260,6 +367,8 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
embedding_provider=embedding_provider,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
+ splitter_type=splitter_type,
+ text_separators=text_separators,
normalize_vectors=normalize_vectors,
distance_metric=distance_metric,
index_type=index_type,
@@ -268,99 +377,90 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
stats = rag_service.get_rag_stats(collection_id)
if stats.get("indexed_documents", 0) == 0:
logger.debug(
- f"No documents indexed in collection {collection_id}"
+ f"No indexed documents in collection {collection_id}"
)
continue
- # Search this collection's vector index via the new
- # int-id-keyed store (chunk text is rehydrated from
- # the encrypted DB by id — never read from the
+ # Search this collection's vector index via the
+ # new int-id-keyed store (chunk text is rehydrated
+ # from the encrypted DB by id — never read from the
# vector store itself; see the SECURITY INVARIANT
# in vector_stores/base.py).
search_results = rag_service.search(
query, collection_id, limit
)
- # Score per THIS collection's own metric, add
- # collection info to metadata, and append to the
- # cross-collection merge list.
+ # Tag each result with collection ID and pre-score
+ # relevance per THIS collection's metric (so cosine
+ # and L2 sort compatibly on merge).
for r in search_results:
- metadata = dict(r.metadata or {})
- metadata["collection_id"] = collection_id
- metadata["collection_name"] = collection.get(
- "name", "Unknown"
- )
- # Map r.distance to a [0,1] relevance growing with
- # similarity: cosine/dot_product is an inner product
- # in [-1,1] (higher=nearer) -> (d+1)/2 clamped; l2 (or
- # any non-standard metric) is a distance >=0 (lower=
- # nearer) -> 1/(1+d) in (0,1]. Raw IP would give
- # negative relevance and break [0,1] consumers.
- # Scoring here (per-collection, before the merge)
- # keeps different-metric collections comparable once
- # merged. The IP test MUST match faiss_store.
- # _build_base_index (inner-product iff cosine/
- # dot_product, else L2): a bare `metric == "l2"` would
- # score a non-standard metric — which builds an L2
- # index — with the IP formula and invert the ranking.
+ # Map r.distance to a [0, 1] relevance score:
+ # - cosine/dot_product: (d+1)/2 clamped to [0,1]
+ # - l2 / other: 1/(1+d) in (0, 1]
+ # Must match faiss_store._build_base_index metric
+ # selection (cosine/dot_product -> IP, else L2).
relevance = (
max(0.0, min(1.0, (r.distance + 1.0) / 2.0))
if r.metric in ("cosine", "dot_product")
else 1.0 / (1.0 + r.distance)
)
- all_results.append((r, relevance, metadata))
+ all_results.append((r, relevance, collection_id))
except Exception as e:
- # One broken collection must not abort the others, but
- # record the failure so it is not silently equated with
- # "no matching documents" below.
safe_msg = self._scrub_error(e)
logger.exception(
f"Error searching collection {collection_id} ({type(e).__name__}): {safe_msg}"
)
- failed_collections.append(
- collection.get("name") or str(collection_id)
- )
+ failed_collections.append(collection_id)
continue
- # Sort all results by relevance, descending (higher is always
- # better post-transform, regardless of the source collection's
- # metric — see the per-result relevance comment above).
- all_results.sort(key=lambda item: item[1], reverse=True)
-
- # Take top results across all collections
- top_results = all_results[:limit]
+ # Fail loudly if *every* collection errored (returning []
+ # masks outright service failure as "no results"). Partial
+ # failures with 0 hits also raise: distinguishing a genuine
+ # "not found" from "the one collection with answers crashed"
+ # is impossible without this check.
+ if failed_collections and not all_results:
+ failed_names = [
+ collection_names.get(cid) or str(cid)
+ for cid in failed_collections
+ ]
+ self._raise_collections_failed(failed_names)
+
+ if failed_collections and all_results:
+ failed_names = [
+ collection_names.get(cid) or str(cid)
+ for cid in failed_collections
+ ]
+ logger.warning(
+ f"Library search returned partial results: {len(failed_collections)} of "
+ f"{len(collections)} collection(s) failed during search: {failed_names}"
+ )
- if not top_results:
- if failed_collections:
- # Zero results AND at least one collection errored:
- # reporting "no results" here would hide the failure.
- self._raise_collections_failed(failed_collections)
+ if not all_results:
logger.info("No results found across any collections")
return []
- if failed_collections:
- # Partial success: results from the healthy collections are
- # returned, but tell the user they are incomplete. WARNING
- # level streams to the research-log UI via
- # frontend_progress_sink.
- logger.warning(
- f"Library search results are incomplete: "
- f"{len(failed_collections)} of {len(collections)} "
- f"collection(s) failed: {failed_collections}"
- )
+ # Sort by pre-scored relevance (descending) and take top limit
+ all_results.sort(key=lambda x: x[1], reverse=True)
+ top_results = all_results[:limit]
- # Convert to the search-result dict format
+ # Convert to search result format
results = []
- for r, relevance, metadata in top_results:
- # Try both source_id and document_id for compatibility
+ for r, relevance, coll_id in top_results:
+ metadata = dict(r.metadata or {})
+ metadata["collection_id"] = coll_id
+ metadata["collection_name"] = (
+ collection_names.get(coll_id) or "Unknown"
+ )
+
+ # Get document ID
doc_id = (
r.source_id
or metadata.get("source_id")
or metadata.get("document_id")
)
- # Get title from the result, with fallbacks
+ # Get title
title = (
r.document_title
or metadata.get("document_title")
@@ -368,18 +468,17 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
or (f"Document {doc_id}" if doc_id else "Untitled")
)
- # Content is rehydrated from the encrypted DB by chunk id
+ # Create snippet from content
snippet = (
r.text[:SNIPPET_LENGTH_LONG] + "..."
if len(r.text) > SNIPPET_LENGTH_LONG
else r.text
)
- # Generate URL to document content (targeted chunk if
- # chunk_index available). Use the shared helpers so a
- # malformed chunk index (UUID, boolean, negative, etc.)
- # cannot leak into the citation anchor, and so the
- # primary ``r.source_id`` path is sanitised the same way
- # as metadata keys.
+
+ # Generate document URL. Validate chunk_idx and sanitise
+ # doc_id through the shared helpers so a malformed chunk index
+ # (UUID, boolean, negative) or path-traversal doc_id cannot leak
+ # into the citation.
chunk_idx = extract_chunk_index(metadata)
# Authoritative id FIRST. ``extract_document_id`` scans its
# first mapping before any later one, so passing ``metadata``
diff --git a/tests/web/routers/test_rag_indexing_pipeline.py b/tests/web/routers/test_rag_indexing_pipeline.py
index 7a766a7760..32bc4429be 100644
--- a/tests/web/routers/test_rag_indexing_pipeline.py
+++ b/tests/web/routers/test_rag_indexing_pipeline.py
@@ -537,22 +537,19 @@ def _sole_index_row(session, collection_id):
class TestChunkingSettingsAtQueryTime:
"""``splitter_type``/``text_separators`` are part of the index identity
(``_get_index_hash``) and are stamped onto the ``RAGIndex`` row at
- indexing time, but neither search engine reads them back when it
- rebuilds the query-time service -- issue #5745."""
+ indexing time, and both search engines read them back when rebuilding
+ the query-time service -- issue #5745."""
@pytest.mark.parametrize(
"engine",
[_COLLECTION_ENGINE, _LIBRARY_ENGINE],
ids=["collection", "library"],
)
- def test_search_engines_omit_the_chunking_kwargs(self, engine):
- """The static half of the mechanism, and a tripwire: when #5745 is
- fixed this fails and the xfails below must be retired in the same
- change."""
+ def test_search_engines_include_the_chunking_kwargs(self, engine):
+ """The identity-affecting kwargs the engine threads through --
+ verifies that chunking settings are forwarded to LibraryRAGService."""
kwargs = _library_rag_service_call_kwargs(engine)
- # The identity-affecting kwargs the engine DOES thread through --
- # proves the extraction works and the omission is specific.
assert {
"embedding_model",
"embedding_provider",
@@ -561,9 +558,9 @@ class TestChunkingSettingsAtQueryTime:
"distance_metric",
"normalize_vectors",
"index_type",
+ "splitter_type",
+ "text_separators",
} <= kwargs
- assert "splitter_type" not in kwargs
- assert "text_separators" not in kwargs
def test_indexing_persists_the_configured_separators(
self, world, indexed_with_nondefault_separators
@@ -591,25 +588,6 @@ class TestChunkingSettingsAtQueryTime:
hits = svc.search(QUERY, collection_id, 5)
assert any(SECRET in h.text for h in hits)
- @pytest.mark.xfail(
- strict=True,
- reason=(
- "#5745 mechanism. _get_index_hash covers text_separators and "
- "splitter_type, but CollectionSearchEngine/LibrarySearchEngine "
- "rebuild the query-time LibraryRAGService from the RAGIndex row "
- "WITHOUT them, so the service falls back to the "
- "["
- "'\\n\\n', '\\n', '. ', ' ', ''] default. The hash differs, "
- "_find_matching_rag_index's fallback also rejects the row "
- "(_matches_current_index_configuration compares separators), and "
- "_get_or_create_rag_index creates a fresh EMPTY index instead. "
- "The engines' get_rag_stats pre-check is config-independent so it "
- "passes, and the user sees a silently empty result set. Fix by "
- "threading rag_index.splitter_type and rag_index.text_separators "
- "into both LibraryRAGService(...) call sites, exactly as "
- "distance_metric/normalize_vectors/index_type already are."
- ),
- )
def test_query_time_reconstruction_finds_the_content(
self, world, indexed_with_nondefault_separators
):
@@ -627,22 +605,6 @@ class TestChunkingSettingsAtQueryTime:
"unreachable from the search engines' query-time service"
)
- @pytest.mark.xfail(
- strict=True,
- reason=(
- "The mismatched query-time search does not merely return "
- "nothing: _get_or_create_rag_index_in_session sets "
- "mutated = created or (...), so the row it CREATES is promoted "
- "to is_current and the real index is demoted -- on a read path, "
- "despite promote_current=False. Both engines select the "
- "collection's index with filter_by(is_current=True), so after "
- "one such search they resolve the empty index permanently and "
- "the collection stays empty even for correctly-configured "
- "callers. Fix by not promoting an index created on a read path "
- "(gate the promotion on promote_current), in addition to "
- "threading the chunking settings through."
- ),
- )
def test_query_time_mismatch_does_not_disturb_the_real_index(
self, world, indexed_with_nondefault_separators
):
diff --git a/tests/web_search_engines/engines/test_local_search_coverage.py b/tests/web_search_engines/engines/test_local_search_coverage.py
index 9bc93f19d1..fa8c99ce96 100644
--- a/tests/web_search_engines/engines/test_local_search_coverage.py
+++ b/tests/web_search_engines/engines/test_local_search_coverage.py
@@ -66,6 +66,11 @@ def _mock_rag_index(
idx.embedding_model_type = Mock(value=embedding_provider)
idx.chunk_size = chunk_size
idx.chunk_overlap = chunk_overlap
+ idx.splitter_type = "recursive"
+ idx.text_separators = None
+ idx.distance_metric = "cosine"
+ idx.normalize_vectors = True
+ idx.index_type = "flat"
return idx
@@ -796,3 +801,5 @@ class TestResultStructure:
assert r["source_type"] == "library"
assert r["link"] == r["url"]
assert isinstance(r["relevance_score"], float)
+ assert r["metadata"]["collection_id"] == 1
+ assert r["metadata"]["collection_name"] == "C"
diff --git a/tests/web_search_engines/engines/test_search_engine_collection.py b/tests/web_search_engines/engines/test_search_engine_collection.py
index 33837edbe0..671c3890b1 100644
--- a/tests/web_search_engines/engines/test_search_engine_collection.py
+++ b/tests/web_search_engines/engines/test_search_engine_collection.py
@@ -124,19 +124,24 @@ class TestLoadCollectionEmbeddingSettings:
# Should not raise, just log warning
assert engine.username is None
- def test_load_settings_with_rag_index(self):
- """Load settings from RAG index."""
+ def test_load_settings_with_collection(self):
+ """Load settings directly from Collection table."""
from local_deep_research.web_search_engines.engines.search_engine_collection import (
CollectionSearchEngine,
)
settings = {"_username": "testuser"}
- mock_rag_index = Mock()
- mock_rag_index.embedding_model = "custom-model"
- mock_rag_index.embedding_model_type = Mock(value="ollama")
- mock_rag_index.chunk_size = 2000
- mock_rag_index.chunk_overlap = 400
+ mock_collection = Mock()
+ mock_collection.embedding_model = "custom-model"
+ mock_collection.embedding_model_type = Mock(value="ollama")
+ mock_collection.chunk_size = 2000
+ mock_collection.chunk_overlap = 400
+ mock_collection.splitter_type = "recursive"
+ mock_collection.text_separators = ["\n\n", "\n"]
+ mock_collection.distance_metric = "cosine"
+ mock_collection.normalize_vectors = True
+ mock_collection.index_type = "flat"
with patch(
"local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
@@ -149,7 +154,7 @@ class TestLoadCollectionEmbeddingSettings:
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
) as mock_session:
- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
engine = CollectionSearchEngine(
collection_id="abc123",
@@ -162,8 +167,95 @@ class TestLoadCollectionEmbeddingSettings:
assert engine.chunk_size == 2000
assert engine.chunk_overlap == 400
- def test_load_settings_no_rag_index(self):
- """Load settings handles missing RAG index."""
+ def test_load_settings_zero_chunk_overlap(self):
+ """Zero chunk_overlap must be preserved as 0, not overridden by default."""
+ from local_deep_research.web_search_engines.engines.search_engine_collection import (
+ CollectionSearchEngine,
+ )
+
+ settings = {"_username": "testuser"}
+
+ mock_collection = Mock()
+ mock_collection.embedding_model = "custom-model"
+ mock_collection.embedding_model_type = Mock(value="openai")
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 0
+ mock_collection.splitter_type = "token"
+ mock_collection.text_separators = ["\n\n\n", "\n"]
+ mock_collection.distance_metric = "cosine"
+ mock_collection.normalize_vectors = True
+ mock_collection.index_type = "flat"
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
+ return_value=None,
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",
+ return_value="http://localhost:5000",
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
+ ) as mock_session:
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
+
+ engine = CollectionSearchEngine(
+ collection_id="abc123",
+ collection_name="Test Collection",
+ settings_snapshot=settings,
+ )
+
+ assert engine.chunk_overlap == 0
+ assert engine.splitter_type == "token"
+ assert engine.text_separators == ["\n\n\n", "\n"]
+
+ def test_load_settings_nullable_embedding_model_type(self):
+ """When collection.embedding_model_type is None, embedding_provider is not set to 'None'."""
+ from local_deep_research.web_search_engines.engines.search_engine_collection import (
+ CollectionSearchEngine,
+ )
+
+ settings = {
+ "_username": "testuser",
+ "local_search_embedding_provider": "sentence_transformers",
+ }
+
+ mock_collection = Mock()
+ mock_collection.embedding_model = "custom-model"
+ mock_collection.embedding_model_type = None
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 200
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.distance_metric = None
+ mock_collection.normalize_vectors = None
+ mock_collection.index_type = None
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
+ return_value="sentence_transformers",
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",
+ return_value="http://localhost:5000",
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
+ ) as mock_session:
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
+
+ engine = CollectionSearchEngine(
+ collection_id="abc123",
+ collection_name="Test Collection",
+ settings_snapshot=settings,
+ )
+
+ assert engine.embedding_model == "custom-model"
+ assert engine.embedding_provider != "None"
+ assert engine.embedding_provider == "sentence_transformers"
+
+ def test_load_settings_no_collection(self):
+ """Load settings handles missing collection."""
from local_deep_research.web_search_engines.engines.search_engine_collection import (
CollectionSearchEngine,
)
@@ -251,8 +343,8 @@ class TestSearch:
assert results == []
- def test_search_no_rag_index(self):
- """Search returns empty when no RAG index."""
+ def test_search_no_collection(self):
+ """Search returns empty when collection not found."""
from local_deep_research.web_search_engines.engines.search_engine_collection import (
CollectionSearchEngine,
)
@@ -289,14 +381,18 @@ class TestSearch:
settings = {"_username": "testuser"}
- mock_rag_index = Mock()
- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"
- mock_rag_index.embedding_model_type = Mock(
+ mock_collection = Mock()
+ mock_collection.embedding_model = "all-MiniLM-L6-v2"
+ mock_collection.embedding_model_type = Mock(
value="sentence_transformers"
)
- mock_rag_index.chunk_size = 1000
- mock_rag_index.chunk_overlap = 200
-
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 200
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.distance_metric = None
+ mock_collection.normalize_vectors = None
+ mock_collection.index_type = None
with patch(
"local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
return_value=None,
@@ -308,7 +404,7 @@ class TestSearch:
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
) as mock_session:
- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"
@@ -342,13 +438,18 @@ class TestSearch:
settings = {"_username": "testuser"}
- mock_rag_index = Mock()
- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"
- mock_rag_index.embedding_model_type = Mock(
+ mock_collection = Mock()
+ mock_collection.embedding_model = "all-MiniLM-L6-v2"
+ mock_collection.embedding_model_type = Mock(
value="sentence_transformers"
)
- mock_rag_index.chunk_size = 1000
- mock_rag_index.chunk_overlap = 200
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 200
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.distance_metric = None
+ mock_collection.normalize_vectors = None
+ mock_collection.index_type = None
mock_doc = Mock()
mock_doc.page_content = "This is the document content for testing."
@@ -368,7 +469,7 @@ class TestSearch:
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
) as mock_session:
- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"
@@ -415,6 +516,171 @@ class TestSearch:
assert results[0]["source_type"] == "library"
assert results[0]["url"] == "/library/document/123"
+ def test_search_forwards_all_rag_parameters(self):
+ """Search forwards chunk_overlap=0, splitter_type, text_separators, distance_metric, index_type."""
+ from local_deep_research.web_search_engines.engines.search_engine_collection import (
+ CollectionSearchEngine,
+ )
+
+ settings = {"_username": "testuser"}
+
+ mock_collection = Mock()
+ mock_collection.embedding_model = "nomic-embed"
+ mock_collection.embedding_model_type = Mock(value="openai")
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 0
+ mock_collection.splitter_type = "token"
+ mock_collection.text_separators = ["\n\n\n", "\n\n", "\n"]
+ mock_collection.distance_metric = "cosine"
+ mock_collection.normalize_vectors = True
+ mock_collection.index_type = "flat"
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
+ return_value=None,
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",
+ return_value="http://localhost:5000",
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
+ ) as mock_session:
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"
+ ) as mock_rag_service:
+ mock_rag_instance = Mock()
+ mock_rag_instance.get_rag_stats.return_value = {
+ "indexed_documents": 1
+ }
+ mock_rag_instance.search.return_value = [
+ SearchResult(
+ chunk_id=1,
+ text="content",
+ distance=0.8,
+ metric="cosine",
+ metadata={"source_id": "123"},
+ document_title="Doc",
+ source_id=None,
+ source_type=None,
+ )
+ ]
+ mock_rag_service.return_value.__enter__.return_value = (
+ mock_rag_instance
+ )
+ mock_rag_service.return_value.__exit__.return_value = (
+ None
+ )
+
+ engine = CollectionSearchEngine(
+ collection_id="abc123",
+ collection_name="Test Collection",
+ settings_snapshot=settings,
+ )
+ results = engine.search("test query")
+
+ assert len(results) == 1
+ # Verify LibraryRAGService instantiation kwargs
+ call_kwargs = mock_rag_service.call_args.kwargs
+ assert call_kwargs["embedding_model"] == "nomic-embed"
+ assert call_kwargs["embedding_provider"] == "openai"
+ assert call_kwargs["chunk_size"] == 1000
+ assert call_kwargs["chunk_overlap"] == 0
+ assert call_kwargs["splitter_type"] == "token"
+ assert call_kwargs["text_separators"] == [
+ "\n\n\n",
+ "\n\n",
+ "\n",
+ ]
+ assert call_kwargs["distance_metric"] == "cosine"
+ assert call_kwargs["normalize_vectors"] is True
+ assert call_kwargs["index_type"] == "flat"
+
+ def test_search_relevance_metric_mapping(self):
+ """Cosine and dot_product use (d+1)/2 clamped, L2 uses 1/(1+d)."""
+ from local_deep_research.web_search_engines.engines.search_engine_collection import (
+ CollectionSearchEngine,
+ )
+
+ settings = {"_username": "testuser"}
+
+ mock_collection = Mock()
+ mock_collection.embedding_model = "test-model"
+ mock_collection.embedding_model_type = Mock(value="openai")
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 0
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.distance_metric = None
+ mock_collection.normalize_vectors = None
+ mock_collection.index_type = None
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
+ return_value=None,
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",
+ return_value="http://localhost:5000",
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
+ ) as mock_session:
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"
+ ) as mock_rag_service:
+ mock_rag_instance = Mock()
+ mock_rag_instance.get_rag_stats.return_value = {
+ "indexed_documents": 2
+ }
+ mock_rag_instance.search.return_value = [
+ SearchResult(
+ chunk_id=1,
+ text="cosine hit",
+ distance=0.6,
+ metric="cosine",
+ metadata={"source_id": "1"},
+ document_title="Doc 1",
+ source_id=None,
+ source_type=None,
+ ),
+ SearchResult(
+ chunk_id=2,
+ text="l2 hit",
+ distance=0.5,
+ metric="l2",
+ metadata={"source_id": "2"},
+ document_title="Doc 2",
+ source_id=None,
+ source_type=None,
+ ),
+ ]
+ mock_rag_service.return_value.__enter__.return_value = (
+ mock_rag_instance
+ )
+ mock_rag_service.return_value.__exit__.return_value = (
+ None
+ )
+
+ engine = CollectionSearchEngine(
+ collection_id="abc123",
+ collection_name="Test Collection",
+ settings_snapshot=settings,
+ )
+ results = engine.search("test query")
+
+ assert len(results) == 2
+ # Cosine relevance: (0.6 + 1.0) / 2.0 = 0.8
+ assert abs(results[0]["relevance_score"] - 0.8) < 1e-6
+ # L2 relevance: 1.0 / (1.0 + 0.5) = 0.666666...
+ assert (
+ abs(results[1]["relevance_score"] - (1.0 / 1.5))
+ < 1e-6
+ )
+
def test_search_empty_vector_results(self):
"""Search handles empty vector search results."""
from local_deep_research.web_search_engines.engines.search_engine_collection import (
@@ -423,14 +689,18 @@ class TestSearch:
settings = {"_username": "testuser"}
- mock_rag_index = Mock()
- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"
- mock_rag_index.embedding_model_type = Mock(
+ mock_collection = Mock()
+ mock_collection.embedding_model = "all-MiniLM-L6-v2"
+ mock_collection.embedding_model_type = Mock(
value="sentence_transformers"
)
- mock_rag_index.chunk_size = 1000
- mock_rag_index.chunk_overlap = 200
-
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 200
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.distance_metric = None
+ mock_collection.normalize_vectors = None
+ mock_collection.index_type = None
with patch(
"local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
return_value=None,
@@ -442,7 +712,7 @@ class TestSearch:
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
) as mock_session:
- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"
@@ -479,14 +749,18 @@ class TestSearch:
settings = {"_username": "testuser"}
- mock_rag_index = Mock()
- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"
- mock_rag_index.embedding_model_type = Mock(
+ mock_collection = Mock()
+ mock_collection.embedding_model = "all-MiniLM-L6-v2"
+ mock_collection.embedding_model_type = Mock(
value="sentence_transformers"
)
- mock_rag_index.chunk_size = 1000
- mock_rag_index.chunk_overlap = 200
-
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 200
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.distance_metric = None
+ mock_collection.normalize_vectors = None
+ mock_collection.index_type = None
with patch(
"local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
return_value=None,
@@ -498,7 +772,7 @@ class TestSearch:
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
) as mock_session:
- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
with patch(
"local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService",
@@ -514,6 +788,69 @@ class TestSearch:
):
engine.search("test query")
+ def test_search_nullable_embedding_model_type(self):
+ """When collection.embedding_model_type is None, search uses default provider, not 'None'."""
+ from local_deep_research.web_search_engines.engines.search_engine_collection import (
+ CollectionSearchEngine,
+ )
+
+ settings = {
+ "_username": "testuser",
+ "local_search_embedding_provider": "sentence_transformers",
+ }
+
+ mock_collection = Mock()
+ mock_collection.embedding_model = "custom-model"
+ mock_collection.embedding_model_type = None
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 200
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.distance_metric = None
+ mock_collection.normalize_vectors = None
+ mock_collection.index_type = None
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
+ return_value="sentence_transformers",
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",
+ return_value="http://localhost:5000",
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_collection.get_user_db_session"
+ ) as mock_session:
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_collection.LibraryRAGService"
+ ) as mock_rag_service:
+ mock_rag_instance = Mock()
+ mock_rag_instance.get_rag_stats.return_value = {
+ "indexed_documents": 0
+ }
+ mock_rag_service.return_value.__enter__.return_value = (
+ mock_rag_instance
+ )
+ mock_rag_service.return_value.__exit__.return_value = (
+ None
+ )
+
+ engine = CollectionSearchEngine(
+ collection_id="abc123",
+ collection_name="Test Collection",
+ settings_snapshot=settings,
+ )
+ engine.search("test query")
+
+ call_kwargs = mock_rag_service.call_args.kwargs
+ assert (
+ call_kwargs["embedding_provider"]
+ == "sentence_transformers"
+ )
+ assert call_kwargs["embedding_provider"] != "None"
+
class TestGetDocumentUrl:
"""Tests for _get_document_url method."""
diff --git a/tests/web_search_engines/engines/test_search_engine_library.py b/tests/web_search_engines/engines/test_search_engine_library.py
index 931cc24e46..8c3a3e8aec 100644
--- a/tests/web_search_engines/engines/test_search_engine_library.py
+++ b/tests/web_search_engines/engines/test_search_engine_library.py
@@ -189,15 +189,20 @@ class TestSearch:
with patch(
"local_deep_research.web_search_engines.engines.search_engine_library.get_user_db_session"
) as mock_session:
- mock_rag_index = Mock()
- mock_rag_index.embedding_model = "all-MiniLM-L6-v2"
- mock_rag_index.embedding_model_type = Mock(
+ mock_collection = Mock()
+ mock_collection.embedding_model = "all-MiniLM-L6-v2"
+ mock_collection.embedding_model_type = Mock(
value="sentence_transformers"
)
- mock_rag_index.chunk_size = 1000
- mock_rag_index.chunk_overlap = 200
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 200
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.normalize_vectors = True
+ mock_collection.distance_metric = "l2"
+ mock_collection.index_type = "flat"
- mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_rag_index
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
with patch(
"local_deep_research.web_search_engines.engines.search_engine_library.LibraryRAGService"
@@ -234,6 +239,171 @@ class TestSearch:
assert results[0]["source_type"] == "library"
assert "/library/document/123" in results[0]["url"]
+ def test_search_forwards_zero_overlap_and_splitter_settings(self):
+ """Search forwards chunk_overlap=0, splitter_type, and text_separators to LibraryRAGService."""
+ from local_deep_research.web_search_engines.engines.search_engine_library import (
+ LibraryRAGSearchEngine,
+ )
+
+ settings = {"_username": "testuser"}
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_setting_from_snapshot",
+ return_value=None,
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_server_url",
+ return_value="http://localhost:5000",
+ ):
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryService"
+ ) as mock_lib_service:
+ mock_lib_service.return_value.get_all_collections.return_value = [
+ {"id": "col_1", "name": "Zero Overlap Collection"}
+ ]
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_user_db_session"
+ ) as mock_session:
+ mock_collection = Mock()
+ mock_collection.embedding_model = "nomic-embed"
+ mock_collection.embedding_model_type = Mock(
+ value="openai"
+ )
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 0
+ mock_collection.splitter_type = "token"
+ mock_collection.text_separators = [
+ "\n\n\n",
+ "\n\n",
+ "\n",
+ ]
+ mock_collection.distance_metric = "cosine"
+ mock_collection.normalize_vectors = True
+ mock_collection.index_type = "flat"
+
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryRAGService"
+ ) as mock_rag_service:
+ mock_rag_instance = Mock()
+ mock_rag_instance.get_rag_stats.return_value = {
+ "indexed_documents": 1
+ }
+ mock_rag_instance.search.return_value = [
+ SearchResult(
+ chunk_id=1,
+ text="test text",
+ distance=0.8,
+ metric="cosine",
+ metadata={"source_id": "doc1"},
+ document_title="Title",
+ source_id=None,
+ source_type=None,
+ )
+ ]
+ mock_rag_service.return_value.__enter__.return_value = mock_rag_instance
+ mock_rag_service.return_value.__exit__.return_value = None
+
+ engine = LibraryRAGSearchEngine(
+ settings_snapshot=settings
+ )
+ results = engine.search("query")
+
+ assert len(results) == 1
+ call_kwargs = mock_rag_service.call_args.kwargs
+ assert call_kwargs["chunk_overlap"] == 0
+ assert call_kwargs["splitter_type"] == "token"
+ assert call_kwargs["text_separators"] == [
+ "\n\n\n",
+ "\n\n",
+ "\n",
+ ]
+ assert call_kwargs["distance_metric"] == "cosine"
+ assert call_kwargs["normalize_vectors"] is True
+ assert call_kwargs["index_type"] == "flat"
+
+ def test_search_nullable_embedding_model_type_uses_default(self):
+ """When collection.embedding_model_type is NULL, search falls back to default provider, not 'None'."""
+ from local_deep_research.web_search_engines.engines.search_engine_library import (
+ LibraryRAGSearchEngine,
+ )
+
+ settings = {
+ "_username": "testuser",
+ "local_search_embedding_provider": "sentence_transformers",
+ }
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryService"
+ ) as mock_lib_service:
+ mock_lib_service.return_value.get_all_collections.return_value = [
+ {"id": "col1", "name": "Collection 1"}
+ ]
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_user_db_session"
+ ) as mock_session:
+ mock_collection = Mock()
+ mock_collection.embedding_model = "all-MiniLM-L6-v2"
+ mock_collection.embedding_model_type = None
+ mock_collection.chunk_size = 1000
+ mock_collection.chunk_overlap = 200
+ mock_collection.splitter_type = None
+ mock_collection.text_separators = None
+ mock_collection.distance_metric = None
+ mock_collection.normalize_vectors = None
+ mock_collection.index_type = None
+
+ mock_session.return_value.__enter__.return_value.query.return_value.filter_by.return_value.first.return_value = mock_collection
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryRAGService"
+ ) as mock_rag_service:
+ mock_rag_instance = Mock()
+ mock_rag_instance.get_rag_stats.return_value = {
+ "indexed_documents": 0
+ }
+ mock_rag_service.return_value.__enter__.return_value = (
+ mock_rag_instance
+ )
+ mock_rag_service.return_value.__exit__.return_value = None
+
+ engine = LibraryRAGSearchEngine(settings_snapshot=settings)
+ engine.search("test query")
+
+ call_kwargs = mock_rag_service.call_args.kwargs
+ assert (
+ call_kwargs["embedding_provider"]
+ == "sentence_transformers"
+ )
+ assert call_kwargs["embedding_provider"] != "None"
+
+ def test_all_collections_fail_passes_collection_names(self):
+ """When all collections fail, _raise_collections_failed receives collection names, not raw UUIDs."""
+ import pytest
+ from local_deep_research.web_search_engines.engines.search_engine_library import (
+ LibraryRAGSearchEngine,
+ )
+
+ settings = {"_username": "testuser"}
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.LibraryService"
+ ) as mock_lib:
+ mock_lib.return_value.get_all_collections.return_value = [
+ {"id": "uuid-1234", "name": "Important Docs"}
+ ]
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_library.get_user_db_session",
+ side_effect=RuntimeError("db error"),
+ ):
+ engine = LibraryRAGSearchEngine(settings_snapshot=settings)
+ with pytest.raises(RuntimeError) as exc_info:
+ engine.search("test query")
+
+ assert "Important Docs" in str(exc_info.value)
+
def test_search_exception(self):
"""Search re-raises so failures are not mistaken for no results."""
import pytest
@@ -596,3 +766,18 @@ class TestClassAttributes:
)
assert LibraryRAGSearchEngine.is_local is True
+
+ def test_egress_attributes(self):
+ """LibraryRAGSearchEngine has egress sensitivity and exposure configured."""
+ from local_deep_research.web_search_engines.engines.search_engine_library import (
+ LibraryRAGSearchEngine,
+ )
+ from local_deep_research.web_search_engines.search_engine_base import (
+ Exposure,
+ Sensitivity,
+ )
+
+ assert (
+ LibraryRAGSearchEngine.egress_sensitivity == Sensitivity.SENSITIVE
+ )
+ assert LibraryRAGSearchEngine.egress_exposure == Exposure.CONTAINED
diff --git a/tests/web_search_engines/engines/test_search_engine_library_coverage.py b/tests/web_search_engines/engines/test_search_engine_library_coverage.py
index e20d89bc6d..a6947f6c6e 100644
--- a/tests/web_search_engines/engines/test_search_engine_library_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_library_coverage.py
@@ -1,57 +1,57 @@
"""
-Tests for LibraryRAGSearchEngine (search_engine_library.py, 0% coverage).
-
-Covers:
-- __init__: initialization with/without username
-- search: no username, no collections, exception handling
-- _get_previews: delegates to search
-- _get_full_content: no username, no doc_id, success path
-- close: no-op
+Coverage tests for search_engine_library.py error handling and edge cases.
"""
from unittest.mock import MagicMock, Mock, patch
from local_deep_research.vector_stores.facade import SearchResult
-
MODULE = "local_deep_research.web_search_engines.engines.search_engine_library"
-def _make_engine(username="testuser", settings_snapshot=None):
- """Create a LibraryRAGSearchEngine with mocked dependencies."""
+def _make_engine(username="test_user", settings_snapshot=None):
from local_deep_research.web_search_engines.engines.search_engine_library import (
LibraryRAGSearchEngine,
)
- snapshot = settings_snapshot or {"_username": username}
- engine = LibraryRAGSearchEngine(
- llm=MagicMock(),
- max_results=10,
- settings_snapshot=snapshot,
- )
- return engine
+ snap = {"_username": username} if username else {}
+ if settings_snapshot:
+ snap.update(settings_snapshot)
+ return LibraryRAGSearchEngine(settings_snapshot=snap if snap else None)
class TestLibraryRAGSearchEngineInit:
+ """Tests for LibraryRAGSearchEngine initialization edge cases."""
+
def test_init_with_username(self):
- engine = _make_engine(username="testuser")
- assert engine.username == "testuser"
+ engine = _make_engine("custom_user")
+ assert engine.username == "custom_user"
assert engine.is_local is True
def test_init_without_username(self):
- engine = _make_engine(settings_snapshot={"_username": None})
+ engine = _make_engine(username=None)
assert engine.username is None
def test_init_reads_embedding_settings(self):
- engine = _make_engine(username="user1")
- # Defaults should be set
- assert engine.embedding_model is not None
- assert engine.chunk_size is not None
+ snap = {
+ "_username": "test_user",
+ "local_search_embedding_model": "custom-model",
+ "local_search_embedding_provider": "openai",
+ "local_search_chunk_size": 500,
+ "local_search_chunk_overlap": 100,
+ }
+ engine = _make_engine(settings_snapshot=snap)
+ assert engine.embedding_model == "custom-model"
+ assert engine.embedding_provider == "openai"
+ assert engine.chunk_size == 500
+ assert engine.chunk_overlap == 100
class TestSearch:
+ """Tests for LibraryRAGSearchEngine.search error handling."""
+
def test_search_no_username_returns_empty(self):
- engine = _make_engine(settings_snapshot={"_username": None})
+ engine = _make_engine(username=None)
result = engine.search("test query")
assert result == []
@@ -59,32 +59,33 @@ class TestSearch:
engine = _make_engine()
mock_service = MagicMock()
mock_service.get_all_collections.return_value = []
+
with patch(f"{MODULE}.LibraryService", return_value=mock_service):
result = engine.search("test query")
assert result == []
def test_search_exception_propagates(self):
- """A failed search raises instead of masquerading as no results."""
+ """Top-level errors bubble up so callers see failure, not empty results."""
import pytest
engine = _make_engine()
with patch(
- f"{MODULE}.LibraryService", side_effect=RuntimeError("fail")
+ f"{MODULE}.LibraryService",
+ side_effect=Exception("Database error"),
):
- with pytest.raises(RuntimeError, match="fail"):
+ with pytest.raises(Exception, match="Database error"):
engine.search("test query")
def test_search_collection_no_rag_index_skips(self):
engine = _make_engine()
mock_service = MagicMock()
mock_service.get_all_collections.return_value = [
- {"id": "col1", "name": "Test Collection"}
+ {"id": "col1", "name": "Collection 1"}
]
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
- # RAGIndex query returns None
mock_session.query.return_value.filter_by.return_value.first.return_value = None
with patch(f"{MODULE}.LibraryService", return_value=mock_service):
@@ -132,6 +133,11 @@ class TestSearch:
)
mock_rag_index.chunk_size = 1000
mock_rag_index.chunk_overlap = 200
+ mock_rag_index.splitter_type = "recursive"
+ mock_rag_index.text_separators = None
+ mock_rag_index.distance_metric = "cosine"
+ mock_rag_index.normalize_vectors = True
+ mock_rag_index.index_type = "flat"
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_rag_index
# col1 fails when its RAG service is created; col2 succeeds
@@ -198,7 +204,6 @@ class TestSearch:
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
-
mock_rag_index = MagicMock()
mock_rag_index.embedding_model = "all-MiniLM-L6-v2"
mock_rag_index.embedding_model_type = MagicMock(
@@ -218,7 +223,8 @@ class TestSearch:
f"{MODULE}.get_user_db_session", return_value=mock_session
):
with patch(
- f"{MODULE}.LibraryRAGService", return_value=mock_rag_service
+ f"{MODULE}.LibraryRAGService",
+ return_value=mock_rag_service,
):
result = engine.search("test query")
@@ -226,46 +232,48 @@ class TestSearch:
class TestGetPreviews:
+ """Tests for LibraryRAGSearchEngine._get_previews delegation."""
+
def test_delegates_to_search(self):
engine = _make_engine()
with patch.object(
- engine, "search", return_value=[{"title": "test"}]
+ engine, "search", return_value=[{"title": "Doc"}]
) as mock_search:
- result = engine._get_previews("test query", limit=5)
- mock_search.assert_called_once_with("test query", 5, None, None)
- assert result == [{"title": "test"}]
+ result = engine._get_previews("query", limit=5)
+ mock_search.assert_called_once_with("query", 5, None, None)
+ assert result == [{"title": "Doc"}]
class TestGetFullContent:
+ """Tests for LibraryRAGSearchEngine._get_full_content."""
+
def test_items_without_document_id_returned_unchanged(self):
- """Items lacking metadata.document_id are skipped and returned as-is."""
engine = _make_engine()
- items = [{"title": "test", "snippet": "content"}]
-
+ items = [{"title": "Doc 1", "metadata": {}}]
result = engine._get_full_content(items)
-
assert result == items
def test_no_username_returns_items(self):
- engine = _make_engine(settings_snapshot={"_username": None})
- items = [{"title": "test"}]
+ engine = _make_engine(username=None)
+ items = [{"title": "Doc 1", "metadata": {"document_id": "doc1"}}]
result = engine._get_full_content(items)
assert result == items
def test_exception_returns_items(self):
engine = _make_engine()
- items = [{"title": "test", "metadata": {"document_id": "doc1"}}]
+ items = [{"title": "Doc 1", "metadata": {"document_id": "doc1"}}]
with patch(
f"{MODULE}.get_user_db_session",
- side_effect=RuntimeError("fail"),
+ side_effect=Exception("DB error"),
):
result = engine._get_full_content(items)
-
assert result == items
class TestClose:
+ """Tests for LibraryRAGSearchEngine.close."""
+
def test_close_is_noop(self):
engine = _make_engine()
- engine.close() # Should not raise
+ engine.close() # should not raise
diff --git a/tests/web_search_engines/test_engine_success_path.py b/tests/web_search_engines/test_engine_success_path.py
index 55f3cb4da5..b1670a6959 100644
--- a/tests/web_search_engines/test_engine_success_path.py
+++ b/tests/web_search_engines/test_engine_success_path.py
@@ -1408,6 +1408,8 @@ def _fake_rag_index_session(module):
embedding_model_type=SimpleNamespace(value="sentence_transformers"),
chunk_size=1000,
chunk_overlap=200,
+ splitter_type="recursive",
+ text_separators=None,
normalize_vectors=True,
distance_metric="cosine",
index_type="flat",