Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

refactor: extract shared ensure_in_collection utility (#2978)

LearningCircuitb05d8a9dd634
diff --git a/src/local_deep_research/research_library/routes/rag_routes.py b/src/local_deep_research/research_library/routes/rag_routes.py
index eba557047c..c6448aed3a 100644
--- a/src/local_deep_research/research_library/routes/rag_routes.py
+++ b/src/local_deep_research/research_library/routes/rag_routes.py
@@ -43,7 +43,7 @@ from ...security.rate_limiter import (
upload_rate_limit_ip,
upload_rate_limit_user,
)
-from ..utils import handle_api_error
+from ..utils import ensure_in_collection, handle_api_error
from ...database.models.library import (
Document,
Collection,
@@ -1450,14 +1450,9 @@ def upload_to_collection(collection_id):
)
if not existing_link:
- # Add to collection
- collection_link = DocumentCollection(
- document_id=existing_doc.id,
- collection_id=collection_id,
- indexed=False,
- chunk_count=0,
+ ensure_in_collection(
+ db_session, existing_doc.id, collection_id
)
- db_session.add(collection_link)
status = "added_to_collection"
if pdf_upgraded:
status = "added_to_collection_pdf_upgraded"
@@ -1614,13 +1609,9 @@ def upload_to_collection(collection_id):
# Continue without PDF storage - text is still saved
# Add to collection
- collection_link = DocumentCollection(
- document_id=new_doc.id,
- collection_id=collection_id,
- indexed=False,
- chunk_count=0,
+ ensure_in_collection(
+ db_session, new_doc.id, collection_id
)
- db_session.add(collection_link)
uploaded_files.append(
{
diff --git a/src/local_deep_research/research_library/search/services/research_history_indexer.py b/src/local_deep_research/research_library/search/services/research_history_indexer.py
index d01426a325..d1628662cd 100644
--- a/src/local_deep_research/research_library/search/services/research_history_indexer.py
+++ b/src/local_deep_research/research_library/search/services/research_history_indexer.py
@@ -19,12 +19,12 @@ from ....constants import ResearchStatus
from ....database.library_init import ensure_research_history_collection
from ....database.models.library import (
Document,
- DocumentCollection,
DocumentStatus,
SourceType,
)
from ....database.models.research import ResearchHistory
from ....database.session_context import get_user_db_session
+from ...utils import ensure_in_collection
class ResearchHistoryIndexer:
@@ -301,7 +301,7 @@ class ResearchHistoryIndexer:
if existing_doc:
# Ensure it's in the collection
- self._ensure_in_collection(existing_doc.id, collection_id, session)
+ ensure_in_collection(session, existing_doc.id, collection_id)
return existing_doc
# Create document or reuse existing one with same content hash
@@ -337,27 +337,9 @@ class ResearchHistoryIndexer:
session.add(document)
session.flush()
- self._ensure_in_collection(document.id, collection_id, session)
+ ensure_in_collection(session, document.id, collection_id)
return document
- def _ensure_in_collection(
- self, document_id: str, collection_id: str, session
- ) -> None:
- """Add document to collection if not already there."""
- existing = (
- session.query(DocumentCollection)
- .filter_by(document_id=document_id, collection_id=collection_id)
- .first()
- )
- if not existing:
- session.add(
- DocumentCollection(
- document_id=document_id,
- collection_id=collection_id,
- indexed=False,
- )
- )
-
def auto_convert_research(
username: str, research_id: str, db_password: str | None = None
diff --git a/src/local_deep_research/research_library/services/download_service.py b/src/local_deep_research/research_library/services/download_service.py
index a532bfcad0..95da2aa318 100644
--- a/src/local_deep_research/research_library/services/download_service.py
+++ b/src/local_deep_research/research_library/services/download_service.py
@@ -36,7 +36,6 @@ from ...database.models.library import (
Document as Document,
DocumentStatus,
DownloadQueue as LibraryDownloadQueue,
- DocumentCollection,
)
from .pdf_storage_manager import PDFStorageManager
from ...database.models.research import ResearchResource
@@ -46,6 +45,7 @@ from ...utilities.db_utils import get_settings_manager
from ...library.download_management import RetryManager
from ...config.paths import get_library_directory
from ..utils import (
+ ensure_in_collection,
get_document_for_resource,
get_url_hash,
get_absolute_path_from_settings,
@@ -695,12 +695,7 @@ class DownloadService:
)
# Link document to default Library collection
- doc_collection = DocumentCollection(
- document_id=doc_id,
- collection_id=library_collection_id,
- indexed=False,
- )
- session.add(doc_collection)
+ ensure_in_collection(session, doc_id, library_collection_id)
# Update attempt
attempt.succeeded = True
@@ -1659,13 +1654,7 @@ class DownloadService:
session.query(Collection).filter_by(name="Library").first()
)
if library_collection:
- doc_collection = DocumentCollection(
- document_id=doc_id,
- collection_id=library_collection.id,
- indexed=False,
- chunk_count=0,
- )
- session.add(doc_collection)
+ ensure_in_collection(session, doc_id, library_collection.id)
else:
logger.warning(
f"Library collection not found - document {doc_id} will not be linked to default collection"
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.py
index 0e2634f86a..769d889e4b 100644
--- a/src/local_deep_research/research_library/services/library_rag_service.py
+++ b/src/local_deep_research/research_library/services/library_rag_service.py
@@ -28,6 +28,7 @@ from ...database.models.library import (
)
from ...database.session_context import get_user_db_session
from ...utilities.type_utils import to_bool
+from ..utils import ensure_in_collection
from ...embeddings.splitters import get_text_splitter
from ...web_search_engines.engines.local_embedding_manager import (
LocalEmbeddingManager,
@@ -543,33 +544,10 @@ class LibraryRAGService:
return {"status": "error", "error": "Document not found"}
# Get or create DocumentCollection entry
- all_doc_collections = (
- session.query(DocumentCollection)
- .filter_by(document_id=document_id, collection_id=collection_id)
- .all()
- )
-
- logger.info(
- f"Found {len(all_doc_collections)} DocumentCollection entries for doc={document_id}, coll={collection_id}"
- )
-
- doc_collection = (
- all_doc_collections[0] if all_doc_collections else None
+ doc_collection = ensure_in_collection(
+ session, document_id, collection_id
)
- if not doc_collection:
- # Create new DocumentCollection entry
- doc_collection = DocumentCollection(
- document_id=document_id,
- collection_id=collection_id,
- indexed=False,
- chunk_count=0,
- )
- session.add(doc_collection)
- logger.info(
- f"Created new DocumentCollection entry for doc={document_id}, coll={collection_id}"
- )
-
# Check if already indexed for this collection
if doc_collection.indexed and not force_reindex:
return {
diff --git a/src/local_deep_research/research_library/utils/__init__.py b/src/local_deep_research/research_library/utils/__init__.py
index 5a840845e8..89835368ba 100644
--- a/src/local_deep_research/research_library/utils/__init__.py
+++ b/src/local_deep_research/research_library/utils/__init__.py
@@ -12,7 +12,7 @@ from flask import jsonify
from loguru import logger
from ...config.paths import get_library_directory
-from ...database.models.library import Document
+from ...database.models.library import Document, DocumentCollection
from ...security.path_validator import PathValidator
@@ -150,6 +150,36 @@ def get_url_hash(url: str) -> str:
return hashlib.sha256(url.lower().encode()).hexdigest()
+def ensure_in_collection(
+ session, document_id: str, collection_id: str
+) -> "DocumentCollection":
+ """Get or create a DocumentCollection link between a document and a collection.
+
+ Args:
+ session: SQLAlchemy session
+ document_id: UUID of the document
+ collection_id: UUID of the collection
+
+ Returns:
+ The existing or newly created DocumentCollection row
+ """
+ existing = (
+ session.query(DocumentCollection)
+ .filter_by(document_id=document_id, collection_id=collection_id)
+ .first()
+ )
+ if existing:
+ return existing
+
+ doc_collection = DocumentCollection(
+ document_id=document_id,
+ collection_id=collection_id,
+ indexed=False,
+ )
+ session.add(doc_collection)
+ return doc_collection
+
+
def get_library_storage_path(username: str) -> Path:
"""
Get the storage path for a user's library.
diff --git a/tests/research_library/services/test_download_service_coverage.py b/tests/research_library/services/test_download_service_coverage.py
index 2bb9866b12..223b9de28c 100644
--- a/tests/research_library/services/test_download_service_coverage.py
+++ b/tests/research_library/services/test_download_service_coverage.py
@@ -1370,6 +1370,7 @@ class TestSaveTextWithDb:
patch(f"{MODULE}.get_document_for_resource", return_value=None),
patch(f"{MODULE}.get_source_type_id", return_value="src-1"),
patch(f"{MODULE}.uuid.uuid4", return_value="new-uuid"),
+ patch(f"{MODULE}.ensure_in_collection") as mock_ensure,
):
# session.query(Collection).filter_by(name="Library").first() -> library_col
session.query.return_value.filter_by.return_value.first.return_value = library_col
@@ -1380,7 +1381,10 @@ class TestSaveTextWithDb:
extraction_method="native_api",
extraction_source="arxiv_api",
)
- assert session.add.call_count == 2 # doc + doc_collection
+ assert session.add.call_count == 1 # doc only
+ mock_ensure.assert_called_once_with(
+ session, "new-uuid", "lib-col-1"
+ )
def test_create_new_doc_no_library_collection(self, svc):
session = MagicMock()
diff --git a/tests/research_library/services/test_library_rag_service.py b/tests/research_library/services/test_library_rag_service.py
index 5b525beab4..216d695f3f 100644
--- a/tests/research_library/services/test_library_rag_service.py
+++ b/tests/research_library/services/test_library_rag_service.py
@@ -368,13 +368,19 @@ class TestLibraryRAGServiceIndexDocument:
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc
- mock_session.query.return_value.filter_by.return_value.all.return_value = []
mocker.patch(
"local_deep_research.research_library.services.library_rag_service.get_user_db_session",
return_value=mock_session,
)
+ # Mock ensure_in_collection to return a non-indexed DocumentCollection
+ mock_dc = MagicMock(indexed=False, chunk_count=0)
+ mocker.patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=mock_dc,
+ )
+
# Mock embedding manager
mock_embedding_manager = Mock()
mock_embedding_manager.embeddings = Mock()
@@ -414,24 +420,23 @@ class TestLibraryRAGServiceIndexDocument:
mock_doc.id = "doc-123"
mock_doc.text_content = "Some text content"
- # Mock document collection (already indexed)
- mock_doc_collection = Mock()
- mock_doc_collection.indexed = True
- mock_doc_collection.chunk_count = 5
-
mock_session = MagicMock()
mock_session.__enter__ = Mock(return_value=mock_session)
mock_session.__exit__ = Mock(return_value=False)
mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc
- mock_session.query.return_value.filter_by.return_value.all.return_value = [
- mock_doc_collection
- ]
mocker.patch(
"local_deep_research.research_library.services.library_rag_service.get_user_db_session",
return_value=mock_session,
)
+ # Mock ensure_in_collection to return an already-indexed DocumentCollection
+ mock_doc_collection = MagicMock(indexed=True, chunk_count=5)
+ mocker.patch(
+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",
+ return_value=mock_doc_collection,
+ )
+
# Mock embedding manager
mock_embedding_manager = Mock()
mock_embedding_manager.embeddings = Mock()
diff --git a/tests/research_library/services/test_library_rag_service_coverage.py b/tests/research_library/services/test_library_rag_service_coverage.py
index 58906406cd..b83e19740b 100644
--- a/tests/research_library/services/test_library_rag_service_coverage.py
+++ b/tests/research_library/services/test_library_rag_service_coverage.py
@@ -544,8 +544,11 @@ class TestIndexDocument:
assert result["status"] == "error"
assert "not found" in result["error"]
+ @patch(f"{_MOD}.ensure_in_collection")
@patch(f"{_MOD}.get_user_db_session")
- def test_returns_error_when_no_text_content(self, mock_session_ctx):
+ def test_returns_error_when_no_text_content(
+ self, mock_session_ctx, mock_ensure
+ ):
svc = _make_service()
mock_session = MagicMock()
mock_session_ctx.return_value.__enter__ = MagicMock(
@@ -556,29 +559,19 @@ class TestIndexDocument:
mock_document = MagicMock()
mock_document.text_content = None
- mock_doc_collection = MagicMock()
- mock_doc_collection.indexed = False
-
- # query(Document).filter_by().first() -> document
- # query(DocumentCollection).filter_by().all() -> [doc_collection]
- def side_effect_query(model):
- q = MagicMock()
- if model.__name__ == "Document":
- q.filter_by.return_value.first.return_value = mock_document
- elif model.__name__ == "DocumentCollection":
- q.filter_by.return_value.all.return_value = [
- mock_doc_collection
- ]
- return q
+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_document
- mock_session.query = MagicMock(side_effect=side_effect_query)
+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)
result = svc.index_document("doc-1", "coll-1")
assert result["status"] == "error"
assert "no text content" in result["error"]
+ @patch(f"{_MOD}.ensure_in_collection")
@patch(f"{_MOD}.get_user_db_session")
- def test_skips_already_indexed_document(self, mock_session_ctx):
+ def test_skips_already_indexed_document(
+ self, mock_session_ctx, mock_ensure
+ ):
svc = _make_service()
mock_session = MagicMock()
mock_session_ctx.return_value.__enter__ = MagicMock(
@@ -587,21 +580,9 @@ class TestIndexDocument:
mock_session_ctx.return_value.__exit__ = MagicMock(return_value=None)
mock_document = MagicMock()
- mock_doc_collection = MagicMock()
- mock_doc_collection.indexed = True
- mock_doc_collection.chunk_count = 42
-
- def side_effect_query(model):
- q = MagicMock()
- if model.__name__ == "Document":
- q.filter_by.return_value.first.return_value = mock_document
- elif model.__name__ == "DocumentCollection":
- q.filter_by.return_value.all.return_value = [
- mock_doc_collection
- ]
- return q
-
- mock_session.query = MagicMock(side_effect=side_effect_query)
+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_document
+
+ mock_ensure.return_value = MagicMock(indexed=True, chunk_count=42)
result = svc.index_document("doc-1", "coll-1", force_reindex=False)
assert result["status"] == "skipped"
diff --git a/tests/research_library/services/test_library_rag_service_deep_coverage.py b/tests/research_library/services/test_library_rag_service_deep_coverage.py
index b7cdf0186f..aae9cc5e5f 100644
--- a/tests/research_library/services/test_library_rag_service_deep_coverage.py
+++ b/tests/research_library/services/test_library_rag_service_deep_coverage.py
@@ -886,8 +886,11 @@ class TestRemoveCollectionFromIndex:
# index_document — exception mid-way
# =========================================================================
class TestIndexDocumentExceptionPath:
+ @patch(f"{_MOD}.ensure_in_collection")
@patch(f"{_MOD}.get_user_db_session")
- def test_exception_during_splitting_returns_error(self, mock_session_ctx):
+ def test_exception_during_splitting_returns_error(
+ self, mock_session_ctx, mock_ensure
+ ):
svc = _make_service()
mock_session = MagicMock()
mock_session_ctx.return_value = _make_session_ctx(mock_session)
@@ -908,24 +911,19 @@ class TestIndexDocumentExceptionPath:
mock_document.extraction_method = None
mock_document.word_count = None
- mock_doc_collection = MagicMock()
- mock_doc_collection.indexed = False
-
def query_side(model):
q = MagicMock()
model_name = getattr(model, "__name__", str(model))
if model_name == "Document":
q.filter_by.return_value.first.return_value = mock_document
- elif model_name == "DocumentCollection":
- q.filter_by.return_value.all.return_value = [
- mock_doc_collection
- ]
elif model_name == "Collection":
q.filter_by.return_value.first.return_value = MagicMock()
return q
mock_session.query = MagicMock(side_effect=query_side)
+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)
+
svc.text_splitter = MagicMock()
svc.text_splitter.split_documents.side_effect = RuntimeError(
"splitter broke"
@@ -935,9 +933,12 @@ class TestIndexDocumentExceptionPath:
assert result["status"] == "error"
assert "RuntimeError" in result["error"]
+ @patch(f"{_MOD}.ensure_in_collection")
@patch(f"{_MOD}.get_user_db_session")
- def test_creates_doc_collection_when_missing(self, mock_session_ctx):
- """When no DocumentCollection exists, a new one is created before proceeding."""
+ def test_creates_doc_collection_when_missing(
+ self, mock_session_ctx, mock_ensure
+ ):
+ """When no DocumentCollection exists, ensure_in_collection is called."""
svc = _make_service()
mock_session = MagicMock()
mock_session_ctx.return_value = _make_session_ctx(mock_session)
@@ -945,19 +946,12 @@ class TestIndexDocumentExceptionPath:
mock_document = MagicMock()
mock_document.text_content = None # triggers error after dc creation
- def query_side(model):
- q = MagicMock()
- model_name = getattr(model, "__name__", str(model))
- if model_name == "Document":
- q.filter_by.return_value.first.return_value = mock_document
- elif model_name == "DocumentCollection":
- q.filter_by.return_value.all.return_value = [] # empty -> create new
- return q
+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_document
- mock_session.query = MagicMock(side_effect=query_side)
+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)
result = svc.index_document("doc-new", "coll-1")
- # After creating the DocumentCollection, text_content is None → error
+ # After ensure_in_collection, text_content is None → error
assert result["status"] == "error"
assert "no text content" in result["error"]
- mock_session.add.assert_called_once()
+ mock_ensure.assert_called_once_with(mock_session, "doc-new", "coll-1")
diff --git a/tests/research_library/services/test_library_rag_service_index_coverage.py b/tests/research_library/services/test_library_rag_service_index_coverage.py
index f18cd1c8c5..5810297cb5 100644
--- a/tests/research_library/services/test_library_rag_service_index_coverage.py
+++ b/tests/research_library/services/test_library_rag_service_index_coverage.py
@@ -93,50 +93,40 @@ def _make_doc_collection(
class TestIndexDocumentNoTextContent:
"""index_document returns {'status': 'error'} when document has no text."""
+ @patch(f"{_MOD}.ensure_in_collection")
@patch(f"{_MOD}.get_user_db_session")
- def test_none_text_content_returns_error(self, mock_session_ctx):
+ def test_none_text_content_returns_error(
+ self, mock_session_ctx, mock_ensure
+ ):
svc = _make_service()
mock_session = MagicMock()
mock_session_ctx.return_value = _make_session_ctx(mock_session)
mock_doc = _make_document(text_content=None)
- mock_dc = _make_doc_collection(indexed=False)
- def query_side(model):
- q = MagicMock()
- name = getattr(model, "__name__", str(model))
- if "Document" == name:
- q.filter_by.return_value.first.return_value = mock_doc
- elif "DocumentCollection" in name:
- q.filter_by.return_value.all.return_value = [mock_dc]
- return q
+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc
- mock_session.query = MagicMock(side_effect=query_side)
+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)
result = svc.index_document("doc-1", "coll-1")
assert result["status"] == "error"
assert "no text content" in result["error"]
+ @patch(f"{_MOD}.ensure_in_collection")
@patch(f"{_MOD}.get_user_db_session")
- def test_empty_string_text_content_returns_error(self, mock_session_ctx):
+ def test_empty_string_text_content_returns_error(
+ self, mock_session_ctx, mock_ensure
+ ):
svc = _make_service()
mock_session = MagicMock()
mock_session_ctx.return_value = _make_session_ctx(mock_session)
mock_doc = _make_document(text_content="")
- mock_dc = _make_doc_collection(indexed=False)
- def query_side(model):
- q = MagicMock()
- name = getattr(model, "__name__", str(model))
- if "Document" == name:
- q.filter_by.return_value.first.return_value = mock_doc
- elif "DocumentCollection" in name:
- q.filter_by.return_value.all.return_value = [mock_dc]
- return q
+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc
- mock_session.query = MagicMock(side_effect=query_side)
+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)
result = svc.index_document("doc-1", "coll-1")
@@ -163,27 +153,20 @@ class TestIndexDocumentNoTextContent:
assert result["status"] == "error"
assert "not found" in result["error"]
+ @patch(f"{_MOD}.ensure_in_collection")
@patch(f"{_MOD}.get_user_db_session")
def test_already_indexed_without_force_returns_skipped(
- self, mock_session_ctx
+ self, mock_session_ctx, mock_ensure
):
svc = _make_service()
mock_session = MagicMock()
mock_session_ctx.return_value = _make_session_ctx(mock_session)
mock_doc = _make_document()
- mock_dc = _make_doc_collection(indexed=True, chunk_count=5)
- def query_side(model):
- q = MagicMock()
- name = getattr(model, "__name__", str(model))
- if "Document" == name:
- q.filter_by.return_value.first.return_value = mock_doc
- elif "DocumentCollection" in name:
- q.filter_by.return_value.all.return_value = [mock_dc]
- return q
+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc
- mock_session.query = MagicMock(side_effect=query_side)
+ mock_ensure.return_value = MagicMock(indexed=True, chunk_count=5)
result = svc.index_document("doc-1", "coll-1", force_reindex=False)
diff --git a/tests/research_library/utils/test_utils.py b/tests/research_library/utils/test_utils.py
index 023527fcd0..9e251d0975 100644
--- a/tests/research_library/utils/test_utils.py
+++ b/tests/research_library/utils/test_utils.py
@@ -289,3 +289,48 @@ class TestHandleApiError:
with app.app_context():
_, status_code = handle_api_error("op", RuntimeError("err"))
assert status_code == 500
+
+
+# ---------------------------------------------------------------------------
+# ensure_in_collection
+# ---------------------------------------------------------------------------
+
+
+class TestEnsureInCollection:
+ """Tests for ensure_in_collection()."""
+
+ def test_returns_existing_row_without_adding(self):
+ from local_deep_research.research_library.utils import (
+ ensure_in_collection,
+ )
+
+ existing = MagicMock(name="existing_DocumentCollection")
+ session = MagicMock()
+ session.query.return_value.filter_by.return_value.first.return_value = (
+ existing
+ )
+
+ result = ensure_in_collection(session, "doc-1", "coll-1")
+
+ assert result is existing
+ session.query.return_value.filter_by.assert_called_once_with(
+ document_id="doc-1", collection_id="coll-1"
+ )
+ session.add.assert_not_called()
+
+ def test_creates_and_adds_when_missing(self):
+ from local_deep_research.research_library.utils import (
+ ensure_in_collection,
+ )
+
+ session = MagicMock()
+ session.query.return_value.filter_by.return_value.first.return_value = (
+ None
+ )
+
+ result = ensure_in_collection(session, "doc-2", "coll-2")
+
+ session.add.assert_called_once_with(result)
+ assert result.document_id == "doc-2"
+ assert result.collection_id == "coll-2"
+ assert result.indexed is False