~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
refactor: extract shared ensure_in_collection utility (#2978)
LearningCircuitb05d8a9dd634
diff --git a/src/local_deep_research/research_library/routes/rag_routes.py b/src/local_deep_research/research_library/routes/rag_routes.pyindex eba557047c..c6448aed3a 100644--- a/src/local_deep_research/research_library/routes/rag_routes.py+++ b/src/local_deep_research/research_library/routes/rag_routes.py@@ -43,7 +43,7 @@ from ...security.rate_limiter import (upload_rate_limit_ip,upload_rate_limit_user,)-from ..utils import handle_api_error+from ..utils import ensure_in_collection, handle_api_errorfrom ...database.models.library import (Document,Collection,@@ -1450,14 +1450,9 @@ def upload_to_collection(collection_id):)if not existing_link:- # Add to collection- collection_link = DocumentCollection(- document_id=existing_doc.id,- collection_id=collection_id,- indexed=False,- chunk_count=0,+ ensure_in_collection(+ db_session, existing_doc.id, collection_id)- db_session.add(collection_link)status = "added_to_collection"if pdf_upgraded:status = "added_to_collection_pdf_upgraded"@@ -1614,13 +1609,9 @@ def upload_to_collection(collection_id):# Continue without PDF storage - text is still saved# Add to collection- collection_link = DocumentCollection(- document_id=new_doc.id,- collection_id=collection_id,- indexed=False,- chunk_count=0,+ ensure_in_collection(+ db_session, new_doc.id, collection_id)- db_session.add(collection_link)uploaded_files.append({diff --git a/src/local_deep_research/research_library/search/services/research_history_indexer.py b/src/local_deep_research/research_library/search/services/research_history_indexer.pyindex d01426a325..d1628662cd 100644--- a/src/local_deep_research/research_library/search/services/research_history_indexer.py+++ b/src/local_deep_research/research_library/search/services/research_history_indexer.py@@ -19,12 +19,12 @@ from ....constants import ResearchStatusfrom ....database.library_init import ensure_research_history_collectionfrom ....database.models.library import (Document,- DocumentCollection,DocumentStatus,SourceType,)from ....database.models.research import ResearchHistoryfrom ....database.session_context import get_user_db_session+from ...utils import ensure_in_collectionclass ResearchHistoryIndexer:@@ -301,7 +301,7 @@ class ResearchHistoryIndexer:if existing_doc:# Ensure it's in the collection- self._ensure_in_collection(existing_doc.id, collection_id, session)+ ensure_in_collection(session, existing_doc.id, collection_id)return existing_doc# Create document or reuse existing one with same content hash@@ -337,27 +337,9 @@ class ResearchHistoryIndexer:session.add(document)session.flush()- self._ensure_in_collection(document.id, collection_id, session)+ ensure_in_collection(session, document.id, collection_id)return document- def _ensure_in_collection(- self, document_id: str, collection_id: str, session- ) -> None:- """Add document to collection if not already there."""- existing = (- session.query(DocumentCollection)- .filter_by(document_id=document_id, collection_id=collection_id)- .first()- )- if not existing:- session.add(- DocumentCollection(- document_id=document_id,- collection_id=collection_id,- indexed=False,- )- )-def auto_convert_research(username: str, research_id: str, db_password: str | None = Nonediff --git a/src/local_deep_research/research_library/services/download_service.py b/src/local_deep_research/research_library/services/download_service.pyindex a532bfcad0..95da2aa318 100644--- a/src/local_deep_research/research_library/services/download_service.py+++ b/src/local_deep_research/research_library/services/download_service.py@@ -36,7 +36,6 @@ from ...database.models.library import (Document as Document,DocumentStatus,DownloadQueue as LibraryDownloadQueue,- DocumentCollection,)from .pdf_storage_manager import PDFStorageManagerfrom ...database.models.research import ResearchResource@@ -46,6 +45,7 @@ from ...utilities.db_utils import get_settings_managerfrom ...library.download_management import RetryManagerfrom ...config.paths import get_library_directoryfrom ..utils import (+ ensure_in_collection,get_document_for_resource,get_url_hash,get_absolute_path_from_settings,@@ -695,12 +695,7 @@ class DownloadService:)# Link document to default Library collection- doc_collection = DocumentCollection(- document_id=doc_id,- collection_id=library_collection_id,- indexed=False,- )- session.add(doc_collection)+ ensure_in_collection(session, doc_id, library_collection_id)# Update attemptattempt.succeeded = True@@ -1659,13 +1654,7 @@ class DownloadService:session.query(Collection).filter_by(name="Library").first())if library_collection:- doc_collection = DocumentCollection(- document_id=doc_id,- collection_id=library_collection.id,- indexed=False,- chunk_count=0,- )- session.add(doc_collection)+ ensure_in_collection(session, doc_id, library_collection.id)else:logger.warning(f"Library collection not found - document {doc_id} will not be linked to default collection"diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.pyindex 0e2634f86a..769d889e4b 100644--- a/src/local_deep_research/research_library/services/library_rag_service.py+++ b/src/local_deep_research/research_library/services/library_rag_service.py@@ -28,6 +28,7 @@ from ...database.models.library import ()from ...database.session_context import get_user_db_sessionfrom ...utilities.type_utils import to_bool+from ..utils import ensure_in_collectionfrom ...embeddings.splitters import get_text_splitterfrom ...web_search_engines.engines.local_embedding_manager import (LocalEmbeddingManager,@@ -543,33 +544,10 @@ class LibraryRAGService:return {"status": "error", "error": "Document not found"}# Get or create DocumentCollection entry- all_doc_collections = (- session.query(DocumentCollection)- .filter_by(document_id=document_id, collection_id=collection_id)- .all()- )-- logger.info(- f"Found {len(all_doc_collections)} DocumentCollection entries for doc={document_id}, coll={collection_id}"- )-- doc_collection = (- all_doc_collections[0] if all_doc_collections else None+ doc_collection = ensure_in_collection(+ session, document_id, collection_id)- if not doc_collection:- # Create new DocumentCollection entry- doc_collection = DocumentCollection(- document_id=document_id,- collection_id=collection_id,- indexed=False,- chunk_count=0,- )- session.add(doc_collection)- logger.info(- f"Created new DocumentCollection entry for doc={document_id}, coll={collection_id}"- )-# Check if already indexed for this collectionif doc_collection.indexed and not force_reindex:return {diff --git a/src/local_deep_research/research_library/utils/__init__.py b/src/local_deep_research/research_library/utils/__init__.pyindex 5a840845e8..89835368ba 100644--- a/src/local_deep_research/research_library/utils/__init__.py+++ b/src/local_deep_research/research_library/utils/__init__.py@@ -12,7 +12,7 @@ from flask import jsonifyfrom loguru import loggerfrom ...config.paths import get_library_directory-from ...database.models.library import Document+from ...database.models.library import Document, DocumentCollectionfrom ...security.path_validator import PathValidator@@ -150,6 +150,36 @@ def get_url_hash(url: str) -> str:return hashlib.sha256(url.lower().encode()).hexdigest()+def ensure_in_collection(+ session, document_id: str, collection_id: str+) -> "DocumentCollection":+ """Get or create a DocumentCollection link between a document and a collection.++ Args:+ session: SQLAlchemy session+ document_id: UUID of the document+ collection_id: UUID of the collection++ Returns:+ The existing or newly created DocumentCollection row+ """+ existing = (+ session.query(DocumentCollection)+ .filter_by(document_id=document_id, collection_id=collection_id)+ .first()+ )+ if existing:+ return existing++ doc_collection = DocumentCollection(+ document_id=document_id,+ collection_id=collection_id,+ indexed=False,+ )+ session.add(doc_collection)+ return doc_collection++def get_library_storage_path(username: str) -> Path:"""Get the storage path for a user's library.diff --git a/tests/research_library/services/test_download_service_coverage.py b/tests/research_library/services/test_download_service_coverage.pyindex 2bb9866b12..223b9de28c 100644--- a/tests/research_library/services/test_download_service_coverage.py+++ b/tests/research_library/services/test_download_service_coverage.py@@ -1370,6 +1370,7 @@ class TestSaveTextWithDb:patch(f"{MODULE}.get_document_for_resource", return_value=None),patch(f"{MODULE}.get_source_type_id", return_value="src-1"),patch(f"{MODULE}.uuid.uuid4", return_value="new-uuid"),+ patch(f"{MODULE}.ensure_in_collection") as mock_ensure,):# session.query(Collection).filter_by(name="Library").first() -> library_colsession.query.return_value.filter_by.return_value.first.return_value = library_col@@ -1380,7 +1381,10 @@ class TestSaveTextWithDb:extraction_method="native_api",extraction_source="arxiv_api",)- assert session.add.call_count == 2 # doc + doc_collection+ assert session.add.call_count == 1 # doc only+ mock_ensure.assert_called_once_with(+ session, "new-uuid", "lib-col-1"+ )def test_create_new_doc_no_library_collection(self, svc):session = MagicMock()diff --git a/tests/research_library/services/test_library_rag_service.py b/tests/research_library/services/test_library_rag_service.pyindex 5b525beab4..216d695f3f 100644--- a/tests/research_library/services/test_library_rag_service.py+++ b/tests/research_library/services/test_library_rag_service.py@@ -368,13 +368,19 @@ class TestLibraryRAGServiceIndexDocument:mock_session.__enter__ = Mock(return_value=mock_session)mock_session.__exit__ = Mock(return_value=False)mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc- mock_session.query.return_value.filter_by.return_value.all.return_value = []mocker.patch("local_deep_research.research_library.services.library_rag_service.get_user_db_session",return_value=mock_session,)+ # Mock ensure_in_collection to return a non-indexed DocumentCollection+ mock_dc = MagicMock(indexed=False, chunk_count=0)+ mocker.patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=mock_dc,+ )+# Mock embedding managermock_embedding_manager = Mock()mock_embedding_manager.embeddings = Mock()@@ -414,24 +420,23 @@ class TestLibraryRAGServiceIndexDocument:mock_doc.id = "doc-123"mock_doc.text_content = "Some text content"- # Mock document collection (already indexed)- mock_doc_collection = Mock()- mock_doc_collection.indexed = True- mock_doc_collection.chunk_count = 5-mock_session = MagicMock()mock_session.__enter__ = Mock(return_value=mock_session)mock_session.__exit__ = Mock(return_value=False)mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc- mock_session.query.return_value.filter_by.return_value.all.return_value = [- mock_doc_collection- ]mocker.patch("local_deep_research.research_library.services.library_rag_service.get_user_db_session",return_value=mock_session,)+ # Mock ensure_in_collection to return an already-indexed DocumentCollection+ mock_doc_collection = MagicMock(indexed=True, chunk_count=5)+ mocker.patch(+ "local_deep_research.research_library.services.library_rag_service.ensure_in_collection",+ return_value=mock_doc_collection,+ )+# Mock embedding managermock_embedding_manager = Mock()mock_embedding_manager.embeddings = Mock()diff --git a/tests/research_library/services/test_library_rag_service_coverage.py b/tests/research_library/services/test_library_rag_service_coverage.pyindex 58906406cd..b83e19740b 100644--- a/tests/research_library/services/test_library_rag_service_coverage.py+++ b/tests/research_library/services/test_library_rag_service_coverage.py@@ -544,8 +544,11 @@ class TestIndexDocument:assert result["status"] == "error"assert "not found" in result["error"]+ @patch(f"{_MOD}.ensure_in_collection")@patch(f"{_MOD}.get_user_db_session")- def test_returns_error_when_no_text_content(self, mock_session_ctx):+ def test_returns_error_when_no_text_content(+ self, mock_session_ctx, mock_ensure+ ):svc = _make_service()mock_session = MagicMock()mock_session_ctx.return_value.__enter__ = MagicMock(@@ -556,29 +559,19 @@ class TestIndexDocument:mock_document = MagicMock()mock_document.text_content = None- mock_doc_collection = MagicMock()- mock_doc_collection.indexed = False-- # query(Document).filter_by().first() -> document- # query(DocumentCollection).filter_by().all() -> [doc_collection]- def side_effect_query(model):- q = MagicMock()- if model.__name__ == "Document":- q.filter_by.return_value.first.return_value = mock_document- elif model.__name__ == "DocumentCollection":- q.filter_by.return_value.all.return_value = [- mock_doc_collection- ]- return q+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_document- mock_session.query = MagicMock(side_effect=side_effect_query)+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)result = svc.index_document("doc-1", "coll-1")assert result["status"] == "error"assert "no text content" in result["error"]+ @patch(f"{_MOD}.ensure_in_collection")@patch(f"{_MOD}.get_user_db_session")- def test_skips_already_indexed_document(self, mock_session_ctx):+ def test_skips_already_indexed_document(+ self, mock_session_ctx, mock_ensure+ ):svc = _make_service()mock_session = MagicMock()mock_session_ctx.return_value.__enter__ = MagicMock(@@ -587,21 +580,9 @@ class TestIndexDocument:mock_session_ctx.return_value.__exit__ = MagicMock(return_value=None)mock_document = MagicMock()- mock_doc_collection = MagicMock()- mock_doc_collection.indexed = True- mock_doc_collection.chunk_count = 42-- def side_effect_query(model):- q = MagicMock()- if model.__name__ == "Document":- q.filter_by.return_value.first.return_value = mock_document- elif model.__name__ == "DocumentCollection":- q.filter_by.return_value.all.return_value = [- mock_doc_collection- ]- return q-- mock_session.query = MagicMock(side_effect=side_effect_query)+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_document++ mock_ensure.return_value = MagicMock(indexed=True, chunk_count=42)result = svc.index_document("doc-1", "coll-1", force_reindex=False)assert result["status"] == "skipped"diff --git a/tests/research_library/services/test_library_rag_service_deep_coverage.py b/tests/research_library/services/test_library_rag_service_deep_coverage.pyindex b7cdf0186f..aae9cc5e5f 100644--- a/tests/research_library/services/test_library_rag_service_deep_coverage.py+++ b/tests/research_library/services/test_library_rag_service_deep_coverage.py@@ -886,8 +886,11 @@ class TestRemoveCollectionFromIndex:# index_document — exception mid-way# =========================================================================class TestIndexDocumentExceptionPath:+ @patch(f"{_MOD}.ensure_in_collection")@patch(f"{_MOD}.get_user_db_session")- def test_exception_during_splitting_returns_error(self, mock_session_ctx):+ def test_exception_during_splitting_returns_error(+ self, mock_session_ctx, mock_ensure+ ):svc = _make_service()mock_session = MagicMock()mock_session_ctx.return_value = _make_session_ctx(mock_session)@@ -908,24 +911,19 @@ class TestIndexDocumentExceptionPath:mock_document.extraction_method = Nonemock_document.word_count = None- mock_doc_collection = MagicMock()- mock_doc_collection.indexed = False-def query_side(model):q = MagicMock()model_name = getattr(model, "__name__", str(model))if model_name == "Document":q.filter_by.return_value.first.return_value = mock_document- elif model_name == "DocumentCollection":- q.filter_by.return_value.all.return_value = [- mock_doc_collection- ]elif model_name == "Collection":q.filter_by.return_value.first.return_value = MagicMock()return qmock_session.query = MagicMock(side_effect=query_side)+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)+svc.text_splitter = MagicMock()svc.text_splitter.split_documents.side_effect = RuntimeError("splitter broke"@@ -935,9 +933,12 @@ class TestIndexDocumentExceptionPath:assert result["status"] == "error"assert "RuntimeError" in result["error"]+ @patch(f"{_MOD}.ensure_in_collection")@patch(f"{_MOD}.get_user_db_session")- def test_creates_doc_collection_when_missing(self, mock_session_ctx):- """When no DocumentCollection exists, a new one is created before proceeding."""+ def test_creates_doc_collection_when_missing(+ self, mock_session_ctx, mock_ensure+ ):+ """When no DocumentCollection exists, ensure_in_collection is called."""svc = _make_service()mock_session = MagicMock()mock_session_ctx.return_value = _make_session_ctx(mock_session)@@ -945,19 +946,12 @@ class TestIndexDocumentExceptionPath:mock_document = MagicMock()mock_document.text_content = None # triggers error after dc creation- def query_side(model):- q = MagicMock()- model_name = getattr(model, "__name__", str(model))- if model_name == "Document":- q.filter_by.return_value.first.return_value = mock_document- elif model_name == "DocumentCollection":- q.filter_by.return_value.all.return_value = [] # empty -> create new- return q+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_document- mock_session.query = MagicMock(side_effect=query_side)+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)result = svc.index_document("doc-new", "coll-1")- # After creating the DocumentCollection, text_content is None → error+ # After ensure_in_collection, text_content is None → errorassert result["status"] == "error"assert "no text content" in result["error"]- mock_session.add.assert_called_once()+ mock_ensure.assert_called_once_with(mock_session, "doc-new", "coll-1")diff --git a/tests/research_library/services/test_library_rag_service_index_coverage.py b/tests/research_library/services/test_library_rag_service_index_coverage.pyindex f18cd1c8c5..5810297cb5 100644--- a/tests/research_library/services/test_library_rag_service_index_coverage.py+++ b/tests/research_library/services/test_library_rag_service_index_coverage.py@@ -93,50 +93,40 @@ def _make_doc_collection(class TestIndexDocumentNoTextContent:"""index_document returns {'status': 'error'} when document has no text."""+ @patch(f"{_MOD}.ensure_in_collection")@patch(f"{_MOD}.get_user_db_session")- def test_none_text_content_returns_error(self, mock_session_ctx):+ def test_none_text_content_returns_error(+ self, mock_session_ctx, mock_ensure+ ):svc = _make_service()mock_session = MagicMock()mock_session_ctx.return_value = _make_session_ctx(mock_session)mock_doc = _make_document(text_content=None)- mock_dc = _make_doc_collection(indexed=False)- def query_side(model):- q = MagicMock()- name = getattr(model, "__name__", str(model))- if "Document" == name:- q.filter_by.return_value.first.return_value = mock_doc- elif "DocumentCollection" in name:- q.filter_by.return_value.all.return_value = [mock_dc]- return q+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc- mock_session.query = MagicMock(side_effect=query_side)+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)result = svc.index_document("doc-1", "coll-1")assert result["status"] == "error"assert "no text content" in result["error"]+ @patch(f"{_MOD}.ensure_in_collection")@patch(f"{_MOD}.get_user_db_session")- def test_empty_string_text_content_returns_error(self, mock_session_ctx):+ def test_empty_string_text_content_returns_error(+ self, mock_session_ctx, mock_ensure+ ):svc = _make_service()mock_session = MagicMock()mock_session_ctx.return_value = _make_session_ctx(mock_session)mock_doc = _make_document(text_content="")- mock_dc = _make_doc_collection(indexed=False)- def query_side(model):- q = MagicMock()- name = getattr(model, "__name__", str(model))- if "Document" == name:- q.filter_by.return_value.first.return_value = mock_doc- elif "DocumentCollection" in name:- q.filter_by.return_value.all.return_value = [mock_dc]- return q+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc- mock_session.query = MagicMock(side_effect=query_side)+ mock_ensure.return_value = MagicMock(indexed=False, chunk_count=0)result = svc.index_document("doc-1", "coll-1")@@ -163,27 +153,20 @@ class TestIndexDocumentNoTextContent:assert result["status"] == "error"assert "not found" in result["error"]+ @patch(f"{_MOD}.ensure_in_collection")@patch(f"{_MOD}.get_user_db_session")def test_already_indexed_without_force_returns_skipped(- self, mock_session_ctx+ self, mock_session_ctx, mock_ensure):svc = _make_service()mock_session = MagicMock()mock_session_ctx.return_value = _make_session_ctx(mock_session)mock_doc = _make_document()- mock_dc = _make_doc_collection(indexed=True, chunk_count=5)- def query_side(model):- q = MagicMock()- name = getattr(model, "__name__", str(model))- if "Document" == name:- q.filter_by.return_value.first.return_value = mock_doc- elif "DocumentCollection" in name:- q.filter_by.return_value.all.return_value = [mock_dc]- return q+ mock_session.query.return_value.filter_by.return_value.first.return_value = mock_doc- mock_session.query = MagicMock(side_effect=query_side)+ mock_ensure.return_value = MagicMock(indexed=True, chunk_count=5)result = svc.index_document("doc-1", "coll-1", force_reindex=False)diff --git a/tests/research_library/utils/test_utils.py b/tests/research_library/utils/test_utils.pyindex 023527fcd0..9e251d0975 100644--- a/tests/research_library/utils/test_utils.py+++ b/tests/research_library/utils/test_utils.py@@ -289,3 +289,48 @@ class TestHandleApiError:with app.app_context():_, status_code = handle_api_error("op", RuntimeError("err"))assert status_code == 500+++# ---------------------------------------------------------------------------+# ensure_in_collection+# ---------------------------------------------------------------------------+++class TestEnsureInCollection:+ """Tests for ensure_in_collection()."""++ def test_returns_existing_row_without_adding(self):+ from local_deep_research.research_library.utils import (+ ensure_in_collection,+ )++ existing = MagicMock(name="existing_DocumentCollection")+ session = MagicMock()+ session.query.return_value.filter_by.return_value.first.return_value = (+ existing+ )++ result = ensure_in_collection(session, "doc-1", "coll-1")++ assert result is existing+ session.query.return_value.filter_by.assert_called_once_with(+ document_id="doc-1", collection_id="coll-1"+ )+ session.add.assert_not_called()++ def test_creates_and_adds_when_missing(self):+ from local_deep_research.research_library.utils import (+ ensure_in_collection,+ )++ session = MagicMock()+ session.query.return_value.filter_by.return_value.first.return_value = (+ None+ )++ result = ensure_in_collection(session, "doc-2", "coll-2")++ session.add.assert_called_once_with(result)+ assert result.document_id == "doc-2"+ assert result.collection_id == "coll-2"+ assert result.indexed is False