~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
refactor(library): remove obsolete index_user_document (#5077)
LearningCircuit790604ebd880
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.pyindex 04bac31942..e7bde612d9 100644--- a/src/local_deep_research/research_library/services/library_rag_service.py+++ b/src/local_deep_research/research_library/services/library_rag_service.py@@ -1514,152 +1514,6 @@ class LibraryRAGService:"chunk_overlap": self.chunk_overlap,}- def index_user_document(- self, user_doc, collection_name: str, force_reindex: bool = False- ) -> Dict[str, Any]:- """- Index a user-uploaded document into a specific collection.-- Args:- user_doc: UserDocument object- collection_name: Name of the collection (e.g., "collection_123")- force_reindex: Whether to force reindexing-- Returns:- Dict with status, chunk_count, and any errors- """-- try:- # Use the pre-extracted text content- content = user_doc.text_content-- if not content or len(content.strip()) < 10:- return {- "status": "error",- "error": "Document has no extractable text content",- }-- # Create LangChain Document- doc = LangchainDocument(- page_content=content,- metadata={- "source": f"user_upload_{user_doc.id}",- "source_id": user_doc.id,- "title": user_doc.filename,- "document_title": user_doc.filename,- "file_type": user_doc.file_type,- "file_size": user_doc.file_size,- "collection": collection_name,- },- )-- # Split into chunks- chunks = self.text_splitter.split_documents([doc])- logger.info(- f"Split user document {user_doc.filename} into {len(chunks)} chunks"- )-- # Store chunks in database- embedding_ids = self.embedding_manager._store_chunks_to_db(- chunks=chunks,- collection_name=collection_name,- source_type="user_document",- source_id=user_doc.id,- )-- # Load or create FAISS index for this collection (lazy;- # merge step below reloads under the lock anyway).- if self.faiss_index is None:- # Extract collection_id from collection_name (format: "collection_<uuid>")- collection_id = collection_name.removeprefix("collection_")- self.faiss_index = self.load_or_create_faiss_index(- collection_id- )-- unique_count = len(set(embedding_ids))- batch_dups = len(chunks) - unique_count-- # Read-modify-write the on-disk FAISS under the lock so- # concurrent uploads to the same collection don't lose- # each other's chunks.- index_path = (- Path(self.rag_index_record.index_path)- if self.rag_index_record- else None- )- if index_path:- merge_stats = self._merge_and_persist_locked(- index_path,- chunks,- embedding_ids,- force_reindex=force_reindex,- )- else:- # No persistent index path — in-memory only path.- # Preserve old behavior: handle force_reindex deletion- # and dedup add against in-memory state without saving.- if force_reindex and hasattr(self.faiss_index, "docstore"):- existing_ids = set(self.faiss_index.docstore._dict.keys())- old_chunk_ids = list(- {eid for eid in embedding_ids if eid in existing_ids}- )- if old_chunk_ids:- logger.info(- f"Force re-index: removing {len(old_chunk_ids)} "- f"existing chunks from FAISS"- )- self.faiss_index.delete(old_chunk_ids)- if not force_reindex and hasattr(self.faiss_index, "docstore"):- existing_ids = set(self.faiss_index.docstore._dict.keys())- else:- existing_ids = None- new_chunks, new_ids = self._deduplicate_chunks(- chunks, embedding_ids, existing_ids- )- if new_chunks:- self.faiss_index.add_documents(new_chunks, ids=new_ids)- merge_stats = {- "added": len(new_chunks),- "skipped": len(chunks) - len(new_chunks),- "added_ids": new_ids,- }- if merge_stats["added"]:- if force_reindex:- logger.info(- f"Force re-index: added {merge_stats['added']} "- f"chunks with updated metadata to FAISS index"- )- else:- already_exist = unique_count - merge_stats["added"]- logger.info(- f"Added {merge_stats['added']} new chunks to FAISS "- f"({already_exist} already exist, "- f"{batch_dups} batch duplicates removed)"- )- else:- logger.info(- f"All {len(chunks)} chunks already exist in FAISS index, skipping"- )-- logger.info(- f"Successfully indexed user document {user_doc.filename} with {len(chunks)} chunks"- )-- return {- "status": "success",- "chunk_count": len(chunks),- "embedding_ids": embedding_ids,- }-- except Exception as e:- logger.exception(- f"Error indexing user document {user_doc.filename}"- )- return {- "status": "error",- "error": f"Operation failed: {type(e).__name__}",- }-def remove_collection_from_index(self, collection_name: str) -> Dict[str, Any]:diff --git a/tests/research_library/services/test_library_rag_service.py b/tests/research_library/services/test_library_rag_service.pyindex 5802c27180..a01b01d36b 100644--- a/tests/research_library/services/test_library_rag_service.py+++ b/tests/research_library/services/test_library_rag_service.py@@ -1067,9 +1067,8 @@ class TestRemoveCollectionFromIndex:class TestLoadOrCreateFaissIndexCollectionId:"""Tests that load_or_create_faiss_index receives the correct collection_id.- Fixes a pre-existing bug where index_user_document and- remove_collection_from_index called load_or_create_faiss_index()- without the required collection_id argument.+ Fixes a pre-existing bug where remove_collection_from_index called+ load_or_create_faiss_index() without the required collection_id argument."""def _create_service(self, mocker):@@ -1109,52 +1108,6 @@ class TestLoadOrCreateFaissIndexCollectionId:)return service- def test_index_user_document_passes_extracted_collection_id(self, mocker):- """index_user_document extracts collection_id from collection_name."""- service = self._create_service(mocker)-- # Create mock user document- mock_user_doc = Mock()- mock_user_doc.id = "user-doc-123"- mock_user_doc.text_content = (- "Enough content for indexing test purposes."- )- mock_user_doc.filename = "test.pdf"- mock_user_doc.file_type = "pdf"- mock_user_doc.file_size = 1024-- # Mock FAISS index as None to trigger load- service.faiss_index = None- mock_faiss_index = MagicMock()- mock_faiss_index.docstore = Mock()- mock_faiss_index.docstore._dict = {}- service.load_or_create_faiss_index = Mock(return_value=mock_faiss_index)-- # Mock _store_chunks_to_db- service.embedding_manager._store_chunks_to_db = Mock(- return_value=["chunk-1", "chunk-2"]- )-- # Mock text_splitter- from langchain_core.documents import Document as LangchainDocument-- mock_chunks = [- LangchainDocument(page_content="chunk 1"),- LangchainDocument(page_content="chunk 2"),- ]- service.text_splitter.split_documents.return_value = mock_chunks-- # Mock rag_index_record for save path- service.rag_index_record = None-- collection_name = "collection_abc-uuid-123"- service.index_user_document(mock_user_doc, collection_name)-- # Verify collection_id was correctly extracted and passed- service.load_or_create_faiss_index.assert_called_once_with(- "abc-uuid-123"- )-def test_remove_collection_passes_extracted_collection_id(self, mocker):"""remove_collection_from_index extracts collection_id from collection_name."""service = self._create_service(mocker)@@ -1440,52 +1393,3 @@ class TestForceReindexDedup:assert len(delete_call) == len(set(delete_call)), ("old_chunk_ids passed to FAISS.delete() must not contain duplicates")-- def test_force_reindex_old_chunk_ids_deduplicated_index_user_document(- self, mocker- ):- """Verify deletion path in index_user_document uses unique IDs."""- service = _create_rag_service(mocker)-- # Mock user document- mock_user_doc = Mock()- mock_user_doc.id = "udoc-1"- mock_user_doc.filename = "test.txt"- mock_user_doc.text_content = "Some content here for testing"- mock_user_doc.file_type = "txt"- mock_user_doc.file_size = 100-- chunks = [- LangchainDocument(page_content="chunk A"),- LangchainDocument(page_content="chunk B"),- LangchainDocument(page_content="chunk A dup"),- ]- service.text_splitter.split_documents.return_value = chunks-- # _store_chunks_to_db returns IDs with duplicates- duplicate_ids = ["id-1", "id-2", "id-1"]- service.embedding_manager._store_chunks_to_db.return_value = (- duplicate_ids- )-- # Setup FAISS mock- mock_faiss = Mock()- mock_docstore = Mock()- mock_docstore._dict = {"id-1": "doc", "id-2": "doc"}- mock_faiss.docstore = mock_docstore- service.faiss_index = mock_faiss-- mock_rag_index = Mock()- mock_rag_index.index_path = "/tmp/test.faiss"- mock_rag_index.id = "rag-1"- service.rag_index_record = mock_rag_index-- service.index_user_document(- mock_user_doc, "collection_123", force_reindex=True- )-- # Verify delete was called with deduplicated IDs- delete_call = mock_faiss.delete.call_args[0][0]- assert len(delete_call) == len(set(delete_call)), (- "old_chunk_ids passed to FAISS.delete() must not contain duplicates"- )diff --git a/tests/research_library/services/test_library_rag_service_deep_coverage.py b/tests/research_library/services/test_library_rag_service_deep_coverage.pyindex 770b9a7783..2ccf33ca4c 100644--- a/tests/research_library/services/test_library_rag_service_deep_coverage.py+++ b/tests/research_library/services/test_library_rag_service_deep_coverage.py@@ -6,14 +6,12 @@ Targets ~146 missing statements not covered by existing test files:- remove_document_from_rag (success, not-in-collection error, exception path)- index_documents_batch (not found, already indexed skip, no text, batch exception)- get_rag_stats (default collection id, with collection, chunk_sample present/absent)-- index_user_document (no content, success no rag_index_record, force_reindex path, exception)- remove_collection_from_index (no chunks, has chunks success, faiss delete exception, outer exception)- close (idempotent second call, clears resources)- index_document exception path (embedding fails mid-way)"""from unittest.mock import MagicMock, patch-from langchain_core.documents import Document as LangchainDocument# ---------------------------------------------------------------------------# Module-level patch path prefix@@ -505,125 +503,6 @@ class TestGetRagStats:assert result["embedding_info"]["model_type"] is None-# =========================================================================-# index_user_document-# =========================================================================-class TestIndexUserDocument:- def _make_user_doc(self, text_content="some content here for testing"):- user_doc = MagicMock()- user_doc.id = "udoc-1"- user_doc.filename = "test.pdf"- user_doc.file_type = "pdf"- user_doc.file_size = 1024- user_doc.text_content = text_content- return user_doc-- def test_no_content_returns_error(self):- svc = _make_service()- user_doc = self._make_user_doc(text_content=None)- result = svc.index_user_document(user_doc, "collection_abc")- assert result["status"] == "error"- assert "no extractable text" in result["error"]-- def test_short_content_returns_error(self):- svc = _make_service()- user_doc = self._make_user_doc(text_content="hi") # < 10 chars- result = svc.index_user_document(user_doc, "collection_abc")- assert result["status"] == "error"-- def test_success_without_rag_index_record(self):- svc = _make_service()- svc.embedding_manager = MagicMock()- svc.embedding_manager._store_chunks_to_db.return_value = ["uid-1"]-- mock_faiss = MagicMock()- mock_faiss.docstore._dict = {}- svc.faiss_index = mock_faiss- svc.rag_index_record = None # no record → save is skipped-- mock_chunks = [- LangchainDocument(page_content="user uploaded content text")- ]- svc.text_splitter = MagicMock()- svc.text_splitter.split_documents.return_value = mock_chunks-- user_doc = self._make_user_doc()- result = svc.index_user_document(user_doc, "collection_abc")- assert result["status"] == "success"- assert result["chunk_count"] == 1-- def test_success_with_rag_index_record(self, tmp_path):- svc = _make_service()- svc.embedding_manager = MagicMock()- svc.embedding_manager._store_chunks_to_db.return_value = ["uid-1"]- svc.integrity_manager = MagicMock()-- mock_faiss = MagicMock()- mock_faiss.docstore._dict = {}- svc.faiss_index = mock_faiss-- mock_record = MagicMock()- idx_path = tmp_path / "idx.faiss"- mock_record.index_path = str(idx_path)- mock_record.id = "rag-1"- svc.rag_index_record = mock_record-- mock_chunks = [- LangchainDocument(page_content="user uploaded content text")- ]- svc.text_splitter = MagicMock()- svc.text_splitter.split_documents.return_value = mock_chunks-- user_doc = self._make_user_doc()- result = svc.index_user_document(user_doc, "collection_abc")- assert result["status"] == "success"- mock_faiss.save_local.assert_called_once()-- def test_force_reindex_removes_old_chunks(self):- svc = _make_service()- svc.embedding_manager = MagicMock()- svc.embedding_manager._store_chunks_to_db.return_value = ["uid-old"]-- mock_faiss = MagicMock()- mock_faiss.docstore._dict = {"uid-old": MagicMock()}- svc.faiss_index = mock_faiss- svc.rag_index_record = None-- mock_chunks = [- LangchainDocument(page_content="updated content text here")- ]- svc.text_splitter = MagicMock()- svc.text_splitter.split_documents.return_value = mock_chunks-- user_doc = self._make_user_doc()- result = svc.index_user_document(- user_doc, "collection_abc", force_reindex=True- )- assert result["status"] == "success"- # Old chunk ids should have been deleted from FAISS- mock_faiss.delete.assert_called_once()-- def test_exception_returns_error(self):- svc = _make_service()- svc.embedding_manager = MagicMock()- svc.embedding_manager._store_chunks_to_db.side_effect = RuntimeError(- "db exploded"- )-- mock_chunks = [- LangchainDocument(page_content="valid content text here")- ]- svc.text_splitter = MagicMock()- svc.text_splitter.split_documents.return_value = mock_chunks-- svc.faiss_index = MagicMock()-- user_doc = self._make_user_doc()- result = svc.index_user_document(user_doc, "collection_abc")- assert result["status"] == "error"- assert "RuntimeError" in result["error"]--# =========================================================================# remove_collection_from_index# =========================================================================