Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

refactor(library): remove obsolete index_user_document (#5077)

LearningCircuit790604ebd880
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.py
index 04bac31942..e7bde612d9 100644
--- a/src/local_deep_research/research_library/services/library_rag_service.py
+++ b/src/local_deep_research/research_library/services/library_rag_service.py
@@ -1514,152 +1514,6 @@ class LibraryRAGService:
"chunk_overlap": self.chunk_overlap,
}
- def index_user_document(
- self, user_doc, collection_name: str, force_reindex: bool = False
- ) -> Dict[str, Any]:
- """
- Index a user-uploaded document into a specific collection.
-
- Args:
- user_doc: UserDocument object
- collection_name: Name of the collection (e.g., "collection_123")
- force_reindex: Whether to force reindexing
-
- Returns:
- Dict with status, chunk_count, and any errors
- """
-
- try:
- # Use the pre-extracted text content
- content = user_doc.text_content
-
- if not content or len(content.strip()) < 10:
- return {
- "status": "error",
- "error": "Document has no extractable text content",
- }
-
- # Create LangChain Document
- doc = LangchainDocument(
- page_content=content,
- metadata={
- "source": f"user_upload_{user_doc.id}",
- "source_id": user_doc.id,
- "title": user_doc.filename,
- "document_title": user_doc.filename,
- "file_type": user_doc.file_type,
- "file_size": user_doc.file_size,
- "collection": collection_name,
- },
- )
-
- # Split into chunks
- chunks = self.text_splitter.split_documents([doc])
- logger.info(
- f"Split user document {user_doc.filename} into {len(chunks)} chunks"
- )
-
- # Store chunks in database
- embedding_ids = self.embedding_manager._store_chunks_to_db(
- chunks=chunks,
- collection_name=collection_name,
- source_type="user_document",
- source_id=user_doc.id,
- )
-
- # Load or create FAISS index for this collection (lazy;
- # merge step below reloads under the lock anyway).
- if self.faiss_index is None:
- # Extract collection_id from collection_name (format: "collection_<uuid>")
- collection_id = collection_name.removeprefix("collection_")
- self.faiss_index = self.load_or_create_faiss_index(
- collection_id
- )
-
- unique_count = len(set(embedding_ids))
- batch_dups = len(chunks) - unique_count
-
- # Read-modify-write the on-disk FAISS under the lock so
- # concurrent uploads to the same collection don't lose
- # each other's chunks.
- index_path = (
- Path(self.rag_index_record.index_path)
- if self.rag_index_record
- else None
- )
- if index_path:
- merge_stats = self._merge_and_persist_locked(
- index_path,
- chunks,
- embedding_ids,
- force_reindex=force_reindex,
- )
- else:
- # No persistent index path — in-memory only path.
- # Preserve old behavior: handle force_reindex deletion
- # and dedup add against in-memory state without saving.
- if force_reindex and hasattr(self.faiss_index, "docstore"):
- existing_ids = set(self.faiss_index.docstore._dict.keys())
- old_chunk_ids = list(
- {eid for eid in embedding_ids if eid in existing_ids}
- )
- if old_chunk_ids:
- logger.info(
- f"Force re-index: removing {len(old_chunk_ids)} "
- f"existing chunks from FAISS"
- )
- self.faiss_index.delete(old_chunk_ids)
- if not force_reindex and hasattr(self.faiss_index, "docstore"):
- existing_ids = set(self.faiss_index.docstore._dict.keys())
- else:
- existing_ids = None
- new_chunks, new_ids = self._deduplicate_chunks(
- chunks, embedding_ids, existing_ids
- )
- if new_chunks:
- self.faiss_index.add_documents(new_chunks, ids=new_ids)
- merge_stats = {
- "added": len(new_chunks),
- "skipped": len(chunks) - len(new_chunks),
- "added_ids": new_ids,
- }
- if merge_stats["added"]:
- if force_reindex:
- logger.info(
- f"Force re-index: added {merge_stats['added']} "
- f"chunks with updated metadata to FAISS index"
- )
- else:
- already_exist = unique_count - merge_stats["added"]
- logger.info(
- f"Added {merge_stats['added']} new chunks to FAISS "
- f"({already_exist} already exist, "
- f"{batch_dups} batch duplicates removed)"
- )
- else:
- logger.info(
- f"All {len(chunks)} chunks already exist in FAISS index, skipping"
- )
-
- logger.info(
- f"Successfully indexed user document {user_doc.filename} with {len(chunks)} chunks"
- )
-
- return {
- "status": "success",
- "chunk_count": len(chunks),
- "embedding_ids": embedding_ids,
- }
-
- except Exception as e:
- logger.exception(
- f"Error indexing user document {user_doc.filename}"
- )
- return {
- "status": "error",
- "error": f"Operation failed: {type(e).__name__}",
- }
-
def remove_collection_from_index(
self, collection_name: str
) -> Dict[str, Any]:
diff --git a/tests/research_library/services/test_library_rag_service.py b/tests/research_library/services/test_library_rag_service.py
index 5802c27180..a01b01d36b 100644
--- a/tests/research_library/services/test_library_rag_service.py
+++ b/tests/research_library/services/test_library_rag_service.py
@@ -1067,9 +1067,8 @@ class TestRemoveCollectionFromIndex:
class TestLoadOrCreateFaissIndexCollectionId:
"""Tests that load_or_create_faiss_index receives the correct collection_id.
- Fixes a pre-existing bug where index_user_document and
- remove_collection_from_index called load_or_create_faiss_index()
- without the required collection_id argument.
+ Fixes a pre-existing bug where remove_collection_from_index called
+ load_or_create_faiss_index() without the required collection_id argument.
"""
def _create_service(self, mocker):
@@ -1109,52 +1108,6 @@ class TestLoadOrCreateFaissIndexCollectionId:
)
return service
- def test_index_user_document_passes_extracted_collection_id(self, mocker):
- """index_user_document extracts collection_id from collection_name."""
- service = self._create_service(mocker)
-
- # Create mock user document
- mock_user_doc = Mock()
- mock_user_doc.id = "user-doc-123"
- mock_user_doc.text_content = (
- "Enough content for indexing test purposes."
- )
- mock_user_doc.filename = "test.pdf"
- mock_user_doc.file_type = "pdf"
- mock_user_doc.file_size = 1024
-
- # Mock FAISS index as None to trigger load
- service.faiss_index = None
- mock_faiss_index = MagicMock()
- mock_faiss_index.docstore = Mock()
- mock_faiss_index.docstore._dict = {}
- service.load_or_create_faiss_index = Mock(return_value=mock_faiss_index)
-
- # Mock _store_chunks_to_db
- service.embedding_manager._store_chunks_to_db = Mock(
- return_value=["chunk-1", "chunk-2"]
- )
-
- # Mock text_splitter
- from langchain_core.documents import Document as LangchainDocument
-
- mock_chunks = [
- LangchainDocument(page_content="chunk 1"),
- LangchainDocument(page_content="chunk 2"),
- ]
- service.text_splitter.split_documents.return_value = mock_chunks
-
- # Mock rag_index_record for save path
- service.rag_index_record = None
-
- collection_name = "collection_abc-uuid-123"
- service.index_user_document(mock_user_doc, collection_name)
-
- # Verify collection_id was correctly extracted and passed
- service.load_or_create_faiss_index.assert_called_once_with(
- "abc-uuid-123"
- )
-
def test_remove_collection_passes_extracted_collection_id(self, mocker):
"""remove_collection_from_index extracts collection_id from collection_name."""
service = self._create_service(mocker)
@@ -1440,52 +1393,3 @@ class TestForceReindexDedup:
assert len(delete_call) == len(set(delete_call)), (
"old_chunk_ids passed to FAISS.delete() must not contain duplicates"
)
-
- def test_force_reindex_old_chunk_ids_deduplicated_index_user_document(
- self, mocker
- ):
- """Verify deletion path in index_user_document uses unique IDs."""
- service = _create_rag_service(mocker)
-
- # Mock user document
- mock_user_doc = Mock()
- mock_user_doc.id = "udoc-1"
- mock_user_doc.filename = "test.txt"
- mock_user_doc.text_content = "Some content here for testing"
- mock_user_doc.file_type = "txt"
- mock_user_doc.file_size = 100
-
- chunks = [
- LangchainDocument(page_content="chunk A"),
- LangchainDocument(page_content="chunk B"),
- LangchainDocument(page_content="chunk A dup"),
- ]
- service.text_splitter.split_documents.return_value = chunks
-
- # _store_chunks_to_db returns IDs with duplicates
- duplicate_ids = ["id-1", "id-2", "id-1"]
- service.embedding_manager._store_chunks_to_db.return_value = (
- duplicate_ids
- )
-
- # Setup FAISS mock
- mock_faiss = Mock()
- mock_docstore = Mock()
- mock_docstore._dict = {"id-1": "doc", "id-2": "doc"}
- mock_faiss.docstore = mock_docstore
- service.faiss_index = mock_faiss
-
- mock_rag_index = Mock()
- mock_rag_index.index_path = "/tmp/test.faiss"
- mock_rag_index.id = "rag-1"
- service.rag_index_record = mock_rag_index
-
- service.index_user_document(
- mock_user_doc, "collection_123", force_reindex=True
- )
-
- # Verify delete was called with deduplicated IDs
- delete_call = mock_faiss.delete.call_args[0][0]
- assert len(delete_call) == len(set(delete_call)), (
- "old_chunk_ids passed to FAISS.delete() must not contain duplicates"
- )
diff --git a/tests/research_library/services/test_library_rag_service_deep_coverage.py b/tests/research_library/services/test_library_rag_service_deep_coverage.py
index 770b9a7783..2ccf33ca4c 100644
--- a/tests/research_library/services/test_library_rag_service_deep_coverage.py
+++ b/tests/research_library/services/test_library_rag_service_deep_coverage.py
@@ -6,14 +6,12 @@ Targets ~146 missing statements not covered by existing test files:
- remove_document_from_rag (success, not-in-collection error, exception path)
- index_documents_batch (not found, already indexed skip, no text, batch exception)
- get_rag_stats (default collection id, with collection, chunk_sample present/absent)
-- index_user_document (no content, success no rag_index_record, force_reindex path, exception)
- remove_collection_from_index (no chunks, has chunks success, faiss delete exception, outer exception)
- close (idempotent second call, clears resources)
- index_document exception path (embedding fails mid-way)
"""
from unittest.mock import MagicMock, patch
-from langchain_core.documents import Document as LangchainDocument
# ---------------------------------------------------------------------------
# Module-level patch path prefix
@@ -505,125 +503,6 @@ class TestGetRagStats:
assert result["embedding_info"]["model_type"] is None
-# =========================================================================
-# index_user_document
-# =========================================================================
-class TestIndexUserDocument:
- def _make_user_doc(self, text_content="some content here for testing"):
- user_doc = MagicMock()
- user_doc.id = "udoc-1"
- user_doc.filename = "test.pdf"
- user_doc.file_type = "pdf"
- user_doc.file_size = 1024
- user_doc.text_content = text_content
- return user_doc
-
- def test_no_content_returns_error(self):
- svc = _make_service()
- user_doc = self._make_user_doc(text_content=None)
- result = svc.index_user_document(user_doc, "collection_abc")
- assert result["status"] == "error"
- assert "no extractable text" in result["error"]
-
- def test_short_content_returns_error(self):
- svc = _make_service()
- user_doc = self._make_user_doc(text_content="hi") # < 10 chars
- result = svc.index_user_document(user_doc, "collection_abc")
- assert result["status"] == "error"
-
- def test_success_without_rag_index_record(self):
- svc = _make_service()
- svc.embedding_manager = MagicMock()
- svc.embedding_manager._store_chunks_to_db.return_value = ["uid-1"]
-
- mock_faiss = MagicMock()
- mock_faiss.docstore._dict = {}
- svc.faiss_index = mock_faiss
- svc.rag_index_record = None # no record → save is skipped
-
- mock_chunks = [
- LangchainDocument(page_content="user uploaded content text")
- ]
- svc.text_splitter = MagicMock()
- svc.text_splitter.split_documents.return_value = mock_chunks
-
- user_doc = self._make_user_doc()
- result = svc.index_user_document(user_doc, "collection_abc")
- assert result["status"] == "success"
- assert result["chunk_count"] == 1
-
- def test_success_with_rag_index_record(self, tmp_path):
- svc = _make_service()
- svc.embedding_manager = MagicMock()
- svc.embedding_manager._store_chunks_to_db.return_value = ["uid-1"]
- svc.integrity_manager = MagicMock()
-
- mock_faiss = MagicMock()
- mock_faiss.docstore._dict = {}
- svc.faiss_index = mock_faiss
-
- mock_record = MagicMock()
- idx_path = tmp_path / "idx.faiss"
- mock_record.index_path = str(idx_path)
- mock_record.id = "rag-1"
- svc.rag_index_record = mock_record
-
- mock_chunks = [
- LangchainDocument(page_content="user uploaded content text")
- ]
- svc.text_splitter = MagicMock()
- svc.text_splitter.split_documents.return_value = mock_chunks
-
- user_doc = self._make_user_doc()
- result = svc.index_user_document(user_doc, "collection_abc")
- assert result["status"] == "success"
- mock_faiss.save_local.assert_called_once()
-
- def test_force_reindex_removes_old_chunks(self):
- svc = _make_service()
- svc.embedding_manager = MagicMock()
- svc.embedding_manager._store_chunks_to_db.return_value = ["uid-old"]
-
- mock_faiss = MagicMock()
- mock_faiss.docstore._dict = {"uid-old": MagicMock()}
- svc.faiss_index = mock_faiss
- svc.rag_index_record = None
-
- mock_chunks = [
- LangchainDocument(page_content="updated content text here")
- ]
- svc.text_splitter = MagicMock()
- svc.text_splitter.split_documents.return_value = mock_chunks
-
- user_doc = self._make_user_doc()
- result = svc.index_user_document(
- user_doc, "collection_abc", force_reindex=True
- )
- assert result["status"] == "success"
- # Old chunk ids should have been deleted from FAISS
- mock_faiss.delete.assert_called_once()
-
- def test_exception_returns_error(self):
- svc = _make_service()
- svc.embedding_manager = MagicMock()
- svc.embedding_manager._store_chunks_to_db.side_effect = RuntimeError(
- "db exploded"
- )
-
- mock_chunks = [
- LangchainDocument(page_content="valid content text here")
- ]
- svc.text_splitter = MagicMock()
- svc.text_splitter.split_documents.return_value = mock_chunks
-
- svc.faiss_index = MagicMock()
-
- user_doc = self._make_user_doc()
- result = svc.index_user_document(user_doc, "collection_abc")
- assert result["status"] == "error"
- assert "RuntimeError" in result["error"]
-
-
# =========================================================================
# remove_collection_from_index
# =========================================================================