~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
refactor(library): remove orphaned RAG index-local and index-research endpoints (#4867)
LearningCircuit5c419343efef
diff --git a/changelog.d/+remove-orphaned-rag-index-endpoints.removal.md b/changelog.d/+remove-orphaned-rag-index-endpoints.removal.mdnew file mode 100644index 0000000000..1ef7df0e1a--- /dev/null+++ b/changelog.d/+remove-orphaned-rag-index-endpoints.removal.md@@ -0,0 +1 @@+Removed two orphaned RAG HTTP endpoints that had no UI and no other caller: `GET /library/api/rag/index-local` (local-folder indexing — superseded by collection-based indexing) and `POST /library/api/rag/index-research` (which called a method that no longer exists and always errored). The unused `index_local_file` service method and glob-pattern allowlist were removed with them.diff --git a/src/local_deep_research/research_library/routes/rag_routes.py b/src/local_deep_research/research_library/routes/rag_routes.pyindex 6ee7f03a4e..50df5dfac7 100644--- a/src/local_deep_research/research_library/routes/rag_routes.py+++ b/src/local_deep_research/research_library/routes/rag_routes.py@@ -9,7 +9,6 @@ Provides endpoints for managing RAG indexing of library documents:"""import os-import refrom flask import (Blueprint,@@ -23,7 +22,6 @@ from flask import (from loguru import loggerfrom sqlalchemy import case, funcimport atexit-import globimport jsonimport uuidimport time@@ -51,7 +49,6 @@ from ...utilities.resource_utils import safe_closefrom ..services.library_rag_service import LibraryRAGServicefrom ...settings.manager import SettingsManagerfrom ...security.file_upload_validator import FileUploadValidator-from ...security.path_validator import PathValidatorfrom ...security.rate_limiter import (upload_rate_limit_ip,upload_rate_limit_user,@@ -73,25 +70,6 @@ from ...constants import DEFAULT_SEARCH_TOOLrag_bp = Blueprint("rag", __name__, url_prefix="/library")-# Allowlist for user-supplied glob patterns. Each pattern is joined onto an-# already-validated base directory and globbed, so it must not be able to-# escape that directory. Only in-directory filename globs are allowed:-# any path separator, parent reference, or character outside this set is-# rejected. This also blocks a leading "/" — pathlib's "/" operator resets-# to an absolute path when the right operand is absolute (``base / "/abs"``-# yields ``/abs``), so a leading-slash absolute pattern would otherwise-# escape the base too.-# Anchored with ``\Z`` (not ``$``) so a trailing newline cannot slip through —-# ``$`` matches just before a final "\n", which would make the predicate-# depend on the caller stripping the pattern first.-_SAFE_GLOB_PATTERN = re.compile(r"^[A-Za-z0-9._*?\[\]-]+\Z")---def _is_safe_glob_pattern(pattern: str) -> bool:- """Return True for in-directory filename globs that cannot traverse out."""- return bool(_SAFE_GLOB_PATTERN.match(pattern)) and ".." not in pattern--# NOTE: Routes use session["username"] (not .get()) intentionally.# @login_required guarantees the key exists; direct access fails fast# if the decorator is ever removed.@@ -852,31 +830,6 @@ def remove_document():return handle_api_error(f"removing document {text_doc_id}", e)-@rag_bp.route("/api/rag/index-research", methods=["POST"])-@login_required-@require_json_body(error_format="success")-def index_research():- """Index all documents from a research."""- try:- data = request.get_json()- research_id = data.get("research_id")- force_reindex = data.get("force_reindex", False)-- if not research_id:- return jsonify(- {"success": False, "error": "research_id is required"}- ), 400-- with get_rag_service() as rag_service:- results = rag_service.index_research_documents(- research_id, force_reindex- )-- return jsonify({"success": True, "results": results})- except Exception as e:- return handle_api_error(f"indexing research {research_id}", e)--@rag_bp.route("/api/rag/index-all", methods=["GET"])@login_requireddef index_all():@@ -1252,144 +1205,6 @@ def get_documents():return handle_api_error("getting documents", e)-@rag_bp.route("/api/rag/index-local", methods=["GET"])-@login_required-def index_local_library():- """Index documents from a local folder with Server-Sent Events progress."""- folder_path = request.args.get("path")- raw_patterns = request.args.get(- "patterns", "*.pdf,*.txt,*.md,*.html"- ).split(",")- file_patterns = [p.strip() for p in raw_patterns if p.strip()]- recursive = parse_bool_arg("recursive", default=True)-- if not folder_path:- return jsonify({"success": False, "error": "Path is required"}), 400-- # Reject glob patterns that could escape the validated base directory.- # The folder is validated below, but each pattern is joined onto it and- # globbed; a pattern with ".." or a leading-slash absolute pattern would- # otherwise read files outside the folder. Only in-directory filename- # globs are allowed.- if not file_patterns or not all(- _is_safe_glob_pattern(p) for p in file_patterns- ):- return jsonify({"success": False, "error": "Invalid file pattern"}), 400-- # Validate and sanitize the path to prevent traversal attacks- try:- validated_path = PathValidator.validate_local_filesystem_path(- folder_path- )- # Re-sanitize for static analyzer recognition (CodeQL)- path = PathValidator.sanitize_for_filesystem_ops(validated_path)- except ValueError:- # Don't log the raw user-submitted path — it can contain a username.- # A static message keeps operator visibility without leaking the input- # (project policy disallows exception variables in logger.warning).- logger.warning("Path validation failed")- return jsonify({"success": False, "error": "Invalid path"}), 400-- # Check path exists and is a directory- if not path.exists():- return jsonify({"success": False, "error": "Path does not exist"}), 400- if not path.is_dir():- return jsonify(- {"success": False, "error": "Path is not a directory"}- ), 400-- # Create RAG service in request context- rag_service = get_rag_service()-- def generate():- """Generator function for SSE progress updates."""- try:- # Send initial status- yield f"data: {json.dumps({'type': 'start', 'message': f'Scanning folder: {path}'})}\n\n"-- # Find all matching files- files_to_index = []- for pattern in file_patterns:- pattern = pattern.strip()- if recursive:- search_pattern = str(path / "**" / pattern)- else:- search_pattern = str(path / pattern)-- matching_files = glob.glob(search_pattern, recursive=recursive)- files_to_index.extend(matching_files)-- # Remove duplicates and sort- files_to_index = sorted(set(files_to_index))-- # Confine matches to the indexed folder: glob follows symlinks, so- # a symlink inside the folder could otherwise expose files outside- # it. Drop any match whose resolved path escapes the base.- files_to_index = PathValidator.confine_to_base(files_to_index, path)-- if not files_to_index:- yield f"data: {json.dumps({'type': 'complete', 'results': {'successful': 0, 'skipped': 0, 'failed': 0, 'message': 'No matching files found'}})}\n\n"- return-- results = {"successful": 0, "skipped": 0, "failed": 0, "errors": []}- total = len(files_to_index)-- # Index each file- for idx, file_path in enumerate(files_to_index, 1):- file_name = Path(file_path).name-- # Send progress update- yield f"data: {json.dumps({'type': 'progress', 'current': idx, 'total': total, 'filename': file_name, 'percent': int((idx / total) * 100)})}\n\n"-- try:- # Index the file directly using RAG service- result = rag_service.index_local_file(file_path)-- if result.get("status") == "success":- results["successful"] += 1- elif result.get("status") == "skipped":- results["skipped"] += 1- else:- results["failed"] += 1- results["errors"].append(- {- "file": file_name,- "error": result.get("error", "Unknown error"),- }- )- except Exception:- results["failed"] += 1- results["errors"].append(- {"file": file_name, "error": "Failed to index file"}- )- # Log the file name only — the full path can contain a- # username.- logger.exception(f"Error indexing file {file_name}")-- # Send completion status- yield f"data: {json.dumps({'type': 'complete', 'results': results})}\n\n"-- # Log the folder name only — the full path can contain a username.- logger.info(- f"Local library indexing complete for {path.name}: "- f"{results['successful']} successful, "- f"{results['skipped']} skipped, "- f"{results['failed']} failed"- )-- except Exception:- logger.exception("Error in local library indexing")- yield f"data: {json.dumps({'type': 'error', 'error': 'An internal error occurred during indexing'})}\n\n"- finally:- # See parallel comment in index-all generator above — generator- # ``finally`` is the safe close site for streamed RAG services.- safe_close(rag_service, "rag_service (index-local SSE)")-- return Response(- stream_with_context(generate()), mimetype="text/event-stream"- )--# Collection Management Routes@@ -1643,7 +1458,6 @@ def upload_to_collection(collection_id):safe_rollback,)from ...security import sanitize_filename, UnsafeFilenameError- from pathlib import Pathimport hashlibimport uuidfrom ..services.pdf_storage_manager import PDFStorageManagerdiff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.pyindex 41e2904ce2..4b21d44d1a 100644--- a/src/local_deep_research/research_library/services/library_rag_service.py+++ b/src/local_deep_research/research_library/services/library_rag_service.py@@ -1500,158 +1500,6 @@ class LibraryRAGService:"chunk_overlap": self.chunk_overlap,}- def index_local_file(self, file_path: str) -> Dict[str, Any]:- """- Index a local file from the filesystem into RAG.-- Args:- file_path: Path to the file to index-- Returns:- Dict with status, chunk_count, and any errors- """- from pathlib import Path- import mimetypes-- file_path = Path(file_path)-- if not file_path.exists():- return {"status": "error", "error": f"File not found: {file_path}"}-- if not file_path.is_file():- return {"status": "error", "error": f"Not a file: {file_path}"}-- # Determine file type- mime_type, _ = mimetypes.guess_type(str(file_path))-- # Read file content based on type- try:- if file_path.suffix.lower() in [".txt", ".md", ".markdown"]:- # Text files- with open(file_path, "r", encoding="utf-8") as f:- content = f.read()- elif file_path.suffix.lower() in [".html", ".htm"]:- # HTML files - strip tags- from bs4 import BeautifulSoup-- with open(file_path, "r", encoding="utf-8") as f:- soup = BeautifulSoup(f.read(), "html.parser")- content = soup.get_text()- elif file_path.suffix.lower() == ".pdf":- # PDF files - extract text- from pypdf import PdfReader-- content = ""- with open(file_path, "rb") as f:- pdf_reader = PdfReader(f)- for page in pdf_reader.pages:- content += page.extract_text()- else:- return {- "status": "skipped",- "error": f"Unsupported file type: {file_path.suffix}",- }-- if not content or len(content.strip()) < 10:- return {- "status": "error",- "error": "File has no extractable text content",- }-- # Create LangChain Document from text- doc = LangchainDocument(- page_content=content,- metadata={- "source": str(file_path),- "source_id": f"local_{file_path.stem}_{hash(str(file_path))}",- "title": file_path.stem,- "document_title": file_path.stem,- "file_type": file_path.suffix.lower(),- "file_size": file_path.stat().st_size,- "source_type": "local_file",- "collection": "local_library",- },- )-- # Split into chunks- chunks = self.text_splitter.split_documents([doc])- logger.info(- f"Split local file {file_path} into {len(chunks)} chunks"- )-- # Store chunks in database (returns UUID-based IDs)- embedding_ids = self.embedding_manager._store_chunks_to_db(- chunks=chunks,- collection_name="local_library",- source_type="local_file",- source_id=str(file_path),- )-- # Load or create FAISS index using default library collection- # (lazy; merge step below reloads under the lock anyway).- if self.faiss_index is None:- from ...database.library_init import get_default_library_id-- default_collection_id = get_default_library_id(- self.username, self.db_password- )- self.faiss_index = self.load_or_create_faiss_index(- default_collection_id- )-- # Read-modify-write the on-disk FAISS under the lock.- index_path = (- Path(self.rag_index_record.index_path)- if self.rag_index_record- else None- )- if index_path:- merge_stats = self._merge_and_persist_locked(- index_path, chunks, embedding_ids, force_reindex=False- )- logger.info(- f"Saved FAISS index to {index_path} with integrity tracking"- )- else:- # No persistent index path — in-memory only path.- # Preserve old behavior: dedup against in-memory state- # and add directly without saving.- if self.faiss_index is not None and hasattr(- self.faiss_index, "docstore"- ):- existing_ids = set(self.faiss_index.docstore._dict.keys())- else:- existing_ids = None- new_chunks, new_ids = self._deduplicate_chunks(- chunks, embedding_ids, existing_ids- )- if new_chunks:- self.faiss_index.add_documents(new_chunks, ids=new_ids)- merge_stats = {- "added": len(new_chunks),- "skipped": len(chunks) - len(new_chunks),- "added_ids": new_ids,- }-- logger.info(- f"Successfully indexed local file {file_path} with "- f"{merge_stats['added']} new chunks "- f"({merge_stats['skipped']} skipped)"- )-- return {- "status": "success",- "chunk_count": merge_stats["added"],- "embedding_ids": merge_stats["added_ids"],- }-- except Exception as e:- logger.exception(f"Error indexing local file {file_path}")- return {- "status": "error",- "error": f"Operation failed: {type(e).__name__}",- }-def index_user_document(self, user_doc, collection_name: str, force_reindex: bool = False) -> Dict[str, Any]:diff --git a/tests/research_library/routes/test_rag_routes.py b/tests/research_library/routes/test_rag_routes.pyindex ea1a7449ab..f144b02226 100644--- a/tests/research_library/routes/test_rag_routes.py+++ b/tests/research_library/routes/test_rag_routes.py@@ -762,27 +762,6 @@ class TestRagIndexRoutes:)assert response.status_code == 401, response.status_code- def test_index_research_route(self):- """Test /api/rag/index-research POST endpoint exists."""- # audit: PUNCHLIST reviewed 2026-05 — issue resolved by prior PR (recommendation: REWRITE).- from flask import Flask- from local_deep_research.research_library.routes.rag_routes import (- rag_bp,- )-- app = Flask(__name__)- app.config["SECRET_KEY"] = "test-secret"- app.register_blueprint(rag_bp)- app.register_blueprint(auth_bp)-- with app.test_client() as client:- response = client.post(- "/library/api/rag/index-research",- json={"research_id": "research123"},- content_type="application/json",- )- assert response.status_code == 401, response.status_code-def test_index_all_route(self):"""Test /api/rag/index-all GET endpoint exists."""from flask import Flask@@ -1765,88 +1744,6 @@ class TestRemoveDocumentEndpoint:assert response.status_code == 401, response.status_code-class TestIndexResearchEndpoint:- """Extended tests for index research endpoint."""-- def test_index_research_missing_research_id(self):- """Test index research without research_id."""- from flask import Flask- from local_deep_research.research_library.routes.rag_routes import (- rag_bp,- )-- app = Flask(__name__)- app.config["SECRET_KEY"] = "test-secret"- app.register_blueprint(rag_bp)- app.register_blueprint(auth_bp)-- with app.test_client() as client:- response = client.post(- "/library/api/rag/index-research",- json={},- content_type="application/json",- )- assert response.status_code == 401, response.status_code---class TestIndexLocalEndpoint:- """Extended tests for index local library endpoint."""-- def test_index_local_missing_path(self):- """Test index local without path."""- from flask import Flask- from local_deep_research.research_library.routes.rag_routes import (- rag_bp,- )-- app = Flask(__name__)- app.config["SECRET_KEY"] = "test-secret"- app.register_blueprint(rag_bp)- app.register_blueprint(auth_bp)-- with app.test_client() as client:- response = client.get("/library/api/rag/index-local")- assert response.status_code == 401, response.status_code-- def test_index_local_path_traversal_attempt(self):- """Test index local with path traversal attempt."""- # audit: PUNCHLIST reviewed 2026-05 — issue resolved by prior PR (recommendation: REWRITE).- from flask import Flask- from local_deep_research.research_library.routes.rag_routes import (- rag_bp,- )-- app = Flask(__name__)- app.config["SECRET_KEY"] = "test-secret"- app.register_blueprint(rag_bp)- app.register_blueprint(auth_bp)-- with app.test_client() as client:- response = client.get(- "/library/api/rag/index-local?path=../../etc/passwd"- )- assert response.status_code == 401, response.status_code-- def test_index_local_with_patterns(self):- """Test index local with custom patterns."""- # audit: PUNCHLIST reviewed 2026-05 — issue resolved by prior PR (recommendation: REWRITE).- from flask import Flask- from local_deep_research.research_library.routes.rag_routes import (- rag_bp,- )-- app = Flask(__name__)- app.config["SECRET_KEY"] = "test-secret"- app.register_blueprint(rag_bp)- app.register_blueprint(auth_bp)-- with app.test_client() as client:- response = client.get(- "/library/api/rag/index-local?path=/tmp&patterns=*.pdf,*.txt"- )- assert response.status_code == 401, response.status_code--class TestGetDocumentsEndpoint:"""Extended tests for get documents endpoint."""diff --git a/tests/research_library/routes/test_rag_routes_coverage.py b/tests/research_library/routes/test_rag_routes_coverage.pyindex 975d374e5e..500e9e723b 100644--- a/tests/research_library/routes/test_rag_routes_coverage.py+++ b/tests/research_library/routes/test_rag_routes_coverage.py@@ -854,40 +854,6 @@ class TestRemoveDocument:assert resp.status_code == 400-# ----------------------------------------------------------------------------# Tests: POST /api/rag/index-research-# ------------------------------------------------------------------------------class TestIndexResearch:- def test_missing_research_id(self, app):- with _auth_client(app) as (client, ctx):- resp = client.post("/library/api/rag/index-research", json={})- assert resp.status_code == 400-- def test_success(self, app):- mock_rag = MagicMock()- # rag_routes now uses ``with get_rag_service(...) as rag_service:``- # MagicMock's default __enter__ returns a child mock, so route body- # would see a different object. Pin __enter__ to self.- mock_rag.__enter__.return_value = mock_rag- mock_rag.index_research_documents.return_value = [{"status": "success"}]-- with _auth_client(- app,- extra_patches=[- patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),- ],- ) as (client, ctx):- resp = client.post(- "/library/api/rag/index-research",- json={"research_id": "research-1"},- )- assert resp.status_code == 200- data = resp.get_json()- assert data["success"] is True--# ---------------------------------------------------------------------------# Tests: POST /api/rag/configure# ---------------------------------------------------------------------------@@ -2109,74 +2075,6 @@ class TestCancelIndexing:assert resp.status_code == 404-# ----------------------------------------------------------------------------# Tests: index_local_library (GET /api/rag/index-local)-# ------------------------------------------------------------------------------class TestIndexLocalLibrary:- def test_missing_path(self, app):- with _auth_client(app) as (client, ctx):- resp = client.get("/library/api/rag/index-local")- assert resp.status_code == 400-- def test_invalid_path(self, app):- with _auth_client(- app,- extra_patches=[- patch(- f"{_ROUTES}.PathValidator.validate_local_filesystem_path",- side_effect=ValueError("bad path"),- ),- ],- ) as (client, ctx):- resp = client.get("/library/api/rag/index-local?path=/etc/../root")- assert resp.status_code == 400-- def test_path_not_exist(self, app):- mock_path = Mock()- mock_path.exists.return_value = False-- with _auth_client(- app,- extra_patches=[- patch(- f"{_ROUTES}.PathValidator.validate_local_filesystem_path",- return_value=mock_path,- ),- patch(- f"{_ROUTES}.PathValidator.sanitize_for_filesystem_ops",- return_value=mock_path,- ),- ],- ) as (client, ctx):- resp = client.get("/library/api/rag/index-local?path=/nonexistent")- assert resp.status_code == 400-- def test_path_not_dir(self, app):- mock_path = Mock()- mock_path.exists.return_value = True- mock_path.is_dir.return_value = False-- with _auth_client(- app,- extra_patches=[- patch(- f"{_ROUTES}.PathValidator.validate_local_filesystem_path",- return_value=mock_path,- ),- patch(- f"{_ROUTES}.PathValidator.sanitize_for_filesystem_ops",- return_value=mock_path,- ),- ],- ) as (client, ctx):- resp = client.get(- "/library/api/rag/index-local?path=/some/file.txt"- )- assert resp.status_code == 400--# ---------------------------------------------------------------------------# Tests: helper functions# ---------------------------------------------------------------------------diff --git a/tests/research_library/routes/test_rag_routes_indexing.py b/tests/research_library/routes/test_rag_routes_indexing.pyindex b056df5677..4167f691dd 100644--- a/tests/research_library/routes/test_rag_routes_indexing.py+++ b/tests/research_library/routes/test_rag_routes_indexing.py@@ -1,303 +1,11 @@# allow: no-sut-import — black-box HTTP test; drives real routes through the Flask test client-"""Tests for index_local_library and view_document_chunks in rag_routes.py."""+"""Tests for view_document_chunks in rag_routes.py."""-from pathlib import Pathfrom unittest.mock import patch, MagicMockRAG_PREFIX = "/library"-class TestIndexLocalLibrary:- """Tests for /api/rag/index-local SSE endpoint."""-- def test_requires_authentication(self, client):- response = client.get(- f"{RAG_PREFIX}/api/rag/index-local?path=/tmp/docs"- )- assert response.status_code in [401, 302]-- def test_missing_path_returns_400(self, authenticated_client):- response = authenticated_client.get(f"{RAG_PREFIX}/api/rag/index-local")- assert response.status_code == 400- data = response.get_json()- assert data["success"] is False- assert "path" in data["error"].lower()-- def test_traversal_glob_pattern_returns_400(self, authenticated_client):- """A glob pattern that escapes the base dir is rejected before any- path handling, so no traversal read can occur."""- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local"- "?path=/tmp/docs&patterns=../../../etc/passwd"- )- assert response.status_code == 400- data = response.get_json()- assert data["success"] is False- assert "pattern" in data["error"].lower()-- def test_absolute_glob_pattern_returns_400(self, authenticated_client):- """An absolute pattern is rejected: pathlib's "/" resets to an absolute- path, so ``base / "/etc/*"`` would otherwise escape the folder."""- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local?path=/tmp/docs&patterns=/etc/*"- )- assert response.status_code == 400- data = response.get_json()- assert data["success"] is False- assert "pattern" in data["error"].lower()-- def test_empty_patterns_returns_400(self, authenticated_client):- """A patterns value that is only commas/whitespace leaves nothing to- match and is rejected rather than silently indexing everything."""- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local?path=/tmp/docs&patterns=,%20,"- )- assert response.status_code == 400- data = response.get_json()- assert data["success"] is False- assert "pattern" in data["error"].lower()-- def test_one_bad_pattern_in_list_rejects_whole_request(- self, authenticated_client- ):- """A single unsafe pattern alongside valid ones rejects the request- (the validation is an all()-gate, not a silent filter)."""- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local"- "?path=/tmp/docs&patterns=*.pdf,../etc/passwd"- )- assert response.status_code == 400- data = response.get_json()- assert data["success"] is False- assert "pattern" in data["error"].lower()-- def test_valid_custom_patterns_pass_the_allowlist(- self, authenticated_client- ):- """Realistic custom globs must NOT be rejected by the allowlist —- guards against a future over-tightening of the regex. The request may- still fail because the path does not exist, but never for a pattern- reason."""- for pattern in ("data_*.csv", "*.PDF", "report-[0-9].txt"):- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local"- f"?path=/tmp/docs&patterns={pattern}"- )- error = (response.get_json() or {}).get("error") or ""- assert "pattern" not in error.lower(), (- f"{pattern!r} was wrongly rejected as an invalid pattern"- )-- @patch(- "local_deep_research.research_library.routes.rag_routes.PathValidator"- )- def test_invalid_path_returns_400(- self, mock_validator, authenticated_client- ):- mock_validator.validate_local_filesystem_path.side_effect = ValueError(- "blocked"- )-- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local?path=/etc/passwd"- )- assert response.status_code == 400- data = response.get_json()- assert data["success"] is False-- @patch(- "local_deep_research.research_library.routes.rag_routes.PathValidator"- )- def test_invalid_path_does_not_log_submitted_path(- self, mock_validator, authenticated_client- ):- """On a path-validation failure the submitted path must not reach the- logs (it can contain a username); only the generic reason is logged."""- from loguru import logger-- mock_validator.validate_local_filesystem_path.side_effect = ValueError(- "Cannot access system directories"- )- secret_path = "/home/alice-secret-9f3a/private"-- # The package disables loguru by default; enable it so the warning- # emitted from the route reaches our sink.- logger.enable("local_deep_research")- logged: list[str] = []- sink_id = logger.add(- lambda m: logged.append(m.record["message"]), level="WARNING"- )- try:- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local?path={secret_path}"- )- finally:- logger.remove(sink_id)- logger.disable("local_deep_research")-- assert response.status_code == 400- failed = [m for m in logged if "Path validation failed" in m]- assert failed, "expected a path-validation-failure log"- # the submitted path (which can contain a username) is never logged- assert all("alice-secret-9f3a" not in m for m in failed)-- def test_indexing_complete_log_uses_folder_basename(- self, authenticated_client, loguru_caplog_full, tmp_path- ):- """The success "indexing complete" log names the folder by basename,- never the full path (which can contain a username)."""- folder = tmp_path / "alice-secret-9f3a" / "mydocs"- folder.mkdir(parents=True)- (folder / "report.txt").write_text("hello", encoding="utf-8")-- rag_service = MagicMock()- rag_service.index_local_file.return_value = {"status": "success"}-- with (- loguru_caplog_full.at_level("DEBUG"),- patch(- "local_deep_research.research_library.routes.rag_routes.get_rag_service",- return_value=rag_service,- ),- ):- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local?path={folder}&recursive=false"- )- response.get_data() # drive the SSE generator-- assert response.status_code == 200- text = loguru_caplog_full.text- assert "alice-secret-9f3a" not in text- assert "indexing complete for mydocs" in text-- def test_indexing_error_log_uses_file_basename(- self, authenticated_client, loguru_caplog_full, tmp_path- ):- """A per-file indexing error logs the file's basename, never its full- path (the exception block is captured too via loguru_caplog_full)."""- folder = tmp_path / "bob-secret-1c2d" / "papers"- folder.mkdir(parents=True)- (folder / "thesis.txt").write_text("x", encoding="utf-8")-- rag_service = MagicMock()- rag_service.index_local_file.side_effect = RuntimeError("boom")-- with (- loguru_caplog_full.at_level("DEBUG"),- patch(- "local_deep_research.research_library.routes.rag_routes.get_rag_service",- return_value=rag_service,- ),- ):- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local?path={folder}&recursive=false"- )- response.get_data()-- text = loguru_caplog_full.text- assert "bob-secret-1c2d" not in text- assert "Error indexing file thesis.txt" in text-- @patch(- "local_deep_research.research_library.routes.rag_routes.PathValidator"- )- def test_nonexistent_path_returns_400(- self, mock_validator, authenticated_client, tmp_path- ):- nonexistent = tmp_path / "nonexistent"- mock_validator.validate_local_filesystem_path.return_value = nonexistent- mock_validator.sanitize_for_filesystem_ops.return_value = nonexistent-- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local?path={nonexistent}"- )- assert response.status_code == 400-- @patch(- "local_deep_research.research_library.routes.rag_routes.PathValidator"- )- def test_file_not_directory_returns_400(- self, mock_validator, authenticated_client, tmp_path- ):- file_path = tmp_path / "file.txt"- file_path.write_text("content")- mock_validator.validate_local_filesystem_path.return_value = file_path- mock_validator.sanitize_for_filesystem_ops.return_value = file_path-- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local?path={file_path}"- )- assert response.status_code == 400-- def test_symlink_escaping_folder_is_not_indexed(- self, authenticated_client, tmp_path- ):- """glob follows symlinks, so a symlink inside the indexed folder that- points outside it would otherwise be read. The route confines matches- to the folder, so the escaping symlink is skipped while the real file- inside the folder is still indexed."""- folder = tmp_path / "docs"- folder.mkdir()- (folder / "real.txt").write_text("inside", encoding="utf-8")- secret = tmp_path / "secret.txt"- secret.write_text("OUTSIDE THE FOLDER", encoding="utf-8")- (folder / "evil.txt").symlink_to(secret)-- indexed: list[str] = []- rag_service = MagicMock()- rag_service.index_local_file.side_effect = lambda fp: (- indexed.append(fp) or {"status": "success"}- )-- with patch(- "local_deep_research.research_library.routes.rag_routes.get_rag_service",- return_value=rag_service,- ):- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local"- f"?path={folder}&patterns=*.txt&recursive=false"- )- response.get_data() # drive the SSE generator-- assert response.status_code == 200- names = {Path(p).name for p in indexed}- assert "real.txt" in names- assert "evil.txt" not in names-- def test_symlink_loop_does_not_abort_indexing(- self, authenticated_client, tmp_path- ):- """A symlink loop in the folder (Path.resolve raises RuntimeError) must- be skipped, not abort the whole run — the real file is still indexed."""- folder = tmp_path / "docs"- folder.mkdir()- (folder / "real.txt").write_text("inside", encoding="utf-8")- loop = folder / "loop.txt"- loop.symlink_to(loop) # self-referential loop-- indexed: list[str] = []- rag_service = MagicMock()- rag_service.index_local_file.side_effect = lambda fp: (- indexed.append(fp) or {"status": "success"}- )-- with patch(- "local_deep_research.research_library.routes.rag_routes.get_rag_service",- return_value=rag_service,- ):- response = authenticated_client.get(- f"{RAG_PREFIX}/api/rag/index-local"- f"?path={folder}&patterns=*.txt&recursive=false"- )- response.get_data() # drive the SSE generator-- assert response.status_code == 200- names = {Path(p).name for p in indexed}- # the run completed and indexed the real file despite the loop- assert "real.txt" in names- assert "loop.txt" not in names--class TestViewDocumentChunks:"""Tests for /api/rag/document/<id>/chunks endpoint."""diff --git a/tests/research_library/services/test_library_rag_service.py b/tests/research_library/services/test_library_rag_service.pyindex e9449f1770..5802c27180 100644--- a/tests/research_library/services/test_library_rag_service.py+++ b/tests/research_library/services/test_library_rag_service.py@@ -1067,8 +1067,8 @@ class TestRemoveCollectionFromIndex:class TestLoadOrCreateFaissIndexCollectionId:"""Tests that load_or_create_faiss_index receives the correct collection_id.- Fixes a pre-existing bug where index_local_file, index_user_document,- and remove_collection_from_index called load_or_create_faiss_index()+ Fixes a pre-existing bug where index_user_document and+ remove_collection_from_index called load_or_create_faiss_index()without the required collection_id argument."""@@ -1109,49 +1109,6 @@ class TestLoadOrCreateFaissIndexCollectionId:)return service- def test_index_local_file_passes_default_library_id(self, mocker, tmp_path):- """index_local_file passes the default library collection_id."""- service = self._create_service(mocker)-- # Create a real temp file- test_file = tmp_path / "test.txt"- test_file.write_text("This is test content for indexing purposes.")-- # Mock the FAISS index (simulate faiss_index being None)- service.faiss_index = None- mock_faiss_index = MagicMock()- service.load_or_create_faiss_index = Mock(return_value=mock_faiss_index)-- # Mock _store_chunks_to_db- service.embedding_manager._store_chunks_to_db = Mock(- return_value=["id-1", "id-2"]- )-- # Mock text_splitter to return chunks- from langchain_core.documents import Document as LangchainDocument-- mock_chunks = [- LangchainDocument(page_content="chunk 1"),- LangchainDocument(page_content="chunk 2"),- ]- service.text_splitter.split_documents.return_value = mock_chunks-- # Mock get_default_library_id- mocker.patch(- "local_deep_research.database.library_init.get_default_library_id",- return_value="default-lib-uuid",- )-- # Mock rag_index_record for save path- service.rag_index_record = None-- service.index_local_file(str(test_file))-- # Verify load_or_create_faiss_index was called with the default library id- service.load_or_create_faiss_index.assert_called_once_with(- "default-lib-uuid"- )-def test_index_user_document_passes_extracted_collection_id(self, mocker):"""index_user_document extracts collection_id from collection_name."""service = self._create_service(mocker)@@ -1231,37 +1188,6 @@ class TestLoadOrCreateFaissIndexCollectionId:"xyz-uuid-456")- def test_index_local_file_skips_load_when_faiss_already_set(- self, mocker, tmp_path- ):- """index_local_file does not call load_or_create_faiss_index when faiss_index is already set."""- service = self._create_service(mocker)-- test_file = tmp_path / "test.txt"- test_file.write_text("This is test content for indexing purposes.")-- # Set faiss_index to non-None (already loaded)- mock_faiss_index = MagicMock()- service.faiss_index = mock_faiss_index- service.load_or_create_faiss_index = Mock()-- service.embedding_manager._store_chunks_to_db = Mock(- return_value=["id-1"]- )-- from langchain_core.documents import Document as LangchainDocument-- service.text_splitter.split_documents.return_value = [- LangchainDocument(page_content="chunk 1"),- ]-- service.rag_index_record = None-- service.index_local_file(str(test_file))-- # load_or_create_faiss_index should NOT be called- service.load_or_create_faiss_index.assert_not_called()-class TestDeduplicateChunks:"""Tests for _deduplicate_chunks static method.@@ -1563,128 +1489,3 @@ class TestForceReindexDedup:assert len(delete_call) == len(set(delete_call)), ("old_chunk_ids passed to FAISS.delete() must not contain duplicates")---class TestIndexLocalFileDedup:- """Tests for index_local_file deduplication and ID consistency."""-- def test_index_local_file_uses_db_embedding_ids(self, mocker, tmp_path):- """Verify IDs come from _store_chunks_to_db, not manual generation."""- service = _create_rag_service(mocker)-- # Create a real temp file- test_file = tmp_path / "test.txt"- test_file.write_text(- "This is a test file with enough content to pass validation."- )-- chunks = [- LangchainDocument(page_content="chunk 1"),- LangchainDocument(page_content="chunk 2"),- ]- service.text_splitter.split_documents.return_value = chunks-- db_ids = ["db-uuid-1", "db-uuid-2"]- service.embedding_manager._store_chunks_to_db.return_value = db_ids-- # Setup FAISS mock- mock_faiss = Mock()- mock_docstore = Mock()- mock_docstore._dict = {} # Empty index- mock_faiss.docstore = mock_docstore- service.faiss_index = mock_faiss-- mock_rag_index = Mock()- mock_rag_index.index_path = "/tmp/test.faiss"- mock_rag_index.id = "rag-1"- service.rag_index_record = mock_rag_index-- result = service.index_local_file(str(test_file))-- assert result["status"] == "success"- # The IDs in the result should come from DB, not manual local_* generation- assert result["embedding_ids"] == db_ids- # Verify add_documents was called with DB-based IDs- add_call = mock_faiss.add_documents.call_args- assert add_call[1]["ids"] == db_ids-- def test_index_local_file_deduplicates_before_faiss(self, mocker, tmp_path):- """Verify dedup is applied before adding to FAISS."""- service = _create_rag_service(mocker)-- test_file = tmp_path / "test.txt"- test_file.write_text(- "This is a test file with enough content to pass validation."- )-- chunks = [- LangchainDocument(page_content="chunk 1"),- LangchainDocument(page_content="chunk 2"),- LangchainDocument(page_content="chunk 1 dup"),- ]- service.text_splitter.split_documents.return_value = chunks-- # Return IDs with a duplicate- db_ids = ["db-uuid-1", "db-uuid-2", "db-uuid-1"]- service.embedding_manager._store_chunks_to_db.return_value = db_ids-- # Setup FAISS mock- mock_faiss = Mock()- mock_docstore = Mock()- mock_docstore._dict = {}- mock_faiss.docstore = mock_docstore- service.faiss_index = mock_faiss-- mock_rag_index = Mock()- mock_rag_index.index_path = "/tmp/test.faiss"- mock_rag_index.id = "rag-1"- service.rag_index_record = mock_rag_index-- result = service.index_local_file(str(test_file))-- assert result["status"] == "success"- # Only 2 unique chunks should be added- assert result["chunk_count"] == 2- add_call = mock_faiss.add_documents.call_args- added_ids = add_call[1]["ids"]- assert len(added_ids) == 2- assert len(added_ids) == len(set(added_ids))-- def test_index_local_file_skips_existing_chunks(self, mocker, tmp_path):- """Verify existing_ids filtering works — chunks already in FAISS are skipped."""- service = _create_rag_service(mocker)-- test_file = tmp_path / "test.txt"- test_file.write_text(- "This is a test file with enough content to pass validation."- )-- chunks = [- LangchainDocument(page_content="already in index"),- LangchainDocument(page_content="new chunk"),- ]- service.text_splitter.split_documents.return_value = chunks-- db_ids = ["existing-id", "new-id"]- service.embedding_manager._store_chunks_to_db.return_value = db_ids-- # Setup FAISS mock with one existing ID- mock_faiss = Mock()- mock_docstore = Mock()- mock_docstore._dict = {"existing-id": "some_doc"}- mock_faiss.docstore = mock_docstore- service.faiss_index = mock_faiss-- mock_rag_index = Mock()- mock_rag_index.index_path = "/tmp/test.faiss"- mock_rag_index.id = "rag-1"- service.rag_index_record = mock_rag_index-- result = service.index_local_file(str(test_file))-- assert result["status"] == "success"- assert result["chunk_count"] == 1- assert result["embedding_ids"] == ["new-id"]- # Verify only the new chunk was added- add_call = mock_faiss.add_documents.call_args- assert add_call[1]["ids"] == ["new-id"]diff --git a/tests/research_library/services/test_library_rag_service_deep_coverage.py b/tests/research_library/services/test_library_rag_service_deep_coverage.pyindex aae9cc5e5f..770b9a7783 100644--- a/tests/research_library/services/test_library_rag_service_deep_coverage.py+++ b/tests/research_library/services/test_library_rag_service_deep_coverage.py@@ -6,8 +6,6 @@ Targets ~146 missing statements not covered by existing test files:- remove_document_from_rag (success, not-in-collection error, exception path)- index_documents_batch (not found, already indexed skip, no text, batch exception)- get_rag_stats (default collection id, with collection, chunk_sample present/absent)-- index_local_file (file not found, not a file, unsupported type, empty content,- txt success, html success, pdf success, exception path, no rag_index_record)- index_user_document (no content, success no rag_index_record, force_reindex path, exception)- remove_collection_from_index (no chunks, has chunks success, faiss delete exception, outer exception)- close (idempotent second call, clears resources)@@ -507,152 +505,6 @@ class TestGetRagStats:assert result["embedding_info"]["model_type"] is None-# =========================================================================-# index_local_file-# =========================================================================-class TestIndexLocalFile:- def test_file_not_found(self, tmp_path):- svc = _make_service()- result = svc.index_local_file(str(tmp_path / "no_such_file.txt"))- assert result["status"] == "error"- assert "not found" in result["error"]-- def test_not_a_file(self, tmp_path):- svc = _make_service()- d = tmp_path / "some_dir"- d.mkdir()- result = svc.index_local_file(str(d))- assert result["status"] == "error"- assert "Not a file" in result["error"]-- def test_unsupported_file_type(self, tmp_path):- svc = _make_service()- f = tmp_path / "data.xyz"- f.write_text("some content")- result = svc.index_local_file(str(f))- assert result["status"] == "skipped"- assert "Unsupported" in result["error"]-- def test_empty_content_returns_error(self, tmp_path):- svc = _make_service()- f = tmp_path / "empty.txt"- f.write_text(" ") # whitespace-only → < 10 chars after strip- result = svc.index_local_file(str(f))- assert result["status"] == "error"- assert "no extractable text" in result["error"]-- def test_txt_file_success_no_rag_index_record(self, tmp_path):- svc = _make_service()- svc.embedding_manager = MagicMock()- svc.embedding_manager._store_chunks_to_db.return_value = [- "id-1",- "id-2",- ]-- mock_faiss = MagicMock()- mock_faiss.docstore._dict = {}-- svc.faiss_index = mock_faiss- svc.rag_index_record = None # no record → save branch skipped-- mock_chunks = [LangchainDocument(page_content="chunk text")]- svc.text_splitter = MagicMock()- svc.text_splitter.split_documents.return_value = mock_chunks-- f = tmp_path / "doc.txt"- f.write_text("This is a long enough document text content for testing.")-- result = svc.index_local_file(str(f))- assert result["status"] == "success"- assert result["chunk_count"] >= 0-- def test_txt_file_success_with_rag_index_record(self, tmp_path):- svc = _make_service()- svc.embedding_manager = MagicMock()- svc.embedding_manager._store_chunks_to_db.return_value = ["id-1"]- svc.integrity_manager = MagicMock()-- mock_faiss = MagicMock()- mock_faiss.docstore._dict = {}- svc.faiss_index = mock_faiss-- mock_record = MagicMock()- index_file = tmp_path / "idx.faiss"- mock_record.index_path = str(index_file)- mock_record.id = "rag-rec-1"- svc.rag_index_record = mock_record-- mock_chunks = [LangchainDocument(page_content="some content text")]- svc.text_splitter = MagicMock()- svc.text_splitter.split_documents.return_value = mock_chunks-- f = tmp_path / "report.txt"- f.write_text("This is a long enough document text content for testing.")-- result = svc.index_local_file(str(f))- assert result["status"] == "success"- mock_faiss.save_local.assert_called_once()- svc.integrity_manager.record_file.assert_called_once()-- def test_txt_exception_path(self, tmp_path):- svc = _make_service()- svc.embedding_manager = MagicMock()- svc.embedding_manager._store_chunks_to_db.side_effect = RuntimeError(- "boom"- )-- mock_faiss = MagicMock()- mock_faiss.docstore._dict = {}- svc.faiss_index = mock_faiss-- mock_chunks = [- LangchainDocument(page_content="long enough content here")- ]- svc.text_splitter = MagicMock()- svc.text_splitter.split_documents.return_value = mock_chunks-- f = tmp_path / "broken.txt"- f.write_text("This is a long enough document text content for testing.")-- result = svc.index_local_file(str(f))- assert result["status"] == "error"- assert "RuntimeError" in result["error"]-- def test_html_file_success(self, tmp_path):- svc = _make_service()- svc.embedding_manager = MagicMock()- svc.embedding_manager._store_chunks_to_db.return_value = ["id-h1"]- svc.integrity_manager = MagicMock()-- mock_faiss = MagicMock()- mock_faiss.docstore._dict = {}- svc.faiss_index = mock_faiss- svc.rag_index_record = None-- mock_chunks = [LangchainDocument(page_content="html content text")]- svc.text_splitter = MagicMock()- svc.text_splitter.split_documents.return_value = mock_chunks-- f = tmp_path / "page.html"- f.write_text(- "<html><body><p>This is a long enough HTML document.</p></body></html>"- )-- result = svc.index_local_file(str(f))- assert result["status"] == "success"-- def test_pdf_file_exception_returns_error(self, tmp_path):- svc = _make_service()- svc.text_splitter = MagicMock()-- f = tmp_path / "doc.pdf"- f.write_bytes(b"%PDF invalid pdf content")-- # pypdf will raise on invalid PDF- result = svc.index_local_file(str(f))- assert result["status"] == "error"--# =========================================================================# index_user_document# =========================================================================