Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

refactor(library): remove orphaned RAG index-local and index-research endpoints (#4867)

LearningCircuit5c419343efef
diff --git a/changelog.d/+remove-orphaned-rag-index-endpoints.removal.md b/changelog.d/+remove-orphaned-rag-index-endpoints.removal.md
new file mode 100644
index 0000000000..1ef7df0e1a
--- /dev/null
+++ b/changelog.d/+remove-orphaned-rag-index-endpoints.removal.md
@@ -0,0 +1 @@
+Removed two orphaned RAG HTTP endpoints that had no UI and no other caller: `GET /library/api/rag/index-local` (local-folder indexing — superseded by collection-based indexing) and `POST /library/api/rag/index-research` (which called a method that no longer exists and always errored). The unused `index_local_file` service method and glob-pattern allowlist were removed with them.
diff --git a/src/local_deep_research/research_library/routes/rag_routes.py b/src/local_deep_research/research_library/routes/rag_routes.py
index 6ee7f03a4e..50df5dfac7 100644
--- a/src/local_deep_research/research_library/routes/rag_routes.py
+++ b/src/local_deep_research/research_library/routes/rag_routes.py
@@ -9,7 +9,6 @@ Provides endpoints for managing RAG indexing of library documents:
"""
import os
-import re
from flask import (
Blueprint,
@@ -23,7 +22,6 @@ from flask import (
from loguru import logger
from sqlalchemy import case, func
import atexit
-import glob
import json
import uuid
import time
@@ -51,7 +49,6 @@ from ...utilities.resource_utils import safe_close
from ..services.library_rag_service import LibraryRAGService
from ...settings.manager import SettingsManager
from ...security.file_upload_validator import FileUploadValidator
-from ...security.path_validator import PathValidator
from ...security.rate_limiter import (
upload_rate_limit_ip,
upload_rate_limit_user,
@@ -73,25 +70,6 @@ from ...constants import DEFAULT_SEARCH_TOOL
rag_bp = Blueprint("rag", __name__, url_prefix="/library")
-# Allowlist for user-supplied glob patterns. Each pattern is joined onto an
-# already-validated base directory and globbed, so it must not be able to
-# escape that directory. Only in-directory filename globs are allowed:
-# any path separator, parent reference, or character outside this set is
-# rejected. This also blocks a leading "/" — pathlib's "/" operator resets
-# to an absolute path when the right operand is absolute (``base / "/abs"``
-# yields ``/abs``), so a leading-slash absolute pattern would otherwise
-# escape the base too.
-# Anchored with ``\Z`` (not ``$``) so a trailing newline cannot slip through —
-# ``$`` matches just before a final "\n", which would make the predicate
-# depend on the caller stripping the pattern first.
-_SAFE_GLOB_PATTERN = re.compile(r"^[A-Za-z0-9._*?\[\]-]+\Z")
-
-
-def _is_safe_glob_pattern(pattern: str) -> bool:
- """Return True for in-directory filename globs that cannot traverse out."""
- return bool(_SAFE_GLOB_PATTERN.match(pattern)) and ".." not in pattern
-
-
# NOTE: Routes use session["username"] (not .get()) intentionally.
# @login_required guarantees the key exists; direct access fails fast
# if the decorator is ever removed.
@@ -852,31 +830,6 @@ def remove_document():
return handle_api_error(f"removing document {text_doc_id}", e)
-@rag_bp.route("/api/rag/index-research", methods=["POST"])
-@login_required
-@require_json_body(error_format="success")
-def index_research():
- """Index all documents from a research."""
- try:
- data = request.get_json()
- research_id = data.get("research_id")
- force_reindex = data.get("force_reindex", False)
-
- if not research_id:
- return jsonify(
- {"success": False, "error": "research_id is required"}
- ), 400
-
- with get_rag_service() as rag_service:
- results = rag_service.index_research_documents(
- research_id, force_reindex
- )
-
- return jsonify({"success": True, "results": results})
- except Exception as e:
- return handle_api_error(f"indexing research {research_id}", e)
-
-
@rag_bp.route("/api/rag/index-all", methods=["GET"])
@login_required
def index_all():
@@ -1252,144 +1205,6 @@ def get_documents():
return handle_api_error("getting documents", e)
-@rag_bp.route("/api/rag/index-local", methods=["GET"])
-@login_required
-def index_local_library():
- """Index documents from a local folder with Server-Sent Events progress."""
- folder_path = request.args.get("path")
- raw_patterns = request.args.get(
- "patterns", "*.pdf,*.txt,*.md,*.html"
- ).split(",")
- file_patterns = [p.strip() for p in raw_patterns if p.strip()]
- recursive = parse_bool_arg("recursive", default=True)
-
- if not folder_path:
- return jsonify({"success": False, "error": "Path is required"}), 400
-
- # Reject glob patterns that could escape the validated base directory.
- # The folder is validated below, but each pattern is joined onto it and
- # globbed; a pattern with ".." or a leading-slash absolute pattern would
- # otherwise read files outside the folder. Only in-directory filename
- # globs are allowed.
- if not file_patterns or not all(
- _is_safe_glob_pattern(p) for p in file_patterns
- ):
- return jsonify({"success": False, "error": "Invalid file pattern"}), 400
-
- # Validate and sanitize the path to prevent traversal attacks
- try:
- validated_path = PathValidator.validate_local_filesystem_path(
- folder_path
- )
- # Re-sanitize for static analyzer recognition (CodeQL)
- path = PathValidator.sanitize_for_filesystem_ops(validated_path)
- except ValueError:
- # Don't log the raw user-submitted path — it can contain a username.
- # A static message keeps operator visibility without leaking the input
- # (project policy disallows exception variables in logger.warning).
- logger.warning("Path validation failed")
- return jsonify({"success": False, "error": "Invalid path"}), 400
-
- # Check path exists and is a directory
- if not path.exists():
- return jsonify({"success": False, "error": "Path does not exist"}), 400
- if not path.is_dir():
- return jsonify(
- {"success": False, "error": "Path is not a directory"}
- ), 400
-
- # Create RAG service in request context
- rag_service = get_rag_service()
-
- def generate():
- """Generator function for SSE progress updates."""
- try:
- # Send initial status
- yield f"data: {json.dumps({'type': 'start', 'message': f'Scanning folder: {path}'})}\n\n"
-
- # Find all matching files
- files_to_index = []
- for pattern in file_patterns:
- pattern = pattern.strip()
- if recursive:
- search_pattern = str(path / "**" / pattern)
- else:
- search_pattern = str(path / pattern)
-
- matching_files = glob.glob(search_pattern, recursive=recursive)
- files_to_index.extend(matching_files)
-
- # Remove duplicates and sort
- files_to_index = sorted(set(files_to_index))
-
- # Confine matches to the indexed folder: glob follows symlinks, so
- # a symlink inside the folder could otherwise expose files outside
- # it. Drop any match whose resolved path escapes the base.
- files_to_index = PathValidator.confine_to_base(files_to_index, path)
-
- if not files_to_index:
- yield f"data: {json.dumps({'type': 'complete', 'results': {'successful': 0, 'skipped': 0, 'failed': 0, 'message': 'No matching files found'}})}\n\n"
- return
-
- results = {"successful": 0, "skipped": 0, "failed": 0, "errors": []}
- total = len(files_to_index)
-
- # Index each file
- for idx, file_path in enumerate(files_to_index, 1):
- file_name = Path(file_path).name
-
- # Send progress update
- yield f"data: {json.dumps({'type': 'progress', 'current': idx, 'total': total, 'filename': file_name, 'percent': int((idx / total) * 100)})}\n\n"
-
- try:
- # Index the file directly using RAG service
- result = rag_service.index_local_file(file_path)
-
- if result.get("status") == "success":
- results["successful"] += 1
- elif result.get("status") == "skipped":
- results["skipped"] += 1
- else:
- results["failed"] += 1
- results["errors"].append(
- {
- "file": file_name,
- "error": result.get("error", "Unknown error"),
- }
- )
- except Exception:
- results["failed"] += 1
- results["errors"].append(
- {"file": file_name, "error": "Failed to index file"}
- )
- # Log the file name only — the full path can contain a
- # username.
- logger.exception(f"Error indexing file {file_name}")
-
- # Send completion status
- yield f"data: {json.dumps({'type': 'complete', 'results': results})}\n\n"
-
- # Log the folder name only — the full path can contain a username.
- logger.info(
- f"Local library indexing complete for {path.name}: "
- f"{results['successful']} successful, "
- f"{results['skipped']} skipped, "
- f"{results['failed']} failed"
- )
-
- except Exception:
- logger.exception("Error in local library indexing")
- yield f"data: {json.dumps({'type': 'error', 'error': 'An internal error occurred during indexing'})}\n\n"
- finally:
- # See parallel comment in index-all generator above — generator
- # ``finally`` is the safe close site for streamed RAG services.
- safe_close(rag_service, "rag_service (index-local SSE)")
-
- return Response(
- stream_with_context(generate()), mimetype="text/event-stream"
- )
-
-
# Collection Management Routes
@@ -1643,7 +1458,6 @@ def upload_to_collection(collection_id):
safe_rollback,
)
from ...security import sanitize_filename, UnsafeFilenameError
- from pathlib import Path
import hashlib
import uuid
from ..services.pdf_storage_manager import PDFStorageManager
diff --git a/src/local_deep_research/research_library/services/library_rag_service.py b/src/local_deep_research/research_library/services/library_rag_service.py
index 41e2904ce2..4b21d44d1a 100644
--- a/src/local_deep_research/research_library/services/library_rag_service.py
+++ b/src/local_deep_research/research_library/services/library_rag_service.py
@@ -1500,158 +1500,6 @@ class LibraryRAGService:
"chunk_overlap": self.chunk_overlap,
}
- def index_local_file(self, file_path: str) -> Dict[str, Any]:
- """
- Index a local file from the filesystem into RAG.
-
- Args:
- file_path: Path to the file to index
-
- Returns:
- Dict with status, chunk_count, and any errors
- """
- from pathlib import Path
- import mimetypes
-
- file_path = Path(file_path)
-
- if not file_path.exists():
- return {"status": "error", "error": f"File not found: {file_path}"}
-
- if not file_path.is_file():
- return {"status": "error", "error": f"Not a file: {file_path}"}
-
- # Determine file type
- mime_type, _ = mimetypes.guess_type(str(file_path))
-
- # Read file content based on type
- try:
- if file_path.suffix.lower() in [".txt", ".md", ".markdown"]:
- # Text files
- with open(file_path, "r", encoding="utf-8") as f:
- content = f.read()
- elif file_path.suffix.lower() in [".html", ".htm"]:
- # HTML files - strip tags
- from bs4 import BeautifulSoup
-
- with open(file_path, "r", encoding="utf-8") as f:
- soup = BeautifulSoup(f.read(), "html.parser")
- content = soup.get_text()
- elif file_path.suffix.lower() == ".pdf":
- # PDF files - extract text
- from pypdf import PdfReader
-
- content = ""
- with open(file_path, "rb") as f:
- pdf_reader = PdfReader(f)
- for page in pdf_reader.pages:
- content += page.extract_text()
- else:
- return {
- "status": "skipped",
- "error": f"Unsupported file type: {file_path.suffix}",
- }
-
- if not content or len(content.strip()) < 10:
- return {
- "status": "error",
- "error": "File has no extractable text content",
- }
-
- # Create LangChain Document from text
- doc = LangchainDocument(
- page_content=content,
- metadata={
- "source": str(file_path),
- "source_id": f"local_{file_path.stem}_{hash(str(file_path))}",
- "title": file_path.stem,
- "document_title": file_path.stem,
- "file_type": file_path.suffix.lower(),
- "file_size": file_path.stat().st_size,
- "source_type": "local_file",
- "collection": "local_library",
- },
- )
-
- # Split into chunks
- chunks = self.text_splitter.split_documents([doc])
- logger.info(
- f"Split local file {file_path} into {len(chunks)} chunks"
- )
-
- # Store chunks in database (returns UUID-based IDs)
- embedding_ids = self.embedding_manager._store_chunks_to_db(
- chunks=chunks,
- collection_name="local_library",
- source_type="local_file",
- source_id=str(file_path),
- )
-
- # Load or create FAISS index using default library collection
- # (lazy; merge step below reloads under the lock anyway).
- if self.faiss_index is None:
- from ...database.library_init import get_default_library_id
-
- default_collection_id = get_default_library_id(
- self.username, self.db_password
- )
- self.faiss_index = self.load_or_create_faiss_index(
- default_collection_id
- )
-
- # Read-modify-write the on-disk FAISS under the lock.
- index_path = (
- Path(self.rag_index_record.index_path)
- if self.rag_index_record
- else None
- )
- if index_path:
- merge_stats = self._merge_and_persist_locked(
- index_path, chunks, embedding_ids, force_reindex=False
- )
- logger.info(
- f"Saved FAISS index to {index_path} with integrity tracking"
- )
- else:
- # No persistent index path — in-memory only path.
- # Preserve old behavior: dedup against in-memory state
- # and add directly without saving.
- if self.faiss_index is not None and hasattr(
- self.faiss_index, "docstore"
- ):
- existing_ids = set(self.faiss_index.docstore._dict.keys())
- else:
- existing_ids = None
- new_chunks, new_ids = self._deduplicate_chunks(
- chunks, embedding_ids, existing_ids
- )
- if new_chunks:
- self.faiss_index.add_documents(new_chunks, ids=new_ids)
- merge_stats = {
- "added": len(new_chunks),
- "skipped": len(chunks) - len(new_chunks),
- "added_ids": new_ids,
- }
-
- logger.info(
- f"Successfully indexed local file {file_path} with "
- f"{merge_stats['added']} new chunks "
- f"({merge_stats['skipped']} skipped)"
- )
-
- return {
- "status": "success",
- "chunk_count": merge_stats["added"],
- "embedding_ids": merge_stats["added_ids"],
- }
-
- except Exception as e:
- logger.exception(f"Error indexing local file {file_path}")
- return {
- "status": "error",
- "error": f"Operation failed: {type(e).__name__}",
- }
-
def index_user_document(
self, user_doc, collection_name: str, force_reindex: bool = False
) -> Dict[str, Any]:
diff --git a/tests/research_library/routes/test_rag_routes.py b/tests/research_library/routes/test_rag_routes.py
index ea1a7449ab..f144b02226 100644
--- a/tests/research_library/routes/test_rag_routes.py
+++ b/tests/research_library/routes/test_rag_routes.py
@@ -762,27 +762,6 @@ class TestRagIndexRoutes:
)
assert response.status_code == 401, response.status_code
- def test_index_research_route(self):
- """Test /api/rag/index-research POST endpoint exists."""
- # audit: PUNCHLIST reviewed 2026-05 — issue resolved by prior PR (recommendation: REWRITE).
- from flask import Flask
- from local_deep_research.research_library.routes.rag_routes import (
- rag_bp,
- )
-
- app = Flask(__name__)
- app.config["SECRET_KEY"] = "test-secret"
- app.register_blueprint(rag_bp)
- app.register_blueprint(auth_bp)
-
- with app.test_client() as client:
- response = client.post(
- "/library/api/rag/index-research",
- json={"research_id": "research123"},
- content_type="application/json",
- )
- assert response.status_code == 401, response.status_code
-
def test_index_all_route(self):
"""Test /api/rag/index-all GET endpoint exists."""
from flask import Flask
@@ -1765,88 +1744,6 @@ class TestRemoveDocumentEndpoint:
assert response.status_code == 401, response.status_code
-class TestIndexResearchEndpoint:
- """Extended tests for index research endpoint."""
-
- def test_index_research_missing_research_id(self):
- """Test index research without research_id."""
- from flask import Flask
- from local_deep_research.research_library.routes.rag_routes import (
- rag_bp,
- )
-
- app = Flask(__name__)
- app.config["SECRET_KEY"] = "test-secret"
- app.register_blueprint(rag_bp)
- app.register_blueprint(auth_bp)
-
- with app.test_client() as client:
- response = client.post(
- "/library/api/rag/index-research",
- json={},
- content_type="application/json",
- )
- assert response.status_code == 401, response.status_code
-
-
-class TestIndexLocalEndpoint:
- """Extended tests for index local library endpoint."""
-
- def test_index_local_missing_path(self):
- """Test index local without path."""
- from flask import Flask
- from local_deep_research.research_library.routes.rag_routes import (
- rag_bp,
- )
-
- app = Flask(__name__)
- app.config["SECRET_KEY"] = "test-secret"
- app.register_blueprint(rag_bp)
- app.register_blueprint(auth_bp)
-
- with app.test_client() as client:
- response = client.get("/library/api/rag/index-local")
- assert response.status_code == 401, response.status_code
-
- def test_index_local_path_traversal_attempt(self):
- """Test index local with path traversal attempt."""
- # audit: PUNCHLIST reviewed 2026-05 — issue resolved by prior PR (recommendation: REWRITE).
- from flask import Flask
- from local_deep_research.research_library.routes.rag_routes import (
- rag_bp,
- )
-
- app = Flask(__name__)
- app.config["SECRET_KEY"] = "test-secret"
- app.register_blueprint(rag_bp)
- app.register_blueprint(auth_bp)
-
- with app.test_client() as client:
- response = client.get(
- "/library/api/rag/index-local?path=../../etc/passwd"
- )
- assert response.status_code == 401, response.status_code
-
- def test_index_local_with_patterns(self):
- """Test index local with custom patterns."""
- # audit: PUNCHLIST reviewed 2026-05 — issue resolved by prior PR (recommendation: REWRITE).
- from flask import Flask
- from local_deep_research.research_library.routes.rag_routes import (
- rag_bp,
- )
-
- app = Flask(__name__)
- app.config["SECRET_KEY"] = "test-secret"
- app.register_blueprint(rag_bp)
- app.register_blueprint(auth_bp)
-
- with app.test_client() as client:
- response = client.get(
- "/library/api/rag/index-local?path=/tmp&patterns=*.pdf,*.txt"
- )
- assert response.status_code == 401, response.status_code
-
-
class TestGetDocumentsEndpoint:
"""Extended tests for get documents endpoint."""
diff --git a/tests/research_library/routes/test_rag_routes_coverage.py b/tests/research_library/routes/test_rag_routes_coverage.py
index 975d374e5e..500e9e723b 100644
--- a/tests/research_library/routes/test_rag_routes_coverage.py
+++ b/tests/research_library/routes/test_rag_routes_coverage.py
@@ -854,40 +854,6 @@ class TestRemoveDocument:
assert resp.status_code == 400
-# ---------------------------------------------------------------------------
-# Tests: POST /api/rag/index-research
-# ---------------------------------------------------------------------------
-
-
-class TestIndexResearch:
- def test_missing_research_id(self, app):
- with _auth_client(app) as (client, ctx):
- resp = client.post("/library/api/rag/index-research", json={})
- assert resp.status_code == 400
-
- def test_success(self, app):
- mock_rag = MagicMock()
- # rag_routes now uses ``with get_rag_service(...) as rag_service:``
- # MagicMock's default __enter__ returns a child mock, so route body
- # would see a different object. Pin __enter__ to self.
- mock_rag.__enter__.return_value = mock_rag
- mock_rag.index_research_documents.return_value = [{"status": "success"}]
-
- with _auth_client(
- app,
- extra_patches=[
- patch(f"{_ROUTES}.get_rag_service", return_value=mock_rag),
- ],
- ) as (client, ctx):
- resp = client.post(
- "/library/api/rag/index-research",
- json={"research_id": "research-1"},
- )
- assert resp.status_code == 200
- data = resp.get_json()
- assert data["success"] is True
-
-
# ---------------------------------------------------------------------------
# Tests: POST /api/rag/configure
# ---------------------------------------------------------------------------
@@ -2109,74 +2075,6 @@ class TestCancelIndexing:
assert resp.status_code == 404
-# ---------------------------------------------------------------------------
-# Tests: index_local_library (GET /api/rag/index-local)
-# ---------------------------------------------------------------------------
-
-
-class TestIndexLocalLibrary:
- def test_missing_path(self, app):
- with _auth_client(app) as (client, ctx):
- resp = client.get("/library/api/rag/index-local")
- assert resp.status_code == 400
-
- def test_invalid_path(self, app):
- with _auth_client(
- app,
- extra_patches=[
- patch(
- f"{_ROUTES}.PathValidator.validate_local_filesystem_path",
- side_effect=ValueError("bad path"),
- ),
- ],
- ) as (client, ctx):
- resp = client.get("/library/api/rag/index-local?path=/etc/../root")
- assert resp.status_code == 400
-
- def test_path_not_exist(self, app):
- mock_path = Mock()
- mock_path.exists.return_value = False
-
- with _auth_client(
- app,
- extra_patches=[
- patch(
- f"{_ROUTES}.PathValidator.validate_local_filesystem_path",
- return_value=mock_path,
- ),
- patch(
- f"{_ROUTES}.PathValidator.sanitize_for_filesystem_ops",
- return_value=mock_path,
- ),
- ],
- ) as (client, ctx):
- resp = client.get("/library/api/rag/index-local?path=/nonexistent")
- assert resp.status_code == 400
-
- def test_path_not_dir(self, app):
- mock_path = Mock()
- mock_path.exists.return_value = True
- mock_path.is_dir.return_value = False
-
- with _auth_client(
- app,
- extra_patches=[
- patch(
- f"{_ROUTES}.PathValidator.validate_local_filesystem_path",
- return_value=mock_path,
- ),
- patch(
- f"{_ROUTES}.PathValidator.sanitize_for_filesystem_ops",
- return_value=mock_path,
- ),
- ],
- ) as (client, ctx):
- resp = client.get(
- "/library/api/rag/index-local?path=/some/file.txt"
- )
- assert resp.status_code == 400
-
-
# ---------------------------------------------------------------------------
# Tests: helper functions
# ---------------------------------------------------------------------------
diff --git a/tests/research_library/routes/test_rag_routes_indexing.py b/tests/research_library/routes/test_rag_routes_indexing.py
index b056df5677..4167f691dd 100644
--- a/tests/research_library/routes/test_rag_routes_indexing.py
+++ b/tests/research_library/routes/test_rag_routes_indexing.py
@@ -1,303 +1,11 @@
# allow: no-sut-import — black-box HTTP test; drives real routes through the Flask test client
-"""Tests for index_local_library and view_document_chunks in rag_routes.py."""
+"""Tests for view_document_chunks in rag_routes.py."""
-from pathlib import Path
from unittest.mock import patch, MagicMock
RAG_PREFIX = "/library"
-class TestIndexLocalLibrary:
- """Tests for /api/rag/index-local SSE endpoint."""
-
- def test_requires_authentication(self, client):
- response = client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path=/tmp/docs"
- )
- assert response.status_code in [401, 302]
-
- def test_missing_path_returns_400(self, authenticated_client):
- response = authenticated_client.get(f"{RAG_PREFIX}/api/rag/index-local")
- assert response.status_code == 400
- data = response.get_json()
- assert data["success"] is False
- assert "path" in data["error"].lower()
-
- def test_traversal_glob_pattern_returns_400(self, authenticated_client):
- """A glob pattern that escapes the base dir is rejected before any
- path handling, so no traversal read can occur."""
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local"
- "?path=/tmp/docs&patterns=../../../etc/passwd"
- )
- assert response.status_code == 400
- data = response.get_json()
- assert data["success"] is False
- assert "pattern" in data["error"].lower()
-
- def test_absolute_glob_pattern_returns_400(self, authenticated_client):
- """An absolute pattern is rejected: pathlib's "/" resets to an absolute
- path, so ``base / "/etc/*"`` would otherwise escape the folder."""
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path=/tmp/docs&patterns=/etc/*"
- )
- assert response.status_code == 400
- data = response.get_json()
- assert data["success"] is False
- assert "pattern" in data["error"].lower()
-
- def test_empty_patterns_returns_400(self, authenticated_client):
- """A patterns value that is only commas/whitespace leaves nothing to
- match and is rejected rather than silently indexing everything."""
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path=/tmp/docs&patterns=,%20,"
- )
- assert response.status_code == 400
- data = response.get_json()
- assert data["success"] is False
- assert "pattern" in data["error"].lower()
-
- def test_one_bad_pattern_in_list_rejects_whole_request(
- self, authenticated_client
- ):
- """A single unsafe pattern alongside valid ones rejects the request
- (the validation is an all()-gate, not a silent filter)."""
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local"
- "?path=/tmp/docs&patterns=*.pdf,../etc/passwd"
- )
- assert response.status_code == 400
- data = response.get_json()
- assert data["success"] is False
- assert "pattern" in data["error"].lower()
-
- def test_valid_custom_patterns_pass_the_allowlist(
- self, authenticated_client
- ):
- """Realistic custom globs must NOT be rejected by the allowlist —
- guards against a future over-tightening of the regex. The request may
- still fail because the path does not exist, but never for a pattern
- reason."""
- for pattern in ("data_*.csv", "*.PDF", "report-[0-9].txt"):
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local"
- f"?path=/tmp/docs&patterns={pattern}"
- )
- error = (response.get_json() or {}).get("error") or ""
- assert "pattern" not in error.lower(), (
- f"{pattern!r} was wrongly rejected as an invalid pattern"
- )
-
- @patch(
- "local_deep_research.research_library.routes.rag_routes.PathValidator"
- )
- def test_invalid_path_returns_400(
- self, mock_validator, authenticated_client
- ):
- mock_validator.validate_local_filesystem_path.side_effect = ValueError(
- "blocked"
- )
-
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path=/etc/passwd"
- )
- assert response.status_code == 400
- data = response.get_json()
- assert data["success"] is False
-
- @patch(
- "local_deep_research.research_library.routes.rag_routes.PathValidator"
- )
- def test_invalid_path_does_not_log_submitted_path(
- self, mock_validator, authenticated_client
- ):
- """On a path-validation failure the submitted path must not reach the
- logs (it can contain a username); only the generic reason is logged."""
- from loguru import logger
-
- mock_validator.validate_local_filesystem_path.side_effect = ValueError(
- "Cannot access system directories"
- )
- secret_path = "/home/alice-secret-9f3a/private"
-
- # The package disables loguru by default; enable it so the warning
- # emitted from the route reaches our sink.
- logger.enable("local_deep_research")
- logged: list[str] = []
- sink_id = logger.add(
- lambda m: logged.append(m.record["message"]), level="WARNING"
- )
- try:
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path={secret_path}"
- )
- finally:
- logger.remove(sink_id)
- logger.disable("local_deep_research")
-
- assert response.status_code == 400
- failed = [m for m in logged if "Path validation failed" in m]
- assert failed, "expected a path-validation-failure log"
- # the submitted path (which can contain a username) is never logged
- assert all("alice-secret-9f3a" not in m for m in failed)
-
- def test_indexing_complete_log_uses_folder_basename(
- self, authenticated_client, loguru_caplog_full, tmp_path
- ):
- """The success "indexing complete" log names the folder by basename,
- never the full path (which can contain a username)."""
- folder = tmp_path / "alice-secret-9f3a" / "mydocs"
- folder.mkdir(parents=True)
- (folder / "report.txt").write_text("hello", encoding="utf-8")
-
- rag_service = MagicMock()
- rag_service.index_local_file.return_value = {"status": "success"}
-
- with (
- loguru_caplog_full.at_level("DEBUG"),
- patch(
- "local_deep_research.research_library.routes.rag_routes.get_rag_service",
- return_value=rag_service,
- ),
- ):
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path={folder}&recursive=false"
- )
- response.get_data() # drive the SSE generator
-
- assert response.status_code == 200
- text = loguru_caplog_full.text
- assert "alice-secret-9f3a" not in text
- assert "indexing complete for mydocs" in text
-
- def test_indexing_error_log_uses_file_basename(
- self, authenticated_client, loguru_caplog_full, tmp_path
- ):
- """A per-file indexing error logs the file's basename, never its full
- path (the exception block is captured too via loguru_caplog_full)."""
- folder = tmp_path / "bob-secret-1c2d" / "papers"
- folder.mkdir(parents=True)
- (folder / "thesis.txt").write_text("x", encoding="utf-8")
-
- rag_service = MagicMock()
- rag_service.index_local_file.side_effect = RuntimeError("boom")
-
- with (
- loguru_caplog_full.at_level("DEBUG"),
- patch(
- "local_deep_research.research_library.routes.rag_routes.get_rag_service",
- return_value=rag_service,
- ),
- ):
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path={folder}&recursive=false"
- )
- response.get_data()
-
- text = loguru_caplog_full.text
- assert "bob-secret-1c2d" not in text
- assert "Error indexing file thesis.txt" in text
-
- @patch(
- "local_deep_research.research_library.routes.rag_routes.PathValidator"
- )
- def test_nonexistent_path_returns_400(
- self, mock_validator, authenticated_client, tmp_path
- ):
- nonexistent = tmp_path / "nonexistent"
- mock_validator.validate_local_filesystem_path.return_value = nonexistent
- mock_validator.sanitize_for_filesystem_ops.return_value = nonexistent
-
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path={nonexistent}"
- )
- assert response.status_code == 400
-
- @patch(
- "local_deep_research.research_library.routes.rag_routes.PathValidator"
- )
- def test_file_not_directory_returns_400(
- self, mock_validator, authenticated_client, tmp_path
- ):
- file_path = tmp_path / "file.txt"
- file_path.write_text("content")
- mock_validator.validate_local_filesystem_path.return_value = file_path
- mock_validator.sanitize_for_filesystem_ops.return_value = file_path
-
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local?path={file_path}"
- )
- assert response.status_code == 400
-
- def test_symlink_escaping_folder_is_not_indexed(
- self, authenticated_client, tmp_path
- ):
- """glob follows symlinks, so a symlink inside the indexed folder that
- points outside it would otherwise be read. The route confines matches
- to the folder, so the escaping symlink is skipped while the real file
- inside the folder is still indexed."""
- folder = tmp_path / "docs"
- folder.mkdir()
- (folder / "real.txt").write_text("inside", encoding="utf-8")
- secret = tmp_path / "secret.txt"
- secret.write_text("OUTSIDE THE FOLDER", encoding="utf-8")
- (folder / "evil.txt").symlink_to(secret)
-
- indexed: list[str] = []
- rag_service = MagicMock()
- rag_service.index_local_file.side_effect = lambda fp: (
- indexed.append(fp) or {"status": "success"}
- )
-
- with patch(
- "local_deep_research.research_library.routes.rag_routes.get_rag_service",
- return_value=rag_service,
- ):
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local"
- f"?path={folder}&patterns=*.txt&recursive=false"
- )
- response.get_data() # drive the SSE generator
-
- assert response.status_code == 200
- names = {Path(p).name for p in indexed}
- assert "real.txt" in names
- assert "evil.txt" not in names
-
- def test_symlink_loop_does_not_abort_indexing(
- self, authenticated_client, tmp_path
- ):
- """A symlink loop in the folder (Path.resolve raises RuntimeError) must
- be skipped, not abort the whole run — the real file is still indexed."""
- folder = tmp_path / "docs"
- folder.mkdir()
- (folder / "real.txt").write_text("inside", encoding="utf-8")
- loop = folder / "loop.txt"
- loop.symlink_to(loop) # self-referential loop
-
- indexed: list[str] = []
- rag_service = MagicMock()
- rag_service.index_local_file.side_effect = lambda fp: (
- indexed.append(fp) or {"status": "success"}
- )
-
- with patch(
- "local_deep_research.research_library.routes.rag_routes.get_rag_service",
- return_value=rag_service,
- ):
- response = authenticated_client.get(
- f"{RAG_PREFIX}/api/rag/index-local"
- f"?path={folder}&patterns=*.txt&recursive=false"
- )
- response.get_data() # drive the SSE generator
-
- assert response.status_code == 200
- names = {Path(p).name for p in indexed}
- # the run completed and indexed the real file despite the loop
- assert "real.txt" in names
- assert "loop.txt" not in names
-
-
class TestViewDocumentChunks:
"""Tests for /api/rag/document/<id>/chunks endpoint."""
diff --git a/tests/research_library/services/test_library_rag_service.py b/tests/research_library/services/test_library_rag_service.py
index e9449f1770..5802c27180 100644
--- a/tests/research_library/services/test_library_rag_service.py
+++ b/tests/research_library/services/test_library_rag_service.py
@@ -1067,8 +1067,8 @@ class TestRemoveCollectionFromIndex:
class TestLoadOrCreateFaissIndexCollectionId:
"""Tests that load_or_create_faiss_index receives the correct collection_id.
- Fixes a pre-existing bug where index_local_file, index_user_document,
- and remove_collection_from_index called load_or_create_faiss_index()
+ Fixes a pre-existing bug where index_user_document and
+ remove_collection_from_index called load_or_create_faiss_index()
without the required collection_id argument.
"""
@@ -1109,49 +1109,6 @@ class TestLoadOrCreateFaissIndexCollectionId:
)
return service
- def test_index_local_file_passes_default_library_id(self, mocker, tmp_path):
- """index_local_file passes the default library collection_id."""
- service = self._create_service(mocker)
-
- # Create a real temp file
- test_file = tmp_path / "test.txt"
- test_file.write_text("This is test content for indexing purposes.")
-
- # Mock the FAISS index (simulate faiss_index being None)
- service.faiss_index = None
- mock_faiss_index = MagicMock()
- service.load_or_create_faiss_index = Mock(return_value=mock_faiss_index)
-
- # Mock _store_chunks_to_db
- service.embedding_manager._store_chunks_to_db = Mock(
- return_value=["id-1", "id-2"]
- )
-
- # Mock text_splitter to return chunks
- from langchain_core.documents import Document as LangchainDocument
-
- mock_chunks = [
- LangchainDocument(page_content="chunk 1"),
- LangchainDocument(page_content="chunk 2"),
- ]
- service.text_splitter.split_documents.return_value = mock_chunks
-
- # Mock get_default_library_id
- mocker.patch(
- "local_deep_research.database.library_init.get_default_library_id",
- return_value="default-lib-uuid",
- )
-
- # Mock rag_index_record for save path
- service.rag_index_record = None
-
- service.index_local_file(str(test_file))
-
- # Verify load_or_create_faiss_index was called with the default library id
- service.load_or_create_faiss_index.assert_called_once_with(
- "default-lib-uuid"
- )
-
def test_index_user_document_passes_extracted_collection_id(self, mocker):
"""index_user_document extracts collection_id from collection_name."""
service = self._create_service(mocker)
@@ -1231,37 +1188,6 @@ class TestLoadOrCreateFaissIndexCollectionId:
"xyz-uuid-456"
)
- def test_index_local_file_skips_load_when_faiss_already_set(
- self, mocker, tmp_path
- ):
- """index_local_file does not call load_or_create_faiss_index when faiss_index is already set."""
- service = self._create_service(mocker)
-
- test_file = tmp_path / "test.txt"
- test_file.write_text("This is test content for indexing purposes.")
-
- # Set faiss_index to non-None (already loaded)
- mock_faiss_index = MagicMock()
- service.faiss_index = mock_faiss_index
- service.load_or_create_faiss_index = Mock()
-
- service.embedding_manager._store_chunks_to_db = Mock(
- return_value=["id-1"]
- )
-
- from langchain_core.documents import Document as LangchainDocument
-
- service.text_splitter.split_documents.return_value = [
- LangchainDocument(page_content="chunk 1"),
- ]
-
- service.rag_index_record = None
-
- service.index_local_file(str(test_file))
-
- # load_or_create_faiss_index should NOT be called
- service.load_or_create_faiss_index.assert_not_called()
-
class TestDeduplicateChunks:
"""Tests for _deduplicate_chunks static method.
@@ -1563,128 +1489,3 @@ class TestForceReindexDedup:
assert len(delete_call) == len(set(delete_call)), (
"old_chunk_ids passed to FAISS.delete() must not contain duplicates"
)
-
-
-class TestIndexLocalFileDedup:
- """Tests for index_local_file deduplication and ID consistency."""
-
- def test_index_local_file_uses_db_embedding_ids(self, mocker, tmp_path):
- """Verify IDs come from _store_chunks_to_db, not manual generation."""
- service = _create_rag_service(mocker)
-
- # Create a real temp file
- test_file = tmp_path / "test.txt"
- test_file.write_text(
- "This is a test file with enough content to pass validation."
- )
-
- chunks = [
- LangchainDocument(page_content="chunk 1"),
- LangchainDocument(page_content="chunk 2"),
- ]
- service.text_splitter.split_documents.return_value = chunks
-
- db_ids = ["db-uuid-1", "db-uuid-2"]
- service.embedding_manager._store_chunks_to_db.return_value = db_ids
-
- # Setup FAISS mock
- mock_faiss = Mock()
- mock_docstore = Mock()
- mock_docstore._dict = {} # Empty index
- mock_faiss.docstore = mock_docstore
- service.faiss_index = mock_faiss
-
- mock_rag_index = Mock()
- mock_rag_index.index_path = "/tmp/test.faiss"
- mock_rag_index.id = "rag-1"
- service.rag_index_record = mock_rag_index
-
- result = service.index_local_file(str(test_file))
-
- assert result["status"] == "success"
- # The IDs in the result should come from DB, not manual local_* generation
- assert result["embedding_ids"] == db_ids
- # Verify add_documents was called with DB-based IDs
- add_call = mock_faiss.add_documents.call_args
- assert add_call[1]["ids"] == db_ids
-
- def test_index_local_file_deduplicates_before_faiss(self, mocker, tmp_path):
- """Verify dedup is applied before adding to FAISS."""
- service = _create_rag_service(mocker)
-
- test_file = tmp_path / "test.txt"
- test_file.write_text(
- "This is a test file with enough content to pass validation."
- )
-
- chunks = [
- LangchainDocument(page_content="chunk 1"),
- LangchainDocument(page_content="chunk 2"),
- LangchainDocument(page_content="chunk 1 dup"),
- ]
- service.text_splitter.split_documents.return_value = chunks
-
- # Return IDs with a duplicate
- db_ids = ["db-uuid-1", "db-uuid-2", "db-uuid-1"]
- service.embedding_manager._store_chunks_to_db.return_value = db_ids
-
- # Setup FAISS mock
- mock_faiss = Mock()
- mock_docstore = Mock()
- mock_docstore._dict = {}
- mock_faiss.docstore = mock_docstore
- service.faiss_index = mock_faiss
-
- mock_rag_index = Mock()
- mock_rag_index.index_path = "/tmp/test.faiss"
- mock_rag_index.id = "rag-1"
- service.rag_index_record = mock_rag_index
-
- result = service.index_local_file(str(test_file))
-
- assert result["status"] == "success"
- # Only 2 unique chunks should be added
- assert result["chunk_count"] == 2
- add_call = mock_faiss.add_documents.call_args
- added_ids = add_call[1]["ids"]
- assert len(added_ids) == 2
- assert len(added_ids) == len(set(added_ids))
-
- def test_index_local_file_skips_existing_chunks(self, mocker, tmp_path):
- """Verify existing_ids filtering works — chunks already in FAISS are skipped."""
- service = _create_rag_service(mocker)
-
- test_file = tmp_path / "test.txt"
- test_file.write_text(
- "This is a test file with enough content to pass validation."
- )
-
- chunks = [
- LangchainDocument(page_content="already in index"),
- LangchainDocument(page_content="new chunk"),
- ]
- service.text_splitter.split_documents.return_value = chunks
-
- db_ids = ["existing-id", "new-id"]
- service.embedding_manager._store_chunks_to_db.return_value = db_ids
-
- # Setup FAISS mock with one existing ID
- mock_faiss = Mock()
- mock_docstore = Mock()
- mock_docstore._dict = {"existing-id": "some_doc"}
- mock_faiss.docstore = mock_docstore
- service.faiss_index = mock_faiss
-
- mock_rag_index = Mock()
- mock_rag_index.index_path = "/tmp/test.faiss"
- mock_rag_index.id = "rag-1"
- service.rag_index_record = mock_rag_index
-
- result = service.index_local_file(str(test_file))
-
- assert result["status"] == "success"
- assert result["chunk_count"] == 1
- assert result["embedding_ids"] == ["new-id"]
- # Verify only the new chunk was added
- add_call = mock_faiss.add_documents.call_args
- assert add_call[1]["ids"] == ["new-id"]
diff --git a/tests/research_library/services/test_library_rag_service_deep_coverage.py b/tests/research_library/services/test_library_rag_service_deep_coverage.py
index aae9cc5e5f..770b9a7783 100644
--- a/tests/research_library/services/test_library_rag_service_deep_coverage.py
+++ b/tests/research_library/services/test_library_rag_service_deep_coverage.py
@@ -6,8 +6,6 @@ Targets ~146 missing statements not covered by existing test files:
- remove_document_from_rag (success, not-in-collection error, exception path)
- index_documents_batch (not found, already indexed skip, no text, batch exception)
- get_rag_stats (default collection id, with collection, chunk_sample present/absent)
-- index_local_file (file not found, not a file, unsupported type, empty content,
- txt success, html success, pdf success, exception path, no rag_index_record)
- index_user_document (no content, success no rag_index_record, force_reindex path, exception)
- remove_collection_from_index (no chunks, has chunks success, faiss delete exception, outer exception)
- close (idempotent second call, clears resources)
@@ -507,152 +505,6 @@ class TestGetRagStats:
assert result["embedding_info"]["model_type"] is None
-# =========================================================================
-# index_local_file
-# =========================================================================
-class TestIndexLocalFile:
- def test_file_not_found(self, tmp_path):
- svc = _make_service()
- result = svc.index_local_file(str(tmp_path / "no_such_file.txt"))
- assert result["status"] == "error"
- assert "not found" in result["error"]
-
- def test_not_a_file(self, tmp_path):
- svc = _make_service()
- d = tmp_path / "some_dir"
- d.mkdir()
- result = svc.index_local_file(str(d))
- assert result["status"] == "error"
- assert "Not a file" in result["error"]
-
- def test_unsupported_file_type(self, tmp_path):
- svc = _make_service()
- f = tmp_path / "data.xyz"
- f.write_text("some content")
- result = svc.index_local_file(str(f))
- assert result["status"] == "skipped"
- assert "Unsupported" in result["error"]
-
- def test_empty_content_returns_error(self, tmp_path):
- svc = _make_service()
- f = tmp_path / "empty.txt"
- f.write_text(" ") # whitespace-only → < 10 chars after strip
- result = svc.index_local_file(str(f))
- assert result["status"] == "error"
- assert "no extractable text" in result["error"]
-
- def test_txt_file_success_no_rag_index_record(self, tmp_path):
- svc = _make_service()
- svc.embedding_manager = MagicMock()
- svc.embedding_manager._store_chunks_to_db.return_value = [
- "id-1",
- "id-2",
- ]
-
- mock_faiss = MagicMock()
- mock_faiss.docstore._dict = {}
-
- svc.faiss_index = mock_faiss
- svc.rag_index_record = None # no record → save branch skipped
-
- mock_chunks = [LangchainDocument(page_content="chunk text")]
- svc.text_splitter = MagicMock()
- svc.text_splitter.split_documents.return_value = mock_chunks
-
- f = tmp_path / "doc.txt"
- f.write_text("This is a long enough document text content for testing.")
-
- result = svc.index_local_file(str(f))
- assert result["status"] == "success"
- assert result["chunk_count"] >= 0
-
- def test_txt_file_success_with_rag_index_record(self, tmp_path):
- svc = _make_service()
- svc.embedding_manager = MagicMock()
- svc.embedding_manager._store_chunks_to_db.return_value = ["id-1"]
- svc.integrity_manager = MagicMock()
-
- mock_faiss = MagicMock()
- mock_faiss.docstore._dict = {}
- svc.faiss_index = mock_faiss
-
- mock_record = MagicMock()
- index_file = tmp_path / "idx.faiss"
- mock_record.index_path = str(index_file)
- mock_record.id = "rag-rec-1"
- svc.rag_index_record = mock_record
-
- mock_chunks = [LangchainDocument(page_content="some content text")]
- svc.text_splitter = MagicMock()
- svc.text_splitter.split_documents.return_value = mock_chunks
-
- f = tmp_path / "report.txt"
- f.write_text("This is a long enough document text content for testing.")
-
- result = svc.index_local_file(str(f))
- assert result["status"] == "success"
- mock_faiss.save_local.assert_called_once()
- svc.integrity_manager.record_file.assert_called_once()
-
- def test_txt_exception_path(self, tmp_path):
- svc = _make_service()
- svc.embedding_manager = MagicMock()
- svc.embedding_manager._store_chunks_to_db.side_effect = RuntimeError(
- "boom"
- )
-
- mock_faiss = MagicMock()
- mock_faiss.docstore._dict = {}
- svc.faiss_index = mock_faiss
-
- mock_chunks = [
- LangchainDocument(page_content="long enough content here")
- ]
- svc.text_splitter = MagicMock()
- svc.text_splitter.split_documents.return_value = mock_chunks
-
- f = tmp_path / "broken.txt"
- f.write_text("This is a long enough document text content for testing.")
-
- result = svc.index_local_file(str(f))
- assert result["status"] == "error"
- assert "RuntimeError" in result["error"]
-
- def test_html_file_success(self, tmp_path):
- svc = _make_service()
- svc.embedding_manager = MagicMock()
- svc.embedding_manager._store_chunks_to_db.return_value = ["id-h1"]
- svc.integrity_manager = MagicMock()
-
- mock_faiss = MagicMock()
- mock_faiss.docstore._dict = {}
- svc.faiss_index = mock_faiss
- svc.rag_index_record = None
-
- mock_chunks = [LangchainDocument(page_content="html content text")]
- svc.text_splitter = MagicMock()
- svc.text_splitter.split_documents.return_value = mock_chunks
-
- f = tmp_path / "page.html"
- f.write_text(
- "<html><body><p>This is a long enough HTML document.</p></body></html>"
- )
-
- result = svc.index_local_file(str(f))
- assert result["status"] == "success"
-
- def test_pdf_file_exception_returns_error(self, tmp_path):
- svc = _make_service()
- svc.text_splitter = MagicMock()
-
- f = tmp_path / "doc.pdf"
- f.write_bytes(b"%PDF invalid pdf content")
-
- # pypdf will raise on invalid PDF
- result = svc.index_local_file(str(f))
- assert result["status"] == "error"
-
-
# =========================================================================
# index_user_document
# =========================================================================