~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(rag): withhold indexing exception details from clients (#6994)
LearningCircuitee65664bd025
diff --git a/changelog.d/6273.security.md b/changelog.d/6273.security.mdnew file mode 100644index 0000000000..38ba860c00--- /dev/null+++ b/changelog.d/6273.security.md@@ -0,0 +1 @@+Use fixed client-facing messages for RAG indexing and reconciliation failures, and filter legacy task rows when served. This keeps server paths, SQL text, and provider endpoints from exception details out of the indexing status endpoint, the indexing SSE streams, the RAG document index and remove routes, and the note index route. The RAG service's fixed per-document messages ("Document not found", "Document not found in collection", "Document has no text content" and "Document deleted during indexing") still reach those responses unchanged. Error messages that the indexing worker builds only from fixed phrases and counts, such as the vector-store drop retry guidance and "Collection not found", still reach the status endpoint unchanged.diff --git a/src/local_deep_research/web/routers/notes.py b/src/local_deep_research/web/routers/notes.pyindex 08252bc053..217e50b944 100644--- a/src/local_deep_research/web/routers/notes.py+++ b/src/local_deep_research/web/routers/notes.py@@ -2270,7 +2270,7 @@ def index_note_to_collection(collection_id: The collection to index the note into (required)force_reindex: Whether to force reindexing (optional, default false)"""- from ...web.routers.rag import get_rag_service+ from ...web.routers.rag import _client_indexing_error, get_rag_serviceif isinstance(body, JSONResponse):return body@@ -2320,8 +2320,13 @@ def index_note_to_collection()if result["status"] == "error":+ # Same boundary as the RAG document routes: known document+ # guidance passes, any other service text is withheld.return JSONResponse(- {"success": False, "error": result.get("error")},+ {+ "success": False,+ "error": _client_indexing_error(result.get("error")),+ },status_code=400,)diff --git a/src/local_deep_research/web/routers/rag.py b/src/local_deep_research/web/routers/rag.pyindex 1feda59625..9a1e3c7ca0 100644--- a/src/local_deep_research/web/routers/rag.py+++ b/src/local_deep_research/web/routers/rag.py@@ -1161,7 +1161,10 @@ async def index_document(if result["status"] == "error":return JSONResponse(- {"success": False, "error": result.get("error")},+ {+ "success": False,+ "error": _client_indexing_error(result.get("error")),+ },status_code=400,)@@ -1211,7 +1214,10 @@ async def remove_document(if result["status"] == "error":return JSONResponse(- {"success": False, "error": result.get("error")},+ {+ "success": False,+ "error": _client_indexing_error(result.get("error")),+ },status_code=400,)@@ -1429,7 +1435,9 @@ def index_all({"doc_id": doc_id,"title": title,- "error": result.get("error"),+ "error": _client_indexing_error(+ result.get("error")+ ),})@@ -3712,14 +3720,12 @@ def index_collection(results["successful"] = _aggregate["successful"]results["skipped"] = _aggregate["skipped"]results["failed"] = _aggregate["failed"]- # Replace placeholder errors with the structured ones- # from the parallel helper so the final ``complete``- # event matches what previous serial runs returned.+ # Keep document identity for the UI, but never put an+ # extension's arbitrary error detail in the SSE event.results["errors"] = [{"filename": err.get("title"),- "error": sanitize_error_message(str(err.get("error")))- or "Failed to index document",+ "error": _client_indexing_error(err.get("error")),}for err in _aggregate.get("errors", [])]@@ -4019,23 +4025,219 @@ def _auto_index_documents_worker(logger.exception("Auto-indexing worker failed")-def _sanitized_indexing_errors(results: dict, limit: int = 50) -> list:- """Return a sanitized, bounded list of per-document errors for the- indexing task metadata. Used by both terminal paths of- :func:`_background_index_worker` so the scrubbing logic is defined once.+# Fixed per-document guidance that LibraryRAGService returns as a result's+# "error". An exact match reaches the client; _client_indexing_error replaces+# any other text. TestIndexingErrorAllowlistMatchesService in+# tests/security/test_library_rag_security_fastapi.py checks this set, and+# _EMPTY_VECTOR_STORE_REASON below, against the text the service builds.+_SAFE_INDEXING_ERRORS = frozenset(+ {+ "Document not found",+ "Document not found in collection",+ "Document has no text content",+ "Document deleted during indexing",+ }+)+_EMPTY_VECTOR_STORE_REASON = (+ "live vector store reported zero ids while DocumentChunk rows exist; "+ "refusing to clear indexed flags"+)+_COLLECTION_NOT_FOUND = "Collection not found"+_REMOTE_DROP_OUTCOME = "indexing was aborted"+# Codes for ``metadata_json["failure"]``. The indexing worker records one next+# to an ``error_message`` it built only from fixed phrases and counts, and the+# status endpoint rebuilds that message from the record instead of serving the+# stored text.+_FAILURE_COLLECTION_NOT_FOUND = "collection_not_found"+_FAILURE_REMOTE_DROP = "remote_vector_store_drop"+++def _client_indexing_error(error: Any) -> str:+ """Return only messages known to be safe for indexing clients.++ Service extensions and exception handlers can supply arbitrary error text.+ Credential-shape scrubbing does not remove paths, SQL, or provider URLs.+ """+ if error is None or (isinstance(error, str) and not error):+ return "Indexing failed"+ if isinstance(error, str) and error in _SAFE_INDEXING_ERRORS:+ return error+ return "Indexing failed. Check the server logs for details."+++def _client_reconciliation_reason(reason: Any) -> str:+ """Preserve the known durable-state warning, never arbitrary detail."""+ if isinstance(reason, str) and reason == _EMPTY_VECTOR_STORE_REASON:+ return _EMPTY_VECTOR_STORE_REASON+ return (+ "Could not verify the vector store. Check the server logs for details."+ )+++def _reconciliation_skipped_message(reason: str) -> str:+ """Task error for a skipped reconciliation, given a client-safe reason.++ The worker stores it and the status endpoint rebuilds it, both through+ this function, so the two texts cannot drift apart.+ """+ return (+ f"Reconciliation skipped: {reason.rstrip('.')}. Indexed flags "+ "preserved; verify the FAISS index before retrying."+ )+++def _reconciled_failures_message(failed: int) -> str:+ """Task error when documents failed and reconciliation then completed."""+ return (+ f"{failed} document(s) failed to index. The collection was "+ "reconciled to the durable vector store."+ )+++def _remote_drop_failure(exc: Any) -> dict:+ """The failure record for a force-reindex whose remote drop failed.++ It keeps only the two fields ``remote_drop_failure_message`` reads: a+ flag and a count. The error's own text is never recorded. A count that is+ not a non-negative int is stored as None, so the status endpoint withholds+ the message rather than rebuild it from a bad value.+ """+ dropped = exc.dropped+ return {+ "code": _FAILURE_REMOTE_DROP,+ "retryable": bool(exc.retryable),+ "dropped": dropped if type(dropped) is int and dropped >= 0 else None,+ }+++def _client_task_failure_message(failure: Any) -> Optional[str]:+ """Rebuild the client text for a worker failure record.++ Returns None for anything else: no record, an unknown code, or a field of+ the wrong type. The caller then withholds the stored message."""+ if not isinstance(failure, dict):+ return None+ code = failure.get("code")+ if code == _FAILURE_COLLECTION_NOT_FOUND:+ return _COLLECTION_NOT_FOUND+ if code == _FAILURE_REMOTE_DROP:+ retryable = failure.get("retryable")+ dropped = failure.get("dropped")+ if type(retryable) is bool and type(dropped) is int and dropped >= 0:+ from ...research_library.deletion.utils.cascade_helper import (+ RemoteVectorStoreDropError,+ remote_drop_failure_message,+ )++ # The same template the worker used, fed only the recorded+ # flag and count.+ return remote_drop_failure_message(+ RemoteVectorStoreDropError(+ "recorded remote-drop failure",+ retryable=retryable,+ dropped=dropped,+ ),+ _REMOTE_DROP_OUTCOME,+ )+ return None+++def _sanitized_indexing_errors(results: dict, limit: int = 50) -> list:+ """Return a bounded list of client-safe per-document errors."""+ errors = results.get("errors")+ if not isinstance(errors, list):+ return []return [{"doc_id": item.get("doc_id"),"title": item.get("title"),- "error": sanitize_error_message(- str(item.get("error") or "Indexing failed")- ),+ "error": _client_indexing_error(item.get("error")),}- for item in results.get("errors", [])[:limit]+ for item in errors[: max(0, limit)]+ if isinstance(item, dict)]+def _client_indexing_summary(results: dict) -> dict:+ """Keep only the non-negative integer counts and the ``cancelled`` flag.++ Every other key is dropped.+ """+ summary = {}+ for key in ("successful", "skipped", "failed", "total"):+ value = results.get(key)+ if type(value) is int and value >= 0:+ summary[key] = value+ if type(results.get("cancelled")) is bool:+ summary["cancelled"] = results["cancelled"]+ return summary+++def _client_indexing_result(result: Any) -> Optional[dict]:+ """Filter old and new task rows at the HTTP boundary."""+ if not isinstance(result, dict):+ return None+ safe = _client_indexing_summary(result)+ for key in (+ "durable_indexed_documents",+ "durable_indexed_chunks",+ "live_vectors",+ "orphan_vectors",+ ):+ value = result.get(key)+ if type(value) is int and value >= 0:+ safe[key] = value+ if result.get("reconciliation_skipped") is True:+ safe["reconciliation_skipped"] = True+ safe["reconciliation_reason"] = _client_reconciliation_reason(+ result.get("reconciliation_reason")+ )+ if "errors" in result:+ safe["errors"] = _sanitized_indexing_errors(result)+ return safe+++def _client_indexing_task_error(+ message: Any, result: Optional[dict], failure: Any = None+) -> Optional[str]:+ """Rebuild known task errors rather than trusting persisted text.++ The client text comes from the worker's failure record (``failure``) or+ from the result counts (``result``, already filtered by+ ``_client_indexing_result``). Any other stored message gets the generic+ text. That includes rows written before these rules, whose text can hold+ exception detail.++ A stored message may be served unchanged only through an exact match on+ text that this module writes to an indexing task row.+ ``TestIndexingTaskErrorPassThroughs`` in+ tests/security/test_library_rag_security_fastapi.py fails on a match+ that nothing writes, and on fixed text written without a failure record+ that is neither matched nor listed there as withheld.+ """+ if message is None:+ return None+ rebuilt = _client_task_failure_message(failure)+ if rebuilt is not None:+ return rebuilt+ if message == "Failed to start indexing. Please try again.":+ return message+ if result is not None:+ if result.get("reconciliation_skipped") is True:+ return _reconciliation_skipped_message(+ _client_reconciliation_reason(+ result.get("reconciliation_reason")+ )+ )+ failed = result.get("failed")+ if type(failed) is int and failed > 0:+ if "durable_indexed_documents" in result:+ return _reconciled_failures_message(failed)+ return f"{failed} document(s) failed to index. Check the server logs for details."+ return "Indexing failed. Check the server logs for details."++@thread_cleanupdef _background_index_worker(task_id: str,@@ -4086,7 +4288,8 @@ def _background_index_worker(db_password,task_id,status="failed",- error_message="Collection not found",+ error_message=_COLLECTION_NOT_FOUND,+ failure={"code": _FAILURE_COLLECTION_NOT_FOUND},)return@@ -4224,7 +4427,7 @@ def _background_index_worker("live_vectors": 0,"orphan_vectors": 0,"reconciliation_skipped": True,- "reconciliation_reason": sanitize_error_message(str(exc)),+ "reconciliation_reason": _client_reconciliation_reason(exc),}if not isinstance(reconciliation, dict):reconciliation = {@@ -4245,10 +4448,10 @@ def _background_index_worker(reconciliation.get("reason")or "stored vectors could not be loaded")- sanitized_reason = sanitize_error_message(reason)+ sanitized_reason = _client_reconciliation_reason(reason)logger.warning(f"Background indexing task {task_id}: reconciliation "- f"skipped — {sanitized_reason}"+ f"skipped — {sanitize_error_message(str(reason))}")_update_task_status(username,@@ -4262,13 +4465,11 @@ def _background_index_worker(f"{results['skipped']} skipped; reconciliation "f"skipped: {sanitized_reason}"),- error_message=(- f"Reconciliation skipped: {sanitized_reason}. "- "Indexed flags preserved; verify the FAISS index "- "before retrying."+ error_message=_reconciliation_skipped_message(+ sanitized_reason),result_metadata={- **{k: v for k, v in results.items() if k != "errors"},+ **_client_indexing_summary(results),# Omit durable_indexed_documents/chunks and# live_vectors/orphan_vectors: true durable state is# unverified on the skipped path, and the UI@@ -4287,7 +4488,7 @@ def _background_index_worker(durable_documents = reconciliation["indexed_documents"]durable_chunks = reconciliation["indexed_chunks"]task_result = {- **results,+ **_client_indexing_summary(results),"durable_indexed_documents": durable_documents,"durable_indexed_chunks": durable_chunks,"live_vectors": reconciliation["live_vectors"],@@ -4310,8 +4511,7 @@ def _background_index_worker(progress_current=total,progress_message=message,error_message=(- f"{results['failed']} document(s) failed to index. "- "The collection was reconciled to the durable vector store."+ _reconciled_failures_message(results["failed"])if has_failureselse None),@@ -4337,8 +4537,10 @@ def _background_index_worker(task_id,status="failed",error_message=remote_drop_failure_message(- exc, "indexing was aborted"+ exc, _REMOTE_DROP_OUTCOME),+ # Lets the status endpoint rebuild this fixed-phrase message.+ failure=_remote_drop_failure(exc),)except Exception as e:logger.exception(f"Background indexing task {task_id} failed")@@ -4347,9 +4549,10 @@ def _background_index_worker(db_password,task_id,status="failed",- # Scrub creds/keys: this is later returned to the client by the- # index-status endpoint; full trace stays in server logs above.- error_message=sanitize_error_message(str(e)),+ # Always the generic text for an exception, which is also what+ # the status endpoint serves for a message it cannot rebuild. The+ # full exception stays in the server log above.+ error_message=_client_indexing_error(e),)@@ -4380,6 +4583,7 @@ def _do_update_task_status(progress_message: str = None,error_message: str = None,result_metadata: dict = None,+ failure: Optional[dict] = None,):"""Perform the task-status update under a tenacity-managed retry loop.@@ -4387,6 +4591,10 @@ def _do_update_task_status(with 0.05s exponential backoff); any other exception escapes so theouter caller can log it without retrying. See PR #5235 review comment3669857779.++ ``result_metadata`` is stored as ``metadata_json["result"]`` and+ ``failure`` as ``metadata_json["failure"]`` (see+ ``_client_task_failure_message``)."""from ...database.session_context import get_user_db_session@@ -4417,9 +4625,12 @@ def _do_update_task_status(task.progress_message = progress_messageif error_message is not None:task.error_message = error_message- if result_metadata is not None:+ if result_metadata is not None or failure is not None:metadata = dict(task.metadata_json or {})- metadata["result"] = result_metadata+ if result_metadata is not None:+ metadata["result"] = result_metadata+ if failure is not None:+ metadata["failure"] = failuretask.metadata_json = metadatadb_session.commit()@@ -4434,6 +4645,7 @@ def _update_task_status(progress_message: str = None,error_message: str = None,result_metadata: dict = None,+ failure: Optional[dict] = None,):"""Update task metadata in the database."""try:@@ -4447,6 +4659,7 @@ def _update_task_status(progress_message=progress_message,error_message=error_message,result_metadata=result_metadata,+ failure=failure,)except Exception:logger.exception(f"Failed to update task status for {task_id}")@@ -4717,15 +4930,25 @@ def get_index_status("message": "No indexing task for this collection",}+ task_metadata = task.metadata_json or {}+ safe_result = _client_indexing_result(task_metadata.get("result"))return {"task_id": task.task_id,"collection_id": collection_id,"status": task.status,"progress_current": task.progress_current or 0,"progress_total": task.progress_total or 0,- "progress_message": task.progress_message,- "error_message": task.error_message,- "result": (task.metadata_json or {}).get("result"),+ "progress_message": (+ "Indexing failed"+ if task.status == "failed"+ else task.progress_message+ ),+ "error_message": _client_indexing_task_error(+ task.error_message,+ safe_result,+ task_metadata.get("failure"),+ ),+ "result": safe_result,"created_at": task.created_at.isoformat()if task.created_atelse None,diff --git a/tests/notes/test_notes_api.py b/tests/notes/test_notes_api.pyindex c4ca9ba427..fdebc67cb0 100644--- a/tests/notes/test_notes_api.py+++ b/tests/notes/test_notes_api.py@@ -559,7 +559,8 @@ class TestIndexNoteToCollectionRoute:collection_id before any RAG work, (b) 404 an unknown note id BEFOREreaching ``get_rag_service`` (so an arbitrary user document can't beindexed through this path), and (c) map a RAG-service error result to a- 400 error contract rather than letting it surface as a 500.+ 400 error contract, with a client-safe message, rather than letting it+ surface as a 500.The branch pins only the unknown-note 404 (via``tests/security/test_two_user_attack_simulation.py``) and the@@ -653,12 +654,39 @@ class TestIndexNoteToCollectionRoute:)rag_service.__exit__.assert_called_once_with(None, None, None)+ @pytest.mark.parametrize(+ ("service_error", "client_error"),+ [+ pytest.param(+ "embedding provider unavailable at "+ "http://embed-internal.corp.example:11434",+ "Indexing failed. Check the server logs for details.",+ id="arbitrary-service-text-is-withheld",+ ),+ pytest.param(+ "Document has no text content",+ "Document has no text content",+ id="known-document-guidance-passes",+ ),+ pytest.param(+ "Document deleted during indexing",+ "Document deleted during indexing",+ id="deleted-during-indexing-guidance-passes",+ ),+ ],+ )def test_rag_error_result_maps_to_400_not_500(- self, monkeypatch, db_session, note_source_type+ self,+ monkeypatch,+ db_session,+ note_source_type,+ service_error,+ client_error,):"""A RAG-service result with status='error' is a handled failure --- the route must return the route's 400 error contract carrying the- service message, NOT let it fall through to a generic 500."""+ the route must return the route's 400 error contract, NOT let it fall+ through to a generic 500. Like the RAG document routes, it passes+ known document guidance and withholds any other service text."""notes_routes = _route_ctx(monkeypatch, db_session)note = _seed_note(db_session, note_source_type)@@ -666,7 +694,7 @@ class TestIndexNoteToCollectionRoute:rag_service.__enter__.return_value = rag_servicerag_service.index_document.return_value = {"status": "error",- "error": "embedding provider unavailable",+ "error": service_error,}self._patch_rag(monkeypatch, lambda request, username, cid: rag_service)@@ -680,7 +708,7 @@ class TestIndexNoteToCollectionRoute:assert status == 400assert payload["success"] is False- assert payload["error"] == "embedding provider unavailable"+ assert payload["error"] == client_errorrag_service.__exit__.assert_called_once_with(None, None, None)diff --git a/tests/research_library/routes/test_rag_routes_background_main_port.py b/tests/research_library/routes/test_rag_routes_background_main_port.pyindex 0645bbbde5..8ec221101a 100644--- a/tests/research_library/routes/test_rag_routes_background_main_port.py+++ b/tests/research_library/routes/test_rag_routes_background_main_port.py@@ -618,9 +618,9 @@ class TestReconciliationSkippedOmitsDurableCounts:chunks" sentence instead of implying data loss where state is merelyunverified. Regression for PR #5235 review comment 5085604502.- No branch test asserts the ABSENCE of those keys — the only branch tests- on this path (``test_rag_indexing_pipeline.py::TestIndexStatusErrorDisclosure``)- are strict-xfail credential-disclosure tests that read the reason text.+ No branch test asserts the ABSENCE of those keys — the branch tests on+ this path (``test_rag_indexing_pipeline.py::TestIndexStatusErrorDisclosure``)+ check only the client-visible reason and error text."""def _run(self, reconciliation_return=None, reconciliation_raises=None):@@ -665,7 +665,10 @@ class TestReconciliationSkippedOmitsDurableCounts:final = self._run(reconciliation_return={"reconciliation_skipped": True,- "reconciliation_reason": "live vector store reported zero ids",+ "reconciliation_reason": (+ "live vector store reported zero ids while DocumentChunk "+ "rows exist; refusing to clear indexed flags"+ ),"indexed_documents": 0,"indexed_chunks": 0,"live_vectors": 0,diff --git a/tests/research_library/routes/test_rag_routes_coverage_main_port.py b/tests/research_library/routes/test_rag_routes_coverage_main_port.pyindex 56f9840b40..c4a055ba29 100644--- a/tests/research_library/routes/test_rag_routes_coverage_main_port.py+++ b/tests/research_library/routes/test_rag_routes_coverage_main_port.py@@ -1155,13 +1155,28 @@ class TestIndexDocument:assert data["success"] is True- def test_error_result(self):+ @pytest.mark.parametrize(+ ("service_error", "client_error"),+ [+ pytest.param(+ "No text",+ "Indexing failed. Check the server logs for details.",+ id="arbitrary-service-text-is-withheld",+ ),+ pytest.param(+ "Document deleted during indexing",+ "Document deleted during indexing",+ id="known-document-guidance-passes",+ ),+ ],+ )+ def test_error_result(self, service_error, client_error):"""A service-level ``status: error`` must surface as a 400, not a 200carrying a failure the UI would render as success."""from local_deep_research.web.routers.rag import index_documentmock_rag = _ctx_rag_service(- index_document={"status": "error", "error": "No text"}+ index_document={"status": "error", "error": service_error})with (@@ -1179,7 +1194,10 @@ class TestIndexDocument:)assert resp.status_code == 400- assert _body(resp)["error"] == "No text"+ # _client_indexing_error passes only exact known document guidance,+ # such as a document deleted while it was being indexed. Any other+ # service text ("No text") gets the generic client message.+ assert _body(resp)["error"] == client_errordef test_with_collection_id(self):from local_deep_research.web.routers.rag import index_document@@ -1325,7 +1343,11 @@ class TestRemoveDocument:)assert resp.status_code == 400- assert _body(resp)["error"] == "not found"+ # Same sanitization boundary: "not found" from the service seam+ # is replaced with the generic client message.+ assert _body(resp)["error"] == (+ "Indexing failed. Check the server logs for details."+ )# ---------------------------------------------------------------------------diff --git a/tests/research_library/routes/test_rag_routes_gaps_main_port.py b/tests/research_library/routes/test_rag_routes_gaps_main_port.pyindex 40b34f747c..75b7a7fddb 100644--- a/tests/research_library/routes/test_rag_routes_gaps_main_port.py+++ b/tests/research_library/routes/test_rag_routes_gaps_main_port.py@@ -203,17 +203,32 @@ def _user_db_session(db_session):(``database/session_context.py``) so rollback assertions matchproduction rather than a stub that would pass with the check removed."""+ from local_deep_research.database.session_context import (+ get_user_db_session as real_get_user_db_session,+ )++ active = True@contextmanagerdef _fake(*_a, **_kw):+ # A service imported while the source module is patched can retain+ # this function after teardown. Later requests must reach their own+ # database, not the prior test's mock session.+ if not active:+ with real_get_user_db_session(*_a, **_kw) as session:+ yield session+ returntry:yield db_sessionexcept Exception:db_session.rollback()raise- with patch(f"{_DB_CTX}.get_user_db_session", side_effect=_fake):- yield db_session+ try:+ with patch(f"{_DB_CTX}.get_user_db_session", _fake):+ yield db_session+ finally:+ active = False@contextmanager@@ -864,8 +879,26 @@ class TestIndexCollection:assert complete["results"]["successful"] == 1assert complete["results"]["failed"] == 0- def test_indexing_with_failed_document(self):- """Failed document is counted and error event emitted."""+ @pytest.mark.parametrize(+ ("service_error", "client_error"),+ [+ pytest.param(+ "Parse failed at /srv/private/index.db "+ "[SQL: SELECT * FROM secrets]",+ "Indexing failed. Check the server logs for details.",+ id="arbitrary-service-text-is-withheld",+ ),+ pytest.param(+ "Document deleted during indexing",+ "Document deleted during indexing",+ id="known-document-guidance-passes",+ ),+ ],+ )+ def test_indexing_with_failed_document(self, service_error, client_error):+ """Failed document is counted and error event emitted. Its error+ reaches the complete event as the client message: known document+ guidance passes, and any other service text is withheld."""mock_coll, docs = self._one_document("bad.pdf")db_session = _make_db_session(query_side_effect=_collection_then_docs(mock_coll, docs)@@ -883,7 +916,7 @@ class TestIndexCollection:{"doc_id": doc_info[0][0],"title": doc_info[0][1],- "error": "RuntimeError: Parse error",+ "error": service_error,}],"results": {@@ -906,6 +939,7 @@ class TestIndexCollection:assert complete["results"]["failed"] == 1assert len(complete["results"]["errors"]) == 1assert "bad.pdf" in complete["results"]["errors"][0]["filename"]+ assert complete["results"]["errors"][0]["error"] == client_errordef test_skipped_document(self):"""Document returning 'skipped' status is counted correctly."""@@ -1129,6 +1163,95 @@ class TestIndexAll:mock_reset.assert_not_called()+ @pytest.mark.parametrize(+ ("service_error", "client_error"),+ [+ pytest.param(+ "Parse failed at /srv/private/index.db "+ "[SQL: SELECT * FROM secrets]",+ "Indexing failed. Check the server logs for details.",+ id="arbitrary-service-text-is-withheld",+ ),+ pytest.param(+ "Document deleted during indexing",+ "Document deleted during indexing",+ id="known-document-guidance-passes",+ ),+ ],+ )+ def test_index_all_failed_document_error_is_sanitized(+ self, service_error, client_error+ ):+ """The index-all SSE boundary sanitizes per-doc errors (B1 fix).++ index_documents_parallel is a service seam: an extension can+ supply arbitrary error text. The complete event must carry the+ generic client message, never the raw detail. Known document+ guidance, such as a document deleted while it was being indexed,+ passes unchanged."""+ mock_coll = Mock()+ mock_coll.id = "coll-1"+ mock_coll.name = "Test Collection"+ mock_coll.embedding_model = None+ document = SimpleNamespace(+ id="doc-1", filename="bad.pdf", title="bad.pdf"+ )++ db_session = _make_db_session(+ query_side_effect=lambda *a: _build_mock_query(+ first_result=mock_coll+ )+ )+ mock_rag = Mock()+ mock_rag.index_documents_parallel.side_effect = (+ _parallel_progress_side_effect(+ "error",+ lambda doc_info: {+ "successful": 0,+ "skipped": 0,+ "failed": 1,+ "errors": [+ {+ "doc_id": "doc-1",+ "title": "bad.pdf",+ "error": service_error,+ }+ ],+ "results": {+ "doc-1": {"status": "error", "error": service_error}+ },+ "cancelled": False,+ "total": len(doc_info),+ },+ )+ )++ with (+ patch(f"{MODULE}._store_collection_embedding_metadata"),+ patch(+ f"{MODULE}._query_documents_to_index",+ return_value=[(Mock(), document)],+ ),+ ):+ events, _ = _drive_index_all(+ db_session=db_session,+ rag_service=mock_rag,+ query_params={"collection_id": "coll-1"},+ )++ mock_rag.index_documents_parallel.assert_called_once()+ assert mock_rag.index_documents_parallel.call_args.args == (+ [("doc-1", "bad.pdf")],+ "coll-1",+ )+ assert "/srv/private" not in json.dumps(events)+ assert "SELECT * FROM secrets" not in json.dumps(events)+ complete = [e for e in events if e["type"] == "complete"][0]+ assert complete["results"]["failed"] == 1+ assert complete["results"]["errors"] == [+ {"doc_id": "doc-1", "title": "bad.pdf", "error": client_error}+ ]+def test_index_all_collection_not_found_yields_error_and_short_circuits(self,):diff --git a/tests/security/test_library_rag_security_fastapi.py b/tests/security/test_library_rag_security_fastapi.pyindex d37faa83b0..2222be2c63 100644--- a/tests/security/test_library_rag_security_fastapi.py+++ b/tests/security/test_library_rag_security_fastapi.py@@ -48,6 +48,13 @@ output is``_background_index_worker`` and read back by the index-status / SSEendpoints, i.e. it reaches the browser. Unscrubbed per-document errors area disclosure; an unbounded list is a response-size problem.+ ``TestIndexingErrorAllowlistMatchesService`` checks the allowlist behind it+ against the error text ``LibraryRAGService`` returns, so a fixed message+ the service adds cannot be withheld unnoticed and an allowlisted one+ cannot outlive the code that returns it.+ ``TestIndexingTaskErrorPassThroughs`` does the same for the stored task+ error the index-status endpoint serves, against the text+ ``web/routers/rag.py`` writes to indexing task rows.COVERAGE AREA 5 -- ``research_library/utils/is_downloadable_domain``. It had nodirect coverage at the review snapshot; the tests below and@@ -872,14 +879,16 @@ class TestTestEmbeddingEndpointDoesNotLeak:class TestSanitizedIndexingErrors:- """``rag.py::_sanitized_indexing_errors`` is the single scrubbing point+ """``rag.py::_sanitized_indexing_errors`` is the client-message gatefor the per-document error list that both terminal paths of``_background_index_worker`` write into ``TaskMetadata.result_metadata``,which the index-status / SSE endpoints hand to the browser. The tests belowpin its redaction and length-bound contracts."""- def test_error_text_is_credential_scrubbed(self):+ def test_exception_detail_is_replaced_without_losing_document_identity(+ self,+ ):from local_deep_research.web.routers.rag import (_sanitized_indexing_errors,)@@ -898,14 +907,37 @@ class TestSanitizedIndexingErrors:assert len(out) == 1entry = out[0]- # Positive: the identifying fields and the useful part of the message- # are preserved (an empty list would satisfy the negative alone).+ # The UI still identifies the failed document.assert entry["doc_id"] == "doc-1"assert entry["title"] == "Quarterly report"- assert "embedding call rejected key" in entry["error"]- # Negative: the credential is gone.+ assert (+ entry["error"]+ == "Indexing failed. Check the server logs for details."+ )assert _FAKE_API_KEY not in entry["error"]- assert "[REDACTED_KEY]" in entry["error"]++ def test_path_sql_and_provider_detail_are_not_returned(self):+ from local_deep_research.web.routers.rag import (+ _sanitized_indexing_errors,+ )++ out = _sanitized_indexing_errors(+ {+ "errors": [+ {+ "doc_id": "doc-1",+ "error": (+ "DB at /srv/private/user.db failed [SQL: SELECT * "+ "FROM secrets] via http://internal.example:11434"+ ),+ }+ ]+ }+ )+ assert (+ out[0]["error"]+ == "Indexing failed. Check the server logs for details."+ )def test_output_is_bounded_to_fifty_entries_by_default(self):"""A failed index of a large collection can produce one error per@@ -932,7 +964,11 @@ class TestSanitizedIndexingErrors:# It is the FIRST 50, in order -- not a random or trailing slice.assert out[0]["doc_id"] == "doc-0"assert out[-1]["doc_id"] == "doc-49"- assert all(entry["error"] == f"boom {i}" for i, entry in enumerate(out))+ assert all(+ entry["error"]+ == "Indexing failed. Check the server logs for details."+ for entry in out+ )def test_limit_is_overridable_and_still_applied(self):from local_deep_research.web.routers.rag import (@@ -994,10 +1030,8 @@ class TestSanitizedIndexingErrors:assert _sanitized_indexing_errors({}) == []assert _sanitized_indexing_errors({"errors": []}) == []- def test_non_string_error_values_are_coerced_before_scrubbing(self):- """``str(...)`` around the value: an exception object (the common- case at the call sites) must be rendered and scrubbed, not crash.- """+ def test_non_string_error_values_use_the_generic_message(self):+ """An exception object must never be rendered for the client."""from local_deep_research.web.routers.rag import (_sanitized_indexing_errors,)@@ -1013,9 +1047,499 @@ class TestSanitizedIndexingErrors:})- assert "bad key" in out[0]["error"]+ assert (+ out[0]["error"]+ == "Indexing failed. Check the server logs for details."+ )assert _FAKE_API_KEY not in out[0]["error"]+ def test_only_known_service_messages_are_preserved(self):+ from local_deep_research.web.routers.rag import (+ _sanitized_indexing_errors,+ )++ out = _sanitized_indexing_errors(+ {+ "errors": [+ {"error": "Document deleted during indexing"},+ {"error": "Document deleted during indexing /srv/private"},+ ]+ }+ )+ assert out[0]["error"] == "Document deleted during indexing"+ assert (+ out[1]["error"]+ == "Indexing failed. Check the server logs for details."+ )+++#: Text ``LibraryRAGService`` returns as a result's ``"error"`` that clients+#: must not see verbatim: ``_client_indexing_error`` gives them the generic+#: message instead. ``{}`` marks each f-string placeholder.+_WITHHELD_SERVICE_ERRORS = frozenset(+ {+ "Operation failed: {}",+ "Indexing failed: {}",+ "Worker crashed: {}",+ "Indexing must run outside an existing database session",+ }+)++#: ``"error"`` values that copy the error of a result the service's own+#: methods returned, as ``(enclosing function, source)``. The text they copy+#: is classified by ``_SAFE_INDEXING_ERRORS`` and the set above.+_FORWARDED_SERVICE_ERRORS = frozenset(+ {+ ("index_all_documents", "result.get('error')"),+ ("index_documents_parallel", "result.get('error')"),+ ("index_documents_parallel", "scope_error['error']"),+ }+)++#: Reconciliation reasons the service returns that clients must not see+#: verbatim. The status endpoint shows only ``_EMPTY_VECTOR_STORE_REASON``.+_WITHHELD_SERVICE_REASONS = frozenset()+++def _library_rag_service_values(key):+ """Every value the ``LibraryRAGService`` module builds under ``key``.++ A value counts when it is a dict-literal entry, a ``dict(...)`` or+ ``.update(...)`` keyword, or an ``x[key] = ...`` assignment; a+ conditional expression counts as both of its branches. Returns+ ``(texts, others)``: ``texts`` holds each string and f-string, with+ ``{}`` for each placeholder, and ``others`` holds every other value as+ ``(enclosing function, source)``.+ """+ import ast+ import inspect++ from local_deep_research.research_library.services import (+ library_rag_service,+ )++ texts, others = set(), set()++ def record(value, function):+ if isinstance(value, ast.IfExp):+ record(value.body, function)+ record(value.orelse, function)+ elif isinstance(value, ast.Constant) and isinstance(value.value, str):+ texts.add(value.value)+ elif isinstance(value, ast.JoinedStr):+ texts.add(+ "".join(+ part.value if isinstance(part, ast.Constant) else "{}"+ for part in value.values+ )+ )+ else:+ others.add((function, ast.unparse(value)))++ def visit(node, function):+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):+ function = node.name+ if isinstance(node, ast.Dict):+ for dict_key, value in zip(node.keys, node.values):+ if isinstance(dict_key, ast.Constant) and dict_key.value == key:+ record(value, function)+ elif isinstance(node, ast.Call) and (+ getattr(node.func, "id", None) == "dict"+ or getattr(node.func, "attr", None) == "update"+ ):+ for keyword in node.keywords:+ if keyword.arg == key:+ record(keyword.value, function)+ elif (+ isinstance(node, (ast.Assign, ast.AnnAssign))+ and node.value is not None+ ):+ targets = (+ node.targets if isinstance(node, ast.Assign) else [node.target]+ )+ for target in targets:+ if (+ isinstance(target, ast.Subscript)+ and isinstance(target.slice, ast.Constant)+ and target.slice.value == key+ ):+ record(node.value, function)+ for child in ast.iter_child_nodes(node):+ visit(child, function)++ visit(ast.parse(inspect.getsource(library_rag_service)), "<module>")+ return texts, others+++class TestIndexingErrorAllowlistMatchesService:+ """``_SAFE_INDEXING_ERRORS`` must match the error text that+ ``LibraryRAGService`` returns.++ The RAG document index and remove routes, the note index route, both+ indexing SSE streams and the index-status endpoint pass each+ per-document ``"error"`` through ``_client_indexing_error``. An exact+ match in ``_SAFE_INDEXING_ERRORS`` reaches the client, and any other text+ becomes the generic message. Every such value comes from+ ``LibraryRAGService``, so these tests collect the values its module+ builds under ``"error"``, in the forms ``_library_rag_service_values``+ lists. Each string or f-string must be allowlisted or+ listed in ``_WITHHELD_SERVICE_ERRORS``, and each allowlisted string must+ still be returned. Reconciliation ``"reason"`` values get the same check+ against ``_EMPTY_VECTOR_STORE_REASON``, the one reason the status+ endpoint shows. New or reworded text fails here until someone decides+ whether clients may see it.+ """++ def test_every_error_the_service_returns_is_classified(self):+ from local_deep_research.web.routers.rag import _SAFE_INDEXING_ERRORS++ texts, others = _library_rag_service_values("error")++ unclassified = sorted(+ texts - _SAFE_INDEXING_ERRORS - _WITHHELD_SERVICE_ERRORS+ )+ assert unclassified == [], (+ f"LibraryRAGService returns unclassified error text {unclassified}."+ " Add fixed guidance a user can act on to _SAFE_INDEXING_ERRORS"+ " in web/routers/rag.py, and anything else to"+ " _WITHHELD_SERVICE_ERRORS in this file."+ )+ assert others == _FORWARDED_SERVICE_ERRORS, (+ "LibraryRAGService builds an error value that is neither text nor"+ f" a known copy of another result's error: {sorted(others)}"+ )++ def test_every_classified_error_is_still_returned(self):+ """An allowlisted string that nothing returns hides drift: a test+ that pins it passes while the service's real message is withheld."""+ from local_deep_research.web.routers.rag import _SAFE_INDEXING_ERRORS++ texts, _ = _library_rag_service_values("error")++ assert not _SAFE_INDEXING_ERRORS & _WITHHELD_SERVICE_ERRORS+ assert sorted(_SAFE_INDEXING_ERRORS - texts) == [], (+ "allowlisted error text that LibraryRAGService no longer returns"+ )+ assert sorted(_WITHHELD_SERVICE_ERRORS - texts) == [], (+ "withheld error text that LibraryRAGService no longer returns"+ )++ def test_every_reconciliation_reason_is_classified(self):+ from local_deep_research.web.routers.rag import (+ _EMPTY_VECTOR_STORE_REASON,+ )++ texts, others = _library_rag_service_values("reason")++ unclassified = sorted(+ texts - {_EMPTY_VECTOR_STORE_REASON} - _WITHHELD_SERVICE_REASONS+ )+ assert unclassified == [], (+ f"LibraryRAGService returns unclassified reason {unclassified}."+ " Make it _EMPTY_VECTOR_STORE_REASON in web/routers/rag.py if"+ " clients should see it, or add it to _WITHHELD_SERVICE_REASONS"+ " in this file."+ )+ assert others == set()+ assert _EMPTY_VECTOR_STORE_REASON in texts+ assert _WITHHELD_SERVICE_REASONS <= texts++ def test_clients_receive_each_text_as_classified(self):+ from local_deep_research.web.routers.rag import (+ _EMPTY_VECTOR_STORE_REASON,+ _SAFE_INDEXING_ERRORS,+ _client_indexing_error,+ _client_reconciliation_reason,+ )++ for text in _SAFE_INDEXING_ERRORS:+ assert _client_indexing_error(text) == text+ for template in _WITHHELD_SERVICE_ERRORS:+ assert _client_indexing_error(+ template.replace("{}", "OperationalError")+ ) == ("Indexing failed. Check the server logs for details.")+ assert (+ _client_reconciliation_reason(_EMPTY_VECTOR_STORE_REASON)+ == _EMPTY_VECTOR_STORE_REASON+ )+++#: Fixed text that ``web/routers/rag.py`` writes to an indexing task's+#: ``error_message`` without a failure record, and that the index-status+#: endpoint must not serve as stored.+_WITHHELD_TASK_ERRORS = frozenset()+++def _indexing_task_error_writes():+ """The fixed texts ``web/routers/rag.py`` writes to an indexing task's+ ``error_message``, as ``(with_failure, without_failure)``.++ That module creates every indexing ``TaskMetadata`` row and is the only+ one that writes them. A write is an ``error_message=`` keyword of any+ call, such as ``_update_task_status(...)`` or ``TaskMetadata(...)``, or+ an ``x.error_message = ...`` assignment. Its text counts when it is a+ string, an f-string (``{}`` for each placeholder) or a module-level+ string constant; a conditional expression counts as both of its+ branches. A write whose call also passes a ``failure`` record goes in+ ``with_failure``: for a record it recognises, the status endpoint+ rebuilds the text instead of matching it.+ """+ import ast+ import inspect++ from local_deep_research.web.routers import rag++ namespace = vars(rag)+ with_failure, without_failure = set(), set()++ def texts(value):+ if isinstance(value, ast.IfExp):+ return texts(value.body) | texts(value.orelse)+ if isinstance(value, ast.Constant) and isinstance(value.value, str):+ return {value.value}+ if isinstance(value, ast.JoinedStr):+ return {+ "".join(+ part.value if isinstance(part, ast.Constant) else "{}"+ for part in value.values+ )+ }+ if isinstance(value, ast.Name) and isinstance(+ namespace.get(value.id), str+ ):+ return {namespace[value.id]}+ return set()++ for node in ast.walk(ast.parse(inspect.getsource(rag))):+ if isinstance(node, ast.Call):+ keywords = {keyword.arg: keyword.value for keyword in node.keywords}+ if "error_message" in keywords:+ failure = keywords.get("failure")+ records = failure is not None and not (+ isinstance(failure, ast.Constant) and failure.value is None+ )+ (with_failure if records else without_failure).update(+ texts(keywords["error_message"])+ )+ elif isinstance(node, ast.Assign) and any(+ isinstance(target, ast.Attribute) and target.attr == "error_message"+ for target in node.targets+ ):+ without_failure.update(texts(node.value))+ return with_failure, without_failure+++def _stored_message_pass_throughs(source, namespace):+ """The stored texts the task-error function in ``source`` serves+ unchanged.++ The function's first parameter is the stored ``error_message``. Its+ pass-throughs are the texts it compares that parameter with exactly:+ ``==`` with a string, or ``in`` with a collection of strings, each+ written out or a name bound in ``namespace``. Returns+ ``(texts, others)``. ``others`` holds the source of every other use of+ the parameter, so a new shape fails the guard instead of passing+ unseen. Three uses are allowed: an ``is None`` or ``is not None`` check,+ an ``isinstance`` check, and a ``return`` of the parameter directly in+ the body of an ``if`` whose test is such a comparison, alone or in an+ ``and``.+ """+ import ast+ import textwrap++ function = ast.parse(textwrap.dedent(source)).body[0]+ param = function.args.args[0].arg+ parents = {+ child: node+ for node in ast.walk(function)+ for child in ast.iter_child_nodes(node)+ }++ def is_param(node):+ return isinstance(node, ast.Name) and node.id == param++ def text_of(node):+ if isinstance(node, ast.Constant) and isinstance(node.value, str):+ return node.value+ if isinstance(node, ast.Name) and isinstance(+ namespace.get(node.id), str+ ):+ return namespace[node.id]+ return None++ def texts_in(node):+ if isinstance(node, (ast.Set, ast.Tuple, ast.List)):+ items = [text_of(element) for element in node.elts]+ elif isinstance(node, ast.Name) and isinstance(+ namespace.get(node.id), (frozenset, set, tuple, list)+ ):+ items = list(namespace[node.id])+ else:+ return None+ if all(isinstance(item, str) for item in items):+ return set(items)+ return None++ def exact_match(node):+ if not (isinstance(node, ast.Compare) and len(node.ops) == 1):+ return None+ left, op, right = node.left, node.ops[0], node.comparators[0]+ if isinstance(op, ast.Eq):+ if is_param(left) and text_of(right) is not None:+ return {text_of(right)}+ if is_param(right) and text_of(left) is not None:+ return {text_of(left)}+ if isinstance(op, ast.In) and is_param(left):+ return texts_in(right)+ return None++ def guarded(statement):+ branch = parents.get(statement)+ if not (isinstance(branch, ast.If) and statement in branch.body):+ return False+ test = branch.test+ if isinstance(test, ast.BoolOp) and isinstance(test.op, ast.And):+ tests = test.values+ else:+ tests = [test]+ return any(exact_match(each) is not None for each in tests)++ def allowed(use, parent):+ if isinstance(parent, ast.Compare):+ return (+ parent.left is use+ and len(parent.ops) == 1+ and isinstance(parent.ops[0], (ast.Is, ast.IsNot))+ and isinstance(parent.comparators[0], ast.Constant)+ and parent.comparators[0].value is None+ )+ if isinstance(parent, ast.Call):+ return (+ getattr(parent.func, "id", None) == "isinstance"+ and bool(parent.args)+ and parent.args[0] is use+ )+ return isinstance(parent, ast.Return) and guarded(parent)++ texts, others = set(), set()+ for node in ast.walk(function):+ if not (is_param(node) and isinstance(node.ctx, ast.Load)):+ continue+ parent = parents[node]+ matched = exact_match(parent)+ if matched is not None:+ texts |= matched+ elif not allowed(node, parent):+ others.add(ast.unparse(parent))+ return texts, others+++def _task_error_pass_throughs():+ """``_stored_message_pass_throughs`` for ``_client_indexing_task_error``."""+ import inspect++ from local_deep_research.web.routers import rag++ return _stored_message_pass_throughs(+ inspect.getsource(rag._client_indexing_task_error), vars(rag)+ )+++class TestIndexingTaskErrorPassThroughs:+ """Stored task errors reach clients unchanged only for text the indexing+ code writes.++ ``get_index_status`` passes each task's stored ``error_message`` through+ ``_client_indexing_task_error``. That function rebuilds the client text+ from the failure record or the result counts, and serves the stored text+ itself only on an exact match, a pass-through. These tests check the+ pass-throughs against the text ``web/routers/rag.py`` writes to indexing+ task rows. A pass-through that nothing writes is dead: a test that seeds+ its text into a row passes while no code path produces it. Fixed text+ written without a failure record must be passed through or listed in+ ``_WITHHELD_TASK_ERRORS``, so it cannot be withheld unnoticed.+ """++ def test_every_pass_through_is_written_to_a_task_row(self):+ from local_deep_research.web.routers.rag import (+ _client_indexing_task_error,+ )++ with_failure, without_failure = _indexing_task_error_writes()+ pass_throughs, _ = _task_error_pass_throughs()++ dead = sorted(pass_throughs - with_failure - without_failure)+ assert dead == [], (+ f"_client_indexing_task_error serves stored text {dead} unchanged,"+ " but web/routers/rag.py never writes it to an indexing task row."+ " Remove the pass-through, or add the code that writes the text."+ )+ for text in pass_throughs:+ assert _client_indexing_task_error(text, None) == text++ def test_every_fixed_text_written_without_a_record_is_classified(self):+ from local_deep_research.web.routers.rag import (+ _COLLECTION_NOT_FOUND,+ _client_indexing_task_error,+ )++ with_failure, without_failure = _indexing_task_error_writes()+ pass_throughs, _ = _task_error_pass_throughs()++ unclassified = sorted(+ without_failure - pass_throughs - _WITHHELD_TASK_ERRORS+ )+ assert unclassified == [], (+ f"web/routers/rag.py writes task error text {unclassified} with no"+ " failure record, and the index-status endpoint does not serve it."+ " Add an exact-match pass-through to _client_indexing_task_error"+ " if clients should see it, or add it to _WITHHELD_TASK_ERRORS in"+ " this file."+ )+ assert not pass_throughs & _WITHHELD_TASK_ERRORS+ assert sorted(_WITHHELD_TASK_ERRORS - without_failure) == [], (+ "withheld task error text that web/routers/rag.py no longer writes"+ )+ for text in _WITHHELD_TASK_ERRORS:+ assert _client_indexing_task_error(text, None) == (+ "Indexing failed. Check the server logs for details."+ )+ # Control: the scan sees the worker's writes.+ assert _COLLECTION_NOT_FOUND in with_failure++ def test_stored_text_is_served_only_on_an_exact_match(self):+ _, others = _task_error_pass_throughs()++ assert sorted(others) == [], (+ "_client_indexing_task_error uses the stored message in a way"+ f" these checks do not classify: {sorted(others)}"+ )++ def test_the_scan_recognises_each_pass_through_shape(self):+ """Control for the checks above, which find no pass-through in the+ current code."""+ source = """+ def task_error(message, result, failure=None):+ if message is None:+ return None+ if message == "Literal text":+ return message+ if isinstance(message, str) and message in NAMED:+ return message+ if message in {"Set text"}:+ return message+ if message.startswith("Prefix"):+ return message+ return "generic"+ """++ texts, others = _stored_message_pass_throughs(+ source, {"NAMED": frozenset({"Named text"})}+ )++ assert texts == {"Literal text", "Named text", "Set text"}+ assert others == {"message.startswith", "return message"}+# ===========================================================================# COVERAGE AREA 5 -- is_downloadable_domain URL allowlistdiff --git a/tests/web/routers/test_rag_indexing_admission.py b/tests/web/routers/test_rag_indexing_admission.pyindex 6b2d9319ce..24cff8f44d 100644--- a/tests/web/routers/test_rag_indexing_admission.py+++ b/tests/web/routers/test_rag_indexing_admission.py@@ -228,6 +228,32 @@ def test_failed_thread_start_releases_admission_and_fails_task(task = session.query(TaskMetadata).one()assert task.status == "failed"assert task.completed_at is not None+ assert (+ task.error_message == "Failed to start indexing. Please try again."+ )+ task_id = task.task_id++ status = rag.get_index_status(+ SimpleNamespace(session={}), "collection", "user"+ )+ assert status["status"] == "failed"+ assert (+ status["error_message"] == "Failed to start indexing. Please try again."+ )++ # Only the exact fixed text is public. An older row containing extra+ # exception detail must not pass merely because it starts the same way.+ with admission.sessions.begin() as session:+ task = session.query(TaskMetadata).filter_by(task_id=task_id).one()+ task.error_message += " [SQL: SELECT private] at /srv/private.db"+ status = rag.get_index_status(+ SimpleNamespace(session={}), "collection", "user"+ )+ assert status["error_message"] == (+ "Indexing failed. Check the server logs for details."+ )+ assert "[SQL:" not in str(status)+ assert "/srv/private.db" not in str(status)def test_racing_stream_and_background_request_admit_exactly_one(admission):diff --git a/tests/web/routers/test_rag_indexing_pipeline.py b/tests/web/routers/test_rag_indexing_pipeline.pyindex d296a9a086..b07d226505 100644--- a/tests/web/routers/test_rag_indexing_pipeline.py+++ b/tests/web/routers/test_rag_indexing_pipeline.py@@ -12,9 +12,8 @@ mirroring the real-components harness in``tests/research_library/services/test_multi_model_switch_revert.py``.No embedding model or LLM is ever loaded or called.-Four claims are examined. Each defect is recorded as a *strict xfail*-carrying its mechanism, and each has an adjacent PASSING positive control-so the xfail cannot be an artifact of the harness:+Four claims are examined. Open defects use *strict xfail* markers, with+passing controls beside them so the harness cannot mask a failure:1. **force-reindex is destroy-then-rebuild, not non-destructive.**``_reset_collection_for_reindex`` deletes the chunk rows, deletes the@@ -34,12 +33,12 @@ so the xfail cannot be an artifact of the harness:is precisely the row both engines look up, so every later search isempty too.-3. **Embedding-backend failure detail reaches the client through the- background indexing status endpoint.** ``/api/rag/test-embedding``- was hardened to default-deny (CWE-209), but ``_background_index_worker``- still passes raw ``str(exc)`` through ``sanitize_error_message``- (credential *shapes* only) into ``TaskMetadata.error_message``, which- ``get_index_status`` returns verbatim.+3. **Embedding-backend failure detail stays out of the background indexing+ status endpoint.** ``_background_index_worker`` stores fixed messages in+ ``TaskMetadata`` instead of scrubbed exception text (CWE-209). The status+ endpoint rebuilds each message from typed data, so the worker's fixed+ guidance (the remote-drop retry advice, "Collection not found") still+ reaches the client unchanged.4. **Concurrent collection indexing is mutually excluded.** The sharedadmission gate checks both streaming workers and background tasks before@@ -66,7 +65,7 @@ import astimport hashlibimport importlibimport uuid-from contextlib import contextmanager+from contextlib import ExitStack, contextmanagerfrom datetime import UTC, datetimefrom pathlib import Pathfrom types import SimpleNamespace@@ -148,6 +147,10 @@ class _FakeEmbeddings(Embeddings):def world(tmp_path, monkeypatch, mocker):"""One in-memory DB shared by every ``get_user_db_session`` call site,and a real on-disk ``cache/rag_indices`` tree under ``tmp_path``."""+ from local_deep_research.database.session_context import (+ get_user_db_session as real_get_user_db_session,+ )+monkeypatch.setenv("LDR_DATA_DIR", str(tmp_path))engine = create_engine("sqlite:///:memory:")@@ -167,8 +170,15 @@ def world(tmp_path, monkeypatch, mocker):session = sessionmaker(bind=engine)()@contextmanager- def _shared_session(*_a, **_k):- yield session+ def _shared_session(username, *args, **kwargs):+ if username == USERNAME:+ yield session+ else:+ # A module imported while this patch is active may retain the+ # wrapper after teardown. Other users must still reach their+ # actual DB rather than this fixture's closed in-memory session.+ with real_get_user_db_session(username, *args, **kwargs) as db:+ yield dbmocker.patch(f"{_RAG_SVC_MOD}.get_user_db_session", _shared_session)mocker.patch(f"{_FACADE_MOD}.get_user_db_session", _shared_session)@@ -723,9 +733,7 @@ class TestChunkingSettingsAtQueryTime:#: A backend failure carrying a server path and the provider endpoint.-#: Neither is a credential *shape*, so ``sanitize_error_message`` -- which-#: matches Bearer tokens, ``sk-``/``ghp_``/``AKIA`` prefixes, URL-embedded-#: credentials and credential query params -- leaves both intact.+#: Neither is a credential shape, so credential scrubbing alone leaves both._LEAKY_BACKEND_ERROR = OSError("cannot load embedding model from ""/srv/ldr-data/models/nomic-embed-text/pytorch_model.bin "@@ -738,19 +746,154 @@ _LEAKY_RECONCILE_ERROR = RuntimeError("[parameters: ('/srv/ldr-data/raguser.db',)]")+#: Served for any stored task error the status endpoint cannot rebuild.+_GENERIC_TASK_ERROR = "Indexing failed. Check the server logs for details."++#: What the worker stores on each terminal failure path. Every one is built+#: only from fixed phrases and counts, so the status endpoint must serve it+#: unchanged.+_WORKER_FAILURE_MESSAGES = {+ "collection-not-found": "Collection not found",+ "remote-drop-retryable-partial": (+ "Vector store unreachable — indexing was aborted. Retry once the "+ "vector store is available. 2 vector-store collection(s) had already "+ "been removed before the failure; their index records were kept, so "+ "search over them returns nothing until this is retried successfully "+ "or the collection is force re-indexed."+ ),+ "remote-drop-permanent": (+ "The vector store could not drop the collection — indexing was "+ "aborted. Retrying will not help until the vector store configuration "+ "or permissions are fixed; see the server log."+ ),+ "reconciliation-error": (+ "Reconciliation skipped: Could not verify the vector store. Check the "+ "server logs for details. Indexed flags preserved; verify the FAISS "+ "index before retrying."+ ),+ "reconciliation-refused": (+ "Reconciliation skipped: live vector store reported zero ids while "+ "DocumentChunk rows exist; refusing to clear indexed flags. Indexed "+ "flags preserved; verify the FAISS index before retrying."+ ),+ "reconciled-failures": (+ "1 document(s) failed to index. The collection was reconciled to the "+ "durable vector store."+ ),+ "unexpected-exception": _GENERIC_TASK_ERROR,+}++#: Must never reach the client: the leaky errors' details, and the provider+#: and index-hash prefix in a failed drop's own message.+_CLIENT_SECRETS = (+ "/srv/ldr-data",+ "embed-internal.corp.example",+ "[SQL:",+ "qdrant",+ "0123456789ab",+)+++def _remote_drop_error(retryable, dropped):+ """A failed drop whose own text names the provider and an index hash."""+ from local_deep_research.research_library.deletion.utils.cascade_helper import (+ RemoteVectorStoreDropError,+ )++ return RemoteVectorStoreDropError(+ "Failed to drop qdrant collection for index 0123456789ab...: "+ "ConnectionError. The index record was NOT deleted.",+ retryable=retryable,+ dropped=dropped,+ cause_type="ConnectionError",+ )+++def _fail_indexing(rag_mod, path, task_id, collection_id):+ """Drive ``_background_index_worker`` down one terminal failure path.++ For ``collection-not-found`` the caller passes the id of a collection+ that does not exist; every other path indexes the seeded collection.+ """+ svc = _make_service()+ svc.index_documents_parallel = MagicMock(+ return_value={+ "successful": 1,+ "skipped": 0,+ "failed": 0,+ "cancelled": False,+ "errors": [],+ }+ )+ svc.reconcile_collection_index = MagicMock(+ return_value={+ "indexed_documents": 1,+ "indexed_chunks": 4,+ "live_vectors": 4,+ "orphan_vectors": 0,+ }+ )+ force_reindex = False+ with ExitStack() as stack:+ get_service = stack.enter_context(+ patch.object(+ rag_mod, "_get_rag_service_for_thread", return_value=svc+ )+ )+ if path == "unexpected-exception":+ get_service.side_effect = _LEAKY_BACKEND_ERROR+ elif path.startswith("remote-drop"):+ force_reindex = True+ if path == "remote-drop-retryable-partial":+ error = _remote_drop_error(retryable=True, dropped=2)+ else:+ error = _remote_drop_error(retryable=False, dropped=0)+ # Exercise the typed failure boundary without depending on a+ # destructive reset helper that staged rebuilds no longer use.+ get_service.side_effect = error+ elif path == "reconciliation-error":+ svc.reconcile_collection_index.side_effect = _LEAKY_RECONCILE_ERROR+ elif path == "reconciliation-refused":+ svc.reconcile_collection_index.return_value = {+ "reconciliation_skipped": True,+ "reason": (+ "live vector store reported zero ids while DocumentChunk "+ "rows exist; refusing to clear indexed flags"+ ),+ "indexed_documents": 0,+ "indexed_chunks": 0,+ "live_vectors": 0,+ "orphan_vectors": 0,+ }+ elif path == "reconciled-failures":+ svc.index_documents_parallel.return_value = {+ "successful": 0,+ "skipped": 0,+ "failed": 1,+ "cancelled": False,+ "errors": [+ {+ "doc_id": "doc-1",+ "title": "report",+ "error": "Document has no text content",+ }+ ],+ }+ rag_mod._background_index_worker(+ task_id, collection_id, USERNAME, DB_PASSWORD, force_reindex, 2+ )+ get_service.assert_called_once_with(+ collection_id, USERNAME, DB_PASSWORD, use_defaults=force_reindex+ )+class TestIndexStatusErrorDisclosure:- """``/api/rag/test-embedding`` was hardened to default-deny, but the- background indexing path echoes ``str(exc)`` into- ``TaskMetadata.error_message``, which ``get_index_status`` returns to- the browser."""+ """Background indexing must match the embedding route's default deny."""def test_test_embedding_route_withholds_the_same_detail(self):"""Control: the hardened surface, given the SAME exception, returnsa fixed message carrying nothing from it -- not even the class- name. The difference from the xfails below is a real- inconsistency between two surfaces, not a property of the- exception text."""+ name. This proves the exception text itself contains unsafe detail."""from local_deep_research.web.routers.rag import (_format_test_embedding_error,)@@ -773,23 +916,6 @@ class TestIndexStatusErrorDisclosure:assert "/srv/ldr-data/models" not in messageassert "embed-internal.corp.example" not in message- @pytest.mark.xfail(- strict=True,- reason=(- "CWE-209. _background_index_worker's outer handler writes "- "error_message=sanitize_error_message(str(e)) and "- "get_index_status returns task.error_message verbatim. "- "sanitize_error_message scrubs credential SHAPES only -- not "- "server filesystem paths, not provider endpoints, not SQL "- "text -- which is exactly the reasoning _format_test_embedding_"- "error's docstring gives for its default-deny allowlist. The "- "exception here comes from _get_rag_service_for_thread, i.e. "- "the embedding-manager construction boundary that was hardened "- "on the other route. Fix by routing this sink through the same "- "module-allowlist formatter (or storing the class name only) "- "and keeping the full text in the server log. Tracked as #6273."- ),- )def test_backend_failure_detail_does_not_reach_the_task_error(self, world, seeded):@@ -812,6 +938,10 @@ class TestIndexStatusErrorDisclosure:task = session.query(TaskMetadata).filter_by(task_id=task_id).one()assert task.status == "failed"client_visible = task.error_message or ""+ assert (+ client_visible+ == "Indexing failed. Check the server logs for details."+ )assert "/srv/ldr-data/models" not in client_visible, ("a server filesystem path reached the client-visible indexing "f"task error: {client_visible!r}"@@ -821,19 +951,6 @@ class TestIndexStatusErrorDisclosure:f"indexing task error: {client_visible!r}")- @pytest.mark.xfail(- strict=True,- reason=(- "Second sink, same shape: the reconciliation handler builds "- "reconciliation_reason=sanitize_error_message(str(exc)) and the "- "reconciliation_skipped branch copies it into BOTH "- "error_message and result_metadata, both of which "- "get_index_status returns. A DBAPIError renders driver text "- "plus [SQL: ...] and [parameters: ...] (the exact example "- "_format_test_embedding_error cites when refusing to echo "- "sqlalchemy.exc.*). Fix as above. Tracked as #6273."- ),- )def test_reconciliation_failure_detail_does_not_reach_the_task(self, world, seeded):@@ -870,6 +987,7 @@ class TestIndexStatusErrorDisclosure:session.expire_all()task = session.query(TaskMetadata).filter_by(task_id=task_id).one()client_visible = f"{task.error_message or ''}{task.metadata_json or {}}"+ assert "Could not verify the vector store" in client_visibleassert "[SQL:" not in client_visible, ("SQL text reached the client-visible indexing task state: "f"{client_visible!r}"@@ -879,6 +997,358 @@ class TestIndexStatusErrorDisclosure:f"task state: {client_visible!r}")+ def test_nested_per_document_result_does_not_bypass_error_boundary(+ self, world, seeded+ ):+ from local_deep_research.web.routers import rag as rag_mod++ session = world+ collection_id, _ = seeded+ svc = _make_service()+ svc.index_documents_parallel = MagicMock(+ return_value={+ "successful": 0,+ "skipped": 0,+ "failed": 1,+ "cancelled": False,+ "total": 1,+ "errors": [+ {+ "doc_id": "doc-1",+ "title": "report",+ "error": _LEAKY_BACKEND_ERROR,+ }+ ],+ "results": {+ "doc-1": {+ "status": "error",+ "error": str(_LEAKY_BACKEND_ERROR),+ }+ },+ }+ )+ svc.reconcile_collection_index = MagicMock(+ return_value={+ "indexed_documents": 0,+ "indexed_chunks": 0,+ "live_vectors": 0,+ "orphan_vectors": 0,+ }+ )++ task_id = _seed_task(session, uuid.uuid4().hex, collection_id)+ with patch.object(+ rag_mod, "_get_rag_service_for_thread", return_value=svc+ ):+ rag_mod._background_index_worker(+ task_id, collection_id, USERNAME, DB_PASSWORD, False, 2+ )++ session.expire_all()+ task = session.query(TaskMetadata).filter_by(task_id=task_id).one()+ result = task.metadata_json["result"]+ assert result["failed"] == 1+ assert "results" not in result+ assert result["errors"] == [+ {+ "doc_id": "doc-1",+ "title": "report",+ "error": "Indexing failed. Check the server logs for details.",+ }+ ]+ assert "/srv/ldr-data/models" not in str(result)++ def test_status_filters_legacy_task_rows_written_before_the_fix(+ self, world, seeded+ ):+ from local_deep_research.web.routers import rag as rag_mod++ session = world+ collection_id, _ = seeded+ task_id = _seed_task(session, uuid.uuid4().hex, collection_id)+ task = session.query(TaskMetadata).filter_by(task_id=task_id).one()+ task.status = "failed"+ task.progress_message = "Failed: [SQL: SELECT * FROM secrets]"+ task.error_message = "Failed at /srv/ldr-data/private.db"+ task.metadata_json = {+ "collection_id": collection_id,+ "result": {+ "successful": 0,+ "failed": 1,+ "results": {+ "doc-1": {+ "error": "http://embed-internal.corp.example:11434"+ }+ },+ "errors": [+ {+ "doc_id": "doc-1",+ "title": "report",+ "error": "[SQL: SELECT 1]",+ }+ ],+ },+ }+ session.commit()++ status = rag_mod.get_index_status(+ SimpleNamespace(session={}), collection_id, USERNAME+ )++ assert status["status"] == "failed"+ assert status["progress_message"] == "Indexing failed"+ assert status["error_message"] == (+ "1 document(s) failed to index. Check the server logs for details."+ )+ assert status["result"]["errors"][0]["error"] == (+ "Indexing failed. Check the server logs for details."+ )+ assert "results" not in status["result"]+ for secret in ("[SQL:", "/srv/ldr-data", "embed-internal.corp.example"):+ assert secret not in str(status)++ def test_status_keeps_safe_counts_and_document_guidance(+ self, world, seeded+ ):+ from local_deep_research.web.routers import rag as rag_mod++ session = world+ collection_id, _ = seeded+ task_id = _seed_task(session, uuid.uuid4().hex, collection_id)+ task = session.query(TaskMetadata).filter_by(task_id=task_id).one()+ task.status = "completed"+ task.metadata_json = {+ "collection_id": collection_id,+ "result": {+ "successful": 2,+ "skipped": 0,+ "failed": 0,+ "durable_indexed_documents": 2,+ "durable_indexed_chunks": 12,+ "live_vectors": 12,+ "orphan_vectors": 0,+ "errors": [+ {+ "doc_id": "doc-1",+ "error": "Document deleted during indexing",+ }+ ],+ },+ }+ session.commit()++ status = rag_mod.get_index_status(+ SimpleNamespace(session={}), collection_id, USERNAME+ )++ assert status["status"] == "completed"+ assert status["result"]["durable_indexed_documents"] == 2+ assert status["result"]["durable_indexed_chunks"] == 12+ assert status["result"]["errors"][0]["error"] == (+ "Document deleted during indexing"+ )++ def test_worker_stores_and_status_serves_document_guidance(+ self, world, seeded+ ):+ """A document deleted while it was being indexed: the worker stores+ the service's fixed guidance, and the status endpoint serves it+ exactly."""+ from local_deep_research.web.routers import rag as rag_mod++ session = world+ collection_id, _ = seeded+ svc = _make_service()+ svc.index_documents_parallel = MagicMock(+ return_value={+ "successful": 0,+ "skipped": 0,+ "failed": 1,+ "cancelled": False,+ "total": 1,+ "errors": [+ {+ "doc_id": "doc-1",+ "title": "report",+ "error": "Document deleted during indexing",+ }+ ],+ }+ )+ svc.reconcile_collection_index = MagicMock(+ return_value={+ "indexed_documents": 0,+ "indexed_chunks": 0,+ "live_vectors": 0,+ "orphan_vectors": 0,+ }+ )++ task_id = _seed_task(session, uuid.uuid4().hex, collection_id)+ with patch.object(+ rag_mod, "_get_rag_service_for_thread", return_value=svc+ ):+ rag_mod._background_index_worker(+ task_id, collection_id, USERNAME, DB_PASSWORD, False, 2+ )++ session.expire_all()+ status = rag_mod.get_index_status(+ SimpleNamespace(session={}), collection_id, USERNAME+ )++ assert status["status"] == "failed"+ assert status["error_message"] == (+ "1 document(s) failed to index. The collection was reconciled to "+ "the durable vector store."+ )+ assert status["result"]["errors"] == [+ {+ "doc_id": "doc-1",+ "title": "report",+ "error": "Document deleted during indexing",+ }+ ]++ @pytest.mark.parametrize("path", sorted(_WORKER_FAILURE_MESSAGES))+ def test_status_serves_each_worker_failure_message_unchanged(+ self, world, seeded, path+ ):+ """Each terminal message the worker builds only from fixed phrases+ and counts reaches the client exactly as stored. The remote-drop+ text says whether a retry can help and warns that collections it+ already dropped need a forced re-index."""+ from local_deep_research.web.routers import rag as rag_mod++ session = world+ collection_id, _ = seeded+ if path == "collection-not-found":+ collection_id = uuid.uuid4().hex+ task_id = _seed_task(session, uuid.uuid4().hex, collection_id)++ _fail_indexing(rag_mod, path, task_id, collection_id)++ session.expire_all()+ task = session.query(TaskMetadata).filter_by(task_id=task_id).one()+ assert task.status == "failed"+ assert task.error_message == _WORKER_FAILURE_MESSAGES[path]++ status = rag_mod.get_index_status(+ SimpleNamespace(session={}), collection_id, USERNAME+ )++ assert status["status"] == "failed"+ assert status["error_message"] == task.error_message+ for secret in _CLIENT_SECRETS:+ assert secret not in str(status), (path, secret)++ @pytest.mark.parametrize(+ ("failure", "expected"),+ [+ pytest.param(None, _GENERIC_TASK_ERROR, id="no-record"),+ pytest.param(+ {"code": "unknown"}, _GENERIC_TASK_ERROR, id="unknown-code"+ ),+ pytest.param(+ "collection_not_found",+ _GENERIC_TASK_ERROR,+ id="record-not-a-dict",+ ),+ pytest.param(+ {+ "code": "remote_vector_store_drop",+ "retryable": "yes",+ "dropped": 0,+ },+ _GENERIC_TASK_ERROR,+ id="flag-not-a-bool",+ ),+ pytest.param(+ {+ "code": "remote_vector_store_drop",+ "retryable": True,+ "dropped": "2 at /srv/ldr-data",+ },+ _GENERIC_TASK_ERROR,+ id="count-not-an-int",+ ),+ pytest.param(+ {+ "code": "remote_vector_store_drop",+ "retryable": True,+ "dropped": True,+ },+ _GENERIC_TASK_ERROR,+ id="count-is-a-bool",+ ),+ pytest.param(+ {+ "code": "remote_vector_store_drop",+ "retryable": True,+ "dropped": -1,+ },+ _GENERIC_TASK_ERROR,+ id="count-negative",+ ),+ pytest.param(+ {+ "code": "remote_vector_store_drop",+ "retryable": True,+ "dropped": None,+ },+ _GENERIC_TASK_ERROR,+ id="count-missing",+ ),+ pytest.param(+ {"code": "collection_not_found"},+ "Collection not found",+ id="valid-record-rebuilds-its-text",+ ),+ pytest.param(+ {+ "code": "remote_vector_store_drop",+ "retryable": False,+ "dropped": 0,+ },+ _WORKER_FAILURE_MESSAGES["remote-drop-permanent"],+ id="valid-drop-record-rebuilds-its-text",+ ),+ ],+ )+ def test_status_never_serves_the_stored_error_text(+ self, world, seeded, failure, expected+ ):+ """The client text comes from the failure record, never from the+ stored message. A row with no record (such as one written before+ records existed) or with a malformed record gets the generic text,+ even when its message starts like the drop guidance. A valid record+ rebuilds its own fixed text and drops whatever the stored message+ holds."""+ from local_deep_research.web.routers import rag as rag_mod++ session = world+ collection_id, _ = seeded+ task_id = _seed_task(session, uuid.uuid4().hex, collection_id)+ task = session.query(TaskMetadata).filter_by(task_id=task_id).one()+ task.status = "failed"+ task.error_message = (+ "Vector store unreachable — indexing was aborted. "+ "[SQL: SELECT * FROM rag_indices] at /srv/ldr-data/raguser.db"+ )+ metadata = {"collection_id": collection_id}+ if failure is not None:+ metadata["failure"] = failure+ task.metadata_json = metadata+ session.commit()++ status = rag_mod.get_index_status(+ SimpleNamespace(session={}), collection_id, USERNAME+ )++ assert status["error_message"] == expected+ for secret in _CLIENT_SECRETS:+ assert secret not in str(status), secret+# ---------------------------------------------------------------------------# (4) concurrent indexing of one collection