~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
Purge research RAG chunks and history counters on deletion (#7083)
LearningCircuitdd6191fe8d19
diff --git a/changelog.d/+research-history-rag-cleanup.bugfix.md b/changelog.d/+research-history-rag-cleanup.bugfix.mdnew file mode 100644index 0000000000..4e0bf1d1a2--- /dev/null+++ b/changelog.d/+research-history-rag-cleanup.bugfix.md@@ -0,0 +1 @@+- Deleting research also removes the RAG chunk text and vector entries of documents actually deleted (if the post-deletion check fails, that document's search data is left in place), while preserving search data for documents still shared by surviving research. Clearing history also releases per-user active-research slots for the research it actually deleted.diff --git a/src/local_deep_research/chat/service.py b/src/local_deep_research/chat/service.pyindex c66d8d9b39..26910c02e6 100644--- a/src/local_deep_research/chat/service.py+++ b/src/local_deep_research/chat/service.py@@ -36,7 +36,10 @@ from ..web.routes.globals import (is_research_thread_alive,set_termination_flag,)-from ..database.research_cascade import delete_research_with_dependents+from ..database.research_cascade import (+ collect_research_document_rag_targets,+ delete_research_with_dependents,+)from ..constants import ResearchStatusfrom ..utilities.formatting import short_id@@ -1230,7 +1233,10 @@ class ChatService:# the shape ``#5074`` was protecting against, but# applied through the chat service's own liveness# gate rather than the helper's IN_PROGRESS check.- delete_research_with_dependents(+ rag_targets = collect_research_document_rag_targets(+ db, research_id+ )+ deleted = delete_research_with_dependents(db,research_id,require_status_not_in_progress=False,@@ -1254,6 +1260,15 @@ class ChatService:db.commit()+ if deleted and rag_targets:+ from ..research_library.deletion.services.document_deletion import (+ DocumentDeletionService,+ )++ DocumentDeletionService(+ self.username+ ).purge_deleted_research_documents(rag_targets)+# Phase 3 — post-commit in-memory cleanup. The worker's own# finally block calls cleanup_research too, so this is a# no-op for the in-progress path; for the stale-IN_PROGRESSdiff --git a/src/local_deep_research/database/research_cascade.py b/src/local_deep_research/database/research_cascade.pyindex 6a3ed16a4f..02ee57afda 100644--- a/src/local_deep_research/database/research_cascade.py+++ b/src/local_deep_research/database/research_cascade.py@@ -28,6 +28,8 @@ from ..constants import ResearchStatusfrom .models import (Base,Document,+ DocumentChunk,+ DocumentCollection,DownloadDuplicates,DownloadTracker,QueuedResearch,@@ -37,6 +39,59 @@ from .models import (from ..utilities.download_url import get_url_hash, get_url_hash_candidates+def collect_research_document_rag_targets(+ db_session: Session, research_id: str+) -> list[tuple[str, list[str]]]:+ """Snapshot owned documents with chunk rows before the research cascade.++ ``DocumentChunk.source_id`` has no foreign key. The cascade deletes the+ owning Document and its collection links, but must leave chunk rows until+ the post-commit vector purge has used their FAISS IDs. Include linked+ collections as well as those named by chunks, since an interrupted index+ can leave the two sets different.+ """+ # Legacy SQLite SELECTs do not open a database transaction. Discovery+ # must share the cascade's snapshot, or a newly committed document can+ # be deleted without appearing in these post-commit cleanup targets.+ _ensure_outer_db_transaction(db_session)+ chunk_rows = (+ db_session.query(Document.id, DocumentChunk.collection_name)+ .join(+ DocumentChunk,+ (DocumentChunk.source_id == Document.id)+ & (DocumentChunk.source_type == "document"),+ )+ .filter(Document.research_id == research_id)+ .distinct()+ .all()+ )+ targets: dict[str, set[str]] = {}+ for document_id, collection_name in chunk_rows:+ collections = targets.setdefault(document_id, set())+ if collection_name and collection_name.startswith("collection_"):+ collections.add(collection_name.removeprefix("collection_"))++ if not targets:+ return []++ links = (+ db_session.query(+ DocumentCollection.document_id, DocumentCollection.collection_id+ )+ .join(Document, Document.id == DocumentCollection.document_id)+ .filter(Document.research_id == research_id)+ .all()+ )+ for document_id, collection_id in links:+ if document_id in targets:+ targets[document_id].add(collection_id)++ return [+ (document_id, sorted(collections))+ for document_id, collections in sorted(targets.items())+ ]++_CASCADE_ROOT_TABLE = "research_history"# (child_table, child_column) pairs whose rows SURVIVE a researchdiff --git a/src/local_deep_research/research_library/deletion/services/document_deletion.py b/src/local_deep_research/research_library/deletion/services/document_deletion.pyindex d97ff65aa5..075b0f8ee2 100644--- a/src/local_deep_research/research_library/deletion/services/document_deletion.py+++ b/src/local_deep_research/research_library/deletion/services/document_deletion.py@@ -101,6 +101,33 @@ class DocumentDeletionService:"""self.username = username+ def purge_deleted_research_documents(+ self, targets: List[Tuple[str, List[str]]]+ ) -> None:+ """Purge vectors and residual chunks after research deletion commits."""+ for document_id, collection_ids in targets:+ # The pre-delete snapshot also contains documents transferred to+ # surviving research by the cascade. Only a missing document may+ # lose its RAG data; query committed state, not the old ownership.+ try:+ with get_user_db_session(self.username) as session:+ if (+ session.query(Document.id)+ .filter_by(id=document_id)+ .first()+ is not None+ ):+ continue+ except Exception:+ logger.warning(+ "Could not verify research document removal; "+ "skipping RAG cleanup"+ )+ continue+ self._purge_document_rag(+ document_id, collection_ids, full_delete=True+ )+def delete_document(self, document_id: str) -> Dict[str, Any]:"""Delete one document, serializing duplicate in-process requests."""with _hold_document_delete_lock(self.username, document_id):@@ -360,11 +387,13 @@ class DocumentDeletionService:logged, not surfaced."""if collection_ids:- dbpw = capture_request_db_password(self.username)- from ...services.rag_service_factory import get_rag_service-for collection_id in collection_ids:try:+ dbpw = capture_request_db_password(self.username)+ from ...services.rag_service_factory import (+ get_rag_service,+ )+# A fresh service per collection so the factory resolves# THAT collection's stored embedding model/provider — a# shared instance would compute the wrong index hash anddiff --git a/src/local_deep_research/web/routers/research.py b/src/local_deep_research/web/routers/research.pyindex cac4627f09..6cf921ece5 100644--- a/src/local_deep_research/web/routers/research.py+++ b/src/local_deep_research/web/routers/research.py@@ -72,6 +72,7 @@ from ..models.database import ()from ...database.models.library import Document as Documentfrom ...database.research_cascade import (+ collect_research_document_rag_targets,SharedDownloadIndex,delete_research_with_dependents as _delete_research_with_dependents,)@@ -1934,6 +1935,9 @@ def delete_research(# Application-level cascade with the #5074 predicates# baked into the parent DELETE (see# ``_delete_research_with_dependents``).+ rag_targets = collect_research_document_rag_targets(+ db_session, research_id+ )deleted = _delete_research_with_dependents(db_session, research_id)if not deleted:# A concurrent request claimed the row between our@@ -1973,6 +1977,17 @@ def delete_research(# Files cannot roll back with the transaction. Remove the report only# after its database deletion commits, retaining the containment guard._unlink_report_file_if_inside_root(report_path)+ # The chunk rows must outlive the parent transaction: vector deletion+ # resolves FAISS IDs from them. Purge only after the guarded delete+ # committed, never after a refusal or rollback.+ if rag_targets:+ from ...research_library.deletion.services.document_deletion import (+ DocumentDeletionService,+ )++ DocumentDeletionService(username).purge_deleted_research_documents(+ rag_targets+ )return {"status": "success"}try:@@ -2028,6 +2043,7 @@ def clear_history(db_session, deletable_ids)deleted_ids = []+ rag_targets = []for research_id in deletable_ids:# The #5074 atomic guard is applied INSIDE the helper's# parent DELETE; a row whose queue entry was claimed@@ -2048,12 +2064,23 @@ def clear_history(# Application-level cascade with the IN_PROGRESS and# claimed-queue predicates baked into the parent# DELETE; see ``_delete_research_with_dependents``.+ document_targets = collect_research_document_rag_targets(+ db_session, research_id+ )if _delete_research_with_dependents(db_session, research_id, shared_downloads=shared_downloads):deleted_ids.append(research_id)+ rag_targets.extend(document_targets)if deleted_ids:+ # The cascade has no FK path to this per-user counter table.+ # Remove only rows whose guarded parent DELETE succeeded;+ # protected or raced-to-active research keeps its slot.+ db_session.query(UserActiveResearch).filter(+ UserActiveResearch.research_id.in_(deleted_ids)+ ).delete(synchronize_session=False)+from ..queue.lifecycle_cleanup import (cleanup_queued_research_state,)@@ -2065,6 +2092,14 @@ def clear_history(for research in research_records:if research.id in deleted_ids:_unlink_report_file_if_inside_root(research.report_path)+ if rag_targets:+ from ...research_library.deletion.services.document_deletion import (+ DocumentDeletionService,+ )++ DocumentDeletionService(username).purge_deleted_research_documents(+ rag_targets+ )return {"status": "success"}try:diff --git a/tests/chat/test_chat_attempt_delete_retry.py b/tests/chat/test_chat_attempt_delete_retry.pyindex d856c915d2..60de741f97 100644--- a/tests/chat/test_chat_attempt_delete_retry.py+++ b/tests/chat/test_chat_attempt_delete_retry.py@@ -21,6 +21,7 @@ Coverage in this file:"""import json+import hashlibimport uuidfrom datetime import datetime, UTCfrom contextlib import contextmanager@@ -257,6 +258,141 @@ class TestDeleteAttemptService:assert session is not Noneassert session.message_count == 0+ def test_delete_attempt_purges_research_document_chunks_after_commit(+ self, setup_database_for_all_tests+ ):+ """Chat teardown uses the same vector-before-chunk purge as history."""+ from local_deep_research.database.models import (+ Collection,+ Document,+ DocumentChunk,+ DocumentCollection,+ SourceType,+ )+ from local_deep_research.database.models.library import (+ EmbeddingProvider,+ )++ SessionLocal = setup_database_for_all_tests+ username = f"alice_{uuid.uuid4().hex[:8]}"+ service = ChatService(username=username)+ research_id = str(uuid.uuid4())+ document_id = str(uuid.uuid4())+ collection_id = str(uuid.uuid4())+ chunk_text = "private chat research chunk"++ with SessionLocal() as db:+ session_id = _make_session(db, username)+ _seed_attempt(db, username, session_id, research_id)+ source_type_id = str(uuid.uuid4())+ db.add(+ SourceType(+ id=source_type_id,+ name=source_type_id,+ display_name="Research Download",+ icon="download",+ )+ )+ db.add(+ Document(+ id=document_id,+ research_id=research_id,+ source_type_id=source_type_id,+ document_hash=f"hash-{document_id}",+ file_size=len(chunk_text),+ file_type="txt",+ )+ )+ db.add(+ Collection(+ id=collection_id,+ name=f"chat-{collection_id}",+ collection_type="user_collection",+ )+ )+ db.add(+ DocumentCollection(+ document_id=document_id,+ collection_id=collection_id,+ indexed=True,+ chunk_count=1,+ )+ )+ db.add(+ DocumentChunk(+ chunk_hash=hashlib.sha256(chunk_text.encode()).hexdigest(),+ source_type="document",+ source_id=document_id,+ collection_name=f"collection_{collection_id}",+ chunk_text=chunk_text,+ chunk_index=0,+ start_char=0,+ end_char=len(chunk_text),+ word_count=len(chunk_text.split()),+ embedding_id=str(uuid.uuid4()),+ embedding_model="fake-model",+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,+ embedding_dimension=8,+ )+ )+ db.commit()++ events = []++ class _Rag:+ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ def purge_document_vectors(self, doc_id, coll_id):+ with SessionLocal() as db:+ assert db.query(Document).filter_by(id=doc_id).count() == 0+ assert (+ db.query(DocumentChunk)+ .filter_by(source_id=doc_id)+ .count()+ == 1+ )+ events.append(("vectors", doc_id, coll_id))++ def purge_document_chunks(self, doc_id, coll_id):+ events.append(("chunks", doc_id, coll_id))++ @contextmanager+ def _document_db(*_args, **_kwargs):+ with SessionLocal() as db:+ yield db++ with (+ _service_db(SessionLocal),+ patch(+ "local_deep_research.research_library.deletion.services.document_deletion.get_user_db_session",+ _document_db,+ ),+ patch(+ "local_deep_research.research_library.deletion.services.document_deletion.capture_request_db_password",+ return_value="test-password",+ ),+ patch(+ "local_deep_research.research_library.services.rag_service_factory.get_rag_service",+ side_effect=lambda *_args, **_kwargs: _Rag(),+ ),+ ):+ assert service.delete_attempt(session_id, research_id) is True++ with SessionLocal() as db:+ assert db.query(Document).filter_by(id=document_id).count() == 0+ assert (+ db.query(DocumentChunk).filter_by(source_id=document_id).count()+ == 0+ )+ assert events == [+ ("vectors", document_id, collection_id),+ ("chunks", document_id, collection_id),+ ]+def test_delete_attempt_failed_no_assistant_removes_user_msg(self, setup_database_for_all_tests):diff --git a/tests/database/test_research_retry_rag_cleanup.py b/tests/database/test_research_retry_rag_cleanup.pynew file mode 100644index 0000000000..daf0219f83--- /dev/null+++ b/tests/database/test_research_retry_rag_cleanup.py@@ -0,0 +1,332 @@+"""Research retries preserve files and RAG data until deletion commits."""++from contextlib import contextmanager, nullcontext+import sqlite3++import pytest+from sqlalchemy.exc import OperationalError+from sqlalchemy.orm import Session++from tests.database import test_shared_document_lifetime as shared_lifetime+from tests.database.test_shared_document_lifetime import (+ BODY,+ seed_family,+ call_delete_route,+)++from local_deep_research.database import research_cascade+from local_deep_research.database.models import ResearchHistory+from local_deep_research.database.models.library import (+ Collection,+ Document,+ DocumentChunk,+ DocumentCollection,+ EmbeddingProvider,+)+from local_deep_research.research_library.deletion.services import (+ document_deletion,+)+from local_deep_research.research_library.services import rag_service_factory+++# Reuse the two real SQLite FK variants and production-style rollback fixture.+database = shared_lifetime.database+route_session = shared_lifetime.route_session+++def observe_actual_purge(monkeypatch, database, events):+ @contextmanager+ def independent_session(_username):+ with Session(database) as session:+ yield session++ class VectorObserver:+ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ def purge_document_vectors(self, document_id, collection_id):+ # A fresh connection observes the committed delete and the IDs+ # needed by the real vector operation, not a dirty ORM identity map.+ with Session(database) as verification:+ assert verification.get(ResearchHistory, "owner") is None+ assert verification.get(Document, document_id) is None+ chunk = (+ verification.query(DocumentChunk)+ .filter_by(+ source_id=document_id,+ collection_name=f"collection_{collection_id}",+ )+ .one()+ )+ assert chunk.embedding_id == collection_id+ events.append(("vectors", document_id, collection_id))++ def purge_document_chunks(self, document_id, collection_id):+ events.append(("chunks", document_id, collection_id))+ # Keep rows for the real production residual sweep below.++ monkeypatch.setattr(+ document_deletion, "get_user_db_session", independent_session+ )+ monkeypatch.setattr(+ document_deletion,+ "capture_request_db_password",+ lambda _: "test-password",+ )+ monkeypatch.setattr(+ rag_service_factory,+ "get_rag_service",+ lambda *args, **kwargs: VectorObserver(),+ )++[email protected]("batch", [False, True], ids=["single", "history"])+ "persistent", [False, True], ids=["transient", "exhausted"]+)+def test_combined_retry_never_purges_before_commit(+ database, route_session, monkeypatch, tmp_path, batch, persistent+):+ from local_deep_research.config import paths++ seed_family(database, references=False)+ report = tmp_path / "report.md"+ report.write_text("Persist this file until the deletion commits.")+ monkeypatch.setattr(+ paths, "get_research_outputs_directory", lambda: tmp_path+ )+ with Session(database) as writer:+ writer.get(ResearchHistory, "owner").report_path = str(report)+ writer.commit()+ events = []+ observe_actual_purge(monkeypatch, database, events)+ original_commit = route_session.commit+ attempts = 0++ def conflicting_commit():+ nonlocal attempts+ attempts += 1+ assert report.is_file()+ assert events == []+ with Session(database) as verification:+ assert verification.get(ResearchHistory, "owner") is not None+ assert (+ verification.query(DocumentChunk)+ .filter_by(source_id="canonical")+ .count()+ == 2+ )+ if persistent or attempts == 1:+ raise OperationalError(+ "COMMIT", None, sqlite3.OperationalError("database is locked")+ )+ original_commit()++ monkeypatch.setattr(route_session, "commit", conflicting_commit)+ response = call_delete_route(batch)+ with Session(database) as verification:+ if persistent:+ assert response.status_code == 500+ assert attempts == 3+ assert report.is_file()+ assert events == []+ assert verification.get(ResearchHistory, "owner") is not None+ assert verification.get(Document, "canonical") is not None+ assert (+ verification.query(DocumentChunk)+ .filter_by(source_id="canonical")+ .count()+ == 2+ )+ else:+ assert response == {"status": "success"}+ assert attempts == 2+ assert not report.exists()+ assert verification.get(ResearchHistory, "owner") is None+ assert verification.get(Document, "canonical") is None+ assert (+ verification.query(DocumentChunk)+ .filter_by(source_id="canonical")+ .count()+ == 0+ )+ assert events == [+ (kind, "canonical", collection)+ for collection in ("library", "selected")+ for kind in ("vectors", "chunks")+ ]++[email protected]("batch", [False, True], ids=["single", "history"])+def test_combined_retry_refreshes_collection_targets(+ database, route_session, monkeypatch, batch+):+ seed_family(database, references=False)+ events = []+ observe_actual_purge(monkeypatch, database, events)+ original_lookup = research_cascade._shared_document_replacements+ lookups = 0++ def add_collection_after_lookup(*args):+ nonlocal lookups+ found = original_lookup(*args)+ lookups += 1+ if lookups == 1:+ with Session(database) as writer:+ writer.add(Collection(id="added", name="Added during deletion"))+ writer.flush()+ writer.add(+ DocumentCollection(+ document_id="canonical",+ collection_id="added",+ indexed=True,+ chunk_count=1,+ )+ )+ writer.add(+ DocumentChunk(+ chunk_hash="added",+ source_type="document",+ source_id="canonical",+ collection_name="collection_added",+ chunk_text=BODY,+ chunk_index=0,+ start_char=0,+ end_char=len(BODY),+ word_count=len(BODY.split()),+ embedding_id="added",+ embedding_model="test",+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,+ )+ )+ writer.commit()+ return found++ monkeypatch.setattr(+ research_cascade,+ "_shared_document_replacements",+ add_collection_after_lookup,+ )+ assert call_delete_route(batch) == {"status": "success"}+ assert lookups == 2+ assert events == [+ (kind, "canonical", collection)+ for collection in ("added", "library", "selected")+ for kind in ("vectors", "chunks")+ ]+ with Session(database) as verification:+ assert verification.get(ResearchHistory, "owner") is None+ assert verification.get(Document, "canonical") is None+ assert (+ verification.query(DocumentChunk)+ .filter_by(source_id="canonical")+ .count()+ == 0+ )++[email protected]("batch", [False, True], ids=["single", "history"])+def test_target_discovery_shares_the_cascade_snapshot(+ database, route_session, monkeypatch, batch+):+ """A document committed after target discovery must not escape its purge.++ Legacy SQLite SELECTs do not start a database transaction. If discovery+ runs before BEGIN, the cascade can see and delete a newer document while+ cleanup still has the older target list. A shared snapshot forces the+ existing lock retry to rediscover the new document and its vector IDs.+ """+ from local_deep_research.web.routers import research++ seed_family(database, references=False)+ events = []+ observe_actual_purge(monkeypatch, database, events)+ collect = research.collect_research_document_rag_targets+ snapshot_calls = 0++ def collect_then_insert(*args):+ nonlocal snapshot_calls+ targets = collect(*args)+ snapshot_calls += 1+ if snapshot_calls == 1:+ # A separate WAL writer commits a newly indexed document after+ # the target SELECTs but before the cascade's reference lookup.+ with Session(database) as writer:+ writer.add(Collection(id="late", name="Late collection"))+ writer.flush()+ writer.add(+ Document(+ id="late-document",+ source_type_id="source",+ research_id="owner",+ document_hash="late-document-hash",+ file_type="text",+ file_size=len(BODY),+ text_content=BODY,+ )+ )+ writer.flush()+ writer.add(+ DocumentChunk(+ chunk_hash="late-document-chunk",+ source_type="document",+ source_id="late-document",+ collection_name="collection_late",+ chunk_text=BODY,+ chunk_index=0,+ start_char=0,+ end_char=len(BODY),+ word_count=len(BODY.split()),+ embedding_id="late",+ embedding_model="test",+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,+ )+ )+ writer.commit()+ return targets++ monkeypatch.setattr(+ research, "collect_research_document_rag_targets", collect_then_insert+ )+ assert call_delete_route(batch) == {"status": "success"}+ with Session(database) as verification:+ assert verification.get(ResearchHistory, "owner") is None+ assert verification.get(Document, "late-document") is None+ assert (+ verification.query(DocumentChunk)+ .filter_by(source_id="late-document")+ .count()+ == 0+ ), "Deleted document retained its chunk text and vector ID"+ assert snapshot_calls == 2+ assert events == [+ (kind, document_id, collection)+ for document_id, collection in (+ ("canonical", "library"),+ ("canonical", "selected"),+ ("late-document", "late"),+ )+ for kind in ("vectors", "chunks")+ ]++[email protected]("nested", [False, True], ids=["outer", "savepoint"])+def test_target_discovery_preserves_caller_transaction(database, nested):+ """Discovery must neither restart nor commit an active transaction."""+ seed_family(database, references=False)+ with Session(database) as session:+ session.get(ResearchHistory, "owner").query = "Uncommitted change"+ session.flush()+ with session.begin_nested() if nested else nullcontext():+ session.get(Document, "canonical").title = "Uncommitted title"+ session.flush()+ assert research_cascade.collect_research_document_rag_targets(+ session, "owner"+ ) == [("canonical", ["library", "selected"])]+ session.rollback()+ with Session(database) as verification:+ assert verification.get(ResearchHistory, "owner").query == "q"+ assert verification.get(Document, "canonical").title == "Shared paper"diff --git a/tests/web/routers/test_delete_research_cascade.py b/tests/web/routers/test_delete_research_cascade.pyindex 0e8e0902de..f42418f62b 100644--- a/tests/web/routers/test_delete_research_cascade.py+++ b/tests/web/routers/test_delete_research_cascade.py@@ -62,6 +62,8 @@ the production schema (same ``Base.metadata.create_all``) and enablefrom contextlib import contextmanagerfrom datetime import UTC, datetimefrom unittest.mock import patch+import hashlib+import uuidimport pytestfrom sqlalchemy import create_engine, event@@ -274,6 +276,95 @@ def _populate_all_child_tables(session, research_id):}+def _add_rag_chunk(session, document_id, collection_id):+ from local_deep_research.database.models.library import (+ DocumentChunk,+ DocumentCollection,+ EmbeddingProvider,+ )++ text = f"private chunk of {document_id}"+ session.add(+ DocumentCollection(+ document_id=document_id,+ collection_id=collection_id,+ indexed=True,+ chunk_count=1,+ )+ )+ session.add(+ DocumentChunk(+ chunk_hash=hashlib.sha256(text.encode()).hexdigest(),+ source_type="document",+ source_id=document_id,+ collection_name=f"collection_{collection_id}",+ chunk_text=text,+ chunk_index=0,+ start_char=0,+ end_char=len(text),+ word_count=len(text.split()),+ embedding_id=str(uuid.uuid4()),+ embedding_model="fake-model",+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,+ embedding_dimension=8,+ )+ )+ session.commit()+++@contextmanager+def _patch_real_rag_purge(session, events):+ """Run the production residual-chunk sweep with an inert vector store."""+ from local_deep_research.database.models.library import (+ Document,+ DocumentChunk,+ )++ @contextmanager+ def _ctx(*_args, **_kwargs):+ yield session++ class _Rag:+ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ def purge_document_vectors(self, document_id, collection_id):+ # Vector deletion must see the committed parent deletion while+ # chunk rows still exist to resolve their FAISS IDs.+ assert (+ session.query(Document).filter_by(id=document_id).count() == 0+ )+ assert (+ session.query(DocumentChunk)+ .filter_by(source_id=document_id)+ .count()+ == 1+ )+ events.append(("vectors", document_id, collection_id))++ def purge_document_chunks(self, document_id, collection_id):+ events.append(("chunks", document_id, collection_id))++ with (+ patch(+ "local_deep_research.research_library.deletion.services.document_deletion.get_user_db_session",+ _ctx,+ ),+ patch(+ "local_deep_research.research_library.deletion.services.document_deletion.capture_request_db_password",+ return_value="test-password",+ ),+ patch(+ "local_deep_research.research_library.services.rag_service_factory.get_rag_service",+ side_effect=lambda *_args, **_kwargs: _Rag(),+ ),+ ):+ yield++# ---------------------------------------------------------------------------# Direct unit tests on the helper# ---------------------------------------------------------------------------@@ -750,6 +841,287 @@ class TestClearHistoryRouteWithFKEnforcement:assert db_with_fk.query(ResearchResource).count() == 0+class TestResearchRagPurgeRoutes:+ """History deletion purges chunk text only after the guarded commit."""++ def test_unverified_document_removal_does_not_purge_search_data(self):+ from local_deep_research.research_library.deletion.services import (+ document_deletion,+ )++ service = document_deletion.DocumentDeletionService("owner")+ with (+ patch.object(+ document_deletion,+ "get_user_db_session",+ side_effect=RuntimeError("database unavailable"),+ ),+ patch.object(service, "_purge_document_rag") as purge,+ ):+ service.purge_deleted_research_documents([("doc", ["collection"])])+ purge.assert_not_called()++ @pytest.mark.parametrize("entrypoint", ["single", "history", "chat"])+ def test_transferred_document_keeps_rag_until_last_reference_deleted(+ self, authenticated_client, db_with_fk, entrypoint+ ):+ from local_deep_research.chat.service import ChatService+ from local_deep_research.database.models import (+ ChatSession,+ Document,+ DocumentChunk,+ DocumentCollection,+ ResearchHistory,+ ResearchResource,+ )++ _add_research(db_with_fk, "owner")+ owned = _populate_all_child_tables(db_with_fk, "owner")+ document_id = owned["document"].id+ collection_id = owned["collection"].id+ _add_rag_chunk(db_with_fk, document_id, collection_id)+ _add_research(db_with_fk, "survivor", status="in_progress")+ survivor = ResearchResource(+ research_id="survivor",+ title="Shared document",+ url="https://example.com/another-download-url",+ document_id=document_id,+ source_type="web",+ created_at=_NOW,+ )+ db_with_fk.add(survivor)+ db_with_fk.get(ResearchHistory, "owner").chat_session_id = "sess-owner"+ db_with_fk.get(ChatSession, "sess-owner").message_count = 1+ db_with_fk.commit()+ resource_id = survivor.id+ chunk = (+ db_with_fk.query(DocumentChunk)+ .filter_by(source_id=document_id)+ .one()+ )+ snapshot = (chunk.id, chunk.chunk_text, chunk.embedding_id)+ events = []++ @contextmanager+ def _chat_db(*_args, **_kwargs):+ yield db_with_fk++ with (+ _patch_session(db_with_fk),+ _patch_real_rag_purge(db_with_fk, events),+ patch(f"{_RR}.get_active_research_ids", return_value=[]),+ patch(_LIFECYCLE_CLEANUP),+ patch(+ "local_deep_research.chat.service.get_user_db_session", _chat_db+ ),+ patch("local_deep_research.chat.service.cleanup_research"),+ ):+ if entrypoint == "chat":+ assert ChatService("owner").delete_attempt(+ "sess-owner", "owner"+ )+ elif entrypoint == "history":+ response = authenticated_client.post("/api/clear_history")+ assert response.status_code == 200, response.text[:300]+ else:+ response = authenticated_client.delete("/api/delete/owner")+ assert response.status_code == 200, response.text[:300]++ db_with_fk.expire_all()+ assert db_with_fk.get(ResearchHistory, "owner") is None+ retained = db_with_fk.get(Document, document_id)+ assert retained.research_id == "survivor"+ assert retained.resource_id == resource_id+ chunk = (+ db_with_fk.query(DocumentChunk)+ .filter_by(source_id=document_id)+ .one()+ )+ assert (chunk.id, chunk.chunk_text, chunk.embedding_id) == snapshot+ link = (+ db_with_fk.query(DocumentCollection)+ .filter_by(document_id=document_id, collection_id=collection_id)+ .one()+ )+ assert link.indexed and link.chunk_count == 1+ assert events == []++ # Once the final owner is removed, the same data must be purged.+ db_with_fk.get(ResearchHistory, "survivor").status = "completed"+ db_with_fk.commit()+ response = authenticated_client.delete("/api/delete/survivor")+ assert response.status_code == 200, response.text[:300]++ db_with_fk.expire_all()+ assert db_with_fk.get(Document, document_id) is None+ assert (+ db_with_fk.query(DocumentChunk)+ .filter_by(source_id=document_id)+ .count()+ == 0+ )+ assert events == [+ ("vectors", document_id, collection_id),+ ("chunks", document_id, collection_id),+ ]++ def test_single_delete_purges_owned_chunks_after_commit(+ self, authenticated_client, db_with_fk+ ):+ from local_deep_research.database.models.library import (+ Document,+ DocumentChunk,+ )++ _add_research(db_with_fk, "done")+ owned = _populate_all_child_tables(db_with_fk, "done")+ document_id = owned["document"].id+ collection_id = owned["collection"].id+ _add_rag_chunk(db_with_fk, document_id, collection_id)+ events = []++ with (+ _patch_session(db_with_fk),+ _patch_real_rag_purge(db_with_fk, events),+ patch(_LIFECYCLE_CLEANUP),+ ):+ response = authenticated_client.delete("/api/delete/done")++ assert response.status_code == 200, response.text[:300]+ assert db_with_fk.query(Document).filter_by(id=document_id).count() == 0+ assert (+ db_with_fk.query(DocumentChunk)+ .filter_by(source_id=document_id)+ .count()+ == 0+ ), "deleted research retained its private chunk text"+ assert events == [+ ("vectors", document_id, collection_id),+ ("chunks", document_id, collection_id),+ ]++ def test_clear_history_purges_only_deleted_research_chunks(+ self, authenticated_client, db_with_fk+ ):+ from local_deep_research.database.models.library import (+ Document,+ DocumentChunk,+ )++ _add_research(db_with_fk, "done")+ done = _populate_all_child_tables(db_with_fk, "done")+ done_id = done["document"].id+ done_collection_id = done["collection"].id+ _add_rag_chunk(db_with_fk, done_id, done_collection_id)++ _add_research(db_with_fk, "live", status="in_progress")+ live = _populate_all_child_tables(db_with_fk, "live")+ live_id = live["document"].id+ _add_rag_chunk(db_with_fk, live_id, live["collection"].id)+ events = []++ with (+ _patch_session(db_with_fk),+ _patch_real_rag_purge(db_with_fk, events),+ patch(f"{_RR}.get_active_research_ids", return_value=[]),+ patch(_LIFECYCLE_CLEANUP),+ ):+ response = authenticated_client.post("/api/clear_history")++ assert response.status_code == 200, response.text[:300]+ assert db_with_fk.query(Document).filter_by(id=done_id).count() == 0+ assert (+ db_with_fk.query(DocumentChunk).filter_by(source_id=done_id).count()+ == 0+ )+ assert db_with_fk.query(Document).filter_by(id=live_id).count() == 1+ assert (+ db_with_fk.query(DocumentChunk).filter_by(source_id=live_id).count()+ == 1+ )+ assert events == [+ ("vectors", done_id, done_collection_id),+ ("chunks", done_id, done_collection_id),+ ]++ def test_unavailable_rag_credentials_still_sweep_private_chunks(+ self, authenticated_client, db_with_fk+ ):+ from local_deep_research.database.models.library import DocumentChunk++ _add_research(db_with_fk, "done")+ owned = _populate_all_child_tables(db_with_fk, "done")+ document_id = owned["document"].id+ _add_rag_chunk(db_with_fk, document_id, owned["collection"].id)+ events = []++ with (+ _patch_session(db_with_fk),+ _patch_real_rag_purge(db_with_fk, events),+ patch(+ "local_deep_research.research_library.deletion.services.document_deletion.capture_request_db_password",+ side_effect=RuntimeError("unavailable"),+ ),+ patch(_LIFECYCLE_CLEANUP),+ ):+ response = authenticated_client.delete("/api/delete/done")++ assert response.status_code == 200, response.text[:300]+ assert events == []+ assert (+ db_with_fk.query(DocumentChunk)+ .filter_by(source_id=document_id)+ .count()+ == 0+ )++ def test_later_failure_rolls_back_without_purging_chunks(+ self, authenticated_client, db_with_fk+ ):+ from local_deep_research.database.models import ResearchHistory+ from local_deep_research.database.models.library import (+ Document,+ DocumentChunk,+ )++ _add_research(db_with_fk, "done")+ owned = _populate_all_child_tables(db_with_fk, "done")+ document_id = owned["document"].id+ _add_rag_chunk(db_with_fk, document_id, owned["collection"].id)++ @contextmanager+ def _rollback_on_error(*_args, **_kwargs):+ try:+ yield db_with_fk+ except Exception:+ db_with_fk.rollback()+ raise++ with (+ patch(f"{_RR}.get_user_db_session", _rollback_on_error),+ patch(+ _LIFECYCLE_CLEANUP, side_effect=RuntimeError("later failure")+ ),+ patch(+ "local_deep_research.research_library.deletion.services.document_deletion.DocumentDeletionService.purge_deleted_research_documents"+ ) as purge,+ ):+ response = authenticated_client.delete("/api/delete/done")++ assert response.status_code == 500+ assert (+ db_with_fk.query(ResearchHistory).filter_by(id="done").count() == 1+ )+ assert db_with_fk.query(Document).filter_by(id=document_id).count() == 1+ assert (+ db_with_fk.query(DocumentChunk)+ .filter_by(source_id=document_id)+ .count()+ == 1+ )+ purge.assert_not_called()++class TestClearHistorySharedDownloadIndex:"""``clear_history`` must build one ``SharedDownloadIndex`` for thewhole batch and hand it to every per-research delete; otherwise eachdiff --git a/tests/web/routers/test_research_mutating_route_effects.py b/tests/web/routers/test_research_mutating_route_effects.pyindex f40381197b..6ed5cb36ec 100644--- a/tests/web/routers/test_research_mutating_route_effects.py+++ b/tests/web/routers/test_research_mutating_route_effects.py@@ -173,6 +173,36 @@ def test_clear_history_protects_an_in_progress_row(authenticated_client, db):assert _ids(db) == {"live"}+def test_clear_history_removes_only_deleted_research_counters(+ authenticated_client, db+):+ """A cleared row must not keep occupying a per-user research slot."""+ from local_deep_research.database.models import UserActiveResearch++ _add_research(db, "done", status="completed")+ _add_research(db, "live", status="in_progress")+ db.add_all(+ [+ UserActiveResearch(username="owner", research_id="done"),+ UserActiveResearch(username="owner", research_id="live"),+ ]+ )+ db.commit()++ with (+ _patch_session(db),+ patch(f"{_RR}.get_active_research_ids", return_value=[]),+ patch(_LIFECYCLE_CLEANUP),+ ):+ response = authenticated_client.post("/api/clear_history")++ assert response.status_code == 200, response.text[:300]+ assert _ids(db) == {"live"}+ assert {row.research_id for row in db.query(UserActiveResearch).all()} == {+ "live"+ }++# ---------------------------------------------------------------------------# DELETE /api/delete/{research_id}# ---------------------------------------------------------------------------