Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

Purge research RAG chunks and history counters on deletion (#7083)

LearningCircuitdd6191fe8d19
diff --git a/changelog.d/+research-history-rag-cleanup.bugfix.md b/changelog.d/+research-history-rag-cleanup.bugfix.md
new file mode 100644
index 0000000000..4e0bf1d1a2
--- /dev/null
+++ b/changelog.d/+research-history-rag-cleanup.bugfix.md
@@ -0,0 +1 @@
+- Deleting research also removes the RAG chunk text and vector entries of documents actually deleted (if the post-deletion check fails, that document's search data is left in place), while preserving search data for documents still shared by surviving research. Clearing history also releases per-user active-research slots for the research it actually deleted.
diff --git a/src/local_deep_research/chat/service.py b/src/local_deep_research/chat/service.py
index c66d8d9b39..26910c02e6 100644
--- a/src/local_deep_research/chat/service.py
+++ b/src/local_deep_research/chat/service.py
@@ -36,7 +36,10 @@ from ..web.routes.globals import (
is_research_thread_alive,
set_termination_flag,
)
-from ..database.research_cascade import delete_research_with_dependents
+from ..database.research_cascade import (
+ collect_research_document_rag_targets,
+ delete_research_with_dependents,
+)
from ..constants import ResearchStatus
from ..utilities.formatting import short_id
@@ -1230,7 +1233,10 @@ class ChatService:
# the shape ``#5074`` was protecting against, but
# applied through the chat service's own liveness
# gate rather than the helper's IN_PROGRESS check.
- delete_research_with_dependents(
+ rag_targets = collect_research_document_rag_targets(
+ db, research_id
+ )
+ deleted = delete_research_with_dependents(
db,
research_id,
require_status_not_in_progress=False,
@@ -1254,6 +1260,15 @@ class ChatService:
db.commit()
+ if deleted and rag_targets:
+ from ..research_library.deletion.services.document_deletion import (
+ DocumentDeletionService,
+ )
+
+ DocumentDeletionService(
+ self.username
+ ).purge_deleted_research_documents(rag_targets)
+
# Phase 3 — post-commit in-memory cleanup. The worker's own
# finally block calls cleanup_research too, so this is a
# no-op for the in-progress path; for the stale-IN_PROGRESS
diff --git a/src/local_deep_research/database/research_cascade.py b/src/local_deep_research/database/research_cascade.py
index 6a3ed16a4f..02ee57afda 100644
--- a/src/local_deep_research/database/research_cascade.py
+++ b/src/local_deep_research/database/research_cascade.py
@@ -28,6 +28,8 @@ from ..constants import ResearchStatus
from .models import (
Base,
Document,
+ DocumentChunk,
+ DocumentCollection,
DownloadDuplicates,
DownloadTracker,
QueuedResearch,
@@ -37,6 +39,59 @@ from .models import (
from ..utilities.download_url import get_url_hash, get_url_hash_candidates
+def collect_research_document_rag_targets(
+ db_session: Session, research_id: str
+) -> list[tuple[str, list[str]]]:
+ """Snapshot owned documents with chunk rows before the research cascade.
+
+ ``DocumentChunk.source_id`` has no foreign key. The cascade deletes the
+ owning Document and its collection links, but must leave chunk rows until
+ the post-commit vector purge has used their FAISS IDs. Include linked
+ collections as well as those named by chunks, since an interrupted index
+ can leave the two sets different.
+ """
+ # Legacy SQLite SELECTs do not open a database transaction. Discovery
+ # must share the cascade's snapshot, or a newly committed document can
+ # be deleted without appearing in these post-commit cleanup targets.
+ _ensure_outer_db_transaction(db_session)
+ chunk_rows = (
+ db_session.query(Document.id, DocumentChunk.collection_name)
+ .join(
+ DocumentChunk,
+ (DocumentChunk.source_id == Document.id)
+ & (DocumentChunk.source_type == "document"),
+ )
+ .filter(Document.research_id == research_id)
+ .distinct()
+ .all()
+ )
+ targets: dict[str, set[str]] = {}
+ for document_id, collection_name in chunk_rows:
+ collections = targets.setdefault(document_id, set())
+ if collection_name and collection_name.startswith("collection_"):
+ collections.add(collection_name.removeprefix("collection_"))
+
+ if not targets:
+ return []
+
+ links = (
+ db_session.query(
+ DocumentCollection.document_id, DocumentCollection.collection_id
+ )
+ .join(Document, Document.id == DocumentCollection.document_id)
+ .filter(Document.research_id == research_id)
+ .all()
+ )
+ for document_id, collection_id in links:
+ if document_id in targets:
+ targets[document_id].add(collection_id)
+
+ return [
+ (document_id, sorted(collections))
+ for document_id, collections in sorted(targets.items())
+ ]
+
+
_CASCADE_ROOT_TABLE = "research_history"
# (child_table, child_column) pairs whose rows SURVIVE a research
diff --git a/src/local_deep_research/research_library/deletion/services/document_deletion.py b/src/local_deep_research/research_library/deletion/services/document_deletion.py
index d97ff65aa5..075b0f8ee2 100644
--- a/src/local_deep_research/research_library/deletion/services/document_deletion.py
+++ b/src/local_deep_research/research_library/deletion/services/document_deletion.py
@@ -101,6 +101,33 @@ class DocumentDeletionService:
"""
self.username = username
+ def purge_deleted_research_documents(
+ self, targets: List[Tuple[str, List[str]]]
+ ) -> None:
+ """Purge vectors and residual chunks after research deletion commits."""
+ for document_id, collection_ids in targets:
+ # The pre-delete snapshot also contains documents transferred to
+ # surviving research by the cascade. Only a missing document may
+ # lose its RAG data; query committed state, not the old ownership.
+ try:
+ with get_user_db_session(self.username) as session:
+ if (
+ session.query(Document.id)
+ .filter_by(id=document_id)
+ .first()
+ is not None
+ ):
+ continue
+ except Exception:
+ logger.warning(
+ "Could not verify research document removal; "
+ "skipping RAG cleanup"
+ )
+ continue
+ self._purge_document_rag(
+ document_id, collection_ids, full_delete=True
+ )
+
def delete_document(self, document_id: str) -> Dict[str, Any]:
"""Delete one document, serializing duplicate in-process requests."""
with _hold_document_delete_lock(self.username, document_id):
@@ -360,11 +387,13 @@ class DocumentDeletionService:
logged, not surfaced.
"""
if collection_ids:
- dbpw = capture_request_db_password(self.username)
- from ...services.rag_service_factory import get_rag_service
-
for collection_id in collection_ids:
try:
+ dbpw = capture_request_db_password(self.username)
+ from ...services.rag_service_factory import (
+ get_rag_service,
+ )
+
# A fresh service per collection so the factory resolves
# THAT collection's stored embedding model/provider — a
# shared instance would compute the wrong index hash and
diff --git a/src/local_deep_research/web/routers/research.py b/src/local_deep_research/web/routers/research.py
index cac4627f09..6cf921ece5 100644
--- a/src/local_deep_research/web/routers/research.py
+++ b/src/local_deep_research/web/routers/research.py
@@ -72,6 +72,7 @@ from ..models.database import (
)
from ...database.models.library import Document as Document
from ...database.research_cascade import (
+ collect_research_document_rag_targets,
SharedDownloadIndex,
delete_research_with_dependents as _delete_research_with_dependents,
)
@@ -1934,6 +1935,9 @@ def delete_research(
# Application-level cascade with the #5074 predicates
# baked into the parent DELETE (see
# ``_delete_research_with_dependents``).
+ rag_targets = collect_research_document_rag_targets(
+ db_session, research_id
+ )
deleted = _delete_research_with_dependents(db_session, research_id)
if not deleted:
# A concurrent request claimed the row between our
@@ -1973,6 +1977,17 @@ def delete_research(
# Files cannot roll back with the transaction. Remove the report only
# after its database deletion commits, retaining the containment guard.
_unlink_report_file_if_inside_root(report_path)
+ # The chunk rows must outlive the parent transaction: vector deletion
+ # resolves FAISS IDs from them. Purge only after the guarded delete
+ # committed, never after a refusal or rollback.
+ if rag_targets:
+ from ...research_library.deletion.services.document_deletion import (
+ DocumentDeletionService,
+ )
+
+ DocumentDeletionService(username).purge_deleted_research_documents(
+ rag_targets
+ )
return {"status": "success"}
try:
@@ -2028,6 +2043,7 @@ def clear_history(
db_session, deletable_ids
)
deleted_ids = []
+ rag_targets = []
for research_id in deletable_ids:
# The #5074 atomic guard is applied INSIDE the helper's
# parent DELETE; a row whose queue entry was claimed
@@ -2048,12 +2064,23 @@ def clear_history(
# Application-level cascade with the IN_PROGRESS and
# claimed-queue predicates baked into the parent
# DELETE; see ``_delete_research_with_dependents``.
+ document_targets = collect_research_document_rag_targets(
+ db_session, research_id
+ )
if _delete_research_with_dependents(
db_session, research_id, shared_downloads=shared_downloads
):
deleted_ids.append(research_id)
+ rag_targets.extend(document_targets)
if deleted_ids:
+ # The cascade has no FK path to this per-user counter table.
+ # Remove only rows whose guarded parent DELETE succeeded;
+ # protected or raced-to-active research keeps its slot.
+ db_session.query(UserActiveResearch).filter(
+ UserActiveResearch.research_id.in_(deleted_ids)
+ ).delete(synchronize_session=False)
+
from ..queue.lifecycle_cleanup import (
cleanup_queued_research_state,
)
@@ -2065,6 +2092,14 @@ def clear_history(
for research in research_records:
if research.id in deleted_ids:
_unlink_report_file_if_inside_root(research.report_path)
+ if rag_targets:
+ from ...research_library.deletion.services.document_deletion import (
+ DocumentDeletionService,
+ )
+
+ DocumentDeletionService(username).purge_deleted_research_documents(
+ rag_targets
+ )
return {"status": "success"}
try:
diff --git a/tests/chat/test_chat_attempt_delete_retry.py b/tests/chat/test_chat_attempt_delete_retry.py
index d856c915d2..60de741f97 100644
--- a/tests/chat/test_chat_attempt_delete_retry.py
+++ b/tests/chat/test_chat_attempt_delete_retry.py
@@ -21,6 +21,7 @@ Coverage in this file:
"""
import json
+import hashlib
import uuid
from datetime import datetime, UTC
from contextlib import contextmanager
@@ -257,6 +258,141 @@ class TestDeleteAttemptService:
assert session is not None
assert session.message_count == 0
+ def test_delete_attempt_purges_research_document_chunks_after_commit(
+ self, setup_database_for_all_tests
+ ):
+ """Chat teardown uses the same vector-before-chunk purge as history."""
+ from local_deep_research.database.models import (
+ Collection,
+ Document,
+ DocumentChunk,
+ DocumentCollection,
+ SourceType,
+ )
+ from local_deep_research.database.models.library import (
+ EmbeddingProvider,
+ )
+
+ SessionLocal = setup_database_for_all_tests
+ username = f"alice_{uuid.uuid4().hex[:8]}"
+ service = ChatService(username=username)
+ research_id = str(uuid.uuid4())
+ document_id = str(uuid.uuid4())
+ collection_id = str(uuid.uuid4())
+ chunk_text = "private chat research chunk"
+
+ with SessionLocal() as db:
+ session_id = _make_session(db, username)
+ _seed_attempt(db, username, session_id, research_id)
+ source_type_id = str(uuid.uuid4())
+ db.add(
+ SourceType(
+ id=source_type_id,
+ name=source_type_id,
+ display_name="Research Download",
+ icon="download",
+ )
+ )
+ db.add(
+ Document(
+ id=document_id,
+ research_id=research_id,
+ source_type_id=source_type_id,
+ document_hash=f"hash-{document_id}",
+ file_size=len(chunk_text),
+ file_type="txt",
+ )
+ )
+ db.add(
+ Collection(
+ id=collection_id,
+ name=f"chat-{collection_id}",
+ collection_type="user_collection",
+ )
+ )
+ db.add(
+ DocumentCollection(
+ document_id=document_id,
+ collection_id=collection_id,
+ indexed=True,
+ chunk_count=1,
+ )
+ )
+ db.add(
+ DocumentChunk(
+ chunk_hash=hashlib.sha256(chunk_text.encode()).hexdigest(),
+ source_type="document",
+ source_id=document_id,
+ collection_name=f"collection_{collection_id}",
+ chunk_text=chunk_text,
+ chunk_index=0,
+ start_char=0,
+ end_char=len(chunk_text),
+ word_count=len(chunk_text.split()),
+ embedding_id=str(uuid.uuid4()),
+ embedding_model="fake-model",
+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
+ embedding_dimension=8,
+ )
+ )
+ db.commit()
+
+ events = []
+
+ class _Rag:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def purge_document_vectors(self, doc_id, coll_id):
+ with SessionLocal() as db:
+ assert db.query(Document).filter_by(id=doc_id).count() == 0
+ assert (
+ db.query(DocumentChunk)
+ .filter_by(source_id=doc_id)
+ .count()
+ == 1
+ )
+ events.append(("vectors", doc_id, coll_id))
+
+ def purge_document_chunks(self, doc_id, coll_id):
+ events.append(("chunks", doc_id, coll_id))
+
+ @contextmanager
+ def _document_db(*_args, **_kwargs):
+ with SessionLocal() as db:
+ yield db
+
+ with (
+ _service_db(SessionLocal),
+ patch(
+ "local_deep_research.research_library.deletion.services.document_deletion.get_user_db_session",
+ _document_db,
+ ),
+ patch(
+ "local_deep_research.research_library.deletion.services.document_deletion.capture_request_db_password",
+ return_value="test-password",
+ ),
+ patch(
+ "local_deep_research.research_library.services.rag_service_factory.get_rag_service",
+ side_effect=lambda *_args, **_kwargs: _Rag(),
+ ),
+ ):
+ assert service.delete_attempt(session_id, research_id) is True
+
+ with SessionLocal() as db:
+ assert db.query(Document).filter_by(id=document_id).count() == 0
+ assert (
+ db.query(DocumentChunk).filter_by(source_id=document_id).count()
+ == 0
+ )
+ assert events == [
+ ("vectors", document_id, collection_id),
+ ("chunks", document_id, collection_id),
+ ]
+
def test_delete_attempt_failed_no_assistant_removes_user_msg(
self, setup_database_for_all_tests
):
diff --git a/tests/database/test_research_retry_rag_cleanup.py b/tests/database/test_research_retry_rag_cleanup.py
new file mode 100644
index 0000000000..daf0219f83
--- /dev/null
+++ b/tests/database/test_research_retry_rag_cleanup.py
@@ -0,0 +1,332 @@
+"""Research retries preserve files and RAG data until deletion commits."""
+
+from contextlib import contextmanager, nullcontext
+import sqlite3
+
+import pytest
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import Session
+
+from tests.database import test_shared_document_lifetime as shared_lifetime
+from tests.database.test_shared_document_lifetime import (
+ BODY,
+ seed_family,
+ call_delete_route,
+)
+
+from local_deep_research.database import research_cascade
+from local_deep_research.database.models import ResearchHistory
+from local_deep_research.database.models.library import (
+ Collection,
+ Document,
+ DocumentChunk,
+ DocumentCollection,
+ EmbeddingProvider,
+)
+from local_deep_research.research_library.deletion.services import (
+ document_deletion,
+)
+from local_deep_research.research_library.services import rag_service_factory
+
+
+# Reuse the two real SQLite FK variants and production-style rollback fixture.
+database = shared_lifetime.database
+route_session = shared_lifetime.route_session
+
+
+def observe_actual_purge(monkeypatch, database, events):
+ @contextmanager
+ def independent_session(_username):
+ with Session(database) as session:
+ yield session
+
+ class VectorObserver:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def purge_document_vectors(self, document_id, collection_id):
+ # A fresh connection observes the committed delete and the IDs
+ # needed by the real vector operation, not a dirty ORM identity map.
+ with Session(database) as verification:
+ assert verification.get(ResearchHistory, "owner") is None
+ assert verification.get(Document, document_id) is None
+ chunk = (
+ verification.query(DocumentChunk)
+ .filter_by(
+ source_id=document_id,
+ collection_name=f"collection_{collection_id}",
+ )
+ .one()
+ )
+ assert chunk.embedding_id == collection_id
+ events.append(("vectors", document_id, collection_id))
+
+ def purge_document_chunks(self, document_id, collection_id):
+ events.append(("chunks", document_id, collection_id))
+ # Keep rows for the real production residual sweep below.
+
+ monkeypatch.setattr(
+ document_deletion, "get_user_db_session", independent_session
+ )
+ monkeypatch.setattr(
+ document_deletion,
+ "capture_request_db_password",
+ lambda _: "test-password",
+ )
+ monkeypatch.setattr(
+ rag_service_factory,
+ "get_rag_service",
+ lambda *args, **kwargs: VectorObserver(),
+ )
+
+
[email protected]("batch", [False, True], ids=["single", "history"])
+ "persistent", [False, True], ids=["transient", "exhausted"]
+)
+def test_combined_retry_never_purges_before_commit(
+ database, route_session, monkeypatch, tmp_path, batch, persistent
+):
+ from local_deep_research.config import paths
+
+ seed_family(database, references=False)
+ report = tmp_path / "report.md"
+ report.write_text("Persist this file until the deletion commits.")
+ monkeypatch.setattr(
+ paths, "get_research_outputs_directory", lambda: tmp_path
+ )
+ with Session(database) as writer:
+ writer.get(ResearchHistory, "owner").report_path = str(report)
+ writer.commit()
+ events = []
+ observe_actual_purge(monkeypatch, database, events)
+ original_commit = route_session.commit
+ attempts = 0
+
+ def conflicting_commit():
+ nonlocal attempts
+ attempts += 1
+ assert report.is_file()
+ assert events == []
+ with Session(database) as verification:
+ assert verification.get(ResearchHistory, "owner") is not None
+ assert (
+ verification.query(DocumentChunk)
+ .filter_by(source_id="canonical")
+ .count()
+ == 2
+ )
+ if persistent or attempts == 1:
+ raise OperationalError(
+ "COMMIT", None, sqlite3.OperationalError("database is locked")
+ )
+ original_commit()
+
+ monkeypatch.setattr(route_session, "commit", conflicting_commit)
+ response = call_delete_route(batch)
+ with Session(database) as verification:
+ if persistent:
+ assert response.status_code == 500
+ assert attempts == 3
+ assert report.is_file()
+ assert events == []
+ assert verification.get(ResearchHistory, "owner") is not None
+ assert verification.get(Document, "canonical") is not None
+ assert (
+ verification.query(DocumentChunk)
+ .filter_by(source_id="canonical")
+ .count()
+ == 2
+ )
+ else:
+ assert response == {"status": "success"}
+ assert attempts == 2
+ assert not report.exists()
+ assert verification.get(ResearchHistory, "owner") is None
+ assert verification.get(Document, "canonical") is None
+ assert (
+ verification.query(DocumentChunk)
+ .filter_by(source_id="canonical")
+ .count()
+ == 0
+ )
+ assert events == [
+ (kind, "canonical", collection)
+ for collection in ("library", "selected")
+ for kind in ("vectors", "chunks")
+ ]
+
+
[email protected]("batch", [False, True], ids=["single", "history"])
+def test_combined_retry_refreshes_collection_targets(
+ database, route_session, monkeypatch, batch
+):
+ seed_family(database, references=False)
+ events = []
+ observe_actual_purge(monkeypatch, database, events)
+ original_lookup = research_cascade._shared_document_replacements
+ lookups = 0
+
+ def add_collection_after_lookup(*args):
+ nonlocal lookups
+ found = original_lookup(*args)
+ lookups += 1
+ if lookups == 1:
+ with Session(database) as writer:
+ writer.add(Collection(id="added", name="Added during deletion"))
+ writer.flush()
+ writer.add(
+ DocumentCollection(
+ document_id="canonical",
+ collection_id="added",
+ indexed=True,
+ chunk_count=1,
+ )
+ )
+ writer.add(
+ DocumentChunk(
+ chunk_hash="added",
+ source_type="document",
+ source_id="canonical",
+ collection_name="collection_added",
+ chunk_text=BODY,
+ chunk_index=0,
+ start_char=0,
+ end_char=len(BODY),
+ word_count=len(BODY.split()),
+ embedding_id="added",
+ embedding_model="test",
+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
+ )
+ )
+ writer.commit()
+ return found
+
+ monkeypatch.setattr(
+ research_cascade,
+ "_shared_document_replacements",
+ add_collection_after_lookup,
+ )
+ assert call_delete_route(batch) == {"status": "success"}
+ assert lookups == 2
+ assert events == [
+ (kind, "canonical", collection)
+ for collection in ("added", "library", "selected")
+ for kind in ("vectors", "chunks")
+ ]
+ with Session(database) as verification:
+ assert verification.get(ResearchHistory, "owner") is None
+ assert verification.get(Document, "canonical") is None
+ assert (
+ verification.query(DocumentChunk)
+ .filter_by(source_id="canonical")
+ .count()
+ == 0
+ )
+
+
[email protected]("batch", [False, True], ids=["single", "history"])
+def test_target_discovery_shares_the_cascade_snapshot(
+ database, route_session, monkeypatch, batch
+):
+ """A document committed after target discovery must not escape its purge.
+
+ Legacy SQLite SELECTs do not start a database transaction. If discovery
+ runs before BEGIN, the cascade can see and delete a newer document while
+ cleanup still has the older target list. A shared snapshot forces the
+ existing lock retry to rediscover the new document and its vector IDs.
+ """
+ from local_deep_research.web.routers import research
+
+ seed_family(database, references=False)
+ events = []
+ observe_actual_purge(monkeypatch, database, events)
+ collect = research.collect_research_document_rag_targets
+ snapshot_calls = 0
+
+ def collect_then_insert(*args):
+ nonlocal snapshot_calls
+ targets = collect(*args)
+ snapshot_calls += 1
+ if snapshot_calls == 1:
+ # A separate WAL writer commits a newly indexed document after
+ # the target SELECTs but before the cascade's reference lookup.
+ with Session(database) as writer:
+ writer.add(Collection(id="late", name="Late collection"))
+ writer.flush()
+ writer.add(
+ Document(
+ id="late-document",
+ source_type_id="source",
+ research_id="owner",
+ document_hash="late-document-hash",
+ file_type="text",
+ file_size=len(BODY),
+ text_content=BODY,
+ )
+ )
+ writer.flush()
+ writer.add(
+ DocumentChunk(
+ chunk_hash="late-document-chunk",
+ source_type="document",
+ source_id="late-document",
+ collection_name="collection_late",
+ chunk_text=BODY,
+ chunk_index=0,
+ start_char=0,
+ end_char=len(BODY),
+ word_count=len(BODY.split()),
+ embedding_id="late",
+ embedding_model="test",
+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
+ )
+ )
+ writer.commit()
+ return targets
+
+ monkeypatch.setattr(
+ research, "collect_research_document_rag_targets", collect_then_insert
+ )
+ assert call_delete_route(batch) == {"status": "success"}
+ with Session(database) as verification:
+ assert verification.get(ResearchHistory, "owner") is None
+ assert verification.get(Document, "late-document") is None
+ assert (
+ verification.query(DocumentChunk)
+ .filter_by(source_id="late-document")
+ .count()
+ == 0
+ ), "Deleted document retained its chunk text and vector ID"
+ assert snapshot_calls == 2
+ assert events == [
+ (kind, document_id, collection)
+ for document_id, collection in (
+ ("canonical", "library"),
+ ("canonical", "selected"),
+ ("late-document", "late"),
+ )
+ for kind in ("vectors", "chunks")
+ ]
+
+
[email protected]("nested", [False, True], ids=["outer", "savepoint"])
+def test_target_discovery_preserves_caller_transaction(database, nested):
+ """Discovery must neither restart nor commit an active transaction."""
+ seed_family(database, references=False)
+ with Session(database) as session:
+ session.get(ResearchHistory, "owner").query = "Uncommitted change"
+ session.flush()
+ with session.begin_nested() if nested else nullcontext():
+ session.get(Document, "canonical").title = "Uncommitted title"
+ session.flush()
+ assert research_cascade.collect_research_document_rag_targets(
+ session, "owner"
+ ) == [("canonical", ["library", "selected"])]
+ session.rollback()
+ with Session(database) as verification:
+ assert verification.get(ResearchHistory, "owner").query == "q"
+ assert verification.get(Document, "canonical").title == "Shared paper"
diff --git a/tests/web/routers/test_delete_research_cascade.py b/tests/web/routers/test_delete_research_cascade.py
index 0e8e0902de..f42418f62b 100644
--- a/tests/web/routers/test_delete_research_cascade.py
+++ b/tests/web/routers/test_delete_research_cascade.py
@@ -62,6 +62,8 @@ the production schema (same ``Base.metadata.create_all``) and enable
from contextlib import contextmanager
from datetime import UTC, datetime
from unittest.mock import patch
+import hashlib
+import uuid
import pytest
from sqlalchemy import create_engine, event
@@ -274,6 +276,95 @@ def _populate_all_child_tables(session, research_id):
}
+def _add_rag_chunk(session, document_id, collection_id):
+ from local_deep_research.database.models.library import (
+ DocumentChunk,
+ DocumentCollection,
+ EmbeddingProvider,
+ )
+
+ text = f"private chunk of {document_id}"
+ session.add(
+ DocumentCollection(
+ document_id=document_id,
+ collection_id=collection_id,
+ indexed=True,
+ chunk_count=1,
+ )
+ )
+ session.add(
+ DocumentChunk(
+ chunk_hash=hashlib.sha256(text.encode()).hexdigest(),
+ source_type="document",
+ source_id=document_id,
+ collection_name=f"collection_{collection_id}",
+ chunk_text=text,
+ chunk_index=0,
+ start_char=0,
+ end_char=len(text),
+ word_count=len(text.split()),
+ embedding_id=str(uuid.uuid4()),
+ embedding_model="fake-model",
+ embedding_model_type=EmbeddingProvider.SENTENCE_TRANSFORMERS,
+ embedding_dimension=8,
+ )
+ )
+ session.commit()
+
+
+@contextmanager
+def _patch_real_rag_purge(session, events):
+ """Run the production residual-chunk sweep with an inert vector store."""
+ from local_deep_research.database.models.library import (
+ Document,
+ DocumentChunk,
+ )
+
+ @contextmanager
+ def _ctx(*_args, **_kwargs):
+ yield session
+
+ class _Rag:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def purge_document_vectors(self, document_id, collection_id):
+ # Vector deletion must see the committed parent deletion while
+ # chunk rows still exist to resolve their FAISS IDs.
+ assert (
+ session.query(Document).filter_by(id=document_id).count() == 0
+ )
+ assert (
+ session.query(DocumentChunk)
+ .filter_by(source_id=document_id)
+ .count()
+ == 1
+ )
+ events.append(("vectors", document_id, collection_id))
+
+ def purge_document_chunks(self, document_id, collection_id):
+ events.append(("chunks", document_id, collection_id))
+
+ with (
+ patch(
+ "local_deep_research.research_library.deletion.services.document_deletion.get_user_db_session",
+ _ctx,
+ ),
+ patch(
+ "local_deep_research.research_library.deletion.services.document_deletion.capture_request_db_password",
+ return_value="test-password",
+ ),
+ patch(
+ "local_deep_research.research_library.services.rag_service_factory.get_rag_service",
+ side_effect=lambda *_args, **_kwargs: _Rag(),
+ ),
+ ):
+ yield
+
+
# ---------------------------------------------------------------------------
# Direct unit tests on the helper
# ---------------------------------------------------------------------------
@@ -750,6 +841,287 @@ class TestClearHistoryRouteWithFKEnforcement:
assert db_with_fk.query(ResearchResource).count() == 0
+class TestResearchRagPurgeRoutes:
+ """History deletion purges chunk text only after the guarded commit."""
+
+ def test_unverified_document_removal_does_not_purge_search_data(self):
+ from local_deep_research.research_library.deletion.services import (
+ document_deletion,
+ )
+
+ service = document_deletion.DocumentDeletionService("owner")
+ with (
+ patch.object(
+ document_deletion,
+ "get_user_db_session",
+ side_effect=RuntimeError("database unavailable"),
+ ),
+ patch.object(service, "_purge_document_rag") as purge,
+ ):
+ service.purge_deleted_research_documents([("doc", ["collection"])])
+ purge.assert_not_called()
+
+ @pytest.mark.parametrize("entrypoint", ["single", "history", "chat"])
+ def test_transferred_document_keeps_rag_until_last_reference_deleted(
+ self, authenticated_client, db_with_fk, entrypoint
+ ):
+ from local_deep_research.chat.service import ChatService
+ from local_deep_research.database.models import (
+ ChatSession,
+ Document,
+ DocumentChunk,
+ DocumentCollection,
+ ResearchHistory,
+ ResearchResource,
+ )
+
+ _add_research(db_with_fk, "owner")
+ owned = _populate_all_child_tables(db_with_fk, "owner")
+ document_id = owned["document"].id
+ collection_id = owned["collection"].id
+ _add_rag_chunk(db_with_fk, document_id, collection_id)
+ _add_research(db_with_fk, "survivor", status="in_progress")
+ survivor = ResearchResource(
+ research_id="survivor",
+ title="Shared document",
+ url="https://example.com/another-download-url",
+ document_id=document_id,
+ source_type="web",
+ created_at=_NOW,
+ )
+ db_with_fk.add(survivor)
+ db_with_fk.get(ResearchHistory, "owner").chat_session_id = "sess-owner"
+ db_with_fk.get(ChatSession, "sess-owner").message_count = 1
+ db_with_fk.commit()
+ resource_id = survivor.id
+ chunk = (
+ db_with_fk.query(DocumentChunk)
+ .filter_by(source_id=document_id)
+ .one()
+ )
+ snapshot = (chunk.id, chunk.chunk_text, chunk.embedding_id)
+ events = []
+
+ @contextmanager
+ def _chat_db(*_args, **_kwargs):
+ yield db_with_fk
+
+ with (
+ _patch_session(db_with_fk),
+ _patch_real_rag_purge(db_with_fk, events),
+ patch(f"{_RR}.get_active_research_ids", return_value=[]),
+ patch(_LIFECYCLE_CLEANUP),
+ patch(
+ "local_deep_research.chat.service.get_user_db_session", _chat_db
+ ),
+ patch("local_deep_research.chat.service.cleanup_research"),
+ ):
+ if entrypoint == "chat":
+ assert ChatService("owner").delete_attempt(
+ "sess-owner", "owner"
+ )
+ elif entrypoint == "history":
+ response = authenticated_client.post("/api/clear_history")
+ assert response.status_code == 200, response.text[:300]
+ else:
+ response = authenticated_client.delete("/api/delete/owner")
+ assert response.status_code == 200, response.text[:300]
+
+ db_with_fk.expire_all()
+ assert db_with_fk.get(ResearchHistory, "owner") is None
+ retained = db_with_fk.get(Document, document_id)
+ assert retained.research_id == "survivor"
+ assert retained.resource_id == resource_id
+ chunk = (
+ db_with_fk.query(DocumentChunk)
+ .filter_by(source_id=document_id)
+ .one()
+ )
+ assert (chunk.id, chunk.chunk_text, chunk.embedding_id) == snapshot
+ link = (
+ db_with_fk.query(DocumentCollection)
+ .filter_by(document_id=document_id, collection_id=collection_id)
+ .one()
+ )
+ assert link.indexed and link.chunk_count == 1
+ assert events == []
+
+ # Once the final owner is removed, the same data must be purged.
+ db_with_fk.get(ResearchHistory, "survivor").status = "completed"
+ db_with_fk.commit()
+ response = authenticated_client.delete("/api/delete/survivor")
+ assert response.status_code == 200, response.text[:300]
+
+ db_with_fk.expire_all()
+ assert db_with_fk.get(Document, document_id) is None
+ assert (
+ db_with_fk.query(DocumentChunk)
+ .filter_by(source_id=document_id)
+ .count()
+ == 0
+ )
+ assert events == [
+ ("vectors", document_id, collection_id),
+ ("chunks", document_id, collection_id),
+ ]
+
+ def test_single_delete_purges_owned_chunks_after_commit(
+ self, authenticated_client, db_with_fk
+ ):
+ from local_deep_research.database.models.library import (
+ Document,
+ DocumentChunk,
+ )
+
+ _add_research(db_with_fk, "done")
+ owned = _populate_all_child_tables(db_with_fk, "done")
+ document_id = owned["document"].id
+ collection_id = owned["collection"].id
+ _add_rag_chunk(db_with_fk, document_id, collection_id)
+ events = []
+
+ with (
+ _patch_session(db_with_fk),
+ _patch_real_rag_purge(db_with_fk, events),
+ patch(_LIFECYCLE_CLEANUP),
+ ):
+ response = authenticated_client.delete("/api/delete/done")
+
+ assert response.status_code == 200, response.text[:300]
+ assert db_with_fk.query(Document).filter_by(id=document_id).count() == 0
+ assert (
+ db_with_fk.query(DocumentChunk)
+ .filter_by(source_id=document_id)
+ .count()
+ == 0
+ ), "deleted research retained its private chunk text"
+ assert events == [
+ ("vectors", document_id, collection_id),
+ ("chunks", document_id, collection_id),
+ ]
+
+ def test_clear_history_purges_only_deleted_research_chunks(
+ self, authenticated_client, db_with_fk
+ ):
+ from local_deep_research.database.models.library import (
+ Document,
+ DocumentChunk,
+ )
+
+ _add_research(db_with_fk, "done")
+ done = _populate_all_child_tables(db_with_fk, "done")
+ done_id = done["document"].id
+ done_collection_id = done["collection"].id
+ _add_rag_chunk(db_with_fk, done_id, done_collection_id)
+
+ _add_research(db_with_fk, "live", status="in_progress")
+ live = _populate_all_child_tables(db_with_fk, "live")
+ live_id = live["document"].id
+ _add_rag_chunk(db_with_fk, live_id, live["collection"].id)
+ events = []
+
+ with (
+ _patch_session(db_with_fk),
+ _patch_real_rag_purge(db_with_fk, events),
+ patch(f"{_RR}.get_active_research_ids", return_value=[]),
+ patch(_LIFECYCLE_CLEANUP),
+ ):
+ response = authenticated_client.post("/api/clear_history")
+
+ assert response.status_code == 200, response.text[:300]
+ assert db_with_fk.query(Document).filter_by(id=done_id).count() == 0
+ assert (
+ db_with_fk.query(DocumentChunk).filter_by(source_id=done_id).count()
+ == 0
+ )
+ assert db_with_fk.query(Document).filter_by(id=live_id).count() == 1
+ assert (
+ db_with_fk.query(DocumentChunk).filter_by(source_id=live_id).count()
+ == 1
+ )
+ assert events == [
+ ("vectors", done_id, done_collection_id),
+ ("chunks", done_id, done_collection_id),
+ ]
+
+ def test_unavailable_rag_credentials_still_sweep_private_chunks(
+ self, authenticated_client, db_with_fk
+ ):
+ from local_deep_research.database.models.library import DocumentChunk
+
+ _add_research(db_with_fk, "done")
+ owned = _populate_all_child_tables(db_with_fk, "done")
+ document_id = owned["document"].id
+ _add_rag_chunk(db_with_fk, document_id, owned["collection"].id)
+ events = []
+
+ with (
+ _patch_session(db_with_fk),
+ _patch_real_rag_purge(db_with_fk, events),
+ patch(
+ "local_deep_research.research_library.deletion.services.document_deletion.capture_request_db_password",
+ side_effect=RuntimeError("unavailable"),
+ ),
+ patch(_LIFECYCLE_CLEANUP),
+ ):
+ response = authenticated_client.delete("/api/delete/done")
+
+ assert response.status_code == 200, response.text[:300]
+ assert events == []
+ assert (
+ db_with_fk.query(DocumentChunk)
+ .filter_by(source_id=document_id)
+ .count()
+ == 0
+ )
+
+ def test_later_failure_rolls_back_without_purging_chunks(
+ self, authenticated_client, db_with_fk
+ ):
+ from local_deep_research.database.models import ResearchHistory
+ from local_deep_research.database.models.library import (
+ Document,
+ DocumentChunk,
+ )
+
+ _add_research(db_with_fk, "done")
+ owned = _populate_all_child_tables(db_with_fk, "done")
+ document_id = owned["document"].id
+ _add_rag_chunk(db_with_fk, document_id, owned["collection"].id)
+
+ @contextmanager
+ def _rollback_on_error(*_args, **_kwargs):
+ try:
+ yield db_with_fk
+ except Exception:
+ db_with_fk.rollback()
+ raise
+
+ with (
+ patch(f"{_RR}.get_user_db_session", _rollback_on_error),
+ patch(
+ _LIFECYCLE_CLEANUP, side_effect=RuntimeError("later failure")
+ ),
+ patch(
+ "local_deep_research.research_library.deletion.services.document_deletion.DocumentDeletionService.purge_deleted_research_documents"
+ ) as purge,
+ ):
+ response = authenticated_client.delete("/api/delete/done")
+
+ assert response.status_code == 500
+ assert (
+ db_with_fk.query(ResearchHistory).filter_by(id="done").count() == 1
+ )
+ assert db_with_fk.query(Document).filter_by(id=document_id).count() == 1
+ assert (
+ db_with_fk.query(DocumentChunk)
+ .filter_by(source_id=document_id)
+ .count()
+ == 1
+ )
+ purge.assert_not_called()
+
+
class TestClearHistorySharedDownloadIndex:
"""``clear_history`` must build one ``SharedDownloadIndex`` for the
whole batch and hand it to every per-research delete; otherwise each
diff --git a/tests/web/routers/test_research_mutating_route_effects.py b/tests/web/routers/test_research_mutating_route_effects.py
index f40381197b..6ed5cb36ec 100644
--- a/tests/web/routers/test_research_mutating_route_effects.py
+++ b/tests/web/routers/test_research_mutating_route_effects.py
@@ -173,6 +173,36 @@ def test_clear_history_protects_an_in_progress_row(authenticated_client, db):
assert _ids(db) == {"live"}
+def test_clear_history_removes_only_deleted_research_counters(
+ authenticated_client, db
+):
+ """A cleared row must not keep occupying a per-user research slot."""
+ from local_deep_research.database.models import UserActiveResearch
+
+ _add_research(db, "done", status="completed")
+ _add_research(db, "live", status="in_progress")
+ db.add_all(
+ [
+ UserActiveResearch(username="owner", research_id="done"),
+ UserActiveResearch(username="owner", research_id="live"),
+ ]
+ )
+ db.commit()
+
+ with (
+ _patch_session(db),
+ patch(f"{_RR}.get_active_research_ids", return_value=[]),
+ patch(_LIFECYCLE_CLEANUP),
+ ):
+ response = authenticated_client.post("/api/clear_history")
+
+ assert response.status_code == 200, response.text[:300]
+ assert _ids(db) == {"live"}
+ assert {row.research_id for row in db.query(UserActiveResearch).all()} == {
+ "live"
+ }
+
+
# ---------------------------------------------------------------------------
# DELETE /api/delete/{research_id}
# ---------------------------------------------------------------------------