Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: delete a chat's uploaded documents when the chat is deleted (#11233)

Nilayfd51e602d152
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.py
index 0e741cbb3d8..df12d7f23f3 100644
--- a/studio/backend/core/rag/conversation_archive.py
+++ b/studio/backend/core/rag/conversation_archive.py
@@ -2481,11 +2481,11 @@ def recall(
def _scope_select(scope: str, created_before: Optional[str]) -> tuple:
- """The scope's document ids, optionally cut at ``created_before``. See `delete_for_thread`."""
+ """The scope's documents, optionally cut at ``created_before`` (see `delete_for_thread`)."""
if not created_before:
- return ("SELECT id FROM documents WHERE scope=?", (scope,))
+ return ("SELECT id, stored_path FROM documents WHERE scope=?", (scope,))
return (
- "SELECT id FROM documents WHERE scope=? AND created_at<?",
+ "SELECT id, stored_path FROM documents WHERE scope=? AND created_at<?",
(scope, created_before),
)
@@ -2495,35 +2495,33 @@ def _delete_scope_without_vec(
thread_id: str,
*,
created_before: Optional[str] = None,
-) -> int:
+) -> list:
"""Delete a scope's text-bearing rows over a connection with no sqlite-vec.
- Deletion must not depend on the optional native extension. Archives are only WRITTEN while vec0
+ Deletion must not depend on the optional native extension. Rows are only WRITTEN while vec0
loads, but it can stop loading afterwards (a venv change, common on macOS), and a delete that
- silently does nothing leaves a deleted conversation on disk ready to answer again once vec0
- returns.
+ silently does nothing leaves a deleted chat's turns and uploads on disk, retrievable again once
+ vec0 returns.
The chunks_vec rows are unreachable from here and left behind. They carry vectors, not text, and
every read path resolves through ``chunks`` joined to ``documents``, both gone, so nothing can
retrieve an orphan.
"""
conn = None
- removed = 0
+ removed = []
try:
conn = rag_db.get_metadata_connection()
- documents = [
- row["id"] for row in conn.execute(*_scope_select(scope, created_before)).fetchall()
- ]
- for document_id in documents:
+ documents = conn.execute(*_scope_select(scope, created_before)).fetchall()
+ for row in documents:
conn.execute(
"DELETE FROM chunks_fts WHERE chunk_id IN "
"(SELECT id FROM chunks WHERE document_id=?)",
- (document_id,),
+ (row["id"],),
)
- conn.execute("DELETE FROM chunks WHERE document_id=?", (document_id,))
- conn.execute("DELETE FROM documents WHERE id=?", (document_id,))
- removed += 1
+ conn.execute("DELETE FROM chunks WHERE document_id=?", (row["id"],))
+ conn.execute("DELETE FROM documents WHERE id=?", (row["id"],))
conn.commit()
+ removed = [row["stored_path"] for row in documents]
except Exception:
logger.warning(
"conversation_archive.delete_without_vec_failed thread_id=%s", thread_id, exc_info = True
@@ -2550,8 +2548,30 @@ def delete_for_thread(thread_id: str, *, created_before: Optional[str] = None) -
if not thread_id:
return 0
scope = store.conversation_archive_scope(thread_id)
+ return len(_delete_scope(scope, thread_id, created_before = created_before))
+
+
+def delete_thread_documents(thread_id: str, *, created_before: Optional[str] = None) -> int:
+ """Drop a thread's uploaded documents and their stored files, cut as in `delete_for_thread`."""
+ if not thread_id:
+ return 0
+ from .ingestion import _remove_upload
+
+ scope = store.thread_scope(thread_id)
+ removed = _delete_scope(scope, thread_id, created_before = created_before)
+ for stored_path in removed:
+ _remove_upload(stored_path)
+ return len(removed)
+
+
+def _delete_scope(
+ scope: str,
+ thread_id: str,
+ *,
+ created_before: Optional[str] = None,
+) -> list:
conn = None
- removed = 0
+ removed = []
try:
try:
conn = rag_db.get_connection()
@@ -2560,7 +2580,7 @@ def delete_for_thread(thread_id: str, *, created_before: Optional[str] = None) -
return _delete_scope_without_vec(scope, thread_id, created_before = created_before)
for row in conn.execute(*_scope_select(scope, created_before)).fetchall():
store.delete_document(conn, row["id"])
- removed += 1
+ removed.append(row["stored_path"])
except Exception:
logger.warning("conversation_archive.delete_failed thread_id=%s", thread_id, exc_info = True)
finally:
diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py
index 39bd8c891b7..61f403afff7 100644
--- a/studio/backend/routes/chat_history.py
+++ b/studio/backend/routes/chat_history.py
@@ -823,9 +823,9 @@ async def delete_threads(
# Keyed by thread id, so the folder is unreachable once the thread is gone; done in a worker because the
# post-upgrade legacy move can be a cross-filesystem copy.
removed, kept = await _remove_sandboxes(payload.ids, payload.delete_files)
- # Archived turns are keyed by thread id and unreferenced once the thread is gone, so
- # drop them rather than leaking a scope per deleted chat.
- await run_in_threadpool(_remove_conversation_archives, payload.ids, cutoff = cutoff)
+ # Archived turns and uploaded documents are keyed by thread id and unreferenced once the thread
+ # is gone, so drop them rather than leaking scopes per deleted chat.
+ await run_in_threadpool(_remove_thread_rag_data, payload.ids, cutoff = cutoff)
return {"status": "deleted", "sandboxes_removed": removed, "sandboxes_kept": kept}
@@ -835,25 +835,30 @@ def _archive_cutoff() -> str:
return datetime.now(timezone.utc).isoformat()
-def _remove_conversation_archives(thread_ids, *, cutoff: "str | None" = None) -> None:
- """Drop each deleted thread's archived turns. Never raises."""
+def _remove_thread_rag_data(thread_ids, *, cutoff: "str | None" = None) -> None:
+ """Drop each deleted thread's archived turns and uploaded documents. Never raises."""
try:
from core.rag import conversation_archive
except Exception:
return
for thread_id in thread_ids or []:
# Cut at the instant the delete was accepted, not on recreation: another tab can have recreated this
- # id, and skipping the scope left the deleted conversation recallable. Everything archived before
+ # id, and skipping the scope left the deleted conversation recallable. Everything stored before
# that instant belongs to the deleted conversation, everything after to the new one.
recreated = get_chat_thread(str(thread_id)) is not None
if recreated and not cutoff:
continue
+ created_before = cutoff if recreated else None
try:
- conversation_archive.delete_for_thread(
- str(thread_id), created_before = cutoff if recreated else None
- )
+ conversation_archive.delete_for_thread(str(thread_id), created_before = created_before)
except Exception:
logger.warning("Could not remove the conversation archive for %s", thread_id)
+ try:
+ conversation_archive.delete_thread_documents(
+ str(thread_id), created_before = created_before
+ )
+ except Exception:
+ logger.warning("Could not remove the uploaded documents for %s", thread_id)
async def _remove_sandboxes(thread_ids, delete_files: bool) -> "tuple[int, list[str]]":
@@ -1174,8 +1179,8 @@ async def delete_project(
await run_in_threadpool(_delete_project_rag_sources, project_id)
except Exception: # noqa: BLE001 - source cleanup must not block project deletion
logger.warning("failed to delete RAG sources for project %s", project_id, exc_info = True)
- # The project's chats go with it, so their archives have to as well.
- await run_in_threadpool(_remove_conversation_archives, member_ids, cutoff = cutoff)
+ # The project's chats go with it, so their archives and documents have to as well.
+ await run_in_threadpool(_remove_thread_rag_data, member_ids, cutoff = cutoff)
if project.get("sandboxPath"):
from core.inference.tools import (
finish_workspace_delete_when_idle,
@@ -1513,10 +1518,10 @@ async def clear_history(
# By id: the rows went with the threads, so nothing can look them up now.
_cancel_research_runs(request, cleared_runs)
_cancel_chat_generation_runs(request, cleared_chat_runs)
- # Same archive cleanup as DELETE /threads. Without it "Clear all chats" leaves every
- # conversation searchable in rag.db, and a reused thread id reads the old archive.
+ # Same cleanup as DELETE /threads. Without it "Clear all chats" leaves every conversation
+ # searchable in rag.db, and a reused thread id reads the old archive.
await run_in_threadpool(
- _remove_conversation_archives, list(dict.fromkeys(thread_ids + cleared)), cutoff = cutoff
+ _remove_thread_rag_data, list(dict.fromkeys(thread_ids + cleared)), cutoff = cutoff
)
# "Clear all chats" is the common bulk delete.
# delete_files matches DELETE /threads: off by default, since the files are the user's.
diff --git a/studio/backend/tests/test_account_chat_settings.py b/studio/backend/tests/test_account_chat_settings.py
index eec9faf4adf..27a14ea37c4 100644
--- a/studio/backend/tests/test_account_chat_settings.py
+++ b/studio/backend/tests/test_account_chat_settings.py
@@ -87,7 +87,7 @@ def client(monkeypatch):
return 0, []
monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)
- monkeypatch.setattr(chat_history, "_remove_conversation_archives", lambda *args, **kwargs: None)
+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda *args, **kwargs: None)
with TestClient(app) as test_client:
yield test_client
diff --git a/studio/backend/tests/test_chat_history_routes.py b/studio/backend/tests/test_chat_history_routes.py
index d07b337b1c4..e18dbee1d69 100644
--- a/studio/backend/tests/test_chat_history_routes.py
+++ b/studio/backend/tests/test_chat_history_routes.py
@@ -287,9 +287,7 @@ def test_clear_history_reaps_search_thumbnails_with_a_body(monkeypatch):
monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)
monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)
monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)
- monkeypatch.setattr(
- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None
- )
+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)
request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))
asyncio.run(
@@ -939,9 +937,7 @@ def test_a_clear_does_not_reap_an_image_registered_while_it_was_running(tmp_path
monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)
monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)
monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)
- monkeypatch.setattr(
- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None
- )
+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)
request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))
studio_db.upsert_chat_thread(_clear_thread_row("before-clear"))
@@ -989,9 +985,7 @@ def test_replayed_clear_keeps_the_thumbnails_of_a_chat_it_did_not_delete(tmp_pat
monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)
monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)
monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)
- monkeypatch.setattr(
- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None
- )
+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)
request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))
def clear():
@@ -1051,9 +1045,7 @@ def test_the_replay_bit_comes_from_the_clear_transaction(monkeypatch, tmp_path):
monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)
monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)
monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)
- monkeypatch.setattr(
- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None
- )
+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)
request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))
studio_db.upsert_chat_thread(_clear_thread_row("before-clear"))
@@ -1141,9 +1133,7 @@ def test_a_chat_created_in_the_gap_after_the_clear_keeps_its_images(monkeypatch,
monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)
monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)
monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)
- monkeypatch.setattr(
- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None
- )
+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)
# The other tab's image, registered in the gap. Straight into the registry: this is about
# WHEN the id becomes visible to the snapshot, not about how it got there.
@@ -1250,9 +1240,7 @@ def test_a_replay_finishes_a_reap_the_original_clear_died_before_running(monkeyp
monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)
monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)
monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)
- monkeypatch.setattr(
- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None
- )
+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)
request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))
def clear():
@@ -1314,9 +1302,7 @@ def test_a_plain_replay_with_nothing_outstanding_still_reaps_nothing(monkeypatch
monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)
monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)
monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)
- monkeypatch.setattr(
- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None
- )
+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)
request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))
def clear():
diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.py
index 7e5d7f4ed56..f581599a177 100644
--- a/studio/backend/tests/test_conversation_archive.py
+++ b/studio/backend/tests/test_conversation_archive.py
@@ -1435,13 +1435,13 @@ def test_the_late_archive_cleanup_spares_a_recreated_thread(conn):
_save_thread(thread_id, turns, append = True)
assert conversation_archive.archive_turns(thread_id, turns) == 1
- chat_history._remove_conversation_archives([thread_id])
+ chat_history._remove_thread_rag_data([thread_id])
assert conversation_archive.has_archive(thread_id) is True
# And a thread that really is gone still has its archive dropped.
studio_db.delete_chat_threads([thread_id])
- chat_history._remove_conversation_archives([thread_id])
+ chat_history._remove_thread_rag_data([thread_id])
assert conversation_archive.has_archive(thread_id) is False
@@ -3517,7 +3517,7 @@ def test_the_deleted_conversation_goes_even_when_its_id_comes_back(conn):
_save_thread(thread_id, old_turns + fresh, append = True)
assert conversation_archive.archive_turns(thread_id, fresh) == 1
- chat_history._remove_conversation_archives([thread_id], cutoff = cutoff)
+ chat_history._remove_thread_rag_data([thread_id], cutoff = cutoff)
scope = store.conversation_archive_scope(thread_id)
remaining = " ".join(
diff --git a/studio/backend/tests/test_rag_thread_document_cleanup.py b/studio/backend/tests/test_rag_thread_document_cleanup.py
new file mode 100644
index 00000000000..e9431723cd8
--- /dev/null
+++ b/studio/backend/tests/test_rag_thread_document_cleanup.py
@@ -0,0 +1,174 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import os
+import time
+from datetime import datetime, timezone
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from auth.authentication import get_current_subject
+from core.rag import ingestion, store
+from routes import chat_history, rag as rag_routes
+from storage import rag_db
+
+
+def client(rag_home, stub_embeddings):
+ app = FastAPI()
+ app.include_router(chat_history.router, prefix = "/api/chat")
+ app.include_router(rag_routes.router, prefix = "/api/rag")
+ app.dependency_overrides[get_current_subject] = lambda: "tester"
+ return TestClient(app)
+
+
+def _create_thread(client, thread_id):
+ response = client.post(
+ "/api/chat/threads",
+ json = {"id": thread_id, "title": "t", "modelType": "base", "createdAt": 1},
+ )
+ assert response.status_code == 200, response.text
+
+
+def _upload(client, thread_id, name, text):
+ response = client.post(
+ f"/api/rag/threads/{thread_id}/documents",
+ files = {"file": (name, text.encode("utf-8"), "text/plain")},
+ )
+ assert response.status_code == 200, response.text
+ body = response.json()
+ deadline = time.time() + 30
+ while time.time() < deadline:
+ status = ingestion.get_job_status(body["jobId"])
+ if status and status["status"] in ("completed", "failed"):
+ break
+ time.sleep(0.05)
+ assert status["status"] == "completed"
+ return body["documentId"]
+
+
+def _stored_path(document_id):
+ conn = rag_db.get_connection()
+ try:
+ return store.get_document(conn, document_id)["stored_path"]
+ finally:
+ conn.close()
+
+
+def _document_ids(client):
+ response = client.get("/api/rag/documents")
+ assert response.status_code == 200, response.text
+ return {doc["id"] for doc in response.json()["documents"]}
+
+
+def _chunk_count(thread_id):
+ conn = rag_db.get_metadata_connection()
+ try:
+ scope = store.thread_scope(thread_id)
+ chunks = conn.execute("SELECT COUNT(*) FROM chunks WHERE scope=?", (scope,)).fetchone()[0]
+ fts = conn.execute("SELECT COUNT(*) FROM chunks_fts WHERE scope=?", (scope,)).fetchone()[0]
+ return chunks + fts
+ finally:
+ conn.close()
+
+
+def test_deleting_a_thread_removes_its_uploaded_documents(client):
+ _create_thread(client, "doomed")
+ _create_thread(client, "kept")
+ doomed = [
+ _upload(client, "doomed", f"doomed{i}.txt", f"alpha bravo charlie {i} " * 50)
+ for i in range(2)
+ ]
+ kept = _upload(client, "kept", "kept.txt", "delta echo foxtrot " * 50)
+ doomed_paths, kept_path = [_stored_path(d) for d in doomed], _stored_path(kept)
+ assert all(os.path.isfile(path) for path in doomed_paths)
+
+ response = client.request("DELETE", "/api/chat/threads", json = {"ids": ["doomed"]})
+
+ assert response.status_code == 200, response.text
+ assert _document_ids(client) == {kept}
+ assert _chunk_count("doomed") == 0
+ assert not any(os.path.exists(path) for path in doomed_paths)
+ assert os.path.isfile(kept_path)
+ assert _chunk_count("kept") > 0
+
+
+def test_clearing_history_removes_every_threads_uploaded_documents(client):
+ paths = []
+ for thread_id in ("first", "second"):
+ _create_thread(client, thread_id)
+ paths.append(
+ _stored_path(_upload(client, thread_id, f"{thread_id}.txt", f"{thread_id} words " * 50))
+ )
+
+ response = client.request("DELETE", "/api/chat")
+
+ assert response.status_code == 200, response.text
+ assert _document_ids(client) == set()
+ assert not any(os.path.exists(path) for path in paths)
+
+
+def test_deleting_a_project_removes_its_member_threads_documents(client):
+ response = client.post(
+ "/api/chat/projects", json = {"id": "proj", "name": "p", "createdAt": 1, "updatedAt": 1}
+ )
+ assert response.status_code == 200, response.text
+ response = client.post(
+ "/api/chat/threads",
+ json = {
+ "id": "member",
+ "title": "t",
+ "modelType": "base",
+ "createdAt": 1,
+ "projectId": "proj",
+ },
+ )
+ assert response.status_code == 200, response.text
+ path = _stored_path(_upload(client, "member", "member.txt", "golf hotel india " * 50))
+ _create_thread(client, "outsider")
+ outsider = _upload(client, "outsider", "outsider.txt", "sierra tango uniform " * 50)
+
+ response = client.delete("/api/chat/projects/proj")
+
+ assert response.status_code == 200, response.text
+ assert _document_ids(client) == {outsider}
+ assert not os.path.exists(path)
+
+
+def test_a_recreated_thread_keeps_documents_uploaded_after_the_cutoff(client):
+ _create_thread(client, "recreated")
+ old = _upload(client, "recreated", "old.txt", "juliet kilo lima " * 50)
+ old_path = _stored_path(old)
+ cutoff = datetime.now(timezone.utc).isoformat()
+ fresh = _upload(client, "recreated", "fresh.txt", "mike november oscar " * 50)
+
+ chat_history._remove_thread_rag_data(["recreated"])
+ assert _document_ids(client) == {old, fresh}
+
+ chat_history._remove_thread_rag_data(["recreated"], cutoff = cutoff)
+
+ assert _document_ids(client) == {fresh}
+ assert not os.path.exists(old_path)
+ assert os.path.isfile(_stored_path(fresh))
+
+
+def test_thread_documents_go_without_sqlite_vec(client, monkeypatch):
+ from storage import studio_db
+
+ _create_thread(client, "vecless")
+ document_id = _upload(client, "vecless", "vecless.txt", "papa quebec romeo " * 50)
+ path = _stored_path(document_id)
+ studio_db.delete_chat_threads(["vecless"])
+
+ def no_vec():
+ raise rag_db.RagExtensionUnavailable("vec0 will not load")
+
+ with monkeypatch.context() as patch:
+ patch.setattr(rag_db, "get_connection", no_vec)
+ chat_history._remove_thread_rag_data(["vecless"])
+
+ assert _document_ids(client) == set()
+ assert _chunk_count("vecless") == 0
+ assert not os.path.exists(path)