Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: delete a chat's uploaded documents when the chat is deleted (#11233)
Nilayfd51e602d152
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.pyindex 0e741cbb3d8..df12d7f23f3 100644--- a/studio/backend/core/rag/conversation_archive.py+++ b/studio/backend/core/rag/conversation_archive.py@@ -2481,11 +2481,11 @@ def recall(def _scope_select(scope: str, created_before: Optional[str]) -> tuple:- """The scope's document ids, optionally cut at ``created_before``. See `delete_for_thread`."""+ """The scope's documents, optionally cut at ``created_before`` (see `delete_for_thread`)."""if not created_before:- return ("SELECT id FROM documents WHERE scope=?", (scope,))+ return ("SELECT id, stored_path FROM documents WHERE scope=?", (scope,))return (- "SELECT id FROM documents WHERE scope=? AND created_at<?",+ "SELECT id, stored_path FROM documents WHERE scope=? AND created_at<?",(scope, created_before),)@@ -2495,35 +2495,33 @@ def _delete_scope_without_vec(thread_id: str,*,created_before: Optional[str] = None,-) -> int:+) -> list:"""Delete a scope's text-bearing rows over a connection with no sqlite-vec.- Deletion must not depend on the optional native extension. Archives are only WRITTEN while vec0+ Deletion must not depend on the optional native extension. Rows are only WRITTEN while vec0loads, but it can stop loading afterwards (a venv change, common on macOS), and a delete that- silently does nothing leaves a deleted conversation on disk ready to answer again once vec0- returns.+ silently does nothing leaves a deleted chat's turns and uploads on disk, retrievable again once+ vec0 returns.The chunks_vec rows are unreachable from here and left behind. They carry vectors, not text, andevery read path resolves through ``chunks`` joined to ``documents``, both gone, so nothing canretrieve an orphan."""conn = None- removed = 0+ removed = []try:conn = rag_db.get_metadata_connection()- documents = [- row["id"] for row in conn.execute(*_scope_select(scope, created_before)).fetchall()- ]- for document_id in documents:+ documents = conn.execute(*_scope_select(scope, created_before)).fetchall()+ for row in documents:conn.execute("DELETE FROM chunks_fts WHERE chunk_id IN ""(SELECT id FROM chunks WHERE document_id=?)",- (document_id,),+ (row["id"],),)- conn.execute("DELETE FROM chunks WHERE document_id=?", (document_id,))- conn.execute("DELETE FROM documents WHERE id=?", (document_id,))- removed += 1+ conn.execute("DELETE FROM chunks WHERE document_id=?", (row["id"],))+ conn.execute("DELETE FROM documents WHERE id=?", (row["id"],))conn.commit()+ removed = [row["stored_path"] for row in documents]except Exception:logger.warning("conversation_archive.delete_without_vec_failed thread_id=%s", thread_id, exc_info = True@@ -2550,8 +2548,30 @@ def delete_for_thread(thread_id: str, *, created_before: Optional[str] = None) -if not thread_id:return 0scope = store.conversation_archive_scope(thread_id)+ return len(_delete_scope(scope, thread_id, created_before = created_before))+++def delete_thread_documents(thread_id: str, *, created_before: Optional[str] = None) -> int:+ """Drop a thread's uploaded documents and their stored files, cut as in `delete_for_thread`."""+ if not thread_id:+ return 0+ from .ingestion import _remove_upload++ scope = store.thread_scope(thread_id)+ removed = _delete_scope(scope, thread_id, created_before = created_before)+ for stored_path in removed:+ _remove_upload(stored_path)+ return len(removed)+++def _delete_scope(+ scope: str,+ thread_id: str,+ *,+ created_before: Optional[str] = None,+) -> list:conn = None- removed = 0+ removed = []try:try:conn = rag_db.get_connection()@@ -2560,7 +2580,7 @@ def delete_for_thread(thread_id: str, *, created_before: Optional[str] = None) -return _delete_scope_without_vec(scope, thread_id, created_before = created_before)for row in conn.execute(*_scope_select(scope, created_before)).fetchall():store.delete_document(conn, row["id"])- removed += 1+ removed.append(row["stored_path"])except Exception:logger.warning("conversation_archive.delete_failed thread_id=%s", thread_id, exc_info = True)finally:diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.pyindex 39bd8c891b7..61f403afff7 100644--- a/studio/backend/routes/chat_history.py+++ b/studio/backend/routes/chat_history.py@@ -823,9 +823,9 @@ async def delete_threads(# Keyed by thread id, so the folder is unreachable once the thread is gone; done in a worker because the# post-upgrade legacy move can be a cross-filesystem copy.removed, kept = await _remove_sandboxes(payload.ids, payload.delete_files)- # Archived turns are keyed by thread id and unreferenced once the thread is gone, so- # drop them rather than leaking a scope per deleted chat.- await run_in_threadpool(_remove_conversation_archives, payload.ids, cutoff = cutoff)+ # Archived turns and uploaded documents are keyed by thread id and unreferenced once the thread+ # is gone, so drop them rather than leaking scopes per deleted chat.+ await run_in_threadpool(_remove_thread_rag_data, payload.ids, cutoff = cutoff)return {"status": "deleted", "sandboxes_removed": removed, "sandboxes_kept": kept}@@ -835,25 +835,30 @@ def _archive_cutoff() -> str:return datetime.now(timezone.utc).isoformat()-def _remove_conversation_archives(thread_ids, *, cutoff: "str | None" = None) -> None:- """Drop each deleted thread's archived turns. Never raises."""+def _remove_thread_rag_data(thread_ids, *, cutoff: "str | None" = None) -> None:+ """Drop each deleted thread's archived turns and uploaded documents. Never raises."""try:from core.rag import conversation_archiveexcept Exception:returnfor thread_id in thread_ids or []:# Cut at the instant the delete was accepted, not on recreation: another tab can have recreated this- # id, and skipping the scope left the deleted conversation recallable. Everything archived before+ # id, and skipping the scope left the deleted conversation recallable. Everything stored before# that instant belongs to the deleted conversation, everything after to the new one.recreated = get_chat_thread(str(thread_id)) is not Noneif recreated and not cutoff:continue+ created_before = cutoff if recreated else Nonetry:- conversation_archive.delete_for_thread(- str(thread_id), created_before = cutoff if recreated else None- )+ conversation_archive.delete_for_thread(str(thread_id), created_before = created_before)except Exception:logger.warning("Could not remove the conversation archive for %s", thread_id)+ try:+ conversation_archive.delete_thread_documents(+ str(thread_id), created_before = created_before+ )+ except Exception:+ logger.warning("Could not remove the uploaded documents for %s", thread_id)async def _remove_sandboxes(thread_ids, delete_files: bool) -> "tuple[int, list[str]]":@@ -1174,8 +1179,8 @@ async def delete_project(await run_in_threadpool(_delete_project_rag_sources, project_id)except Exception: # noqa: BLE001 - source cleanup must not block project deletionlogger.warning("failed to delete RAG sources for project %s", project_id, exc_info = True)- # The project's chats go with it, so their archives have to as well.- await run_in_threadpool(_remove_conversation_archives, member_ids, cutoff = cutoff)+ # The project's chats go with it, so their archives and documents have to as well.+ await run_in_threadpool(_remove_thread_rag_data, member_ids, cutoff = cutoff)if project.get("sandboxPath"):from core.inference.tools import (finish_workspace_delete_when_idle,@@ -1513,10 +1518,10 @@ async def clear_history(# By id: the rows went with the threads, so nothing can look them up now._cancel_research_runs(request, cleared_runs)_cancel_chat_generation_runs(request, cleared_chat_runs)- # Same archive cleanup as DELETE /threads. Without it "Clear all chats" leaves every- # conversation searchable in rag.db, and a reused thread id reads the old archive.+ # Same cleanup as DELETE /threads. Without it "Clear all chats" leaves every conversation+ # searchable in rag.db, and a reused thread id reads the old archive.await run_in_threadpool(- _remove_conversation_archives, list(dict.fromkeys(thread_ids + cleared)), cutoff = cutoff+ _remove_thread_rag_data, list(dict.fromkeys(thread_ids + cleared)), cutoff = cutoff)# "Clear all chats" is the common bulk delete.# delete_files matches DELETE /threads: off by default, since the files are the user's.diff --git a/studio/backend/tests/test_account_chat_settings.py b/studio/backend/tests/test_account_chat_settings.pyindex eec9faf4adf..27a14ea37c4 100644--- a/studio/backend/tests/test_account_chat_settings.py+++ b/studio/backend/tests/test_account_chat_settings.py@@ -87,7 +87,7 @@ def client(monkeypatch):return 0, []monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)- monkeypatch.setattr(chat_history, "_remove_conversation_archives", lambda *args, **kwargs: None)+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda *args, **kwargs: None)with TestClient(app) as test_client:yield test_clientdiff --git a/studio/backend/tests/test_chat_history_routes.py b/studio/backend/tests/test_chat_history_routes.pyindex d07b337b1c4..e18dbee1d69 100644--- a/studio/backend/tests/test_chat_history_routes.py+++ b/studio/backend/tests/test_chat_history_routes.py@@ -287,9 +287,7 @@ def test_clear_history_reaps_search_thumbnails_with_a_body(monkeypatch):monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)- monkeypatch.setattr(- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None- )+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))asyncio.run(@@ -939,9 +937,7 @@ def test_a_clear_does_not_reap_an_image_registered_while_it_was_running(tmp_pathmonkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)- monkeypatch.setattr(- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None- )+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))studio_db.upsert_chat_thread(_clear_thread_row("before-clear"))@@ -989,9 +985,7 @@ def test_replayed_clear_keeps_the_thumbnails_of_a_chat_it_did_not_delete(tmp_patmonkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)- monkeypatch.setattr(- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None- )+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))def clear():@@ -1051,9 +1045,7 @@ def test_the_replay_bit_comes_from_the_clear_transaction(monkeypatch, tmp_path):monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)- monkeypatch.setattr(- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None- )+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))studio_db.upsert_chat_thread(_clear_thread_row("before-clear"))@@ -1141,9 +1133,7 @@ def test_a_chat_created_in_the_gap_after_the_clear_keeps_its_images(monkeypatch,monkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)- monkeypatch.setattr(- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None- )+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)# The other tab's image, registered in the gap. Straight into the registry: this is about# WHEN the id becomes visible to the snapshot, not about how it got there.@@ -1250,9 +1240,7 @@ def test_a_replay_finishes_a_reap_the_original_clear_died_before_running(monkeypmonkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)- monkeypatch.setattr(- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None- )+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))def clear():@@ -1314,9 +1302,7 @@ def test_a_plain_replay_with_nothing_outstanding_still_reaps_nothing(monkeypatchmonkeypatch.setattr(chat_history, "_remove_sandboxes", remove_sandboxes)monkeypatch.setattr(chat_history, "_cancel_active_generations", lambda _ids: None)monkeypatch.setattr(chat_history, "_cancel_research_runs", lambda _request, _ids: None)- monkeypatch.setattr(- chat_history, "_remove_conversation_archives", lambda _ids, cutoff = None: None- )+ monkeypatch.setattr(chat_history, "_remove_thread_rag_data", lambda _ids, cutoff = None: None)request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace()))def clear():diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.pyindex 7e5d7f4ed56..f581599a177 100644--- a/studio/backend/tests/test_conversation_archive.py+++ b/studio/backend/tests/test_conversation_archive.py@@ -1435,13 +1435,13 @@ def test_the_late_archive_cleanup_spares_a_recreated_thread(conn):_save_thread(thread_id, turns, append = True)assert conversation_archive.archive_turns(thread_id, turns) == 1- chat_history._remove_conversation_archives([thread_id])+ chat_history._remove_thread_rag_data([thread_id])assert conversation_archive.has_archive(thread_id) is True# And a thread that really is gone still has its archive dropped.studio_db.delete_chat_threads([thread_id])- chat_history._remove_conversation_archives([thread_id])+ chat_history._remove_thread_rag_data([thread_id])assert conversation_archive.has_archive(thread_id) is False@@ -3517,7 +3517,7 @@ def test_the_deleted_conversation_goes_even_when_its_id_comes_back(conn):_save_thread(thread_id, old_turns + fresh, append = True)assert conversation_archive.archive_turns(thread_id, fresh) == 1- chat_history._remove_conversation_archives([thread_id], cutoff = cutoff)+ chat_history._remove_thread_rag_data([thread_id], cutoff = cutoff)scope = store.conversation_archive_scope(thread_id)remaining = " ".join(diff --git a/studio/backend/tests/test_rag_thread_document_cleanup.py b/studio/backend/tests/test_rag_thread_document_cleanup.pynew file mode 100644index 00000000000..e9431723cd8--- /dev/null+++ b/studio/backend/tests/test_rag_thread_document_cleanup.py@@ -0,0 +1,174 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import os+import time+from datetime import datetime, timezone++import pytest+from fastapi import FastAPI+from fastapi.testclient import TestClient++from auth.authentication import get_current_subject+from core.rag import ingestion, store+from routes import chat_history, rag as rag_routes+from storage import rag_db+++def client(rag_home, stub_embeddings):+ app = FastAPI()+ app.include_router(chat_history.router, prefix = "/api/chat")+ app.include_router(rag_routes.router, prefix = "/api/rag")+ app.dependency_overrides[get_current_subject] = lambda: "tester"+ return TestClient(app)+++def _create_thread(client, thread_id):+ response = client.post(+ "/api/chat/threads",+ json = {"id": thread_id, "title": "t", "modelType": "base", "createdAt": 1},+ )+ assert response.status_code == 200, response.text+++def _upload(client, thread_id, name, text):+ response = client.post(+ f"/api/rag/threads/{thread_id}/documents",+ files = {"file": (name, text.encode("utf-8"), "text/plain")},+ )+ assert response.status_code == 200, response.text+ body = response.json()+ deadline = time.time() + 30+ while time.time() < deadline:+ status = ingestion.get_job_status(body["jobId"])+ if status and status["status"] in ("completed", "failed"):+ break+ time.sleep(0.05)+ assert status["status"] == "completed"+ return body["documentId"]+++def _stored_path(document_id):+ conn = rag_db.get_connection()+ try:+ return store.get_document(conn, document_id)["stored_path"]+ finally:+ conn.close()+++def _document_ids(client):+ response = client.get("/api/rag/documents")+ assert response.status_code == 200, response.text+ return {doc["id"] for doc in response.json()["documents"]}+++def _chunk_count(thread_id):+ conn = rag_db.get_metadata_connection()+ try:+ scope = store.thread_scope(thread_id)+ chunks = conn.execute("SELECT COUNT(*) FROM chunks WHERE scope=?", (scope,)).fetchone()[0]+ fts = conn.execute("SELECT COUNT(*) FROM chunks_fts WHERE scope=?", (scope,)).fetchone()[0]+ return chunks + fts+ finally:+ conn.close()+++def test_deleting_a_thread_removes_its_uploaded_documents(client):+ _create_thread(client, "doomed")+ _create_thread(client, "kept")+ doomed = [+ _upload(client, "doomed", f"doomed{i}.txt", f"alpha bravo charlie {i} " * 50)+ for i in range(2)+ ]+ kept = _upload(client, "kept", "kept.txt", "delta echo foxtrot " * 50)+ doomed_paths, kept_path = [_stored_path(d) for d in doomed], _stored_path(kept)+ assert all(os.path.isfile(path) for path in doomed_paths)++ response = client.request("DELETE", "/api/chat/threads", json = {"ids": ["doomed"]})++ assert response.status_code == 200, response.text+ assert _document_ids(client) == {kept}+ assert _chunk_count("doomed") == 0+ assert not any(os.path.exists(path) for path in doomed_paths)+ assert os.path.isfile(kept_path)+ assert _chunk_count("kept") > 0+++def test_clearing_history_removes_every_threads_uploaded_documents(client):+ paths = []+ for thread_id in ("first", "second"):+ _create_thread(client, thread_id)+ paths.append(+ _stored_path(_upload(client, thread_id, f"{thread_id}.txt", f"{thread_id} words " * 50))+ )++ response = client.request("DELETE", "/api/chat")++ assert response.status_code == 200, response.text+ assert _document_ids(client) == set()+ assert not any(os.path.exists(path) for path in paths)+++def test_deleting_a_project_removes_its_member_threads_documents(client):+ response = client.post(+ "/api/chat/projects", json = {"id": "proj", "name": "p", "createdAt": 1, "updatedAt": 1}+ )+ assert response.status_code == 200, response.text+ response = client.post(+ "/api/chat/threads",+ json = {+ "id": "member",+ "title": "t",+ "modelType": "base",+ "createdAt": 1,+ "projectId": "proj",+ },+ )+ assert response.status_code == 200, response.text+ path = _stored_path(_upload(client, "member", "member.txt", "golf hotel india " * 50))+ _create_thread(client, "outsider")+ outsider = _upload(client, "outsider", "outsider.txt", "sierra tango uniform " * 50)++ response = client.delete("/api/chat/projects/proj")++ assert response.status_code == 200, response.text+ assert _document_ids(client) == {outsider}+ assert not os.path.exists(path)+++def test_a_recreated_thread_keeps_documents_uploaded_after_the_cutoff(client):+ _create_thread(client, "recreated")+ old = _upload(client, "recreated", "old.txt", "juliet kilo lima " * 50)+ old_path = _stored_path(old)+ cutoff = datetime.now(timezone.utc).isoformat()+ fresh = _upload(client, "recreated", "fresh.txt", "mike november oscar " * 50)++ chat_history._remove_thread_rag_data(["recreated"])+ assert _document_ids(client) == {old, fresh}++ chat_history._remove_thread_rag_data(["recreated"], cutoff = cutoff)++ assert _document_ids(client) == {fresh}+ assert not os.path.exists(old_path)+ assert os.path.isfile(_stored_path(fresh))+++def test_thread_documents_go_without_sqlite_vec(client, monkeypatch):+ from storage import studio_db++ _create_thread(client, "vecless")+ document_id = _upload(client, "vecless", "vecless.txt", "papa quebec romeo " * 50)+ path = _stored_path(document_id)+ studio_db.delete_chat_threads(["vecless"])++ def no_vec():+ raise rag_db.RagExtensionUnavailable("vec0 will not load")++ with monkeypatch.context() as patch:+ patch.setattr(rag_db, "get_connection", no_vec)+ chat_history._remove_thread_rag_data(["vecless"])++ assert _document_ids(client) == set()+ assert _chunk_count("vecless") == 0+ assert not os.path.exists(path)