Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio RAG: delete a retired folder's or scope's chunks in batches instead of one index scan per document (#17281)
Daniel Han78abc68191fd
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.pyindex 579238ea214..bdee360f336 100644--- a/studio/backend/core/rag/conversation_archive.py+++ b/studio/backend/core/rag/conversation_archive.py@@ -2735,15 +2735,7 @@ def _delete_scope_without_vec(try:conn = rag_db.get_metadata_connection()documents = conn.execute(*_scope_select(scope, created_before)).fetchall()- for row in documents:- conn.execute(- "DELETE FROM chunks_fts WHERE chunk_id IN "- "(SELECT id FROM chunks WHERE document_id=?)",- (row["id"],),- )- conn.execute("DELETE FROM chunks WHERE document_id=?", (row["id"],))- conn.execute("DELETE FROM documents WHERE id=?", (row["id"],))- conn.commit()+ store.delete_documents(conn, [row["id"] for row in documents], vec = False)removed = [row["stored_path"] for row in documents]except Exception:logger.warning(@@ -2863,9 +2855,9 @@ def _delete_scope(except Exception:# No vec0 here. Delete what can be deleted rather than nothing at all.return _delete_scope_without_vec(scope, thread_id, created_before = created_before)- for row in conn.execute(*_scope_select(scope, created_before)).fetchall():- store.delete_document(conn, row["id"])- removed.append(row["stored_path"])+ documents = conn.execute(*_scope_select(scope, created_before)).fetchall()+ store.delete_documents(conn, [row["id"] for row in documents])+ removed = [row["stored_path"] for row in documents]except Exception:logger.warning("conversation_archive.delete_failed thread_id=%s", thread_id, exc_info = True)finally:diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.pyindex 918cd087dcf..f2929cb38e0 100644--- a/studio/backend/core/rag/folder_sync.py+++ b/studio/backend/core/rag/folder_sync.py@@ -628,10 +628,10 @@ def _delete_retired_folder(folder_id: str) -> bool | None:for snapshot in dict.fromkeys(doc["stored_path"] for doc in docs):_remove_retired_snapshot(snapshot)conn.execute("DELETE FROM linked_folder_files WHERE folder_id=?", (folder_id,))- for doc in docs:- if folder["delete_remove_index"]:- store.delete_document(conn, doc["id"], commit = False)- else:+ if folder["delete_remove_index"]:+ store.delete_documents(conn, [doc["id"] for doc in docs], commit = False)+ else:+ for doc in docs:conn.execute("UPDATE documents SET linked_folder_id=NULL, linked_relative_path=NULL ""WHERE id=?",@@ -857,8 +857,7 @@ def delete_retired_scope(scope: str) -> bool:).fetchall()for stored_path in dict.fromkeys(document["stored_path"] for document in documents):_remove_retired_snapshot(stored_path)- for document in documents:- store.delete_document(conn, document["id"], commit = False)+ store.delete_documents(conn, [document["id"] for document in documents], commit = False)conn.execute("DELETE FROM rag_job_leases WHERE kind=? AND job_id IN ""(SELECT id FROM ingestion_jobs WHERE scope=?)",diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.pyindex 564fd6601a3..72e1a8abae4 100644--- a/studio/backend/core/rag/store.py+++ b/studio/backend/core/rag/store.py@@ -569,8 +569,20 @@ def delete_document(commit: bool = True,) -> None:"""Remove a document and all its chunks (+ fts + vec rows)."""- rag_db._delete_document_chunks(conn, document_id)- conn.execute("DELETE FROM documents WHERE id=?", (document_id,))+ delete_documents(conn, [document_id], commit = commit)+++def delete_documents(+ conn: sqlite3.Connection,+ document_ids,+ *,+ commit: bool = True,+ vec: bool = True,+) -> None:+ ids = list(document_ids)+ rag_db._delete_documents_chunks(conn, ids, vec = vec)+ for batch, marks in rag_db._id_batches(ids):+ conn.execute(f"DELETE FROM documents WHERE id IN ({marks})", batch)if commit:conn.commit()diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.pyindex 29a614500d3..ad0ac007f11 100644--- a/studio/backend/storage/rag_db.py+++ b/studio/backend/storage/rag_db.py@@ -601,20 +601,41 @@ def vec_table_exists(conn: sqlite3.Connection) -> bool:return row is not None+# Under the oldest SQLITE_MAX_VARIABLE_NUMBER (999).+_DOC_ID_BATCH = 900+++def _id_batches(ids):+ ids = list(dict.fromkeys(ids))+ for start in range(0, len(ids), _DOC_ID_BATCH):+ batch = ids[start : start + _DOC_ID_BATCH]+ yield batch, ",".join("?" * len(batch))++def _delete_document_chunks(conn, document_id: str) -> None:"""Deletes chunk rows, keeps the documents row; retrieval filters by scope, so leftovers staycitable."""- if (- conn.execute("SELECT 1 FROM chunks WHERE document_id=? LIMIT 1", (document_id,)).fetchone()- is None- ):- return- # chunk_id is unindexed in fts5 and vec0: one IN scan per table, not one scan per chunk.- owned = "SELECT id FROM chunks WHERE document_id=?"- conn.execute(f"DELETE FROM chunks_fts WHERE chunk_id IN ({owned})", (document_id,))- if vec_table_exists(conn):- conn.execute(f"DELETE FROM chunks_vec WHERE chunk_id IN ({owned})", (document_id,))- conn.execute("DELETE FROM chunks WHERE document_id=?", (document_id,))+ _delete_documents_chunks(conn, [document_id])+++def _delete_documents_chunks(+ conn,+ document_ids,+ *,+ vec: bool = True,+) -> None:+ """chunk_id is unindexed in fts5 and vec0: one full scan per id batch, not per document."""+ has_vec = None if vec else False+ for batch, marks in _id_batches(document_ids):+ owned = f"SELECT id FROM chunks WHERE document_id IN ({marks})"+ if conn.execute(f"{owned} LIMIT 1", batch).fetchone() is None:+ continue+ conn.execute(f"DELETE FROM chunks_fts WHERE chunk_id IN ({owned})", batch)+ if has_vec is None:+ has_vec = vec_table_exists(conn)+ if has_vec:+ conn.execute(f"DELETE FROM chunks_vec WHERE chunk_id IN ({owned})", batch)+ conn.execute(f"DELETE FROM chunks WHERE document_id IN ({marks})", batch)def reconcile_orphaned_ingestion_jobs() -> int:diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.pyindex ebce6252ee7..c53ebea74fc 100644--- a/studio/backend/tests/test_conversation_archive.py+++ b/studio/backend/tests/test_conversation_archive.py@@ -5243,3 +5243,42 @@ def test_a_repeat_that_came_back_into_the_prompt_keeps_one_copy(conn):assert found is not Nonetexts = [source["text"] for source in found[1]]assert len(texts) == len(set(texts)), texts++[email protected]("vec", [True, False])+def test_deleting_a_thread_scans_the_fts_table_once_not_once_per_turn(conn, monkeypatch, vec):+ thread = f"many-turns-{vec}"+ turns = []+ for i in range(4):+ turns += _turn(f"question {i} about ducks", f"answer {i}: SCANONCE-{i}")+ _save_thread(thread, turns, append = True)+ assert conversation_archive.archive_turns(thread, turns) >= 2+ archived = len(store.list_documents(conn, store.conversation_archive_scope(thread)))++ statements = []++ def traced(open_conn):+ def opener(*args, **kwargs):+ c = open_conn(*args, **kwargs)+ c.set_trace_callback(statements.append)+ return c++ return opener++ if vec:+ monkeypatch.setattr(rag_db, "get_connection", traced(rag_db.get_connection))+ else:++ def no_vec():+ raise rag_db.RagExtensionUnavailable("vec0 will not load")++ monkeypatch.setattr(rag_db, "get_connection", no_vec)+ monkeypatch.setattr(+ rag_db, "get_metadata_connection", traced(rag_db.get_metadata_connection)+ )+ removed = conversation_archive.delete_for_thread(thread)+ monkeypatch.undo()++ assert removed == archived+ assert conversation_archive.has_archive(thread) is False+ assert sum("DELETE FROM chunks_fts" in s for s in statements) == 1diff --git a/studio/backend/tests/test_rag_delete_document_scan.py b/studio/backend/tests/test_rag_delete_document_scan.pyindex cbe9a11e33c..3b785249fc9 100644--- a/studio/backend/tests/test_rag_delete_document_scan.py+++ b/studio/backend/tests/test_rag_delete_document_scan.py@@ -200,3 +200,103 @@ def test_delete_cost_does_not_scale_with_chunks_times_corpus(rag_conn):conn.execute("SELECT count(*) FROM chunks_vec WHERE chunk_id LIKE 'large:%'").fetchone()[0]== 0)+++def _scope_fixture(conn, rng):+ conn.execute(+ "INSERT INTO linked_folders(id, scope_type, scope_id, scope, path, name, delete_remove_index, "+ "status, created_at, updated_at) VALUES('F1','kb','keep','kb:keep','/tmp/x','x',1,'idle',?,?)",+ ("2026-01-01T00:00:00", "2026-01-01T00:00:00"),+ )+ for d in range(12):+ _add(conn, rng, "kb:gone", f"gone{d}", [f"alpha gone {d} {i}" for i in range(d % 4 * 3)])+ _add(conn, rng, "kb:keep", f"keep{d}", [f"alpha keep {d} {i}" for i in range(3)])+ _add(+ conn, rng, "kb:keep", f"linked{d}", [f"alpha linked {d} {i}" for i in range(d % 3 + 1)]+ )+ conn.execute(+ "INSERT INTO linked_folder_files(folder_id, relative_path, size_bytes, mtime_ns, "+ "document_id, synced_at) VALUES('F1', ?, 1, 1, ?, '2026-01-01T00:00:00')",+ (f"r{d}.txt", f"linked{d}"),+ )+ conn.execute(+ "INSERT INTO linked_folder_retired_scopes(scope, retired_at) "+ "VALUES('kb:gone', '2026-01-01T00:00:00')"+ )+ conn.commit()+++def _full_dump(conn):+ return _dump(conn) + [+ [tuple(r) for r in conn.execute(sql)]+ for sql in (+ "SELECT * FROM rag_vec_epoch ORDER BY scope",+ "SELECT * FROM linked_folder_files ORDER BY folder_id, relative_path",+ "SELECT id FROM linked_folders ORDER BY id",+ )+ ]++[email protected]("op", ["scope", "folder"])+def test_retired_scope_and_folder_deletes_scan_indexes_once(rag_conn, monkeypatch, op):+ from core.rag import folder_sync++ conn = rag_conn+ _scope_fixture(conn, random.Random(3))+ reference = _copy(conn)+ statements = []+ real_connection = rag_db.get_connection++ def traced_connection():+ traced = real_connection()+ traced.set_trace_callback(statements.append)+ return traced++ monkeypatch.setattr(folder_sync.rag_db, "get_connection", traced_connection)+ try:+ if op == "scope":+ assert folder_sync.delete_retired_scope("kb:gone") is True+ gone = [+ r[0] for r in reference.execute("SELECT id FROM documents WHERE scope='kb:gone'")+ ]+ else:+ assert folder_sync._delete_retired_folder("F1") is True+ gone = [r[0] for r in reference.execute("SELECT document_id FROM linked_folder_files")]+ assert len(gone) == 12+ if op == "folder":+ reference.execute("DELETE FROM linked_folder_files WHERE folder_id='F1'")+ reference.execute("DELETE FROM linked_folders WHERE id='F1'")+ for doc_id in gone:+ store.delete_document(reference, doc_id, commit = False)+ reference.commit()++ conn.rollback()+ assert _full_dump(conn) == _full_dump(reference)+ for table in ("chunks_fts", "chunks_vec"):+ scans = [s for s in statements if s.startswith(f"DELETE FROM {table} ")]+ assert len(scans) == 1, (table, scans)+ conn.execute("INSERT INTO chunks_fts(chunks_fts) VALUES('integrity-check')")+ assert store.search_lexical(conn, "kb:keep", "alpha", 50) == store.search_lexical(+ reference, "kb:keep", "alpha", 50+ )+ assert store.search_lexical(conn, "kb:gone", "alpha", 50) == []+ finally:+ reference.close()+++def test_delete_documents_batches_past_the_variable_limit(rag_conn, monkeypatch):+ conn = rag_conn+ rng = random.Random(4)+ for d in range(7):+ _add(conn, rng, "kb:a", f"b{d}", [f"alpha {d} {i}" for i in range(d % 3)])+ _add(conn, rng, "kb:a", "stay", ["alpha stay"])+ reference = _copy(conn)+ try:+ monkeypatch.setattr(rag_db, "_DOC_ID_BATCH", 2)+ ids = [f"b{d}" for d in range(7)] + ["b0", "missing"]+ store.delete_documents(conn, ids)+ for doc_id in ids:+ store.delete_document(reference, doc_id)+ assert _full_dump(conn) == _full_dump(reference)+ finally:+ reference.close()