Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio RAG: delete a retired folder's or scope's chunks in batches instead of one index scan per document (#17281)

Daniel Han78abc68191fd
diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.py
index 579238ea214..bdee360f336 100644
--- a/studio/backend/core/rag/conversation_archive.py
+++ b/studio/backend/core/rag/conversation_archive.py
@@ -2735,15 +2735,7 @@ def _delete_scope_without_vec(
try:
conn = rag_db.get_metadata_connection()
documents = conn.execute(*_scope_select(scope, created_before)).fetchall()
- for row in documents:
- conn.execute(
- "DELETE FROM chunks_fts WHERE chunk_id IN "
- "(SELECT id FROM chunks WHERE document_id=?)",
- (row["id"],),
- )
- conn.execute("DELETE FROM chunks WHERE document_id=?", (row["id"],))
- conn.execute("DELETE FROM documents WHERE id=?", (row["id"],))
- conn.commit()
+ store.delete_documents(conn, [row["id"] for row in documents], vec = False)
removed = [row["stored_path"] for row in documents]
except Exception:
logger.warning(
@@ -2863,9 +2855,9 @@ def _delete_scope(
except Exception:
# No vec0 here. Delete what can be deleted rather than nothing at all.
return _delete_scope_without_vec(scope, thread_id, created_before = created_before)
- for row in conn.execute(*_scope_select(scope, created_before)).fetchall():
- store.delete_document(conn, row["id"])
- removed.append(row["stored_path"])
+ documents = conn.execute(*_scope_select(scope, created_before)).fetchall()
+ store.delete_documents(conn, [row["id"] for row in documents])
+ removed = [row["stored_path"] for row in documents]
except Exception:
logger.warning("conversation_archive.delete_failed thread_id=%s", thread_id, exc_info = True)
finally:
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.py
index 918cd087dcf..f2929cb38e0 100644
--- a/studio/backend/core/rag/folder_sync.py
+++ b/studio/backend/core/rag/folder_sync.py
@@ -628,10 +628,10 @@ def _delete_retired_folder(folder_id: str) -> bool | None:
for snapshot in dict.fromkeys(doc["stored_path"] for doc in docs):
_remove_retired_snapshot(snapshot)
conn.execute("DELETE FROM linked_folder_files WHERE folder_id=?", (folder_id,))
- for doc in docs:
- if folder["delete_remove_index"]:
- store.delete_document(conn, doc["id"], commit = False)
- else:
+ if folder["delete_remove_index"]:
+ store.delete_documents(conn, [doc["id"] for doc in docs], commit = False)
+ else:
+ for doc in docs:
conn.execute(
"UPDATE documents SET linked_folder_id=NULL, linked_relative_path=NULL "
"WHERE id=?",
@@ -857,8 +857,7 @@ def delete_retired_scope(scope: str) -> bool:
).fetchall()
for stored_path in dict.fromkeys(document["stored_path"] for document in documents):
_remove_retired_snapshot(stored_path)
- for document in documents:
- store.delete_document(conn, document["id"], commit = False)
+ store.delete_documents(conn, [document["id"] for document in documents], commit = False)
conn.execute(
"DELETE FROM rag_job_leases WHERE kind=? AND job_id IN "
"(SELECT id FROM ingestion_jobs WHERE scope=?)",
diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.py
index 564fd6601a3..72e1a8abae4 100644
--- a/studio/backend/core/rag/store.py
+++ b/studio/backend/core/rag/store.py
@@ -569,8 +569,20 @@ def delete_document(
commit: bool = True,
) -> None:
"""Remove a document and all its chunks (+ fts + vec rows)."""
- rag_db._delete_document_chunks(conn, document_id)
- conn.execute("DELETE FROM documents WHERE id=?", (document_id,))
+ delete_documents(conn, [document_id], commit = commit)
+
+
+def delete_documents(
+ conn: sqlite3.Connection,
+ document_ids,
+ *,
+ commit: bool = True,
+ vec: bool = True,
+) -> None:
+ ids = list(document_ids)
+ rag_db._delete_documents_chunks(conn, ids, vec = vec)
+ for batch, marks in rag_db._id_batches(ids):
+ conn.execute(f"DELETE FROM documents WHERE id IN ({marks})", batch)
if commit:
conn.commit()
diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.py
index 29a614500d3..ad0ac007f11 100644
--- a/studio/backend/storage/rag_db.py
+++ b/studio/backend/storage/rag_db.py
@@ -601,20 +601,41 @@ def vec_table_exists(conn: sqlite3.Connection) -> bool:
return row is not None
+# Under the oldest SQLITE_MAX_VARIABLE_NUMBER (999).
+_DOC_ID_BATCH = 900
+
+
+def _id_batches(ids):
+ ids = list(dict.fromkeys(ids))
+ for start in range(0, len(ids), _DOC_ID_BATCH):
+ batch = ids[start : start + _DOC_ID_BATCH]
+ yield batch, ",".join("?" * len(batch))
+
+
def _delete_document_chunks(conn, document_id: str) -> None:
"""Deletes chunk rows, keeps the documents row; retrieval filters by scope, so leftovers stay
citable."""
- if (
- conn.execute("SELECT 1 FROM chunks WHERE document_id=? LIMIT 1", (document_id,)).fetchone()
- is None
- ):
- return
- # chunk_id is unindexed in fts5 and vec0: one IN scan per table, not one scan per chunk.
- owned = "SELECT id FROM chunks WHERE document_id=?"
- conn.execute(f"DELETE FROM chunks_fts WHERE chunk_id IN ({owned})", (document_id,))
- if vec_table_exists(conn):
- conn.execute(f"DELETE FROM chunks_vec WHERE chunk_id IN ({owned})", (document_id,))
- conn.execute("DELETE FROM chunks WHERE document_id=?", (document_id,))
+ _delete_documents_chunks(conn, [document_id])
+
+
+def _delete_documents_chunks(
+ conn,
+ document_ids,
+ *,
+ vec: bool = True,
+) -> None:
+ """chunk_id is unindexed in fts5 and vec0: one full scan per id batch, not per document."""
+ has_vec = None if vec else False
+ for batch, marks in _id_batches(document_ids):
+ owned = f"SELECT id FROM chunks WHERE document_id IN ({marks})"
+ if conn.execute(f"{owned} LIMIT 1", batch).fetchone() is None:
+ continue
+ conn.execute(f"DELETE FROM chunks_fts WHERE chunk_id IN ({owned})", batch)
+ if has_vec is None:
+ has_vec = vec_table_exists(conn)
+ if has_vec:
+ conn.execute(f"DELETE FROM chunks_vec WHERE chunk_id IN ({owned})", batch)
+ conn.execute(f"DELETE FROM chunks WHERE document_id IN ({marks})", batch)
def reconcile_orphaned_ingestion_jobs() -> int:
diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.py
index ebce6252ee7..c53ebea74fc 100644
--- a/studio/backend/tests/test_conversation_archive.py
+++ b/studio/backend/tests/test_conversation_archive.py
@@ -5243,3 +5243,42 @@ def test_a_repeat_that_came_back_into_the_prompt_keeps_one_copy(conn):
assert found is not None
texts = [source["text"] for source in found[1]]
assert len(texts) == len(set(texts)), texts
+
+
[email protected]("vec", [True, False])
+def test_deleting_a_thread_scans_the_fts_table_once_not_once_per_turn(conn, monkeypatch, vec):
+ thread = f"many-turns-{vec}"
+ turns = []
+ for i in range(4):
+ turns += _turn(f"question {i} about ducks", f"answer {i}: SCANONCE-{i}")
+ _save_thread(thread, turns, append = True)
+ assert conversation_archive.archive_turns(thread, turns) >= 2
+ archived = len(store.list_documents(conn, store.conversation_archive_scope(thread)))
+
+ statements = []
+
+ def traced(open_conn):
+ def opener(*args, **kwargs):
+ c = open_conn(*args, **kwargs)
+ c.set_trace_callback(statements.append)
+ return c
+
+ return opener
+
+ if vec:
+ monkeypatch.setattr(rag_db, "get_connection", traced(rag_db.get_connection))
+ else:
+
+ def no_vec():
+ raise rag_db.RagExtensionUnavailable("vec0 will not load")
+
+ monkeypatch.setattr(rag_db, "get_connection", no_vec)
+ monkeypatch.setattr(
+ rag_db, "get_metadata_connection", traced(rag_db.get_metadata_connection)
+ )
+ removed = conversation_archive.delete_for_thread(thread)
+ monkeypatch.undo()
+
+ assert removed == archived
+ assert conversation_archive.has_archive(thread) is False
+ assert sum("DELETE FROM chunks_fts" in s for s in statements) == 1
diff --git a/studio/backend/tests/test_rag_delete_document_scan.py b/studio/backend/tests/test_rag_delete_document_scan.py
index cbe9a11e33c..3b785249fc9 100644
--- a/studio/backend/tests/test_rag_delete_document_scan.py
+++ b/studio/backend/tests/test_rag_delete_document_scan.py
@@ -200,3 +200,103 @@ def test_delete_cost_does_not_scale_with_chunks_times_corpus(rag_conn):
conn.execute("SELECT count(*) FROM chunks_vec WHERE chunk_id LIKE 'large:%'").fetchone()[0]
== 0
)
+
+
+def _scope_fixture(conn, rng):
+ conn.execute(
+ "INSERT INTO linked_folders(id, scope_type, scope_id, scope, path, name, delete_remove_index, "
+ "status, created_at, updated_at) VALUES('F1','kb','keep','kb:keep','/tmp/x','x',1,'idle',?,?)",
+ ("2026-01-01T00:00:00", "2026-01-01T00:00:00"),
+ )
+ for d in range(12):
+ _add(conn, rng, "kb:gone", f"gone{d}", [f"alpha gone {d} {i}" for i in range(d % 4 * 3)])
+ _add(conn, rng, "kb:keep", f"keep{d}", [f"alpha keep {d} {i}" for i in range(3)])
+ _add(
+ conn, rng, "kb:keep", f"linked{d}", [f"alpha linked {d} {i}" for i in range(d % 3 + 1)]
+ )
+ conn.execute(
+ "INSERT INTO linked_folder_files(folder_id, relative_path, size_bytes, mtime_ns, "
+ "document_id, synced_at) VALUES('F1', ?, 1, 1, ?, '2026-01-01T00:00:00')",
+ (f"r{d}.txt", f"linked{d}"),
+ )
+ conn.execute(
+ "INSERT INTO linked_folder_retired_scopes(scope, retired_at) "
+ "VALUES('kb:gone', '2026-01-01T00:00:00')"
+ )
+ conn.commit()
+
+
+def _full_dump(conn):
+ return _dump(conn) + [
+ [tuple(r) for r in conn.execute(sql)]
+ for sql in (
+ "SELECT * FROM rag_vec_epoch ORDER BY scope",
+ "SELECT * FROM linked_folder_files ORDER BY folder_id, relative_path",
+ "SELECT id FROM linked_folders ORDER BY id",
+ )
+ ]
+
+
[email protected]("op", ["scope", "folder"])
+def test_retired_scope_and_folder_deletes_scan_indexes_once(rag_conn, monkeypatch, op):
+ from core.rag import folder_sync
+
+ conn = rag_conn
+ _scope_fixture(conn, random.Random(3))
+ reference = _copy(conn)
+ statements = []
+ real_connection = rag_db.get_connection
+
+ def traced_connection():
+ traced = real_connection()
+ traced.set_trace_callback(statements.append)
+ return traced
+
+ monkeypatch.setattr(folder_sync.rag_db, "get_connection", traced_connection)
+ try:
+ if op == "scope":
+ assert folder_sync.delete_retired_scope("kb:gone") is True
+ gone = [
+ r[0] for r in reference.execute("SELECT id FROM documents WHERE scope='kb:gone'")
+ ]
+ else:
+ assert folder_sync._delete_retired_folder("F1") is True
+ gone = [r[0] for r in reference.execute("SELECT document_id FROM linked_folder_files")]
+ assert len(gone) == 12
+ if op == "folder":
+ reference.execute("DELETE FROM linked_folder_files WHERE folder_id='F1'")
+ reference.execute("DELETE FROM linked_folders WHERE id='F1'")
+ for doc_id in gone:
+ store.delete_document(reference, doc_id, commit = False)
+ reference.commit()
+
+ conn.rollback()
+ assert _full_dump(conn) == _full_dump(reference)
+ for table in ("chunks_fts", "chunks_vec"):
+ scans = [s for s in statements if s.startswith(f"DELETE FROM {table} ")]
+ assert len(scans) == 1, (table, scans)
+ conn.execute("INSERT INTO chunks_fts(chunks_fts) VALUES('integrity-check')")
+ assert store.search_lexical(conn, "kb:keep", "alpha", 50) == store.search_lexical(
+ reference, "kb:keep", "alpha", 50
+ )
+ assert store.search_lexical(conn, "kb:gone", "alpha", 50) == []
+ finally:
+ reference.close()
+
+
+def test_delete_documents_batches_past_the_variable_limit(rag_conn, monkeypatch):
+ conn = rag_conn
+ rng = random.Random(4)
+ for d in range(7):
+ _add(conn, rng, "kb:a", f"b{d}", [f"alpha {d} {i}" for i in range(d % 3)])
+ _add(conn, rng, "kb:a", "stay", ["alpha stay"])
+ reference = _copy(conn)
+ try:
+ monkeypatch.setattr(rag_db, "_DOC_ID_BATCH", 2)
+ ids = [f"b{d}" for d in range(7)] + ["b0", "missing"]
+ store.delete_documents(conn, ids)
+ for doc_id in ids:
+ store.delete_document(reference, doc_id)
+ assert _full_dump(conn) == _full_dump(reference)
+ finally:
+ reference.close()