Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: do not permanently purge RAG when a project is recreated (#10583)
Apurv Gupta1e212fe6035d
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.pyindex 747c5b9f0a9..b09e218b970 100644--- a/studio/backend/core/rag/folder_sync.py+++ b/studio/backend/core/rag/folder_sync.py@@ -698,10 +698,9 @@ def retire_and_delete_kb(kb_id: str) -> bool:def delete_retired_scope(scope: str) -> bool:"""Purge an ownerless scope and retain its tombstone permanently.- Scope identifiers are treated as non-reusable lifecycle IDs. Keeping the small- tombstone closes late cross-database upload/link races without a distributed transaction.- File removal happens before database rows are discarded, so any failure remains- retryable from durable metadata.+ The tombstone closes late cross-database upload/link races while the scope is ownerless; only a+ same-id project recreate clears it (``unretire_scope``). File removal happens before database rows+ are discarded, so any failure remains retryable from durable metadata."""conn = rag_db.get_connection()try:@@ -845,7 +844,7 @@ def reconcile_retired_scopes(project_exists) -> dict[str, list[str]]:def unretire_scope(scope: str) -> bool:- """Drop the tombstone of a scope whose owner exists again, so it can be used."""+ """Drop the tombstone of a scope whose owner exists again, purged or not, so it can be used."""with _scope_lock(scope):conn = _retirement_connection()try:diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.pyindex 986a431b6a2..55f823043ef 100644--- a/studio/backend/routes/chat_history.py+++ b/studio/backend/routes/chat_history.py@@ -1230,7 +1230,7 @@ def _delete_project_rag_sources(project_id: str) -> None:returnfolder_sync.retire_scope(scope, owned)# The purge takes the whole scope, `owned` or not, so bounding retirement buys nothing- # unless the purge is skipped too.+ # unless the purge is skipped too. A recreate racing this is unretired by upsert_chat_project.if rag_db.rag_available():folder_sync.delete_retired_scope(scope)diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.pyindex 742e4f75724..6cfc35c16c1 100644--- a/studio/backend/storage/studio_db.py+++ b/studio/backend/storage/studio_db.py@@ -2856,6 +2856,32 @@ def count_chat_threads() -> int:conn.close()+def _unretire_project_rag_scope(project_id: str) -> None:+ """Give a recreated project id back its RAG scope (#10567).++ Runs after the Studio row commits and under the scope lock the delete route purges in, so a+ purged tombstone left by a racing delete is cleared instead of disabling RAG for good. The+ owner is re-read under the lock: a delete that won it must keep its tombstone.+ """+ from utils.paths import rag_db_path+ try:+ if not rag_db_path().exists():+ return+ from core.rag import folder_sync, store as rag_store++ scope = rag_store.project_scope(project_id)+ with folder_sync.scope_lock(scope):+ if get_chat_project(project_id) is None:+ return+ folder_sync.unretire_scope(scope)+ except Exception:+ logger.warning(+ "could not clear RAG retirement for project %s after the Studio row committed",+ project_id,+ exc_info = True,+ )++def upsert_chat_project(project: dict) -> dict:existing = get_chat_project(project["id"])root_path = existing.get("rootPath") if existing else None@@ -2888,9 +2914,11 @@ def upsert_chat_project(project: dict) -> dict:),)conn.commit()- return get_chat_project(project["id"]) or project+ saved = get_chat_project(project["id"]) or projectfinally:conn.close()+ _unretire_project_rag_scope(project["id"])+ return saveddef update_chat_project(id: str, patch: dict) -> Optional[dict]:diff --git a/studio/backend/tests/test_rag_linked_folders.py b/studio/backend/tests/test_rag_linked_folders.pyindex 859c0bceec9..2621bef41cb 100644--- a/studio/backend/tests/test_rag_linked_folders.py+++ b/studio/backend/tests/test_rag_linked_folders.py@@ -2518,6 +2518,209 @@ def test_the_purge_is_skipped_for_a_project_recreated_after_the_ownership_check(assert folder_sync.scope_retired(scope) is False+@requires_sqlite_vec+def test_a_project_recreated_before_the_purge_keeps_a_usable_rag_scope(rag_home, monkeypatch):+ """A recreate between the last owner check and the purge must not lock RAG out (#10567)."""+ from routes import chat_history+ from storage import studio_db++ project_id = "p1"+ scope = store.project_scope(project_id)+ source = rag_home / "before-delete"+ source.mkdir()+ folder_sync.create_folder(scope_type = "project", scope_id = project_id, path = str(source))+ owner = {"row": None}++ def current_owner(pid):+ return owner["row"]++ real_purge = folder_sync.delete_retired_scope++ def recreate_then_purge(purged_scope, **kwargs):+ owner["row"] = studio_db.upsert_chat_project(+ {+ "id": project_id,+ "name": "Recreated",+ "createdAt": 1,+ "updatedAt": 1,+ }+ )+ return real_purge(purged_scope, **kwargs)++ monkeypatch.setattr(chat_history, "get_chat_project", current_owner)+ monkeypatch.setattr(folder_sync, "delete_retired_scope", recreate_then_purge)++ chat_history._delete_project_rag_sources(project_id)++ assert owner["row"] is not None, "the project was never recreated, so this proves nothing"+ assert folder_sync.scope_retired(scope) is False+ with _connection() as conn:+ tombstone = conn.execute(+ "SELECT 1 FROM linked_folder_retired_scopes WHERE scope=?", (scope,)+ ).fetchone()+ assert tombstone is None+ replacement = rag_home / "after-recreate"+ replacement.mkdir()+ linked = folder_sync.create_folder(+ scope_type = "project",+ scope_id = project_id,+ path = str(replacement),+ )+ assert linked["status"] == "pending"+++@requires_sqlite_vec+def test_upsert_still_returns_when_rag_unretire_cannot_write(rag_home, monkeypatch):+ """A locked rag.db must not fail a project whose Studio row already committed."""+ from storage import studio_db++ project_id = "p-locked"+ scope = store.project_scope(project_id)+ source = rag_home / "locked-rag"+ source.mkdir()+ folder_sync.create_folder(scope_type = "project", scope_id = project_id, path = str(source))+ folder_sync.retire_scope(scope)+ folder_sync.delete_retired_scope(scope)+ assert folder_sync.scope_retired(scope) is True++ def boom(purged_scope):+ raise sqlite3.OperationalError("database is locked")++ monkeypatch.setattr(folder_sync, "unretire_scope", boom)+ saved = studio_db.upsert_chat_project(+ {+ "id": project_id,+ "name": "Locked RAG",+ "createdAt": 1,+ "updatedAt": 1,+ }+ )+ assert saved["id"] == project_id+ assert studio_db.get_chat_project(project_id) is not None+++@requires_sqlite_vec+def test_upsert_clears_a_tombstone_even_when_the_studio_row_already_existed(rag_home):+ """A leftover tombstone must not depend on the pre-upsert existing snapshot."""+ from storage import studio_db++ project_id = "p-existing"+ scope = store.project_scope(project_id)+ studio_db.upsert_chat_project(+ {+ "id": project_id,+ "name": "First",+ "createdAt": 1,+ "updatedAt": 1,+ }+ )+ folder_sync.retire_scope(scope)+ folder_sync.delete_retired_scope(scope)+ assert folder_sync.scope_retired(scope) is True++ studio_db.upsert_chat_project(+ {+ "id": project_id,+ "name": "Second",+ "createdAt": 2,+ "updatedAt": 2,+ }+ )+ assert folder_sync.scope_retired(scope) is False+++@requires_sqlite_vec+def test_late_upload_after_unretire_is_the_live_projects(rag_home, monkeypatch):+ """A same-id recreate reopens the scope; while ownerless the tombstone still refuses."""+ from routes import chat_history, rag as rag_routes+ from storage import studio_db+ from utils.paths import ensure_dir, rag_uploads_root++ project_id = "p-late-upload"+ scope = store.project_scope(project_id)+ studio_db.upsert_chat_project(+ {+ "id": project_id,+ "name": "Live",+ "createdAt": 1,+ "updatedAt": 1,+ }+ )+ source = rag_home / "live-scope"+ source.mkdir()+ folder_sync.create_folder(scope_type = "project", scope_id = project_id, path = str(source))+ assert folder_sync.scope_retired(scope) is False++ studio_db.delete_chat_project(project_id)+ chat_history._delete_project_rag_sources(project_id)+ assert folder_sync.scope_retired(scope) is True++ ingested = []+ saved_path = ensure_dir(rag_uploads_root()) / "late.txt"++ def resolve_upload(*args, **kwargs):+ saved_path.write_text("saved", encoding = "utf-8")+ return str(saved_path), "late.txt", "0" * 64++ def capture_ingest(*args, **kwargs):+ ingested.append(kwargs.get("project_id") or project_id)+ return "doc-late", "job-late"++ monkeypatch.setattr(rag_routes.rag_db, "rag_available", lambda: True)+ monkeypatch.setattr(rag_routes, "_resolve_document_upload", resolve_upload)+ monkeypatch.setattr(rag_routes.ingestion, "start_ingestion", capture_ingest)++ with pytest.raises(Exception) as exc_info:+ rag_routes.upload_project_document(project_id, subject = "test")+ assert getattr(exc_info.value, "status_code", None) == 404+ assert ingested == []++ with monkeypatch.context() as patched:+ patched.setattr(studio_db, "get_chat_project", lambda value: {"id": value})+ with pytest.raises(Exception) as retired_exc:+ rag_routes.upload_project_document(project_id, subject = "test")+ assert getattr(retired_exc.value, "status_code", None) == 409+ assert ingested == []+ assert not saved_path.exists()++ replacement = rag_home / "before-unretire"+ replacement.mkdir()+ with pytest.raises(ValueError, match = "no longer exists"):+ folder_sync.create_folder(scope_type = "project", scope_id = project_id, path = str(replacement))++ studio_db.upsert_chat_project(+ {+ "id": project_id,+ "name": "Recreated",+ "createdAt": 2,+ "updatedAt": 2,+ }+ )+ assert folder_sync.scope_retired(scope) is False++ uploaded = rag_routes.upload_project_document(project_id, subject = "test")+ assert uploaded == {"documentId": "doc-late", "jobId": "job-late", "filename": "late.txt"}+ assert ingested == [project_id]++ relinked = rag_home / "after-unretire"+ relinked.mkdir()+ linked = folder_sync.create_folder(+ scope_type = "project", scope_id = project_id, path = str(relinked)+ )+ assert linked["status"] == "pending"++ studio_db.delete_chat_project(project_id)+ chat_history._delete_project_rag_sources(project_id)+ assert folder_sync.scope_retired(scope) is True+ with pytest.raises(Exception) as second_exc:+ rag_routes.upload_project_document(project_id, subject = "test")+ assert getattr(second_exc.value, "status_code", None) == 404+ assert ingested == [project_id]++ studio_db._unretire_project_rag_scope(project_id)+ assert folder_sync.scope_retired(scope) is True++@requires_sqlite_vecdef test_reconciliation_restores_a_scope_whose_project_came_back(rag_home):scope = store.project_scope("p1")