Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep RAG WAL sidecars between database operations (#15719)
Lee Jackson47f064c8e560
diff --git a/studio/backend/main.py b/studio/backend/main.pyindex fc432a8416a..f83853dffc0 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -860,6 +860,10 @@ async def lifespan(app: FastAPI):if overlay_dir.is_dir():shutil.rmtree(overlay_dir, ignore_errors = True)+ from storage import rag_db++ rag_db.start_wal_keepers()+# Before the first writer, so startup's own connections stop checkpointing too.try:from storage.studio_db import open_wal_keeper@@ -1110,6 +1114,7 @@ async def lifespan(app: FastAPI):from storage.studio_db import close_wal_keeperclose_wal_keeper()+ rag_db.close_wal_keeper()try:storage.close_wal_keeper()except Exception as exc:diff --git a/studio/backend/routes/accounts.py b/studio/backend/routes/accounts.pyindex 79ce4142a60..41237e69046 100644--- a/studio/backend/routes/accounts.py+++ b/studio/backend/routes/accounts.py@@ -98,6 +98,10 @@ def retire_account_roots(account: AccountContext):from storage.studio_db import close_wal_keeper_forclose_wal_keeper_for(run_as(account, storage_roots.workspace_root) / "studio.db")+ from storage import rag_db++ # Built from the workspace like studio.db: rag_db_path() refuses an escaping link, which must not block deletion.+ rag_db.close_wal_keeper_for(run_as(account, storage_roots.workspace_root) / "rag" / "rag.db")from core.inference import file_historyfile_history.close_store(run_as(account, storage_roots.account_path, "file_history"))diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.pyindex fee96efc981..c89fec38cd9 100644--- a/studio/backend/storage/rag_db.py+++ b/studio/backend/storage/rag_db.py@@ -37,6 +37,11 @@ class RagExtensionUnavailable(RuntimeError):_schema_lock = threading.Lock()_schema_ready: set[Path] = set()+# Held idle so per-operation closes do not delete and recreate WAL/SHM (#15552).+_wal_keepers: dict[Path, sqlite3.Connection] = {}+_wal_keeper_paths: dict[Path, Path] = {}+_wal_keeper_lock = threading.Lock()+_wal_keepers_enabled = True# Warn once per process: the UI polls the KB list on a timer._unavailable_lock = threading.Lock()_unavailable_warned = False@@ -323,6 +328,11 @@ def get_connection() -> sqlite3.Connection:except Exception:conn.close()raise+ try:+ _ensure_wal_keeper(db_path)+ except Exception:+ conn.close()+ raisereturn conn@@ -335,9 +345,87 @@ def get_metadata_connection() -> sqlite3.Connection:conn = sqlite_gate.connect(db_path)conn.row_factory = sqlite3.Rowconn.execute("PRAGMA busy_timeout = 5000")+ try:+ _ensure_wal_keeper(db_path)+ except Exception:+ conn.close()+ raisereturn conn+def _ensure_wal_keeper(db_path: Path) -> None:+ if not _wal_keepers_enabled or db_path in _wal_keeper_paths:+ return+ with _wal_keeper_lock:+ if not _wal_keepers_enabled or _account_retired():+ return+ resolved = db_path.resolve()+ if resolved not in _wal_keepers:+ # No vec0 load: metadata-only callers must work without sqlite-vec.+ keeper = sqlite_gate.connect(db_path, check_same_thread = False)+ try:+ mode = keeper.execute("PRAGMA journal_mode").fetchone()[0]+ if not isinstance(mode, str) or mode.lower() != "wal":+ keeper.close()+ # Not cached: a metadata open can precede RAG switching the file to WAL.+ return+ keeper.execute("SELECT 1 FROM sqlite_schema LIMIT 1").fetchall()+ except Exception:+ keeper.close()+ raise+ _wal_keepers[resolved] = keeper+ _wal_keeper_paths[db_path] = resolved+ _wal_keeper_paths[resolved] = resolved+++def _account_retired() -> bool:+ # Under _wal_keeper_lock: a late open must not recreate a keeper retirement closed (blocks the Windows rename).+ from utils.account_context import is_owner_context++ if is_owner_context():+ return False+ from core.training.account_jobs import account_is_retired++ return account_is_retired()+++def start_wal_keepers() -> None:+ """Enable lazy keepers for a new backend lifespan without initializing RAG."""+ global _wal_keepers_enabled+ with _wal_keeper_lock:+ _wal_keepers_enabled = True+++def _close_keeper(conn: sqlite3.Connection) -> None:+ try:+ conn.close()+ except Exception as exc:+ logger.warning("Could not close the rag.db WAL keeper: %s", exc)+++def close_wal_keeper_for(path: str | Path) -> None:+ """Release one account's RAG database before its directories are retired."""+ resolved = Path(path).resolve()+ with _wal_keeper_lock:+ keeper = _wal_keepers.pop(resolved, None)+ if keeper is not None:+ _close_keeper(keeper)+ for spelling, target in tuple(_wal_keeper_paths.items()):+ if target == resolved:+ del _wal_keeper_paths[spelling]+++def close_wal_keeper() -> None:+ """Release all keepers; late workers cannot reopen them after shutdown."""+ global _wal_keepers_enabled+ with _wal_keeper_lock:+ _wal_keepers_enabled = False+ for keeper in _wal_keepers.values():+ _close_keeper(keeper)+ _wal_keepers.clear()+ _wal_keeper_paths.clear()++def vec_table_dim(conn: sqlite3.Connection) -> int | None:"""Embedding width baked into ``chunks_vec``, or None when absent."""row = conn.execute(diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex a6cb256a360..a402f71cd34 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -498,14 +498,23 @@ def _forget_the_cached_owner_identity():def _isolate_wal_keepers():"""Close the WAL keepers a test opened, so a later test's assertion on _wal_keepers sees onlyits own."""- from storage import studio_db+ from storage import rag_db, studio_dbbefore = set(studio_db._wal_keepers)unsupported = set(studio_db._wal_unsupported)+ rag_before = set(rag_db._wal_keepers)+ rag_enabled = rag_db._wal_keepers_enabled+ rag_db.start_wal_keepers()yieldfor path in set(studio_db._wal_keepers) - before:studio_db.close_wal_keeper_for(path)studio_db._wal_unsupported.intersection_update(unsupported)+ for path in set(rag_db._wal_keepers) - rag_before:+ rag_db.close_wal_keeper_for(path)+ if rag_enabled:+ rag_db.start_wal_keepers()+ else:+ rag_db.close_wal_keeper()@pytest.fixture(autouse = True)diff --git a/studio/backend/tests/test_account_storage.py b/studio/backend/tests/test_account_storage.pyindex ab0e75c9944..eadc3d34a4f 100644--- a/studio/backend/tests/test_account_storage.py+++ b/studio/backend/tests/test_account_storage.py@@ -11,7 +11,7 @@ from pathlib import Pathimport pytest-from storage import api_usage_db, profile_stats_db, studio_db+from storage import api_usage_db, profile_stats_db, rag_db, studio_dbfrom utils.account_context import OWNER, AccountContext, account_thread, current_account_id, run_asfrom utils.paths import storage_roots as rootsfrom utils.paths.lazy import LazyPath@@ -19,6 +19,13 @@ from utils.paths.lazy import LazyPathALICE = AccountContext("11111111111111111111111111111111", "alice")BOB = AccountContext("22222222222222222222222222222222", "bob")ACCOUNTS = (OWNER, ALICE, BOB)+RAG_DB_PARAM = pytest.param(+ "rag_db",+ marks = pytest.mark.skipif(+ not rag_db.RAG_AVAILABLE or not hasattr(sqlite3.Connection, "enable_load_extension"),+ reason = "sqlite-vec cannot be loaded here",+ ),+)@pytest.fixture(autouse = True)@@ -34,6 +41,8 @@ def account_home(tmp_path, monkeypatch):monkeypatch.setenv("UNSLOTH_STUDIO_DOCUMENTS_HOME", str(tmp_path / "Documents"))monkeypatch.setenv("UNSLOTH_STUDIO_PROJECTS_HOME", str(tmp_path / "Projects"))studio_db.close_wal_keeper()+ rag_db.close_wal_keeper()+ rag_db.start_wal_keepers()profile_stats_db._cache.clear()memos = []for name in (@@ -52,6 +61,7 @@ def account_home(tmp_path, monkeypatch):for memo in memos:memo.clear()studio_db.close_wal_keeper()+ rag_db.close_wal_keeper()profile_stats_db._cache.clear()@@ -96,18 +106,24 @@ def test_each_database_gets_its_schema(account_home, module_name, table):assert module._schema_ready == set()-def test_wal_keepers_can_close_one_account(account_home):[email protected]("module_name", ["studio_db", RAG_DB_PARAM])+def test_wal_keepers_can_close_one_account(account_home, module_name):+ module = importlib.import_module(f"storage.{module_name}")+ path_fn = roots.rag_db_path if module_name == "rag_db" else roots.studio_db_pathfor account in ACCOUNTS:- assert run_as(account, studio_db.open_wal_keeper)- paths = {account: run_as(account, roots.studio_db_path) for account in ACCOUNTS}- alice = studio_db._wal_keepers[paths[ALICE]]- studio_db.close_wal_keeper_for(paths[ALICE])+ if module_name == "studio_db":+ assert run_as(account, studio_db.open_wal_keeper)+ else:+ run_as(account, module.get_connection).close()+ paths = {account: run_as(account, path_fn).resolve() for account in ACCOUNTS}+ alice = module._wal_keepers[paths[ALICE]]+ module.close_wal_keeper_for(paths[ALICE])with pytest.raises(sqlite3.ProgrammingError, match = "closed"):alice.execute("SELECT 1")for account in (OWNER, BOB):- assert studio_db._wal_keepers[paths[account]].execute("SELECT 1").fetchone()[0] == 1- studio_db.close_wal_keeper()- assert studio_db._wal_keepers == {}+ assert module._wal_keepers[paths[account]].execute("SELECT 1").fetchone()[0] == 1+ module.close_wal_keeper()+ assert module._wal_keepers == {}def test_a_managed_database_keeps_its_wal_across_a_connection_close(account_home):@@ -130,19 +146,61 @@ def test_a_managed_database_keeps_its_wal_across_a_connection_close(account_homeassert alice_db.with_name("studio.db-wal").exists()-def test_retiring_an_account_releases_its_wal_keeper(account_home):[email protected]("module_name", ["studio_db", RAG_DB_PARAM])+def test_retiring_an_account_releases_its_wal_keeper(account_home, module_name):from routes.accounts import retire_account_roots- studio_db.reset_schema_state_for_tests()- alice_db = run_as(ALICE, roots.studio_db_path)- run_as(ALICE, studio_db.get_connection).close()- keeper = studio_db._wal_keepers[alice_db]+ module = importlib.import_module(f"storage.{module_name}")+ path_fn = roots.rag_db_path if module_name == "rag_db" else roots.studio_db_path+ module.reset_schema_state_for_tests()+ alice_db = run_as(ALICE, path_fn).resolve()+ workspace = run_as(ALICE, roots.workspace_root)+ run_as(ALICE, module.get_connection).close()+ keeper = module._wal_keepers[alice_db]retire_account_roots(ALICE)- assert alice_db not in studio_db._wal_keepers+ assert alice_db not in module._wal_keepers+ assert not workspace.exists(), "the keeper must not prevent the Windows retirement rename"with pytest.raises(sqlite3.ProgrammingError, match = "closed"):keeper.execute("SELECT 1")+def test_retirement_ignores_an_escaping_rag_link(account_home, monkeypatch, tmp_path):+ from routes.accounts import retire_account_roots++ monkeypatch.setattr("core.inference.mcp_client.close_mcp_sessions", lambda: None)+ monkeypatch.setattr("core.inference.mcp_client.invalidate_tool_cache", lambda: None)+ studio_db.reset_schema_state_for_tests()+ workspace = run_as(ALICE, roots.workspace_root)+ run_as(ALICE, studio_db.get_connection).close()+ outside = tmp_path / "outside_rag"+ outside.mkdir()+ try:+ (workspace / "rag").symlink_to(outside, target_is_directory = True)+ except OSError:+ pytest.skip("symlinks are not permitted here")+ retire_account_roots(ALICE)+ assert not workspace.exists()+ assert outside.is_dir()+++def test_a_late_rag_open_does_not_reopen_a_retired_accounts_keeper(account_home):+ """An open that passed ensure_dir before deletion and resumes after retirement released the+ keeper must not leave a new one holding the workspace open."""+ from core.training import account_jobs++ alice_db = run_as(ALICE, roots.rag_db_path)+ run_as(ALICE, rag_db.get_metadata_connection).close()+ with sqlite3.connect(str(alice_db)) as conn:+ conn.execute("PRAGMA journal_mode=WAL")+ account_jobs._retired.add(ALICE.account_id)+ rag_db.close_wal_keeper_for(alice_db)+ run_as(ALICE, rag_db._ensure_wal_keeper, alice_db)+ assert alice_db.resolve() not in rag_db._wal_keepers+ account_jobs._retired.discard(ALICE.account_id)+ run_as(ALICE, rag_db._ensure_wal_keeper, alice_db)+ assert alice_db.resolve() in rag_db._wal_keepers++def _receipt() -> api_usage_db.ApiUsageReceipt:return api_usage_db.ApiUsageReceipt(id = "same-receipt",diff --git a/studio/backend/tests/test_rag_wal_keeper.py b/studio/backend/tests/test_rag_wal_keeper.pynew file mode 100644index 00000000000..84e51021227--- /dev/null+++ b/studio/backend/tests/test_rag_wal_keeper.py@@ -0,0 +1,155 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""RAG operations retain their WAL files without blocking writes or storage teardown."""++from contextlib import closing+from pathlib import Path+import sqlite3+import threading+from concurrent.futures import ThreadPoolExecutor+from types import SimpleNamespace++import pytest++from storage import rag_db++# Some macOS Pythons build sqlite3 without extension loading, so the import alone is not enough.+requires_sqlite_vec = pytest.mark.skipif(+ not rag_db.RAG_AVAILABLE or not hasattr(sqlite3.Connection, "enable_load_extension"),+ reason = "sqlite-vec cannot be loaded here",+)+++def db(tmp_path, monkeypatch):+ path = tmp_path / "rag.db"+ rag_db.close_wal_keeper()+ rag_db.start_wal_keepers()+ monkeypatch.setattr(rag_db, "rag_db_path", lambda: path)+ monkeypatch.setattr(rag_db, "_schema_ready", set())+ yield path+ rag_db.close_wal_keeper()+++def _sidecars(path):+ return Path(str(path) + "-wal"), Path(str(path) + "-shm")+++@requires_sqlite_vec+def test_rag_operations_retain_sidecars_until_keeper_closes(db):+ with closing(rag_db.get_connection()) as conn:+ conn.execute("CREATE TABLE keeper_probe (value INTEGER)")+ conn.execute("INSERT INTO keeper_probe VALUES (42)")+ conn.commit()+ wal, shm = _sidecars(db)+ identities = [(path.stat().st_dev, path.stat().st_ino) for path in (wal, shm)]++ for _ in range(6):+ with closing(rag_db.get_connection()) as conn:+ assert conn.execute("SELECT value FROM keeper_probe").fetchone()[0] == 42+ assert [(path.stat().st_dev, path.stat().st_ino) for path in (wal, shm)] == identities++ rag_db.close_wal_keeper()+ assert not wal.exists()+ assert not shm.exists()+ with closing(sqlite3.connect(str(db))) as conn:+ assert conn.execute("SELECT value FROM keeper_probe").fetchone()[0] == 42+++@requires_sqlite_vec+def test_idle_keeper_does_not_block_a_writer_or_checkpoint(db):+ with closing(rag_db.get_connection()) as conn:+ conn.execute("CREATE TABLE keeper_probe (value INTEGER)")+ conn.commit()++ with closing(sqlite3.connect(str(db), timeout = 1)) as conn:+ conn.execute("INSERT INTO keeper_probe VALUES (7)")+ conn.commit()+ assert conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() == (0, 0, 0)+ assert _sidecars(db)[0].stat().st_size == 0+ with closing(rag_db.get_connection()) as conn:+ assert conn.execute("SELECT value FROM keeper_probe").fetchone()[0] == 7+++@requires_sqlite_vec+def test_metadata_first_does_not_suppress_a_later_wal_keeper(db):+ with closing(rag_db.get_metadata_connection()) as conn:+ assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "delete"+ assert db.resolve() not in rag_db._wal_keepers++ with closing(rag_db.get_connection()) as conn:+ assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal"+ assert db.resolve() in rag_db._wal_keepers+ assert all(path.exists() for path in _sidecars(db))+++ "with_vectors",+ [False, pytest.param(True, marks = requires_sqlite_vec)],+ ids = ["plain-metadata", "vec0-schema"],+)+def test_metadata_keeper_does_not_require_vector_extension(db, monkeypatch, with_vectors):+ connection = rag_db.get_connection() if with_vectors else sqlite3.connect(str(db))+ with closing(connection) as conn:+ if with_vectors:+ rag_db.ensure_vec(conn, 3)+ else:+ conn.execute("PRAGMA journal_mode=WAL")+ conn.execute("CREATE TABLE metadata_probe (value INTEGER)")+ conn.execute("INSERT INTO metadata_probe VALUES (3)")+ conn.commit()+ rag_db.close_wal_keeper()+ rag_db.start_wal_keepers()++ def unavailable(_conn):+ pytest.fail("A metadata-only keeper must not load sqlite-vec")++ monkeypatch.setattr(rag_db, "sqlite_vec", SimpleNamespace(load = unavailable))+ monkeypatch.setattr(rag_db, "RAG_AVAILABLE", False)+ with closing(rag_db.get_metadata_connection()) as conn:+ assert conn.execute("SELECT value FROM metadata_probe").fetchone()[0] == 3+ if with_vectors:+ assert (+ conn.execute("SELECT 1 FROM sqlite_schema WHERE name='chunks_vec'").fetchone()+ is not None+ )+ assert db.resolve() in rag_db._wal_keepers+ assert all(path.exists() for path in _sidecars(db))+++@requires_sqlite_vec+def test_shutdown_prevents_late_keeper_reopen_and_restart_enables_it(db):+ rag_db.get_connection().close()+ assert db.resolve() in rag_db._wal_keepers+ rag_db.close_wal_keeper()++ # A worker finishing after the bounded shutdown join may still record metadata.+ with closing(rag_db.get_metadata_connection()) as conn:+ conn.execute("SELECT count(*) FROM documents").fetchone()+ assert not rag_db._wal_keepers+ assert not any(path.exists() for path in _sidecars(db))++ rag_db.start_wal_keepers()+ rag_db.get_metadata_connection().close()+ assert db.resolve() in rag_db._wal_keepers+ assert all(path.exists() for path in _sidecars(db))+++@requires_sqlite_vec+def test_concurrent_first_opens_share_one_keeper(db):+ ready = threading.Barrier(8)++ def read():+ ready.wait(timeout = 10)+ with closing(rag_db.get_connection()) as conn:+ return conn.execute("SELECT count(*) FROM documents").fetchone()[0]++ with ThreadPoolExecutor(max_workers = 8) as pool:+ futures = [pool.submit(read) for _ in range(8)]+ assert [future.result(timeout = 30) for future in futures] == [0] * 8+ assert set(rag_db._wal_keepers) == {db.resolve()}+ keeper = rag_db._wal_keepers[db.resolve()]+ rag_db.get_connection().close()+ assert rag_db._wal_keepers[db.resolve()] is keeper+ assert all(path.exists() for path in _sidecars(db))