Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep RAG WAL sidecars between database operations (#15719)

Lee Jackson47f064c8e560
diff --git a/studio/backend/main.py b/studio/backend/main.py
index fc432a8416a..f83853dffc0 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -860,6 +860,10 @@ async def lifespan(app: FastAPI):
if overlay_dir.is_dir():
shutil.rmtree(overlay_dir, ignore_errors = True)
+ from storage import rag_db
+
+ rag_db.start_wal_keepers()
+
# Before the first writer, so startup's own connections stop checkpointing too.
try:
from storage.studio_db import open_wal_keeper
@@ -1110,6 +1114,7 @@ async def lifespan(app: FastAPI):
from storage.studio_db import close_wal_keeper
close_wal_keeper()
+ rag_db.close_wal_keeper()
try:
storage.close_wal_keeper()
except Exception as exc:
diff --git a/studio/backend/routes/accounts.py b/studio/backend/routes/accounts.py
index 79ce4142a60..41237e69046 100644
--- a/studio/backend/routes/accounts.py
+++ b/studio/backend/routes/accounts.py
@@ -98,6 +98,10 @@ def retire_account_roots(account: AccountContext):
from storage.studio_db import close_wal_keeper_for
close_wal_keeper_for(run_as(account, storage_roots.workspace_root) / "studio.db")
+ from storage import rag_db
+
+ # Built from the workspace like studio.db: rag_db_path() refuses an escaping link, which must not block deletion.
+ rag_db.close_wal_keeper_for(run_as(account, storage_roots.workspace_root) / "rag" / "rag.db")
from core.inference import file_history
file_history.close_store(run_as(account, storage_roots.account_path, "file_history"))
diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.py
index fee96efc981..c89fec38cd9 100644
--- a/studio/backend/storage/rag_db.py
+++ b/studio/backend/storage/rag_db.py
@@ -37,6 +37,11 @@ class RagExtensionUnavailable(RuntimeError):
_schema_lock = threading.Lock()
_schema_ready: set[Path] = set()
+# Held idle so per-operation closes do not delete and recreate WAL/SHM (#15552).
+_wal_keepers: dict[Path, sqlite3.Connection] = {}
+_wal_keeper_paths: dict[Path, Path] = {}
+_wal_keeper_lock = threading.Lock()
+_wal_keepers_enabled = True
# Warn once per process: the UI polls the KB list on a timer.
_unavailable_lock = threading.Lock()
_unavailable_warned = False
@@ -323,6 +328,11 @@ def get_connection() -> sqlite3.Connection:
except Exception:
conn.close()
raise
+ try:
+ _ensure_wal_keeper(db_path)
+ except Exception:
+ conn.close()
+ raise
return conn
@@ -335,9 +345,87 @@ def get_metadata_connection() -> sqlite3.Connection:
conn = sqlite_gate.connect(db_path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA busy_timeout = 5000")
+ try:
+ _ensure_wal_keeper(db_path)
+ except Exception:
+ conn.close()
+ raise
return conn
+def _ensure_wal_keeper(db_path: Path) -> None:
+ if not _wal_keepers_enabled or db_path in _wal_keeper_paths:
+ return
+ with _wal_keeper_lock:
+ if not _wal_keepers_enabled or _account_retired():
+ return
+ resolved = db_path.resolve()
+ if resolved not in _wal_keepers:
+ # No vec0 load: metadata-only callers must work without sqlite-vec.
+ keeper = sqlite_gate.connect(db_path, check_same_thread = False)
+ try:
+ mode = keeper.execute("PRAGMA journal_mode").fetchone()[0]
+ if not isinstance(mode, str) or mode.lower() != "wal":
+ keeper.close()
+ # Not cached: a metadata open can precede RAG switching the file to WAL.
+ return
+ keeper.execute("SELECT 1 FROM sqlite_schema LIMIT 1").fetchall()
+ except Exception:
+ keeper.close()
+ raise
+ _wal_keepers[resolved] = keeper
+ _wal_keeper_paths[db_path] = resolved
+ _wal_keeper_paths[resolved] = resolved
+
+
+def _account_retired() -> bool:
+ # Under _wal_keeper_lock: a late open must not recreate a keeper retirement closed (blocks the Windows rename).
+ from utils.account_context import is_owner_context
+
+ if is_owner_context():
+ return False
+ from core.training.account_jobs import account_is_retired
+
+ return account_is_retired()
+
+
+def start_wal_keepers() -> None:
+ """Enable lazy keepers for a new backend lifespan without initializing RAG."""
+ global _wal_keepers_enabled
+ with _wal_keeper_lock:
+ _wal_keepers_enabled = True
+
+
+def _close_keeper(conn: sqlite3.Connection) -> None:
+ try:
+ conn.close()
+ except Exception as exc:
+ logger.warning("Could not close the rag.db WAL keeper: %s", exc)
+
+
+def close_wal_keeper_for(path: str | Path) -> None:
+ """Release one account's RAG database before its directories are retired."""
+ resolved = Path(path).resolve()
+ with _wal_keeper_lock:
+ keeper = _wal_keepers.pop(resolved, None)
+ if keeper is not None:
+ _close_keeper(keeper)
+ for spelling, target in tuple(_wal_keeper_paths.items()):
+ if target == resolved:
+ del _wal_keeper_paths[spelling]
+
+
+def close_wal_keeper() -> None:
+ """Release all keepers; late workers cannot reopen them after shutdown."""
+ global _wal_keepers_enabled
+ with _wal_keeper_lock:
+ _wal_keepers_enabled = False
+ for keeper in _wal_keepers.values():
+ _close_keeper(keeper)
+ _wal_keepers.clear()
+ _wal_keeper_paths.clear()
+
+
def vec_table_dim(conn: sqlite3.Connection) -> int | None:
"""Embedding width baked into ``chunks_vec``, or None when absent."""
row = conn.execute(
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index a6cb256a360..a402f71cd34 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -498,14 +498,23 @@ def _forget_the_cached_owner_identity():
def _isolate_wal_keepers():
"""Close the WAL keepers a test opened, so a later test's assertion on _wal_keepers sees only
its own."""
- from storage import studio_db
+ from storage import rag_db, studio_db
before = set(studio_db._wal_keepers)
unsupported = set(studio_db._wal_unsupported)
+ rag_before = set(rag_db._wal_keepers)
+ rag_enabled = rag_db._wal_keepers_enabled
+ rag_db.start_wal_keepers()
yield
for path in set(studio_db._wal_keepers) - before:
studio_db.close_wal_keeper_for(path)
studio_db._wal_unsupported.intersection_update(unsupported)
+ for path in set(rag_db._wal_keepers) - rag_before:
+ rag_db.close_wal_keeper_for(path)
+ if rag_enabled:
+ rag_db.start_wal_keepers()
+ else:
+ rag_db.close_wal_keeper()
@pytest.fixture(autouse = True)
diff --git a/studio/backend/tests/test_account_storage.py b/studio/backend/tests/test_account_storage.py
index ab0e75c9944..eadc3d34a4f 100644
--- a/studio/backend/tests/test_account_storage.py
+++ b/studio/backend/tests/test_account_storage.py
@@ -11,7 +11,7 @@ from pathlib import Path
import pytest
-from storage import api_usage_db, profile_stats_db, studio_db
+from storage import api_usage_db, profile_stats_db, rag_db, studio_db
from utils.account_context import OWNER, AccountContext, account_thread, current_account_id, run_as
from utils.paths import storage_roots as roots
from utils.paths.lazy import LazyPath
@@ -19,6 +19,13 @@ from utils.paths.lazy import LazyPath
ALICE = AccountContext("11111111111111111111111111111111", "alice")
BOB = AccountContext("22222222222222222222222222222222", "bob")
ACCOUNTS = (OWNER, ALICE, BOB)
+RAG_DB_PARAM = pytest.param(
+ "rag_db",
+ marks = pytest.mark.skipif(
+ not rag_db.RAG_AVAILABLE or not hasattr(sqlite3.Connection, "enable_load_extension"),
+ reason = "sqlite-vec cannot be loaded here",
+ ),
+)
@pytest.fixture(autouse = True)
@@ -34,6 +41,8 @@ def account_home(tmp_path, monkeypatch):
monkeypatch.setenv("UNSLOTH_STUDIO_DOCUMENTS_HOME", str(tmp_path / "Documents"))
monkeypatch.setenv("UNSLOTH_STUDIO_PROJECTS_HOME", str(tmp_path / "Projects"))
studio_db.close_wal_keeper()
+ rag_db.close_wal_keeper()
+ rag_db.start_wal_keepers()
profile_stats_db._cache.clear()
memos = []
for name in (
@@ -52,6 +61,7 @@ def account_home(tmp_path, monkeypatch):
for memo in memos:
memo.clear()
studio_db.close_wal_keeper()
+ rag_db.close_wal_keeper()
profile_stats_db._cache.clear()
@@ -96,18 +106,24 @@ def test_each_database_gets_its_schema(account_home, module_name, table):
assert module._schema_ready == set()
-def test_wal_keepers_can_close_one_account(account_home):
[email protected]("module_name", ["studio_db", RAG_DB_PARAM])
+def test_wal_keepers_can_close_one_account(account_home, module_name):
+ module = importlib.import_module(f"storage.{module_name}")
+ path_fn = roots.rag_db_path if module_name == "rag_db" else roots.studio_db_path
for account in ACCOUNTS:
- assert run_as(account, studio_db.open_wal_keeper)
- paths = {account: run_as(account, roots.studio_db_path) for account in ACCOUNTS}
- alice = studio_db._wal_keepers[paths[ALICE]]
- studio_db.close_wal_keeper_for(paths[ALICE])
+ if module_name == "studio_db":
+ assert run_as(account, studio_db.open_wal_keeper)
+ else:
+ run_as(account, module.get_connection).close()
+ paths = {account: run_as(account, path_fn).resolve() for account in ACCOUNTS}
+ alice = module._wal_keepers[paths[ALICE]]
+ module.close_wal_keeper_for(paths[ALICE])
with pytest.raises(sqlite3.ProgrammingError, match = "closed"):
alice.execute("SELECT 1")
for account in (OWNER, BOB):
- assert studio_db._wal_keepers[paths[account]].execute("SELECT 1").fetchone()[0] == 1
- studio_db.close_wal_keeper()
- assert studio_db._wal_keepers == {}
+ assert module._wal_keepers[paths[account]].execute("SELECT 1").fetchone()[0] == 1
+ module.close_wal_keeper()
+ assert module._wal_keepers == {}
def test_a_managed_database_keeps_its_wal_across_a_connection_close(account_home):
@@ -130,19 +146,61 @@ def test_a_managed_database_keeps_its_wal_across_a_connection_close(account_home
assert alice_db.with_name("studio.db-wal").exists()
-def test_retiring_an_account_releases_its_wal_keeper(account_home):
[email protected]("module_name", ["studio_db", RAG_DB_PARAM])
+def test_retiring_an_account_releases_its_wal_keeper(account_home, module_name):
from routes.accounts import retire_account_roots
- studio_db.reset_schema_state_for_tests()
- alice_db = run_as(ALICE, roots.studio_db_path)
- run_as(ALICE, studio_db.get_connection).close()
- keeper = studio_db._wal_keepers[alice_db]
+ module = importlib.import_module(f"storage.{module_name}")
+ path_fn = roots.rag_db_path if module_name == "rag_db" else roots.studio_db_path
+ module.reset_schema_state_for_tests()
+ alice_db = run_as(ALICE, path_fn).resolve()
+ workspace = run_as(ALICE, roots.workspace_root)
+ run_as(ALICE, module.get_connection).close()
+ keeper = module._wal_keepers[alice_db]
retire_account_roots(ALICE)
- assert alice_db not in studio_db._wal_keepers
+ assert alice_db not in module._wal_keepers
+ assert not workspace.exists(), "the keeper must not prevent the Windows retirement rename"
with pytest.raises(sqlite3.ProgrammingError, match = "closed"):
keeper.execute("SELECT 1")
+def test_retirement_ignores_an_escaping_rag_link(account_home, monkeypatch, tmp_path):
+ from routes.accounts import retire_account_roots
+
+ monkeypatch.setattr("core.inference.mcp_client.close_mcp_sessions", lambda: None)
+ monkeypatch.setattr("core.inference.mcp_client.invalidate_tool_cache", lambda: None)
+ studio_db.reset_schema_state_for_tests()
+ workspace = run_as(ALICE, roots.workspace_root)
+ run_as(ALICE, studio_db.get_connection).close()
+ outside = tmp_path / "outside_rag"
+ outside.mkdir()
+ try:
+ (workspace / "rag").symlink_to(outside, target_is_directory = True)
+ except OSError:
+ pytest.skip("symlinks are not permitted here")
+ retire_account_roots(ALICE)
+ assert not workspace.exists()
+ assert outside.is_dir()
+
+
+def test_a_late_rag_open_does_not_reopen_a_retired_accounts_keeper(account_home):
+ """An open that passed ensure_dir before deletion and resumes after retirement released the
+ keeper must not leave a new one holding the workspace open."""
+ from core.training import account_jobs
+
+ alice_db = run_as(ALICE, roots.rag_db_path)
+ run_as(ALICE, rag_db.get_metadata_connection).close()
+ with sqlite3.connect(str(alice_db)) as conn:
+ conn.execute("PRAGMA journal_mode=WAL")
+ account_jobs._retired.add(ALICE.account_id)
+ rag_db.close_wal_keeper_for(alice_db)
+ run_as(ALICE, rag_db._ensure_wal_keeper, alice_db)
+ assert alice_db.resolve() not in rag_db._wal_keepers
+ account_jobs._retired.discard(ALICE.account_id)
+ run_as(ALICE, rag_db._ensure_wal_keeper, alice_db)
+ assert alice_db.resolve() in rag_db._wal_keepers
+
+
def _receipt() -> api_usage_db.ApiUsageReceipt:
return api_usage_db.ApiUsageReceipt(
id = "same-receipt",
diff --git a/studio/backend/tests/test_rag_wal_keeper.py b/studio/backend/tests/test_rag_wal_keeper.py
new file mode 100644
index 00000000000..84e51021227
--- /dev/null
+++ b/studio/backend/tests/test_rag_wal_keeper.py
@@ -0,0 +1,155 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""RAG operations retain their WAL files without blocking writes or storage teardown."""
+
+from contextlib import closing
+from pathlib import Path
+import sqlite3
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from types import SimpleNamespace
+
+import pytest
+
+from storage import rag_db
+
+# Some macOS Pythons build sqlite3 without extension loading, so the import alone is not enough.
+requires_sqlite_vec = pytest.mark.skipif(
+ not rag_db.RAG_AVAILABLE or not hasattr(sqlite3.Connection, "enable_load_extension"),
+ reason = "sqlite-vec cannot be loaded here",
+)
+
+
+def db(tmp_path, monkeypatch):
+ path = tmp_path / "rag.db"
+ rag_db.close_wal_keeper()
+ rag_db.start_wal_keepers()
+ monkeypatch.setattr(rag_db, "rag_db_path", lambda: path)
+ monkeypatch.setattr(rag_db, "_schema_ready", set())
+ yield path
+ rag_db.close_wal_keeper()
+
+
+def _sidecars(path):
+ return Path(str(path) + "-wal"), Path(str(path) + "-shm")
+
+
+@requires_sqlite_vec
+def test_rag_operations_retain_sidecars_until_keeper_closes(db):
+ with closing(rag_db.get_connection()) as conn:
+ conn.execute("CREATE TABLE keeper_probe (value INTEGER)")
+ conn.execute("INSERT INTO keeper_probe VALUES (42)")
+ conn.commit()
+ wal, shm = _sidecars(db)
+ identities = [(path.stat().st_dev, path.stat().st_ino) for path in (wal, shm)]
+
+ for _ in range(6):
+ with closing(rag_db.get_connection()) as conn:
+ assert conn.execute("SELECT value FROM keeper_probe").fetchone()[0] == 42
+ assert [(path.stat().st_dev, path.stat().st_ino) for path in (wal, shm)] == identities
+
+ rag_db.close_wal_keeper()
+ assert not wal.exists()
+ assert not shm.exists()
+ with closing(sqlite3.connect(str(db))) as conn:
+ assert conn.execute("SELECT value FROM keeper_probe").fetchone()[0] == 42
+
+
+@requires_sqlite_vec
+def test_idle_keeper_does_not_block_a_writer_or_checkpoint(db):
+ with closing(rag_db.get_connection()) as conn:
+ conn.execute("CREATE TABLE keeper_probe (value INTEGER)")
+ conn.commit()
+
+ with closing(sqlite3.connect(str(db), timeout = 1)) as conn:
+ conn.execute("INSERT INTO keeper_probe VALUES (7)")
+ conn.commit()
+ assert conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() == (0, 0, 0)
+ assert _sidecars(db)[0].stat().st_size == 0
+ with closing(rag_db.get_connection()) as conn:
+ assert conn.execute("SELECT value FROM keeper_probe").fetchone()[0] == 7
+
+
+@requires_sqlite_vec
+def test_metadata_first_does_not_suppress_a_later_wal_keeper(db):
+ with closing(rag_db.get_metadata_connection()) as conn:
+ assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "delete"
+ assert db.resolve() not in rag_db._wal_keepers
+
+ with closing(rag_db.get_connection()) as conn:
+ assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
+ assert db.resolve() in rag_db._wal_keepers
+ assert all(path.exists() for path in _sidecars(db))
+
+
+ "with_vectors",
+ [False, pytest.param(True, marks = requires_sqlite_vec)],
+ ids = ["plain-metadata", "vec0-schema"],
+)
+def test_metadata_keeper_does_not_require_vector_extension(db, monkeypatch, with_vectors):
+ connection = rag_db.get_connection() if with_vectors else sqlite3.connect(str(db))
+ with closing(connection) as conn:
+ if with_vectors:
+ rag_db.ensure_vec(conn, 3)
+ else:
+ conn.execute("PRAGMA journal_mode=WAL")
+ conn.execute("CREATE TABLE metadata_probe (value INTEGER)")
+ conn.execute("INSERT INTO metadata_probe VALUES (3)")
+ conn.commit()
+ rag_db.close_wal_keeper()
+ rag_db.start_wal_keepers()
+
+ def unavailable(_conn):
+ pytest.fail("A metadata-only keeper must not load sqlite-vec")
+
+ monkeypatch.setattr(rag_db, "sqlite_vec", SimpleNamespace(load = unavailable))
+ monkeypatch.setattr(rag_db, "RAG_AVAILABLE", False)
+ with closing(rag_db.get_metadata_connection()) as conn:
+ assert conn.execute("SELECT value FROM metadata_probe").fetchone()[0] == 3
+ if with_vectors:
+ assert (
+ conn.execute("SELECT 1 FROM sqlite_schema WHERE name='chunks_vec'").fetchone()
+ is not None
+ )
+ assert db.resolve() in rag_db._wal_keepers
+ assert all(path.exists() for path in _sidecars(db))
+
+
+@requires_sqlite_vec
+def test_shutdown_prevents_late_keeper_reopen_and_restart_enables_it(db):
+ rag_db.get_connection().close()
+ assert db.resolve() in rag_db._wal_keepers
+ rag_db.close_wal_keeper()
+
+ # A worker finishing after the bounded shutdown join may still record metadata.
+ with closing(rag_db.get_metadata_connection()) as conn:
+ conn.execute("SELECT count(*) FROM documents").fetchone()
+ assert not rag_db._wal_keepers
+ assert not any(path.exists() for path in _sidecars(db))
+
+ rag_db.start_wal_keepers()
+ rag_db.get_metadata_connection().close()
+ assert db.resolve() in rag_db._wal_keepers
+ assert all(path.exists() for path in _sidecars(db))
+
+
+@requires_sqlite_vec
+def test_concurrent_first_opens_share_one_keeper(db):
+ ready = threading.Barrier(8)
+
+ def read():
+ ready.wait(timeout = 10)
+ with closing(rag_db.get_connection()) as conn:
+ return conn.execute("SELECT count(*) FROM documents").fetchone()[0]
+
+ with ThreadPoolExecutor(max_workers = 8) as pool:
+ futures = [pool.submit(read) for _ in range(8)]
+ assert [future.result(timeout = 30) for future in futures] == [0] * 8
+ assert set(rag_db._wal_keepers) == {db.resolve()}
+ keeper = rag_db._wal_keepers[db.resolve()]
+ rag_db.get_connection().close()
+ assert rag_db._wal_keepers[db.resolve()] is keeper
+ assert all(path.exists() for path in _sidecars(db))