Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio RAG: reuse one rag.db connection per thread within a linked-folder sync pass (#16607)

Daniel Hanf240d743490c
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.py
index 1b1b54d3322..fcc3f78378e 100644
--- a/studio/backend/core/rag/folder_sync.py
+++ b/studio/backend/core/rag/folder_sync.py
@@ -209,6 +209,7 @@ def _now() -> str:
def _hash_file(path: str) -> str:
+ rag_db.release_reused()
digest = hashlib.sha256()
with open(path, "rb") as source:
for block in iter(lambda: source.read(1 << 20), b""):
@@ -552,6 +553,7 @@ def _update_folder(
def _remove_snapshot(path: str | None) -> None:
if not path:
return
+ rag_db.release_reused()
try:
root = os.path.realpath(str(rag_uploads_root()))
target = os.path.realpath(path)
@@ -1211,6 +1213,7 @@ def _scan(
def _snapshot(root: str, metadata: dict) -> str:
+ rag_db.release_reused()
source = metadata["path"]
resolved = os.path.realpath(source)
if not _is_within(root, resolved):
@@ -1298,6 +1301,7 @@ def _check_running() -> None:
def _check_root_identity(root: str, expected: tuple[int, int]) -> None:
+ rag_db.release_reused()
try:
current = _root_identity(root)
except RuntimeError as exc:
@@ -1308,6 +1312,7 @@ def _check_root_identity(root: str, expected: tuple[int, int]) -> None:
def _source_reappeared(root: str, relative_path: str, excluded: set[str]) -> bool:
"""check scanner eligibility without following a replaced path component."""
+ rag_db.release_reused()
path = PurePosixPath(relative_path)
parts = path.parts
if (
@@ -1549,35 +1554,36 @@ def _ingest_folder_document(
_worker_state.stop_event = stop_event
document_id = None
ingestion_job = None
- try:
- _check_running()
- document_id, ingestion_job = ingestion.start_ingestion(
- folder["scope"],
- folder["scope_id"] if folder["scope_type"] == "knowledge_base" else None,
- None,
- relative_path,
- snapshot,
- project_id = folder["scope_id"] if folder["scope_type"] == "project" else None,
- model_name = embedding_model,
- content_hash = content_hash,
- dedupe = False,
- linked_folder_id = folder["id"],
- linked_relative_path = relative_path,
- background = False,
- reuse_identical = not rebuild,
- )
- result = ingestion.get_job_status(ingestion_job)
- if result is None:
- raise RuntimeError("Ingestion job disappeared")
- if result["status"] != "completed":
- raise RuntimeError(result.get("error") or "Ingestion failed")
- _check_running()
- return _IngestionResult(document_id, ingestion_job)
- except Exception as exc: # noqa: BLE001 - the coordinator classifies it
- if is_cancel_requested(folder["id"]):
- # The unlink cut the ingest short; that is not a file failure.
- return _IngestionResult(document_id, ingestion_job, _SyncCancelled())
- return _IngestionResult(document_id, ingestion_job, exc)
+ with rag_db.reuse_connections():
+ try:
+ _check_running()
+ document_id, ingestion_job = ingestion.start_ingestion(
+ folder["scope"],
+ folder["scope_id"] if folder["scope_type"] == "knowledge_base" else None,
+ None,
+ relative_path,
+ snapshot,
+ project_id = folder["scope_id"] if folder["scope_type"] == "project" else None,
+ model_name = embedding_model,
+ content_hash = content_hash,
+ dedupe = False,
+ linked_folder_id = folder["id"],
+ linked_relative_path = relative_path,
+ background = False,
+ reuse_identical = not rebuild,
+ )
+ result = ingestion.get_job_status(ingestion_job)
+ if result is None:
+ raise RuntimeError("Ingestion job disappeared")
+ if result["status"] != "completed":
+ raise RuntimeError(result.get("error") or "Ingestion failed")
+ _check_running()
+ return _IngestionResult(document_id, ingestion_job)
+ except Exception as exc: # noqa: BLE001 - the coordinator classifies it
+ if is_cancel_requested(folder["id"]):
+ # The unlink cut the ingest short; that is not a file failure.
+ return _IngestionResult(document_id, ingestion_job, _SyncCancelled())
+ return _IngestionResult(document_id, ingestion_job, exc)
def _cleanup_pending_ingestion(pending: _PendingIngestion, result: _IngestionResult) -> None:
@@ -1713,72 +1719,75 @@ def _reconcile_folder(job_id: str) -> None:
def finish_pending(pending: _PendingIngestion) -> None:
nonlocal added, changed_count, processed
- result = _IngestionResult()
- try:
+ with rag_db.reuse_connections():
+ result = _IngestionResult()
try:
- result = pending.future.result()
- except Exception as exc: # noqa: BLE001
- result = _IngestionResult(error = exc)
-
- if result.error is not None:
- if isinstance(
- result.error, (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged)
- ):
- _cleanup_pending_ingestion(pending, result)
- raise result.error
- failures.append(pending.relative_path)
- reason = _error_text(result.error, folder["path"])
- failure_errors[pending.relative_path] = reason
- ingest_failed[pending.relative_path] = {
- "sig": _failure_signature(pending.metadata),
- "at": time.time(),
- "error": reason,
- }
- # A failed file may be a rename or copy of a vanished path, so grant one pass.
- withheld.update(missing - already_withheld)
- logger.warning(
- "linked-folder ingestion failed for %s",
- pending.relative_path,
- exc_info = (type(result.error), result.error, result.error.__traceback__),
- )
- _cleanup_pending_ingestion(pending, result)
- else:
try:
- _check_running()
- _check_root_identity(folder["path"], scanned_identity)
- _install_mapping(
- folder,
- pending.relative_path,
- pending.metadata,
- result.document_id,
- pending.content_hash,
- )
- except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):
- _cleanup_pending_ingestion(pending, result)
- raise
+ result = pending.future.result()
except Exception as exc: # noqa: BLE001
+ result = _IngestionResult(error = exc)
+
+ if result.error is not None:
+ if isinstance(
+ result.error, (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged)
+ ):
+ _cleanup_pending_ingestion(pending, result)
+ raise result.error
failures.append(pending.relative_path)
- failure_errors[pending.relative_path] = _error_text(exc, folder["path"])
+ reason = _error_text(result.error, folder["path"])
+ failure_errors[pending.relative_path] = reason
+ ingest_failed[pending.relative_path] = {
+ "sig": _failure_signature(pending.metadata),
+ "at": time.time(),
+ "error": reason,
+ }
+ # A failed file may be a rename or copy of a vanished path, so grant one pass.
withheld.update(missing - already_withheld)
logger.warning(
"linked-folder ingestion failed for %s",
pending.relative_path,
- exc_info = (type(exc), exc, exc.__traceback__),
+ exc_info = (type(result.error), result.error, result.error.__traceback__),
)
_cleanup_pending_ingestion(pending, result)
else:
- if pending.relative_path in new:
- added += 1
+ try:
+ _check_running()
+ _check_root_identity(folder["path"], scanned_identity)
+ _install_mapping(
+ folder,
+ pending.relative_path,
+ pending.metadata,
+ result.document_id,
+ pending.content_hash,
+ )
+ except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):
+ _cleanup_pending_ingestion(pending, result)
+ raise
+ except Exception as exc: # noqa: BLE001
+ failures.append(pending.relative_path)
+ failure_errors[pending.relative_path] = _error_text(exc, folder["path"])
+ withheld.update(missing - already_withheld)
+ logger.warning(
+ "linked-folder ingestion failed for %s",
+ pending.relative_path,
+ exc_info = (type(exc), exc, exc.__traceback__),
+ )
+ _cleanup_pending_ingestion(pending, result)
else:
- changed_count += 1
- finally:
- prune_ingestion_job(result)
- processed += 1
- update_progress()
+ if pending.relative_path in new:
+ added += 1
+ else:
+ changed_count += 1
+ finally:
+ prune_ingestion_job(result)
+ processed += 1
+ update_progress()
def drain_one() -> None:
if not in_flight:
return
+ # Hold no rag.db handle while the workers run, as before the shared connection.
+ rag_db.release_reused()
completed, _ = wait(tuple(in_flight), return_when = FIRST_COMPLETED)
for future in completed:
pending = in_flight.pop(future)
@@ -1812,94 +1821,97 @@ def _reconcile_folder(job_id: str) -> None:
thread_name_prefix = "rag-folder-ingest",
) as executor:
try:
- for rel in work:
- if known_failed.get(rel, {}).get("sig") == _failure_signature(current[rel]):
- # No per-file write: a folder can hold thousands of these.
- if is_cancel_requested(folder["id"]):
- raise _SyncCancelled
- failures.append(rel)
- ingest_failed[rel] = known_failed[rel]
- failure_errors[rel] = known_failed[rel].get("error") or "Indexing failed"
- if not cached_grace:
- # missing only shrinks during the pass, so once covers every later failure
- withheld.update(missing - already_withheld)
- cached_grace = True
- processed += 1
- continue
- if len(in_flight) >= workers:
- drain_one()
- snapshot = None
- try:
- _check_running()
- metadata = current[rel]
- snapshot = _snapshot(folder["path"], metadata)
- content_hash = _hash_file(snapshot)
- _check_running()
- if (
- not rebuild
- and rel in changed
- and content_hash == known[rel].get("content_hash")
- ):
- _check_root_identity(folder["path"], scanned_identity)
- _update_mapping_metadata(folder["id"], rel, metadata, content_hash)
- _remove_snapshot(snapshot)
- snapshot = None
- elif not rebuild and rel in new:
- rename_key = (content_hash, os.path.splitext(rel)[1].lower())
- rename_candidates = missing_by_content.get(rename_key)
- if rename_candidates:
- old_rel = rename_candidates.pop(0)
- missing.discard(old_rel)
+ with rag_db.reuse_connections():
+ for rel in work:
+ if known_failed.get(rel, {}).get("sig") == _failure_signature(current[rel]):
+ # No per-file write: a folder can hold thousands of these.
+ if is_cancel_requested(folder["id"]):
+ raise _SyncCancelled
+ failures.append(rel)
+ ingest_failed[rel] = known_failed[rel]
+ failure_errors[rel] = known_failed[rel].get("error") or "Indexing failed"
+ if not cached_grace:
+ # missing only shrinks during the pass, so once covers every later failure
+ withheld.update(missing - already_withheld)
+ cached_grace = True
+ processed += 1
+ continue
+ if len(in_flight) >= workers:
+ drain_one()
+ snapshot = None
+ try:
+ _check_running()
+ metadata = current[rel]
+ # Also before the call: a wrapped _snapshot must not see the handle either.
+ rag_db.release_reused()
+ snapshot = _snapshot(folder["path"], metadata)
+ content_hash = _hash_file(snapshot)
+ _check_running()
+ if (
+ not rebuild
+ and rel in changed
+ and content_hash == known[rel].get("content_hash")
+ ):
_check_root_identity(folder["path"], scanned_identity)
- _rename_mapping(folder["id"], old_rel, rel)
_update_mapping_metadata(folder["id"], rel, metadata, content_hash)
_remove_snapshot(snapshot)
snapshot = None
- renamed += 1
- if snapshot is not None:
- # reuse_identical needs a finished donor, so a copy waits for its in-flight twin.
- while not rebuild and any(
- p.content_hash == content_hash for p in in_flight.values()
- ):
- drain_one()
- # Pool threads start as the owner account; rebind to this job's.
- future = executor.submit(
- run_as,
- account,
- _ingest_folder_document,
- job_id,
- stop_event,
- folder,
- rel,
- snapshot,
- content_hash,
- embedding_model,
- rebuild,
- )
- in_flight[future] = _PendingIngestion(
- relative_path = rel,
- snapshot = snapshot,
- metadata = metadata,
- content_hash = content_hash,
- future = future,
- )
- snapshot = None
- except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):
- _remove_snapshot(snapshot)
- raise
- except Exception as exc: # noqa: BLE001
- failures.append(rel)
- failure_errors[rel] = _error_text(exc, folder["path"])
- withheld.update(missing - already_withheld)
- logger.warning("linked-folder ingestion failed for %s", rel, exc_info = True)
- _remove_snapshot(snapshot)
- if any(pending.relative_path == rel for pending in in_flight.values()):
- continue
- processed += 1
+ elif not rebuild and rel in new:
+ rename_key = (content_hash, os.path.splitext(rel)[1].lower())
+ rename_candidates = missing_by_content.get(rename_key)
+ if rename_candidates:
+ old_rel = rename_candidates.pop(0)
+ missing.discard(old_rel)
+ _check_root_identity(folder["path"], scanned_identity)
+ _rename_mapping(folder["id"], old_rel, rel)
+ _update_mapping_metadata(folder["id"], rel, metadata, content_hash)
+ _remove_snapshot(snapshot)
+ snapshot = None
+ renamed += 1
+ if snapshot is not None:
+ # reuse_identical needs a finished donor, so a copy waits for its in-flight twin.
+ while not rebuild and any(
+ p.content_hash == content_hash for p in in_flight.values()
+ ):
+ drain_one()
+ # Pool threads start as the owner account; rebind to this job's.
+ future = executor.submit(
+ run_as,
+ account,
+ _ingest_folder_document,
+ job_id,
+ stop_event,
+ folder,
+ rel,
+ snapshot,
+ content_hash,
+ embedding_model,
+ rebuild,
+ )
+ in_flight[future] = _PendingIngestion(
+ relative_path = rel,
+ snapshot = snapshot,
+ metadata = metadata,
+ content_hash = content_hash,
+ future = future,
+ )
+ snapshot = None
+ except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):
+ _remove_snapshot(snapshot)
+ raise
+ except Exception as exc: # noqa: BLE001
+ failures.append(rel)
+ failure_errors[rel] = _error_text(exc, folder["path"])
+ withheld.update(missing - already_withheld)
+ logger.warning("linked-folder ingestion failed for %s", rel, exc_info = True)
+ _remove_snapshot(snapshot)
+ if any(pending.relative_path == rel for pending in in_flight.values()):
+ continue
+ processed += 1
+ update_progress()
update_progress()
- update_progress()
- while in_flight:
- drain_one()
+ while in_flight:
+ drain_one()
except Exception:
abort_in_flight()
raise
@@ -1915,19 +1927,20 @@ def _reconcile_folder(job_id: str) -> None:
raise _FolderChanged(str(exc)) from exc
if not rescanned.keys().isdisjoint(removable):
raise _FolderChanged("Linked source reappeared during reconciliation")
- for rel in removable:
- _check_running()
- _check_root_identity(folder["path"], scanned_identity)
- if _source_reappeared(folder["path"], rel, excluded):
- raise _FolderChanged("Linked source reappeared during reconciliation")
- _check_root_identity(folder["path"], scanned_identity)
- _delete_mapping(folder["id"], rel)
- deleted += 1
- _set_job(
- job_id,
- deleted = deleted,
- progress = (len(work) + deleted) / max(total, 1),
- )
+ with rag_db.reuse_connections():
+ for rel in removable:
+ _check_running()
+ _check_root_identity(folder["path"], scanned_identity)
+ if _source_reappeared(folder["path"], rel, excluded):
+ raise _FolderChanged("Linked source reappeared during reconciliation")
+ _check_root_identity(folder["path"], scanned_identity)
+ _delete_mapping(folder["id"], rel)
+ deleted += 1
+ _set_job(
+ job_id,
+ deleted = deleted,
+ progress = (len(work) + deleted) / max(total, 1),
+ )
_check_running()
_store_withheld(folder["id"], withheld)
diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.py
index 506bbc32513..39335609dfc 100644
--- a/studio/backend/storage/rag_db.py
+++ b/studio/backend/storage/rag_db.py
@@ -8,6 +8,7 @@ import logging
import re
import sqlite3
import threading
+from contextlib import contextmanager
from pathlib import Path
from datetime import datetime, timezone
@@ -315,6 +316,85 @@ def reset_schema_state_for_tests() -> None:
_schema_ready.clear()
+class _ReusedConnection(sqlite_gate.GatedConnection):
+ """Shared by a reuse_connections() block: close() rolls back the uncommitted rest (what a real close
+ discards) and keeps the connection open."""
+
+ _reuse_slot = None
+
+ def close(self) -> None:
+ slot = self._reuse_slot
+ if slot is None or slot.conn is not self:
+ super().close()
+ return
+ if not slot.checked_out:
+ return
+ slot.checked_out = False
+ try:
+ self.rollback()
+ except Exception: # noqa: BLE001 - a real close would not raise here either
+ slot.discard()
+
+
+class _ReuseSlot:
+ __slots__ = ("conn", "db_path", "checked_out")
+
+ def __init__(self) -> None:
+ self.conn = None
+ self.db_path = None
+ self.checked_out = False
+
+ def checkout(self, db_path: Path):
+ conn = self.conn
+ if conn is None or self.checked_out or self.db_path != db_path:
+ return None
+ try:
+ conn.row_factory = sqlite3.Row
+ conn.execute("PRAGMA busy_timeout = 5000")
+ except Exception: # noqa: BLE001 - unusable now, so the caller opens a new one
+ self.discard()
+ return None
+ self.checked_out = True
+ return conn
+
+ def discard(self) -> None:
+ conn, self.conn, self.db_path, self.checked_out = self.conn, None, None, False
+ if conn is not None:
+ conn._reuse_slot = None
+ try:
+ conn.close()
+ except Exception: # noqa: BLE001
+ logger.warning("Could not close a reused rag.db connection", exc_info = True)
+
+
+_reuse_state = threading.local()
+
+
+def release_reused() -> None:
+ """Close this thread's idle shared connection before file work or waits, so an account rename on
+ Windows sees the same open handles as without the block. No-op outside a block or while checked out."""
+ slot = getattr(_reuse_state, "slot", None)
+ if slot is not None and slot.conn is not None and not slot.checked_out:
+ slot.discard()
+
+
+@contextmanager
+def reuse_connections():
+ """On this thread, get_connection() reuses one connection (a fresh open costs ~0.5 ms) unless it is
+ still checked out, then opens a normal one. Callers commit and close as before. Nested blocks share
+ the outer one, which closes it."""
+ if getattr(_reuse_state, "slot", None) is not None:
+ yield
+ return
+ slot = _ReuseSlot()
+ _reuse_state.slot = slot
+ try:
+ yield
+ finally:
+ _reuse_state.slot = None
+ slot.discard()
+
+
def get_connection() -> sqlite3.Connection:
"""Open rag.db (WAL + sqlite-vec loaded, schema created once). Raises if the extension is unavailable."""
global _extension_loaded
@@ -323,18 +403,28 @@ def get_connection() -> sqlite3.Connection:
db_path = rag_db_path()
ensure_dir(db_path.parent)
- conn = sqlite_gate.connect(db_path)
- conn.row_factory = sqlite3.Row
- # Wait for a lock instead of erroring: figure-heavy ingests hold connections across many seconds.
- conn.execute("PRAGMA busy_timeout = 5000")
- try:
- conn.enable_load_extension(True)
- sqlite_vec.load(conn)
- conn.enable_load_extension(False)
- except Exception as exc: # noqa: BLE001
- conn.close()
- _warn_unavailable_once(exc)
- raise RagExtensionUnavailable(_RAG_UNAVAILABLE_MSG) from exc
+ slot = getattr(_reuse_state, "slot", None)
+ conn = slot.checkout(db_path) if slot is not None else None
+ if conn is None:
+ adopt = slot is not None and slot.conn is None
+ if adopt:
+ conn = sqlite_gate.connect(db_path, factory = _ReusedConnection)
+ else:
+ conn = sqlite_gate.connect(db_path)
+ conn.row_factory = sqlite3.Row
+ # Wait for a lock instead of erroring: figure-heavy ingests hold connections across many seconds.
+ conn.execute("PRAGMA busy_timeout = 5000")
+ try:
+ conn.enable_load_extension(True)
+ sqlite_vec.load(conn)
+ conn.enable_load_extension(False)
+ except Exception as exc: # noqa: BLE001
+ conn.close()
+ _warn_unavailable_once(exc)
+ raise RagExtensionUnavailable(_RAG_UNAVAILABLE_MSG) from exc
+ if adopt:
+ slot.conn, slot.db_path, slot.checked_out = conn, db_path, True
+ conn._reuse_slot = slot
# Set before the schema step: the library loaded, whatever the db does next. Monotonic, no lock.
_extension_loaded = True
diff --git a/studio/backend/storage/sqlite_gate.py b/studio/backend/storage/sqlite_gate.py
index a75542347fb..75fdc0953be 100644
--- a/studio/backend/storage/sqlite_gate.py
+++ b/studio/backend/storage/sqlite_gate.py
@@ -31,9 +31,11 @@ class GatedConnection(sqlite3.Connection):
def connect(database: str | os.PathLike[str], **kwargs: Any) -> sqlite3.Connection:
- """sqlite3.connect with opens and closes serialized process-wide."""
+ """sqlite3.connect with opens and closes serialized process-wide. A ``factory`` must subclass
+ GatedConnection so its close stays gated."""
+ kwargs.setdefault("factory", GatedConnection)
with _CONNECTION_GATE:
- return sqlite3.connect(str(database), factory = GatedConnection, **kwargs)
+ return sqlite3.connect(str(database), **kwargs)
def is_busy_error(exc: sqlite3.OperationalError) -> bool:
diff --git a/studio/backend/tests/test_rag_connection_reuse.py b/studio/backend/tests/test_rag_connection_reuse.py
new file mode 100644
index 00000000000..438c1057528
--- /dev/null
+++ b/studio/backend/tests/test_rag_connection_reuse.py
@@ -0,0 +1,360 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""rag_db.reuse_connections(): one connection per thread inside a block, with every caller's commit
+and close keeping its meaning, and linked-folder sync using it without changing what is durable."""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+import threading
+from contextlib import closing
+
+import pytest
+
+from storage import rag_db, sqlite_gate
+from utils.paths import rag_db_path
+
+pytestmark = pytest.mark.skipif(
+ not rag_db.RAG_AVAILABLE or not hasattr(sqlite3.Connection, "enable_load_extension"),
+ reason = "sqlite-vec cannot be loaded by this Python's sqlite3",
+)
+
+
+def _make_table() -> None:
+ with closing(rag_db.get_connection()) as conn:
+ conn.execute("CREATE TABLE IF NOT EXISTS reuse_probe(k TEXT PRIMARY KEY, v INTEGER)")
+ conn.commit()
+
+
+def _independent_rows() -> list[tuple]:
+ conn = sqlite3.connect(str(rag_db_path()))
+ try:
+ return conn.execute("SELECT k, v FROM reuse_probe ORDER BY k").fetchall()
+ finally:
+ conn.close()
+
+
+def _closed(conn) -> bool:
+ try:
+ conn.execute("SELECT 1")
+ except sqlite3.ProgrammingError:
+ return True
+ return False
+
+
+def test_outside_a_block_every_open_is_new_and_close_closes(rag_home):
+ first = rag_db.get_connection()
+ second = rag_db.get_connection()
+ assert first is not second
+ assert type(first) is sqlite_gate.GatedConnection
+ first.close()
+ second.close()
+ assert _closed(first) and _closed(second)
+
+
+def test_block_hands_back_one_connection_and_close_discards_uncommitted(rag_home):
+ _make_table()
+ with rag_db.reuse_connections():
+ conn = rag_db.get_connection()
+ conn.execute("INSERT INTO reuse_probe VALUES('kept', 1)")
+ conn.commit()
+ conn.execute("INSERT INTO reuse_probe VALUES('dropped', 2)")
+ assert conn.in_transaction
+ conn.close()
+ again = rag_db.get_connection()
+ assert again is conn
+ assert not again.in_transaction
+ assert [tuple(row) for row in again.execute("SELECT k FROM reuse_probe ORDER BY k")] == [
+ ("kept",)
+ ]
+ again.close()
+ assert _independent_rows() == [("kept", 1)]
+ assert _closed(conn)
+
+
+def test_an_open_while_the_shared_one_is_held_gets_its_own(rag_home):
+ _make_table()
+ with rag_db.reuse_connections():
+ outer = rag_db.get_connection()
+ outer.execute("INSERT INTO reuse_probe VALUES('outer', 1)")
+ inner = rag_db.get_connection()
+ assert inner is not outer
+ inner.close()
+ assert _closed(inner)
+ # The inner close must not end the outer caller's transaction.
+ assert outer.in_transaction
+ outer.commit()
+ outer.close()
+ assert _independent_rows() == [("outer", 1)]
+
+
+def test_nested_blocks_share_and_only_the_outer_one_closes(rag_home):
+ with rag_db.reuse_connections():
+ conn = rag_db.get_connection()
+ conn.close()
+ with rag_db.reuse_connections():
+ assert rag_db.get_connection() is conn
+ conn.close()
+ assert not _closed(conn)
+ assert _closed(conn)
+
+
+def test_checkout_restores_fresh_connection_settings(rag_home):
+ with rag_db.reuse_connections():
+ conn = rag_db.get_connection()
+ conn.row_factory = None
+ conn.execute("PRAGMA busy_timeout = 60000")
+ conn.close()
+ again = rag_db.get_connection()
+ assert again is conn
+ assert again.row_factory is sqlite3.Row
+ assert again.execute("PRAGMA busy_timeout").fetchone()[0] == 5000
+ again.close()
+
+
+def test_a_dead_shared_connection_is_replaced(rag_home):
+ with rag_db.reuse_connections():
+ conn = rag_db.get_connection()
+ conn.close()
+ sqlite3.Connection.close(conn)
+ fresh = rag_db.get_connection()
+ assert fresh is not conn
+ assert fresh.execute("SELECT 1").fetchone()[0] == 1
+ fresh.close()
+
+
+def test_another_database_path_is_not_shared(rag_home, monkeypatch):
+ with rag_db.reuse_connections():
+ conn = rag_db.get_connection()
+ conn.close()
+ other = rag_home / "other"
+ other.mkdir()
+ monkeypatch.setattr(rag_db, "rag_db_path", lambda: other / "rag.db")
+ elsewhere = rag_db.get_connection()
+ assert elsewhere is not conn
+ elsewhere.close()
+ assert _closed(elsewhere)
+
+
+def test_blocks_are_per_thread(rag_home):
+ seen = {}
+ with rag_db.reuse_connections():
+ mine = rag_db.get_connection()
+ mine.close()
+
+ def worker():
+ conn = rag_db.get_connection()
+ seen["other"] = conn
+ conn.close()
+ seen["closed"] = _closed(conn)
+
+ thread = threading.Thread(target = worker)
+ thread.start()
+ thread.join()
+ assert seen["other"] is not mine
+ assert seen["closed"]
+
+
+def test_an_idle_shared_connection_does_not_block_writers_or_checkpoints(rag_home):
+ _make_table()
+ with rag_db.reuse_connections():
+ conn = rag_db.get_connection()
+ conn.execute("SELECT k FROM reuse_probe").fetchall()
+ conn.execute("INSERT INTO reuse_probe VALUES('a', 1)")
+ conn.commit()
+ conn.close()
+ other = sqlite3.connect(str(rag_db_path()), timeout = 0)
+ try:
+ other.execute("BEGIN IMMEDIATE")
+ other.execute("INSERT INTO reuse_probe VALUES('b', 2)")
+ other.commit()
+ busy, _, _ = other.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
+ assert busy == 0
+ finally:
+ other.close()
+
+
+def test_retired_account_still_refuses_inside_a_block(rag_home, monkeypatch):
+ from core.rag import account_db
+ with rag_db.reuse_connections():
+ account_db.get_connection().close()
+ monkeypatch.setattr(account_db, "account_is_retired", lambda: True)
+ with pytest.raises(RuntimeError, match = "Account is retired"):
+ account_db.get_connection()
+
+
+def _run(folder_id: str) -> dict:
+ from core.rag import folder_sync
+
+ job_id = folder_sync.request_sync(folder_id)
+ folder_sync.reconcile_folder(job_id)
+ return folder_sync.get_job(job_id)
+
+
+def test_folder_sync_reuses_connections_and_commits_each_file(
+ rag_home, stub_embeddings, monkeypatch
+):
+ from core.rag import folder_sync
+
+ source = rag_home / "source"
+ source.mkdir()
+ for index in range(12):
+ (source / f"f{index:02d}.txt").write_text(
+ f"file {index} " + "words " * 30, encoding = "utf-8"
+ )
+ folder = folder_sync.create_folder(
+ scope_type = "knowledge_base",
+ scope_id = "scope-1",
+ path = str(source),
+ name = "Docs",
+ )
+ opens = []
+ connect = sqlite_gate.connect
+
+ def counting_connect(*args, **kwargs):
+ opens.append(1)
+ return connect(*args, **kwargs)
+
+ monkeypatch.setattr(rag_db.sqlite_gate, "connect", counting_connect)
+ assert _run(folder["id"])["status"] == "completed"
+ attach_opens = len(opens)
+ # Base opens ~14 per file, head ~3.
+ assert attach_opens < 12 * 8
+
+ # Metadata-only changes: each file's row must be committed before the next file starts.
+ for path in source.iterdir():
+ os.utime(path, ns = (10**18, 10**18))
+ durable = []
+ update = folder_sync._update_mapping_metadata
+
+ def observed_update(folder_id, rel, metadata, content_hash):
+ update(folder_id, rel, metadata, content_hash)
+ conn = sqlite3.connect(str(rag_db_path()))
+ try:
+ durable.append(
+ conn.execute(
+ "SELECT mtime_ns FROM linked_folder_files WHERE folder_id=? AND relative_path=?",
+ (folder_id, rel),
+ ).fetchone()[0]
+ )
+ finally:
+ conn.close()
+
+ monkeypatch.setattr(folder_sync, "_update_mapping_metadata", observed_update)
+ opens.clear()
+ assert _run(folder["id"])["status"] == "completed"
+ assert durable == [10**18] * 12
+ # Base opens 4 per file, head 3.
+ assert len(opens) < 20 + 12 * 4 - 6
+
+
+def test_release_reused_closes_only_an_idle_shared_connection(rag_home):
+ rag_db.release_reused() # outside a block: nothing to do
+ with rag_db.reuse_connections():
+ held = rag_db.get_connection()
+ rag_db.release_reused()
+ assert not _closed(held)
+ held.close()
+ rag_db.release_reused()
+ assert _closed(held)
+ fresh = rag_db.get_connection()
+ assert fresh is not held
+ fresh.close()
+ assert rag_db.get_connection() is fresh
+ fresh.close()
+ assert _closed(fresh)
+
+
+def test_a_stale_close_does_not_end_another_callers_work(rag_home):
+ _make_table()
+ with rag_db.reuse_connections():
+ first = rag_db.get_connection()
+ first.close()
+ first.close()
+ second = rag_db.get_connection()
+ assert second is first
+ second.close()
+ rag_db.release_reused()
+ third = rag_db.get_connection()
+ assert third is not first
+ third.execute("INSERT INTO reuse_probe VALUES('live', 1)")
+ first.close()
+ assert third.in_transaction
+ third.commit()
+ third.close()
+ assert _independent_rows() == [("live", 1)]
+
+
+def test_folder_sync_holds_no_handle_while_waiting_statting_or_copying(
+ rag_home, stub_embeddings, monkeypatch
+):
+ from core.rag import folder_sync
+
+ source = rag_home / "source"
+ source.mkdir()
+ for index in range(6):
+ (source / f"f{index}.txt").write_text(f"file {index} " + "words " * 20, encoding = "utf-8")
+ folder = folder_sync.create_folder(
+ scope_type = "knowledge_base",
+ scope_id = "scope-1",
+ path = str(source),
+ name = "Docs",
+ )
+ created: list[tuple[int, sqlite3.Connection]] = []
+ connect = sqlite_gate.connect
+
+ def tracking_connect(*args, **kwargs):
+ conn = connect(*args, **kwargs)
+ created.append((threading.get_ident(), conn))
+ return conn
+
+ def open_on_this_thread() -> int:
+ keepers = set(map(id, rag_db._wal_keepers.values()))
+ me = threading.get_ident()
+ return sum(
+ 1
+ for owner, conn in created
+ if owner == me and id(conn) not in keepers and not _closed(conn)
+ )
+
+ # Probes sit inside the file work (after each helper's release), counting this thread's open handles.
+ seen = {
+ "wait": [],
+ "reappeared": [],
+ "snapshot_call": [],
+ "snapshot": [],
+ "identity": [],
+ "uploads": [],
+ }
+
+ def probe(name, real):
+ def wrapper(*args, **kwargs):
+ seen[name].append(open_on_this_thread())
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(folder_sync, real.__name__, wrapper)
+
+ probe("wait", folder_sync.wait)
+ probe("reappeared", folder_sync.PurePosixPath)
+ # A wrapper around _snapshot runs before the helper's own release, as a slow copy start would.
+ probe("snapshot_call", folder_sync._snapshot)
+ probe("snapshot", folder_sync._copy_exact)
+ probe("identity", folder_sync._root_identity)
+ probe("uploads", folder_sync.rag_uploads_root)
+ monkeypatch.setattr(rag_db.sqlite_gate, "connect", tracking_connect)
+ assert _run(folder["id"])["status"] == "completed"
+ for path in source.iterdir():
+ os.utime(path, ns = (10**18, 10**18))
+ assert _run(folder["id"])["status"] == "completed"
+ for index in range(3):
+ (source / f"f{index}.txt").unlink()
+ assert _run(folder["id"])["status"] == "completed"
+ assert seen["wait"] and set(seen["wait"]) == {0}
+ assert len(seen["reappeared"]) == 3 and set(seen["reappeared"]) == {0}
+ assert len(seen["snapshot_call"]) == 12 and set(seen["snapshot_call"]) == {0}
+ assert len(seen["snapshot"]) == 12 and set(seen["snapshot"]) == {0}
+ assert len(seen["identity"]) >= 6 + 6 + 3 * 2 and set(seen["identity"]) == {0}
+ # twelve snapshot targets, six touch-pass removals, three removals after _delete_mapping commits
+ assert len(seen["uploads"]) >= 12 + 6 + 3 and set(seen["uploads"]) == {0}