Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio RAG: reuse one rag.db connection per thread within a linked-folder sync pass (#16607)
Daniel Hanf240d743490c
diff --git a/studio/backend/core/rag/folder_sync.py b/studio/backend/core/rag/folder_sync.pyindex 1b1b54d3322..fcc3f78378e 100644--- a/studio/backend/core/rag/folder_sync.py+++ b/studio/backend/core/rag/folder_sync.py@@ -209,6 +209,7 @@ def _now() -> str:def _hash_file(path: str) -> str:+ rag_db.release_reused()digest = hashlib.sha256()with open(path, "rb") as source:for block in iter(lambda: source.read(1 << 20), b""):@@ -552,6 +553,7 @@ def _update_folder(def _remove_snapshot(path: str | None) -> None:if not path:return+ rag_db.release_reused()try:root = os.path.realpath(str(rag_uploads_root()))target = os.path.realpath(path)@@ -1211,6 +1213,7 @@ def _scan(def _snapshot(root: str, metadata: dict) -> str:+ rag_db.release_reused()source = metadata["path"]resolved = os.path.realpath(source)if not _is_within(root, resolved):@@ -1298,6 +1301,7 @@ def _check_running() -> None:def _check_root_identity(root: str, expected: tuple[int, int]) -> None:+ rag_db.release_reused()try:current = _root_identity(root)except RuntimeError as exc:@@ -1308,6 +1312,7 @@ def _check_root_identity(root: str, expected: tuple[int, int]) -> None:def _source_reappeared(root: str, relative_path: str, excluded: set[str]) -> bool:"""check scanner eligibility without following a replaced path component."""+ rag_db.release_reused()path = PurePosixPath(relative_path)parts = path.partsif (@@ -1549,35 +1554,36 @@ def _ingest_folder_document(_worker_state.stop_event = stop_eventdocument_id = Noneingestion_job = None- try:- _check_running()- document_id, ingestion_job = ingestion.start_ingestion(- folder["scope"],- folder["scope_id"] if folder["scope_type"] == "knowledge_base" else None,- None,- relative_path,- snapshot,- project_id = folder["scope_id"] if folder["scope_type"] == "project" else None,- model_name = embedding_model,- content_hash = content_hash,- dedupe = False,- linked_folder_id = folder["id"],- linked_relative_path = relative_path,- background = False,- reuse_identical = not rebuild,- )- result = ingestion.get_job_status(ingestion_job)- if result is None:- raise RuntimeError("Ingestion job disappeared")- if result["status"] != "completed":- raise RuntimeError(result.get("error") or "Ingestion failed")- _check_running()- return _IngestionResult(document_id, ingestion_job)- except Exception as exc: # noqa: BLE001 - the coordinator classifies it- if is_cancel_requested(folder["id"]):- # The unlink cut the ingest short; that is not a file failure.- return _IngestionResult(document_id, ingestion_job, _SyncCancelled())- return _IngestionResult(document_id, ingestion_job, exc)+ with rag_db.reuse_connections():+ try:+ _check_running()+ document_id, ingestion_job = ingestion.start_ingestion(+ folder["scope"],+ folder["scope_id"] if folder["scope_type"] == "knowledge_base" else None,+ None,+ relative_path,+ snapshot,+ project_id = folder["scope_id"] if folder["scope_type"] == "project" else None,+ model_name = embedding_model,+ content_hash = content_hash,+ dedupe = False,+ linked_folder_id = folder["id"],+ linked_relative_path = relative_path,+ background = False,+ reuse_identical = not rebuild,+ )+ result = ingestion.get_job_status(ingestion_job)+ if result is None:+ raise RuntimeError("Ingestion job disappeared")+ if result["status"] != "completed":+ raise RuntimeError(result.get("error") or "Ingestion failed")+ _check_running()+ return _IngestionResult(document_id, ingestion_job)+ except Exception as exc: # noqa: BLE001 - the coordinator classifies it+ if is_cancel_requested(folder["id"]):+ # The unlink cut the ingest short; that is not a file failure.+ return _IngestionResult(document_id, ingestion_job, _SyncCancelled())+ return _IngestionResult(document_id, ingestion_job, exc)def _cleanup_pending_ingestion(pending: _PendingIngestion, result: _IngestionResult) -> None:@@ -1713,72 +1719,75 @@ def _reconcile_folder(job_id: str) -> None:def finish_pending(pending: _PendingIngestion) -> None:nonlocal added, changed_count, processed- result = _IngestionResult()- try:+ with rag_db.reuse_connections():+ result = _IngestionResult()try:- result = pending.future.result()- except Exception as exc: # noqa: BLE001- result = _IngestionResult(error = exc)-- if result.error is not None:- if isinstance(- result.error, (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged)- ):- _cleanup_pending_ingestion(pending, result)- raise result.error- failures.append(pending.relative_path)- reason = _error_text(result.error, folder["path"])- failure_errors[pending.relative_path] = reason- ingest_failed[pending.relative_path] = {- "sig": _failure_signature(pending.metadata),- "at": time.time(),- "error": reason,- }- # A failed file may be a rename or copy of a vanished path, so grant one pass.- withheld.update(missing - already_withheld)- logger.warning(- "linked-folder ingestion failed for %s",- pending.relative_path,- exc_info = (type(result.error), result.error, result.error.__traceback__),- )- _cleanup_pending_ingestion(pending, result)- else:try:- _check_running()- _check_root_identity(folder["path"], scanned_identity)- _install_mapping(- folder,- pending.relative_path,- pending.metadata,- result.document_id,- pending.content_hash,- )- except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):- _cleanup_pending_ingestion(pending, result)- raise+ result = pending.future.result()except Exception as exc: # noqa: BLE001+ result = _IngestionResult(error = exc)++ if result.error is not None:+ if isinstance(+ result.error, (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged)+ ):+ _cleanup_pending_ingestion(pending, result)+ raise result.errorfailures.append(pending.relative_path)- failure_errors[pending.relative_path] = _error_text(exc, folder["path"])+ reason = _error_text(result.error, folder["path"])+ failure_errors[pending.relative_path] = reason+ ingest_failed[pending.relative_path] = {+ "sig": _failure_signature(pending.metadata),+ "at": time.time(),+ "error": reason,+ }+ # A failed file may be a rename or copy of a vanished path, so grant one pass.withheld.update(missing - already_withheld)logger.warning("linked-folder ingestion failed for %s",pending.relative_path,- exc_info = (type(exc), exc, exc.__traceback__),+ exc_info = (type(result.error), result.error, result.error.__traceback__),)_cleanup_pending_ingestion(pending, result)else:- if pending.relative_path in new:- added += 1+ try:+ _check_running()+ _check_root_identity(folder["path"], scanned_identity)+ _install_mapping(+ folder,+ pending.relative_path,+ pending.metadata,+ result.document_id,+ pending.content_hash,+ )+ except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):+ _cleanup_pending_ingestion(pending, result)+ raise+ except Exception as exc: # noqa: BLE001+ failures.append(pending.relative_path)+ failure_errors[pending.relative_path] = _error_text(exc, folder["path"])+ withheld.update(missing - already_withheld)+ logger.warning(+ "linked-folder ingestion failed for %s",+ pending.relative_path,+ exc_info = (type(exc), exc, exc.__traceback__),+ )+ _cleanup_pending_ingestion(pending, result)else:- changed_count += 1- finally:- prune_ingestion_job(result)- processed += 1- update_progress()+ if pending.relative_path in new:+ added += 1+ else:+ changed_count += 1+ finally:+ prune_ingestion_job(result)+ processed += 1+ update_progress()def drain_one() -> None:if not in_flight:return+ # Hold no rag.db handle while the workers run, as before the shared connection.+ rag_db.release_reused()completed, _ = wait(tuple(in_flight), return_when = FIRST_COMPLETED)for future in completed:pending = in_flight.pop(future)@@ -1812,94 +1821,97 @@ def _reconcile_folder(job_id: str) -> None:thread_name_prefix = "rag-folder-ingest",) as executor:try:- for rel in work:- if known_failed.get(rel, {}).get("sig") == _failure_signature(current[rel]):- # No per-file write: a folder can hold thousands of these.- if is_cancel_requested(folder["id"]):- raise _SyncCancelled- failures.append(rel)- ingest_failed[rel] = known_failed[rel]- failure_errors[rel] = known_failed[rel].get("error") or "Indexing failed"- if not cached_grace:- # missing only shrinks during the pass, so once covers every later failure- withheld.update(missing - already_withheld)- cached_grace = True- processed += 1- continue- if len(in_flight) >= workers:- drain_one()- snapshot = None- try:- _check_running()- metadata = current[rel]- snapshot = _snapshot(folder["path"], metadata)- content_hash = _hash_file(snapshot)- _check_running()- if (- not rebuild- and rel in changed- and content_hash == known[rel].get("content_hash")- ):- _check_root_identity(folder["path"], scanned_identity)- _update_mapping_metadata(folder["id"], rel, metadata, content_hash)- _remove_snapshot(snapshot)- snapshot = None- elif not rebuild and rel in new:- rename_key = (content_hash, os.path.splitext(rel)[1].lower())- rename_candidates = missing_by_content.get(rename_key)- if rename_candidates:- old_rel = rename_candidates.pop(0)- missing.discard(old_rel)+ with rag_db.reuse_connections():+ for rel in work:+ if known_failed.get(rel, {}).get("sig") == _failure_signature(current[rel]):+ # No per-file write: a folder can hold thousands of these.+ if is_cancel_requested(folder["id"]):+ raise _SyncCancelled+ failures.append(rel)+ ingest_failed[rel] = known_failed[rel]+ failure_errors[rel] = known_failed[rel].get("error") or "Indexing failed"+ if not cached_grace:+ # missing only shrinks during the pass, so once covers every later failure+ withheld.update(missing - already_withheld)+ cached_grace = True+ processed += 1+ continue+ if len(in_flight) >= workers:+ drain_one()+ snapshot = None+ try:+ _check_running()+ metadata = current[rel]+ # Also before the call: a wrapped _snapshot must not see the handle either.+ rag_db.release_reused()+ snapshot = _snapshot(folder["path"], metadata)+ content_hash = _hash_file(snapshot)+ _check_running()+ if (+ not rebuild+ and rel in changed+ and content_hash == known[rel].get("content_hash")+ ):_check_root_identity(folder["path"], scanned_identity)- _rename_mapping(folder["id"], old_rel, rel)_update_mapping_metadata(folder["id"], rel, metadata, content_hash)_remove_snapshot(snapshot)snapshot = None- renamed += 1- if snapshot is not None:- # reuse_identical needs a finished donor, so a copy waits for its in-flight twin.- while not rebuild and any(- p.content_hash == content_hash for p in in_flight.values()- ):- drain_one()- # Pool threads start as the owner account; rebind to this job's.- future = executor.submit(- run_as,- account,- _ingest_folder_document,- job_id,- stop_event,- folder,- rel,- snapshot,- content_hash,- embedding_model,- rebuild,- )- in_flight[future] = _PendingIngestion(- relative_path = rel,- snapshot = snapshot,- metadata = metadata,- content_hash = content_hash,- future = future,- )- snapshot = None- except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):- _remove_snapshot(snapshot)- raise- except Exception as exc: # noqa: BLE001- failures.append(rel)- failure_errors[rel] = _error_text(exc, folder["path"])- withheld.update(missing - already_withheld)- logger.warning("linked-folder ingestion failed for %s", rel, exc_info = True)- _remove_snapshot(snapshot)- if any(pending.relative_path == rel for pending in in_flight.values()):- continue- processed += 1+ elif not rebuild and rel in new:+ rename_key = (content_hash, os.path.splitext(rel)[1].lower())+ rename_candidates = missing_by_content.get(rename_key)+ if rename_candidates:+ old_rel = rename_candidates.pop(0)+ missing.discard(old_rel)+ _check_root_identity(folder["path"], scanned_identity)+ _rename_mapping(folder["id"], old_rel, rel)+ _update_mapping_metadata(folder["id"], rel, metadata, content_hash)+ _remove_snapshot(snapshot)+ snapshot = None+ renamed += 1+ if snapshot is not None:+ # reuse_identical needs a finished donor, so a copy waits for its in-flight twin.+ while not rebuild and any(+ p.content_hash == content_hash for p in in_flight.values()+ ):+ drain_one()+ # Pool threads start as the owner account; rebind to this job's.+ future = executor.submit(+ run_as,+ account,+ _ingest_folder_document,+ job_id,+ stop_event,+ folder,+ rel,+ snapshot,+ content_hash,+ embedding_model,+ rebuild,+ )+ in_flight[future] = _PendingIngestion(+ relative_path = rel,+ snapshot = snapshot,+ metadata = metadata,+ content_hash = content_hash,+ future = future,+ )+ snapshot = None+ except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):+ _remove_snapshot(snapshot)+ raise+ except Exception as exc: # noqa: BLE001+ failures.append(rel)+ failure_errors[rel] = _error_text(exc, folder["path"])+ withheld.update(missing - already_withheld)+ logger.warning("linked-folder ingestion failed for %s", rel, exc_info = True)+ _remove_snapshot(snapshot)+ if any(pending.relative_path == rel for pending in in_flight.values()):+ continue+ processed += 1+ update_progress()update_progress()- update_progress()- while in_flight:- drain_one()+ while in_flight:+ drain_one()except Exception:abort_in_flight()raise@@ -1915,19 +1927,20 @@ def _reconcile_folder(job_id: str) -> None:raise _FolderChanged(str(exc)) from excif not rescanned.keys().isdisjoint(removable):raise _FolderChanged("Linked source reappeared during reconciliation")- for rel in removable:- _check_running()- _check_root_identity(folder["path"], scanned_identity)- if _source_reappeared(folder["path"], rel, excluded):- raise _FolderChanged("Linked source reappeared during reconciliation")- _check_root_identity(folder["path"], scanned_identity)- _delete_mapping(folder["id"], rel)- deleted += 1- _set_job(- job_id,- deleted = deleted,- progress = (len(work) + deleted) / max(total, 1),- )+ with rag_db.reuse_connections():+ for rel in removable:+ _check_running()+ _check_root_identity(folder["path"], scanned_identity)+ if _source_reappeared(folder["path"], rel, excluded):+ raise _FolderChanged("Linked source reappeared during reconciliation")+ _check_root_identity(folder["path"], scanned_identity)+ _delete_mapping(folder["id"], rel)+ deleted += 1+ _set_job(+ job_id,+ deleted = deleted,+ progress = (len(work) + deleted) / max(total, 1),+ )_check_running()_store_withheld(folder["id"], withheld)diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.pyindex 506bbc32513..39335609dfc 100644--- a/studio/backend/storage/rag_db.py+++ b/studio/backend/storage/rag_db.py@@ -8,6 +8,7 @@ import loggingimport reimport sqlite3import threading+from contextlib import contextmanagerfrom pathlib import Pathfrom datetime import datetime, timezone@@ -315,6 +316,85 @@ def reset_schema_state_for_tests() -> None:_schema_ready.clear()+class _ReusedConnection(sqlite_gate.GatedConnection):+ """Shared by a reuse_connections() block: close() rolls back the uncommitted rest (what a real close+ discards) and keeps the connection open."""++ _reuse_slot = None++ def close(self) -> None:+ slot = self._reuse_slot+ if slot is None or slot.conn is not self:+ super().close()+ return+ if not slot.checked_out:+ return+ slot.checked_out = False+ try:+ self.rollback()+ except Exception: # noqa: BLE001 - a real close would not raise here either+ slot.discard()+++class _ReuseSlot:+ __slots__ = ("conn", "db_path", "checked_out")++ def __init__(self) -> None:+ self.conn = None+ self.db_path = None+ self.checked_out = False++ def checkout(self, db_path: Path):+ conn = self.conn+ if conn is None or self.checked_out or self.db_path != db_path:+ return None+ try:+ conn.row_factory = sqlite3.Row+ conn.execute("PRAGMA busy_timeout = 5000")+ except Exception: # noqa: BLE001 - unusable now, so the caller opens a new one+ self.discard()+ return None+ self.checked_out = True+ return conn++ def discard(self) -> None:+ conn, self.conn, self.db_path, self.checked_out = self.conn, None, None, False+ if conn is not None:+ conn._reuse_slot = None+ try:+ conn.close()+ except Exception: # noqa: BLE001+ logger.warning("Could not close a reused rag.db connection", exc_info = True)+++_reuse_state = threading.local()+++def release_reused() -> None:+ """Close this thread's idle shared connection before file work or waits, so an account rename on+ Windows sees the same open handles as without the block. No-op outside a block or while checked out."""+ slot = getattr(_reuse_state, "slot", None)+ if slot is not None and slot.conn is not None and not slot.checked_out:+ slot.discard()+++@contextmanager+def reuse_connections():+ """On this thread, get_connection() reuses one connection (a fresh open costs ~0.5 ms) unless it is+ still checked out, then opens a normal one. Callers commit and close as before. Nested blocks share+ the outer one, which closes it."""+ if getattr(_reuse_state, "slot", None) is not None:+ yield+ return+ slot = _ReuseSlot()+ _reuse_state.slot = slot+ try:+ yield+ finally:+ _reuse_state.slot = None+ slot.discard()++def get_connection() -> sqlite3.Connection:"""Open rag.db (WAL + sqlite-vec loaded, schema created once). Raises if the extension is unavailable."""global _extension_loaded@@ -323,18 +403,28 @@ def get_connection() -> sqlite3.Connection:db_path = rag_db_path()ensure_dir(db_path.parent)- conn = sqlite_gate.connect(db_path)- conn.row_factory = sqlite3.Row- # Wait for a lock instead of erroring: figure-heavy ingests hold connections across many seconds.- conn.execute("PRAGMA busy_timeout = 5000")- try:- conn.enable_load_extension(True)- sqlite_vec.load(conn)- conn.enable_load_extension(False)- except Exception as exc: # noqa: BLE001- conn.close()- _warn_unavailable_once(exc)- raise RagExtensionUnavailable(_RAG_UNAVAILABLE_MSG) from exc+ slot = getattr(_reuse_state, "slot", None)+ conn = slot.checkout(db_path) if slot is not None else None+ if conn is None:+ adopt = slot is not None and slot.conn is None+ if adopt:+ conn = sqlite_gate.connect(db_path, factory = _ReusedConnection)+ else:+ conn = sqlite_gate.connect(db_path)+ conn.row_factory = sqlite3.Row+ # Wait for a lock instead of erroring: figure-heavy ingests hold connections across many seconds.+ conn.execute("PRAGMA busy_timeout = 5000")+ try:+ conn.enable_load_extension(True)+ sqlite_vec.load(conn)+ conn.enable_load_extension(False)+ except Exception as exc: # noqa: BLE001+ conn.close()+ _warn_unavailable_once(exc)+ raise RagExtensionUnavailable(_RAG_UNAVAILABLE_MSG) from exc+ if adopt:+ slot.conn, slot.db_path, slot.checked_out = conn, db_path, True+ conn._reuse_slot = slot# Set before the schema step: the library loaded, whatever the db does next. Monotonic, no lock._extension_loaded = Truediff --git a/studio/backend/storage/sqlite_gate.py b/studio/backend/storage/sqlite_gate.pyindex a75542347fb..75fdc0953be 100644--- a/studio/backend/storage/sqlite_gate.py+++ b/studio/backend/storage/sqlite_gate.py@@ -31,9 +31,11 @@ class GatedConnection(sqlite3.Connection):def connect(database: str | os.PathLike[str], **kwargs: Any) -> sqlite3.Connection:- """sqlite3.connect with opens and closes serialized process-wide."""+ """sqlite3.connect with opens and closes serialized process-wide. A ``factory`` must subclass+ GatedConnection so its close stays gated."""+ kwargs.setdefault("factory", GatedConnection)with _CONNECTION_GATE:- return sqlite3.connect(str(database), factory = GatedConnection, **kwargs)+ return sqlite3.connect(str(database), **kwargs)def is_busy_error(exc: sqlite3.OperationalError) -> bool:diff --git a/studio/backend/tests/test_rag_connection_reuse.py b/studio/backend/tests/test_rag_connection_reuse.pynew file mode 100644index 00000000000..438c1057528--- /dev/null+++ b/studio/backend/tests/test_rag_connection_reuse.py@@ -0,0 +1,360 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""rag_db.reuse_connections(): one connection per thread inside a block, with every caller's commit+and close keeping its meaning, and linked-folder sync using it without changing what is durable."""++from __future__ import annotations++import os+import sqlite3+import threading+from contextlib import closing++import pytest++from storage import rag_db, sqlite_gate+from utils.paths import rag_db_path++pytestmark = pytest.mark.skipif(+ not rag_db.RAG_AVAILABLE or not hasattr(sqlite3.Connection, "enable_load_extension"),+ reason = "sqlite-vec cannot be loaded by this Python's sqlite3",+)+++def _make_table() -> None:+ with closing(rag_db.get_connection()) as conn:+ conn.execute("CREATE TABLE IF NOT EXISTS reuse_probe(k TEXT PRIMARY KEY, v INTEGER)")+ conn.commit()+++def _independent_rows() -> list[tuple]:+ conn = sqlite3.connect(str(rag_db_path()))+ try:+ return conn.execute("SELECT k, v FROM reuse_probe ORDER BY k").fetchall()+ finally:+ conn.close()+++def _closed(conn) -> bool:+ try:+ conn.execute("SELECT 1")+ except sqlite3.ProgrammingError:+ return True+ return False+++def test_outside_a_block_every_open_is_new_and_close_closes(rag_home):+ first = rag_db.get_connection()+ second = rag_db.get_connection()+ assert first is not second+ assert type(first) is sqlite_gate.GatedConnection+ first.close()+ second.close()+ assert _closed(first) and _closed(second)+++def test_block_hands_back_one_connection_and_close_discards_uncommitted(rag_home):+ _make_table()+ with rag_db.reuse_connections():+ conn = rag_db.get_connection()+ conn.execute("INSERT INTO reuse_probe VALUES('kept', 1)")+ conn.commit()+ conn.execute("INSERT INTO reuse_probe VALUES('dropped', 2)")+ assert conn.in_transaction+ conn.close()+ again = rag_db.get_connection()+ assert again is conn+ assert not again.in_transaction+ assert [tuple(row) for row in again.execute("SELECT k FROM reuse_probe ORDER BY k")] == [+ ("kept",)+ ]+ again.close()+ assert _independent_rows() == [("kept", 1)]+ assert _closed(conn)+++def test_an_open_while_the_shared_one_is_held_gets_its_own(rag_home):+ _make_table()+ with rag_db.reuse_connections():+ outer = rag_db.get_connection()+ outer.execute("INSERT INTO reuse_probe VALUES('outer', 1)")+ inner = rag_db.get_connection()+ assert inner is not outer+ inner.close()+ assert _closed(inner)+ # The inner close must not end the outer caller's transaction.+ assert outer.in_transaction+ outer.commit()+ outer.close()+ assert _independent_rows() == [("outer", 1)]+++def test_nested_blocks_share_and_only_the_outer_one_closes(rag_home):+ with rag_db.reuse_connections():+ conn = rag_db.get_connection()+ conn.close()+ with rag_db.reuse_connections():+ assert rag_db.get_connection() is conn+ conn.close()+ assert not _closed(conn)+ assert _closed(conn)+++def test_checkout_restores_fresh_connection_settings(rag_home):+ with rag_db.reuse_connections():+ conn = rag_db.get_connection()+ conn.row_factory = None+ conn.execute("PRAGMA busy_timeout = 60000")+ conn.close()+ again = rag_db.get_connection()+ assert again is conn+ assert again.row_factory is sqlite3.Row+ assert again.execute("PRAGMA busy_timeout").fetchone()[0] == 5000+ again.close()+++def test_a_dead_shared_connection_is_replaced(rag_home):+ with rag_db.reuse_connections():+ conn = rag_db.get_connection()+ conn.close()+ sqlite3.Connection.close(conn)+ fresh = rag_db.get_connection()+ assert fresh is not conn+ assert fresh.execute("SELECT 1").fetchone()[0] == 1+ fresh.close()+++def test_another_database_path_is_not_shared(rag_home, monkeypatch):+ with rag_db.reuse_connections():+ conn = rag_db.get_connection()+ conn.close()+ other = rag_home / "other"+ other.mkdir()+ monkeypatch.setattr(rag_db, "rag_db_path", lambda: other / "rag.db")+ elsewhere = rag_db.get_connection()+ assert elsewhere is not conn+ elsewhere.close()+ assert _closed(elsewhere)+++def test_blocks_are_per_thread(rag_home):+ seen = {}+ with rag_db.reuse_connections():+ mine = rag_db.get_connection()+ mine.close()++ def worker():+ conn = rag_db.get_connection()+ seen["other"] = conn+ conn.close()+ seen["closed"] = _closed(conn)++ thread = threading.Thread(target = worker)+ thread.start()+ thread.join()+ assert seen["other"] is not mine+ assert seen["closed"]+++def test_an_idle_shared_connection_does_not_block_writers_or_checkpoints(rag_home):+ _make_table()+ with rag_db.reuse_connections():+ conn = rag_db.get_connection()+ conn.execute("SELECT k FROM reuse_probe").fetchall()+ conn.execute("INSERT INTO reuse_probe VALUES('a', 1)")+ conn.commit()+ conn.close()+ other = sqlite3.connect(str(rag_db_path()), timeout = 0)+ try:+ other.execute("BEGIN IMMEDIATE")+ other.execute("INSERT INTO reuse_probe VALUES('b', 2)")+ other.commit()+ busy, _, _ = other.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()+ assert busy == 0+ finally:+ other.close()+++def test_retired_account_still_refuses_inside_a_block(rag_home, monkeypatch):+ from core.rag import account_db+ with rag_db.reuse_connections():+ account_db.get_connection().close()+ monkeypatch.setattr(account_db, "account_is_retired", lambda: True)+ with pytest.raises(RuntimeError, match = "Account is retired"):+ account_db.get_connection()+++def _run(folder_id: str) -> dict:+ from core.rag import folder_sync++ job_id = folder_sync.request_sync(folder_id)+ folder_sync.reconcile_folder(job_id)+ return folder_sync.get_job(job_id)+++def test_folder_sync_reuses_connections_and_commits_each_file(+ rag_home, stub_embeddings, monkeypatch+):+ from core.rag import folder_sync++ source = rag_home / "source"+ source.mkdir()+ for index in range(12):+ (source / f"f{index:02d}.txt").write_text(+ f"file {index} " + "words " * 30, encoding = "utf-8"+ )+ folder = folder_sync.create_folder(+ scope_type = "knowledge_base",+ scope_id = "scope-1",+ path = str(source),+ name = "Docs",+ )+ opens = []+ connect = sqlite_gate.connect++ def counting_connect(*args, **kwargs):+ opens.append(1)+ return connect(*args, **kwargs)++ monkeypatch.setattr(rag_db.sqlite_gate, "connect", counting_connect)+ assert _run(folder["id"])["status"] == "completed"+ attach_opens = len(opens)+ # Base opens ~14 per file, head ~3.+ assert attach_opens < 12 * 8++ # Metadata-only changes: each file's row must be committed before the next file starts.+ for path in source.iterdir():+ os.utime(path, ns = (10**18, 10**18))+ durable = []+ update = folder_sync._update_mapping_metadata++ def observed_update(folder_id, rel, metadata, content_hash):+ update(folder_id, rel, metadata, content_hash)+ conn = sqlite3.connect(str(rag_db_path()))+ try:+ durable.append(+ conn.execute(+ "SELECT mtime_ns FROM linked_folder_files WHERE folder_id=? AND relative_path=?",+ (folder_id, rel),+ ).fetchone()[0]+ )+ finally:+ conn.close()++ monkeypatch.setattr(folder_sync, "_update_mapping_metadata", observed_update)+ opens.clear()+ assert _run(folder["id"])["status"] == "completed"+ assert durable == [10**18] * 12+ # Base opens 4 per file, head 3.+ assert len(opens) < 20 + 12 * 4 - 6+++def test_release_reused_closes_only_an_idle_shared_connection(rag_home):+ rag_db.release_reused() # outside a block: nothing to do+ with rag_db.reuse_connections():+ held = rag_db.get_connection()+ rag_db.release_reused()+ assert not _closed(held)+ held.close()+ rag_db.release_reused()+ assert _closed(held)+ fresh = rag_db.get_connection()+ assert fresh is not held+ fresh.close()+ assert rag_db.get_connection() is fresh+ fresh.close()+ assert _closed(fresh)+++def test_a_stale_close_does_not_end_another_callers_work(rag_home):+ _make_table()+ with rag_db.reuse_connections():+ first = rag_db.get_connection()+ first.close()+ first.close()+ second = rag_db.get_connection()+ assert second is first+ second.close()+ rag_db.release_reused()+ third = rag_db.get_connection()+ assert third is not first+ third.execute("INSERT INTO reuse_probe VALUES('live', 1)")+ first.close()+ assert third.in_transaction+ third.commit()+ third.close()+ assert _independent_rows() == [("live", 1)]+++def test_folder_sync_holds_no_handle_while_waiting_statting_or_copying(+ rag_home, stub_embeddings, monkeypatch+):+ from core.rag import folder_sync++ source = rag_home / "source"+ source.mkdir()+ for index in range(6):+ (source / f"f{index}.txt").write_text(f"file {index} " + "words " * 20, encoding = "utf-8")+ folder = folder_sync.create_folder(+ scope_type = "knowledge_base",+ scope_id = "scope-1",+ path = str(source),+ name = "Docs",+ )+ created: list[tuple[int, sqlite3.Connection]] = []+ connect = sqlite_gate.connect++ def tracking_connect(*args, **kwargs):+ conn = connect(*args, **kwargs)+ created.append((threading.get_ident(), conn))+ return conn++ def open_on_this_thread() -> int:+ keepers = set(map(id, rag_db._wal_keepers.values()))+ me = threading.get_ident()+ return sum(+ 1+ for owner, conn in created+ if owner == me and id(conn) not in keepers and not _closed(conn)+ )++ # Probes sit inside the file work (after each helper's release), counting this thread's open handles.+ seen = {+ "wait": [],+ "reappeared": [],+ "snapshot_call": [],+ "snapshot": [],+ "identity": [],+ "uploads": [],+ }++ def probe(name, real):+ def wrapper(*args, **kwargs):+ seen[name].append(open_on_this_thread())+ return real(*args, **kwargs)++ monkeypatch.setattr(folder_sync, real.__name__, wrapper)++ probe("wait", folder_sync.wait)+ probe("reappeared", folder_sync.PurePosixPath)+ # A wrapper around _snapshot runs before the helper's own release, as a slow copy start would.+ probe("snapshot_call", folder_sync._snapshot)+ probe("snapshot", folder_sync._copy_exact)+ probe("identity", folder_sync._root_identity)+ probe("uploads", folder_sync.rag_uploads_root)+ monkeypatch.setattr(rag_db.sqlite_gate, "connect", tracking_connect)+ assert _run(folder["id"])["status"] == "completed"+ for path in source.iterdir():+ os.utime(path, ns = (10**18, 10**18))+ assert _run(folder["id"])["status"] == "completed"+ for index in range(3):+ (source / f"f{index}.txt").unlink()+ assert _run(folder["id"])["status"] == "completed"+ assert seen["wait"] and set(seen["wait"]) == {0}+ assert len(seen["reappeared"]) == 3 and set(seen["reappeared"]) == {0}+ assert len(seen["snapshot_call"]) == 12 and set(seen["snapshot_call"]) == {0}+ assert len(seen["snapshot"]) == 12 and set(seen["snapshot"]) == {0}+ assert len(seen["identity"]) >= 6 + 6 + 3 * 2 and set(seen["identity"]) == {0}+ # twelve snapshot targets, six touch-pass removals, three removals after _delete_mapping commits+ assert len(seen["uploads"]) >= 12 + 6 + 3 and set(seen["uploads"]) == {0}