Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Durable, bounded-pipeline reconciliation for linked local RAG folders."""
from __future__ import annotations
from core.training.account_jobs import (
account_is_retired,
account_key,
account_path,
job_accounts,
)
from utils.account_context import account_thread, current_account, run_as
import hashlib
import json
import logging
import os
import sqlite3
import stat
import threading
import time
import uuid
import weakref
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
from contextlib import closing
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
from core.rag import account_db as rag_db
from utils.paths import ensure_dir, rag_uploads_root
from . import config, embeddings, gitignore, ingestion, job_leases, store
from utils.paths.path_utils import is_appledouble_metadata
logger = logging.getLogger(__name__)
_wake = threading.Event()
_stop = threading.Event()
_thread: threading.Thread | None = None
_thread_stop: threading.Event | None = None
_thread_lock = threading.Lock()
_worker_lock = threading.Lock()
_worker_state = threading.local()
# Scheduling cursor of the single worker; see _next_account_job.
_last_job_account: str | None = None
_folder_locks: weakref.WeakValueDictionary[str, threading.RLock] = weakref.WeakValueDictionary()
_scope_locks: weakref.WeakValueDictionary[str, threading.RLock] = weakref.WeakValueDictionary()
_named_locks_lock = threading.Lock()
# Folders being unlinked, read by the worker without rag.db: an ingest can hold the writer past the
# default 5 s busy_timeout, so the retire write alone could not stop it (#13095).
_folder_cancels: set = set()
_folder_cancels_lock = threading.Lock()
_UNLINK_BUSY_TIMEOUT_MS = 30_000
_retirement_schema_lock = threading.Lock()
_TERMINAL = {"completed", "failed"}
_MAX_FOLDER_DEPTH = 64
_MAX_NAMED_FAILURES = 3
_MAX_WITHHELD_PATHS = 500
_FAILED_RETRY_S = 6 * 3600
_JOB_EVENT_KEEPALIVE_S = 4.0
_SQLITE_INTEGER_MAX = (1 << 63) - 1
# Dependency, VCS and cache trees would flood the index with third-party source.
_IGNORE_SCAN_DIRS = frozenset(
{
".git",
".svn",
".hg",
".venv",
"venv",
"node_modules",
"bower_components",
"__pycache__",
".pytest_cache",
".mypy_cache",
}
)
_LOCKFILES = frozenset(
{"package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", "pnpm-lock.yml"}
)
def _is_ignored_scan_dir(name: str, path: str) -> bool:
# hidden subtrees can contain application configuration and plugins such as .obsidian.
return (
name.startswith(".")
or name.lower() in _IGNORE_SCAN_DIRS
or os.path.exists(os.path.join(path, "pyvenv.cfg"))
)
# Build output is skipped only beside a project manifest: a bare "build" folder in a vault is content.
_BUILD_OUTPUT_DIRS = frozenset({"dist", "build", "out", "target", "coverage", "htmlcov", "vendor"})
_PROJECT_MANIFESTS = (
"package.json",
"pyproject.toml",
"setup.py",
"setup.cfg",
"Cargo.toml",
"pom.xml",
"build.gradle",
"build.gradle.kts",
"go.mod",
"composer.json",
"CMakeLists.txt",
"Gemfile",
"mix.exs",
)
_MAX_REPORTED_FAILURES = 500
def _is_build_output_dir(name: str, parent: str) -> bool:
return name.lower() in _BUILD_OUTPUT_DIRS and any(
os.path.isfile(os.path.join(parent, manifest)) for manifest in _PROJECT_MANIFESTS
)
def _read_gitignore(directory: str, base: str) -> list[gitignore.Rule]:
"""An unreadable .gitignore fails the scan: indexing without it would take in excluded files."""
path = os.path.join(directory, ".gitignore")
if not os.path.isfile(path) or os.path.islink(path):
return []
shown = f"{base}/.gitignore" if base else ".gitignore"
try:
with open(path, "rb") as handle:
data = handle.read(gitignore.MAX_GITIGNORE_BYTES + 1)
except OSError as exc:
raise RuntimeError(f"Couldn't read {shown}: {exc.strerror or exc}") from exc
if len(data) > gitignore.MAX_GITIGNORE_BYTES:
raise RuntimeError(f"{shown} is larger than {gitignore.MAX_GITIGNORE_BYTES // 1024} KB")
# utf-8-sig drops a leading BOM, as git does (dir.c add_patterns_from_buffer, skip_utf8_bom).
return gitignore.parse(data.decode("utf-8-sig", "replace"), base)
def _is_plain_text_ext(ext: str) -> bool:
return ext == ".txt" or ext in config.SOURCE_TEXT_EXTS
def _is_ignored_scan_file(name: str) -> bool:
# .env and Terraform state can expose plaintext secrets in retrieval; lockfiles add noise.
lower = name.lower()
if lower == ".env" or lower.startswith(".env.") or lower.endswith(".env"):
return True
return lower.endswith((".lock", ".lockb", ".tfstate")) or lower in _LOCKFILES
class _SyncStopped(Exception):
pass
class _SyncCancelled(Exception):
pass
class _LeaseLost(Exception):
pass
class _FolderChanged(RuntimeError):
pass
@dataclass
class _IngestionResult:
document_id: str | None = None
ingestion_job: str | None = None
error: Exception | None = None
@dataclass
class _PendingIngestion:
relative_path: str
snapshot: str
metadata: dict
content_hash: str
future: Future["_IngestionResult"]
def request_cancel(folder_id: str) -> None:
with _folder_cancels_lock:
_folder_cancels.add(account_key(folder_id))
def clear_cancel(folder_id: str) -> None:
with _folder_cancels_lock:
_folder_cancels.discard(account_key(folder_id))
def is_cancel_requested(folder_id: str | None) -> bool:
if folder_id is None:
return False
with _folder_cancels_lock:
return account_key(folder_id) in _folder_cancels
def _local_chat_busy() -> bool:
from state import active_generations
# Cloud-provider chats use no local compute; embedding requests are indexing's own kind of work.
return any(
entry["kind"] != "embeddings"
for entry in active_generations.snapshot(include_external = False)
)
def wait_while_chat_busy() -> None:
"""Hold a linked-folder ingest thread while a local generation runs; stop and unlink end the wait."""
folder_id = getattr(_worker_state, "folder_id", None)
if folder_id is None or not config.FOLDER_YIELD_TO_CHAT:
return
stop_event = getattr(_worker_state, "stop_event", _stop)
while _local_chat_busy():
if account_is_retired() or is_cancel_requested(folder_id) or stop_event.wait(0.5):
return
def _folder_lock(folder_id: str) -> threading.RLock:
with _named_locks_lock:
return _folder_locks.setdefault(account_key(folder_id), threading.RLock())
def _scope_lock(scope: str) -> threading.RLock:
with _named_locks_lock:
return _scope_locks.setdefault(account_key(scope), threading.RLock())
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _hash_file(path: str) -> str:
rag_db.release_reused()
digest = hashlib.sha256()
with open(path, "rb") as source:
for block in iter(lambda: source.read(1 << 20), b""):
digest.update(block)
return digest.hexdigest()
def _path_key(path: str) -> str:
return os.path.normcase(os.path.normpath(path))
def _same_file(left: str, right: str) -> bool:
try:
return os.path.samefile(left, right)
except OSError:
return False
def _paths_overlap(left: str, right: str) -> bool:
try:
if os.path.commonpath((left, right)) in (left, right):
return True
except ValueError:
pass
return any(_same_file(left, str(parent)) for parent in Path(right).parents) or any(
_same_file(right, str(parent)) for parent in Path(left).parents
)
def _error_text(exc: Exception, native_path: str | None = None) -> str:
from utils.native_path_leases import redact_native_paths
error = redact_native_paths(str(exc) or exc.__class__.__name__)
if native_path:
error = error.replace(native_path, "<native_path>")
error = error.replace(os.path.normpath(native_path), "<native_path>")
return error
def _is_within(root: str, path: str) -> bool:
try:
return os.path.normcase(os.path.commonpath([root, path])) == os.path.normcase(root)
except ValueError:
return False
def validate_folder_path(path: str) -> str:
"""Apply the existing model scan-folder policy without persisting there."""
account_path(path)
if not path or not path.strip() or "\x00" in path:
raise ValueError("Path cannot be empty")
expanded = os.path.abspath(os.path.expanduser(path))
try:
if stat.S_ISLNK(os.lstat(expanded).st_mode):
raise ValueError("Symbolic-link folders are not allowed")
except OSError as exc:
raise ValueError("Path does not exist") from exc
normalized = os.path.realpath(expanded)
uploads_root = os.path.realpath(str(rag_uploads_root()))
if _paths_overlap(_path_key(normalized), _path_key(uploads_root)):
raise ValueError("The managed RAG uploads folder cannot be linked")
from hub.storage.scan_folders import (
contains_sensitive_path_component,
is_denied_system_path,
)
from utils.paths.external_media import is_local_filesystem_root
if not os.path.isdir(normalized):
raise ValueError("Path must be a directory, not a file")
if not os.access(normalized, os.R_OK | os.X_OK):
raise ValueError("Path is not readable")
if is_local_filesystem_root(normalized):
raise ValueError("The filesystem root cannot be registered")
try:
if Path(normalized) == Path.home().resolve():
raise ValueError("The entire home folder cannot be registered")
except RuntimeError:
pass
if contains_sensitive_path_component(normalized):
raise ValueError("Credential or configuration directories are not allowed")
if is_denied_system_path(normalized):
raise ValueError("System directories are not allowed")
return normalized
def _root_identity(root: str) -> tuple[int, int]:
try:
root_stat = os.lstat(root)
except OSError as exc:
raise RuntimeError("Linked folder is unavailable") from exc
if stat.S_ISLNK(root_stat.st_mode) or not stat.S_ISDIR(root_stat.st_mode):
raise RuntimeError("Linked folder is no longer a regular directory")
if os.path.normcase(os.path.realpath(root)) != os.path.normcase(root):
raise RuntimeError("Linked folder no longer resolves to its registered path")
return root_stat.st_dev, root_stat.st_ino
def _store_identity(identity: tuple[int, int]) -> tuple[int | str, int | str]:
return tuple(value if value <= _SQLITE_INTEGER_MAX else f"x{value:x}" for value in identity)
def _file_identity(metadata: dict) -> tuple[int | str, int | str]:
return _store_identity((metadata["device"], metadata["inode"]))
def _load_identity(device_id: int | str, file_id: int | str) -> tuple[int, int]:
def load(value: int | str) -> int:
return (
int(value[1:], 16) if isinstance(value, str) and value.startswith("x") else int(value)
)
return load(device_id), load(file_id)
def _mount_points() -> frozenset[str]:
"""Return known mount boundaries without assuming directory identities are unique."""
if os.name != "posix" or not os.path.exists("/proc/self/mountinfo"):
return frozenset()
escaped = {"\\040": " ", "\\011": "\t", "\\012": "\n", "\\134": "\\"}
points = set()
try:
with open("/proc/self/mountinfo", encoding = "utf-8") as mountinfo:
for line in mountinfo:
fields = line.split()
if len(fields) < 5:
continue
path = fields[4]
for encoded, decoded in escaped.items():
path = path.replace(encoded, decoded)
points.add(_path_key(os.path.realpath(path)))
except OSError:
return frozenset()
return frozenset(points)
def create_folder(
*,
scope_type: str,
scope_id: str,
path: str,
expected_identity: tuple[int, int] | None = None,
name: str | None = None,
auto_sync: bool = True,
) -> dict:
if scope_type not in {"knowledge_base", "project"}:
raise ValueError("Linked folders support only knowledge-base and project scopes")
normalized = validate_folder_path(path)
try:
root_device, root_inode = _root_identity(normalized)
except RuntimeError as exc:
raise ValueError(str(exc)) from exc
if expected_identity is not None and (root_device, root_inode) != expected_identity:
raise ValueError("Linked folder changed after it was selected")
scope = (
store.kb_scope(scope_id)
if scope_type == "knowledge_base"
else store.project_scope(scope_id)
)
folder_id = str(uuid.uuid4())
now = _now()
with _scope_lock(scope):
conn = rag_db.get_connection()
try:
conn.execute("BEGIN IMMEDIATE")
if conn.execute(
"SELECT 1 FROM linked_folder_retired_scopes WHERE scope=?", (scope,)
).fetchone():
raise ValueError("The linked-folder scope no longer exists")
normalized_key = _path_key(normalized)
existing = conn.execute(
"SELECT * FROM linked_folders WHERE scope=?", (scope,)
).fetchall()
for row in existing:
existing_key = _path_key(row["path"])
if existing_key == normalized_key or _same_file(row["path"], normalized):
if row["status"] == "retired" or row["delete_remove_index"] is not None:
raise ValueError("Linked folder is still being removed")
conn.rollback()
return _reauthorize_folder(row["id"], normalized, expected_identity)
if _paths_overlap(existing_key, normalized_key):
raise ValueError("Linked folders in the same scope cannot overlap")
try:
current_identity = _root_identity(normalized)
except RuntimeError as exc:
raise ValueError(str(exc)) from exc
if current_identity != (root_device, root_inode) or (
expected_identity is not None and current_identity != expected_identity
):
raise ValueError("Linked folder changed after it was selected")
conn.execute(
"INSERT INTO linked_folders(id, scope_type, scope_id, scope, path, name, "
"root_device, root_inode, auto_sync, status, created_at, updated_at) "
"VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
(
folder_id,
scope_type,
scope_id,
scope,
normalized,
(name or Path(normalized).name or normalized).strip(),
*_store_identity((root_device, root_inode)),
int(auto_sync),
"pending",
now,
now,
),
)
conn.commit()
return dict(
conn.execute("SELECT * FROM linked_folders WHERE id=?", (folder_id,)).fetchone()
)
except Exception:
conn.rollback()
raise
finally:
conn.close()
def _reauthorize_folder(
folder_id: str, path: str, expected_identity: tuple[int, int] | None
) -> dict:
with _folder_lock(folder_id):
conn = rag_db.get_connection()
try:
conn.execute("BEGIN IMMEDIATE")
if (
conn.execute("SELECT 1 FROM linked_folders WHERE id=?", (folder_id,)).fetchone()
is None
):
raise ValueError("Linked folder changed while it was reauthorized")
try:
identity = _root_identity(path)
except RuntimeError as exc:
raise ValueError(str(exc)) from exc
if expected_identity is not None and identity != expected_identity:
raise ValueError("Linked folder changed after it was selected")
conn.execute(
"UPDATE linked_folders SET root_device=?, root_inode=?, updated_at=? WHERE id=?",
(*_store_identity(identity), _now(), folder_id),
)
conn.commit()
return dict(
conn.execute("SELECT * FROM linked_folders WHERE id=?", (folder_id,)).fetchone()
)
except Exception:
conn.rollback()
raise
finally:
conn.close()
def create_folder_with_sync(
*,
scope_type: str,
scope_id: str,
path: str,
expected_identity: tuple[int, int] | None = None,
name: str | None = None,
auto_sync: bool = True,
) -> tuple[dict, str]:
if scope_type not in {"knowledge_base", "project"}:
raise ValueError("Linked folders support only knowledge-base and project scopes")
scope = (
store.kb_scope(scope_id)
if scope_type == "knowledge_base"
else store.project_scope(scope_id)
)
with _scope_lock(scope):
folder = create_folder(
scope_type = scope_type,
scope_id = scope_id,
path = path,
expected_identity = expected_identity,
name = name,
auto_sync = auto_sync,
)
return folder, request_sync(folder["id"])
def get_folder(folder_id: str) -> dict | None:
with closing(rag_db.get_connection()) as conn:
row = conn.execute("SELECT * FROM linked_folders WHERE id=?", (folder_id,)).fetchone()
return dict(row) if row else None
def list_folders(scope: str) -> list[dict]:
with closing(rag_db.get_connection()) as conn:
rows = conn.execute(
"SELECT f.*, COUNT(ff.relative_path) AS file_count, (SELECT j.id FROM "
"linked_folder_sync_jobs j WHERE j.folder_id=f.id AND j.status IN ('pending','running') "
"ORDER BY j.created_at LIMIT 1) AS active_job_id FROM linked_folders f "
"LEFT JOIN linked_folder_files ff ON ff.folder_id=f.id WHERE f.scope=? "
"GROUP BY f.id ORDER BY f.created_at",
(scope,),
).fetchall()
return [dict(row) for row in rows]
def update_folder(
folder_id: str,
*,
name: str | None = None,
auto_sync: bool | None = None,
) -> dict:
with _folder_lock(folder_id):
return _update_folder(folder_id, name = name, auto_sync = auto_sync)
def _update_folder(
folder_id: str,
*,
name: str | None = None,
auto_sync: bool | None = None,
) -> dict:
with closing(rag_db.get_connection()) as conn:
row = conn.execute(
"SELECT status, delete_remove_index FROM linked_folders WHERE id=?", (folder_id,)
).fetchone()
if row is None or row["status"] == "retired" or row["delete_remove_index"] is not None:
raise KeyError(folder_id)
if name is not None:
clean_name = name.strip()
if not clean_name:
raise ValueError("Folder name cannot be empty")
conn.execute(
"UPDATE linked_folders SET name=?, updated_at=? WHERE id=?",
(clean_name, _now(), folder_id),
)
if auto_sync is not None:
conn.execute(
"UPDATE linked_folders SET auto_sync=?, updated_at=? WHERE id=?",
(int(auto_sync), _now(), folder_id),
)
conn.commit()
return dict(
conn.execute("SELECT * FROM linked_folders WHERE id=?", (folder_id,)).fetchone()
)
def _remove_snapshot(path: str | None) -> None:
if not path:
return
rag_db.release_reused()
try:
root = os.path.realpath(str(rag_uploads_root()))
target = os.path.realpath(path)
if _is_within(root, target) and os.path.isfile(target):
os.remove(target)
except Exception:
logger.warning("failed to remove linked-folder snapshot", exc_info = True)
def _remove_retired_snapshot(path: str | None) -> None:
"""Remove a managed snapshot or raise so its retirement remains retryable."""
if not path:
return
root = os.path.realpath(str(rag_uploads_root()))
target = os.path.realpath(path)
if not _is_within(root, target):
return
try:
os.remove(target)
except FileNotFoundError:
pass
def _delete_retired_folder(folder_id: str) -> bool | None:
"""Finish a durable unlink once no process still owns its sync job."""
conn = rag_db.get_connection()
try:
conn.execute(f"PRAGMA busy_timeout = {_UNLINK_BUSY_TIMEOUT_MS}")
conn.execute("BEGIN IMMEDIATE")
folder = conn.execute(
"SELECT delete_remove_index FROM linked_folders "
"WHERE id=? AND delete_remove_index IS NOT NULL",
(folder_id,),
).fetchone()
if folder is None:
conn.rollback()
return None
if conn.execute(
"SELECT 1 FROM linked_folder_sync_jobs j JOIN rag_job_leases l "
"ON l.kind=? AND l.job_id=j.id WHERE j.folder_id=? AND l.expires_at>? LIMIT 1",
(job_leases.FOLDER_SYNC, folder_id, _now()),
).fetchone():
conn.rollback()
return False
docs = conn.execute(
"SELECT d.id, d.stored_path FROM linked_folder_files ff "
"JOIN documents d ON d.id=ff.document_id WHERE ff.folder_id=?",
(folder_id,),
).fetchall()
if folder["delete_remove_index"]:
for snapshot in dict.fromkeys(doc["stored_path"] for doc in docs):
_remove_retired_snapshot(snapshot)
conn.execute("DELETE FROM linked_folder_files WHERE folder_id=?", (folder_id,))
if folder["delete_remove_index"]:
store.delete_documents(conn, [doc["id"] for doc in docs], commit = False)
else:
for doc in docs:
conn.execute(
"UPDATE documents SET linked_folder_id=NULL, linked_relative_path=NULL "
"WHERE id=?",
(doc["id"],),
)
conn.execute(
"DELETE FROM rag_job_leases WHERE kind=? AND job_id IN "
"(SELECT id FROM linked_folder_sync_jobs WHERE folder_id=?)",
(job_leases.FOLDER_SYNC, folder_id),
)
conn.execute("DELETE FROM linked_folder_sync_jobs WHERE folder_id=?", (folder_id,))
conn.execute("DELETE FROM linked_folders WHERE id=?", (folder_id,))
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
return True
def _reconcile_retired_folder_deletions() -> None:
with closing(rag_db.get_connection()) as conn:
folder_ids = [
row["id"]
for row in conn.execute(
"SELECT id FROM linked_folders WHERE delete_remove_index IS NOT NULL"
)
]
for folder_id in folder_ids:
_delete_retired_folder(folder_id)
def delete_folder(folder_id: str, *, remove_index: bool = True) -> bool:
request_cancel(folder_id)
try:
conn = rag_db.get_connection()
try:
conn.execute(f"PRAGMA busy_timeout = {_UNLINK_BUSY_TIMEOUT_MS}")
conn.execute("BEGIN IMMEDIATE")
if (
conn.execute("SELECT 1 FROM linked_folders WHERE id=?", (folder_id,)).fetchone()
is None
):
conn.rollback()
return False
conn.execute(
"UPDATE linked_folders SET auto_sync=0, status='retired', "
"delete_remove_index=COALESCE(delete_remove_index, ?), updated_at=? WHERE id=?",
(int(remove_index), _now(), folder_id),
)
conn.execute(
"UPDATE linked_folder_sync_jobs SET status='failed', stage='error', "
"error='Linked folder was removed', completed_at=? "
"WHERE folder_id=? AND status IN ('pending','running')",
(_now(), folder_id),
)
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
while _delete_retired_folder(folder_id) is False:
time.sleep(0.05)
return True
finally:
clear_cancel(folder_id)
def _retirement_connection():
conn = rag_db.get_metadata_connection()
try:
with _retirement_schema_lock:
conn.execute("BEGIN IMMEDIATE")
conn.execute(
"CREATE TABLE IF NOT EXISTS linked_folder_retired_scopes ("
"scope TEXT NOT NULL PRIMARY KEY, retired_at TEXT NOT NULL, purged_at TEXT)"
)
rag_db.ensure_linked_folder_columns(conn)
conn.commit()
except Exception:
conn.close()
raise
return conn
def _metadata_table_exists(conn, table: str) -> bool:
return (
conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
).fetchone()
is not None
)
# SQLite's default host-parameter cap is 999, so a bound id list is applied in chunks.
_ID_CHUNK = 500
def _id_chunks(folder_ids: list[str]) -> list[list[str]]:
unique = list(dict.fromkeys(folder_ids))
return [unique[start : start + _ID_CHUNK] for start in range(0, len(unique), _ID_CHUNK)]
def linked_folder_ids(scope: str) -> list[str]:
"""Reads only the metadata DB, so deletes finish even when the vector extension cannot load."""
with closing(rag_db.get_metadata_connection()) as conn:
if not _metadata_table_exists(conn, "linked_folders"):
return []
return [
row["id"]
for row in conn.execute("SELECT id FROM linked_folders WHERE scope=?", (scope,))
]
def _retire_scope_rows(
conn,
scope: str,
folder_ids: list[str] | None = None,
rows: bool = True,
) -> None:
folders_exist = rows and _metadata_table_exists(conn, "linked_folders")
jobs_exist = folders_exist and _metadata_table_exists(conn, "linked_folder_sync_jobs")
conn.execute(
"INSERT OR IGNORE INTO linked_folder_retired_scopes(scope, retired_at) VALUES(?, ?)",
(scope, _now()),
)
# Bounded to folder ids, not created_at: Windows clock has ~15.6ms resolution.
batches = [None] if folder_ids is None else _id_chunks(folder_ids)
for batch in batches:
bound = "" if batch is None else f" AND id IN ({','.join('?' * len(batch))})"
params = () if batch is None else tuple(batch)
if folders_exist:
conn.execute(
"UPDATE linked_folders SET auto_sync=0, status='retired', "
f"last_error='Owning scope was removed', updated_at=? WHERE scope=?{bound}",
(_now(), scope, *params),
)
if jobs_exist:
conn.execute(
"UPDATE linked_folder_sync_jobs SET status='failed', stage='error', "
"error='Owning scope was removed', successor_kind=NULL, completed_at=? "
"WHERE folder_id IN (SELECT id FROM linked_folders WHERE scope=?"
f"{bound}) AND (status IN ('pending','running') OR successor_kind IS NOT NULL)",
(_now(), scope, *params),
)
def retire_scope(
scope: str,
folder_ids: list[str] | None = None,
rows: bool = True,
) -> None:
"""rows=False writes only the tombstone: stops new links and uploads, and is all a caller can undo."""
conn = _retirement_connection()
try:
conn.execute("BEGIN IMMEDIATE")
_retire_scope_rows(conn, scope, folder_ids, rows)
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
def retire_and_delete_kb(kb_id: str) -> bool:
"""Retires the scope and deletes its owner row atomically; document rows stay as the cleanup queue."""
scope = store.kb_scope(kb_id)
conn = rag_db.get_connection()
try:
conn.execute("BEGIN IMMEDIATE")
if store.get_kb(conn, kb_id) is None:
conn.rollback()
return False
_retire_scope_rows(conn, scope)
store.delete_kb(conn, kb_id, commit = False, delete_documents = False)
conn.commit()
return True
except Exception:
conn.rollback()
raise
finally:
conn.close()
def delete_retired_scope(scope: str) -> bool:
"""Purges an ownerless scope but keeps its tombstone; files go before DB rows, so retries are safe."""
conn = rag_db.get_connection()
try:
conn.execute("BEGIN IMMEDIATE")
if (
conn.execute(
"SELECT 1 FROM linked_folder_retired_scopes WHERE scope=? AND purged_at IS NULL",
(scope,),
).fetchone()
is None
):
conn.rollback()
return False
live_ingestion = conn.execute(
"SELECT 1 FROM ingestion_jobs j JOIN rag_job_leases l "
"ON l.kind=? AND l.job_id=j.id WHERE j.scope=? "
"AND j.status IN ('pending','running') AND l.expires_at>? LIMIT 1",
(job_leases.INGESTION, scope, _now()),
).fetchone()
if live_ingestion is not None:
conn.rollback()
return False
live_folder_sync = conn.execute(
"SELECT 1 FROM linked_folder_sync_jobs j JOIN linked_folders f "
"ON f.id=j.folder_id JOIN rag_job_leases l "
"ON l.kind=? AND l.job_id=j.id WHERE f.scope=? "
"AND l.expires_at>? LIMIT 1",
(job_leases.FOLDER_SYNC, scope, _now()),
).fetchone()
if live_folder_sync is not None:
conn.rollback()
return False
documents = conn.execute(
"SELECT id, stored_path FROM documents WHERE scope=?", (scope,)
).fetchall()
for stored_path in dict.fromkeys(document["stored_path"] for document in documents):
_remove_retired_snapshot(stored_path)
store.delete_documents(conn, [document["id"] for document in documents], commit = False)
conn.execute(
"DELETE FROM rag_job_leases WHERE kind=? AND job_id IN "
"(SELECT id FROM ingestion_jobs WHERE scope=?)",
(job_leases.INGESTION, scope),
)
conn.execute("DELETE FROM ingestion_jobs WHERE scope=?", (scope,))
conn.execute(
"DELETE FROM linked_folder_files WHERE folder_id IN "
"(SELECT id FROM linked_folders WHERE scope=?)",
(scope,),
)
conn.execute(
"DELETE FROM rag_job_leases WHERE kind=? AND job_id IN "
"(SELECT j.id FROM linked_folder_sync_jobs j JOIN linked_folders f "
"ON f.id=j.folder_id WHERE f.scope=?)",
(job_leases.FOLDER_SYNC, scope),
)
conn.execute(
"DELETE FROM linked_folder_sync_jobs WHERE folder_id IN "
"(SELECT id FROM linked_folders WHERE scope=?)",
(scope,),
)
conn.execute("DELETE FROM linked_folders WHERE scope=?", (scope,))
conn.execute(
"UPDATE linked_folder_retired_scopes SET purged_at=? WHERE scope=?",
(_now(), scope),
)
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
return True
def reconcile_retired_scopes(project_exists) -> dict[str, list[str]]:
"""Retire orphaned project scopes and reap ownerless retired scopes."""
with closing(rag_db.get_connection()) as conn:
retired_scopes = {
row["scope"]
for row in conn.execute(
"SELECT scope FROM linked_folder_retired_scopes WHERE purged_at IS NULL"
)
}
project_scopes = {
row["scope"]
for row in conn.execute(
"SELECT scope FROM linked_folders WHERE scope_type='project' "
"UNION SELECT scope FROM documents WHERE project_id IS NOT NULL"
)
}
retired: list[str] = []
deleted: list[str] = []
restored: list[str] = []
for scope in sorted(project_scopes - retired_scopes):
project_id = scope.removeprefix("project_")
if not project_id:
continue
try:
# Rechecked under create_folder's lock so a recreated project's folders are not retired.
with _scope_lock(scope):
owned = linked_folder_ids(scope)
if project_exists(project_id):
continue
retire_scope(scope, owned)
retired_scopes.add(scope)
retired.append(scope)
except Exception:
logger.warning("failed to retire orphaned RAG scope %s", scope, exc_info = True)
for scope in sorted(retired_scopes):
try:
if scope.startswith("project_"):
project_id = scope[len("project_") :]
if not project_id:
continue
owner_exists = project_exists(project_id)
elif scope.startswith("kb_"):
kb_id = scope[len("kb_") :]
if not kb_id:
continue
with closing(rag_db.get_connection()) as owner_conn:
owner_exists = store.get_kb(owner_conn, kb_id) is not None
else:
continue
if owner_exists:
# retirement only ever meant "ownerless", so a returned id must not keep gating its owner
if unretire_scope(scope):
restored.append(scope)
elif delete_retired_scope(scope):
deleted.append(scope)
except Exception:
logger.warning("failed to reconcile retired RAG scope %s", scope, exc_info = True)
if retired:
logger.info("retired %s orphaned RAG scope(s)", len(retired))
if deleted:
logger.info("deleted %s retired RAG scope(s)", len(deleted))
if restored:
logger.info("restored %s recreated RAG scope(s)", len(restored))
return {"retired": retired, "deleted": deleted, "restored": restored}
def unretire_scope(scope: str) -> bool:
"""Drop the tombstone of a scope whose owner exists again, purged or not, so it can be used."""
with _scope_lock(scope):
conn = _retirement_connection()
try:
conn.execute("BEGIN IMMEDIATE")
removed = conn.execute(
"DELETE FROM linked_folder_retired_scopes WHERE scope=?", (scope,)
).rowcount
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
return bool(removed)
def scope_retired(scope: str) -> bool:
with _scope_lock(scope):
with closing(_retirement_connection()) as conn:
return (
conn.execute(
"SELECT 1 FROM linked_folder_retired_scopes WHERE scope=?", (scope,)
).fetchone()
is not None
)
def scope_lock(scope: str) -> threading.RLock:
return _scope_lock(scope)
def request_sync(folder_id: str, *, rebuild: bool = False) -> str:
with _folder_lock(folder_id):
return _request_sync(folder_id, rebuild = rebuild)
def _fold_into_active_job(conn, active, kind: str) -> None:
"""Pending jobs take the kind; a running job's files are fixed, so the request becomes a successor."""
if active["status"] == "pending":
if kind == "rebuild":
conn.execute(
"UPDATE linked_folder_sync_jobs SET kind='rebuild', successor_kind=NULL WHERE id=?",
(active["id"],),
)
return
# a queued rebuild outranks a queued sync; it covers everything a sync would
conn.execute(
"UPDATE linked_folder_sync_jobs SET successor_kind=? WHERE id=? "
"AND (successor_kind IS NULL OR ?='rebuild')",
(kind, active["id"], kind),
)
def _request_sync(folder_id: str, *, rebuild: bool = False) -> str:
job_id = str(uuid.uuid4())
with closing(rag_db.get_connection()) as conn:
conn.execute("BEGIN IMMEDIATE")
folder = conn.execute(
"SELECT status, delete_remove_index FROM linked_folders WHERE id=?", (folder_id,)
).fetchone()
if (
folder is None
or folder["status"] == "retired"
or folder["delete_remove_index"] is not None
):
raise KeyError(folder_id)
conn.execute("UPDATE linked_folders SET failed_files=NULL WHERE id=?", (folder_id,))
active = conn.execute(
"SELECT id, kind, status FROM linked_folder_sync_jobs WHERE folder_id=? "
"AND status IN ('pending','running') ORDER BY created_at LIMIT 1",
(folder_id,),
).fetchone()
if active is not None:
_fold_into_active_job(conn, active, "rebuild" if rebuild else "sync")
conn.commit()
return active["id"]
conn.execute(
"INSERT INTO linked_folder_sync_jobs(id, folder_id, kind, status, stage, created_at) "
"VALUES(?,?,?,'pending','queued',?)",
(job_id, folder_id, "rebuild" if rebuild else "sync", _now()),
)
conn.commit()
_prune_terminal_jobs(conn)
_wake.set()
return job_id
def _prune_terminal_jobs(conn) -> None:
limit = max(0, config.FOLDER_JOB_HISTORY_LIMIT)
conn.execute(
"DELETE FROM linked_folder_sync_jobs WHERE id IN ("
"SELECT id FROM linked_folder_sync_jobs WHERE status IN ('completed','failed') "
"AND successor_kind IS NULL "
"ORDER BY completed_at DESC, created_at DESC LIMIT -1 OFFSET ?)",
(limit,),
)
conn.commit()
def get_job(job_id: str) -> dict | None:
with closing(rag_db.get_connection()) as conn:
row = conn.execute("SELECT * FROM linked_folder_sync_jobs WHERE id=?", (job_id,)).fetchone()
return dict(row) if row else None
def job_events(job_id: str):
"""Polls persisted state so streams survive restarts; yields None as a keepalive during quiet passes."""
previous = None
last_sent = time.monotonic()
while True:
row = get_job(job_id)
if row is None:
return
state = tuple(row.items())
if state != previous:
yield row
previous = state
last_sent = time.monotonic()
elif time.monotonic() - last_sent >= _JOB_EVENT_KEEPALIVE_S:
yield None
last_sent = time.monotonic()
if row["status"] in _TERMINAL:
return
time.sleep(0.5)
def _scan(
root: str,
expected_identity: tuple[int, int] | None = None,
report: dict[str, int] | None = None,
excluded: set[str] | None = None,
) -> tuple[dict[str, dict], tuple[int, int]]:
"""Map relative path -> metadata; `report` receives counts of deliberately skipped entries."""
from hub.storage.scan_folders import contains_sensitive_path_component, is_denied_system_path
identity = _root_identity(root)
if expected_identity is not None and identity != expected_identity:
raise RuntimeError("Linked folder root identity changed")
skipped = {"gitignored": 0, "build_dirs": 0, "too_large": 0, "over_limit": 0}
excluded = set() if excluded is None else excluded
found: dict[str, dict] = {}
mount_points = _mount_points()
root_identities = frozenset({identity}) if identity[1] not in (None, 0) else frozenset()
pending = [(root, frozenset({_path_key(root)}), root_identities, 0, (), None)]
# Collect past the cap so the kept set is the sorted first FOLDER_MAX_FILES, stable across scans.
hard_cap = config.FOLDER_MAX_FILES * 2 if config.FOLDER_MAX_FILES else 0
while pending:
directory, ancestor_paths, ancestor_identities, depth, rules, matcher = pending.pop()
directory_rel = os.path.relpath(directory, root).replace(os.sep, "/")
directory_rel = "" if directory_rel == "." else directory_rel
own_rules = _read_gitignore(directory, directory_rel)
if own_rules:
rules = (*rules, *own_rules)
matcher = gitignore.Matcher(rules)
with os.scandir(directory) as entries:
for entry in entries:
full = entry.path
if entry.is_symlink():
continue
rel = os.path.relpath(full, root).replace(os.sep, "/")
if entry.is_dir(follow_symlinks = False):
if _is_ignored_scan_dir(entry.name, full):
excluded.add(rel)
continue
if matcher and matcher.is_ignored(rel, True):
skipped["gitignored"] += 1
excluded.add(rel)
continue
if _is_build_output_dir(entry.name, directory):
skipped["build_dirs"] += 1
excluded.add(rel)
continue
resolved = os.path.realpath(full)
if (
not _is_within(root, resolved)
or contains_sensitive_path_component(os.path.relpath(resolved, root))
or is_denied_system_path(resolved)
):
continue
resolved_key = _path_key(resolved)
if resolved_key in ancestor_paths:
continue
directory_stat = entry.stat(follow_symlinks = False)
directory_identity = (directory_stat.st_dev, directory_stat.st_ino)
identity_usable = directory_identity[1] not in (None, 0)
if (
identity_usable
and directory_identity in ancestor_identities
and (resolved_key in mount_points or os.path.ismount(full))
):
continue
if depth >= _MAX_FOLDER_DEPTH:
raise RuntimeError(
f"Folder nesting exceeds the {_MAX_FOLDER_DEPTH}-level scan limit"
)
child_identities = ancestor_identities
if identity_usable:
child_identities = ancestor_identities | {directory_identity}
pending.append(
(
full,
ancestor_paths | {resolved_key},
child_identities,
depth + 1,
rules,
matcher,
)
)
continue
if not entry.is_file(follow_symlinks = False):
continue
if _is_ignored_scan_file(entry.name):
continue
ext = os.path.splitext(entry.name)[1].lower()
if ext not in config.UPLOAD_EXTS:
continue
if matcher and matcher.is_ignored(rel, False):
skipped["gitignored"] += 1
excluded.add(rel)
continue
# Compare only fstat-reproducible ids: scandir has none on Windows; lstat differs on unstable inodes.
if is_appledouble_metadata(Path(full)):
continue
st = entry.stat(follow_symlinks = False)
from_path = False
if st.st_ino in (None, 0):
# Windows DirEntry.stat leaves st_dev/st_ino at 0; os.lstat fills them.
try:
st = os.lstat(full)
from_path = True
except OSError:
# A file vanishing mid-scan must count as a failure: scans never authorize deletion.
pass
if st.st_size == 0:
continue
if (
config.FOLDER_MAX_TEXT_BYTES
and _is_plain_text_ext(ext)
and st.st_size > config.FOLDER_MAX_TEXT_BYTES
):
skipped["too_large"] += 1
excluded.add(rel)
continue
found[rel] = {
"path": full,
"size_bytes": st.st_size,
"mtime_ns": st.st_mtime_ns,
"device": st.st_dev,
"inode": st.st_ino,
# Path-derived ids differ from fstat ids on shared-folder and WebDAV drivers.
"identity_from_path": from_path,
}
if hard_cap and len(found) > hard_cap:
raise RuntimeError(
f"Folder contains more than twice the {config.FOLDER_MAX_FILES} supported "
"files limit. Link a smaller folder, or add a .gitignore for what to skip."
)
if _root_identity(root) != identity:
raise RuntimeError("Linked folder root identity changed during scan")
if config.FOLDER_MAX_FILES and len(found) > config.FOLDER_MAX_FILES:
kept = sorted(found)[: config.FOLDER_MAX_FILES]
skipped["over_limit"] = len(found) - len(kept)
excluded.update(set(found) - set(kept))
found = {rel: found[rel] for rel in kept}
if report is not None:
report.update(skipped)
return found, identity
def _snapshot(root: str, metadata: dict) -> str:
rag_db.release_reused()
source = metadata["path"]
resolved = os.path.realpath(source)
if not _is_within(root, resolved):
raise RuntimeError("File escaped the linked folder")
# os.fdopen already forces this descriptor binary on Windows; O_BINARY only guards a raw os.read.
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0)
fd = os.open(source, flags)
target = None
try:
ext = os.path.splitext(source)[1].lower()
target = ensure_dir(rag_uploads_root()) / f"linked-{uuid.uuid4().hex}{ext}"
before = os.fstat(fd)
if not stat.S_ISREG(before.st_mode):
raise RuntimeError("Linked source is not a regular file")
expected = (
metadata["size_bytes"],
metadata["mtime_ns"],
metadata["device"],
metadata["inode"],
)
actual = (before.st_size, before.st_mtime_ns, before.st_dev, before.st_ino)
# Only an identity os.fstat can reproduce is comparable: os.scandir reports none on Windows, and the
# lstat fallback disagrees with fstat on file systems without stable ids.
usable = metadata["inode"] not in (None, 0) and not metadata.get("identity_from_path")
compared = 4 if usable else 2
if actual[:compared] != expected[:compared]:
raise RuntimeError("Linked source changed during reconciliation")
if config.MAX_UPLOAD_BYTES and before.st_size > config.MAX_UPLOAD_BYTES:
raise RuntimeError("Linked source exceeds the RAG file size limit")
with os.fdopen(fd, "rb", closefd = False) as src, open(target, "xb") as dst:
_copy_exact(src, dst, before.st_size)
after = os.fstat(fd)
# Both sides are fstat here, so the identity is always comparable.
if (after.st_size, after.st_mtime_ns, after.st_dev, after.st_ino) != actual:
raise RuntimeError("Linked source changed while it was copied")
return str(target)
except Exception:
if target is not None:
_remove_snapshot(str(target))
raise
finally:
os.close(fd)
def _copy_exact(source, target, size: int) -> None:
remaining = size
while remaining:
block = source.read(min(1 << 20, remaining))
if not block:
raise RuntimeError("Linked source changed while it was copied")
target.write(block)
remaining -= len(block)
if source.read(1):
raise RuntimeError("Linked source changed while it was copied")
def _check_running() -> None:
if account_is_retired():
raise _SyncStopped
stop_event = getattr(_worker_state, "stop_event", _stop)
if stop_event.is_set():
raise _SyncStopped
if is_cancel_requested(getattr(_worker_state, "folder_id", None)):
raise _SyncCancelled
job_id = getattr(_worker_state, "job_id", None)
if job_id is None:
return
with closing(rag_db.get_connection()) as conn:
if not job_leases.renew_owned(conn, job_leases.FOLDER_SYNC, job_id):
conn.rollback()
raise _LeaseLost
folder = conn.execute(
"SELECT f.status, f.delete_remove_index FROM linked_folder_sync_jobs j "
"JOIN linked_folders f ON f.id=j.folder_id WHERE j.id=?",
(job_id,),
).fetchone()
if (
folder is None
or folder["status"] == "retired"
or folder["delete_remove_index"] is not None
):
conn.commit()
raise _LeaseLost
conn.commit()
def _check_root_identity(root: str, expected: tuple[int, int]) -> None:
rag_db.release_reused()
try:
current = _root_identity(root)
except RuntimeError as exc:
raise _FolderChanged(str(exc)) from exc
if current != expected:
raise _FolderChanged("Linked folder root identity changed during reconciliation")
def _source_reappeared(root: str, relative_path: str, excluded: set[str]) -> bool:
"""check scanner eligibility without following a replaced path component."""
rag_db.release_reused()
path = PurePosixPath(relative_path)
parts = path.parts
if (
not parts
or path.is_absolute()
or "\x00" in relative_path
or any(part == ".." for part in parts)
):
raise _FolderChanged("Linked folder mapping has an invalid relative path")
current = root
for index, part in enumerate(parts):
# previously indexed dot directories are excluded even if they still exist
if index < len(parts) - 1 and part.startswith("."):
return False
if "/".join(parts[: index + 1]) in excluded:
return False
current = os.path.join(current, part)
try:
current_stat = os.lstat(current)
except FileNotFoundError:
return False
except OSError as exc:
raise _FolderChanged("Linked source could not be rechecked") from exc
if stat.S_ISLNK(current_stat.st_mode):
return False
if index < len(parts) - 1:
if not stat.S_ISDIR(current_stat.st_mode):
return False
else:
# Empty files are skipped by the scan; one refilled since then has reappeared.
return stat.S_ISREG(current_stat.st_mode) and current_stat.st_size > 0
return False
def _set_job(job_id: str, **values) -> None:
if not values:
return
with closing(rag_db.get_connection()) as conn:
columns = ", ".join(f"{key}=?" for key in values)
conn.execute(
f"UPDATE linked_folder_sync_jobs SET {columns} WHERE id=?",
(*values.values(), job_id),
)
conn.commit()
def _discard_document(document_id: str) -> None:
conn = rag_db.get_connection()
try:
conn.execute("BEGIN IMMEDIATE")
doc = store.get_document(conn, document_id)
if doc is not None:
_remove_retired_snapshot(doc.get("stored_path"))
store.delete_document(conn, document_id, commit = False)
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
def _install_mapping(
folder: dict, rel: str, metadata: dict, document_id: str, content_hash: str
) -> None:
replaced: list[str] = []
conn = rag_db.get_connection()
try:
old_rows = conn.execute(
"SELECT ff.document_id, d.stored_path FROM linked_folder_files ff "
"LEFT JOIN documents d ON d.id=ff.document_id "
"WHERE ff.folder_id=? AND ff.relative_path=?",
(folder["id"], rel),
).fetchall()
conn.execute("BEGIN IMMEDIATE")
conn.execute(
"INSERT INTO linked_folder_files(folder_id, relative_path, size_bytes, mtime_ns, "
"device, inode, document_id, synced_at, content_hash) VALUES(?,?,?,?,?,?,?,?,?) "
"ON CONFLICT(folder_id, relative_path) DO UPDATE SET size_bytes=excluded.size_bytes, "
"mtime_ns=excluded.mtime_ns, device=excluded.device, inode=excluded.inode, "
"document_id=excluded.document_id, synced_at=excluded.synced_at, "
"content_hash=excluded.content_hash",
(
folder["id"],
rel,
metadata["size_bytes"],
metadata["mtime_ns"],
*_file_identity(metadata),
document_id,
_now(),
content_hash,
),
)
for old in old_rows:
if old["document_id"] != document_id:
store.delete_document(conn, old["document_id"], commit = False)
replaced.append(old["stored_path"])
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
# Only after the commit: a rollback restores a live searchable document.
for stored_path in replaced:
_remove_snapshot(stored_path)
def _update_mapping_metadata(folder_id: str, rel: str, metadata: dict, content_hash: str) -> None:
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folder_files SET size_bytes=?, mtime_ns=?, device=?, inode=?, "
"content_hash=?, synced_at=? WHERE folder_id=? AND relative_path=?",
(
metadata["size_bytes"],
metadata["mtime_ns"],
*_file_identity(metadata),
content_hash,
_now(),
folder_id,
rel,
),
)
conn.commit()
def _delete_mapping(folder_id: str, rel: str) -> None:
conn = rag_db.get_connection()
try:
row = conn.execute(
"SELECT ff.document_id, d.stored_path FROM linked_folder_files ff "
"LEFT JOIN documents d ON d.id=ff.document_id "
"WHERE ff.folder_id=? AND ff.relative_path=?",
(folder_id, rel),
).fetchone()
if row is None:
return
conn.execute("BEGIN IMMEDIATE")
conn.execute(
"DELETE FROM linked_folder_files WHERE folder_id=? AND relative_path=?",
(folder_id, rel),
)
store.delete_document(conn, row["document_id"], commit = False)
conn.commit()
except Exception:
conn.rollback()
raise
finally:
conn.close()
# Only after the commit: a rollback restores a live document.
_remove_snapshot(row["stored_path"])
def _rename_mapping(folder_id: str, old_rel: str, new_rel: str) -> None:
with closing(rag_db.get_connection()) as conn:
row = conn.execute(
"SELECT document_id FROM linked_folder_files WHERE folder_id=? AND relative_path=?",
(folder_id, old_rel),
).fetchone()
if row is None:
return
conn.execute(
"UPDATE linked_folder_files SET relative_path=?, synced_at=? "
"WHERE folder_id=? AND relative_path=?",
(new_rel, _now(), folder_id, old_rel),
)
conn.execute(
"UPDATE documents SET filename=?, linked_relative_path=? WHERE id=?",
(new_rel, new_rel, row["document_id"]),
)
conn.commit()
def _load_withheld(folder: dict) -> set[str]:
try:
stored = json.loads(folder.get("withheld_paths") or "[]")
except ValueError:
return set()
return {path for path in stored if isinstance(path, str)} if isinstance(stored, list) else set()
def _store_withheld(folder_id: str, withheld: set[str]) -> None:
# truncation only shortens a grace pass, so the cap can never block a removal
capped = sorted(withheld)[:_MAX_WITHHELD_PATHS]
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folders SET withheld_paths=? WHERE id=?",
(json.dumps(capped) if capped else None, folder_id),
)
conn.commit()
def _failure_signature(metadata: dict) -> list:
return [metadata["size_bytes"], metadata["mtime_ns"], *_file_identity(metadata)]
def _load_failed(folder: dict) -> dict[str, dict]:
"""Files whose ingestion failed recently, as {relative path: {"sig": [...], "at": epoch}}."""
try:
stored = json.loads(folder.get("failed_files") or "{}")
except ValueError:
return {}
if not isinstance(stored, dict):
return {}
cutoff = time.time() - _FAILED_RETRY_S
return {
rel: entry
for rel, entry in stored.items()
if isinstance(entry, dict)
and isinstance(entry.get("at"), (int, float))
and entry["at"] > cutoff
# A pending embedder download is not the file's fault; also frees entries saved before this check.
and not embeddings.is_download_required_error(entry.get("error"))
}
def _failure_summary(failures: list[str]) -> str | None:
"""Name the files that could not be indexed, so the folder error is actionable."""
if not failures:
return None
named = ", ".join(sorted(failures)[:_MAX_NAMED_FAILURES])
remainder = len(failures) - _MAX_NAMED_FAILURES
if remainder > 0:
named = f"{named} and {remainder} more"
return f"{len(failures)} file(s) could not be indexed ({named})"
def _ingest_folder_document(
job_id: str,
stop_event: threading.Event,
folder: dict,
relative_path: str,
snapshot: str,
content_hash: str,
embedding_model: str,
rebuild: bool,
stage_slots: threading.Semaphore | None = None,
) -> _IngestionResult:
"""Thread-local, so the coordinator lease and stop state are copied in for _check_running."""
_worker_state.job_id = job_id
_worker_state.folder_id = folder["id"]
_worker_state.stop_event = stop_event
document_id = None
ingestion_job = None
with rag_db.reuse_connections():
try:
wait_while_chat_busy()
_check_running()
document_id, ingestion_job = ingestion.start_ingestion(
folder["scope"],
folder["scope_id"] if folder["scope_type"] == "knowledge_base" else None,
None,
relative_path,
snapshot,
project_id = folder["scope_id"] if folder["scope_type"] == "project" else None,
model_name = embedding_model,
content_hash = content_hash,
dedupe = False,
linked_folder_id = folder["id"],
linked_relative_path = relative_path,
background = False,
reuse_identical = not rebuild,
stage_slots = stage_slots,
)
result = ingestion.get_job_status(ingestion_job)
if result is None:
raise RuntimeError("Ingestion job disappeared")
if result["status"] != "completed":
raise RuntimeError(result.get("error") or "Ingestion failed")
_check_running()
return _IngestionResult(document_id, ingestion_job)
except Exception as exc: # noqa: BLE001 - the coordinator classifies it
if is_cancel_requested(folder["id"]):
# The unlink cut the ingest short; that is not a file failure.
return _IngestionResult(document_id, ingestion_job, _SyncCancelled())
return _IngestionResult(document_id, ingestion_job, exc)
def _cleanup_pending_ingestion(pending: _PendingIngestion, result: _IngestionResult) -> None:
"""Remove an uninstalled document, or its snapshot if admission never completed."""
if result.document_id is not None:
_discard_document(result.document_id)
else:
_remove_snapshot(pending.snapshot)
def _folder_ingest_workers() -> int:
return max(1, min(4, config.FOLDER_INGEST_WORKERS))
def _folder_ingest_in_flight(workers: int) -> int:
return max(workers, min(64, config.FOLDER_INGEST_IN_FLIGHT))
def _reconcile_folder(job_id: str) -> None:
"""Run one complete reconciliation; called serially by the coordinator."""
_check_running()
with closing(rag_db.get_connection()) as conn:
row = conn.execute("SELECT * FROM linked_folder_sync_jobs WHERE id=?", (job_id,)).fetchone()
if (
row is None
or row["status"] != "running"
or not job_leases.owned_by_this_process(conn, job_leases.FOLDER_SYNC, job_id)
):
return
job = dict(row)
folder = get_folder(job["folder_id"])
if folder is None or folder["status"] == "retired" or folder["delete_remove_index"] is not None:
_set_job(
job_id,
status = "failed",
stage = "error",
error = "Linked folder not found" if folder is None else "Owning scope was removed",
completed_at = _now(),
)
return
embedding_model = config.effective_embedding_model()
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folders SET status='syncing', last_error=NULL, updated_at=? WHERE id=?",
(_now(), folder["id"]),
)
conn.commit()
try:
_check_running()
expected_identity = _load_identity(folder["root_device"], folder["root_inode"])
skipped: dict[str, int] = {}
excluded: set[str] = set()
current, scanned_identity = _scan(folder["path"], expected_identity, skipped, excluded)
_check_running()
except (_SyncStopped, _SyncCancelled, _LeaseLost):
raise
except Exception as exc:
# A partial/unavailable scan is never authoritative for deletion.
error = _error_text(exc, folder["path"])
_set_job(job_id, status = "failed", stage = "error", error = error, completed_at = _now())
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folders SET status='error', last_error=?, updated_at=? WHERE id=?",
(error, _now(), folder["id"]),
)
conn.commit()
_prune_terminal_jobs(conn)
return
with closing(rag_db.get_connection()) as conn:
known = {
row["relative_path"]: dict(row)
for row in conn.execute(
"SELECT * FROM linked_folder_files WHERE folder_id=?", (folder["id"],)
).fetchall()
}
rebuild = job["kind"] == "rebuild"
missing = set(known) - set(current)
new = set(current) - set(known)
renamed = 0
materially_changed = {
rel
for rel in set(current) & set(known)
if current[rel]["size_bytes"] != known[rel]["size_bytes"]
or current[rel]["mtime_ns"] != known[rel]["mtime_ns"]
or _file_identity(current[rel]) != (known[rel]["device"], known[rel]["inode"])
}
changed = set(current) & set(known) if rebuild else materially_changed
work = sorted(new | changed)
missing_by_content: dict[tuple[str, str], list[str]] = {}
if not rebuild:
for rel in sorted(missing):
content_hash = known[rel].get("content_hash")
if content_hash:
key = (content_hash, os.path.splitext(rel)[1].lower())
missing_by_content.setdefault(key, []).append(rel)
already_withheld = _load_withheld(folder)
# An unchanged file whose ingestion failed fails again: retrying it every periodic sync kept a
# folder of scanned PDFs indexing forever (#13095). Sync changes and Rebuild retry at once.
known_failed = {} if rebuild else _load_failed(folder)
if any(
known_failed.get(rel, {}).get("sig") != _failure_signature(current[rel]) for rel in work
) and embeddings.download_required(embedding_model):
# One job-level wait, not every file failing the same check; the periodic sync resumes it.
error = f"The embedding model {embedding_model!r} is not downloaded yet. " + (
"Indexing starts on its own once its Settings download finishes."
if folder.get("auto_sync")
else "Sync again once its Settings download finishes."
)
_set_job(job_id, status = "failed", stage = "error", error = error, completed_at = _now())
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folders SET status='error', last_error=?, updated_at=? WHERE id=?",
(error, _now(), folder["id"]),
)
conn.commit()
_prune_terminal_jobs(conn)
return
ingest_failed: dict[str, dict] = {}
cached_grace = False
total = len(work) + len(missing)
_set_job(job_id, stage = "ingesting", discovered = len(current), renamed = renamed)
added = changed_count = 0
failures: list[str] = []
failure_errors: dict[str, str] = {}
withheld: set[str] = set()
processed = 0
stop_event = getattr(_worker_state, "stop_event", _stop)
account = current_account()
workers = _folder_ingest_workers()
in_flight_limit = (
_folder_ingest_in_flight(workers)
if ingestion.embeddings.batches_across_documents(embedding_model)
else workers
)
stage_slots = threading.Semaphore(workers)
in_flight: dict[Future["_IngestionResult"], _PendingIngestion] = {}
def update_progress() -> None:
_set_job(
job_id,
added = added,
changed = changed_count,
failed = len(failures),
renamed = renamed,
progress = processed / max(total, 1),
)
def prune_ingestion_job(result: _IngestionResult) -> None:
if result.ingestion_job is None:
return
try:
ingestion.delete_terminal_job(result.ingestion_job)
except Exception:
logger.warning(
"failed to prune linked-folder ingestion job %s",
result.ingestion_job,
exc_info = True,
)
def finish_pending(pending: _PendingIngestion) -> None:
nonlocal added, changed_count, processed
with rag_db.reuse_connections():
result = _IngestionResult()
try:
try:
result = pending.future.result()
except Exception as exc: # noqa: BLE001
result = _IngestionResult(error = exc)
if result.error is not None:
if isinstance(
result.error, (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged)
):
_cleanup_pending_ingestion(pending, result)
raise result.error
failures.append(pending.relative_path)
reason = _error_text(result.error, folder["path"])
failure_errors[pending.relative_path] = reason
if not embeddings.is_download_required_error(reason):
# A transfer that went pending mid-sync is retried by the next sync.
ingest_failed[pending.relative_path] = {
"sig": _failure_signature(pending.metadata),
"at": time.time(),
"error": reason,
}
# A failed file may be a rename or copy of a vanished path, so grant one pass.
withheld.update(missing - already_withheld)
logger.warning(
"linked-folder ingestion failed for %s",
pending.relative_path,
exc_info = (type(result.error), result.error, result.error.__traceback__),
)
_cleanup_pending_ingestion(pending, result)
else:
try:
_check_running()
_check_root_identity(folder["path"], scanned_identity)
_install_mapping(
folder,
pending.relative_path,
pending.metadata,
result.document_id,
pending.content_hash,
)
except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):
_cleanup_pending_ingestion(pending, result)
raise
except Exception as exc: # noqa: BLE001
failures.append(pending.relative_path)
failure_errors[pending.relative_path] = _error_text(exc, folder["path"])
withheld.update(missing - already_withheld)
logger.warning(
"linked-folder ingestion failed for %s",
pending.relative_path,
exc_info = (type(exc), exc, exc.__traceback__),
)
_cleanup_pending_ingestion(pending, result)
else:
if pending.relative_path in new:
added += 1
else:
changed_count += 1
finally:
prune_ingestion_job(result)
processed += 1
update_progress()
def drain_one() -> None:
if not in_flight:
return
# Hold no rag.db handle while the workers run, as before the shared connection.
rag_db.release_reused()
completed, _ = wait(tuple(in_flight), return_when = FIRST_COMPLETED)
for future in completed:
pending = in_flight.pop(future)
finish_pending(pending)
def abort_in_flight() -> None:
pending_items = list(in_flight.items())
for future, _ in pending_items:
future.cancel()
for future, pending in pending_items:
in_flight.pop(future, None)
if future.cancelled():
result = _IngestionResult()
else:
try:
result = future.result()
except Exception as exc: # noqa: BLE001
result = _IngestionResult(error = exc)
try:
_cleanup_pending_ingestion(pending, result)
except Exception:
logger.warning(
"failed to clean up aborted linked-folder ingestion for %s",
pending.relative_path,
exc_info = True,
)
prune_ingestion_job(result)
with ThreadPoolExecutor(
max_workers = in_flight_limit,
thread_name_prefix = "rag-folder-ingest",
) as executor:
try:
with rag_db.reuse_connections():
for rel in work:
if known_failed.get(rel, {}).get("sig") == _failure_signature(current[rel]):
# No per-file write: a folder can hold thousands of these.
if is_cancel_requested(folder["id"]):
raise _SyncCancelled
failures.append(rel)
ingest_failed[rel] = known_failed[rel]
failure_errors[rel] = known_failed[rel].get("error") or "Indexing failed"
if not cached_grace:
# missing only shrinks during the pass, so once covers every later failure
withheld.update(missing - already_withheld)
cached_grace = True
processed += 1
continue
# Only small files queue past the workers: each queued one keeps a snapshot.
cap = (
in_flight_limit
if current[rel]["size_bytes"] <= config.FOLDER_INGEST_BATCH_FILE_BYTES
else workers
)
while len(in_flight) >= cap:
drain_one()
snapshot = None
try:
_check_running()
metadata = current[rel]
# Also before the call: a wrapped _snapshot must not see the handle either.
rag_db.release_reused()
snapshot = _snapshot(folder["path"], metadata)
content_hash = _hash_file(snapshot)
_check_running()
if (
not rebuild
and rel in changed
and content_hash == known[rel].get("content_hash")
):
_check_root_identity(folder["path"], scanned_identity)
_update_mapping_metadata(folder["id"], rel, metadata, content_hash)
_remove_snapshot(snapshot)
snapshot = None
elif not rebuild and rel in new:
rename_key = (content_hash, os.path.splitext(rel)[1].lower())
rename_candidates = missing_by_content.get(rename_key)
if rename_candidates:
old_rel = rename_candidates.pop(0)
missing.discard(old_rel)
_check_root_identity(folder["path"], scanned_identity)
_rename_mapping(folder["id"], old_rel, rel)
_update_mapping_metadata(folder["id"], rel, metadata, content_hash)
_remove_snapshot(snapshot)
snapshot = None
renamed += 1
if snapshot is not None:
# reuse_identical needs a finished donor, so a copy waits for its in-flight twin.
while not rebuild and any(
p.content_hash == content_hash for p in in_flight.values()
):
drain_one()
# Pool threads start as the owner account; rebind to this job's.
future = executor.submit(
run_as,
account,
_ingest_folder_document,
job_id,
stop_event,
folder,
rel,
snapshot,
content_hash,
embedding_model,
rebuild,
stage_slots,
)
in_flight[future] = _PendingIngestion(
relative_path = rel,
snapshot = snapshot,
metadata = metadata,
content_hash = content_hash,
future = future,
)
snapshot = None
except (_SyncStopped, _SyncCancelled, _LeaseLost, _FolderChanged):
_remove_snapshot(snapshot)
raise
except Exception as exc: # noqa: BLE001
failures.append(rel)
failure_errors[rel] = _error_text(exc, folder["path"])
withheld.update(missing - already_withheld)
logger.warning("linked-folder ingestion failed for %s", rel, exc_info = True)
_remove_snapshot(snapshot)
if any(pending.relative_path == rel for pending in in_flight.values()):
continue
processed += 1
update_progress()
update_progress()
while in_flight:
drain_one()
except Exception:
abort_in_flight()
raise
deleted = 0
removable = sorted(missing - withheld)
if removable:
# Exclusions can lapse during ingestion (.gitignore edited, file shrunk): judge removals on a fresh scan.
excluded = set()
try:
rescanned, _ = _scan(folder["path"], scanned_identity, None, excluded)
except RuntimeError as exc:
raise _FolderChanged(str(exc)) from exc
if not rescanned.keys().isdisjoint(removable):
raise _FolderChanged("Linked source reappeared during reconciliation")
with rag_db.reuse_connections():
for rel in removable:
_check_running()
_check_root_identity(folder["path"], scanned_identity)
if _source_reappeared(folder["path"], rel, excluded):
raise _FolderChanged("Linked source reappeared during reconciliation")
_check_root_identity(folder["path"], scanned_identity)
_delete_mapping(folder["id"], rel)
deleted += 1
_set_job(
job_id,
deleted = deleted,
progress = (len(work) + deleted) / max(total, 1),
)
_check_running()
_store_withheld(folder["id"], withheld)
with closing(rag_db.get_connection()) as conn:
# Written while the job is still active, so a later Sync changes clears it in _request_sync and
# one queued behind this run (successor_kind) finds nothing to skip.
conn.execute(
"UPDATE linked_folders SET failed_files=CASE WHEN EXISTS(SELECT 1 FROM "
"linked_folder_sync_jobs WHERE id=? AND successor_kind IS NOT NULL) THEN NULL ELSE ? END "
"WHERE id=?",
(job_id, json.dumps(ingest_failed) if ingest_failed else None, folder["id"]),
)
conn.commit()
error = _failure_summary(failures)
_set_job(
job_id,
status = "completed" if error is None else "failed",
stage = "done" if error is None else "error",
progress = 1.0,
error = error,
completed_at = _now(),
)
scan_report = {
"skipped": {key: count for key, count in skipped.items() if count},
"failures": [
{"path": rel, "error": failure_errors.get(rel, "Indexing failed")}
for rel in sorted(set(failures))[:_MAX_REPORTED_FAILURES]
],
"failureCount": len(set(failures)),
}
# last_change_at moves only when the indexed set did, so idle passes don't make clients refetch.
changed_sources = added + changed_count + deleted + renamed > 0
now = _now()
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folders SET status=?, last_error=?, last_scan_at=?, updated_at=?, "
"scan_report=?, last_change_at=CASE WHEN ? THEN ? ELSE COALESCE(last_change_at, ?) END "
"WHERE id=? AND delete_remove_index IS NULL",
(
"ready" if error is None else "error",
error,
now,
now,
json.dumps(scan_report),
changed_sources,
now,
now,
folder["id"],
),
)
conn.commit()
_prune_terminal_jobs(conn)
def _fail_job(job_id: str, exc: Exception) -> None:
job = get_job(job_id)
folder = get_folder(job["folder_id"]) if job is not None else None
error = _error_text(exc, folder["path"] if folder else None)
_set_job(job_id, status = "failed", stage = "error", error = error, completed_at = _now())
if job is None:
return
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folders SET status='error', last_error=?, updated_at=? WHERE id=?",
(error, _now(), job["folder_id"]),
)
conn.commit()
_prune_terminal_jobs(conn)
def _pause_job(job_id: str) -> None:
job = get_job(job_id)
if job is None:
return
_set_job(job_id, status = "pending", stage = "queued", started_at = None)
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folders SET status='ready', updated_at=? WHERE id=?",
(_now(), job["folder_id"]),
)
conn.commit()
def _stop_job(job_id: str) -> None:
"""End a job an unlink cancelled; terminal, so the queue does not reclaim it before the unlink lands."""
job = get_job(job_id)
if job is None or job["status"] not in ("pending", "running"):
return
_set_job(
job_id,
status = "failed",
stage = "error",
error = "Indexing was stopped",
completed_at = _now(),
)
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folders SET status='ready', last_error=NULL, updated_at=? "
"WHERE id=? AND delete_remove_index IS NULL",
(_now(), job["folder_id"]),
)
conn.commit()
_prune_terminal_jobs(conn)
def _queue_successor(job_id: str) -> None:
"""Hand a finished job's queued follow-up request to the next run."""
conn = rag_db.get_connection()
queued = False
try:
conn.execute("BEGIN IMMEDIATE")
job = conn.execute(
"SELECT folder_id, successor_kind, status FROM linked_folder_sync_jobs WHERE id=?",
(job_id,),
).fetchone()
if job and job["successor_kind"] and job["status"] not in ("pending", "running"):
active = conn.execute(
"SELECT id, kind, status FROM linked_folder_sync_jobs WHERE folder_id=? "
"AND status IN ('pending','running') ORDER BY created_at LIMIT 1",
(job["folder_id"],),
).fetchone()
if active is None:
conn.execute(
"INSERT INTO linked_folder_sync_jobs"
"(id, folder_id, kind, status, stage, created_at) "
"VALUES(?,?,?,'pending','queued',?)",
(str(uuid.uuid4()), job["folder_id"], job["successor_kind"], _now()),
)
else:
_fold_into_active_job(conn, active, job["successor_kind"])
conn.execute(
"UPDATE linked_folder_sync_jobs SET successor_kind=NULL WHERE id=?", (job_id,)
)
queued = True
conn.commit()
if queued:
_prune_terminal_jobs(conn)
finally:
conn.close()
if queued:
_wake.set()
def reconcile_folder(job_id: str) -> None:
"""Reconcile and persist a terminal folder state for every unexpected failure."""
claimed = _claim_job(job_id)
if claimed is None:
# Release the claim here or the heartbeat renews it forever and unlink waits on it.
job_leases.release(job_leases.FOLDER_SYNC, job_id)
return
_, folder_id = claimed
lease_lost = False
cancelled = False
_worker_state.job_id = job_id
_worker_state.folder_id = folder_id
try:
_reconcile_folder(job_id)
except _LeaseLost:
lease_lost = True
except _SyncCancelled:
cancelled = True
if not account_is_retired():
_stop_job(job_id)
except _SyncStopped:
# A retired account's database is closed; its rows moved with the roots.
if not account_is_retired():
_pause_job(job_id)
except Exception as exc:
logger.exception("linked-folder job %s failed unexpectedly", job_id)
if not account_is_retired():
_fail_job(job_id, exc)
finally:
try:
if not lease_lost and not cancelled and not account_is_retired():
_queue_successor(job_id)
finally:
del _worker_state.job_id
del _worker_state.folder_id
job_leases.release(job_leases.FOLDER_SYNC, job_id)
def _enqueue_periodic() -> None:
rag_db.reconcile_orphaned_ingestion_jobs()
_reconcile_retired_folder_deletions()
with closing(rag_db.get_connection()) as conn:
now = _now()
conn.execute("BEGIN IMMEDIATE")
rows = conn.execute(
"SELECT id FROM linked_folders WHERE auto_sync=1 AND status!='retired' "
"AND delete_remove_index IS NULL"
).fetchall()
for row in rows:
conn.execute(
"INSERT OR IGNORE INTO linked_folder_sync_jobs"
"(id, folder_id, kind, status, stage, created_at) "
"VALUES(?,?,'sync','pending','queued',?)",
(str(uuid.uuid4()), row["id"], now),
)
_reap_orphaned_documents(conn, now)
conn.commit()
_prune_terminal_jobs(conn)
def _claim_job(job_id: str) -> tuple[str, str] | None:
with closing(rag_db.get_connection()) as conn:
conn.execute("BEGIN IMMEDIATE")
row = conn.execute(
"SELECT id, folder_id, status FROM linked_folder_sync_jobs WHERE id=?", (job_id,)
).fetchone()
if row is None or row["status"] not in ("pending", "running"):
conn.rollback()
return None
if row["status"] == "running":
if not job_leases.owned_by_this_process(
conn, job_leases.FOLDER_SYNC, job_id
) and not job_leases.claim(conn, job_leases.FOLDER_SYNC, job_id):
conn.rollback()
return None
else:
if not job_leases.claim(conn, job_leases.FOLDER_SYNC, job_id):
conn.rollback()
return None
conn.execute(
"UPDATE linked_folder_sync_jobs SET status='running', stage='scanning', "
"started_at=COALESCE(started_at, ?) WHERE id=? AND status='pending'",
(_now(), job_id),
)
conn.commit()
try:
job_leases.activate(job_leases.FOLDER_SYNC, job_id)
except Exception:
with closing(rag_db.get_connection()) as conn:
conn.execute(
"UPDATE linked_folder_sync_jobs SET status='pending', stage='queued' WHERE id=?",
(job_id,),
)
conn.commit()
job_leases.release(job_leases.FOLDER_SYNC, job_id)
raise
return job_id, row["folder_id"]
def _next_job() -> tuple[str, str] | None:
with closing(rag_db.get_connection()) as conn:
row = conn.execute(
"SELECT j.id FROM linked_folder_sync_jobs j WHERE j.status='pending' OR "
"(j.status='running' AND NOT EXISTS (SELECT 1 FROM rag_job_leases l "
"WHERE l.kind=? AND l.job_id=j.id AND l.expires_at>?)) "
"ORDER BY CASE j.status WHEN 'running' THEN 0 ELSE 1 END, j.created_at LIMIT 1",
(job_leases.FOLDER_SYNC, _now()),
).fetchone()
return _claim_job(row["id"]) if row else None
def _worker(stop_event: threading.Event | None = None, project_exists = None) -> None:
global _thread, _thread_stop
stop_event = stop_event or _stop
try:
with _worker_lock:
_worker_state.stop_event = stop_event
while not stop_event.is_set():
try:
accounts = job_accounts()
except Exception:
logger.warning("linked-folder worker initialization failed", exc_info = True)
stop_event.wait(1.0)
continue
initialized = 0
for account in accounts:
try:
run_as(account, _initialize_account_sync, project_exists, recover = True)
initialized += 1
except Exception:
# One bad account database must not keep the worker out of the loop.
logger.warning(
"linked-folder worker initialization failed for one account",
exc_info = True,
)
if initialized or not accounts:
break
stop_event.wait(1.0)
while not stop_event.is_set():
try:
job = _next_account_job()
except Exception:
# Lock contention must not retire the only worker; the other paths retry.
logger.warning("linked-folder queue selection failed", exc_info = True)
stop_event.wait(1.0)
continue
if job:
account, job_id, folder_id = job
try:
run_as(account, reconcile_folder, job_id)
except Exception as exc:
logger.exception("linked-folder job %s failed unexpectedly", job_id)
try:
run_as(account, _fail_job, job_id, exc)
except Exception:
# One account's bookkeeping must never stop the shared worker.
logger.warning(
"could not record the failure of linked-folder job %s",
job_id,
exc_info = True,
)
continue
_wake.wait(max(1.0, config.FOLDER_SYNC_INTERVAL_S))
_wake.clear()
if not stop_event.is_set():
try:
accounts = job_accounts()
except Exception:
logger.warning("linked-folder periodic scheduling failed", exc_info = True)
accounts = []
for account in accounts:
try:
run_as(account, _initialize_account_sync, project_exists)
except Exception:
logger.warning(
"linked-folder periodic scheduling failed for one account",
exc_info = True,
)
finally:
_worker_state.stop_event = None
with _thread_lock:
if _thread is threading.current_thread():
_thread = None
_thread_stop = None
def _recover_startup_state() -> None:
with closing(rag_db.get_connection()) as conn:
now = _now()
conn.execute("BEGIN IMMEDIATE")
conn.execute(
"UPDATE linked_folder_sync_jobs SET status='pending', stage='queued', "
"error=NULL WHERE status='running' AND NOT EXISTS ("
"SELECT 1 FROM rag_job_leases l WHERE l.kind=? "
"AND l.job_id=linked_folder_sync_jobs.id AND l.expires_at>?)",
(job_leases.FOLDER_SYNC, now),
)
conn.execute("DELETE FROM rag_job_leases WHERE expires_at<=?", (now,))
successor_handoffs = conn.execute(
"SELECT id FROM linked_folder_sync_jobs WHERE status IN ('completed','failed') "
"AND successor_kind IS NOT NULL"
).fetchall()
_reap_orphaned_documents(conn, now)
conn.commit()
for job in successor_handoffs:
_queue_successor(job["id"])
def _reap_orphaned_documents(conn, now: str) -> None:
"""Drops folder-owned orphans from a crash; anything still leased by a live ingestion is kept."""
orphans = conn.execute(
"SELECT d.id, d.stored_path FROM documents d "
"WHERE d.linked_folder_id IS NOT NULL AND NOT EXISTS "
"(SELECT 1 FROM linked_folder_files ff WHERE ff.document_id=d.id) AND NOT EXISTS ("
"SELECT 1 FROM ingestion_jobs j JOIN rag_job_leases l "
"ON l.kind=? AND l.job_id=j.id WHERE j.document_id=d.id "
"AND j.status IN ('pending','running') AND l.expires_at>?) AND NOT EXISTS ("
"SELECT 1 FROM linked_folder_sync_jobs sj JOIN rag_job_leases sl "
"ON sl.kind=? AND sl.job_id=sj.id WHERE sj.folder_id=d.linked_folder_id "
"AND sl.expires_at>?)",
(job_leases.INGESTION, now, job_leases.FOLDER_SYNC, now),
).fetchall()
for orphan in orphans:
_remove_retired_snapshot(orphan["stored_path"])
store.delete_document(conn, orphan["id"], commit = False)
conn.execute(
"DELETE FROM rag_job_leases WHERE kind=? AND job_id IN "
"(SELECT id FROM ingestion_jobs WHERE document_id=?)",
(job_leases.INGESTION, orphan["id"]),
)
conn.execute("DELETE FROM ingestion_jobs WHERE document_id=?", (orphan["id"],))
def start_auto_sync(
*,
admission_lock = None,
admit = None,
project_exists = None,
) -> bool:
global _thread, _thread_stop
try:
if not rag_db.rag_available():
return False
except sqlite3.OperationalError:
# Tile figure pages and merge tile text into the page so small labels and every sub-figure are kept.
pass
with _thread_lock:
retired = _thread if _thread is not None and _thread.is_alive() else None
if retired is not None and not _stop.is_set():
return False
def launch() -> bool:
global _thread, _thread_stop
with _thread_lock:
if _thread is not None and _thread.is_alive() and not _stop.is_set():
return False
stop_event = threading.Event()
_stop.clear()
_thread_stop = stop_event
_thread = account_thread(
target = _worker,
args = (stop_event, project_exists),
daemon = True,
name = "rag-folder-sync",
)
_thread.start()
return True
if admission_lock is None:
return launch()
with admission_lock:
if admit is not None and not admit():
return False
return launch()
def stop_auto_sync(timeout: float = 2.0) -> None:
_stop.set()
_wake.set()
with _thread_lock:
thread = _thread
stop_event = _thread_stop
if stop_event is not None:
stop_event.set()
if thread is not None:
thread.join(timeout = timeout)
def _initialize_account_sync(project_exists, *, recover: bool = False) -> None:
if recover:
_recover_startup_state()
if project_exists is not None:
reconcile_retired_scopes(project_exists)
_enqueue_periodic()
def _next_account_job():
# Round robin across accounts so one backlog cannot starve the rest.
global _last_job_account
accounts = job_accounts()
start = 0
for index, account in enumerate(accounts):
if account.account_id == _last_job_account:
start = index + 1
break
for offset in range(len(accounts)):
account = accounts[(start + offset) % len(accounts)]
try:
job = run_as(account, _next_job)
except Exception:
# The order is stable, so one corrupt database would shadow the accounts behind it.
logger.warning("linked-folder queue selection failed for one account", exc_info = True)
continue
if job:
_last_job_account = account.account_id
return (account, *job)
return None
def retire_account_sync() -> None:
"""Wake the supervisor; retirement checks stop only the retired account."""
_wake.set()