The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Take the SQLite write lock up front for asset scan and output-registration writes (#16480)
Simon Pinfold9d3bfc393b34
diff --git a/app/assets/scanner.py b/app/assets/scanner.pyindex 76821e4c3..5671df0f6 100644--- a/app/assets/scanner.py+++ b/app/assets/scanner.py@@ -11,7 +11,7 @@ import loggingimport osfrom dataclasses import dataclassfrom pathlib import Path-from typing import Callable, Literal, Protocol, TypedDict+from typing import Callable, Literal, NamedTuple, Protocol, TypedDictimport folder_pathsimport sqlalchemy as sa@@ -54,7 +54,7 @@ from app.assets.services.path_utils import ()from app.assets.services.ingest import _discard_unreferenced_contentfrom app.assets.services.snapshot_hash import snapshot_hash-from app.database.db import create_session+from app.database.db import create_session, create_write_session__all__ = ["clear_pending_verifications",@@ -96,6 +96,20 @@ class UnenrichedContent:file_path: str+class _ReferenceObservation(NamedTuple):+ """One live row's file, stat'ed outside the write transaction.++ ``size_bytes`` and ``mtime_ns`` are the row's values when observed, not the file's:+ the write skips a row that no longer matches them. ``stat_result`` is None when the+ file is gone.+ """++ content_id: str+ size_bytes: int | None+ mtime_ns: int | None+ stat_result: os.stat_result | None++def _log_scan_error(phase: str, error: OSError) -> None:error_type = ("permission_denied" if isinstance(error, PermissionError) else "os_error"@@ -152,54 +166,77 @@ def collect_models_files() -> list[str]:return out-def sync_references_with_filesystem(- session,- root: RootType,- collect_existing_paths: bool = False,- progress: _ScanProgress | None = None,-) -> set[str] | None:- return sync_prefixes_with_filesystem(- session,- get_scan_prefixes_for_root(root),- collect_existing_paths=collect_existing_paths,- progress=progress,- )---def sync_prefixes_with_filesystem(- session: Session,- prefixes: list[str],- collect_existing_paths: bool = False,- progress: _ScanProgress | None = None,-) -> set[str] | None:- if not prefixes:- return set() if collect_existing_paths else None-+def observe_references_on_filesystem(+ session: Session, prefixes: list[str], progress: _ScanProgress | None = None+) -> tuple[list[_ReferenceObservation], set[str]]:+ """Stat every live row under ``prefixes`` without writing, so the caller can+ apply the result in a short write transaction. Also returns the paths whose+ file still exists."""+ contents = [+ (content.id, content.path, content.size_bytes, content.mtime_ns)+ for content in live_contents_under_prefixes(session, prefixes)+ ]+ observations: list[_ReferenceObservation] = []survivors: set[str] = set()- for content in live_contents_under_prefixes(session, prefixes):+ for content_id, path, size_bytes, mtime_ns in contents:try:- stat_result = os.stat(content.path, follow_symlinks=True)+ stat_result = os.stat(path, follow_symlinks=True)except FileNotFoundError:- mark_content_missing(session, content.id)+ observations.append(_ReferenceObservation(content_id, size_bytes, mtime_ns, None))except PermissionError as e:_log_scan_error("reference_stat", e)if progress is not None:progress.permission_denied += 1- logging.debug("Permission denied accessing %s", content.path)+ logging.debug("Permission denied accessing %s", path)except OSError as e:_log_scan_error("reference_stat", e)- logging.debug("OSError checking %s: %s", content.path, e)- mark_content_missing(session, content.id)+ logging.debug("OSError checking %s: %s", path, e)+ observations.append(_ReferenceObservation(content_id, size_bytes, mtime_ns, None))else:- detect_content_change(- session,- content,- stat_result,- hashing_is_enabled=mode.hashing_enabled(),- )- survivors.add(os.path.abspath(content.path))+ survivors.add(os.path.abspath(path))+ if stat_result.st_mtime_ns != mtime_ns:+ observations.append(+ _ReferenceObservation(content_id, size_bytes, mtime_ns, stat_result)+ )+ return observations, survivors+++def apply_reference_observations(+ session: Session, observations: list[_ReferenceObservation]+) -> None:+ for observation in observations:+ content = session.get(AssetContent, observation.content_id)+ # Skip a row another writer changed since it was observed; the next scan sees it afresh.+ if (+ content is None+ or content.is_missing+ or content.size_bytes != observation.size_bytes+ or content.mtime_ns != observation.mtime_ns+ ):+ continue+ if observation.stat_result is None:+ mark_content_missing(session, content.id)+ continue+ detect_content_change(+ session,+ content,+ observation.stat_result,+ hashing_is_enabled=mode.hashing_enabled(),+ )- return survivors if collect_existing_paths else None++def _sync_prefixes_in_write_txn(+ prefixes: list[str], progress: _ScanProgress | None+) -> set[str]:+ with create_session() as session:+ observations, survivors = observe_references_on_filesystem(+ session, prefixes, progress+ )+ if observations:+ with create_write_session() as session:+ apply_reference_observations(session, observations)+ session.commit()+ return survivorsdef _is_under_prefixes(path: str, prefixes: list[str]) -> bool:@@ -214,15 +251,7 @@ def sync_root_safely(Returns survivors (existing paths) or empty set on failure."""try:- with create_session() as sess:- survivors = sync_references_with_filesystem(- sess,- root,- collect_existing_paths=True,- progress=progress,- )- sess.commit()- return survivors or set()+ return _sync_prefixes_in_write_txn(get_scan_prefixes_for_root(root), progress)except Exception as exc:logging.exception("fast DB scan failed for %s: %s", root, exc)emit(@@ -238,13 +267,7 @@ def sync_temp_references_safely() -> None:"""Retire temp references whose file is gone; temp is never scanned, so nothing else stats them."""try:- with create_session() as sess:- sync_prefixes_with_filesystem(- sess,- get_temp_prefixes(),- progress=progress,- )- sess.commit()+ _sync_prefixes_in_write_txn(get_temp_prefixes(), progress)except Exception as exc:logging.exception("temp reference sync failed: %s", exc)emit(@@ -375,7 +398,44 @@ def build_asset_specs(return specs, tag_pool, skipped-def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int:+class _SpecObservation(NamedTuple):+ """A spec's file as seen before the write transaction opens.++ ``snapshot`` is None when hashing is off, or when the file changed while being hashed.+ """++ stat_result: os.stat_result+ snapshot: tuple[str, os.stat_result] | None+++def observe_asset_specs(specs: list[SeedAssetSpec]) -> dict[str, _SpecObservation | None]:+ """Stat (and, in hashing mode, hash) each spec before the write transaction opens.++ ``None`` marks a path that vanished or could not be read.+ """+ hashing_is_enabled = mode.hashing_enabled()+ observed: dict[str, _SpecObservation | None] = {}+ for spec in specs:+ path = os.path.abspath(spec["abs_path"])+ try:+ stat_result = os.stat(path, follow_symlinks=True)+ snapshot = snapshot_hash(path) if hashing_is_enabled else None+ except OSError:+ observed[path] = None+ continue+ if snapshot is not None:+ stat_result = snapshot[1] # the stat the hash was verified against+ observed[path] = _SpecObservation(stat_result, snapshot)+ return observed+++def seed_asset_specs(+ session: Session,+ specs: list[SeedAssetSpec],+ observed: dict[str, _SpecObservation | None] | None = None,+) -> int:+ if observed is None:+ observed = observe_asset_specs(specs)created = 0created_content_ids: list[str] = []try:@@ -383,21 +443,17 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int:path = os.path.abspath(spec["abs_path"])try:with session.begin_nested():- try:- stat_result = os.stat(path, follow_symlinks=True)- except OSError:- logging.warning("Skipping vanished asset during scan: %s", path)- continue- try:- recovery = recover_missing_content(- session,- path,- stat_result,- hashing_is_enabled=mode.hashing_enabled(),- )- except OSError:+ observation = observed[path]+ if observation is None:logging.warning("Skipping vanished asset during scan: %s", path)continue+ stat_result = observation.stat_result+ recovery = recover_missing_content(+ session,+ path,+ observation.snapshot,+ hashing_is_enabled=mode.hashing_enabled(),+ )if recovery != "no_match":continuecontent, inserted = create_content_reporting_insert(@@ -438,9 +494,10 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int:def insert_asset_specs(specs: list[SeedAssetSpec], _tag_pool: set[str]) -> int:if not specs:return 0- with create_session() as sess:- created = seed_asset_specs(sess, specs)- sess.commit()+ observed = observe_asset_specs(specs)+ with create_write_session() as session:+ created = seed_asset_specs(session, specs, observed)+ session.commit()return createddiff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.pyindex 43ee8a47a..b7f344a8e 100644--- a/app/assets/scanner_changes.py+++ b/app/assets/scanner_changes.py@@ -44,8 +44,13 @@ def pending_recovery_count() -> int:def recover_missing_content(- session: Session, path: str, stat_result: os.stat_result, hashing_is_enabled: bool+ session: Session,+ path: str,+ snapshot: tuple[str, os.stat_result] | None,+ hashing_is_enabled: bool,) -> Literal["recovered", "no_match", "unstable"]:+ """``snapshot`` is ``snapshot_hash(path)``, taken by the caller before its write+ transaction opens so the file is never hashed while the write lock is held."""if not hashing_is_enabled:return "no_match"occupied = session.scalar(@@ -55,7 +60,6 @@ def recover_missing_content()if occupied is not None:return "no_match"- snapshot = snapshot_hash(path)if snapshot is None:if path not in _pending_recovery_paths:_pending_recovery_paths.append(path)diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.pyindex a8fb3b81d..eaeea7285 100644--- a/app/assets/services/ingest.py+++ b/app/assets/services/ingest.py@@ -47,7 +47,7 @@ from app.assets.services.schemas import (UserMetadata,)from app.assets.services.snapshot_hash import snapshot_hash-from app.database.db import create_session+from app.database.db import create_session, create_write_sessiondef _normalize_hash_input(hash_str: str) -> str:@@ -719,7 +719,7 @@ def register_executed_output(system_metadata = _extract_system_metadata_sync(locator, mime_type, stat_result)- with create_session() as session:+ with create_write_session() as session:created_content_id: str | None = Nonetry:existing = session.scalars(@@ -745,16 +745,17 @@ def register_executed_output(tags=path_tags,system_metadata=system_metadata,)+ # Read before commit: expiry would reload them in a second write transaction.+ record_id = record.id+ record_content_id = record.content_id+ record_job_id = record.job_id+ record_name = record.namesession.commit()except Exception:session.rollback()if created_content_id is not None:_discard_unreferenced_content(session, created_content_id)raise- record_id = record.id- record_content_id = record.content_id- record_job_id = record.job_id- record_name = record.nameexcept Exception:logging.exception("Failed to register executed output: %s", locator)return Nonediff --git a/app/database/db.py b/app/database/db.pyindex 053df2883..1fcff1a76 100644--- a/app/database/db.py+++ b/app/database/db.py@@ -1,6 +1,9 @@import loggingimport osimport shutil+import sqlite3+import time+from contextlib import closingfrom app.logger import log_startup_warningfrom utils.install_util import get_missing_requirements_messagefrom filelock import FileLock, Timeout@@ -8,6 +11,7 @@ from comfy.cli_args import args, database_default_path_DB_AVAILABLE = FalseSession = None+WriteSession = Nonetry:@@ -16,6 +20,7 @@ try:from alembic.runtime.migration import MigrationContextfrom alembic.script import ScriptDirectoryfrom sqlalchemy import create_engine, event+ from sqlalchemy.exc import OperationalErrorfrom sqlalchemy.orm import sessionmakerfrom sqlalchemy.pool import StaticPool@@ -103,6 +108,19 @@ def copy_legacy_default_db(db_path):if os.path.exists(backup_path):return+ if os.path.exists(legacy_db_path + "-wal"):+ # Fold committed WAL pages back into the file before it is renamed and copied.+ try:+ with closing(sqlite3.connect(legacy_db_path)) as legacy:+ busy, _, _ = legacy.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()+ except sqlite3.Error:+ busy = 1+ if busy:+ logging.warning(+ f"Not relocating legacy database '{legacy_db_path}': its WAL could not be "+ f"checkpointed, so it may still be in use."+ )+ returnos.replace(legacy_db_path, backup_path)shutil.copy(backup_path, db_path)logging.info(@@ -118,6 +136,26 @@ def prepare_file_db_path(db_path):copy_legacy_default_db(db_path)+_BACKUP_TIMEOUT_SECONDS = 5.0+_SQLITE_BUSY, _SQLITE_LOCKED = 5, 6 # sqlite3 only exports these names from Python 3.11+++def _backup_database(source_path, destination_path):+ # A plain file copy misses committed pages still in the WAL file.+ # sqlite3's backup() retries a locked database forever, so bound it: another+ # client holding the destination must not hang startup.+ deadline = time.monotonic() + _BACKUP_TIMEOUT_SECONDS++ def give_up_when_locked_too_long(status, remaining, total):+ if status in (_SQLITE_BUSY, _SQLITE_LOCKED) and time.monotonic() > deadline:+ raise TimeoutError(f"'{destination_path}' stayed locked; database backup abandoned")++ with closing(sqlite3.connect(source_path)) as source:+ with closing(sqlite3.connect(destination_path)) as destination:+ source.backup(destination, progress=give_up_when_locked_too_long)+ shutil.copymode(source_path, destination_path)++_db_lock = Nonedef _acquire_file_lock(db_path):@@ -175,8 +213,9 @@ def _init_memory_db(db_url):Base.metadata.create_all(engine)- global Session+ global Session, WriteSessionSession = sessionmaker(bind=engine)+ WriteSession = Sessiondef _init_file_db(db_url):@@ -212,6 +251,7 @@ def _migrate_and_bind(db_url, db_path, db_exists):# Check if we need to upgradeengine = create_engine(db_url)+ write_engine = create_engine(db_url)# Enable foreign key enforcement for SQLite@event.listens_for(engine, "connect")@@ -220,8 +260,30 @@ def _migrate_and_bind(db_url, db_path, db_exists):cursor.execute("PRAGMA foreign_keys=ON")cursor.close()+ # Writes go through a separate engine whose transactions take the write lock up front.+ # pysqlite otherwise defers BEGIN until the first INSERT/UPDATE/DELETE, so a transaction+ # that reads first can fail to upgrade to a write lock without waiting on busy_timeout.+ # Following SQLAlchemy's pysqlite recipe, the driver's own transaction handling is+ # switched off so the only BEGIN issued is the BEGIN IMMEDIATE below.+ @event.listens_for(write_engine, "connect")+ def configure_write_connection(dbapi_connection, connection_record):+ dbapi_connection.isolation_level = None+ dbapi_connection.execute("PRAGMA foreign_keys=ON")++ @event.listens_for(write_engine, "begin")+ def begin_immediate(connection):+ connection.exec_driver_sql("BEGIN IMMEDIATE")+conn = engine.connect()+ try:+ journal_mode = conn.exec_driver_sql("PRAGMA journal_mode=WAL").scalar_one()+ except OperationalError:+ logging.warning("Could not enable SQLite WAL mode; continuing with the default journal mode.")+ else:+ if journal_mode.lower() != "wal":+ logging.warning("SQLite WAL mode unavailable; continuing with %s journal mode.", journal_mode)+context = MigrationContext.configure(conn)current_rev = context.get_current_revision()@@ -234,7 +296,7 @@ def _migrate_and_bind(db_url, db_path, db_exists):# Backup the database pre upgradebackup_path = db_path + ".bkp"if db_exists:- shutil.copy(db_path, backup_path)+ _backup_database(db_path, backup_path)else:backup_path = None@@ -244,7 +306,7 @@ def _migrate_and_bind(db_url, db_path, db_exists):except Exception as e:if backup_path:# Restore the database from backup if upgrade fails- shutil.copy(backup_path, db_path)+ _backup_database(backup_path, db_path)os.remove(backup_path)logging.exception("Error upgrading database: ")raise e@@ -259,9 +321,19 @@ def _migrate_and_bind(db_url, db_path, db_exists):conn.close()- global Session+ global Session, WriteSessionSession = sessionmaker(bind=engine)+ WriteSession = sessionmaker(bind=write_engine)def create_session():return Session()+++def create_write_session():+ """A session whose transactions open with BEGIN IMMEDIATE. Do filesystem work before+ using it: the write lock is held from the first statement until commit. Do not open+ one inside another: the inner one waits out busy_timeout for the outer's lock, then+ fails with "database is locked", indistinguishable from real contention. Rule out a+ nested session before investigating lock contention."""+ return WriteSession()diff --git a/tests-unit/app_test/database_path_test.py b/tests-unit/app_test/database_path_test.pyindex d1387c041..5e6648bea 100644--- a/tests-unit/app_test/database_path_test.py+++ b/tests-unit/app_test/database_path_test.py@@ -1,4 +1,7 @@import os+import shutil+import sqlite3+from contextlib import closingfrom app.database import dbfrom comfy.cli_args import database_default_path@@ -132,3 +135,58 @@ def test_prepare_file_database_accepts_relative_database_path(monkeypatch, tmp_passert os.getcwd() == str(tmp_path)assert not list(tmp_path.iterdir())+++def test_legacy_copy_includes_rows_still_in_the_wal(monkeypatch, tmp_path):+ legacy_db = tmp_path / "install" / "user" / "comfyui.db"+ user_db = tmp_path / "custom_user" / "comfyui.db"+ legacy_db.parent.mkdir(parents=True)+ user_db.parent.mkdir(parents=True)+ writer = sqlite3.connect(legacy_db)+ writer.execute("PRAGMA journal_mode=WAL")+ writer.execute("PRAGMA wal_autocheckpoint=0")+ writer.execute("CREATE TABLE t (x)")+ writer.execute("INSERT INTO t VALUES (1)")+ writer.commit()+ # An unclean exit: the committed rows stay in comfyui.db-wal.+ shutil.copy(legacy_db, tmp_path / "main-only.db")+ shutil.copy(str(legacy_db) + "-wal", tmp_path / "saved-wal")+ writer.close()+ shutil.copy(tmp_path / "main-only.db", legacy_db)+ shutil.copy(tmp_path / "saved-wal", str(legacy_db) + "-wal")++ monkeypatch.setattr(db.args, "database_url", None)+ monkeypatch.setattr(db, "get_legacy_default_db_path", lambda: str(legacy_db))++ db.copy_legacy_default_db(str(user_db))++ with closing(sqlite3.connect(user_db)) as copied:+ assert copied.execute("SELECT x FROM t").fetchall() == [(1,)]+++def test_legacy_copy_skipped_when_its_wal_cannot_be_checkpointed(monkeypatch, tmp_path):+ legacy_db = tmp_path / "install" / "user" / "comfyui.db"+ user_db = tmp_path / "custom_user" / "comfyui.db"+ legacy_db.parent.mkdir(parents=True)+ user_db.parent.mkdir(parents=True)+ writer = sqlite3.connect(legacy_db)+ writer.execute("PRAGMA journal_mode=WAL")+ writer.execute("PRAGMA wal_autocheckpoint=0")+ writer.execute("CREATE TABLE t (x)")+ writer.execute("INSERT INTO t VALUES (1)")+ writer.commit()+ reader = sqlite3.connect(legacy_db, isolation_level=None)+ reader.execute("BEGIN")+ reader.execute("SELECT x FROM t").fetchall() # an open read pins the WAL++ monkeypatch.setattr(db.args, "database_url", None)+ monkeypatch.setattr(db, "get_legacy_default_db_path", lambda: str(legacy_db))++ try:+ db.copy_legacy_default_db(str(user_db))+ finally:+ reader.close()+ writer.close()++ assert not user_db.exists()+ assert legacy_db.exists()diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.pynew file mode 100644index 000000000..179768e7b--- /dev/null+++ b/tests-unit/app_test/test_db_write_txn.py@@ -0,0 +1,81 @@+import sqlite3++import pytest+from sqlalchemy import text++from app.database import db as db_module+++def file_db(tmp_path, monkeypatch):+ db_path = str(tmp_path / "comfyui.db")+ monkeypatch.setattr(db_module.args, "database_url", f"sqlite:///{db_path}")+ monkeypatch.setattr(db_module, "Session", None)+ monkeypatch.setattr(db_module, "WriteSession", None)+ monkeypatch.setattr(db_module, "_db_lock", None)+ db_module._init_file_db(db_module.args.database_url)+ yield db_path+ db_module.Session.kw["bind"].dispose()+ db_module.WriteSession.kw["bind"].dispose()+ db_module._db_lock.release(force=True)+++def _other_writer_can_begin(db_path):+ other = sqlite3.connect(db_path, timeout=0, isolation_level=None)+ try:+ other.execute("BEGIN IMMEDIATE")+ other.execute("ROLLBACK")+ return True+ except sqlite3.OperationalError:+ return False+ finally:+ other.close()+++def test_file_db_uses_wal(file_db):+ with db_module.create_session() as session:+ assert session.execute(text("PRAGMA journal_mode")).scalar_one() == "wal"+++def test_write_session_takes_the_write_lock_before_its_first_write(file_db):+ with db_module.create_write_session() as session:+ session.execute(text("SELECT 1")).scalar_one()+ assert _other_writer_can_begin(file_db) is False+++def test_read_session_does_not_take_the_write_lock(file_db):+ with db_module.create_session() as session:+ session.execute(text("SELECT 1")).scalar_one()+ assert _other_writer_can_begin(file_db) is True+++def test_backup_gives_up_when_the_destination_stays_locked(tmp_path, monkeypatch):+ source = tmp_path / "source.db"+ destination = tmp_path / "destination.db"+ with sqlite3.connect(source) as conn:+ conn.execute("CREATE TABLE t (x)")+ with sqlite3.connect(destination) as conn:+ conn.execute("CREATE TABLE u (y)")+ holder = sqlite3.connect(destination, isolation_level=None, timeout=0)+ holder.execute("BEGIN IMMEDIATE")+ monkeypatch.setattr(db_module, "_BACKUP_TIMEOUT_SECONDS", 0.0)+ try:+ with pytest.raises(TimeoutError):+ db_module._backup_database(str(source), str(destination))+ finally:+ holder.execute("ROLLBACK")+ holder.close()+++def test_backup_past_its_deadline_still_completes_when_nothing_blocks_it(tmp_path, monkeypatch):+ source = tmp_path / "source.db"+ destination = tmp_path / "destination.db"+ with sqlite3.connect(source) as conn:+ conn.execute("CREATE TABLE t (x)")+ conn.execute("INSERT INTO t VALUES (1)")+ monkeypatch.setattr(db_module, "_BACKUP_TIMEOUT_SECONDS", -1.0)++ db_module._backup_database(str(source), str(destination))++ with sqlite3.connect(destination) as conn:+ assert conn.execute("SELECT x FROM t").fetchall() == [(1,)]diff --git a/tests-unit/assets_test/services/conftest.py b/tests-unit/assets_test/services/conftest.pyindex 9c5d1dfbb..3d9bf8c66 100644--- a/tests-unit/assets_test/services/conftest.py+++ b/tests-unit/assets_test/services/conftest.py@@ -5,7 +5,7 @@ from unittest.mock import patchimport pytestfrom sqlalchemy import create_engine, event-from sqlalchemy.orm import Session, Session as SASession+from sqlalchemy.orm import Session, Session as SASession, sessionmakerfrom app.assets import modefrom app.assets.database.models import Base@@ -66,6 +66,7 @@ def mock_create_session(db_engine):yield sesswith patch("app.assets.services.ingest.create_session", _create_session), \+ patch("app.database.db.WriteSession", sessionmaker(bind=db_engine)), \patch("app.assets.services.asset_management.create_session", _create_session), \patch("app.assets.services.tagging.create_session", _create_session):yield _create_sessiondiff --git a/tests-unit/assets_test/services/test_detection_gate.py b/tests-unit/assets_test/services/test_detection_gate.pyindex a0e384d97..02794e0ce 100644--- a/tests-unit/assets_test/services/test_detection_gate.py+++ b/tests-unit/assets_test/services/test_detection_gate.py@@ -10,7 +10,8 @@ from app.assets.helpers import to_stored_hashfrom app.assets.scanner import (clear_pending_verifications,drain_pending_verifications,- sync_prefixes_with_filesystem,+ apply_reference_observations,+ observe_references_on_filesystem,)from app.assets.scanner_changes import queue_pending_verificationfrom app.assets.services.snapshot_hash import snapshot_hash@@ -44,6 +45,11 @@ def _bump_mtime(path: Path) -> None:os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000))+def _sync_references(session, root: Path) -> None:+ observations, _ = observe_references_on_filesystem(session, [str(root)])+ apply_reference_observations(session, observations)++def _stored_hash(path: Path) -> str:snapshot = snapshot_hash(str(path))assert snapshot is not None@@ -63,7 +69,7 @@ def test_off_mode_same_size_touch_does_not_split(session, temp_dir: Path):patch("folder_paths.get_input_directory", return_value=str(input_root)),patch("app.assets.scanner.mode.hashing_enabled", return_value=False),):- sync_prefixes_with_filesystem(session, [str(input_root)])+ _sync_references(session, input_root)session.commit()contents = list(session.scalars(select(AssetContent)))@@ -86,7 +92,7 @@ def test_off_mode_size_change_splits(session, temp_dir: Path):patch("folder_paths.get_input_directory", return_value=str(input_root)),patch("app.assets.scanner.mode.hashing_enabled", return_value=False),):- sync_prefixes_with_filesystem(session, [str(input_root)])+ _sync_references(session, input_root)session.commit()contents = list(session.scalars(select(AssetContent).order_by(AssetContent.created_at)))@@ -107,7 +113,7 @@ def test_hash_mode_touch_refreshes_mtime(session, temp_dir: Path):patch("folder_paths.get_input_directory", return_value=str(input_root)),patch("app.assets.scanner.mode.hashing_enabled", return_value=True),):- sync_prefixes_with_filesystem(session, [str(input_root)])+ _sync_references(session, input_root)processed = drain_pending_verifications(session)session.commit()@@ -133,7 +139,7 @@ def test_hash_mode_real_edit_splits(session, temp_dir: Path):patch("folder_paths.get_input_directory", return_value=str(input_root)),patch("app.assets.scanner.mode.hashing_enabled", return_value=True),):- sync_prefixes_with_filesystem(session, [str(input_root)])+ _sync_references(session, input_root)drain_pending_verifications(session)session.commit()@@ -158,7 +164,7 @@ def test_old_record_id_resolves_to_missing_content_after_split(session, temp_dirpatch("folder_paths.get_input_directory", return_value=str(input_root)),patch("app.assets.scanner.mode.hashing_enabled", return_value=True),):- sync_prefixes_with_filesystem(session, [str(input_root)])+ _sync_references(session, input_root)drain_pending_verifications(session)session.commit()@@ -195,3 +201,35 @@ def test_hash_mode_split_uses_stat_from_the_verified_snapshot(session, temp_dir:assert live_content is not Noneassert live_content.size_bytes == len(new_payload)assert live_content.mtime_ns == path.stat().st_mtime_ns+++def test_observation_is_skipped_when_the_row_changed_before_it_was_applied(+ session, temp_dir: Path+):+ input_root = temp_dir / "input"+ input_root.mkdir()+ path = input_root / "raced.bin"+ path.write_bytes(b"v1")+ content, _ = _seed_content(session, path, hash_value=None)+ path.write_bytes(b"v2 is longer")+ _bump_mtime(path)++ with (+ patch("folder_paths.get_input_directory", return_value=str(input_root)),+ patch("app.assets.scanner.mode.hashing_enabled", return_value=False),+ ):+ observations, _ = observe_references_on_filesystem(session, [str(input_root)])+ assert len(observations) == 1+ # The file changes again and another writer records that before the observation lands.+ path.write_bytes(b"v3 is longer still")+ _bump_mtime(path)+ content.size_bytes = path.stat().st_size+ content.mtime_ns = path.stat().st_mtime_ns+ session.commit()+ apply_reference_observations(session, observations)+ session.commit()++ contents = list(session.scalars(select(AssetContent)))+ assert len(contents) == 1+ assert contents[0].is_missing is False+ assert contents[0].size_bytes == len(b"v3 is longer still")diff --git a/tests-unit/assets_test/services/test_manager_isolated.py b/tests-unit/assets_test/services/test_manager_isolated.pyindex 2b73ba1bb..e195c1184 100644--- a/tests-unit/assets_test/services/test_manager_isolated.py+++ b/tests-unit/assets_test/services/test_manager_isolated.py@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, Mock, callimport folder_pathsimport pytestfrom sqlalchemy import create_engine, select-from sqlalchemy.orm import Session, Session as SASession+from sqlalchemy.orm import Session, Session as SASession, sessionmakerfrom sqlalchemy.pool import StaticPoolfrom app.assets import lifecycle@@ -76,6 +76,7 @@ def threaded_create_session(monkeypatch.setattr(seeder_module, "create_session", _create_session)monkeypatch.setattr(scanner, "create_session", _create_session)monkeypatch.setattr("app.assets.services.ingest.create_session", _create_session)+ monkeypatch.setattr("app.database.db.WriteSession", sessionmaker(bind=engine))yield _create_sessionengine.dispose()diff --git a/tests-unit/assets_test/services/test_recovery_gate.py b/tests-unit/assets_test/services/test_recovery_gate.pyindex eeca7e0b0..885d0100b 100644--- a/tests-unit/assets_test/services/test_recovery_gate.py+++ b/tests-unit/assets_test/services/test_recovery_gate.py@@ -111,7 +111,7 @@ def test_off_mode_no_recovery(session, temp_dir: Path):with (patch("app.assets.scanner.mode.hashing_enabled", return_value=False),- patch("app.assets.scanner_changes.snapshot_hash") as hash_mock,+ patch("app.assets.scanner.snapshot_hash") as hash_mock,):created = seed_asset_specs(session, [_spec(path)])session.commit()@@ -128,7 +128,7 @@ def test_unstable_hash_requeues(session, temp_dir: Path):with (patch("app.assets.scanner.mode.hashing_enabled", return_value=True),- patch("app.assets.scanner_changes.snapshot_hash", return_value=None),+ patch("app.assets.scanner.snapshot_hash", return_value=None),):created = seed_asset_specs(session, [_spec(path)])session.commit()@@ -156,7 +156,7 @@ def test_recovery_skips_a_path_a_live_row_already_occupies(session, temp_dir: Palive_b_id = live_b.idresult = recover_missing_content(- session, str(path), stat_result, hashing_is_enabled=True+ session, str(path), snapshot_hash(str(path)), hashing_is_enabled=True)session.commit()@@ -166,3 +166,24 @@ def test_recovery_skips_a_path_a_live_row_already_occupies(session, temp_dir: Pa)assert session.get(AssetContent, missing_a.id).is_missing is Trueassert session.get(AssetContent, live_b_id).is_missing is False+++def test_seeded_row_takes_the_stat_its_hash_was_verified_against(session, temp_dir: Path):+ path = temp_dir / "rewritten.bin"+ path.write_bytes(b"old")++ def rewrite_then_hash(candidate_path: str):+ path.write_bytes(b"rewritten between the stat and the hash")+ return snapshot_hash(candidate_path)++ with (+ patch("app.assets.scanner.mode.hashing_enabled", return_value=True),+ patch("app.assets.scanner.snapshot_hash", side_effect=rewrite_then_hash),+ ):+ created = seed_asset_specs(session, [_spec(path)])+ session.commit()++ assert created == 1+ content = session.scalars(select(AssetContent)).one()+ assert content.size_bytes == path.stat().st_size+ assert content.mtime_ns == path.stat().st_mtime_nsdiff --git a/tests-unit/assets_test/services/test_scan_lifecycle.py b/tests-unit/assets_test/services/test_scan_lifecycle.pyindex e5a1d2801..134dc2118 100644--- a/tests-unit/assets_test/services/test_scan_lifecycle.py+++ b/tests-unit/assets_test/services/test_scan_lifecycle.py@@ -4,7 +4,12 @@ from unittest.mock import patchfrom sqlalchemy import selectfrom app.assets.database.models import AssetContent-from app.assets.scanner import build_asset_specs, seed_asset_specs, sync_prefixes_with_filesystem+from app.assets.scanner import (+ apply_reference_observations,+ build_asset_specs,+ observe_references_on_filesystem,+ seed_asset_specs,+)def _scan(session, root: Path) -> int:@@ -26,7 +31,9 @@ def test_e2e_scan_seed_detect_prune(session, temp_dir: Path):edited.write_bytes(b"replacement")(root / "partial.part").write_bytes(b"partial")with patch("app.assets.scanner.mode.hashing_enabled", return_value=False):- sync_prefixes_with_filesystem(session, [str(root)])+ apply_reference_observations(+ session, observe_references_on_filesystem(session, [str(root)])[0]+ )_scan(session, root)session.commit()contents = list(session.scalars(select(AssetContent)))diff --git a/tests-unit/assets_test/services/test_scanner_b.py b/tests-unit/assets_test/services/test_scanner_b.pyindex 93c155870..c298381fc 100644--- a/tests-unit/assets_test/services/test_scanner_b.py+++ b/tests-unit/assets_test/services/test_scanner_b.py@@ -14,7 +14,8 @@ from app.assets.scanner import (enrich_asset,mark_contents_missing_outside_prefixes,seed_asset_specs,- sync_prefixes_with_filesystem,+ apply_reference_observations,+ observe_references_on_filesystem,)from app.assets.services.snapshot_hash import snapshot_hash@@ -180,7 +181,9 @@ def test_unhashed_missing_content_gets_tagged(session, temp_dir: Path):session.add(record)session.commit()- sync_prefixes_with_filesystem(session, prefixes=[str(temp_dir)])+ apply_reference_observations(+ session, observe_references_on_filesystem(session, [str(temp_dir)])[0]+ )session.commit()session.expire_all()diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.pyindex f6f8f6ba4..9dc72e82e 100644--- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py+++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py@@ -64,7 +64,7 @@ def test_seed_persists_remaining_specs_when_path_vanishes_during_recovery_hash(raise OSError("file vanished during recovery")return snapshot_hash(path)- monkeypatch.setattr("app.assets.scanner_changes.snapshot_hash", _hash_or_raise)+ monkeypatch.setattr("app.assets.scanner.snapshot_hash", _hash_or_raise)with patch("app.assets.scanner.mode.hashing_enabled", return_value=True):created = seed_asset_specs(session, specs)@@ -85,7 +85,7 @@ def _delete_during_recovery(monkeypatch: pytest.MonkeyPatch, path: Path) -> Noneraise OSError("file vanished during recovery")return snapshot_hash(candidate_path)- monkeypatch.setattr("app.assets.scanner_changes.snapshot_hash", _hash_or_raise)+ monkeypatch.setattr("app.assets.scanner.snapshot_hash", _hash_or_raise)@pytest.mark.parametrize(diff --git a/tests-unit/assets_test/services/test_stored_hash_prefix.py b/tests-unit/assets_test/services/test_stored_hash_prefix.pyindex c28d6fb34..277073774 100644--- a/tests-unit/assets_test/services/test_stored_hash_prefix.py+++ b/tests-unit/assets_test/services/test_stored_hash_prefix.py@@ -183,9 +183,8 @@ def test_recovery_matches_prefixed_stored_hash(session, temp_dir):path.unlink()path.write_bytes(original_bytes)- stat = os.stat(str(path))result = recover_missing_content(- session, str(path), stat, hashing_is_enabled=True+ session, str(path), snapshot_hash(str(path)), hashing_is_enabled=True)assert result == "recovered"diff --git a/tests-unit/assets_test/test_intended_behaviour.py b/tests-unit/assets_test/test_intended_behaviour.pyindex 580637c71..19a00e4c1 100644--- a/tests-unit/assets_test/test_intended_behaviour.py+++ b/tests-unit/assets_test/test_intended_behaviour.py@@ -31,7 +31,8 @@ from app.assets.lifecycle import wipe_temp_db_rowsfrom app.assets.scanner import (build_asset_specs,seed_asset_specs,- sync_prefixes_with_filesystem,+ apply_reference_observations,+ observe_references_on_filesystem,)from app.assets.scanner_admission import _should_skip_extensionfrom app.assets.scanner_changes import (@@ -105,9 +106,8 @@ def _seed_content_row(session, path: Path, hash_value: str | None = None):def _scan_pass(session, root: Path) -> int:- survivors = sync_prefixes_with_filesystem(- session, [str(root)], collect_existing_paths=True- )+ observations, survivors = observe_references_on_filesystem(session, [str(root)])+ apply_reference_observations(session, observations)specs, _tag_pool, _skipped = build_asset_specs(list_files_recursively(str(root)), survivors or set())diff --git a/tests-unit/assets_test/test_scanner_ingest_event_log.py b/tests-unit/assets_test/test_scanner_ingest_event_log.pyindex f81ebee0f..f6eb62803 100644--- a/tests-unit/assets_test/test_scanner_ingest_event_log.py+++ b/tests-unit/assets_test/test_scanner_ingest_event_log.py@@ -119,7 +119,7 @@ def test_permission_error_in_reference_sync_increments_scan_counter(monkeypatch: pytest.MonkeyPatch,) -> None:secret_path = "/private/assets/unreadable.safetensors"- content = SimpleNamespace(id="content", path=secret_path)+ content = SimpleNamespace(id="content", path=secret_path, size_bytes=1, mtime_ns=1)progress = _ScanState()def deny_stat(*_args, **_kwargs):@@ -128,7 +128,7 @@ def test_permission_error_in_reference_sync_increments_scan_counter(monkeypatch.setattr(scanner, "os", SimpleNamespace(stat=deny_stat, path=scanner.os.path))monkeypatch.setattr(scanner, "live_contents_under_prefixes", lambda _session, _prefixes: [content])- scanner.sync_prefixes_with_filesystem(Mock(), ["/private/assets"], progress=progress)+ scanner.observe_references_on_filesystem(Mock(), ["/private/assets"], progress=progress)assert progress.permission_denied == 1