Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Take the SQLite write lock up front for asset scan and output-registration writes (#16480)

Simon Pinfold9d3bfc393b34
diff --git a/app/assets/scanner.py b/app/assets/scanner.py
index 76821e4c3..5671df0f6 100644
--- a/app/assets/scanner.py
+++ b/app/assets/scanner.py
@@ -11,7 +11,7 @@ import logging
import os
from dataclasses import dataclass
from pathlib import Path
-from typing import Callable, Literal, Protocol, TypedDict
+from typing import Callable, Literal, NamedTuple, Protocol, TypedDict
import folder_paths
import sqlalchemy as sa
@@ -54,7 +54,7 @@ from app.assets.services.path_utils import (
)
from app.assets.services.ingest import _discard_unreferenced_content
from app.assets.services.snapshot_hash import snapshot_hash
-from app.database.db import create_session
+from app.database.db import create_session, create_write_session
__all__ = [
"clear_pending_verifications",
@@ -96,6 +96,20 @@ class UnenrichedContent:
file_path: str
+class _ReferenceObservation(NamedTuple):
+ """One live row's file, stat'ed outside the write transaction.
+
+ ``size_bytes`` and ``mtime_ns`` are the row's values when observed, not the file's:
+ the write skips a row that no longer matches them. ``stat_result`` is None when the
+ file is gone.
+ """
+
+ content_id: str
+ size_bytes: int | None
+ mtime_ns: int | None
+ stat_result: os.stat_result | None
+
+
def _log_scan_error(phase: str, error: OSError) -> None:
error_type = (
"permission_denied" if isinstance(error, PermissionError) else "os_error"
@@ -152,54 +166,77 @@ def collect_models_files() -> list[str]:
return out
-def sync_references_with_filesystem(
- session,
- root: RootType,
- collect_existing_paths: bool = False,
- progress: _ScanProgress | None = None,
-) -> set[str] | None:
- return sync_prefixes_with_filesystem(
- session,
- get_scan_prefixes_for_root(root),
- collect_existing_paths=collect_existing_paths,
- progress=progress,
- )
-
-
-def sync_prefixes_with_filesystem(
- session: Session,
- prefixes: list[str],
- collect_existing_paths: bool = False,
- progress: _ScanProgress | None = None,
-) -> set[str] | None:
- if not prefixes:
- return set() if collect_existing_paths else None
-
+def observe_references_on_filesystem(
+ session: Session, prefixes: list[str], progress: _ScanProgress | None = None
+) -> tuple[list[_ReferenceObservation], set[str]]:
+ """Stat every live row under ``prefixes`` without writing, so the caller can
+ apply the result in a short write transaction. Also returns the paths whose
+ file still exists."""
+ contents = [
+ (content.id, content.path, content.size_bytes, content.mtime_ns)
+ for content in live_contents_under_prefixes(session, prefixes)
+ ]
+ observations: list[_ReferenceObservation] = []
survivors: set[str] = set()
- for content in live_contents_under_prefixes(session, prefixes):
+ for content_id, path, size_bytes, mtime_ns in contents:
try:
- stat_result = os.stat(content.path, follow_symlinks=True)
+ stat_result = os.stat(path, follow_symlinks=True)
except FileNotFoundError:
- mark_content_missing(session, content.id)
+ observations.append(_ReferenceObservation(content_id, size_bytes, mtime_ns, None))
except PermissionError as e:
_log_scan_error("reference_stat", e)
if progress is not None:
progress.permission_denied += 1
- logging.debug("Permission denied accessing %s", content.path)
+ logging.debug("Permission denied accessing %s", path)
except OSError as e:
_log_scan_error("reference_stat", e)
- logging.debug("OSError checking %s: %s", content.path, e)
- mark_content_missing(session, content.id)
+ logging.debug("OSError checking %s: %s", path, e)
+ observations.append(_ReferenceObservation(content_id, size_bytes, mtime_ns, None))
else:
- detect_content_change(
- session,
- content,
- stat_result,
- hashing_is_enabled=mode.hashing_enabled(),
- )
- survivors.add(os.path.abspath(content.path))
+ survivors.add(os.path.abspath(path))
+ if stat_result.st_mtime_ns != mtime_ns:
+ observations.append(
+ _ReferenceObservation(content_id, size_bytes, mtime_ns, stat_result)
+ )
+ return observations, survivors
+
+
+def apply_reference_observations(
+ session: Session, observations: list[_ReferenceObservation]
+) -> None:
+ for observation in observations:
+ content = session.get(AssetContent, observation.content_id)
+ # Skip a row another writer changed since it was observed; the next scan sees it afresh.
+ if (
+ content is None
+ or content.is_missing
+ or content.size_bytes != observation.size_bytes
+ or content.mtime_ns != observation.mtime_ns
+ ):
+ continue
+ if observation.stat_result is None:
+ mark_content_missing(session, content.id)
+ continue
+ detect_content_change(
+ session,
+ content,
+ observation.stat_result,
+ hashing_is_enabled=mode.hashing_enabled(),
+ )
- return survivors if collect_existing_paths else None
+
+def _sync_prefixes_in_write_txn(
+ prefixes: list[str], progress: _ScanProgress | None
+) -> set[str]:
+ with create_session() as session:
+ observations, survivors = observe_references_on_filesystem(
+ session, prefixes, progress
+ )
+ if observations:
+ with create_write_session() as session:
+ apply_reference_observations(session, observations)
+ session.commit()
+ return survivors
def _is_under_prefixes(path: str, prefixes: list[str]) -> bool:
@@ -214,15 +251,7 @@ def sync_root_safely(
Returns survivors (existing paths) or empty set on failure.
"""
try:
- with create_session() as sess:
- survivors = sync_references_with_filesystem(
- sess,
- root,
- collect_existing_paths=True,
- progress=progress,
- )
- sess.commit()
- return survivors or set()
+ return _sync_prefixes_in_write_txn(get_scan_prefixes_for_root(root), progress)
except Exception as exc:
logging.exception("fast DB scan failed for %s: %s", root, exc)
emit(
@@ -238,13 +267,7 @@ def sync_temp_references_safely(
) -> None:
"""Retire temp references whose file is gone; temp is never scanned, so nothing else stats them."""
try:
- with create_session() as sess:
- sync_prefixes_with_filesystem(
- sess,
- get_temp_prefixes(),
- progress=progress,
- )
- sess.commit()
+ _sync_prefixes_in_write_txn(get_temp_prefixes(), progress)
except Exception as exc:
logging.exception("temp reference sync failed: %s", exc)
emit(
@@ -375,7 +398,44 @@ def build_asset_specs(
return specs, tag_pool, skipped
-def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int:
+class _SpecObservation(NamedTuple):
+ """A spec's file as seen before the write transaction opens.
+
+ ``snapshot`` is None when hashing is off, or when the file changed while being hashed.
+ """
+
+ stat_result: os.stat_result
+ snapshot: tuple[str, os.stat_result] | None
+
+
+def observe_asset_specs(specs: list[SeedAssetSpec]) -> dict[str, _SpecObservation | None]:
+ """Stat (and, in hashing mode, hash) each spec before the write transaction opens.
+
+ ``None`` marks a path that vanished or could not be read.
+ """
+ hashing_is_enabled = mode.hashing_enabled()
+ observed: dict[str, _SpecObservation | None] = {}
+ for spec in specs:
+ path = os.path.abspath(spec["abs_path"])
+ try:
+ stat_result = os.stat(path, follow_symlinks=True)
+ snapshot = snapshot_hash(path) if hashing_is_enabled else None
+ except OSError:
+ observed[path] = None
+ continue
+ if snapshot is not None:
+ stat_result = snapshot[1] # the stat the hash was verified against
+ observed[path] = _SpecObservation(stat_result, snapshot)
+ return observed
+
+
+def seed_asset_specs(
+ session: Session,
+ specs: list[SeedAssetSpec],
+ observed: dict[str, _SpecObservation | None] | None = None,
+) -> int:
+ if observed is None:
+ observed = observe_asset_specs(specs)
created = 0
created_content_ids: list[str] = []
try:
@@ -383,21 +443,17 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int:
path = os.path.abspath(spec["abs_path"])
try:
with session.begin_nested():
- try:
- stat_result = os.stat(path, follow_symlinks=True)
- except OSError:
- logging.warning("Skipping vanished asset during scan: %s", path)
- continue
- try:
- recovery = recover_missing_content(
- session,
- path,
- stat_result,
- hashing_is_enabled=mode.hashing_enabled(),
- )
- except OSError:
+ observation = observed[path]
+ if observation is None:
logging.warning("Skipping vanished asset during scan: %s", path)
continue
+ stat_result = observation.stat_result
+ recovery = recover_missing_content(
+ session,
+ path,
+ observation.snapshot,
+ hashing_is_enabled=mode.hashing_enabled(),
+ )
if recovery != "no_match":
continue
content, inserted = create_content_reporting_insert(
@@ -438,9 +494,10 @@ def seed_asset_specs(session: Session, specs: list[SeedAssetSpec]) -> int:
def insert_asset_specs(specs: list[SeedAssetSpec], _tag_pool: set[str]) -> int:
if not specs:
return 0
- with create_session() as sess:
- created = seed_asset_specs(sess, specs)
- sess.commit()
+ observed = observe_asset_specs(specs)
+ with create_write_session() as session:
+ created = seed_asset_specs(session, specs, observed)
+ session.commit()
return created
diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py
index 43ee8a47a..b7f344a8e 100644
--- a/app/assets/scanner_changes.py
+++ b/app/assets/scanner_changes.py
@@ -44,8 +44,13 @@ def pending_recovery_count() -> int:
def recover_missing_content(
- session: Session, path: str, stat_result: os.stat_result, hashing_is_enabled: bool
+ session: Session,
+ path: str,
+ snapshot: tuple[str, os.stat_result] | None,
+ hashing_is_enabled: bool,
) -> Literal["recovered", "no_match", "unstable"]:
+ """``snapshot`` is ``snapshot_hash(path)``, taken by the caller before its write
+ transaction opens so the file is never hashed while the write lock is held."""
if not hashing_is_enabled:
return "no_match"
occupied = session.scalar(
@@ -55,7 +60,6 @@ def recover_missing_content(
)
if occupied is not None:
return "no_match"
- snapshot = snapshot_hash(path)
if snapshot is None:
if path not in _pending_recovery_paths:
_pending_recovery_paths.append(path)
diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py
index a8fb3b81d..eaeea7285 100644
--- a/app/assets/services/ingest.py
+++ b/app/assets/services/ingest.py
@@ -47,7 +47,7 @@ from app.assets.services.schemas import (
UserMetadata,
)
from app.assets.services.snapshot_hash import snapshot_hash
-from app.database.db import create_session
+from app.database.db import create_session, create_write_session
def _normalize_hash_input(hash_str: str) -> str:
@@ -719,7 +719,7 @@ def register_executed_output(
system_metadata = _extract_system_metadata_sync(
locator, mime_type, stat_result
)
- with create_session() as session:
+ with create_write_session() as session:
created_content_id: str | None = None
try:
existing = session.scalars(
@@ -745,16 +745,17 @@ def register_executed_output(
tags=path_tags,
system_metadata=system_metadata,
)
+ # Read before commit: expiry would reload them in a second write transaction.
+ record_id = record.id
+ record_content_id = record.content_id
+ record_job_id = record.job_id
+ record_name = record.name
session.commit()
except Exception:
session.rollback()
if created_content_id is not None:
_discard_unreferenced_content(session, created_content_id)
raise
- record_id = record.id
- record_content_id = record.content_id
- record_job_id = record.job_id
- record_name = record.name
except Exception:
logging.exception("Failed to register executed output: %s", locator)
return None
diff --git a/app/database/db.py b/app/database/db.py
index 053df2883..1fcff1a76 100644
--- a/app/database/db.py
+++ b/app/database/db.py
@@ -1,6 +1,9 @@
import logging
import os
import shutil
+import sqlite3
+import time
+from contextlib import closing
from app.logger import log_startup_warning
from utils.install_util import get_missing_requirements_message
from filelock import FileLock, Timeout
@@ -8,6 +11,7 @@ from comfy.cli_args import args, database_default_path
_DB_AVAILABLE = False
Session = None
+WriteSession = None
try:
@@ -16,6 +20,7 @@ try:
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory
from sqlalchemy import create_engine, event
+ from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool
@@ -103,6 +108,19 @@ def copy_legacy_default_db(db_path):
if os.path.exists(backup_path):
return
+ if os.path.exists(legacy_db_path + "-wal"):
+ # Fold committed WAL pages back into the file before it is renamed and copied.
+ try:
+ with closing(sqlite3.connect(legacy_db_path)) as legacy:
+ busy, _, _ = legacy.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
+ except sqlite3.Error:
+ busy = 1
+ if busy:
+ logging.warning(
+ f"Not relocating legacy database '{legacy_db_path}': its WAL could not be "
+ f"checkpointed, so it may still be in use."
+ )
+ return
os.replace(legacy_db_path, backup_path)
shutil.copy(backup_path, db_path)
logging.info(
@@ -118,6 +136,26 @@ def prepare_file_db_path(db_path):
copy_legacy_default_db(db_path)
+_BACKUP_TIMEOUT_SECONDS = 5.0
+_SQLITE_BUSY, _SQLITE_LOCKED = 5, 6 # sqlite3 only exports these names from Python 3.11
+
+
+def _backup_database(source_path, destination_path):
+ # A plain file copy misses committed pages still in the WAL file.
+ # sqlite3's backup() retries a locked database forever, so bound it: another
+ # client holding the destination must not hang startup.
+ deadline = time.monotonic() + _BACKUP_TIMEOUT_SECONDS
+
+ def give_up_when_locked_too_long(status, remaining, total):
+ if status in (_SQLITE_BUSY, _SQLITE_LOCKED) and time.monotonic() > deadline:
+ raise TimeoutError(f"'{destination_path}' stayed locked; database backup abandoned")
+
+ with closing(sqlite3.connect(source_path)) as source:
+ with closing(sqlite3.connect(destination_path)) as destination:
+ source.backup(destination, progress=give_up_when_locked_too_long)
+ shutil.copymode(source_path, destination_path)
+
+
_db_lock = None
def _acquire_file_lock(db_path):
@@ -175,8 +213,9 @@ def _init_memory_db(db_url):
Base.metadata.create_all(engine)
- global Session
+ global Session, WriteSession
Session = sessionmaker(bind=engine)
+ WriteSession = Session
def _init_file_db(db_url):
@@ -212,6 +251,7 @@ def _migrate_and_bind(db_url, db_path, db_exists):
# Check if we need to upgrade
engine = create_engine(db_url)
+ write_engine = create_engine(db_url)
# Enable foreign key enforcement for SQLite
@event.listens_for(engine, "connect")
@@ -220,8 +260,30 @@ def _migrate_and_bind(db_url, db_path, db_exists):
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
+ # Writes go through a separate engine whose transactions take the write lock up front.
+ # pysqlite otherwise defers BEGIN until the first INSERT/UPDATE/DELETE, so a transaction
+ # that reads first can fail to upgrade to a write lock without waiting on busy_timeout.
+ # Following SQLAlchemy's pysqlite recipe, the driver's own transaction handling is
+ # switched off so the only BEGIN issued is the BEGIN IMMEDIATE below.
+ @event.listens_for(write_engine, "connect")
+ def configure_write_connection(dbapi_connection, connection_record):
+ dbapi_connection.isolation_level = None
+ dbapi_connection.execute("PRAGMA foreign_keys=ON")
+
+ @event.listens_for(write_engine, "begin")
+ def begin_immediate(connection):
+ connection.exec_driver_sql("BEGIN IMMEDIATE")
+
conn = engine.connect()
+ try:
+ journal_mode = conn.exec_driver_sql("PRAGMA journal_mode=WAL").scalar_one()
+ except OperationalError:
+ logging.warning("Could not enable SQLite WAL mode; continuing with the default journal mode.")
+ else:
+ if journal_mode.lower() != "wal":
+ logging.warning("SQLite WAL mode unavailable; continuing with %s journal mode.", journal_mode)
+
context = MigrationContext.configure(conn)
current_rev = context.get_current_revision()
@@ -234,7 +296,7 @@ def _migrate_and_bind(db_url, db_path, db_exists):
# Backup the database pre upgrade
backup_path = db_path + ".bkp"
if db_exists:
- shutil.copy(db_path, backup_path)
+ _backup_database(db_path, backup_path)
else:
backup_path = None
@@ -244,7 +306,7 @@ def _migrate_and_bind(db_url, db_path, db_exists):
except Exception as e:
if backup_path:
# Restore the database from backup if upgrade fails
- shutil.copy(backup_path, db_path)
+ _backup_database(backup_path, db_path)
os.remove(backup_path)
logging.exception("Error upgrading database: ")
raise e
@@ -259,9 +321,19 @@ def _migrate_and_bind(db_url, db_path, db_exists):
conn.close()
- global Session
+ global Session, WriteSession
Session = sessionmaker(bind=engine)
+ WriteSession = sessionmaker(bind=write_engine)
def create_session():
return Session()
+
+
+def create_write_session():
+ """A session whose transactions open with BEGIN IMMEDIATE. Do filesystem work before
+ using it: the write lock is held from the first statement until commit. Do not open
+ one inside another: the inner one waits out busy_timeout for the outer's lock, then
+ fails with "database is locked", indistinguishable from real contention. Rule out a
+ nested session before investigating lock contention."""
+ return WriteSession()
diff --git a/tests-unit/app_test/database_path_test.py b/tests-unit/app_test/database_path_test.py
index d1387c041..5e6648bea 100644
--- a/tests-unit/app_test/database_path_test.py
+++ b/tests-unit/app_test/database_path_test.py
@@ -1,4 +1,7 @@
import os
+import shutil
+import sqlite3
+from contextlib import closing
from app.database import db
from comfy.cli_args import database_default_path
@@ -132,3 +135,58 @@ def test_prepare_file_database_accepts_relative_database_path(monkeypatch, tmp_p
assert os.getcwd() == str(tmp_path)
assert not list(tmp_path.iterdir())
+
+
+def test_legacy_copy_includes_rows_still_in_the_wal(monkeypatch, tmp_path):
+ legacy_db = tmp_path / "install" / "user" / "comfyui.db"
+ user_db = tmp_path / "custom_user" / "comfyui.db"
+ legacy_db.parent.mkdir(parents=True)
+ user_db.parent.mkdir(parents=True)
+ writer = sqlite3.connect(legacy_db)
+ writer.execute("PRAGMA journal_mode=WAL")
+ writer.execute("PRAGMA wal_autocheckpoint=0")
+ writer.execute("CREATE TABLE t (x)")
+ writer.execute("INSERT INTO t VALUES (1)")
+ writer.commit()
+ # An unclean exit: the committed rows stay in comfyui.db-wal.
+ shutil.copy(legacy_db, tmp_path / "main-only.db")
+ shutil.copy(str(legacy_db) + "-wal", tmp_path / "saved-wal")
+ writer.close()
+ shutil.copy(tmp_path / "main-only.db", legacy_db)
+ shutil.copy(tmp_path / "saved-wal", str(legacy_db) + "-wal")
+
+ monkeypatch.setattr(db.args, "database_url", None)
+ monkeypatch.setattr(db, "get_legacy_default_db_path", lambda: str(legacy_db))
+
+ db.copy_legacy_default_db(str(user_db))
+
+ with closing(sqlite3.connect(user_db)) as copied:
+ assert copied.execute("SELECT x FROM t").fetchall() == [(1,)]
+
+
+def test_legacy_copy_skipped_when_its_wal_cannot_be_checkpointed(monkeypatch, tmp_path):
+ legacy_db = tmp_path / "install" / "user" / "comfyui.db"
+ user_db = tmp_path / "custom_user" / "comfyui.db"
+ legacy_db.parent.mkdir(parents=True)
+ user_db.parent.mkdir(parents=True)
+ writer = sqlite3.connect(legacy_db)
+ writer.execute("PRAGMA journal_mode=WAL")
+ writer.execute("PRAGMA wal_autocheckpoint=0")
+ writer.execute("CREATE TABLE t (x)")
+ writer.execute("INSERT INTO t VALUES (1)")
+ writer.commit()
+ reader = sqlite3.connect(legacy_db, isolation_level=None)
+ reader.execute("BEGIN")
+ reader.execute("SELECT x FROM t").fetchall() # an open read pins the WAL
+
+ monkeypatch.setattr(db.args, "database_url", None)
+ monkeypatch.setattr(db, "get_legacy_default_db_path", lambda: str(legacy_db))
+
+ try:
+ db.copy_legacy_default_db(str(user_db))
+ finally:
+ reader.close()
+ writer.close()
+
+ assert not user_db.exists()
+ assert legacy_db.exists()
diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py
new file mode 100644
index 000000000..179768e7b
--- /dev/null
+++ b/tests-unit/app_test/test_db_write_txn.py
@@ -0,0 +1,81 @@
+import sqlite3
+
+import pytest
+from sqlalchemy import text
+
+from app.database import db as db_module
+
+
+def file_db(tmp_path, monkeypatch):
+ db_path = str(tmp_path / "comfyui.db")
+ monkeypatch.setattr(db_module.args, "database_url", f"sqlite:///{db_path}")
+ monkeypatch.setattr(db_module, "Session", None)
+ monkeypatch.setattr(db_module, "WriteSession", None)
+ monkeypatch.setattr(db_module, "_db_lock", None)
+ db_module._init_file_db(db_module.args.database_url)
+ yield db_path
+ db_module.Session.kw["bind"].dispose()
+ db_module.WriteSession.kw["bind"].dispose()
+ db_module._db_lock.release(force=True)
+
+
+def _other_writer_can_begin(db_path):
+ other = sqlite3.connect(db_path, timeout=0, isolation_level=None)
+ try:
+ other.execute("BEGIN IMMEDIATE")
+ other.execute("ROLLBACK")
+ return True
+ except sqlite3.OperationalError:
+ return False
+ finally:
+ other.close()
+
+
+def test_file_db_uses_wal(file_db):
+ with db_module.create_session() as session:
+ assert session.execute(text("PRAGMA journal_mode")).scalar_one() == "wal"
+
+
+def test_write_session_takes_the_write_lock_before_its_first_write(file_db):
+ with db_module.create_write_session() as session:
+ session.execute(text("SELECT 1")).scalar_one()
+ assert _other_writer_can_begin(file_db) is False
+
+
+def test_read_session_does_not_take_the_write_lock(file_db):
+ with db_module.create_session() as session:
+ session.execute(text("SELECT 1")).scalar_one()
+ assert _other_writer_can_begin(file_db) is True
+
+
+def test_backup_gives_up_when_the_destination_stays_locked(tmp_path, monkeypatch):
+ source = tmp_path / "source.db"
+ destination = tmp_path / "destination.db"
+ with sqlite3.connect(source) as conn:
+ conn.execute("CREATE TABLE t (x)")
+ with sqlite3.connect(destination) as conn:
+ conn.execute("CREATE TABLE u (y)")
+ holder = sqlite3.connect(destination, isolation_level=None, timeout=0)
+ holder.execute("BEGIN IMMEDIATE")
+ monkeypatch.setattr(db_module, "_BACKUP_TIMEOUT_SECONDS", 0.0)
+ try:
+ with pytest.raises(TimeoutError):
+ db_module._backup_database(str(source), str(destination))
+ finally:
+ holder.execute("ROLLBACK")
+ holder.close()
+
+
+def test_backup_past_its_deadline_still_completes_when_nothing_blocks_it(tmp_path, monkeypatch):
+ source = tmp_path / "source.db"
+ destination = tmp_path / "destination.db"
+ with sqlite3.connect(source) as conn:
+ conn.execute("CREATE TABLE t (x)")
+ conn.execute("INSERT INTO t VALUES (1)")
+ monkeypatch.setattr(db_module, "_BACKUP_TIMEOUT_SECONDS", -1.0)
+
+ db_module._backup_database(str(source), str(destination))
+
+ with sqlite3.connect(destination) as conn:
+ assert conn.execute("SELECT x FROM t").fetchall() == [(1,)]
diff --git a/tests-unit/assets_test/services/conftest.py b/tests-unit/assets_test/services/conftest.py
index 9c5d1dfbb..3d9bf8c66 100644
--- a/tests-unit/assets_test/services/conftest.py
+++ b/tests-unit/assets_test/services/conftest.py
@@ -5,7 +5,7 @@ from unittest.mock import patch
import pytest
from sqlalchemy import create_engine, event
-from sqlalchemy.orm import Session, Session as SASession
+from sqlalchemy.orm import Session, Session as SASession, sessionmaker
from app.assets import mode
from app.assets.database.models import Base
@@ -66,6 +66,7 @@ def mock_create_session(db_engine):
yield sess
with patch("app.assets.services.ingest.create_session", _create_session), \
+ patch("app.database.db.WriteSession", sessionmaker(bind=db_engine)), \
patch("app.assets.services.asset_management.create_session", _create_session), \
patch("app.assets.services.tagging.create_session", _create_session):
yield _create_session
diff --git a/tests-unit/assets_test/services/test_detection_gate.py b/tests-unit/assets_test/services/test_detection_gate.py
index a0e384d97..02794e0ce 100644
--- a/tests-unit/assets_test/services/test_detection_gate.py
+++ b/tests-unit/assets_test/services/test_detection_gate.py
@@ -10,7 +10,8 @@ from app.assets.helpers import to_stored_hash
from app.assets.scanner import (
clear_pending_verifications,
drain_pending_verifications,
- sync_prefixes_with_filesystem,
+ apply_reference_observations,
+ observe_references_on_filesystem,
)
from app.assets.scanner_changes import queue_pending_verification
from app.assets.services.snapshot_hash import snapshot_hash
@@ -44,6 +45,11 @@ def _bump_mtime(path: Path) -> None:
os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000))
+def _sync_references(session, root: Path) -> None:
+ observations, _ = observe_references_on_filesystem(session, [str(root)])
+ apply_reference_observations(session, observations)
+
+
def _stored_hash(path: Path) -> str:
snapshot = snapshot_hash(str(path))
assert snapshot is not None
@@ -63,7 +69,7 @@ def test_off_mode_same_size_touch_does_not_split(session, temp_dir: Path):
patch("folder_paths.get_input_directory", return_value=str(input_root)),
patch("app.assets.scanner.mode.hashing_enabled", return_value=False),
):
- sync_prefixes_with_filesystem(session, [str(input_root)])
+ _sync_references(session, input_root)
session.commit()
contents = list(session.scalars(select(AssetContent)))
@@ -86,7 +92,7 @@ def test_off_mode_size_change_splits(session, temp_dir: Path):
patch("folder_paths.get_input_directory", return_value=str(input_root)),
patch("app.assets.scanner.mode.hashing_enabled", return_value=False),
):
- sync_prefixes_with_filesystem(session, [str(input_root)])
+ _sync_references(session, input_root)
session.commit()
contents = list(session.scalars(select(AssetContent).order_by(AssetContent.created_at)))
@@ -107,7 +113,7 @@ def test_hash_mode_touch_refreshes_mtime(session, temp_dir: Path):
patch("folder_paths.get_input_directory", return_value=str(input_root)),
patch("app.assets.scanner.mode.hashing_enabled", return_value=True),
):
- sync_prefixes_with_filesystem(session, [str(input_root)])
+ _sync_references(session, input_root)
processed = drain_pending_verifications(session)
session.commit()
@@ -133,7 +139,7 @@ def test_hash_mode_real_edit_splits(session, temp_dir: Path):
patch("folder_paths.get_input_directory", return_value=str(input_root)),
patch("app.assets.scanner.mode.hashing_enabled", return_value=True),
):
- sync_prefixes_with_filesystem(session, [str(input_root)])
+ _sync_references(session, input_root)
drain_pending_verifications(session)
session.commit()
@@ -158,7 +164,7 @@ def test_old_record_id_resolves_to_missing_content_after_split(session, temp_dir
patch("folder_paths.get_input_directory", return_value=str(input_root)),
patch("app.assets.scanner.mode.hashing_enabled", return_value=True),
):
- sync_prefixes_with_filesystem(session, [str(input_root)])
+ _sync_references(session, input_root)
drain_pending_verifications(session)
session.commit()
@@ -195,3 +201,35 @@ def test_hash_mode_split_uses_stat_from_the_verified_snapshot(session, temp_dir:
assert live_content is not None
assert live_content.size_bytes == len(new_payload)
assert live_content.mtime_ns == path.stat().st_mtime_ns
+
+
+def test_observation_is_skipped_when_the_row_changed_before_it_was_applied(
+ session, temp_dir: Path
+):
+ input_root = temp_dir / "input"
+ input_root.mkdir()
+ path = input_root / "raced.bin"
+ path.write_bytes(b"v1")
+ content, _ = _seed_content(session, path, hash_value=None)
+ path.write_bytes(b"v2 is longer")
+ _bump_mtime(path)
+
+ with (
+ patch("folder_paths.get_input_directory", return_value=str(input_root)),
+ patch("app.assets.scanner.mode.hashing_enabled", return_value=False),
+ ):
+ observations, _ = observe_references_on_filesystem(session, [str(input_root)])
+ assert len(observations) == 1
+ # The file changes again and another writer records that before the observation lands.
+ path.write_bytes(b"v3 is longer still")
+ _bump_mtime(path)
+ content.size_bytes = path.stat().st_size
+ content.mtime_ns = path.stat().st_mtime_ns
+ session.commit()
+ apply_reference_observations(session, observations)
+ session.commit()
+
+ contents = list(session.scalars(select(AssetContent)))
+ assert len(contents) == 1
+ assert contents[0].is_missing is False
+ assert contents[0].size_bytes == len(b"v3 is longer still")
diff --git a/tests-unit/assets_test/services/test_manager_isolated.py b/tests-unit/assets_test/services/test_manager_isolated.py
index 2b73ba1bb..e195c1184 100644
--- a/tests-unit/assets_test/services/test_manager_isolated.py
+++ b/tests-unit/assets_test/services/test_manager_isolated.py
@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, Mock, call
import folder_paths
import pytest
from sqlalchemy import create_engine, select
-from sqlalchemy.orm import Session, Session as SASession
+from sqlalchemy.orm import Session, Session as SASession, sessionmaker
from sqlalchemy.pool import StaticPool
from app.assets import lifecycle
@@ -76,6 +76,7 @@ def threaded_create_session(
monkeypatch.setattr(seeder_module, "create_session", _create_session)
monkeypatch.setattr(scanner, "create_session", _create_session)
monkeypatch.setattr("app.assets.services.ingest.create_session", _create_session)
+ monkeypatch.setattr("app.database.db.WriteSession", sessionmaker(bind=engine))
yield _create_session
engine.dispose()
diff --git a/tests-unit/assets_test/services/test_recovery_gate.py b/tests-unit/assets_test/services/test_recovery_gate.py
index eeca7e0b0..885d0100b 100644
--- a/tests-unit/assets_test/services/test_recovery_gate.py
+++ b/tests-unit/assets_test/services/test_recovery_gate.py
@@ -111,7 +111,7 @@ def test_off_mode_no_recovery(session, temp_dir: Path):
with (
patch("app.assets.scanner.mode.hashing_enabled", return_value=False),
- patch("app.assets.scanner_changes.snapshot_hash") as hash_mock,
+ patch("app.assets.scanner.snapshot_hash") as hash_mock,
):
created = seed_asset_specs(session, [_spec(path)])
session.commit()
@@ -128,7 +128,7 @@ def test_unstable_hash_requeues(session, temp_dir: Path):
with (
patch("app.assets.scanner.mode.hashing_enabled", return_value=True),
- patch("app.assets.scanner_changes.snapshot_hash", return_value=None),
+ patch("app.assets.scanner.snapshot_hash", return_value=None),
):
created = seed_asset_specs(session, [_spec(path)])
session.commit()
@@ -156,7 +156,7 @@ def test_recovery_skips_a_path_a_live_row_already_occupies(session, temp_dir: Pa
live_b_id = live_b.id
result = recover_missing_content(
- session, str(path), stat_result, hashing_is_enabled=True
+ session, str(path), snapshot_hash(str(path)), hashing_is_enabled=True
)
session.commit()
@@ -166,3 +166,24 @@ def test_recovery_skips_a_path_a_live_row_already_occupies(session, temp_dir: Pa
)
assert session.get(AssetContent, missing_a.id).is_missing is True
assert session.get(AssetContent, live_b_id).is_missing is False
+
+
+def test_seeded_row_takes_the_stat_its_hash_was_verified_against(session, temp_dir: Path):
+ path = temp_dir / "rewritten.bin"
+ path.write_bytes(b"old")
+
+ def rewrite_then_hash(candidate_path: str):
+ path.write_bytes(b"rewritten between the stat and the hash")
+ return snapshot_hash(candidate_path)
+
+ with (
+ patch("app.assets.scanner.mode.hashing_enabled", return_value=True),
+ patch("app.assets.scanner.snapshot_hash", side_effect=rewrite_then_hash),
+ ):
+ created = seed_asset_specs(session, [_spec(path)])
+ session.commit()
+
+ assert created == 1
+ content = session.scalars(select(AssetContent)).one()
+ assert content.size_bytes == path.stat().st_size
+ assert content.mtime_ns == path.stat().st_mtime_ns
diff --git a/tests-unit/assets_test/services/test_scan_lifecycle.py b/tests-unit/assets_test/services/test_scan_lifecycle.py
index e5a1d2801..134dc2118 100644
--- a/tests-unit/assets_test/services/test_scan_lifecycle.py
+++ b/tests-unit/assets_test/services/test_scan_lifecycle.py
@@ -4,7 +4,12 @@ from unittest.mock import patch
from sqlalchemy import select
from app.assets.database.models import AssetContent
-from app.assets.scanner import build_asset_specs, seed_asset_specs, sync_prefixes_with_filesystem
+from app.assets.scanner import (
+ apply_reference_observations,
+ build_asset_specs,
+ observe_references_on_filesystem,
+ seed_asset_specs,
+)
def _scan(session, root: Path) -> int:
@@ -26,7 +31,9 @@ def test_e2e_scan_seed_detect_prune(session, temp_dir: Path):
edited.write_bytes(b"replacement")
(root / "partial.part").write_bytes(b"partial")
with patch("app.assets.scanner.mode.hashing_enabled", return_value=False):
- sync_prefixes_with_filesystem(session, [str(root)])
+ apply_reference_observations(
+ session, observe_references_on_filesystem(session, [str(root)])[0]
+ )
_scan(session, root)
session.commit()
contents = list(session.scalars(select(AssetContent)))
diff --git a/tests-unit/assets_test/services/test_scanner_b.py b/tests-unit/assets_test/services/test_scanner_b.py
index 93c155870..c298381fc 100644
--- a/tests-unit/assets_test/services/test_scanner_b.py
+++ b/tests-unit/assets_test/services/test_scanner_b.py
@@ -14,7 +14,8 @@ from app.assets.scanner import (
enrich_asset,
mark_contents_missing_outside_prefixes,
seed_asset_specs,
- sync_prefixes_with_filesystem,
+ apply_reference_observations,
+ observe_references_on_filesystem,
)
from app.assets.services.snapshot_hash import snapshot_hash
@@ -180,7 +181,9 @@ def test_unhashed_missing_content_gets_tagged(session, temp_dir: Path):
session.add(record)
session.commit()
- sync_prefixes_with_filesystem(session, prefixes=[str(temp_dir)])
+ apply_reference_observations(
+ session, observe_references_on_filesystem(session, [str(temp_dir)])[0]
+ )
session.commit()
session.expire_all()
diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py
index f6f8f6ba4..9dc72e82e 100644
--- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py
+++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py
@@ -64,7 +64,7 @@ def test_seed_persists_remaining_specs_when_path_vanishes_during_recovery_hash(
raise OSError("file vanished during recovery")
return snapshot_hash(path)
- monkeypatch.setattr("app.assets.scanner_changes.snapshot_hash", _hash_or_raise)
+ monkeypatch.setattr("app.assets.scanner.snapshot_hash", _hash_or_raise)
with patch("app.assets.scanner.mode.hashing_enabled", return_value=True):
created = seed_asset_specs(session, specs)
@@ -85,7 +85,7 @@ def _delete_during_recovery(monkeypatch: pytest.MonkeyPatch, path: Path) -> None
raise OSError("file vanished during recovery")
return snapshot_hash(candidate_path)
- monkeypatch.setattr("app.assets.scanner_changes.snapshot_hash", _hash_or_raise)
+ monkeypatch.setattr("app.assets.scanner.snapshot_hash", _hash_or_raise)
@pytest.mark.parametrize(
diff --git a/tests-unit/assets_test/services/test_stored_hash_prefix.py b/tests-unit/assets_test/services/test_stored_hash_prefix.py
index c28d6fb34..277073774 100644
--- a/tests-unit/assets_test/services/test_stored_hash_prefix.py
+++ b/tests-unit/assets_test/services/test_stored_hash_prefix.py
@@ -183,9 +183,8 @@ def test_recovery_matches_prefixed_stored_hash(session, temp_dir):
path.unlink()
path.write_bytes(original_bytes)
- stat = os.stat(str(path))
result = recover_missing_content(
- session, str(path), stat, hashing_is_enabled=True
+ session, str(path), snapshot_hash(str(path)), hashing_is_enabled=True
)
assert result == "recovered"
diff --git a/tests-unit/assets_test/test_intended_behaviour.py b/tests-unit/assets_test/test_intended_behaviour.py
index 580637c71..19a00e4c1 100644
--- a/tests-unit/assets_test/test_intended_behaviour.py
+++ b/tests-unit/assets_test/test_intended_behaviour.py
@@ -31,7 +31,8 @@ from app.assets.lifecycle import wipe_temp_db_rows
from app.assets.scanner import (
build_asset_specs,
seed_asset_specs,
- sync_prefixes_with_filesystem,
+ apply_reference_observations,
+ observe_references_on_filesystem,
)
from app.assets.scanner_admission import _should_skip_extension
from app.assets.scanner_changes import (
@@ -105,9 +106,8 @@ def _seed_content_row(session, path: Path, hash_value: str | None = None):
def _scan_pass(session, root: Path) -> int:
- survivors = sync_prefixes_with_filesystem(
- session, [str(root)], collect_existing_paths=True
- )
+ observations, survivors = observe_references_on_filesystem(session, [str(root)])
+ apply_reference_observations(session, observations)
specs, _tag_pool, _skipped = build_asset_specs(
list_files_recursively(str(root)), survivors or set()
)
diff --git a/tests-unit/assets_test/test_scanner_ingest_event_log.py b/tests-unit/assets_test/test_scanner_ingest_event_log.py
index f81ebee0f..f6eb62803 100644
--- a/tests-unit/assets_test/test_scanner_ingest_event_log.py
+++ b/tests-unit/assets_test/test_scanner_ingest_event_log.py
@@ -119,7 +119,7 @@ def test_permission_error_in_reference_sync_increments_scan_counter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
secret_path = "/private/assets/unreadable.safetensors"
- content = SimpleNamespace(id="content", path=secret_path)
+ content = SimpleNamespace(id="content", path=secret_path, size_bytes=1, mtime_ns=1)
progress = _ScanState()
def deny_stat(*_args, **_kwargs):
@@ -128,7 +128,7 @@ def test_permission_error_in_reference_sync_increments_scan_counter(
monkeypatch.setattr(scanner, "os", SimpleNamespace(stat=deny_stat, path=scanner.os.path))
monkeypatch.setattr(scanner, "live_contents_under_prefixes", lambda _session, _prefixes: [content])
- scanner.sync_prefixes_with_filesystem(Mock(), ["/private/assets"], progress=progress)
+ scanner.observe_references_on_filesystem(Mock(), ["/private/assets"], progress=progress)
assert progress.permission_denied == 1