The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
fix(assets): write scan inserts in short transactions so uploads and output saves aren't locked out + use WAL synchronous=NORMAL (#16748)
Simon Pinfoldae913fdcc4bf
diff --git a/app/assets/seeder.py b/app/assets/seeder.pyindex 29b444f7d..7213ecc5c 100644--- a/app/assets/seeder.py+++ b/app/assets/seeder.py@@ -161,6 +161,16 @@ def _snapshot_progress(state: _ScanState) -> Progress:)+# The fast scan inserts INSERT_BATCH_SIZE files per write transaction and pauses after+# each, so an upload or an output being registered gets the lock.+INSERT_BATCH_SIZE = 200+# A write waiting on the lock retries after sleeps of up to 100 ms; the pause outlasts one,+# plus Windows' 15.6 ms timer tick.+_BUSY_HANDLER_MAX_SLEEP_SECONDS = 0.1+_TIMER_TICK_ALLOWANCE_SECONDS = 0.02+INSERT_PAUSE_SECONDS = _BUSY_HANDLER_MAX_SLEEP_SECONDS + _TIMER_TICK_ALLOWANCE_SECONDS++class _AssetSeeder:"""Background asset scanning manager.@@ -985,11 +995,10 @@ class _AssetSeeder:if self._check_pause_and_cancel(_ScanStage.FAST_SCAN):return total_created, skipped_existing, total_paths- batch_size = 500last_progress_time = time.perf_counter()progress_interval = 1.0- for i in range(0, len(specs), batch_size):+ for i in range(0, len(specs), INSERT_BATCH_SIZE):if self._check_pause_and_cancel(_ScanStage.FAST_SCAN):logging.info("Fast scan cancelled after %d/%d files (created=%d)",@@ -999,7 +1008,7 @@ class _AssetSeeder:)return total_created, skipped_existing, total_paths- batch = specs[i : i + batch_size]+ batch = specs[i : i + INSERT_BATCH_SIZE]batch_tags = {t for spec in batch for t in spec["tags"]}created = 0try:@@ -1043,6 +1052,9 @@ class _AssetSeeder:)last_progress_time = now+ if scanned < len(specs):+ time.sleep(INSERT_PAUSE_SECONDS)+self._update_progress(scanned=len(specs), created=total_created)tick_watch_list(scan_state)logging.info(diff --git a/app/database/db.py b/app/database/db.pyindex 154953913..19e9f4b6a 100644--- a/app/database/db.py+++ b/app/database/db.py@@ -281,6 +281,15 @@ def _init_file_db(db_url):raise+# NORMAL: commits skip the fsync that held the write lock. A power loss or OS crash can+# roll back recent commits but cannot corrupt the database. "FULL" is SQLite's default.+WAL_SYNCHRONOUS = "NORMAL"+++def _set_wal_synchronous(dbapi_connection, connection_record=None):+ dbapi_connection.execute(f"PRAGMA synchronous={WAL_SYNCHRONOUS}")++_DESTRUCTIVE_REVISION = "0007_record_content_split"@@ -328,6 +337,11 @@ def _migrate_and_bind(db_url, db_path, db_exists):else:if journal_mode.lower() != "wal":logging.warning("SQLite WAL mode unavailable; continuing with %s journal mode.", journal_mode)+ else:+ # Only in WAL mode: with a rollback journal, NORMAL risks corruption on power loss.+ event.listen(engine, "connect", _set_wal_synchronous)+ event.listen(write_engine, "connect", _set_wal_synchronous)+ _set_wal_synchronous(conn.connection.dbapi_connection) # opened before the hookscontext = MigrationContext.configure(conn)current_rev = context.get_current_revision()diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.pyindex 179768e7b..48fc99431 100644--- a/tests-unit/app_test/test_db_write_txn.py+++ b/tests-unit/app_test/test_db_write_txn.py@@ -1,25 +1,13 @@import sqlite3+from contextlib import closingimport pytest-from sqlalchemy import text+from sqlalchemy import event, text+from sqlalchemy.engine import Enginefrom app.database import db as db_module-def file_db(tmp_path, monkeypatch):- db_path = str(tmp_path / "comfyui.db")- monkeypatch.setattr(db_module.args, "database_url", f"sqlite:///{db_path}")- monkeypatch.setattr(db_module, "Session", None)- monkeypatch.setattr(db_module, "WriteSession", None)- monkeypatch.setattr(db_module, "_db_lock", None)- db_module._init_file_db(db_module.args.database_url)- yield db_path- db_module.Session.kw["bind"].dispose()- db_module.WriteSession.kw["bind"].dispose()- db_module._db_lock.release(force=True)--def _other_writer_can_begin(db_path):other = sqlite3.connect(db_path, timeout=0, isolation_level=None)try:@@ -37,6 +25,55 @@ def test_file_db_uses_wal(file_db):assert session.execute(text("PRAGMA journal_mode")).scalar_one() == "wal"+def extra_wal_synchronous(monkeypatch):+ # Not SQLite's FULL or WAL default on any build, so only the hooks can produce it.+ monkeypatch.setattr(db_module, "WAL_SYNCHRONOUS", "EXTRA")++[email protected]("factory", ["Session", "WriteSession"])+def test_wal_connections_get_the_wal_synchronous_setting(extra_wal_synchronous, file_db, factory):+ engine = getattr(db_module, factory).kw["bind"]+ # The read engine's first may be the connection opened before WAL was on; holding it+ # makes the second a fresh one through the engine's connect hook.+ with closing(engine.raw_connection()) as first, closing(engine.raw_connection()) as second:+ values = [c.cursor().execute("PRAGMA synchronous").fetchone()[0] for c in (first, second)]+ assert values == [3, 3] # EXTRA+++def refused_wal():+ """Stand in for a filesystem that refuses WAL: the request leaves the rollback journal on."""++ def refuse(conn, cursor, statement, parameters, context, executemany):+ if statement == "PRAGMA journal_mode=WAL":+ statement = "PRAGMA journal_mode=DELETE"+ return statement, parameters++ event.listen(Engine, "before_cursor_execute", refuse, retval=True)+ yield+ event.remove(Engine, "before_cursor_execute", refuse)+++def refused_wal_db(extra_wal_synchronous, refused_wal, file_db):+ """The file database initialised while WAL is refused (fixtures resolve in this order)."""+ return file_db++[email protected]("factory", ["Session", "WriteSession"])+def test_synchronous_is_left_alone_when_wal_is_refused(refused_wal_db, factory):+ with closing(sqlite3.connect(":memory:")) as plain:+ build_default = plain.execute("PRAGMA synchronous").fetchone()[0]+ engine = getattr(db_module, factory).kw["bind"]+ # Two, so the second is a fresh connection through the engine's connect hooks.+ with closing(engine.raw_connection()) as first, closing(engine.raw_connection()) as second:+ for conn in (first, second):+ cursor = conn.cursor()+ assert cursor.execute("PRAGMA journal_mode").fetchone()[0] == "delete"+ assert cursor.execute("PRAGMA synchronous").fetchone()[0] == build_default++def test_write_session_takes_the_write_lock_before_its_first_write(file_db):with db_module.create_write_session() as session:session.execute(text("SELECT 1")).scalar_one()diff --git a/tests-unit/conftest.py b/tests-unit/conftest.pynew file mode 100644index 000000000..ced3470dd--- /dev/null+++ b/tests-unit/conftest.py@@ -0,0 +1,18 @@+import pytest+++def file_db(tmp_path, monkeypatch):+ """The real file database: WAL mode, both engines and their hooks, in ``tmp_path``."""+ from app.database import db as db_module++ db_path = str(tmp_path / "comfyui.db")+ monkeypatch.setattr(db_module.args, "database_url", f"sqlite:///{db_path}")+ monkeypatch.setattr(db_module, "Session", None)+ monkeypatch.setattr(db_module, "WriteSession", None)+ monkeypatch.setattr(db_module, "_db_lock", None)+ db_module._init_file_db(db_module.args.database_url)+ yield db_path+ db_module.Session.kw["bind"].dispose()+ db_module.WriteSession.kw["bind"].dispose()+ db_module._db_lock.release(force=True)diff --git a/tests-unit/seeder_test/test_insert_batches.py b/tests-unit/seeder_test/test_insert_batches.pynew file mode 100644index 000000000..ecee7c5cd--- /dev/null+++ b/tests-unit/seeder_test/test_insert_batches.py@@ -0,0 +1,146 @@+"""The fast scan inserts in small batches with a pause after each, so a foreground write+(an upload, an output being registered) gets the write lock while it runs."""++import os+import threading+import time+from types import SimpleNamespace++import folder_paths+import pytest++from app.assets import mode+from app.assets import scanner as scanner_module+from app.assets import seeder as seeder_module+from app.assets.seeder import ScanPhase, State, _AssetSeeder, _ScanState+from app.assets.services import ingest+++def scan_seeder(monkeypatch: pytest.MonkeyPatch) -> _AssetSeeder:+ instance = _AssetSeeder()+ instance._state = State.RUNNING+ instance._scan_state = _ScanState()+ instance._roots = ("input",)+ instance._phase = ScanPhase.FAST+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)+ monkeypatch.setattr(instance, "_log_scan_config", lambda roots: None)+ return instance+++def _run_fast_phase(scan_seeder, monkeypatch, count, fail_batch=None):+ """Run the fast phase over ``count`` specs with insert_asset_specs stubbed out;+ returns the result, each batch's length, and the pauses taken."""+ batches: list[int] = []+ sleeps: list[float] = []++ def insert(batch, _tags, _progress):+ batches.append(len(batch))+ if len(batches) == fail_batch:+ raise RuntimeError("database is locked")+ return len(batch), None++ specs = [{"tags": []} for _ in range(count)]+ monkeypatch.setattr(seeder_module.time, "sleep", sleeps.append)+ monkeypatch.setattr(seeder_module, "insert_asset_specs", insert)+ monkeypatch.setattr(seeder_module, "sync_root_safely", lambda *_args, **_kwargs: set())+ monkeypatch.setattr(seeder_module, "collect_paths_for_roots", lambda *_args, **_kwargs: [])+ monkeypatch.setattr(seeder_module, "build_asset_specs", lambda *_args, **_kwargs: (specs, set(), 0))+ monkeypatch.setattr(seeder_module, "tick_watch_list", lambda _progress=None: None)+ result = scan_seeder._run_fast_phase(("input",))+ return result, batches, sleeps+++def test_inserts_go_in_batches_with_a_pause_after_all_but_the_last(scan_seeder, monkeypatch):+ (created, _, _), batches, sleeps = _run_fast_phase(scan_seeder, monkeypatch, 450)++ assert created == 450+ assert batches == [seeder_module.INSERT_BATCH_SIZE] * 2 + [50]+ assert sleeps == [seeder_module.INSERT_PAUSE_SECONDS] * 2+++def test_a_failed_batch_is_still_followed_by_a_pause(scan_seeder, monkeypatch):+ # A batch that lost the lock to a foreground write must still let that write go first.+ (created, _, _), batches, sleeps = _run_fast_phase(scan_seeder, monkeypatch, 450, fail_batch=2)++ assert len(batches) == 3+ assert sleeps == [seeder_module.INSERT_PAUSE_SECONDS] * 2+ assert created == seeder_module.INSERT_BATCH_SIZE + 50+ assert scan_seeder._errors == [+ f"Batch insert encountered an error at offset {seeder_module.INSERT_BATCH_SIZE} after creating 0: database is locked"+ ]+++def test_the_pause_outlasts_the_busy_handlers_longest_poll():+ # SQLite's default busy handler sleeps at most 100 ms between retries, and Windows'+ # default timer tick (15.6 ms) can stretch that sleep; a shorter pause can fall+ # between two retries, and a waiting write never sees the lock free.+ assert seeder_module.INSERT_PAUSE_SECONDS > 0.100 + 0.0156+++# --- real threads, real WAL database ---------------------------------------------++# Per scanned file inside the write transaction: 500 files in one transaction, as before+# this change, hold the lock for over 5 s. The test's small batches keep each hold under+# ~1.5 s even on a slow CI runner.+_SLOW_RECORD_SECONDS = 0.01+_TEST_BATCH_SIZE = 20+# A foreground write that gets in at a pause waits about one batch plus a pause.+_FOREGROUND_WAIT_LIMIT_SECONDS = 4.0+++def hashing_off():+ mode.init(SimpleNamespace(enable_asset_hashing=False))+ yield+ mode.init(None)+++def test_output_registration_gets_in_while_the_scan_inserts(hashing_off, file_db, tmp_path, monkeypatch):+ library = tmp_path / "input"+ library.mkdir()+ for i in range(600):+ (library / f"f{i}.png").write_bytes(b"x" * (i + 1))+ output = tmp_path / "output"+ output.mkdir()+ monkeypatch.setattr(folder_paths, "get_input_directory", lambda: str(library))+ monkeypatch.setattr(folder_paths, "get_output_directory", lambda: str(output))+ old = time.time() - 3600+ for path in library.iterdir():+ os.utime(path, (old, old))++ scanning = threading.Event()+ create_record = scanner_module.create_record+ records = []++ def slow_create_record(*args, **kwargs):+ records.append(1)+ # Partway into the second batch, so the writes contend with batches in progress.+ if len(records) == _TEST_BATCH_SIZE + 10:+ scanning.set()+ time.sleep(_SLOW_RECORD_SECONDS)+ return create_record(*args, **kwargs)++ monkeypatch.setattr(scanner_module, "create_record", slow_create_record)+ monkeypatch.setattr(seeder_module, "INSERT_BATCH_SIZE", _TEST_BATCH_SIZE)+ seeder = _AssetSeeder()+ assert seeder.start(roots=("input",), phase=ScanPhase.FAST)+ try:+ assert scanning.wait(30)+ waits = []+ for i in range(3):+ path = output / f"out{i}.png"+ path.write_bytes(b"output")+ started = time.perf_counter()+ registered = ingest.register_executed_output(str(path), job_id="job")+ waits.append(time.perf_counter() - started)+ assert registered is not None+ status = seeder.get_status()+ finally:+ seeder.cancel()+ stopped = seeder.wait(timeout=60)++ assert stopped+ assert status.state == State.RUNNING, "the scan ended before the foreground writes; nothing was contended"+ assert max(waits) < _FOREGROUND_WAIT_LIMIT_SECONDS, waits+ assert status.errors == [] # the scan's own batches got the lock too