Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

fix(assets): write scan inserts in short transactions so uploads and output saves aren't locked out + use WAL synchronous=NORMAL (#16748)

Simon Pinfoldae913fdcc4bf
diff --git a/app/assets/seeder.py b/app/assets/seeder.py
index 29b444f7d..7213ecc5c 100644
--- a/app/assets/seeder.py
+++ b/app/assets/seeder.py
@@ -161,6 +161,16 @@ def _snapshot_progress(state: _ScanState) -> Progress:
)
+# The fast scan inserts INSERT_BATCH_SIZE files per write transaction and pauses after
+# each, so an upload or an output being registered gets the lock.
+INSERT_BATCH_SIZE = 200
+# A write waiting on the lock retries after sleeps of up to 100 ms; the pause outlasts one,
+# plus Windows' 15.6 ms timer tick.
+_BUSY_HANDLER_MAX_SLEEP_SECONDS = 0.1
+_TIMER_TICK_ALLOWANCE_SECONDS = 0.02
+INSERT_PAUSE_SECONDS = _BUSY_HANDLER_MAX_SLEEP_SECONDS + _TIMER_TICK_ALLOWANCE_SECONDS
+
+
class _AssetSeeder:
"""Background asset scanning manager.
@@ -985,11 +995,10 @@ class _AssetSeeder:
if self._check_pause_and_cancel(_ScanStage.FAST_SCAN):
return total_created, skipped_existing, total_paths
- batch_size = 500
last_progress_time = time.perf_counter()
progress_interval = 1.0
- for i in range(0, len(specs), batch_size):
+ for i in range(0, len(specs), INSERT_BATCH_SIZE):
if self._check_pause_and_cancel(_ScanStage.FAST_SCAN):
logging.info(
"Fast scan cancelled after %d/%d files (created=%d)",
@@ -999,7 +1008,7 @@ class _AssetSeeder:
)
return total_created, skipped_existing, total_paths
- batch = specs[i : i + batch_size]
+ batch = specs[i : i + INSERT_BATCH_SIZE]
batch_tags = {t for spec in batch for t in spec["tags"]}
created = 0
try:
@@ -1043,6 +1052,9 @@ class _AssetSeeder:
)
last_progress_time = now
+ if scanned < len(specs):
+ time.sleep(INSERT_PAUSE_SECONDS)
+
self._update_progress(scanned=len(specs), created=total_created)
tick_watch_list(scan_state)
logging.info(
diff --git a/app/database/db.py b/app/database/db.py
index 154953913..19e9f4b6a 100644
--- a/app/database/db.py
+++ b/app/database/db.py
@@ -281,6 +281,15 @@ def _init_file_db(db_url):
raise
+# NORMAL: commits skip the fsync that held the write lock. A power loss or OS crash can
+# roll back recent commits but cannot corrupt the database. "FULL" is SQLite's default.
+WAL_SYNCHRONOUS = "NORMAL"
+
+
+def _set_wal_synchronous(dbapi_connection, connection_record=None):
+ dbapi_connection.execute(f"PRAGMA synchronous={WAL_SYNCHRONOUS}")
+
+
_DESTRUCTIVE_REVISION = "0007_record_content_split"
@@ -328,6 +337,11 @@ def _migrate_and_bind(db_url, db_path, db_exists):
else:
if journal_mode.lower() != "wal":
logging.warning("SQLite WAL mode unavailable; continuing with %s journal mode.", journal_mode)
+ else:
+ # Only in WAL mode: with a rollback journal, NORMAL risks corruption on power loss.
+ event.listen(engine, "connect", _set_wal_synchronous)
+ event.listen(write_engine, "connect", _set_wal_synchronous)
+ _set_wal_synchronous(conn.connection.dbapi_connection) # opened before the hooks
context = MigrationContext.configure(conn)
current_rev = context.get_current_revision()
diff --git a/tests-unit/app_test/test_db_write_txn.py b/tests-unit/app_test/test_db_write_txn.py
index 179768e7b..48fc99431 100644
--- a/tests-unit/app_test/test_db_write_txn.py
+++ b/tests-unit/app_test/test_db_write_txn.py
@@ -1,25 +1,13 @@
import sqlite3
+from contextlib import closing
import pytest
-from sqlalchemy import text
+from sqlalchemy import event, text
+from sqlalchemy.engine import Engine
from app.database import db as db_module
-def file_db(tmp_path, monkeypatch):
- db_path = str(tmp_path / "comfyui.db")
- monkeypatch.setattr(db_module.args, "database_url", f"sqlite:///{db_path}")
- monkeypatch.setattr(db_module, "Session", None)
- monkeypatch.setattr(db_module, "WriteSession", None)
- monkeypatch.setattr(db_module, "_db_lock", None)
- db_module._init_file_db(db_module.args.database_url)
- yield db_path
- db_module.Session.kw["bind"].dispose()
- db_module.WriteSession.kw["bind"].dispose()
- db_module._db_lock.release(force=True)
-
-
def _other_writer_can_begin(db_path):
other = sqlite3.connect(db_path, timeout=0, isolation_level=None)
try:
@@ -37,6 +25,55 @@ def test_file_db_uses_wal(file_db):
assert session.execute(text("PRAGMA journal_mode")).scalar_one() == "wal"
+def extra_wal_synchronous(monkeypatch):
+ # Not SQLite's FULL or WAL default on any build, so only the hooks can produce it.
+ monkeypatch.setattr(db_module, "WAL_SYNCHRONOUS", "EXTRA")
+
+
[email protected]("factory", ["Session", "WriteSession"])
+def test_wal_connections_get_the_wal_synchronous_setting(extra_wal_synchronous, file_db, factory):
+ engine = getattr(db_module, factory).kw["bind"]
+ # The read engine's first may be the connection opened before WAL was on; holding it
+ # makes the second a fresh one through the engine's connect hook.
+ with closing(engine.raw_connection()) as first, closing(engine.raw_connection()) as second:
+ values = [c.cursor().execute("PRAGMA synchronous").fetchone()[0] for c in (first, second)]
+ assert values == [3, 3] # EXTRA
+
+
+def refused_wal():
+ """Stand in for a filesystem that refuses WAL: the request leaves the rollback journal on."""
+
+ def refuse(conn, cursor, statement, parameters, context, executemany):
+ if statement == "PRAGMA journal_mode=WAL":
+ statement = "PRAGMA journal_mode=DELETE"
+ return statement, parameters
+
+ event.listen(Engine, "before_cursor_execute", refuse, retval=True)
+ yield
+ event.remove(Engine, "before_cursor_execute", refuse)
+
+
+def refused_wal_db(extra_wal_synchronous, refused_wal, file_db):
+ """The file database initialised while WAL is refused (fixtures resolve in this order)."""
+ return file_db
+
+
[email protected]("factory", ["Session", "WriteSession"])
+def test_synchronous_is_left_alone_when_wal_is_refused(refused_wal_db, factory):
+ with closing(sqlite3.connect(":memory:")) as plain:
+ build_default = plain.execute("PRAGMA synchronous").fetchone()[0]
+ engine = getattr(db_module, factory).kw["bind"]
+ # Two, so the second is a fresh connection through the engine's connect hooks.
+ with closing(engine.raw_connection()) as first, closing(engine.raw_connection()) as second:
+ for conn in (first, second):
+ cursor = conn.cursor()
+ assert cursor.execute("PRAGMA journal_mode").fetchone()[0] == "delete"
+ assert cursor.execute("PRAGMA synchronous").fetchone()[0] == build_default
+
+
def test_write_session_takes_the_write_lock_before_its_first_write(file_db):
with db_module.create_write_session() as session:
session.execute(text("SELECT 1")).scalar_one()
diff --git a/tests-unit/conftest.py b/tests-unit/conftest.py
new file mode 100644
index 000000000..ced3470dd
--- /dev/null
+++ b/tests-unit/conftest.py
@@ -0,0 +1,18 @@
+import pytest
+
+
+def file_db(tmp_path, monkeypatch):
+ """The real file database: WAL mode, both engines and their hooks, in ``tmp_path``."""
+ from app.database import db as db_module
+
+ db_path = str(tmp_path / "comfyui.db")
+ monkeypatch.setattr(db_module.args, "database_url", f"sqlite:///{db_path}")
+ monkeypatch.setattr(db_module, "Session", None)
+ monkeypatch.setattr(db_module, "WriteSession", None)
+ monkeypatch.setattr(db_module, "_db_lock", None)
+ db_module._init_file_db(db_module.args.database_url)
+ yield db_path
+ db_module.Session.kw["bind"].dispose()
+ db_module.WriteSession.kw["bind"].dispose()
+ db_module._db_lock.release(force=True)
diff --git a/tests-unit/seeder_test/test_insert_batches.py b/tests-unit/seeder_test/test_insert_batches.py
new file mode 100644
index 000000000..ecee7c5cd
--- /dev/null
+++ b/tests-unit/seeder_test/test_insert_batches.py
@@ -0,0 +1,146 @@
+"""The fast scan inserts in small batches with a pause after each, so a foreground write
+(an upload, an output being registered) gets the write lock while it runs."""
+
+import os
+import threading
+import time
+from types import SimpleNamespace
+
+import folder_paths
+import pytest
+
+from app.assets import mode
+from app.assets import scanner as scanner_module
+from app.assets import seeder as seeder_module
+from app.assets.seeder import ScanPhase, State, _AssetSeeder, _ScanState
+from app.assets.services import ingest
+
+
+def scan_seeder(monkeypatch: pytest.MonkeyPatch) -> _AssetSeeder:
+ instance = _AssetSeeder()
+ instance._state = State.RUNNING
+ instance._scan_state = _ScanState()
+ instance._roots = ("input",)
+ instance._phase = ScanPhase.FAST
+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)
+ monkeypatch.setattr(instance, "_log_scan_config", lambda roots: None)
+ return instance
+
+
+def _run_fast_phase(scan_seeder, monkeypatch, count, fail_batch=None):
+ """Run the fast phase over ``count`` specs with insert_asset_specs stubbed out;
+ returns the result, each batch's length, and the pauses taken."""
+ batches: list[int] = []
+ sleeps: list[float] = []
+
+ def insert(batch, _tags, _progress):
+ batches.append(len(batch))
+ if len(batches) == fail_batch:
+ raise RuntimeError("database is locked")
+ return len(batch), None
+
+ specs = [{"tags": []} for _ in range(count)]
+ monkeypatch.setattr(seeder_module.time, "sleep", sleeps.append)
+ monkeypatch.setattr(seeder_module, "insert_asset_specs", insert)
+ monkeypatch.setattr(seeder_module, "sync_root_safely", lambda *_args, **_kwargs: set())
+ monkeypatch.setattr(seeder_module, "collect_paths_for_roots", lambda *_args, **_kwargs: [])
+ monkeypatch.setattr(seeder_module, "build_asset_specs", lambda *_args, **_kwargs: (specs, set(), 0))
+ monkeypatch.setattr(seeder_module, "tick_watch_list", lambda _progress=None: None)
+ result = scan_seeder._run_fast_phase(("input",))
+ return result, batches, sleeps
+
+
+def test_inserts_go_in_batches_with_a_pause_after_all_but_the_last(scan_seeder, monkeypatch):
+ (created, _, _), batches, sleeps = _run_fast_phase(scan_seeder, monkeypatch, 450)
+
+ assert created == 450
+ assert batches == [seeder_module.INSERT_BATCH_SIZE] * 2 + [50]
+ assert sleeps == [seeder_module.INSERT_PAUSE_SECONDS] * 2
+
+
+def test_a_failed_batch_is_still_followed_by_a_pause(scan_seeder, monkeypatch):
+ # A batch that lost the lock to a foreground write must still let that write go first.
+ (created, _, _), batches, sleeps = _run_fast_phase(scan_seeder, monkeypatch, 450, fail_batch=2)
+
+ assert len(batches) == 3
+ assert sleeps == [seeder_module.INSERT_PAUSE_SECONDS] * 2
+ assert created == seeder_module.INSERT_BATCH_SIZE + 50
+ assert scan_seeder._errors == [
+ f"Batch insert encountered an error at offset {seeder_module.INSERT_BATCH_SIZE} after creating 0: database is locked"
+ ]
+
+
+def test_the_pause_outlasts_the_busy_handlers_longest_poll():
+ # SQLite's default busy handler sleeps at most 100 ms between retries, and Windows'
+ # default timer tick (15.6 ms) can stretch that sleep; a shorter pause can fall
+ # between two retries, and a waiting write never sees the lock free.
+ assert seeder_module.INSERT_PAUSE_SECONDS > 0.100 + 0.0156
+
+
+# --- real threads, real WAL database ---------------------------------------------
+
+# Per scanned file inside the write transaction: 500 files in one transaction, as before
+# this change, hold the lock for over 5 s. The test's small batches keep each hold under
+# ~1.5 s even on a slow CI runner.
+_SLOW_RECORD_SECONDS = 0.01
+_TEST_BATCH_SIZE = 20
+# A foreground write that gets in at a pause waits about one batch plus a pause.
+_FOREGROUND_WAIT_LIMIT_SECONDS = 4.0
+
+
+def hashing_off():
+ mode.init(SimpleNamespace(enable_asset_hashing=False))
+ yield
+ mode.init(None)
+
+
+def test_output_registration_gets_in_while_the_scan_inserts(hashing_off, file_db, tmp_path, monkeypatch):
+ library = tmp_path / "input"
+ library.mkdir()
+ for i in range(600):
+ (library / f"f{i}.png").write_bytes(b"x" * (i + 1))
+ output = tmp_path / "output"
+ output.mkdir()
+ monkeypatch.setattr(folder_paths, "get_input_directory", lambda: str(library))
+ monkeypatch.setattr(folder_paths, "get_output_directory", lambda: str(output))
+ old = time.time() - 3600
+ for path in library.iterdir():
+ os.utime(path, (old, old))
+
+ scanning = threading.Event()
+ create_record = scanner_module.create_record
+ records = []
+
+ def slow_create_record(*args, **kwargs):
+ records.append(1)
+ # Partway into the second batch, so the writes contend with batches in progress.
+ if len(records) == _TEST_BATCH_SIZE + 10:
+ scanning.set()
+ time.sleep(_SLOW_RECORD_SECONDS)
+ return create_record(*args, **kwargs)
+
+ monkeypatch.setattr(scanner_module, "create_record", slow_create_record)
+ monkeypatch.setattr(seeder_module, "INSERT_BATCH_SIZE", _TEST_BATCH_SIZE)
+ seeder = _AssetSeeder()
+ assert seeder.start(roots=("input",), phase=ScanPhase.FAST)
+ try:
+ assert scanning.wait(30)
+ waits = []
+ for i in range(3):
+ path = output / f"out{i}.png"
+ path.write_bytes(b"output")
+ started = time.perf_counter()
+ registered = ingest.register_executed_output(str(path), job_id="job")
+ waits.append(time.perf_counter() - started)
+ assert registered is not None
+ status = seeder.get_status()
+ finally:
+ seeder.cancel()
+ stopped = seeder.wait(timeout=60)
+
+ assert stopped
+ assert status.state == State.RUNNING, "the scan ended before the foreground writes; nothing was contended"
+ assert max(waits) < _FOREGROUND_WAIT_LIMIT_SECONDS, waits
+ assert status.errors == [] # the scan's own batches got the lock too