Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

fix(assets): pause the startup scan during its folder walk and file stats (#16764)

Simon Pinfold5c460d8172fe
diff --git a/app/assets/scanner.py b/app/assets/scanner.py
index 193f515c4..b8cfeb048 100644
--- a/app/assets/scanner.py
+++ b/app/assets/scanner.py
@@ -410,21 +410,26 @@ def content_ids_outside_prefixes(session: Session, prefixes: list[str]) -> list[
def collect_paths_for_roots(
- roots: tuple[RootType, ...], progress: _ScanProgress | None = None
+ roots: tuple[RootType, ...],
+ progress: _ScanProgress | None = None,
+ should_stop: ShouldStop = _never_stop,
) -> list[str]:
"""Collect all file paths for the given roots.
``progress.dirs_listed`` counts the input and output walks only. Models are
listed through folder_paths.get_filename_list, which walks the model folders
on a cache miss and re-checks their mtimes on a hit; none of that is counted.
+
+ ``should_stop`` is checked before each input or output directory; once it returns
+ True the list is partial, so callers check it again before using the result.
"""
paths: list[str] = []
if "models" in roots:
paths.extend(collect_models_files())
if "input" in roots:
- paths.extend(list_files_recursively(folder_paths.get_input_directory(), progress))
+ paths.extend(list_files_recursively(folder_paths.get_input_directory(), progress, should_stop))
if "output" in roots:
- paths.extend(list_files_recursively(folder_paths.get_output_directory(), progress))
+ paths.extend(list_files_recursively(folder_paths.get_output_directory(), progress, should_stop))
return paths
@@ -589,6 +594,7 @@ def build_asset_specs(
existing_paths: set[str],
enable_metadata_extraction: bool = True,
progress: _ScanProgress | None = None,
+ should_stop: ShouldStop = _never_stop,
) -> tuple[list[SeedAssetSpec], set[str], int]:
"""Build asset specs from paths, returning (specs, tag_pool, skipped_count).
@@ -597,6 +603,8 @@ def build_asset_specs(
existing_paths: Set of paths that already exist in the database
enable_metadata_extraction: If True, extract tier 1 & 2 metadata
progress: Optional per-scan state for emit-once bookkeeping
+ should_stop: Checked before each path's stats and spec, so a large library
+ can pause here; once it returns True, no specs are returned
"""
specs: list[SeedAssetSpec] = []
tag_pool: set[str] = set()
@@ -604,6 +612,8 @@ def build_asset_specs(
candidates: list[tuple[str, os.stat_result]] = []
for p in paths:
+ if should_stop():
+ return [], set(), skipped
abs_p = os.path.abspath(p)
if _should_skip_extension(abs_p):
skipped += 1
@@ -634,9 +644,11 @@ def build_asset_specs(
continue
candidates.append((abs_p, stat_p))
- admitted_paths, _ = _two_stat_admit(candidates, progress)
+ admitted_paths, _ = _two_stat_admit(candidates, progress, should_stop)
candidate_stats = dict(candidates)
for abs_p in admitted_paths:
+ if should_stop():
+ return [], set(), skipped
yield_gil()
stat_p = candidate_stats[abs_p]
name, tags = get_name_and_tags_from_asset_path(abs_p)
diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.py
index b619ad570..8bbea9c2a 100644
--- a/app/assets/scanner_admission.py
+++ b/app/assets/scanner_admission.py
@@ -13,7 +13,7 @@ import mimetypes
import os
import time
from dataclasses import dataclass
-from typing import TYPE_CHECKING, Final, Protocol
+from typing import TYPE_CHECKING, Callable, Final, Protocol
from app.assets.event_log import emit, error_kind, error_type
from app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path
@@ -47,14 +47,22 @@ class _StatCounter(Protocol):
def _two_stat_admit(
- paths_with_stats: list[tuple[str, os.stat_result]], counter: _StatCounter | None = None
+ paths_with_stats: list[tuple[str, os.stat_result]],
+ counter: _StatCounter | None = None,
+ should_stop: Callable[[], bool] | None = None,
) -> tuple[list[str], list[str]]:
+ """``should_stop`` is called before each second stat; it may block, and True
+ returns ([], []) at once."""
if not paths_with_stats:
return [], []
+ # The caller took the first stat; give a file still being written (a model mid-download)
+ # time to change size or mtime before the second, so it goes on the watch list instead.
time.sleep(0.1)
admitted: list[str] = []
watched: list[str] = []
for path, first_stat in paths_with_stats:
+ if should_stop is not None and should_stop():
+ return [], []
if counter is not None:
counter.files_statted += 1
try:
diff --git a/app/assets/seeder.py b/app/assets/seeder.py
index 7213ecc5c..84dbced3f 100644
--- a/app/assets/seeder.py
+++ b/app/assets/seeder.py
@@ -944,7 +944,11 @@ class _AssetSeeder:
t_collect = time.perf_counter()
walk = list_output_for_rescan() if by_listing else None
- paths = walk.files if walk is not None else collect_paths_for_roots(roots, scan_state)
+ should_stop = lambda: self._check_pause_and_cancel(_ScanStage.FAST_SCAN)
+ paths = walk.files if walk is not None else collect_paths_for_roots(roots, scan_state, should_stop)
+ # A cancel during the walk leaves paths partial.
+ if should_stop():
+ return total_created, skipped_existing, 0
logging.debug(
"Fast scan: collect_paths took %.3fs (%d paths found)",
time.perf_counter() - t_collect,
@@ -983,6 +987,7 @@ class _AssetSeeder:
existing_paths,
enable_metadata_extraction=False,
progress=scan_state,
+ should_stop=should_stop,
)
logging.debug(
"Fast scan: build_asset_specs took %.3fs (%d specs, %d skipped)",
diff --git a/app/assets/services/file_utils.py b/app/assets/services/file_utils.py
index 9d76c74a1..fd29b30f7 100644
--- a/app/assets/services/file_utils.py
+++ b/app/assets/services/file_utils.py
@@ -1,5 +1,5 @@
import os
-from typing import NamedTuple, Protocol
+from typing import Callable, NamedTuple, Protocol
from app.assets.services.gil import yield_gil
@@ -53,11 +53,15 @@ def is_visible(name: str) -> bool:
def list_files_recursively(
- base_dir: str, counter: _DirListingCounter | None = None
+ base_dir: str,
+ counter: _DirListingCounter | None = None,
+ should_stop: Callable[[], bool] | None = None,
) -> list[str]:
"""Recursively list all files in a directory, following symlinks.
- ``counter.dirs_listed`` gains one per directory os.walk listed.
+ ``counter.dirs_listed`` gains one per directory os.walk listed. ``should_stop`` is
+ called before each directory; it may block, and True ends the walk early with a
+ partial list.
"""
out: list[str] = []
base_abs = os.path.abspath(base_dir)
@@ -68,6 +72,8 @@ def list_files_recursively(
for dirpath, subdirs, filenames in os.walk(
base_abs, topdown=True, followlinks=True
):
+ if should_stop is not None and should_stop():
+ break
if counter is not None:
counter.dirs_listed += 1
try:
diff --git a/tests-unit/assets_test/services/test_scan_walk_pause.py b/tests-unit/assets_test/services/test_scan_walk_pause.py
new file mode 100644
index 000000000..0d9f1526d
--- /dev/null
+++ b/tests-unit/assets_test/services/test_scan_walk_pause.py
@@ -0,0 +1,241 @@
+"""The fast scan's folder walk and its per-file stat and spec loops check the pause gate
+for every directory and file, so a prompt that starts during them stops the scan at once
+instead of after the whole library has been walked and stat'ed."""
+
+import os
+import threading
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import sqlalchemy as sa
+from sqlalchemy.orm import Session as SASession, sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.assets import scanner, seeder as seeder_module
+from app.assets.database.models import AssetContent, Base
+from app.assets.scanner_admission import _WATCH_LIST, _two_stat_admit
+from app.assets.services.file_utils import list_files_recursively
+
+FILES = 6
+
+
+class _Gate:
+ """A fake ShouldStop: call ``block_at`` parks until released, calls from ``stop_at`` on return True."""
+
+ def __init__(self, block_at: int | None = None, stop_at: int | None = None) -> None:
+ self.calls = 0
+ self.block_at = block_at
+ self.stop_at = stop_at
+ self.blocked = threading.Event()
+ self.release = threading.Event()
+
+ def __call__(self) -> bool:
+ self.calls += 1
+ if self.calls == self.block_at:
+ self.blocked.set()
+ assert self.release.wait(5)
+ return self.stop_at is not None and self.calls >= self.stop_at
+
+
+class _Counts:
+ dirs_listed = 0
+ files_statted = 0
+
+ def mark_emitted(self, key: str) -> bool:
+ return True
+
+
[email protected](autouse=True)
+def clear_watch_list():
+ _WATCH_LIST.clear()
+ yield
+ _WATCH_LIST.clear()
+
+
+def tree(temp_dir, monkeypatch) -> Path:
+ """The output directory: one file in each of FILES subdirectories."""
+ monkeypatch.setattr("folder_paths.get_output_directory", lambda: str(temp_dir))
+ for d in range(FILES):
+ sub = temp_dir / f"d{d}"
+ sub.mkdir()
+ (sub / "a.png").write_bytes(b"x" * (d + 1))
+ return temp_dir
+
+
+def _in_thread(fn):
+ result: list = []
+ worker = threading.Thread(target=lambda: result.append(fn()), daemon=True)
+ worker.start()
+ return worker, result
+
+
+def test_walk_blocks_on_pause_and_resumes_with_the_same_files(tree):
+ expected = list_files_recursively(str(tree))
+ gate = _Gate(block_at=3)
+ counts = _Counts()
+
+ worker, result = _in_thread(lambda: list_files_recursively(str(tree), counts, gate))
+ assert gate.blocked.wait(5)
+ time.sleep(0.1)
+ assert counts.dirs_listed == 2 # parked before the third directory
+ gate.release.set()
+ worker.join(5)
+
+ assert result == [expected]
+ assert gate.calls == FILES + 1
+
+
+def test_walk_stops_on_cancel(tree):
+ counts = _Counts()
+ gate = _Gate(stop_at=3)
+ files = list_files_recursively(str(tree), counts, gate)
+ assert counts.dirs_listed == 2
+ assert gate.calls == 3 # stopped walking, not just skipping the rest
+ assert len(files) < FILES
+
+
+def _paths(tree: Path) -> list[str]:
+ return sorted(list_files_recursively(str(tree)))
+
+
+def _specs(tree: Path, **kwargs):
+ return scanner.build_asset_specs(_paths(tree), set(), enable_metadata_extraction=False, **kwargs)
+
+
+# Gate call numbers: the first stat loop makes calls 1..FILES, the second stat FILES+1..2*FILES,
+# the spec loop 2*FILES+1..3*FILES.
[email protected]("loop", ["first_stat", "second_stat", "spec"])
+def test_spec_loops_block_on_pause_and_resume_with_the_same_specs(tree, loop):
+ expected = _specs(tree)
+ start = {"first_stat": 0, "second_stat": FILES, "spec": 2 * FILES}[loop]
+ gate = _Gate(block_at=start + 3)
+ counts = _Counts()
+ named: list[str] = []
+ real_name = scanner.get_name_and_tags_from_asset_path
+
+ def name(path):
+ named.append(path)
+ return real_name(path)
+
+ with patch("app.assets.scanner.get_name_and_tags_from_asset_path", name):
+ worker, result = _in_thread(lambda: _specs(tree, progress=counts, should_stop=gate))
+ assert gate.blocked.wait(5)
+ time.sleep(0.1)
+ # Parked before the third item of that loop.
+ assert counts.files_statted == min(start, FILES) + (2 if loop != "spec" else FILES)
+ assert len(named) == (2 if loop == "spec" else 0)
+ gate.release.set()
+ worker.join(5)
+
+ assert result == [expected]
+ assert gate.calls == 3 * FILES
+
+
[email protected]("loop", ["first_stat", "second_stat", "spec"])
+def test_spec_loops_return_nothing_on_cancel(tree, loop):
+ start = {"first_stat": 0, "second_stat": FILES, "spec": 2 * FILES}[loop]
+ counts = _Counts()
+ specs, tags, _ = _specs(tree, progress=counts, should_stop=_Gate(stop_at=start + 3))
+ assert (specs, tags) == ([], set())
+ assert counts.files_statted == min(start, FILES) + (2 if loop != "spec" else FILES)
+
+
+def test_second_stat_returns_nothing_on_cancel(tree):
+ candidates = [(p, os.stat(p)) for p in _paths(tree)]
+ assert _two_stat_admit(candidates, None, _Gate(stop_at=3)) == ([], [])
+
+
+def catalog():
+ engine = sa.create_engine("sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False})
+ Base.metadata.create_all(engine)
+
+ @contextmanager
+ def _create_session():
+ with SASession(engine) as sess:
+ yield sess
+
+ with patch("app.assets.scanner.create_session", _create_session), \
+ patch("app.assets.seeder.create_session", _create_session), \
+ patch("app.assets.scanner.create_write_session", sessionmaker(bind=engine)):
+ yield engine
+
+
+class _HookedState(seeder_module._ScanState):
+ """Calls ``on_count(name, value)`` as each directory or stat is counted, so a test can
+ land a prompt part way through the walk or the stats."""
+
+ on_count = None
+
+ def __setattr__(self, name, value):
+ super().__setattr__(name, value)
+ if name in ("dirs_listed", "files_statted") and self.on_count is not None:
+ self.on_count(name, value)
+
+
+def scan(tree, catalog, monkeypatch, tmp_path):
+ monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(tmp_path))
+ monkeypatch.setattr(scanner, "get_comfy_models_folders", lambda: [])
+ instance = seeder_module._AssetSeeder()
+ instance._state = seeder_module.State.RUNNING
+ instance._scan_state = _HookedState()
+ instance._phase = seeder_module.ScanPhase.FAST
+ instance._run_gate.set()
+ events: list[str] = []
+ instance.set_event_sink(lambda kind, _data: events.append(kind))
+ yield instance, events
+ instance.cancel() # frees a scan thread a failed assertion left parked
+
+
+def _rows(engine) -> int:
+ with SASession(engine) as sess:
+ return sess.scalar(sa.select(sa.func.count()).select_from(AssetContent))
+
+
+# Pause on the 3rd directory listed, or the 3rd and the 9th file stat'ed (the first and
+# second stat loops; the reference sync stats nothing on an empty catalog).
[email protected]("counter,at,parked_at", [
+ ("dirs_listed", 3, (3, 0)),
+ ("files_statted", 3, (FILES + 2, 3)),
+ ("files_statted", FILES + 3, (FILES + 2, FILES + 3)),
+])
+def test_a_prompt_starting_mid_walk_or_stat_parks_the_scan(scan, catalog, counter, at, parked_at):
+ instance, events = scan
+ state = instance._scan_state
+ parked = threading.Event()
+
+ def sink(kind, _data):
+ events.append(kind)
+ if kind == "assets.seed.paused":
+ parked.set()
+
+ instance.set_event_sink(sink)
+ state.on_count = lambda name, n: name == counter and n == at and instance.pause()
+ worker, result = _in_thread(lambda: instance._run_fast_phase(("input", "output")))
+ assert parked.wait(5)
+ time.sleep(0.2)
+ assert (state.dirs_listed, state.files_statted) == parked_at
+ assert instance.resume()
+ worker.join(5)
+
+ assert result[0][0] == FILES
+ assert _rows(catalog) == FILES
+ assert state.paused_s > 0.1 # the pause was timed; margin for the worker's lead-in
+
+
+def test_a_cancel_mid_walk_ends_the_scan_before_it_starts_seeding(scan, catalog):
+ instance, events = scan
+ state = instance._scan_state
+ state.on_count = lambda name, n: name == "dirs_listed" and n == 3 and instance.cancel()
+
+ assert instance._run_fast_phase(("input", "output")) == (0, 0, 0)
+ assert state.dirs_listed == 3 # none counted after the cancel; test_walk_stops_on_cancel pins the break
+ assert state.files_statted == 0
+ assert "assets.seed.started" not in events
+ assert state.cancel_stage == seeder_module._ScanStage.FAST_SCAN.value
+ assert _rows(catalog) == 0
diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py
index 6b1c12cc3..7c40b0653 100644
--- a/tests-unit/seeder_test/test_seeder.py
+++ b/tests-unit/seeder_test/test_seeder.py
@@ -98,7 +98,7 @@ def _configure_fast_phase(
monkeypatch.setattr(
seeder_module,
"collect_paths_for_roots",
- lambda _roots, _progress=None: [str(path) for path in paths],
+ lambda _roots, _progress=None, _should_stop=None: [str(path) for path in paths],
)
monkeypatch.setattr(
seeder_module,
@@ -609,12 +609,12 @@ def test_batch_insert_failure_emits_only_the_exception_type(
monkeypatch.setattr(
seeder_module,
"collect_paths_for_roots",
- lambda roots, progress=None: ["asset.safetensors"],
+ lambda roots, progress=None, should_stop=None: ["asset.safetensors"],
)
monkeypatch.setattr(
seeder_module,
"build_asset_specs",
- lambda paths, existing_paths, enable_metadata_extraction, progress=None: (
+ lambda paths, existing_paths, enable_metadata_extraction, progress=None, should_stop=None: (
[{"tags": []}],
{},
0,