The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
fix(assets): pause the startup scan during its folder walk and file stats (#16764)
Simon Pinfold5c460d8172fe
diff --git a/app/assets/scanner.py b/app/assets/scanner.pyindex 193f515c4..b8cfeb048 100644--- a/app/assets/scanner.py+++ b/app/assets/scanner.py@@ -410,21 +410,26 @@ def content_ids_outside_prefixes(session: Session, prefixes: list[str]) -> list[def collect_paths_for_roots(- roots: tuple[RootType, ...], progress: _ScanProgress | None = None+ roots: tuple[RootType, ...],+ progress: _ScanProgress | None = None,+ should_stop: ShouldStop = _never_stop,) -> list[str]:"""Collect all file paths for the given roots.``progress.dirs_listed`` counts the input and output walks only. Models arelisted through folder_paths.get_filename_list, which walks the model folderson a cache miss and re-checks their mtimes on a hit; none of that is counted.++ ``should_stop`` is checked before each input or output directory; once it returns+ True the list is partial, so callers check it again before using the result."""paths: list[str] = []if "models" in roots:paths.extend(collect_models_files())if "input" in roots:- paths.extend(list_files_recursively(folder_paths.get_input_directory(), progress))+ paths.extend(list_files_recursively(folder_paths.get_input_directory(), progress, should_stop))if "output" in roots:- paths.extend(list_files_recursively(folder_paths.get_output_directory(), progress))+ paths.extend(list_files_recursively(folder_paths.get_output_directory(), progress, should_stop))return paths@@ -589,6 +594,7 @@ def build_asset_specs(existing_paths: set[str],enable_metadata_extraction: bool = True,progress: _ScanProgress | None = None,+ should_stop: ShouldStop = _never_stop,) -> tuple[list[SeedAssetSpec], set[str], int]:"""Build asset specs from paths, returning (specs, tag_pool, skipped_count).@@ -597,6 +603,8 @@ def build_asset_specs(existing_paths: Set of paths that already exist in the databaseenable_metadata_extraction: If True, extract tier 1 & 2 metadataprogress: Optional per-scan state for emit-once bookkeeping+ should_stop: Checked before each path's stats and spec, so a large library+ can pause here; once it returns True, no specs are returned"""specs: list[SeedAssetSpec] = []tag_pool: set[str] = set()@@ -604,6 +612,8 @@ def build_asset_specs(candidates: list[tuple[str, os.stat_result]] = []for p in paths:+ if should_stop():+ return [], set(), skippedabs_p = os.path.abspath(p)if _should_skip_extension(abs_p):skipped += 1@@ -634,9 +644,11 @@ def build_asset_specs(continuecandidates.append((abs_p, stat_p))- admitted_paths, _ = _two_stat_admit(candidates, progress)+ admitted_paths, _ = _two_stat_admit(candidates, progress, should_stop)candidate_stats = dict(candidates)for abs_p in admitted_paths:+ if should_stop():+ return [], set(), skippedyield_gil()stat_p = candidate_stats[abs_p]name, tags = get_name_and_tags_from_asset_path(abs_p)diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.pyindex b619ad570..8bbea9c2a 100644--- a/app/assets/scanner_admission.py+++ b/app/assets/scanner_admission.py@@ -13,7 +13,7 @@ import mimetypesimport osimport timefrom dataclasses import dataclass-from typing import TYPE_CHECKING, Final, Protocol+from typing import TYPE_CHECKING, Callable, Final, Protocolfrom app.assets.event_log import emit, error_kind, error_typefrom app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path@@ -47,14 +47,22 @@ class _StatCounter(Protocol):def _two_stat_admit(- paths_with_stats: list[tuple[str, os.stat_result]], counter: _StatCounter | None = None+ paths_with_stats: list[tuple[str, os.stat_result]],+ counter: _StatCounter | None = None,+ should_stop: Callable[[], bool] | None = None,) -> tuple[list[str], list[str]]:+ """``should_stop`` is called before each second stat; it may block, and True+ returns ([], []) at once."""if not paths_with_stats:return [], []+ # The caller took the first stat; give a file still being written (a model mid-download)+ # time to change size or mtime before the second, so it goes on the watch list instead.time.sleep(0.1)admitted: list[str] = []watched: list[str] = []for path, first_stat in paths_with_stats:+ if should_stop is not None and should_stop():+ return [], []if counter is not None:counter.files_statted += 1try:diff --git a/app/assets/seeder.py b/app/assets/seeder.pyindex 7213ecc5c..84dbced3f 100644--- a/app/assets/seeder.py+++ b/app/assets/seeder.py@@ -944,7 +944,11 @@ class _AssetSeeder:t_collect = time.perf_counter()walk = list_output_for_rescan() if by_listing else None- paths = walk.files if walk is not None else collect_paths_for_roots(roots, scan_state)+ should_stop = lambda: self._check_pause_and_cancel(_ScanStage.FAST_SCAN)+ paths = walk.files if walk is not None else collect_paths_for_roots(roots, scan_state, should_stop)+ # A cancel during the walk leaves paths partial.+ if should_stop():+ return total_created, skipped_existing, 0logging.debug("Fast scan: collect_paths took %.3fs (%d paths found)",time.perf_counter() - t_collect,@@ -983,6 +987,7 @@ class _AssetSeeder:existing_paths,enable_metadata_extraction=False,progress=scan_state,+ should_stop=should_stop,)logging.debug("Fast scan: build_asset_specs took %.3fs (%d specs, %d skipped)",diff --git a/app/assets/services/file_utils.py b/app/assets/services/file_utils.pyindex 9d76c74a1..fd29b30f7 100644--- a/app/assets/services/file_utils.py+++ b/app/assets/services/file_utils.py@@ -1,5 +1,5 @@import os-from typing import NamedTuple, Protocol+from typing import Callable, NamedTuple, Protocolfrom app.assets.services.gil import yield_gil@@ -53,11 +53,15 @@ def is_visible(name: str) -> bool:def list_files_recursively(- base_dir: str, counter: _DirListingCounter | None = None+ base_dir: str,+ counter: _DirListingCounter | None = None,+ should_stop: Callable[[], bool] | None = None,) -> list[str]:"""Recursively list all files in a directory, following symlinks.- ``counter.dirs_listed`` gains one per directory os.walk listed.+ ``counter.dirs_listed`` gains one per directory os.walk listed. ``should_stop`` is+ called before each directory; it may block, and True ends the walk early with a+ partial list."""out: list[str] = []base_abs = os.path.abspath(base_dir)@@ -68,6 +72,8 @@ def list_files_recursively(for dirpath, subdirs, filenames in os.walk(base_abs, topdown=True, followlinks=True):+ if should_stop is not None and should_stop():+ breakif counter is not None:counter.dirs_listed += 1try:diff --git a/tests-unit/assets_test/services/test_scan_walk_pause.py b/tests-unit/assets_test/services/test_scan_walk_pause.pynew file mode 100644index 000000000..0d9f1526d--- /dev/null+++ b/tests-unit/assets_test/services/test_scan_walk_pause.py@@ -0,0 +1,241 @@+"""The fast scan's folder walk and its per-file stat and spec loops check the pause gate+for every directory and file, so a prompt that starts during them stops the scan at once+instead of after the whole library has been walked and stat'ed."""++import os+import threading+import time+from contextlib import contextmanager+from pathlib import Path+from unittest.mock import patch++import pytest+import sqlalchemy as sa+from sqlalchemy.orm import Session as SASession, sessionmaker+from sqlalchemy.pool import StaticPool++from app.assets import scanner, seeder as seeder_module+from app.assets.database.models import AssetContent, Base+from app.assets.scanner_admission import _WATCH_LIST, _two_stat_admit+from app.assets.services.file_utils import list_files_recursively++FILES = 6+++class _Gate:+ """A fake ShouldStop: call ``block_at`` parks until released, calls from ``stop_at`` on return True."""++ def __init__(self, block_at: int | None = None, stop_at: int | None = None) -> None:+ self.calls = 0+ self.block_at = block_at+ self.stop_at = stop_at+ self.blocked = threading.Event()+ self.release = threading.Event()++ def __call__(self) -> bool:+ self.calls += 1+ if self.calls == self.block_at:+ self.blocked.set()+ assert self.release.wait(5)+ return self.stop_at is not None and self.calls >= self.stop_at+++class _Counts:+ dirs_listed = 0+ files_statted = 0++ def mark_emitted(self, key: str) -> bool:+ return True++[email protected](autouse=True)+def clear_watch_list():+ _WATCH_LIST.clear()+ yield+ _WATCH_LIST.clear()+++def tree(temp_dir, monkeypatch) -> Path:+ """The output directory: one file in each of FILES subdirectories."""+ monkeypatch.setattr("folder_paths.get_output_directory", lambda: str(temp_dir))+ for d in range(FILES):+ sub = temp_dir / f"d{d}"+ sub.mkdir()+ (sub / "a.png").write_bytes(b"x" * (d + 1))+ return temp_dir+++def _in_thread(fn):+ result: list = []+ worker = threading.Thread(target=lambda: result.append(fn()), daemon=True)+ worker.start()+ return worker, result+++def test_walk_blocks_on_pause_and_resumes_with_the_same_files(tree):+ expected = list_files_recursively(str(tree))+ gate = _Gate(block_at=3)+ counts = _Counts()++ worker, result = _in_thread(lambda: list_files_recursively(str(tree), counts, gate))+ assert gate.blocked.wait(5)+ time.sleep(0.1)+ assert counts.dirs_listed == 2 # parked before the third directory+ gate.release.set()+ worker.join(5)++ assert result == [expected]+ assert gate.calls == FILES + 1+++def test_walk_stops_on_cancel(tree):+ counts = _Counts()+ gate = _Gate(stop_at=3)+ files = list_files_recursively(str(tree), counts, gate)+ assert counts.dirs_listed == 2+ assert gate.calls == 3 # stopped walking, not just skipping the rest+ assert len(files) < FILES+++def _paths(tree: Path) -> list[str]:+ return sorted(list_files_recursively(str(tree)))+++def _specs(tree: Path, **kwargs):+ return scanner.build_asset_specs(_paths(tree), set(), enable_metadata_extraction=False, **kwargs)+++# Gate call numbers: the first stat loop makes calls 1..FILES, the second stat FILES+1..2*FILES,+# the spec loop 2*FILES+1..3*FILES.[email protected]("loop", ["first_stat", "second_stat", "spec"])+def test_spec_loops_block_on_pause_and_resume_with_the_same_specs(tree, loop):+ expected = _specs(tree)+ start = {"first_stat": 0, "second_stat": FILES, "spec": 2 * FILES}[loop]+ gate = _Gate(block_at=start + 3)+ counts = _Counts()+ named: list[str] = []+ real_name = scanner.get_name_and_tags_from_asset_path++ def name(path):+ named.append(path)+ return real_name(path)++ with patch("app.assets.scanner.get_name_and_tags_from_asset_path", name):+ worker, result = _in_thread(lambda: _specs(tree, progress=counts, should_stop=gate))+ assert gate.blocked.wait(5)+ time.sleep(0.1)+ # Parked before the third item of that loop.+ assert counts.files_statted == min(start, FILES) + (2 if loop != "spec" else FILES)+ assert len(named) == (2 if loop == "spec" else 0)+ gate.release.set()+ worker.join(5)++ assert result == [expected]+ assert gate.calls == 3 * FILES++[email protected]("loop", ["first_stat", "second_stat", "spec"])+def test_spec_loops_return_nothing_on_cancel(tree, loop):+ start = {"first_stat": 0, "second_stat": FILES, "spec": 2 * FILES}[loop]+ counts = _Counts()+ specs, tags, _ = _specs(tree, progress=counts, should_stop=_Gate(stop_at=start + 3))+ assert (specs, tags) == ([], set())+ assert counts.files_statted == min(start, FILES) + (2 if loop != "spec" else FILES)+++def test_second_stat_returns_nothing_on_cancel(tree):+ candidates = [(p, os.stat(p)) for p in _paths(tree)]+ assert _two_stat_admit(candidates, None, _Gate(stop_at=3)) == ([], [])+++def catalog():+ engine = sa.create_engine("sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False})+ Base.metadata.create_all(engine)++ @contextmanager+ def _create_session():+ with SASession(engine) as sess:+ yield sess++ with patch("app.assets.scanner.create_session", _create_session), \+ patch("app.assets.seeder.create_session", _create_session), \+ patch("app.assets.scanner.create_write_session", sessionmaker(bind=engine)):+ yield engine+++class _HookedState(seeder_module._ScanState):+ """Calls ``on_count(name, value)`` as each directory or stat is counted, so a test can+ land a prompt part way through the walk or the stats."""++ on_count = None++ def __setattr__(self, name, value):+ super().__setattr__(name, value)+ if name in ("dirs_listed", "files_statted") and self.on_count is not None:+ self.on_count(name, value)+++def scan(tree, catalog, monkeypatch, tmp_path):+ monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(tmp_path))+ monkeypatch.setattr(scanner, "get_comfy_models_folders", lambda: [])+ instance = seeder_module._AssetSeeder()+ instance._state = seeder_module.State.RUNNING+ instance._scan_state = _HookedState()+ instance._phase = seeder_module.ScanPhase.FAST+ instance._run_gate.set()+ events: list[str] = []+ instance.set_event_sink(lambda kind, _data: events.append(kind))+ yield instance, events+ instance.cancel() # frees a scan thread a failed assertion left parked+++def _rows(engine) -> int:+ with SASession(engine) as sess:+ return sess.scalar(sa.select(sa.func.count()).select_from(AssetContent))+++# Pause on the 3rd directory listed, or the 3rd and the 9th file stat'ed (the first and+# second stat loops; the reference sync stats nothing on an empty catalog).[email protected]("counter,at,parked_at", [+ ("dirs_listed", 3, (3, 0)),+ ("files_statted", 3, (FILES + 2, 3)),+ ("files_statted", FILES + 3, (FILES + 2, FILES + 3)),+])+def test_a_prompt_starting_mid_walk_or_stat_parks_the_scan(scan, catalog, counter, at, parked_at):+ instance, events = scan+ state = instance._scan_state+ parked = threading.Event()++ def sink(kind, _data):+ events.append(kind)+ if kind == "assets.seed.paused":+ parked.set()++ instance.set_event_sink(sink)+ state.on_count = lambda name, n: name == counter and n == at and instance.pause()+ worker, result = _in_thread(lambda: instance._run_fast_phase(("input", "output")))+ assert parked.wait(5)+ time.sleep(0.2)+ assert (state.dirs_listed, state.files_statted) == parked_at+ assert instance.resume()+ worker.join(5)++ assert result[0][0] == FILES+ assert _rows(catalog) == FILES+ assert state.paused_s > 0.1 # the pause was timed; margin for the worker's lead-in+++def test_a_cancel_mid_walk_ends_the_scan_before_it_starts_seeding(scan, catalog):+ instance, events = scan+ state = instance._scan_state+ state.on_count = lambda name, n: name == "dirs_listed" and n == 3 and instance.cancel()++ assert instance._run_fast_phase(("input", "output")) == (0, 0, 0)+ assert state.dirs_listed == 3 # none counted after the cancel; test_walk_stops_on_cancel pins the break+ assert state.files_statted == 0+ assert "assets.seed.started" not in events+ assert state.cancel_stage == seeder_module._ScanStage.FAST_SCAN.value+ assert _rows(catalog) == 0diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.pyindex 6b1c12cc3..7c40b0653 100644--- a/tests-unit/seeder_test/test_seeder.py+++ b/tests-unit/seeder_test/test_seeder.py@@ -98,7 +98,7 @@ def _configure_fast_phase(monkeypatch.setattr(seeder_module,"collect_paths_for_roots",- lambda _roots, _progress=None: [str(path) for path in paths],+ lambda _roots, _progress=None, _should_stop=None: [str(path) for path in paths],)monkeypatch.setattr(seeder_module,@@ -609,12 +609,12 @@ def test_batch_insert_failure_emits_only_the_exception_type(monkeypatch.setattr(seeder_module,"collect_paths_for_roots",- lambda roots, progress=None: ["asset.safetensors"],+ lambda roots, progress=None, should_stop=None: ["asset.safetensors"],)monkeypatch.setattr(seeder_module,"build_asset_specs",- lambda paths, existing_paths, enable_metadata_extraction, progress=None: (+ lambda paths, existing_paths, enable_metadata_extraction, progress=None, should_stop=None: ([{"tags": []}],{},0,