Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

perf(assets): keep the UI responsive during asset scans and rescan output by listing folders (#16546)

Simon Pinfold56c50055cfed
diff --git a/app/assets/scanner.py b/app/assets/scanner.py
index cd3083ddf..8340d6b74 100644
--- a/app/assets/scanner.py
+++ b/app/assets/scanner.py
@@ -8,6 +8,7 @@ can end a batch early, and the cursor holds at the last row the batch attempted,
so the rows it never reached are selected again when the scan resumes.
"""
+import enum
import logging
import os
from dataclasses import dataclass
@@ -45,7 +46,16 @@ from app.assets.scanner_admission import (
_two_stat_admit,
tick_watch_list as tick_watch_list,
)
-from app.assets.services.file_utils import get_mtime_ns, is_visible, list_files_recursively
+from app.assets.services.file_utils import (
+ RESCAN_YIELD_RUN,
+ DirListings,
+ ListingWalk,
+ get_mtime_ns,
+ is_visible,
+ list_files_recursively,
+ walk_listings,
+)
+from app.assets.services.gil import yield_gil
from app.assets.services.image_dimensions import extract_image_dimensions
from app.assets.services.metadata_extract import ExtractedMetadata, extract_file_metadata
from app.assets.services.path_utils import (
@@ -297,7 +307,9 @@ def mark_contents_missing_outside_prefixes(
session: Session, prefixes: list[str]
) -> int:
contents = session.scalars(
- sa.select(AssetContent).where(AssetContent.is_missing.is_(False))
+ sa.select(AssetContent)
+ .where(AssetContent.is_missing.is_(False))
+ .execution_options(yield_per=500)
)
is_owned = path_prefix_matcher(prefixes)
missing = [content for content in contents if not is_owned(content.path)]
@@ -318,6 +330,144 @@ def collect_paths_for_roots(roots: tuple[RootType, ...]) -> list[str]:
return paths
+def rescans_output_by_listing(roots: tuple[RootType, ...]) -> bool:
+ """Whether this scan checks the catalog against directory listings rather than by
+ stat'ing every live row. Only output-only scans do: the rescan queued after each prompt.
+
+ The listing diff catches every add and delete, but nothing stats an already-cataloged
+ file, so an in-place overwrite (same path; new content, size or mtime) goes undetected
+ until the next scan that is not output-only, such as the startup scan. Core save nodes
+ never overwrite, and reported outputs are registered at save time, so this only
+ affects files written by something else.
+ """
+ return tuple(roots) == ("output",)
+
+
+def live_references_safely(root: RootType) -> dict[str, list[_ReferenceObservation]]:
+ """The live rows under ``root`` by path, read without touching the filesystem.
+
+ Each is observed as gone (``stat_result=None``): what apply_reference_observations
+ needs to retire it, should its path turn out not to be listed. Empty on failure, as
+ sync_root_safely is.
+ """
+ prefixes = get_scan_prefixes_for_root(root)
+ live: dict[str, list[_ReferenceObservation]] = {}
+ if not prefixes:
+ return live
+ stmt = sa.select(
+ AssetContent.id, AssetContent.path, AssetContent.size_bytes, AssetContent.mtime_ns
+ ).where(
+ AssetContent.is_missing.is_(False),
+ sa.or_(*(sql_path_under_prefix(AssetContent.path, p) for p in prefixes)),
+ )
+ try:
+ with create_session() as session:
+ for content_id, path, size_bytes, mtime_ns in session.execute(stmt):
+ yield_gil(run=RESCAN_YIELD_RUN)
+ live.setdefault(os.path.abspath(path), []).append(
+ _ReferenceObservation(content_id, size_bytes, mtime_ns, None)
+ )
+ except Exception as exc:
+ logging.exception("fast DB scan failed for %s: %s", root, exc)
+ emit("scanner.fast_scan_failed", root=root, error_type=error_type(exc))
+ return {}
+ return live
+
+
+def unlisted_references(
+ live: dict[str, list[_ReferenceObservation]], listings: DirListings
+) -> tuple[list[_ReferenceObservation], int]:
+ """Split the live rows into (vanished, skipped count).
+
+ A row its parent's listing names is present, with no further check. Every other row
+ is stat'ed, and has vanished only if the stat says the file is gone: that covers a
+ name the listing lacks, a removed directory, and the rows no listing can speak for
+ (a hidden path, a directory that failed to list, a symlink alias the walk did not
+ take). The skipped count is the rows that were stat'ed and kept.
+ """
+ vanished: list[_ReferenceObservation] = []
+ skipped = 0
+ names_by_dir: dict[str, set[str]] = {}
+ for path, observations in live.items():
+ yield_gil(run=RESCAN_YIELD_RUN)
+ verdict = _listing_verdict(path, listings, names_by_dir)
+ if verdict is ListingVerdict.LISTED:
+ continue
+ # Stat before retiring. A listing compares names exactly, but a case-insensitive
+ # (NTFS, APFS) or Unicode-normalizing (HFS+) filesystem resolves a stored path
+ # spelled differently from its entry. Rows that reach here are normally few.
+ if _is_gone(path):
+ vanished.extend(observations)
+ else:
+ skipped += len(observations)
+ return vanished, skipped
+
+
+def _is_gone(path: str) -> bool:
+ """True only when stat says the path does not exist. Any other error (permissions,
+ I/O) leaves it undecided, and the row stays live, as the per-row stat left it."""
+ try:
+ os.stat(path)
+ except (FileNotFoundError, NotADirectoryError):
+ return True
+ except OSError:
+ return False
+ return False
+
+
+class ListingVerdict(enum.Enum):
+ """What this rescan's directory listings say about a cataloged path."""
+
+ LISTED = "listed" # its parent's listing has the name
+ ABSENT = "absent" # the nearest listed directory above it lacks the next component
+ UNKNOWN = "unknown" # no listing can say: hidden, no listed ancestor, or not walked
+
+
+def _listing_verdict(
+ path: str, listings: DirListings, names_by_dir: dict[str, set[str]]
+) -> ListingVerdict:
+ """Classify ``path`` against the listings; see ListingVerdict. An ancestor that
+ still has the directory this walk did not list is UNKNOWN, not LISTED."""
+ child, parent = path, os.path.dirname(path)
+ while parent not in listings:
+ if not is_visible(os.path.basename(child)):
+ return ListingVerdict.UNKNOWN
+ child, parent = parent, os.path.dirname(parent)
+ if parent == child: # reached the filesystem root without meeting a listing
+ return ListingVerdict.UNKNOWN
+ name = os.path.basename(child)
+ if not is_visible(name):
+ return ListingVerdict.UNKNOWN
+ names = names_by_dir.get(parent)
+ if names is None:
+ files, subdirs = listings[parent]
+ names = names_by_dir[parent] = {*files, *subdirs}
+ if name not in names:
+ return ListingVerdict.ABSENT
+ return ListingVerdict.LISTED if child == path else ListingVerdict.UNKNOWN
+
+
+def mark_unlisted_references_missing_safely(
+ root: RootType, observations: list[_ReferenceObservation]
+) -> None:
+ """Retire rows whose file the listing lacks, through the same guarded write
+ sync_root applies to a row whose file has vanished."""
+ if not observations:
+ return
+ try:
+ with create_write_session() as session:
+ apply_reference_observations(session, observations)
+ session.commit()
+ except Exception as exc:
+ logging.exception("fast DB scan failed for %s: %s", root, exc)
+ emit("scanner.fast_scan_failed", root=root, error_type=error_type(exc))
+
+
+def list_output_for_rescan() -> ListingWalk:
+ """Walk the output root, listing every directory."""
+ return walk_listings(folder_paths.get_output_directory())
+
+
def build_asset_specs(
paths: list[str],
existing_paths: set[str],
@@ -364,6 +514,7 @@ def build_asset_specs(
admitted_paths, _ = _two_stat_admit(candidates)
candidate_stats = dict(candidates)
for abs_p in admitted_paths:
+ yield_gil()
stat_p = candidate_stats[abs_p]
name, tags = get_name_and_tags_from_asset_path(abs_p)
rel_fname = compute_loader_path(abs_p)
diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py
index 19b60b5ff..df257b82c 100644
--- a/app/assets/scanner_changes.py
+++ b/app/assets/scanner_changes.py
@@ -8,6 +8,7 @@ so a restored file can never leave two live rows describing one location.
from __future__ import annotations
import os
+from collections.abc import Iterable
from typing import Literal
import sqlalchemy as sa
@@ -202,14 +203,12 @@ def drain_pending_verifications(session: Session, limit: int | None = None) -> i
return processed
-def live_contents_under_prefixes(session: Session, prefixes: list[str]) -> list[AssetContent]:
+def live_contents_under_prefixes(session: Session, prefixes: list[str]) -> Iterable[AssetContent]:
+ """Stream the live contents under the prefixes in batches; consume it inside the session."""
if not prefixes:
return []
- return list(
- session.scalars(
- sa.select(AssetContent).where(
- AssetContent.is_missing.is_(False),
- sa.or_(*(sql_path_under_prefix(AssetContent.path, prefix) for prefix in prefixes)),
- )
- )
+ stmt = sa.select(AssetContent).where(
+ AssetContent.is_missing.is_(False),
+ sa.or_(*(sql_path_under_prefix(AssetContent.path, prefix) for prefix in prefixes)),
)
+ return session.scalars(stmt.execution_options(yield_per=500))
diff --git a/app/assets/seeder.py b/app/assets/seeder.py
index 8e6e8ed18..a538f4ba4 100644
--- a/app/assets/seeder.py
+++ b/app/assets/seeder.py
@@ -24,8 +24,13 @@ from app.assets.scanner import (
get_scan_prefixes_for_root,
get_unenriched_assets_for_roots,
insert_asset_specs,
+ list_output_for_rescan,
+ live_references_safely,
mark_missing_outside_prefixes_safely,
+ mark_unlisted_references_missing_safely,
+ rescans_output_by_listing,
sync_root_safely,
+ unlisted_references,
sync_temp_references_safely,
drain_pending_verifications,
tick_watch_list,
@@ -796,6 +801,8 @@ class _AssetSeeder:
total_created = 0
skipped_existing = 0
+ by_listing = rescans_output_by_listing(roots)
+ live_references: dict[str, list] = {}
existing_paths: set[str] = set()
t_sync = time.perf_counter()
assert self._scan_state is not None
@@ -803,7 +810,11 @@ class _AssetSeeder:
for r in roots:
if self._check_pause_and_cancel(_ScanStage.FAST_SCAN):
return total_created, skipped_existing, 0
- existing_paths.update(sync_root_safely(r, scan_state))
+ if by_listing:
+ live_references = live_references_safely(r)
+ existing_paths.update(live_references)
+ else:
+ existing_paths.update(sync_root_safely(r, scan_state))
logging.debug(
"Fast scan: sync_root phase took %.3fs (%d existing paths)",
time.perf_counter() - t_sync,
@@ -814,12 +825,23 @@ class _AssetSeeder:
return total_created, skipped_existing, 0
t_collect = time.perf_counter()
- paths = collect_paths_for_roots(roots)
+ walk = list_output_for_rescan() if by_listing else None
+ paths = walk.files if walk is not None else collect_paths_for_roots(roots)
logging.debug(
"Fast scan: collect_paths took %.3fs (%d paths found)",
time.perf_counter() - t_collect,
len(paths),
)
+ if walk is not None:
+ vanished, unlisted = unlisted_references(live_references, walk.listings)
+ mark_unlisted_references_missing_safely("output", vanished)
+ logging.debug(
+ "Fast scan: output listing: %d dirs listed, %d rows retired, "
+ "%d rows skipped (not listed, still on disk)",
+ walk.dirs_listed,
+ len(vanished),
+ unlisted,
+ )
total_paths = len(paths)
self._update_progress(total=total_paths)
diff --git a/app/assets/services/file_utils.py b/app/assets/services/file_utils.py
index c47ebe460..4937f0c5d 100644
--- a/app/assets/services/file_utils.py
+++ b/app/assets/services/file_utils.py
@@ -1,4 +1,11 @@
import os
+from typing import NamedTuple
+
+from app.assets.services.gil import yield_gil
+
+# Longer run window for output rescans: they repeat after prompts, so pausing every
+# 2ms would add up to a much slower rescan.
+RESCAN_YIELD_RUN = 0.010
def get_mtime_ns(stat_result: os.stat_result) -> int:
@@ -68,3 +75,73 @@ def list_files_recursively(base_dir: str) -> list[str]:
continue
out.append(os.path.abspath(os.path.join(dirpath, name)))
return out
+
+
+# dir path -> (visible file names, visible subdir names)
+DirListings = dict[str, tuple[list[str], list[str]]]
+
+
+class ListingWalk(NamedTuple):
+ files: list[str]
+ listings: DirListings
+ dirs_listed: int
+
+
+def _list_visible_entries(dirpath: str) -> tuple[list[str], list[str]]:
+ """One directory's visible (file names, subdir names), classified as os.walk does:
+ anything whose is_dir() is false or raises is a file. The exception is a symlink
+ whose target is gone: it is left out, so a row for it reads as vanished, as it did
+ when the rescan stat'ed every row through the link."""
+ files: list[str] = []
+ subdirs: list[str] = []
+ with os.scandir(dirpath) as entries:
+ for entry in entries:
+ yield_gil(run=RESCAN_YIELD_RUN)
+ if not is_visible(entry.name):
+ continue
+ try:
+ is_dir = entry.is_dir()
+ except OSError:
+ is_dir = False
+ if is_dir:
+ subdirs.append(entry.name)
+ elif not (entry.is_symlink() and not os.path.exists(entry.path)):
+ files.append(entry.name)
+ return files, subdirs
+
+
+def walk_listings(base_dir: str) -> ListingWalk:
+ """list_files_recursively, also returning every directory listing it read.
+
+ Same traversal as the os.walk version (visit order, symlink following, device/inode
+ cycle guard, hidden filtering), except each directory is stat'ed before it is listed.
+ ``listings`` holds exactly the directories this walk listed, keyed by normalized
+ absolute path, so it doubles as the record of which directories the walk can vouch for.
+ """
+ files: list[str] = []
+ listings: DirListings = {}
+ # No isdir() precheck, so each directory costs exactly one stat: a root that is
+ # missing or not a directory fails its stat or scandir below and yields nothing.
+ seen_dirs: set[tuple[int, int]] = set()
+ stack = [os.path.abspath(base_dir)]
+ while stack:
+ yield_gil(run=RESCAN_YIELD_RUN)
+ dirpath = stack.pop()
+ try:
+ st = os.stat(dirpath)
+ except OSError:
+ continue
+ dir_id = (st.st_dev, st.st_ino)
+ if dir_id in seen_dirs:
+ continue
+ try:
+ names, subdirs = _list_visible_entries(dirpath)
+ except OSError:
+ continue
+ seen_dirs.add(dir_id)
+ listings[dirpath] = (names, subdirs)
+ for name in names:
+ yield_gil(run=RESCAN_YIELD_RUN)
+ files.append(os.path.abspath(os.path.join(dirpath, name)))
+ stack.extend(os.path.join(dirpath, name) for name in reversed(subdirs))
+ return ListingWalk(files, listings, len(listings))
diff --git a/app/assets/services/gil.py b/app/assets/services/gil.py
new file mode 100644
index 000000000..77ab3dc79
--- /dev/null
+++ b/app/assets/services/gil.py
@@ -0,0 +1,64 @@
+"""Let the event loop run while a scan thread is in a long pure-Python loop.
+
+Without this, the event loop must win the GIL back from the scan after every socket
+syscall, and a page load's requests queue behind the scan for seconds. time.sleep(0)
+is not enough: the scan thread usually retakes the GIL before the loop wakes.
+
+A thread sleeps _SLEEP after running _RUN, so it spends about a third of its time
+asleep. Measured on a 41k-file first scan, that is where page loads stopped getting
+faster (a shorter run bought nothing), for about 2% more scan time. All state is per
+thread; nothing is shared.
+"""
+
+import sys
+import threading
+import time
+
+_RUN = 0.002
+_SLEEP = 0.001
+# The coarse timer tick is ~15.6ms, so a 1ms sleep typically takes about 16x as long there.
+_MAX_SCALE = 16.0
+
+# Indirection so tests can drive the clock without patching the time module.
+_clock = time.perf_counter
+_sleep = time.sleep
+
+_state = threading.local()
+
+
+def _yield_fixed(run: float = _RUN) -> None:
+ """Call once per item in a hot loop on a background thread; sleeps every ``run``
+ seconds of work. A caller that repeats often can pass a longer window to pause less."""
+ now = _clock()
+ next_at = getattr(_state, "next_at", None)
+ if next_at is None:
+ _state.next_at = now + run
+ return
+ if now < next_at:
+ return
+ _sleep(_SLEEP)
+ _state.next_at = _clock() + run
+
+
+def _yield_scaled(run: float = _RUN) -> None:
+ """For a coarse sleep timer: time.sleep(_SLEEP) really takes a whole timer tick, so
+ scale the next run by how long the last sleep took, keeping the same share asleep."""
+ now = _clock()
+ next_at = getattr(_state, "next_at", None)
+ if next_at is None:
+ _state.next_at = now + run
+ return
+ if now < next_at:
+ return
+ _sleep(_SLEEP)
+ after = _clock()
+ # Scale up to the timer tick this exists for, but no further: a sleep that overshoots
+ # because the machine is busy must not buy the scan a longer run.
+ scale = min(max(1.0, (after - now) / _SLEEP), _MAX_SCALE)
+ _state.next_at = after + run * scale
+
+
+# Before Python 3.11, time.sleep on Windows rounds up to the ~15.6ms system timer tick.
+_COARSE_SLEEP = sys.platform == "win32" and sys.version_info < (3, 11)
+
+yield_gil = _yield_scaled if _COARSE_SLEEP else _yield_fixed
diff --git a/tests-unit/assets_test/services/test_gil.py b/tests-unit/assets_test/services/test_gil.py
new file mode 100644
index 000000000..2affd207e
--- /dev/null
+++ b/tests-unit/assets_test/services/test_gil.py
@@ -0,0 +1,158 @@
+import queue
+import threading
+
+import pytest
+
+from app.assets.services import gil
+
+
+SHARE = gil._SLEEP / (gil._SLEEP + gil._RUN)
+
+
+class FakeClock:
+ """Stands in for gil._clock / gil._sleep; a sleep advances the clock by `sleep_cost`."""
+
+ def __init__(self, sleep_cost: float) -> None:
+ self.now = 100.0
+ self.sleep_cost = sleep_cost
+ self.slept = 0.0
+ self.sleeps = 0
+
+ def clock(self) -> float:
+ return self.now
+
+ def sleep(self, _seconds: float) -> None:
+ self.now += self.sleep_cost
+ self.slept += self.sleep_cost
+ self.sleeps += 1
+
+
+def fake(monkeypatch):
+ def install(sleep_cost: float) -> FakeClock:
+ clock = FakeClock(sleep_cost)
+ monkeypatch.setattr(gil, "_clock", clock.clock)
+ monkeypatch.setattr(gil, "_sleep", clock.sleep)
+ monkeypatch.setattr(gil, "_state", threading.local())
+ return clock
+
+ return install
+
+
+def started(fake, sleep_cost: float, yield_gil) -> FakeClock:
+ clock = fake(sleep_cost)
+ yield_gil() # starts this thread's first run window
+ return clock
+
+
+def test_sleeps_only_once_the_run_window_has_passed(fake):
+ clock = started(fake, gil._SLEEP, gil._yield_fixed)
+ clock.now += gil._RUN / 2
+ gil._yield_fixed()
+ assert clock.sleeps == 0
+
+ clock.now += gil._RUN
+ gil._yield_fixed()
+ assert clock.sleeps == 1
+
+
+def test_run_window_restarts_after_each_sleep(fake):
+ clock = started(fake, gil._SLEEP, gil._yield_fixed)
+ clock.now += gil._RUN * 1.5
+ gil._yield_fixed()
+ gil._yield_fixed()
+ assert clock.sleeps == 1
+
+
+def run_hot_loop(clock: FakeClock, yield_gil, seconds: float, work_per_item: float = 0.0001) -> float:
+ """Simulate a loop doing `work_per_item` per call; return the fraction spent asleep."""
+ start = clock.now
+ while clock.now - start < seconds:
+ clock.now += work_per_item
+ yield_gil()
+ return clock.slept / (clock.now - start)
+
+
+def test_accurate_sleep_keeps_the_measured_duty_cycle(fake):
+ clock = started(fake, gil._SLEEP, gil._yield_fixed)
+ assert run_hot_loop(clock, gil._yield_fixed, 10.0) == pytest.approx(SHARE, abs=0.02)
+
+
+def test_coarse_sleep_widens_the_run_window_to_keep_the_duty_cycle(fake):
+ # A 1ms sleep that really takes a 15ms timer tick, as on Windows before Python 3.11.
+ clock = started(fake, 0.015, gil._yield_scaled)
+ assert run_hot_loop(clock, gil._yield_scaled, 30.0) == pytest.approx(SHARE, abs=0.02)
+
+
+def test_slightly_slow_sleep_scales_the_run_window_continuously(fake):
+ clock = started(fake, 0.0015, gil._yield_scaled)
+ assert run_hot_loop(clock, gil._yield_scaled, 10.0) == pytest.approx(SHARE, abs=0.02)
+
+
+def test_sleep_longer_than_the_timer_tick_sleeps_more_rather_than_running_longer(fake):
+ # Past the tick the window stops growing, so the scan's share asleep rises instead.
+ clock = started(fake, 0.040, gil._yield_scaled)
+ assert run_hot_loop(clock, gil._yield_scaled, 60.0) > SHARE + 0.1
+
+
+def test_threads_do_not_consume_each_others_run_window(fake):
+ clock = started(fake, gil._SLEEP, gil._yield_fixed)
+ sleeps_seen: list[int] = []
+
+ def worker(inbox: queue.Queue, done: queue.Queue) -> None:
+ while inbox.get():
+ before = clock.sleeps
+ gil._yield_fixed()
+ sleeps_seen.append(clock.sleeps - before)
+ done.put(True)
+
+ threads = []
+ for _ in range(2):
+ inbox, done = queue.Queue(), queue.Queue()
+ t = threading.Thread(target=worker, args=(inbox, done))
+ t.start()
+ threads.append((t, inbox, done))
+
+ def call(i: int) -> int:
+ _, inbox, done = threads[i]
+ inbox.put(True)
+ done.get(timeout=5)
+ return sleeps_seen[-1]
+
+ try:
+ assert call(0) == 0 and call(1) == 0 # each thread starts its own run window
+ clock.now += gil._RUN * 1.5
+ assert call(0) == 1 # thread 0 yields and restarts only its own run window
+ assert call(1) == 1 # thread 1's run window is untouched, so it yields too
+ finally:
+ for t, inbox, _ in threads:
+ inbox.put(False)
+ t.join(timeout=5)
+
+
+def test_fixed_window_on_a_coarse_timer_sleeps_far_more_than_a_sixth(fake):
+ # Why the scaled version exists: a 1ms sleep that really takes a 15ms tick.
+ clock = started(fake, 0.015, gil._yield_fixed)
+ assert run_hot_loop(clock, gil._yield_fixed, 30.0) > 2 * SHARE
+
+
+def test_scaled_version_is_used_only_for_the_coarse_windows_timer():
+ coarse = gil.sys.platform == "win32" and gil.sys.version_info < (3, 11)
+ assert gil._COARSE_SLEEP is coarse
+ assert gil.yield_gil is (gil._yield_scaled if coarse else gil._yield_fixed)
+
+
+def test_contended_sleep_does_not_stretch_the_run_window_past_the_timer_tick(fake):
+ # A 1ms sleep that overshoots to 400ms under load must not buy an 800ms run window.
+ clock = started(fake, 0.400, gil._yield_scaled)
+ clock.now += gil._RUN
+ gil._yield_scaled() # sleeps, taking 400ms
+ assert gil._state.next_at - clock.now <= gil._RUN * gil._MAX_SCALE + 1e-9
+
+
+
+def test_a_longer_run_window_sleeps_proportionally_less(fake):
+ run = gil._RUN * 5
+ clock = started(fake, gil._SLEEP, lambda: gil._yield_fixed(run=run))
+ share = gil._SLEEP / (gil._SLEEP + run)
+ assert run_hot_loop(clock, lambda: gil._yield_fixed(run=run), 10.0) == pytest.approx(share, abs=0.01)
diff --git a/tests-unit/assets_test/services/test_output_rescan.py b/tests-unit/assets_test/services/test_output_rescan.py
new file mode 100644
index 000000000..eeada768c
--- /dev/null
+++ b/tests-unit/assets_test/services/test_output_rescan.py
@@ -0,0 +1,459 @@
+"""Output-only rescans diff directory listings against the catalog instead of stat'ing
+every live row. Run through the seeder's real fast phase on an in-memory catalog."""
+
+import logging
+import os
+import re
+import shutil
+import stat as stat_module
+from contextlib import contextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import sqlalchemy as sa
+from sqlalchemy.orm import Session as SASession, sessionmaker
+
+from app.assets import scanner, seeder as seeder_module
+from app.assets.database.models import AssetContent
+from app.assets.database.queries.records import create_content, create_record
+from app.assets.scanner_admission import _WATCH_LIST
+from app.assets.services import file_utils
+from app.assets.services.file_utils import list_files_recursively, walk_listings
+
+OUTPUT_ONLY = ("output",)
+ALL_ROOTS = ("models", "input", "output")
+N_FILES = 30
+
+
+def roots(temp_dir: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path]:
+ dirs = {name: temp_dir / name for name in ("input", "output")}
+ for path in dirs.values():
+ path.mkdir()
+ monkeypatch.setattr("folder_paths.get_output_directory", lambda: str(dirs["output"]))
+ monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(dirs["input"]))
+ monkeypatch.setattr(scanner, "get_comfy_models_folders", lambda: [])
+ return dirs
+
+
[email protected](autouse=True)
+def isolated_state(db_engine):
+ @contextmanager
+ def _create_session():
+ with SASession(db_engine) as sess:
+ yield sess
+
+ _WATCH_LIST.clear()
+ with patch("app.assets.scanner.create_session", _create_session), \
+ patch("app.assets.seeder.create_session", _create_session), \
+ patch("app.database.db.WriteSession", sessionmaker(bind=db_engine)):
+ yield
+ _WATCH_LIST.clear()
+
+
+def _scan(roots=OUTPUT_ONLY) -> tuple[int, int, int]:
+ seeder = seeder_module._AssetSeeder()
+ seeder._scan_state = seeder_module._ScanState()
+ seeder._phase = seeder_module.ScanPhase.FAST
+ seeder._run_gate.set()
+ seeder._cancel_event.clear()
+ return seeder._run_fast_phase(roots)
+
+
+def _write(path: Path, payload: bytes = b"png-bytes") -> Path:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(payload)
+ return path
+
+
+def _warm_catalog(output: Path) -> list[Path]:
+ """N files over the root and two levels of nesting."""
+ subdirs = [output, output / "a", output / "b" / "c"]
+ return [_write(subdirs[i % 3] / f"img_{i:03d}.png") for i in range(N_FILES)]
+
+
+def _rows(session, path: Path) -> list[AssetContent]:
+ session.expire_all()
+ return list(session.scalars(sa.select(AssetContent).where(AssetContent.path == str(path))))
+
+
+def _live_paths(session) -> set[str]:
+ session.expire_all()
+ return set(session.scalars(sa.select(AssetContent.path).where(AssetContent.is_missing.is_(False))))
+
+
+_LISTING_LINE = re.compile(
+ r"output listing: (\d+) dirs listed, (\d+) rows retired, (\d+) rows skipped"
+)
+
+
+def _listing_counts(caplog: pytest.LogCaptureFixture) -> tuple[int, int, int]:
+ """(dirs listed, rows retired, rows skipped) from the one rescan in ``caplog``."""
+ matches = [_LISTING_LINE.search(r.getMessage()) for r in caplog.records]
+ found = [m for m in matches if m]
+ assert len(found) == 1, [r.getMessage() for r in caplog.records]
+ listed, retired, skipped = (int(g) for g in found[0].groups())
+ return listed, retired, skipped
+
+
+def _scan_logged(caplog: pytest.LogCaptureFixture, roots=OUTPUT_ONLY) -> tuple[int, int, int]:
+ caplog.clear()
+ with caplog.at_level(logging.DEBUG):
+ return _scan(roots)
+
+
+def test_output_rescan_catalogs_new_files_and_retires_deleted_ones(roots, session, caplog):
+ output = roots["output"]
+ files = _warm_catalog(output)
+ _scan()
+ assert _live_paths(session) == {str(p) for p in files}
+
+ deleted = files[:10]
+ for path in deleted:
+ path.unlink()
+ added = [_write(output / ("a" if i % 2 else "b/c") / f"new_{i}.png") for i in range(10)]
+
+ created, _skipped, total = _scan_logged(caplog)
+
+ assert created == 10
+ assert total == N_FILES
+ assert _live_paths(session) == {str(p) for p in files[10:] + added}
+ for path in deleted:
+ (row,) = _rows(session, path)
+ assert row.is_missing is True
+ _listed, retired, skipped = _listing_counts(caplog)
+ assert (retired, skipped) == (10, 0)
+
+
+def test_in_place_overwrite_is_not_detected_by_an_output_only_rescan(roots, session):
+ """The documented, accepted gap: without a per-row stat, a same-path overwrite is
+ invisible to an output-only rescan until the next scan that stats."""
+ target = _warm_catalog(roots["output"])[0]
+ _scan()
+ (before,) = _rows(session, target)
+
+ _write(target, b"a different, longer payload")
+ _scan()
+
+ (after,) = _rows(session, target)
+ assert after.id == before.id
+ assert after.is_missing is False
+ assert after.size_bytes == before.size_bytes != target.stat().st_size
+
+
+def test_a_scan_that_is_not_output_only_still_stats_every_row(roots, session, caplog):
+ files = _warm_catalog(roots["output"])
+ _write(roots["input"] / "in.png")
+ _scan(ALL_ROOTS)
+ (before,) = _rows(session, files[0])
+
+ _write(files[0], b"a different, longer payload")
+ files[1].unlink()
+ _scan_logged(caplog, ALL_ROOTS)
+
+ assert not [r for r in caplog.records if _LISTING_LINE.search(r.getMessage())]
+ # The stat path ran: the overwrite split the row, the deletion retired its row.
+ assert _rows(session, files[1])[0].is_missing is True
+ rows = _rows(session, files[0])
+ assert {row.id: row.is_missing for row in rows}[before.id] is True
+ assert len(rows) == 2
+
+
+def test_walk_listings_matches_the_os_walk_filtering(temp_dir: Path):
+ base = temp_dir / "tree"
+ _write(base / "a.png")
+ _write(base / ".hidden.png")
+ _write(base / ".hidden_dir" / "x.png")
+ _write(base / "sub" / "empty.png", b"")
+ _write(base / "sub" / "dl.png.part")
+ _write(base / "sub" / "deep" / "d.png")
+ (base / "link_to_sub").symlink_to(base / "sub")
+ (base / "sub" / "loop").symlink_to(base)
+ (base / "broken").symlink_to(base / "nowhere")
+ os.symlink(base / "a.png", base / "file_link.png")
+
+ walk = walk_listings(str(base))
+
+ # The one intended difference: a broken symlink is left out, so its row reads as gone.
+ assert walk.files == [p for p in list_files_recursively(str(base)) if p != str(base / "broken")]
+ assert walk.dirs_listed == len(walk.listings) == 3 # base, sub, sub/deep
+
+
+def _catalog_directly(session, path: Path) -> str:
+ """A live row the walk would never produce itself."""
+ stat = path.stat()
+ content = create_content(session, str(path), None, stat.st_size, stat.st_mtime_ns)
+ create_record(session, content.id, path.name)
+ session.commit()
+ return content.id
+
+
+def test_row_whose_stored_spelling_differs_from_the_entry_stays_live(roots, session, monkeypatch):
+ # A node that saves to "Portraits/" reuses an existing "portraits/" on NTFS or APFS and
+ # reports its own spelling, so the row's path never matches the listing verbatim.
+ real = _write(roots["output"] / "portraits" / "img.png")
+ respelled = roots["output"] / "Portraits" / "img.png"
+ stat = real.stat()
+ content = create_content(session, str(respelled), None, stat.st_size, stat.st_mtime_ns)
+ create_record(session, content.id, respelled.name)
+ session.commit()
+ # Resolve either spelling, as a case-insensitive filesystem does, on any host.
+ real_stat = os.stat
+ monkeypatch.setattr(
+ os, "stat",
+ lambda p, *a, **k: real_stat(os.fspath(p).replace(str(respelled.parent), str(real.parent)), *a, **k),
+ )
+
+ _scan()
+
+ assert session.get(AssetContent, content.id).is_missing is False
+
+
+def test_row_the_listing_misses_but_cannot_be_statted_stays_live(roots, session, monkeypatch):
+ real = _write(roots["output"] / "portraits" / "img.png")
+ respelled = roots["output"] / "Portraits" / "img.png"
+ stat = real.stat()
+ content = create_content(session, str(respelled), None, stat.st_size, stat.st_mtime_ns)
+ create_record(session, content.id, respelled.name)
+ session.commit()
+ real_stat = os.stat
+
+ def stat_denying_the_respelled_path(path, *args, **kwargs):
+ if os.fspath(path) == str(respelled):
+ raise PermissionError(13, "Permission denied", os.fspath(path))
+ return real_stat(path, *args, **kwargs)
+
+ monkeypatch.setattr(os, "stat", stat_denying_the_respelled_path)
+
+ _scan()
+
+ assert session.get(AssetContent, content.id).is_missing is False
+
+
+def test_rows_under_hidden_paths_stay_live(roots, session, caplog):
+ output = roots["output"]
+ _warm_catalog(output)
+ hidden = [_write(output / ".cache" / "x.png"), _write(output / "a" / ".dotfile.png")]
+ for path in hidden:
+ _catalog_directly(session, path)
+
+ _scan_logged(caplog)
+
+ for path in hidden:
+ (row,) = _rows(session, path)
+ assert row.is_missing is False
+ _listed, retired, skipped = _listing_counts(caplog)
+ assert (retired, skipped) == (0, 2)
+
+
+def test_dir_failing_to_list_stats_its_rows_while_siblings_are_diffed(
+ roots, session, monkeypatch, caplog
+):
+ output = roots["output"]
+ files = _warm_catalog(output)
+ _scan()
+ in_failing = [p for p in files if p.parent == output / "a"]
+ in_sibling = [p for p in files if p.parent == output / "b" / "c"]
+ in_failing[0].unlink()
+ in_sibling[0].unlink()
+
+ real_list = file_utils._list_visible_entries
+
+ def failing_list(dirpath: str):
+ if dirpath == str(output / "a"):
+ raise OSError("transient listing failure")
+ return real_list(dirpath)
+
+ monkeypatch.setattr(file_utils, "_list_visible_entries", failing_list)
+ _scan_logged(caplog)
+
+ for path in in_failing:
+ (row,) = _rows(session, path)
+ assert row.is_missing is (path == in_failing[0])
+ (gone,) = _rows(session, in_sibling[0])
+ assert gone.is_missing is True
+ _listed, retired, skipped = _listing_counts(caplog)
+ assert (retired, skipped) == (2, len(in_failing) - 1)
+
+
+def test_deleted_output_under_a_hidden_path_is_retired(roots, session):
+ # SaveImage accepts prefixes like "foo/.bar/img" (hidden dir) and "foo/.bar"
+ # (hidden file), and the node registers the result like any other output.
+ output = roots["output"]
+ _warm_catalog(output)
+ hidden = [_write(output / "foo" / ".bar" / "img.png"), _write(output / "foo" / ".bar_00001_.png")]
+ for path in hidden:
+ _catalog_directly(session, path)
+ _scan()
+ for path in hidden:
+ path.unlink()
+
+ _scan()
+
+ for path in hidden:
+ (row,) = _rows(session, path)
+ assert row.is_missing is True
+
+
+def test_unlistable_output_root_stats_every_row(roots, session, monkeypatch):
+ output = roots["output"]
+ files = _warm_catalog(output)
+ _scan()
+ files[0].unlink()
+ real_list = file_utils._list_visible_entries
+
+ def failing_root(dirpath: str):
+ if dirpath == str(output):
+ raise OSError("root unreadable")
+ return real_list(dirpath)
+
+ monkeypatch.setattr(file_utils, "_list_visible_entries", failing_root)
+ _scan()
+
+ assert _live_paths(session) == {str(p) for p in files[1:]}
+
+
+def test_row_under_an_unvisited_symlink_alias_is_stat_checked(roots, session):
+ output = roots["output"]
+ real = _write(output / "real" / "x.png")
+ (output / "alias").symlink_to(output / "real")
+ _scan()
+ (cataloged,) = _live_paths(session)
+ # The cycle guard walks the directory once, under whichever name came first.
+ visited, unvisited = (
+ (real, output / "alias" / "x.png")
+ if cataloged == str(real)
+ else (output / "alias" / "x.png", real)
+ )
+ alias_id = _catalog_directly(session, unvisited)
+
+ _scan()
+
+ assert session.get(AssetContent, alias_id).is_missing is False
+ assert _live_paths(session) == {str(visited), str(unvisited)}
+
+ real.unlink()
+ _scan()
+
+ assert session.get(AssetContent, alias_id).is_missing is True
+ assert _live_paths(session) == set()
+
+
+DEPTH = 20
+
+
+def _deep_tree(output: Path) -> tuple[list[Path], list[Path]]:
+ """A chain of DEPTH nested dirs under output with one file per level, plus
+ three unchanged sibling dirs at the root. Returns (dirs, files), dirs[0] the root."""
+ dirs = [output]
+ for level in range(1, DEPTH + 1):
+ dirs.append(dirs[-1] / f"d{level:02d}")
+ dirs += [output / f"sib{i}" for i in range(3)]
+ files = [_write(d / f"f_{i:02d}.png") for i, d in enumerate(dirs)]
+ return dirs, files
+
+
+def dir_stats(monkeypatch: pytest.MonkeyPatch):
+ """Counts os.stat calls per directory path; files are not counted."""
+ counts: dict[str, int] = {}
+ real_stat = os.stat
+
+ def counting_stat(path, *args, **kwargs):
+ result = real_stat(path, *args, **kwargs)
+ if stat_module.S_ISDIR(result.st_mode):
+ counts[os.fspath(path)] = counts.get(os.fspath(path), 0) + 1
+ return result
+
+ monkeypatch.setattr(os, "stat", counting_stat)
+ return counts
+
+
+def test_deep_unreported_add_is_admitted(roots, session):
+ dirs, files = _deep_tree(roots["output"])
+ _scan()
+ added = _write(dirs[DEPTH] / "deep_new.png")
+
+ created, _skipped, _total = _scan()
+
+ assert created == 1
+ assert _live_paths(session) == {str(p) for p in files + [added]}
+
+
+def test_deep_deletions_mark_exactly_those_rows_missing(roots, session, caplog):
+ _dirs, files = _deep_tree(roots["output"])
+ _scan()
+ deleted = [files[10], files[15], files[20]] # depths 10, 15 and 20, one per dir
+ for path in deleted:
+ path.unlink()
+
+ _scan_logged(caplog)
+
+ session.expire_all()
+ missing = set(session.scalars(sa.select(AssetContent.path).where(AssetContent.is_missing.is_(True))))
+ assert missing == {str(p) for p in deleted}
+ assert _live_paths(session) == {str(p) for p in files if p not in deleted}
+ _listed, retired, skipped = _listing_counts(caplog)
+ assert (retired, skipped) == (3, 0)
+
+
+def test_removing_a_whole_dir_retires_every_row_beneath_it(roots, session, caplog):
+ dirs, files = _deep_tree(roots["output"])
+ _scan()
+ shutil.rmtree(dirs[5]) # d05 and the 15 levels under it, one file each
+
+ _scan_logged(caplog)
+
+ gone = {str(p) for p in files[5:DEPTH + 1]}
+ assert _live_paths(session) == {str(p) for p in files} - gone
+ _listed, retired, skipped = _listing_counts(caplog)
+ assert (retired, skipped) == (len(gone), 0)
+
+
+def _symlink_or_skip(link: Path, target: Path) -> None:
+ try:
+ link.symlink_to(target)
+ except OSError as e: # Windows without the symlink privilege
+ pytest.skip(f"cannot create symlinks here: {e}")
+
+
+def test_symlink_whose_target_is_removed_is_retired(roots, temp_dir, session):
+ target = _write(temp_dir / "elsewhere" / "t.png")
+ link = roots["output"] / "linked.png"
+ _symlink_or_skip(link, target)
+ kept = _write(roots["output"] / "kept.png")
+ _scan()
+ assert _live_paths(session) == {str(link), str(kept)}
+
+ target.unlink()
+ _scan()
+
+ assert _live_paths(session) == {str(kept)}
+
+
+def test_every_dir_is_stated_and_listed_once_per_rescan(roots, caplog, dir_stats):
+ dirs, _files = _deep_tree(roots["output"])
+ _scan()
+
+ for _ in range(2):
+ dir_stats.clear()
+ _scan_logged(caplog)
+ assert dir_stats == {str(d): 1 for d in dirs}
+ assert _listing_counts(caplog)[0] == len(dirs)
+
+
+def test_the_rescan_yields_the_gil_per_dir_entry_and_row(roots, monkeypatch):
+ dirs, files = _deep_tree(roots["output"])
+ _scan()
+ walk_yields: list[None] = []
+ row_yields: list[None] = []
+ monkeypatch.setattr(file_utils, "yield_gil", lambda run=None: run == file_utils.RESCAN_YIELD_RUN and walk_yields.append(None))
+ monkeypatch.setattr(scanner, "yield_gil", lambda run=None: run == file_utils.RESCAN_YIELD_RUN and row_yields.append(None))
+
+ _scan()
+
+ entries = len(files) + len(dirs) - 1 # every file, and every dir but the root
+ # once per dir walked, per entry listed, and per file path built
+ assert len(walk_yields) == len(dirs) + entries + len(files)
+ assert len(row_yields) == 2 * len(files) # reading the live rows, then diffing them
diff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.py
index f2315bc8b..d92018ded 100644
--- a/tests-unit/test_assets_event_log_static.py
+++ b/tests-unit/test_assets_event_log_static.py
@@ -53,6 +53,10 @@ EXPECTED_CALL_SITES: frozenset[CallSite] = frozenset(
CallSite("app/assets/seeder.py", "_run_fast_phase", "seeder.batch_insert_failed"),
# todo 11 - scanner failure paths
CallSite("app/assets/scanner.py", "sync_root_safely", "scanner.fast_scan_failed"),
+ CallSite("app/assets/scanner.py", "live_references_safely", "scanner.fast_scan_failed"),
+ CallSite(
+ "app/assets/scanner.py", "mark_unlisted_references_missing_safely", "scanner.fast_scan_failed"
+ ),
CallSite("app/assets/scanner.py", "sync_temp_references_safely", "scanner.temp_sync_failed"),
CallSite(
"app/assets/scanner.py", "mark_missing_outside_prefixes_safely", "scanner.mark_missing_failed"