The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
perf(assets): keep the UI responsive during asset scans and rescan output by listing folders (#16546)
Simon Pinfold56c50055cfed
diff --git a/app/assets/scanner.py b/app/assets/scanner.pyindex cd3083ddf..8340d6b74 100644--- a/app/assets/scanner.py+++ b/app/assets/scanner.py@@ -8,6 +8,7 @@ can end a batch early, and the cursor holds at the last row the batch attempted,so the rows it never reached are selected again when the scan resumes."""+import enumimport loggingimport osfrom dataclasses import dataclass@@ -45,7 +46,16 @@ from app.assets.scanner_admission import (_two_stat_admit,tick_watch_list as tick_watch_list,)-from app.assets.services.file_utils import get_mtime_ns, is_visible, list_files_recursively+from app.assets.services.file_utils import (+ RESCAN_YIELD_RUN,+ DirListings,+ ListingWalk,+ get_mtime_ns,+ is_visible,+ list_files_recursively,+ walk_listings,+)+from app.assets.services.gil import yield_gilfrom app.assets.services.image_dimensions import extract_image_dimensionsfrom app.assets.services.metadata_extract import ExtractedMetadata, extract_file_metadatafrom app.assets.services.path_utils import (@@ -297,7 +307,9 @@ def mark_contents_missing_outside_prefixes(session: Session, prefixes: list[str]) -> int:contents = session.scalars(- sa.select(AssetContent).where(AssetContent.is_missing.is_(False))+ sa.select(AssetContent)+ .where(AssetContent.is_missing.is_(False))+ .execution_options(yield_per=500))is_owned = path_prefix_matcher(prefixes)missing = [content for content in contents if not is_owned(content.path)]@@ -318,6 +330,144 @@ def collect_paths_for_roots(roots: tuple[RootType, ...]) -> list[str]:return paths+def rescans_output_by_listing(roots: tuple[RootType, ...]) -> bool:+ """Whether this scan checks the catalog against directory listings rather than by+ stat'ing every live row. Only output-only scans do: the rescan queued after each prompt.++ The listing diff catches every add and delete, but nothing stats an already-cataloged+ file, so an in-place overwrite (same path; new content, size or mtime) goes undetected+ until the next scan that is not output-only, such as the startup scan. Core save nodes+ never overwrite, and reported outputs are registered at save time, so this only+ affects files written by something else.+ """+ return tuple(roots) == ("output",)+++def live_references_safely(root: RootType) -> dict[str, list[_ReferenceObservation]]:+ """The live rows under ``root`` by path, read without touching the filesystem.++ Each is observed as gone (``stat_result=None``): what apply_reference_observations+ needs to retire it, should its path turn out not to be listed. Empty on failure, as+ sync_root_safely is.+ """+ prefixes = get_scan_prefixes_for_root(root)+ live: dict[str, list[_ReferenceObservation]] = {}+ if not prefixes:+ return live+ stmt = sa.select(+ AssetContent.id, AssetContent.path, AssetContent.size_bytes, AssetContent.mtime_ns+ ).where(+ AssetContent.is_missing.is_(False),+ sa.or_(*(sql_path_under_prefix(AssetContent.path, p) for p in prefixes)),+ )+ try:+ with create_session() as session:+ for content_id, path, size_bytes, mtime_ns in session.execute(stmt):+ yield_gil(run=RESCAN_YIELD_RUN)+ live.setdefault(os.path.abspath(path), []).append(+ _ReferenceObservation(content_id, size_bytes, mtime_ns, None)+ )+ except Exception as exc:+ logging.exception("fast DB scan failed for %s: %s", root, exc)+ emit("scanner.fast_scan_failed", root=root, error_type=error_type(exc))+ return {}+ return live+++def unlisted_references(+ live: dict[str, list[_ReferenceObservation]], listings: DirListings+) -> tuple[list[_ReferenceObservation], int]:+ """Split the live rows into (vanished, skipped count).++ A row its parent's listing names is present, with no further check. Every other row+ is stat'ed, and has vanished only if the stat says the file is gone: that covers a+ name the listing lacks, a removed directory, and the rows no listing can speak for+ (a hidden path, a directory that failed to list, a symlink alias the walk did not+ take). The skipped count is the rows that were stat'ed and kept.+ """+ vanished: list[_ReferenceObservation] = []+ skipped = 0+ names_by_dir: dict[str, set[str]] = {}+ for path, observations in live.items():+ yield_gil(run=RESCAN_YIELD_RUN)+ verdict = _listing_verdict(path, listings, names_by_dir)+ if verdict is ListingVerdict.LISTED:+ continue+ # Stat before retiring. A listing compares names exactly, but a case-insensitive+ # (NTFS, APFS) or Unicode-normalizing (HFS+) filesystem resolves a stored path+ # spelled differently from its entry. Rows that reach here are normally few.+ if _is_gone(path):+ vanished.extend(observations)+ else:+ skipped += len(observations)+ return vanished, skipped+++def _is_gone(path: str) -> bool:+ """True only when stat says the path does not exist. Any other error (permissions,+ I/O) leaves it undecided, and the row stays live, as the per-row stat left it."""+ try:+ os.stat(path)+ except (FileNotFoundError, NotADirectoryError):+ return True+ except OSError:+ return False+ return False+++class ListingVerdict(enum.Enum):+ """What this rescan's directory listings say about a cataloged path."""++ LISTED = "listed" # its parent's listing has the name+ ABSENT = "absent" # the nearest listed directory above it lacks the next component+ UNKNOWN = "unknown" # no listing can say: hidden, no listed ancestor, or not walked+++def _listing_verdict(+ path: str, listings: DirListings, names_by_dir: dict[str, set[str]]+) -> ListingVerdict:+ """Classify ``path`` against the listings; see ListingVerdict. An ancestor that+ still has the directory this walk did not list is UNKNOWN, not LISTED."""+ child, parent = path, os.path.dirname(path)+ while parent not in listings:+ if not is_visible(os.path.basename(child)):+ return ListingVerdict.UNKNOWN+ child, parent = parent, os.path.dirname(parent)+ if parent == child: # reached the filesystem root without meeting a listing+ return ListingVerdict.UNKNOWN+ name = os.path.basename(child)+ if not is_visible(name):+ return ListingVerdict.UNKNOWN+ names = names_by_dir.get(parent)+ if names is None:+ files, subdirs = listings[parent]+ names = names_by_dir[parent] = {*files, *subdirs}+ if name not in names:+ return ListingVerdict.ABSENT+ return ListingVerdict.LISTED if child == path else ListingVerdict.UNKNOWN+++def mark_unlisted_references_missing_safely(+ root: RootType, observations: list[_ReferenceObservation]+) -> None:+ """Retire rows whose file the listing lacks, through the same guarded write+ sync_root applies to a row whose file has vanished."""+ if not observations:+ return+ try:+ with create_write_session() as session:+ apply_reference_observations(session, observations)+ session.commit()+ except Exception as exc:+ logging.exception("fast DB scan failed for %s: %s", root, exc)+ emit("scanner.fast_scan_failed", root=root, error_type=error_type(exc))+++def list_output_for_rescan() -> ListingWalk:+ """Walk the output root, listing every directory."""+ return walk_listings(folder_paths.get_output_directory())++def build_asset_specs(paths: list[str],existing_paths: set[str],@@ -364,6 +514,7 @@ def build_asset_specs(admitted_paths, _ = _two_stat_admit(candidates)candidate_stats = dict(candidates)for abs_p in admitted_paths:+ yield_gil()stat_p = candidate_stats[abs_p]name, tags = get_name_and_tags_from_asset_path(abs_p)rel_fname = compute_loader_path(abs_p)diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.pyindex 19b60b5ff..df257b82c 100644--- a/app/assets/scanner_changes.py+++ b/app/assets/scanner_changes.py@@ -8,6 +8,7 @@ so a restored file can never leave two live rows describing one location.from __future__ import annotationsimport os+from collections.abc import Iterablefrom typing import Literalimport sqlalchemy as sa@@ -202,14 +203,12 @@ def drain_pending_verifications(session: Session, limit: int | None = None) -> ireturn processed-def live_contents_under_prefixes(session: Session, prefixes: list[str]) -> list[AssetContent]:+def live_contents_under_prefixes(session: Session, prefixes: list[str]) -> Iterable[AssetContent]:+ """Stream the live contents under the prefixes in batches; consume it inside the session."""if not prefixes:return []- return list(- session.scalars(- sa.select(AssetContent).where(- AssetContent.is_missing.is_(False),- sa.or_(*(sql_path_under_prefix(AssetContent.path, prefix) for prefix in prefixes)),- )- )+ stmt = sa.select(AssetContent).where(+ AssetContent.is_missing.is_(False),+ sa.or_(*(sql_path_under_prefix(AssetContent.path, prefix) for prefix in prefixes)),)+ return session.scalars(stmt.execution_options(yield_per=500))diff --git a/app/assets/seeder.py b/app/assets/seeder.pyindex 8e6e8ed18..a538f4ba4 100644--- a/app/assets/seeder.py+++ b/app/assets/seeder.py@@ -24,8 +24,13 @@ from app.assets.scanner import (get_scan_prefixes_for_root,get_unenriched_assets_for_roots,insert_asset_specs,+ list_output_for_rescan,+ live_references_safely,mark_missing_outside_prefixes_safely,+ mark_unlisted_references_missing_safely,+ rescans_output_by_listing,sync_root_safely,+ unlisted_references,sync_temp_references_safely,drain_pending_verifications,tick_watch_list,@@ -796,6 +801,8 @@ class _AssetSeeder:total_created = 0skipped_existing = 0+ by_listing = rescans_output_by_listing(roots)+ live_references: dict[str, list] = {}existing_paths: set[str] = set()t_sync = time.perf_counter()assert self._scan_state is not None@@ -803,7 +810,11 @@ class _AssetSeeder:for r in roots:if self._check_pause_and_cancel(_ScanStage.FAST_SCAN):return total_created, skipped_existing, 0- existing_paths.update(sync_root_safely(r, scan_state))+ if by_listing:+ live_references = live_references_safely(r)+ existing_paths.update(live_references)+ else:+ existing_paths.update(sync_root_safely(r, scan_state))logging.debug("Fast scan: sync_root phase took %.3fs (%d existing paths)",time.perf_counter() - t_sync,@@ -814,12 +825,23 @@ class _AssetSeeder:return total_created, skipped_existing, 0t_collect = time.perf_counter()- paths = collect_paths_for_roots(roots)+ walk = list_output_for_rescan() if by_listing else None+ paths = walk.files if walk is not None else collect_paths_for_roots(roots)logging.debug("Fast scan: collect_paths took %.3fs (%d paths found)",time.perf_counter() - t_collect,len(paths),)+ if walk is not None:+ vanished, unlisted = unlisted_references(live_references, walk.listings)+ mark_unlisted_references_missing_safely("output", vanished)+ logging.debug(+ "Fast scan: output listing: %d dirs listed, %d rows retired, "+ "%d rows skipped (not listed, still on disk)",+ walk.dirs_listed,+ len(vanished),+ unlisted,+ )total_paths = len(paths)self._update_progress(total=total_paths)diff --git a/app/assets/services/file_utils.py b/app/assets/services/file_utils.pyindex c47ebe460..4937f0c5d 100644--- a/app/assets/services/file_utils.py+++ b/app/assets/services/file_utils.py@@ -1,4 +1,11 @@import os+from typing import NamedTuple++from app.assets.services.gil import yield_gil++# Longer run window for output rescans: they repeat after prompts, so pausing every+# 2ms would add up to a much slower rescan.+RESCAN_YIELD_RUN = 0.010def get_mtime_ns(stat_result: os.stat_result) -> int:@@ -68,3 +75,73 @@ def list_files_recursively(base_dir: str) -> list[str]:continueout.append(os.path.abspath(os.path.join(dirpath, name)))return out+++# dir path -> (visible file names, visible subdir names)+DirListings = dict[str, tuple[list[str], list[str]]]+++class ListingWalk(NamedTuple):+ files: list[str]+ listings: DirListings+ dirs_listed: int+++def _list_visible_entries(dirpath: str) -> tuple[list[str], list[str]]:+ """One directory's visible (file names, subdir names), classified as os.walk does:+ anything whose is_dir() is false or raises is a file. The exception is a symlink+ whose target is gone: it is left out, so a row for it reads as vanished, as it did+ when the rescan stat'ed every row through the link."""+ files: list[str] = []+ subdirs: list[str] = []+ with os.scandir(dirpath) as entries:+ for entry in entries:+ yield_gil(run=RESCAN_YIELD_RUN)+ if not is_visible(entry.name):+ continue+ try:+ is_dir = entry.is_dir()+ except OSError:+ is_dir = False+ if is_dir:+ subdirs.append(entry.name)+ elif not (entry.is_symlink() and not os.path.exists(entry.path)):+ files.append(entry.name)+ return files, subdirs+++def walk_listings(base_dir: str) -> ListingWalk:+ """list_files_recursively, also returning every directory listing it read.++ Same traversal as the os.walk version (visit order, symlink following, device/inode+ cycle guard, hidden filtering), except each directory is stat'ed before it is listed.+ ``listings`` holds exactly the directories this walk listed, keyed by normalized+ absolute path, so it doubles as the record of which directories the walk can vouch for.+ """+ files: list[str] = []+ listings: DirListings = {}+ # No isdir() precheck, so each directory costs exactly one stat: a root that is+ # missing or not a directory fails its stat or scandir below and yields nothing.+ seen_dirs: set[tuple[int, int]] = set()+ stack = [os.path.abspath(base_dir)]+ while stack:+ yield_gil(run=RESCAN_YIELD_RUN)+ dirpath = stack.pop()+ try:+ st = os.stat(dirpath)+ except OSError:+ continue+ dir_id = (st.st_dev, st.st_ino)+ if dir_id in seen_dirs:+ continue+ try:+ names, subdirs = _list_visible_entries(dirpath)+ except OSError:+ continue+ seen_dirs.add(dir_id)+ listings[dirpath] = (names, subdirs)+ for name in names:+ yield_gil(run=RESCAN_YIELD_RUN)+ files.append(os.path.abspath(os.path.join(dirpath, name)))+ stack.extend(os.path.join(dirpath, name) for name in reversed(subdirs))+ return ListingWalk(files, listings, len(listings))diff --git a/app/assets/services/gil.py b/app/assets/services/gil.pynew file mode 100644index 000000000..77ab3dc79--- /dev/null+++ b/app/assets/services/gil.py@@ -0,0 +1,64 @@+"""Let the event loop run while a scan thread is in a long pure-Python loop.++Without this, the event loop must win the GIL back from the scan after every socket+syscall, and a page load's requests queue behind the scan for seconds. time.sleep(0)+is not enough: the scan thread usually retakes the GIL before the loop wakes.++A thread sleeps _SLEEP after running _RUN, so it spends about a third of its time+asleep. Measured on a 41k-file first scan, that is where page loads stopped getting+faster (a shorter run bought nothing), for about 2% more scan time. All state is per+thread; nothing is shared.+"""++import sys+import threading+import time++_RUN = 0.002+_SLEEP = 0.001+# The coarse timer tick is ~15.6ms, so a 1ms sleep typically takes about 16x as long there.+_MAX_SCALE = 16.0++# Indirection so tests can drive the clock without patching the time module.+_clock = time.perf_counter+_sleep = time.sleep++_state = threading.local()+++def _yield_fixed(run: float = _RUN) -> None:+ """Call once per item in a hot loop on a background thread; sleeps every ``run``+ seconds of work. A caller that repeats often can pass a longer window to pause less."""+ now = _clock()+ next_at = getattr(_state, "next_at", None)+ if next_at is None:+ _state.next_at = now + run+ return+ if now < next_at:+ return+ _sleep(_SLEEP)+ _state.next_at = _clock() + run+++def _yield_scaled(run: float = _RUN) -> None:+ """For a coarse sleep timer: time.sleep(_SLEEP) really takes a whole timer tick, so+ scale the next run by how long the last sleep took, keeping the same share asleep."""+ now = _clock()+ next_at = getattr(_state, "next_at", None)+ if next_at is None:+ _state.next_at = now + run+ return+ if now < next_at:+ return+ _sleep(_SLEEP)+ after = _clock()+ # Scale up to the timer tick this exists for, but no further: a sleep that overshoots+ # because the machine is busy must not buy the scan a longer run.+ scale = min(max(1.0, (after - now) / _SLEEP), _MAX_SCALE)+ _state.next_at = after + run * scale+++# Before Python 3.11, time.sleep on Windows rounds up to the ~15.6ms system timer tick.+_COARSE_SLEEP = sys.platform == "win32" and sys.version_info < (3, 11)++yield_gil = _yield_scaled if _COARSE_SLEEP else _yield_fixeddiff --git a/tests-unit/assets_test/services/test_gil.py b/tests-unit/assets_test/services/test_gil.pynew file mode 100644index 000000000..2affd207e--- /dev/null+++ b/tests-unit/assets_test/services/test_gil.py@@ -0,0 +1,158 @@+import queue+import threading++import pytest++from app.assets.services import gil+++SHARE = gil._SLEEP / (gil._SLEEP + gil._RUN)+++class FakeClock:+ """Stands in for gil._clock / gil._sleep; a sleep advances the clock by `sleep_cost`."""++ def __init__(self, sleep_cost: float) -> None:+ self.now = 100.0+ self.sleep_cost = sleep_cost+ self.slept = 0.0+ self.sleeps = 0++ def clock(self) -> float:+ return self.now++ def sleep(self, _seconds: float) -> None:+ self.now += self.sleep_cost+ self.slept += self.sleep_cost+ self.sleeps += 1+++def fake(monkeypatch):+ def install(sleep_cost: float) -> FakeClock:+ clock = FakeClock(sleep_cost)+ monkeypatch.setattr(gil, "_clock", clock.clock)+ monkeypatch.setattr(gil, "_sleep", clock.sleep)+ monkeypatch.setattr(gil, "_state", threading.local())+ return clock++ return install+++def started(fake, sleep_cost: float, yield_gil) -> FakeClock:+ clock = fake(sleep_cost)+ yield_gil() # starts this thread's first run window+ return clock+++def test_sleeps_only_once_the_run_window_has_passed(fake):+ clock = started(fake, gil._SLEEP, gil._yield_fixed)+ clock.now += gil._RUN / 2+ gil._yield_fixed()+ assert clock.sleeps == 0++ clock.now += gil._RUN+ gil._yield_fixed()+ assert clock.sleeps == 1+++def test_run_window_restarts_after_each_sleep(fake):+ clock = started(fake, gil._SLEEP, gil._yield_fixed)+ clock.now += gil._RUN * 1.5+ gil._yield_fixed()+ gil._yield_fixed()+ assert clock.sleeps == 1+++def run_hot_loop(clock: FakeClock, yield_gil, seconds: float, work_per_item: float = 0.0001) -> float:+ """Simulate a loop doing `work_per_item` per call; return the fraction spent asleep."""+ start = clock.now+ while clock.now - start < seconds:+ clock.now += work_per_item+ yield_gil()+ return clock.slept / (clock.now - start)+++def test_accurate_sleep_keeps_the_measured_duty_cycle(fake):+ clock = started(fake, gil._SLEEP, gil._yield_fixed)+ assert run_hot_loop(clock, gil._yield_fixed, 10.0) == pytest.approx(SHARE, abs=0.02)+++def test_coarse_sleep_widens_the_run_window_to_keep_the_duty_cycle(fake):+ # A 1ms sleep that really takes a 15ms timer tick, as on Windows before Python 3.11.+ clock = started(fake, 0.015, gil._yield_scaled)+ assert run_hot_loop(clock, gil._yield_scaled, 30.0) == pytest.approx(SHARE, abs=0.02)+++def test_slightly_slow_sleep_scales_the_run_window_continuously(fake):+ clock = started(fake, 0.0015, gil._yield_scaled)+ assert run_hot_loop(clock, gil._yield_scaled, 10.0) == pytest.approx(SHARE, abs=0.02)+++def test_sleep_longer_than_the_timer_tick_sleeps_more_rather_than_running_longer(fake):+ # Past the tick the window stops growing, so the scan's share asleep rises instead.+ clock = started(fake, 0.040, gil._yield_scaled)+ assert run_hot_loop(clock, gil._yield_scaled, 60.0) > SHARE + 0.1+++def test_threads_do_not_consume_each_others_run_window(fake):+ clock = started(fake, gil._SLEEP, gil._yield_fixed)+ sleeps_seen: list[int] = []++ def worker(inbox: queue.Queue, done: queue.Queue) -> None:+ while inbox.get():+ before = clock.sleeps+ gil._yield_fixed()+ sleeps_seen.append(clock.sleeps - before)+ done.put(True)++ threads = []+ for _ in range(2):+ inbox, done = queue.Queue(), queue.Queue()+ t = threading.Thread(target=worker, args=(inbox, done))+ t.start()+ threads.append((t, inbox, done))++ def call(i: int) -> int:+ _, inbox, done = threads[i]+ inbox.put(True)+ done.get(timeout=5)+ return sleeps_seen[-1]++ try:+ assert call(0) == 0 and call(1) == 0 # each thread starts its own run window+ clock.now += gil._RUN * 1.5+ assert call(0) == 1 # thread 0 yields and restarts only its own run window+ assert call(1) == 1 # thread 1's run window is untouched, so it yields too+ finally:+ for t, inbox, _ in threads:+ inbox.put(False)+ t.join(timeout=5)+++def test_fixed_window_on_a_coarse_timer_sleeps_far_more_than_a_sixth(fake):+ # Why the scaled version exists: a 1ms sleep that really takes a 15ms tick.+ clock = started(fake, 0.015, gil._yield_fixed)+ assert run_hot_loop(clock, gil._yield_fixed, 30.0) > 2 * SHARE+++def test_scaled_version_is_used_only_for_the_coarse_windows_timer():+ coarse = gil.sys.platform == "win32" and gil.sys.version_info < (3, 11)+ assert gil._COARSE_SLEEP is coarse+ assert gil.yield_gil is (gil._yield_scaled if coarse else gil._yield_fixed)+++def test_contended_sleep_does_not_stretch_the_run_window_past_the_timer_tick(fake):+ # A 1ms sleep that overshoots to 400ms under load must not buy an 800ms run window.+ clock = started(fake, 0.400, gil._yield_scaled)+ clock.now += gil._RUN+ gil._yield_scaled() # sleeps, taking 400ms+ assert gil._state.next_at - clock.now <= gil._RUN * gil._MAX_SCALE + 1e-9++++def test_a_longer_run_window_sleeps_proportionally_less(fake):+ run = gil._RUN * 5+ clock = started(fake, gil._SLEEP, lambda: gil._yield_fixed(run=run))+ share = gil._SLEEP / (gil._SLEEP + run)+ assert run_hot_loop(clock, lambda: gil._yield_fixed(run=run), 10.0) == pytest.approx(share, abs=0.01)diff --git a/tests-unit/assets_test/services/test_output_rescan.py b/tests-unit/assets_test/services/test_output_rescan.pynew file mode 100644index 000000000..eeada768c--- /dev/null+++ b/tests-unit/assets_test/services/test_output_rescan.py@@ -0,0 +1,459 @@+"""Output-only rescans diff directory listings against the catalog instead of stat'ing+every live row. Run through the seeder's real fast phase on an in-memory catalog."""++import logging+import os+import re+import shutil+import stat as stat_module+from contextlib import contextmanager+from pathlib import Path+from unittest.mock import patch++import pytest+import sqlalchemy as sa+from sqlalchemy.orm import Session as SASession, sessionmaker++from app.assets import scanner, seeder as seeder_module+from app.assets.database.models import AssetContent+from app.assets.database.queries.records import create_content, create_record+from app.assets.scanner_admission import _WATCH_LIST+from app.assets.services import file_utils+from app.assets.services.file_utils import list_files_recursively, walk_listings++OUTPUT_ONLY = ("output",)+ALL_ROOTS = ("models", "input", "output")+N_FILES = 30+++def roots(temp_dir: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path]:+ dirs = {name: temp_dir / name for name in ("input", "output")}+ for path in dirs.values():+ path.mkdir()+ monkeypatch.setattr("folder_paths.get_output_directory", lambda: str(dirs["output"]))+ monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(dirs["input"]))+ monkeypatch.setattr(scanner, "get_comfy_models_folders", lambda: [])+ return dirs++[email protected](autouse=True)+def isolated_state(db_engine):+ @contextmanager+ def _create_session():+ with SASession(db_engine) as sess:+ yield sess++ _WATCH_LIST.clear()+ with patch("app.assets.scanner.create_session", _create_session), \+ patch("app.assets.seeder.create_session", _create_session), \+ patch("app.database.db.WriteSession", sessionmaker(bind=db_engine)):+ yield+ _WATCH_LIST.clear()+++def _scan(roots=OUTPUT_ONLY) -> tuple[int, int, int]:+ seeder = seeder_module._AssetSeeder()+ seeder._scan_state = seeder_module._ScanState()+ seeder._phase = seeder_module.ScanPhase.FAST+ seeder._run_gate.set()+ seeder._cancel_event.clear()+ return seeder._run_fast_phase(roots)+++def _write(path: Path, payload: bytes = b"png-bytes") -> Path:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_bytes(payload)+ return path+++def _warm_catalog(output: Path) -> list[Path]:+ """N files over the root and two levels of nesting."""+ subdirs = [output, output / "a", output / "b" / "c"]+ return [_write(subdirs[i % 3] / f"img_{i:03d}.png") for i in range(N_FILES)]+++def _rows(session, path: Path) -> list[AssetContent]:+ session.expire_all()+ return list(session.scalars(sa.select(AssetContent).where(AssetContent.path == str(path))))+++def _live_paths(session) -> set[str]:+ session.expire_all()+ return set(session.scalars(sa.select(AssetContent.path).where(AssetContent.is_missing.is_(False))))+++_LISTING_LINE = re.compile(+ r"output listing: (\d+) dirs listed, (\d+) rows retired, (\d+) rows skipped"+)+++def _listing_counts(caplog: pytest.LogCaptureFixture) -> tuple[int, int, int]:+ """(dirs listed, rows retired, rows skipped) from the one rescan in ``caplog``."""+ matches = [_LISTING_LINE.search(r.getMessage()) for r in caplog.records]+ found = [m for m in matches if m]+ assert len(found) == 1, [r.getMessage() for r in caplog.records]+ listed, retired, skipped = (int(g) for g in found[0].groups())+ return listed, retired, skipped+++def _scan_logged(caplog: pytest.LogCaptureFixture, roots=OUTPUT_ONLY) -> tuple[int, int, int]:+ caplog.clear()+ with caplog.at_level(logging.DEBUG):+ return _scan(roots)+++def test_output_rescan_catalogs_new_files_and_retires_deleted_ones(roots, session, caplog):+ output = roots["output"]+ files = _warm_catalog(output)+ _scan()+ assert _live_paths(session) == {str(p) for p in files}++ deleted = files[:10]+ for path in deleted:+ path.unlink()+ added = [_write(output / ("a" if i % 2 else "b/c") / f"new_{i}.png") for i in range(10)]++ created, _skipped, total = _scan_logged(caplog)++ assert created == 10+ assert total == N_FILES+ assert _live_paths(session) == {str(p) for p in files[10:] + added}+ for path in deleted:+ (row,) = _rows(session, path)+ assert row.is_missing is True+ _listed, retired, skipped = _listing_counts(caplog)+ assert (retired, skipped) == (10, 0)+++def test_in_place_overwrite_is_not_detected_by_an_output_only_rescan(roots, session):+ """The documented, accepted gap: without a per-row stat, a same-path overwrite is+ invisible to an output-only rescan until the next scan that stats."""+ target = _warm_catalog(roots["output"])[0]+ _scan()+ (before,) = _rows(session, target)++ _write(target, b"a different, longer payload")+ _scan()++ (after,) = _rows(session, target)+ assert after.id == before.id+ assert after.is_missing is False+ assert after.size_bytes == before.size_bytes != target.stat().st_size+++def test_a_scan_that_is_not_output_only_still_stats_every_row(roots, session, caplog):+ files = _warm_catalog(roots["output"])+ _write(roots["input"] / "in.png")+ _scan(ALL_ROOTS)+ (before,) = _rows(session, files[0])++ _write(files[0], b"a different, longer payload")+ files[1].unlink()+ _scan_logged(caplog, ALL_ROOTS)++ assert not [r for r in caplog.records if _LISTING_LINE.search(r.getMessage())]+ # The stat path ran: the overwrite split the row, the deletion retired its row.+ assert _rows(session, files[1])[0].is_missing is True+ rows = _rows(session, files[0])+ assert {row.id: row.is_missing for row in rows}[before.id] is True+ assert len(rows) == 2+++def test_walk_listings_matches_the_os_walk_filtering(temp_dir: Path):+ base = temp_dir / "tree"+ _write(base / "a.png")+ _write(base / ".hidden.png")+ _write(base / ".hidden_dir" / "x.png")+ _write(base / "sub" / "empty.png", b"")+ _write(base / "sub" / "dl.png.part")+ _write(base / "sub" / "deep" / "d.png")+ (base / "link_to_sub").symlink_to(base / "sub")+ (base / "sub" / "loop").symlink_to(base)+ (base / "broken").symlink_to(base / "nowhere")+ os.symlink(base / "a.png", base / "file_link.png")++ walk = walk_listings(str(base))++ # The one intended difference: a broken symlink is left out, so its row reads as gone.+ assert walk.files == [p for p in list_files_recursively(str(base)) if p != str(base / "broken")]+ assert walk.dirs_listed == len(walk.listings) == 3 # base, sub, sub/deep+++def _catalog_directly(session, path: Path) -> str:+ """A live row the walk would never produce itself."""+ stat = path.stat()+ content = create_content(session, str(path), None, stat.st_size, stat.st_mtime_ns)+ create_record(session, content.id, path.name)+ session.commit()+ return content.id+++def test_row_whose_stored_spelling_differs_from_the_entry_stays_live(roots, session, monkeypatch):+ # A node that saves to "Portraits/" reuses an existing "portraits/" on NTFS or APFS and+ # reports its own spelling, so the row's path never matches the listing verbatim.+ real = _write(roots["output"] / "portraits" / "img.png")+ respelled = roots["output"] / "Portraits" / "img.png"+ stat = real.stat()+ content = create_content(session, str(respelled), None, stat.st_size, stat.st_mtime_ns)+ create_record(session, content.id, respelled.name)+ session.commit()+ # Resolve either spelling, as a case-insensitive filesystem does, on any host.+ real_stat = os.stat+ monkeypatch.setattr(+ os, "stat",+ lambda p, *a, **k: real_stat(os.fspath(p).replace(str(respelled.parent), str(real.parent)), *a, **k),+ )++ _scan()++ assert session.get(AssetContent, content.id).is_missing is False+++def test_row_the_listing_misses_but_cannot_be_statted_stays_live(roots, session, monkeypatch):+ real = _write(roots["output"] / "portraits" / "img.png")+ respelled = roots["output"] / "Portraits" / "img.png"+ stat = real.stat()+ content = create_content(session, str(respelled), None, stat.st_size, stat.st_mtime_ns)+ create_record(session, content.id, respelled.name)+ session.commit()+ real_stat = os.stat++ def stat_denying_the_respelled_path(path, *args, **kwargs):+ if os.fspath(path) == str(respelled):+ raise PermissionError(13, "Permission denied", os.fspath(path))+ return real_stat(path, *args, **kwargs)++ monkeypatch.setattr(os, "stat", stat_denying_the_respelled_path)++ _scan()++ assert session.get(AssetContent, content.id).is_missing is False+++def test_rows_under_hidden_paths_stay_live(roots, session, caplog):+ output = roots["output"]+ _warm_catalog(output)+ hidden = [_write(output / ".cache" / "x.png"), _write(output / "a" / ".dotfile.png")]+ for path in hidden:+ _catalog_directly(session, path)++ _scan_logged(caplog)++ for path in hidden:+ (row,) = _rows(session, path)+ assert row.is_missing is False+ _listed, retired, skipped = _listing_counts(caplog)+ assert (retired, skipped) == (0, 2)+++def test_dir_failing_to_list_stats_its_rows_while_siblings_are_diffed(+ roots, session, monkeypatch, caplog+):+ output = roots["output"]+ files = _warm_catalog(output)+ _scan()+ in_failing = [p for p in files if p.parent == output / "a"]+ in_sibling = [p for p in files if p.parent == output / "b" / "c"]+ in_failing[0].unlink()+ in_sibling[0].unlink()++ real_list = file_utils._list_visible_entries++ def failing_list(dirpath: str):+ if dirpath == str(output / "a"):+ raise OSError("transient listing failure")+ return real_list(dirpath)++ monkeypatch.setattr(file_utils, "_list_visible_entries", failing_list)+ _scan_logged(caplog)++ for path in in_failing:+ (row,) = _rows(session, path)+ assert row.is_missing is (path == in_failing[0])+ (gone,) = _rows(session, in_sibling[0])+ assert gone.is_missing is True+ _listed, retired, skipped = _listing_counts(caplog)+ assert (retired, skipped) == (2, len(in_failing) - 1)+++def test_deleted_output_under_a_hidden_path_is_retired(roots, session):+ # SaveImage accepts prefixes like "foo/.bar/img" (hidden dir) and "foo/.bar"+ # (hidden file), and the node registers the result like any other output.+ output = roots["output"]+ _warm_catalog(output)+ hidden = [_write(output / "foo" / ".bar" / "img.png"), _write(output / "foo" / ".bar_00001_.png")]+ for path in hidden:+ _catalog_directly(session, path)+ _scan()+ for path in hidden:+ path.unlink()++ _scan()++ for path in hidden:+ (row,) = _rows(session, path)+ assert row.is_missing is True+++def test_unlistable_output_root_stats_every_row(roots, session, monkeypatch):+ output = roots["output"]+ files = _warm_catalog(output)+ _scan()+ files[0].unlink()+ real_list = file_utils._list_visible_entries++ def failing_root(dirpath: str):+ if dirpath == str(output):+ raise OSError("root unreadable")+ return real_list(dirpath)++ monkeypatch.setattr(file_utils, "_list_visible_entries", failing_root)+ _scan()++ assert _live_paths(session) == {str(p) for p in files[1:]}+++def test_row_under_an_unvisited_symlink_alias_is_stat_checked(roots, session):+ output = roots["output"]+ real = _write(output / "real" / "x.png")+ (output / "alias").symlink_to(output / "real")+ _scan()+ (cataloged,) = _live_paths(session)+ # The cycle guard walks the directory once, under whichever name came first.+ visited, unvisited = (+ (real, output / "alias" / "x.png")+ if cataloged == str(real)+ else (output / "alias" / "x.png", real)+ )+ alias_id = _catalog_directly(session, unvisited)++ _scan()++ assert session.get(AssetContent, alias_id).is_missing is False+ assert _live_paths(session) == {str(visited), str(unvisited)}++ real.unlink()+ _scan()++ assert session.get(AssetContent, alias_id).is_missing is True+ assert _live_paths(session) == set()+++DEPTH = 20+++def _deep_tree(output: Path) -> tuple[list[Path], list[Path]]:+ """A chain of DEPTH nested dirs under output with one file per level, plus+ three unchanged sibling dirs at the root. Returns (dirs, files), dirs[0] the root."""+ dirs = [output]+ for level in range(1, DEPTH + 1):+ dirs.append(dirs[-1] / f"d{level:02d}")+ dirs += [output / f"sib{i}" for i in range(3)]+ files = [_write(d / f"f_{i:02d}.png") for i, d in enumerate(dirs)]+ return dirs, files+++def dir_stats(monkeypatch: pytest.MonkeyPatch):+ """Counts os.stat calls per directory path; files are not counted."""+ counts: dict[str, int] = {}+ real_stat = os.stat++ def counting_stat(path, *args, **kwargs):+ result = real_stat(path, *args, **kwargs)+ if stat_module.S_ISDIR(result.st_mode):+ counts[os.fspath(path)] = counts.get(os.fspath(path), 0) + 1+ return result++ monkeypatch.setattr(os, "stat", counting_stat)+ return counts+++def test_deep_unreported_add_is_admitted(roots, session):+ dirs, files = _deep_tree(roots["output"])+ _scan()+ added = _write(dirs[DEPTH] / "deep_new.png")++ created, _skipped, _total = _scan()++ assert created == 1+ assert _live_paths(session) == {str(p) for p in files + [added]}+++def test_deep_deletions_mark_exactly_those_rows_missing(roots, session, caplog):+ _dirs, files = _deep_tree(roots["output"])+ _scan()+ deleted = [files[10], files[15], files[20]] # depths 10, 15 and 20, one per dir+ for path in deleted:+ path.unlink()++ _scan_logged(caplog)++ session.expire_all()+ missing = set(session.scalars(sa.select(AssetContent.path).where(AssetContent.is_missing.is_(True))))+ assert missing == {str(p) for p in deleted}+ assert _live_paths(session) == {str(p) for p in files if p not in deleted}+ _listed, retired, skipped = _listing_counts(caplog)+ assert (retired, skipped) == (3, 0)+++def test_removing_a_whole_dir_retires_every_row_beneath_it(roots, session, caplog):+ dirs, files = _deep_tree(roots["output"])+ _scan()+ shutil.rmtree(dirs[5]) # d05 and the 15 levels under it, one file each++ _scan_logged(caplog)++ gone = {str(p) for p in files[5:DEPTH + 1]}+ assert _live_paths(session) == {str(p) for p in files} - gone+ _listed, retired, skipped = _listing_counts(caplog)+ assert (retired, skipped) == (len(gone), 0)+++def _symlink_or_skip(link: Path, target: Path) -> None:+ try:+ link.symlink_to(target)+ except OSError as e: # Windows without the symlink privilege+ pytest.skip(f"cannot create symlinks here: {e}")+++def test_symlink_whose_target_is_removed_is_retired(roots, temp_dir, session):+ target = _write(temp_dir / "elsewhere" / "t.png")+ link = roots["output"] / "linked.png"+ _symlink_or_skip(link, target)+ kept = _write(roots["output"] / "kept.png")+ _scan()+ assert _live_paths(session) == {str(link), str(kept)}++ target.unlink()+ _scan()++ assert _live_paths(session) == {str(kept)}+++def test_every_dir_is_stated_and_listed_once_per_rescan(roots, caplog, dir_stats):+ dirs, _files = _deep_tree(roots["output"])+ _scan()++ for _ in range(2):+ dir_stats.clear()+ _scan_logged(caplog)+ assert dir_stats == {str(d): 1 for d in dirs}+ assert _listing_counts(caplog)[0] == len(dirs)+++def test_the_rescan_yields_the_gil_per_dir_entry_and_row(roots, monkeypatch):+ dirs, files = _deep_tree(roots["output"])+ _scan()+ walk_yields: list[None] = []+ row_yields: list[None] = []+ monkeypatch.setattr(file_utils, "yield_gil", lambda run=None: run == file_utils.RESCAN_YIELD_RUN and walk_yields.append(None))+ monkeypatch.setattr(scanner, "yield_gil", lambda run=None: run == file_utils.RESCAN_YIELD_RUN and row_yields.append(None))++ _scan()++ entries = len(files) + len(dirs) - 1 # every file, and every dir but the root+ # once per dir walked, per entry listed, and per file path built+ assert len(walk_yields) == len(dirs) + entries + len(files)+ assert len(row_yields) == 2 * len(files) # reading the live rows, then diffing themdiff --git a/tests-unit/test_assets_event_log_static.py b/tests-unit/test_assets_event_log_static.pyindex f2315bc8b..d92018ded 100644--- a/tests-unit/test_assets_event_log_static.py+++ b/tests-unit/test_assets_event_log_static.py@@ -53,6 +53,10 @@ EXPECTED_CALL_SITES: frozenset[CallSite] = frozenset(CallSite("app/assets/seeder.py", "_run_fast_phase", "seeder.batch_insert_failed"),# todo 11 - scanner failure pathsCallSite("app/assets/scanner.py", "sync_root_safely", "scanner.fast_scan_failed"),+ CallSite("app/assets/scanner.py", "live_references_safely", "scanner.fast_scan_failed"),+ CallSite(+ "app/assets/scanner.py", "mark_unlisted_references_missing_safely", "scanner.fast_scan_failed"+ ),CallSite("app/assets/scanner.py", "sync_temp_references_safely", "scanner.temp_sync_failed"),CallSite("app/assets/scanner.py", "mark_missing_outside_prefixes_safely", "scanner.mark_missing_failed"