The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat(assets): report scan CPU and paused time, and classify scan failures (#16719)
Simon Pinfold30f15eaf5c34
diff --git a/app/assets/event_log.py b/app/assets/event_log.pyindex 1cbeea6ac..bc88278dc 100644--- a/app/assets/event_log.py+++ b/app/assets/event_log.py@@ -13,8 +13,10 @@ logfmt delimiter or line break — so file names, paths, asset ids and contenthashes cannot ride along."""+import errnoimport loggingimport os+import sqlite3import tracebackfrom collections.abc import Callablefrom typing import Any@@ -28,6 +30,19 @@ ROOTS = frozenset({"models", "input", "output", "user", "temp"})PHASES = frozenset({"fast", "enrich", "full"})STAGES = frozenset({"mark_missing", "pruning", "fast_scan", "enrich", "finalize"})STAT_SITES = frozenset({"discovery", "enrich"})+ERROR_KINDS = frozenset({+ "expression_tree_too_large",+ "too_many_variables",+ "database_locked",+ "disk_full",+ "disk_io",+ "unable_to_open",+ "database_corrupt",+ "permission_denied",+ "file_locked",+ "read_only",+ "other",+})ALLOWED_EVENTS = frozenset({"assets.enabled","seeder.scan_started",@@ -83,6 +98,10 @@ ALLOWED_FIELDS: dict[str, Callable[[Any], bool]] = {"phase": _one_of(PHASES),"stage": _one_of(STAGES),"elapsed_ms": _is_count,+ "cpu_ms": _is_count,+ "paused_ms": _is_count,+ "dirs_listed_count": _is_count,+ "files_statted_count": _is_count,"created": _is_count,"enriched": _is_count,"skipped": _is_count,@@ -93,6 +112,7 @@ ALLOWED_FIELDS: dict[str, Callable[[Any], bool]] = {"recovered_count": _is_count,"count": _is_count,"error_type": _is_safe_string,+ "error_kind": _one_of(ERROR_KINDS),"hashing_enabled": _is_flag,"site": _one_of(STAT_SITES),}@@ -167,3 +187,67 @@ def error_type(exc: BaseException) -> str:and friends embed the path that triggered them."""return type(exc).__name__+++# SQLite primary result codes (sqlite3.Error.sqlite_errorcode & 0xFF, Python 3.11+).+_SQLITE_CODE_KINDS = {+ 5: "database_locked", # SQLITE_BUSY+ 6: "database_locked", # SQLITE_LOCKED+ 8: "read_only", # SQLITE_READONLY+ 10: "disk_io", # SQLITE_IOERR+ 11: "database_corrupt", # SQLITE_CORRUPT+ 13: "disk_full", # SQLITE_FULL+ 14: "unable_to_open", # SQLITE_CANTOPEN+ 26: "database_corrupt", # SQLITE_NOTADB+}+# SQLite's own fixed messages, matched as substrings of the driver exception's first+# argument, which never carries the SQL or its bound parameters (paths). The first two+# share the generic SQLITE_ERROR code, so only the message tells them apart; the rest+# cover Python 3.10, which has no sqlite_errorcode.+_SQLITE_MESSAGE_KINDS = (+ ("expression tree is too large", "expression_tree_too_large"),+ ("too many sql variables", "too_many_variables"),+ ("database is locked", "database_locked"),+ ("database table is locked", "database_locked"),+ ("database or disk is full", "disk_full"),+ ("disk i/o error", "disk_io"),+ ("unable to open database file", "unable_to_open"),+ ("database disk image is malformed", "database_corrupt"),+ ("file is not a database", "database_corrupt"),+ ("attempt to write a readonly database", "read_only"),+)+# Windows reports a file held open by another process (ERROR_SHARING_VIOLATION,+# ERROR_LOCK_VIOLATION) as EACCES; tell it apart from a real permission problem.+_WINERROR_KINDS = {32: "file_locked", 33: "file_locked"}+_ERRNO_KINDS = {+ errno.ENOSPC: "disk_full",+ errno.EIO: "disk_io",+ errno.EACCES: "permission_denied",+ errno.EPERM: "permission_denied",+ errno.EROFS: "read_only",+}+++def error_kind(exc: BaseException) -> str:+ """Classify a failure into :data:`ERROR_KINDS`, without ever emitting its text.++ SQLAlchemy wraps the driver's exception as ``exc.orig``; its str() would carry the+ statement and bound parameters, so only the driver's own message is inspected.+ """+ orig = getattr(exc, "orig", None)+ source = orig if isinstance(orig, BaseException) else exc+ if isinstance(source, sqlite3.Error):+ code = getattr(source, "sqlite_errorcode", None)+ if isinstance(code, int) and (code & 0xFF) in _SQLITE_CODE_KINDS:+ return _SQLITE_CODE_KINDS[code & 0xFF]+ if source.args and isinstance(source.args[0], str):+ message = source.args[0].lower()+ for needle, kind in _SQLITE_MESSAGE_KINDS:+ if needle in message:+ return kind+ if isinstance(source, OSError):+ winerror = getattr(source, "winerror", None)+ if winerror in _WINERROR_KINDS:+ return _WINERROR_KINDS[winerror]+ return _ERRNO_KINDS.get(source.errno, "other")+ return "other"diff --git a/app/assets/scanner.py b/app/assets/scanner.pyindex ec886cb2c..9ed26336c 100644--- a/app/assets/scanner.py+++ b/app/assets/scanner.py@@ -21,7 +21,7 @@ import sqlalchemy as safrom sqlalchemy.exc import IntegrityErrorfrom sqlalchemy.orm import Sessionfrom app.assets import mode-from app.assets.event_log import emit, error_type+from app.assets.event_log import emit, error_kind, error_typefrom app.assets.database.queries import (create_content_reporting_insert,is_live_path_conflict,@@ -93,6 +93,8 @@ class _ScanProgress(Protocol):permission_denied: intmissing_marked: intrecovered: int+ dirs_listed: int+ files_statted: intdef mark_emitted(self, key: str) -> bool: ...@@ -210,6 +212,8 @@ def observe_references_on_filesystem(observations: list[_ReferenceObservation] = []survivors: set[str] = set()for content_id, path, size_bytes, mtime_ns in contents:+ if progress is not None:+ progress.files_statted += 1try:stat_result = os.stat(path, follow_symlinks=True)except (FileNotFoundError, NotADirectoryError):@@ -294,6 +298,7 @@ def sync_root_safely("scanner.fast_scan_failed",root=root,error_type=error_type(exc),+ error_kind=error_kind(exc),)return set()if progress is not None:@@ -313,6 +318,7 @@ def sync_temp_references_safely("scanner.temp_sync_failed",root="temp",error_type=error_type(exc),+ error_kind=error_kind(exc),)@@ -332,6 +338,7 @@ def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int | None:emit("scanner.mark_missing_failed",error_type=error_type(exc),+ error_kind=error_kind(exc),)return None@@ -351,15 +358,22 @@ def mark_contents_missing_outside_prefixes(return len(missing)-def collect_paths_for_roots(roots: tuple[RootType, ...]) -> list[str]:- """Collect all file paths for the given roots."""+def collect_paths_for_roots(+ roots: tuple[RootType, ...], progress: _ScanProgress | None = None+) -> list[str]:+ """Collect all file paths for the given roots.++ ``progress.dirs_listed`` counts the input and output walks only. Models are+ listed through folder_paths.get_filename_list, which walks the model folders+ on a cache miss and re-checks their mtimes on a hit; none of that is counted.+ """paths: list[str] = []if "models" in roots:paths.extend(collect_models_files())if "input" in roots:- paths.extend(list_files_recursively(folder_paths.get_input_directory()))+ paths.extend(list_files_recursively(folder_paths.get_input_directory(), progress))if "output" in roots:- paths.extend(list_files_recursively(folder_paths.get_output_directory()))+ paths.extend(list_files_recursively(folder_paths.get_output_directory(), progress))return paths@@ -404,13 +418,20 @@ def live_references_safely(root: RootType) -> dict[str, list[_ReferenceObservati)except Exception as exc:logging.exception("fast DB scan failed for %s: %s", root, exc)- emit("scanner.fast_scan_failed", root=root, error_type=error_type(exc))+ emit(+ "scanner.fast_scan_failed",+ root=root,+ error_type=error_type(exc),+ error_kind=error_kind(exc),+ )return {}return livedef unlisted_references(- live: dict[str, list[_ReferenceObservation]], listings: DirListings+ live: dict[str, list[_ReferenceObservation]],+ listings: DirListings,+ progress: _ScanProgress | None = None,) -> tuple[list[_ReferenceObservation], int]:"""Split the live rows into (vanished, skipped count).@@ -431,6 +452,8 @@ def unlisted_references(# Stat before retiring. A listing compares names exactly, but a case-insensitive# (NTFS, APFS) or Unicode-normalizing (HFS+) filesystem resolves a stored path# spelled differently from its entry. Rows that reach here are normally few.+ if progress is not None:+ progress.files_statted += 1if _is_gone(path):vanished.extend(observations)else:@@ -497,7 +520,12 @@ def mark_unlisted_references_missing_safely(session.commit()except Exception as exc:logging.exception("fast DB scan failed for %s: %s", root, exc)- emit("scanner.fast_scan_failed", root=root, error_type=error_type(exc))+ emit(+ "scanner.fast_scan_failed",+ root=root,+ error_type=error_type(exc),+ error_kind=error_kind(exc),+ )returnif progress is not None:progress.missing_marked += marked@@ -535,6 +563,8 @@ def build_asset_specs(if abs_p in existing_paths:skipped += 1continue+ if progress is not None:+ progress.files_statted += 1try:stat_p = os.stat(abs_p, follow_symlinks=True)except FileNotFoundError:@@ -545,13 +575,18 @@ def build_asset_specs(if isinstance(e, PermissionError):progress.permission_denied += 1if progress.mark_emitted("stat_failed:discovery"):- emit("scanner.stat_failed", site="discovery", error_type=error_type(e))+ emit(+ "scanner.stat_failed",+ site="discovery",+ error_type=error_type(e),+ error_kind=error_kind(e),+ )continueif not stat_p.st_size:continuecandidates.append((abs_p, stat_p))- admitted_paths, _ = _two_stat_admit(candidates)+ admitted_paths, _ = _two_stat_admit(candidates, progress)candidate_stats = dict(candidates)for abs_p in admitted_paths:yield_gil()@@ -604,7 +639,9 @@ class _SpecObservation(NamedTuple):snapshot: tuple[str, os.stat_result] | None-def observe_asset_specs(specs: list[SeedAssetSpec]) -> dict[str, _SpecObservation | None]:+def observe_asset_specs(+ specs: list[SeedAssetSpec], progress: _ScanProgress | None = None+) -> dict[str, _SpecObservation | None]:"""Stat (and, in hashing mode, hash) each spec before the write transaction opens.``None`` marks a path that vanished or could not be read.@@ -613,6 +650,8 @@ def observe_asset_specs(specs: list[SeedAssetSpec]) -> dict[str, _SpecObservatioobserved: dict[str, _SpecObservation | None] = {}for spec in specs:path = os.path.abspath(spec["abs_path"])+ if progress is not None:+ progress.files_statted += 1try:stat_result = os.stat(path, follow_symlinks=True)snapshot = snapshot_hash(path) if hashing_is_enabled else None@@ -729,7 +768,7 @@ def insert_asset_specs() -> tuple[int, Exception | None]:if not specs:return 0, None- observed = observe_asset_specs(specs)+ observed = observe_asset_specs(specs, progress)missing_ids_by_path = Noneif not mode.hashing_enabled():with create_session() as sess:@@ -852,6 +891,8 @@ def enrich_asset(Returns:Whether enrichment changed the B-schema record or content"""+ if progress is not None:+ progress.files_statted += 1try:stat_p = os.stat(file_path, follow_symlinks=True)except FileNotFoundError:@@ -862,7 +903,12 @@ def enrich_asset(if isinstance(e, PermissionError):progress.permission_denied += 1if progress.mark_emitted("stat_failed:enrich"):- emit("scanner.stat_failed", site="enrich", error_type=error_type(e))+ emit(+ "scanner.stat_failed",+ site="enrich",+ error_type=error_type(e),+ error_kind=error_kind(e),+ )return Falseinitial_mtime_ns = get_mtime_ns(stat_p)diff --git a/app/assets/scanner_admission.py b/app/assets/scanner_admission.pyindex 0fa50ece0..b619ad570 100644--- a/app/assets/scanner_admission.py+++ b/app/assets/scanner_admission.py@@ -13,11 +13,14 @@ import mimetypesimport osimport timefrom dataclasses import dataclass-from typing import Final+from typing import TYPE_CHECKING, Final, Protocol-from app.assets.event_log import emit, error_type+from app.assets.event_log import emit, error_kind, error_typefrom app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path+if TYPE_CHECKING:+ from app.assets.scanner import _ScanProgress+PARTIAL_DOWNLOAD_EXTENSIONS = frozenset({".part", ".partial", ".crdownload", ".download", ".tmp", ".aria2", ".!qb", ".opdownload",})@@ -39,13 +42,21 @@ def _should_skip_extension(path: str) -> bool:return os.path.splitext(path)[1].lower() in PARTIAL_DOWNLOAD_EXTENSIONS-def _two_stat_admit(paths_with_stats: list[tuple[str, os.stat_result]]) -> tuple[list[str], list[str]]:+class _StatCounter(Protocol):+ files_statted: int+++def _two_stat_admit(+ paths_with_stats: list[tuple[str, os.stat_result]], counter: _StatCounter | None = None+) -> tuple[list[str], list[str]]:if not paths_with_stats:return [], []time.sleep(0.1)admitted: list[str] = []watched: list[str] = []for path, first_stat in paths_with_stats:+ if counter is not None:+ counter.files_statted += 1try:second_stat = os.stat(path)except FileNotFoundError:@@ -66,7 +77,7 @@ def _two_stat_admit(paths_with_stats: list[tuple[str, os.stat_result]]) -> tuplereturn admitted, watched-def tick_watch_list() -> None:+def tick_watch_list(progress: _ScanProgress | None = None) -> None:from app.assets.scanner import insert_asset_specs, SeedAssetSpecremaining: list[_WatchEntry] = []@@ -75,10 +86,16 @@ def tick_watch_list() -> None:try:for entry in unvisited:try:+ if progress is not None:+ progress.files_statted += 1current = os.stat(entry.path)except OSError as exc:logging.warning("Dropping watched asset after stat failed: %s", entry.path)- emit("scanner.watch_stat_failed", error_type=error_type(exc))+ emit(+ "scanner.watch_stat_failed",+ error_type=error_type(exc),+ error_kind=error_kind(exc),+ )continueif (current.st_mtime_ns, current.st_size) == (entry.last_stat.st_mtime_ns, entry.last_stat.st_size):try:@@ -106,12 +123,16 @@ def tick_watch_list() -> None:entry.ticks += 1if entry.ticks < _WATCH_SCAN_RETRIES:remaining.append(entry)- _created, seed_error = insert_asset_specs(settled, set())+ _created, seed_error = insert_asset_specs(settled, set(), progress)if seed_error is not None:# The batch reports only its first error, so failed entries can't be named here;# like any settled entry, they leave the watch list either way.logging.warning("Seeding settled watched assets failed for at least one entry")- emit("scanner.watch_seed_failed", error_type=error_type(seed_error))+ emit(+ "scanner.watch_seed_failed",+ error_type=error_type(seed_error),+ error_kind=error_kind(seed_error),+ )finally:# Skipping this write wedges the list: drained entries stay on it and are re-attempted# every tick, while entries past the fault never reach the increment _WATCH_SCAN_RETRIESdiff --git a/app/assets/seeder.py b/app/assets/seeder.pyindex 5b7f68235..3476b90d2 100644--- a/app/assets/seeder.py+++ b/app/assets/seeder.py@@ -14,7 +14,7 @@ from dataclasses import dataclass, field, replacefrom enum import Enumfrom typing import Any, Callable, TypedDict-from app.assets.event_log import emit, error_type+from app.assets.event_log import emit, error_kind, error_typefrom app.assets.scanner import (RootType,build_asset_specs,@@ -119,6 +119,15 @@ class _ScanState:# its own count, and temp retirement is routine, so neither is included.missing_marked: int = 0recovered: int = 0+ # Directories the input/output walks (or the output rescan's listing) listed; the+ # models listing is not counted.+ dirs_listed: int = 0+ # os.stat calls on files in the reference sync, output-listing check, discovery,+ # admission, seed, watch-list and enrich loops. Hashing's own stability stats are+ # not counted.+ files_statted: int = 0+ # Time blocked at the pause gate.+ paused_s: float = 0.0cancel_stage: str | None = None_emitted_keys: set[str] = field(default_factory=set)@@ -543,7 +552,12 @@ class _AssetSeeder:"""if not self._run_gate.is_set():self._emit_event("assets.seed.paused", {})- self._run_gate.wait() # Blocks if paused+ # Re-checked so a pause landing just after the check above still blocks, and is timed.+ if not self._run_gate.is_set():+ t_paused = time.perf_counter()+ self._run_gate.wait() # Blocks until resume or cancel+ if self._scan_state is not None:+ self._scan_state.paused_s += time.perf_counter() - t_pausedcancelled = self._is_cancelled()if cancelled:self._record_cancel_stage(stage)@@ -615,6 +629,8 @@ class _AssetSeeder:def _run_scan(self) -> None:"""Main scan loop running in background thread."""t_start = time.perf_counter()+ # Per-thread CPU clock on Windows, macOS and Linux; excludes time blocked while paused.+ cpu_start = time.thread_time()roots = self._rootsphase = self._phaseroot = roots[0] if len(roots) == 1 else None@@ -712,6 +728,7 @@ class _AssetSeeder:returnelapsed = time.perf_counter() - t_start+ cpu = time.thread_time() - cpu_startlogging.info("Scan(%s, %s) done %.3fs: created=%d enriched=%d skipped=%d",roots,@@ -725,6 +742,10 @@ class _AssetSeeder:"seeder.scan_completed",phase=phase.value,elapsed_ms=round(elapsed * 1000),+ cpu_ms=round(cpu * 1000),+ paused_ms=round(scan_state.paused_s * 1000),+ dirs_listed_count=scan_state.dirs_listed,+ files_statted_count=scan_state.files_statted,created=total_created,enriched=total_enriched,skipped=skipped_existing,@@ -755,6 +776,7 @@ class _AssetSeeder:"seeder.scan_failed",phase=phase.value,error_type=error_type(e),+ error_kind=error_kind(e),root=root,)self._emit_event("assets.seed.error", {"message": str(e)})@@ -845,14 +867,15 @@ class _AssetSeeder:t_collect = time.perf_counter()walk = list_output_for_rescan() if by_listing else None- paths = walk.files if walk is not None else collect_paths_for_roots(roots)+ paths = walk.files if walk is not None else collect_paths_for_roots(roots, scan_state)logging.debug("Fast scan: collect_paths took %.3fs (%d paths found)",time.perf_counter() - t_collect,len(paths),)if walk is not None:- vanished, unlisted = unlisted_references(live_references, walk.listings)+ scan_state.dirs_listed += walk.dirs_listed+ vanished, unlisted = unlisted_references(live_references, walk.listings, scan_state)marked_before = scan_state.missing_markedmark_unlisted_references_missing_safely("output", vanished, scan_state)self._emit_marked_missing("output", scan_state.missing_marked - marked_before)@@ -926,7 +949,11 @@ class _AssetSeeder:i,created,)- emit("seeder.batch_insert_failed", error_type=error_type(e))+ emit(+ "seeder.batch_insert_failed",+ error_type=error_type(e),+ error_kind=error_kind(e),+ )scanned = i + len(batch)now = time.perf_counter()@@ -945,7 +972,7 @@ class _AssetSeeder:last_progress_time = nowself._update_progress(scanned=len(specs), created=total_created)- tick_watch_list()+ tick_watch_list(scan_state)logging.info("Fast scan complete: %.3fs total (created=%d, skipped=%d, total_paths=%d)",time.perf_counter() - t_fast_start,@@ -966,7 +993,7 @@ class _AssetSeeder:with create_session() as session:drain_pending_verifications(session)session.commit()- tick_watch_list()+ tick_watch_list(scan_state)for _ in range(3):drain_transition_queue(session)session.commit()diff --git a/app/assets/services/file_utils.py b/app/assets/services/file_utils.pyindex 4937f0c5d..9d76c74a1 100644--- a/app/assets/services/file_utils.py+++ b/app/assets/services/file_utils.py@@ -1,5 +1,5 @@import os-from typing import NamedTuple+from typing import NamedTuple, Protocolfrom app.assets.services.gil import yield_gil@@ -8,6 +8,10 @@ from app.assets.services.gil import yield_gilRESCAN_YIELD_RUN = 0.010+class _DirListingCounter(Protocol):+ dirs_listed: int++def get_mtime_ns(stat_result: os.stat_result) -> int:"""Extract mtime in nanoseconds from a stat result."""return getattr(@@ -48,8 +52,13 @@ def is_visible(name: str) -> bool:return not name.startswith(".")-def list_files_recursively(base_dir: str) -> list[str]:- """Recursively list all files in a directory, following symlinks."""+def list_files_recursively(+ base_dir: str, counter: _DirListingCounter | None = None+) -> list[str]:+ """Recursively list all files in a directory, following symlinks.++ ``counter.dirs_listed`` gains one per directory os.walk listed.+ """out: list[str] = []base_abs = os.path.abspath(base_dir)if not os.path.isdir(base_abs):@@ -59,6 +68,8 @@ def list_files_recursively(base_dir: str) -> list[str]:for dirpath, subdirs, filenames in os.walk(base_abs, topdown=True, followlinks=True):+ if counter is not None:+ counter.dirs_listed += 1try:st = os.stat(dirpath)dir_id = (st.st_dev, st.st_ino)diff --git a/tests-unit/assets_test/services/test_admission_gate.py b/tests-unit/assets_test/services/test_admission_gate.pyindex 03804af4e..813bcad3c 100644--- a/tests-unit/assets_test/services/test_admission_gate.py+++ b/tests-unit/assets_test/services/test_admission_gate.py@@ -135,7 +135,7 @@ def test_stat_error_drops_entry_and_allows_other_watch_entries_to_commit()assert any(record.getMessage()- == "[assets-event] scanner.watch_stat_failed error_type=PermissionError"+ == "[assets-event] scanner.watch_stat_failed error_kind=other error_type=PermissionError"for record in caplog.records)@@ -151,7 +151,7 @@ def test_seed_failure_does_not_stop_watch_list_drain(_WATCH_LIST[:] = [_WatchEntry(str(path), path.stat()) for path in paths]batches: list[list[str]] = []- def insert_with_one_failure(specs, _tag_pool) -> tuple[int, Exception | None]:+ def insert_with_one_failure(specs, _tag_pool, _progress=None) -> tuple[int, Exception | None]:batches.append([spec["abs_path"] for spec in specs])return 1, RuntimeError("forced watch seed failure")@@ -169,7 +169,7 @@ def test_seed_failure_does_not_stop_watch_list_drain()assert any(record.getMessage()- == "[assets-event] scanner.watch_seed_failed error_type=RuntimeError"+ == "[assets-event] scanner.watch_seed_failed error_kind=other error_type=RuntimeError"for record in caplog.records)@@ -346,6 +346,36 @@ def test_settled_entries_are_seeded_in_one_write_session_batch(temp_dir: Path):tick_watch_list()insert_asset_specs.assert_called_once()- specs, _ = insert_asset_specs.call_args.args+ specs = insert_asset_specs.call_args.args[0]assert [spec["abs_path"] for spec in specs] == [str(path) for path in settled]assert [entry.path for entry in _WATCH_LIST] == [str(moving)]+++def test_two_stat_admit_counts_each_restat_including_a_vanished_file(temp_dir: Path, monkeypatch):+ kept = temp_dir / "kept.bin"+ gone = temp_dir / "gone.bin"+ for path in (kept, gone):+ path.write_bytes(b"x")+ candidates = [(str(kept), kept.stat()), (str(gone), gone.stat())]+ monkeypatch.setattr("app.assets.scanner_admission.time.sleep", lambda _: gone.unlink())+ counter = SimpleNamespace(files_statted=0)++ admitted, _watched = _two_stat_admit(candidates, counter)++ assert admitted == [str(kept)]+ assert counter.files_statted == 2+++def test_tick_watch_list_counts_each_watched_stat(temp_dir: Path):+ watched = temp_dir / "watched.part.bin"+ watched.write_bytes(b"x")+ scanner_admission._WATCH_LIST[:] = [+ scanner_admission._WatchEntry(str(watched), watched.stat()),+ scanner_admission._WatchEntry(str(temp_dir / "gone.bin"), watched.stat()),+ ]+ counter = SimpleNamespace(files_statted=0)++ with patch("app.assets.scanner.insert_asset_specs", return_value=(0, None)):+ tick_watch_list(counter)++ assert counter.files_statted == 2diff --git a/tests-unit/assets_test/services/test_output_rescan.py b/tests-unit/assets_test/services/test_output_rescan.pyindex eeada768c..3ebb73714 100644--- a/tests-unit/assets_test/services/test_output_rescan.py+++ b/tests-unit/assets_test/services/test_output_rescan.py@@ -457,3 +457,17 @@ def test_the_rescan_yields_the_gil_per_dir_entry_and_row(roots, monkeypatch):# once per dir walked, per entry listed, and per file path builtassert len(walk_yields) == len(dirs) + entries + len(files)assert len(row_yields) == 2 * len(files) # reading the live rows, then diffing them+++def test_output_rescan_counts_its_listings_and_row_stats(roots, session):+ files = _warm_catalog(roots["output"])+ _scan()+ files[0].unlink()+ seeder = seeder_module._AssetSeeder()+ seeder._scan_state = state = seeder_module._ScanState()+ seeder._phase = seeder_module.ScanPhase.FAST++ seeder._run_fast_phase(OUTPUT_ONLY)++ assert state.dirs_listed == 4 # output, a, b, b/c+ assert state.files_statted == 1 # only the row its listing lacks is stat'eddiff --git a/tests-unit/assets_test/test_event_log.py b/tests-unit/assets_test/test_event_log.pyindex 410430327..dd946ccd9 100644--- a/tests-unit/assets_test/test_event_log.py+++ b/tests-unit/assets_test/test_event_log.py@@ -1,13 +1,16 @@"""Tests for the structured assets event log lines (``app/assets/event_log.py``)."""+import errnoimport loggingimport re+import sqlite3from pathlib import Pathimport pytest+from sqlalchemy.exc import OperationalErrorfrom app.assets import event_log-from app.assets.event_log import ALLOWED_FIELDS, TAG, EventLogError, emit, error_type+from app.assets.event_log import ALLOWED_FIELDS, ERROR_KINDS, TAG, EventLogError, emit, error_kind, error_type# The line grammar below is the CONTRACT shared with the desktop launcher's log# tap: Comfy-Org/Comfy-Desktop `src/main/lib/assetsTap.ts` holds the equivalent@@ -28,6 +31,10 @@ VALID_VALUES: dict[str, list[object]] = {"phase": ["fast", "enrich", "full"],"stage": ["mark_missing", "pruning", "fast_scan", "enrich", "finalize"],"elapsed_ms": [0, 8123],+ "cpu_ms": [0, 2710],+ "paused_ms": [0, 61250],+ "dirs_listed_count": [0, 42],+ "files_statted_count": [0, 9876],"created": [0, 12],"enriched": [4],"skipped": [3],@@ -38,6 +45,7 @@ VALID_VALUES: dict[str, list[object]] = {"recovered_count": [10],"count": [1],"error_type": ["ValueError", "FileNotFoundError"],+ "error_kind": sorted(ERROR_KINDS),"hashing_enabled": [True, False],"site": ["discovery", "enrich"],}@@ -146,6 +154,101 @@ def test_error_type_is_the_class_name_and_the_path_never_reaches_the_line(caplogassert "model.safetensors" not in line+# --- error_kind ---------------------------------------------------------------------+++def _wrapped(driver_error: BaseException) -> OperationalError:+ # How SQLAlchemy surfaces a driver error: its str() carries the statement and params.+ return OperationalError("SELECT * FROM c WHERE path = ?", ("/home/x/model.safetensors",), driver_error)+++ ("exc", "kind"),+ [+ (_wrapped(sqlite3.OperationalError("Expression tree is too large (maximum depth 1000)")), "expression_tree_too_large"),+ (_wrapped(sqlite3.OperationalError("too many SQL variables")), "too_many_variables"),+ (_wrapped(sqlite3.OperationalError("database is locked")), "database_locked"),+ (_wrapped(sqlite3.OperationalError("database table is locked: assets")), "database_locked"),+ (_wrapped(sqlite3.OperationalError("database or disk is full")), "disk_full"),+ (_wrapped(sqlite3.OperationalError("disk I/O error")), "disk_io"),+ (_wrapped(sqlite3.OperationalError("unable to open database file")), "unable_to_open"),+ (_wrapped(sqlite3.DatabaseError("database disk image is malformed")), "database_corrupt"),+ (sqlite3.OperationalError("database is locked"), "database_locked"),+ (_wrapped(sqlite3.OperationalError("no such table: assets")), "other"),+ (OSError(errno.ENOSPC, "No space left on device", "/home/x/out.png"), "disk_full"),+ (OSError(errno.EIO, "Input/output error"), "disk_io"),+ (PermissionError(errno.EACCES, "Permission denied", "/home/x/out.png"), "permission_denied"),+ (FileNotFoundError("/home/x/model.safetensors"), "other"),+ (ValueError("database is locked"), "other"),+ ],+ ids=[+ "expression-tree", "too-many-variables", "locked", "table-locked", "sqlite-full",+ "sqlite-io", "unable-to-open", "corrupt", "unwrapped-sqlite", "unknown-sqlite",+ "enospc", "eio", "eacces", "no-errno", "not-a-driver-error",+ ],+)+def test_error_kind_classifies_without_reading_the_wrapped_statement(exc, kind):+ assert error_kind(exc) == kind+++def _with_code(exc: sqlite3.Error, code: int) -> sqlite3.Error:+ exc.sqlite_errorcode = code # set by the driver itself on Python 3.11++ return exc+++def _windows_error(winerror: int) -> PermissionError:+ exc = PermissionError(errno.EACCES, "Permission denied")+ exc.winerror = winerror # set by the OS layer on Windows only+ return exc+++ ("exc", "kind"),+ [+ (_wrapped(_with_code(sqlite3.OperationalError("unexpected wording"), 261)), "database_locked"),+ (_wrapped(_with_code(sqlite3.OperationalError("unexpected wording"), 13)), "disk_full"),+ (_wrapped(_with_code(sqlite3.DatabaseError("unexpected wording"), 26)), "database_corrupt"),+ (_wrapped(_with_code(sqlite3.OperationalError("Expression tree is too large"), 1)), "expression_tree_too_large"),+ (_wrapped(sqlite3.DatabaseError("file is not a database")), "database_corrupt"),+ (_wrapped(_with_code(sqlite3.OperationalError("unexpected wording"), 8)), "read_only"),+ (_wrapped(sqlite3.OperationalError("attempt to write a readonly database")), "read_only"),+ (OSError(errno.EROFS, "Read-only file system"), "read_only"),+ (_windows_error(32), "file_locked"),+ (_windows_error(33), "file_locked"),+ (_windows_error(5), "permission_denied"),+ ],+ ids=[+ "busy-extended-code", "full-code", "notadb-code", "generic-code-falls-back-to-message",+ "notadb-message", "readonly-code", "readonly-message", "erofs", "sharing-violation", "lock-violation", "access-denied",+ ],+)+def test_error_kind_prefers_the_sqlite_code_and_windows_error(exc, kind):+ assert error_kind(exc) == kind+++def test_error_kind_classifies_a_real_non_database_file(tmp_path: Path):+ not_a_db = tmp_path / "assets.db"+ not_a_db.write_bytes(b"this is not sqlite" * 100)+ connection = sqlite3.connect(not_a_db)+ try:+ with pytest.raises(sqlite3.DatabaseError) as raised:+ connection.execute("SELECT * FROM sqlite_master")+ finally:+ connection.close()++ assert error_kind(raised.value) == "database_corrupt"+++def test_error_kind_never_carries_the_statement_or_params(caplog):+ exc = _wrapped(sqlite3.OperationalError("database is locked"))+ assert "/home/x/model.safetensors" in str(exc)++ line = emit_line(caplog, "seeder.scan_failed", error_type=error_type(exc), error_kind=error_kind(exc))++ assert "model.safetensors" not in line+ assert "SELECT" not in line++# --- the closed vocabulary ----------------------------------------------------------@@ -190,6 +293,7 @@ def test_a_string_value_carrying_a_forbidden_character_raises(value):("error_type", "x" * 65),("error_type", ""),("error_type", 7),+ ("error_kind", "sqlite_busy"),("elapsed_ms", "8123"),("count", 1.5),("created", True),@@ -206,6 +310,7 @@ def test_a_string_value_carrying_a_forbidden_character_raises(value):"oversized-string","empty-string","non-string-error-type",+ "bad-error-kind","string-into-int-field","float-into-int-field","bool-into-int-field",@@ -282,3 +387,16 @@ def test_production_mode_still_emits_valid_events_after_a_dropped_one(caplog, motagged = [r.getMessage() for r in caplog.records if r.getMessage().startswith(TAG)]assert tagged == ["[assets-event] seeder.scan_started phase=fast"]+++def test_error_kind_classifies_a_real_read_only_database(tmp_path: Path):+ db_path = tmp_path / "assets.db"+ sqlite3.connect(db_path).execute("CREATE TABLE t (x)").connection.close()+ connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)+ try:+ with pytest.raises(sqlite3.OperationalError) as raised:+ connection.execute("INSERT INTO t VALUES (1)")+ finally:+ connection.close()++ assert error_kind(raised.value) == "read_only"diff --git a/tests-unit/assets_test/test_scanner_ingest_event_log.py b/tests-unit/assets_test/test_scanner_ingest_event_log.pyindex b3ae050ce..cd4034dfe 100644--- a/tests-unit/assets_test/test_scanner_ingest_event_log.py+++ b/tests-unit/assets_test/test_scanner_ingest_event_log.py@@ -1,3 +1,4 @@+import errnoimport loggingimport refrom contextlib import nullcontext@@ -86,10 +87,17 @@ def run_hash_failure(session: Mock, path: Path, progress: _ScanState) -> bool:pytest.param(lambda: scanner.sync_root_safely("models"),"scanner.fast_scan_failed",- {"error_type": "FileNotFoundError", "root": "models"},+ {"error_kind": "other", "error_type": "FileNotFoundError", "root": "models"},set(),id="fast-scan",),+ pytest.param(+ lambda: scanner.sync_temp_references_safely(),+ "scanner.temp_sync_failed",+ {"error_kind": "other", "error_type": "FileNotFoundError", "root": "temp"},+ None,+ id="temp-sync",+ ),],)def test_scanner_safe_failures_emit_exception_type_without_path(@@ -258,7 +266,7 @@ def test_locked_files_during_discovery_emit_stat_failed_exactly_once(caplog: pytest.LogCaptureFixture,) -> None:def deny_stat(*_args, **_kwargs):- raise PermissionError("/private/assets/secret.bin")+ raise PermissionError(errno.EACCES, "Permission denied", "/private/assets/secret.bin")monkeypatch.setattr(scanner, "os", SimpleNamespace(stat=deny_stat, path=scanner.os.path))progress = _ScanState()@@ -269,7 +277,7 @@ def test_locked_files_during_discovery_emit_stat_failed_exactly_once(assert specs == []assert events_named(caplog, "scanner.stat_failed") == [- {"error_type": "PermissionError", "site": "discovery"}+ {"error_kind": "permission_denied", "error_type": "PermissionError", "site": "discovery"}]assert progress.permission_denied == 3@@ -341,7 +349,7 @@ def test_locked_files_during_enrichment_emit_stat_failed_exactly_once(assert first is Falseassert second is Falseassert events_named(caplog, "scanner.stat_failed") == [- {"error_type": "PermissionError", "site": "enrich"}+ {"error_kind": "other", "error_type": "PermissionError", "site": "enrich"}]assert progress.permission_denied == 2diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.pyindex 8cda28845..a403170d7 100644--- a/tests-unit/seeder_test/test_seeder.py+++ b/tests-unit/seeder_test/test_seeder.py@@ -1,6 +1,7 @@import loggingimport reimport threading+import timefrom contextlib import contextmanager, nullcontextfrom pathlib import Pathfrom unittest.mock import Mock@@ -95,7 +96,9 @@ def _configure_fast_phase(seeder_module, "sync_root_safely", lambda _root, _progress: set())monkeypatch.setattr(- seeder_module, "collect_paths_for_roots", lambda _roots: [str(path) for path in paths]+ seeder_module,+ "collect_paths_for_roots",+ lambda _roots, _progress=None: [str(path) for path in paths],)monkeypatch.setattr(seeder_module,@@ -104,7 +107,7 @@ def _configure_fast_phase()watch_session = Mock()monkeypatch.setattr(seeder_module, "create_session", lambda: nullcontext(watch_session))- monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None)+ monkeypatch.setattr(seeder_module, "tick_watch_list", lambda _progress=None: None)def _run_faulting_fast_phase(@@ -200,6 +203,8 @@ def test_multi_root_scan_emits_one_started_and_completed_without_root() -> None:clock = iter((10.0, 10.8126))monkeypatch.setattr(seeder_module.time, "perf_counter", lambda: next(clock))+ cpu_clock = iter((2.0, 2.25))+ monkeypatch.setattr(seeder_module.time, "thread_time", lambda: next(cpu_clock))monkeypatch.setattr(scan_seeder, "_run_fast_phase", lambda roots: (3, 2, 5))monkeypatch.setattr(scan_seeder, "_run_enrich_phase", lambda roots: (False, 4))@@ -210,12 +215,16 @@ def test_multi_root_scan_emits_one_started_and_completed_without_root(completed = events_named(caplog, "seeder.scan_completed")assert len(completed) == 1assert completed[0] == {+ "cpu_ms": 250,"created": 3,+ "dirs_listed_count": 0,"elapsed_ms": 813,"enrich_failed": 0,"enriched": 4,+ "files_statted_count": 0,"hash_failed": 0,"missing_marked_count": 0,+ "paused_ms": 0,"permission_denied": 0,"phase": "full","recovered_count": 0,@@ -268,7 +277,7 @@ def test_enrich_phase_does_not_count_returned_ids_as_failures()monkeypatch.setattr(seeder_module, "create_session", lambda: nullcontext(session))monkeypatch.setattr(seeder_module, "drain_pending_verifications", lambda _session: None)- monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None)+ monkeypatch.setattr(seeder_module, "tick_watch_list", lambda _progress=None: None)monkeypatch.setattr(seeder_module, "drain_transition_queue", lambda _session: None)monkeypatch.setattr(seeder_module,@@ -363,7 +372,7 @@ def test_scan_failure_emits_exception_type_without_message(scan_seeder._run_scan()assert events_named(caplog, "seeder.scan_failed") == [- {"error_type": "FileNotFoundError", "phase": "enrich", "root": "models"}+ {"error_kind": "other", "error_type": "FileNotFoundError", "phase": "enrich", "root": "models"}]tagged = "\n".join(record.getMessage() for record in caplog.records if TAG in record.getMessage())assert "/private/models/secret.safetensors" not in tagged@@ -551,7 +560,7 @@ def test_standalone_mark_missing_failure_returns_none_and_emits_no_success_eventassert result is Noneassert events_named(caplog, "scanner.mark_missing_failed") == [- {"error_type": "RuntimeError"}+ {"error_kind": "other", "error_type": "RuntimeError"}]assert events_named(caplog, "seeder.marked_missing") == []@@ -598,7 +607,9 @@ def test_batch_insert_failure_emits_only_the_exception_type(seeder_module, "sync_root_safely", lambda _root, _progress: set())monkeypatch.setattr(- seeder_module, "collect_paths_for_roots", lambda roots: ["asset.safetensors"]+ seeder_module,+ "collect_paths_for_roots",+ lambda roots, progress=None: ["asset.safetensors"],)monkeypatch.setattr(seeder_module,@@ -615,13 +626,13 @@ def test_batch_insert_failure_emits_only_the_exception_type(monkeypatch.setattr(seeder_module, "insert_asset_specs", fail_insert)monkeypatch.setattr(seeder_module, "create_session", lambda: nullcontext(session))- monkeypatch.setattr(seeder_module, "tick_watch_list", lambda: None)+ monkeypatch.setattr(seeder_module, "tick_watch_list", lambda _progress=None: None)with caplog.at_level(logging.INFO):scan_seeder._run_fast_phase(("models",))assert events_named(caplog, "seeder.batch_insert_failed") == [- {"error_type": "PermissionError"}+ {"error_kind": "other", "error_type": "PermissionError"}]tagged = "\n".join(record.getMessage() for record in caplog.records if TAG in record.getMessage())assert "/private/models/asset.safetensors" not in tagged@@ -697,7 +708,7 @@ def test_salvage_commit_failure_reports_the_original_batch_fault("No space left on device"]assert events_named(caplog, "seeder.batch_insert_failed") == [- {"error_type": "OSError"}+ {"error_kind": "other", "error_type": "OSError"}]caller_logs = [record@@ -707,3 +718,196 @@ def test_salvage_commit_failure_reports_the_original_batch_fault(assert len(caller_logs) == 1assert caller_logs[0].exc_info is not Noneassert caller_logs[0].exc_info[1] is original_fault+++# --- scan performance fields on seeder.scan_completed --------------------------------+++def test_scan_completed_reports_the_scan_state_counters(+ scan_seeder: _AssetSeeder,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ scan_seeder._scan_state = _ScanState(dirs_listed=7, files_statted=31, paused_s=1.2344)+ monkeypatch.setattr(scan_seeder, "_run_fast_phase", lambda roots: (0, 0, 0))+ monkeypatch.setattr(scan_seeder, "_run_enrich_phase", lambda roots: (False, 0))++ with caplog.at_level(logging.INFO):+ scan_seeder._run_scan()++ [completed] = events_named(caplog, "seeder.scan_completed")+ assert completed["dirs_listed_count"] == 7+ assert completed["files_statted_count"] == 31+ assert completed["paused_ms"] == 1234+++def test_paused_ms_accumulates_across_pauses(+ scan_seeder: _AssetSeeder,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ pause_s = 0.05++ def fast_phase_paused_twice(roots):+ for _ in range(2):+ assert scan_seeder.pause()+ threading.Timer(pause_s, scan_seeder.resume).start()+ assert scan_seeder._check_pause_and_cancel(_ScanStage.FAST_SCAN) is False+ return 0, 0, 0++ monkeypatch.setattr(scan_seeder, "_run_fast_phase", fast_phase_paused_twice)+ monkeypatch.setattr(scan_seeder, "_run_enrich_phase", lambda roots: (False, 0))++ with caplog.at_level(logging.INFO):+ scan_seeder._run_scan()++ [completed] = events_named(caplog, "seeder.scan_completed")+ # Each wait starts just after its timer, so allow a little under 2 * 50 ms.+ assert completed["paused_ms"] >= 80+ assert completed["paused_ms"] <= completed["elapsed_ms"]+++def test_an_unpaused_checkpoint_adds_no_paused_time(scan_seeder: _AssetSeeder) -> None:+ assert scan_seeder._check_pause_and_cancel(_ScanStage.FAST_SCAN) is False++ assert scan_seeder._scan_state is not None+ assert scan_seeder._scan_state.paused_s == 0.0+++def test_cpu_ms_counts_the_scan_threads_cpu_not_its_sleep(+ scan_seeder: _AssetSeeder,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ def fast_phase_that_spins_then_sleeps(roots):+ spin_until = time.thread_time() + 0.03+ while time.thread_time() < spin_until:+ pass+ time.sleep(0.05)+ return 0, 0, 0++ monkeypatch.setattr(scan_seeder, "_run_fast_phase", fast_phase_that_spins_then_sleeps)+ monkeypatch.setattr(scan_seeder, "_run_enrich_phase", lambda roots: (False, 0))++ with caplog.at_level(logging.INFO):+ scan_seeder._run_scan()++ [completed] = events_named(caplog, "seeder.scan_completed")+ assert completed["cpu_ms"] > 0+ assert completed["cpu_ms"] <= completed["elapsed_ms"]+++def test_dirs_listed_counts_each_directory_the_walk_lists(tmp_path: Path) -> None:+ for relative in ("a/one.png", "a/b/two.png", "c/three.png", ".hidden/four.png"):+ (tmp_path / relative).parent.mkdir(parents=True, exist_ok=True)+ (tmp_path / relative).write_bytes(b"x")+ state = _ScanState()++ files = scanner_module.list_files_recursively(str(tmp_path), state)++ assert len(files) == 3+ assert state.dirs_listed == 4 # root, a, a/b, c; the hidden directory is never listed+++def test_files_statted_counts_discovery_and_admission_stats_only_for_new_files(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(+ scanner_module, "get_name_and_tags_from_asset_path", lambda path: (Path(path).name, ["input"])+ )+ monkeypatch.setattr(scanner_module, "compute_loader_path", lambda path: Path(path).name)+ known = tmp_path / "known.png"+ new = tmp_path / "new.png"+ partial = tmp_path / "download.part"+ for path in (known, new, partial):+ path.write_bytes(b"x")+ state = _ScanState()++ specs, _tags, skipped = scanner_module.build_asset_specs(+ [str(known), str(new), str(partial)],+ existing_paths={str(known)},+ enable_metadata_extraction=False,+ progress=state,+ )++ assert [spec["abs_path"] for spec in specs] == [str(new)]+ assert skipped == 2+ assert state.files_statted == 2 # new.png's discovery stat and its admission re-stat+++def test_files_statted_counts_the_reference_sync_stat_per_live_row(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ present = tmp_path / "present.png"+ present.write_bytes(b"x")+ rows = [+ Mock(id="c1", path=str(present), size_bytes=1, mtime_ns=present.stat().st_mtime_ns),+ Mock(id="c2", path=str(tmp_path / "gone.png"), size_bytes=1, mtime_ns=1),+ ]+ monkeypatch.setattr(scanner_module, "live_contents_under_prefixes", lambda _s, _p: rows)+ state = _ScanState()++ scanner_module.observe_references_on_filesystem(Mock(), [str(tmp_path)], state)++ assert state.files_statted == 2+++def test_files_statted_counts_the_seed_and_enrich_stats(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ asset = tmp_path / "asset.png"+ asset.write_bytes(b"x")+ monkeypatch.setattr(scanner_module.mode, "hashing_enabled", lambda: False)+ state = _ScanState()++ scanner_module.observe_asset_specs([_seed_spec(asset)], state)+ scanner_module.enrich_asset(+ Mock(),+ file_path=str(tmp_path / "vanished.png"),+ content_id="c1",+ record_id="r1",+ progress=state,+ )++ assert state.files_statted == 2+++def test_a_pause_landing_after_the_gate_check_still_blocks_the_checkpoint(+ scan_seeder: _AssetSeeder,+) -> None:+ gate = Mock()+ gate.is_set.side_effect = [True, False] # the pause lands just after the first check+ gate.wait.side_effect = lambda: time.sleep(0.02)+ scan_seeder._run_gate = gate++ assert scan_seeder._check_pause_and_cancel(_ScanStage.FAST_SCAN) is False++ gate.wait.assert_called_once_with()+ assert scan_seeder._scan_state is not None+ assert scan_seeder._scan_state.paused_s >= 0.02+++def test_scan_failure_classifies_a_real_sqlite_expression_tree_error(+ scan_seeder: _AssetSeeder,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ engine = create_engine("sqlite:///:memory:")+ secret_path = "/private/models/secret.safetensors"++ def fail_scan(_roots):+ # One bound path per term; SQLite rejects the expression past depth 1000.+ clause = " OR ".join(["? = 1"] * 1100)+ with engine.connect() as connection:+ connection.exec_driver_sql(f"SELECT 1 WHERE {clause}", tuple([secret_path] * 1100))++ monkeypatch.setattr(scan_seeder, "_run_fast_phase", fail_scan)++ with caplog.at_level(logging.INFO):+ scan_seeder._run_scan()++ [failed] = events_named(caplog, "seeder.scan_failed")+ assert failed["error_type"] == "OperationalError"+ assert failed["error_kind"] == "expression_tree_too_large"+ tagged = "\n".join(record.getMessage() for record in caplog.records if TAG in record.getMessage())+ assert secret_path not in tagged