Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

fix(assets): write the prune and offline marking in short batches so saves aren't locked out (#16696)

Simon Pinfolde9027f2b30f3
diff --git a/app/assets/api/routes.py b/app/assets/api/routes.py
index b6fbfe131..c7fc7022b 100644
--- a/app/assets/api/routes.py
+++ b/app/assets/api/routes.py
@@ -43,7 +43,7 @@ from app.assets.database.queries.records import (
get_preview_file_paths_by_ids,
list_records_page,
)
-from app.assets.seeder import ScanInProgressError, asset_seeder
+from app.assets.seeder import PruneCancelledError, ScanInProgressError, asset_seeder
from app.assets.services import (
DependencyMissingError,
HashMismatchError,
@@ -1030,6 +1030,9 @@ async def get_tags_refine(request: web.Request) -> web.Response:
return web.json_response(payload.model_dump(mode="json", exclude_none=True), status=200)
+_PRUNE_POLL_SECONDS = 0.25
+
+
@ROUTES.post("/api/assets/seed")
@_require_assets_feature_enabled
async def seed_assets(request: web.Request) -> web.Response:
@@ -1059,6 +1062,18 @@ async def seed_assets(request: web.Request) -> web.Response:
started = asset_seeder.start(
roots=valid_roots, compute_hashes=mode.hashing_enabled()
)
+ # A prune from POST /api/assets/prune is not a scan and emits no scan events, so
+ # wait it out rather than answer 409, which a client takes as "a scan is coming".
+ # Polled on the loop: no executor thread is held for the prune's length. The
+ # retry also covers a prune that ended, or another that began, after start() failed.
+ for _ in range(2):
+ if started:
+ break
+ while asset_seeder.standalone_prune_running():
+ await asyncio.sleep(_PRUNE_POLL_SECONDS)
+ started = asset_seeder.start(
+ roots=valid_roots, compute_hashes=mode.hashing_enabled()
+ )
if not started:
return web.json_response({"status": "already_running"}, status=409)
@@ -1125,17 +1140,21 @@ async def mark_missing_assets(request: web.Request) -> web.Response:
Returns:
200 OK with count of marked assets
+ 200 OK with status "cancelled" and the count marked before a cancel stopped it
409 Conflict if a scan is currently running
500 Internal Server Error with PRUNE_FAILED if the marking failed, so a
prune that did not run is never reported as a completed one
"""
try:
- marked = asset_seeder.mark_missing_outside_prefixes()
+ # Off the event loop: a large prune takes seconds to minutes.
+ marked = await asyncio.to_thread(asset_seeder.mark_missing_outside_prefixes)
except ScanInProgressError:
return web.json_response(
{"status": "scan_running", "marked": 0},
status=409,
)
+ except PruneCancelledError as cancelled:
+ return web.json_response({"status": "cancelled", "marked": cancelled.marked}, status=200)
if marked is None:
return _build_error_response(
500,
diff --git a/app/assets/database/queries/__init__.py b/app/assets/database/queries/__init__.py
index 14dff90a8..7293e6f40 100644
--- a/app/assets/database/queries/__init__.py
+++ b/app/assets/database/queries/__init__.py
@@ -16,6 +16,7 @@ from app.assets.database.queries.records import (
is_live_path_conflict,
list_records_page,
mark_content_missing,
+ mark_contents_missing,
rename_record,
unset_content_missing,
update_record_access_time,
@@ -31,6 +32,7 @@ __all__ = [
"is_live_path_conflict",
"list_records_page",
"mark_content_missing",
+ "mark_contents_missing",
"rename_record",
"unset_content_missing",
"update_record_access_time",
diff --git a/app/assets/database/queries/records.py b/app/assets/database/queries/records.py
index bb12e2315..19dec16cf 100644
--- a/app/assets/database/queries/records.py
+++ b/app/assets/database/queries/records.py
@@ -325,6 +325,33 @@ def mark_content_missing(session: Session, content_id: str) -> None:
session.flush()
+def mark_contents_missing(session: Session, content_ids: Sequence[str]) -> list[str]:
+ """mark_content_missing for many rows in a few statements; returns the ids it marked.
+ A row that is gone or already missing is skipped."""
+ if not content_ids:
+ return []
+ # "= 0", not "IS 0": SQLite only uses the partial live-path index for "= 0". This
+ # lookup is by primary key either way; the form matches the other live-row lookups.
+ live = list(session.scalars(sa.select(AssetContent.id).where(AssetContent.id.in_(content_ids), AssetContent.is_missing == sa.false())))
+ if not live:
+ return []
+ session.execute(sa.update(AssetContent).where(AssetContent.id.in_(live)).values(is_missing=True))
+ ensure_tag(session, "missing")
+ unlinked = sa.select(Asset.id, sa.literal("missing"), sa.literal("automatic"), sa.literal(get_utc_now(), sa.DateTime())).where(
+ Asset.content_id.in_(live),
+ ~sa.exists().where(AssetTag.asset_id == Asset.id, AssetTag.tag_name == "missing"),
+ )
+ try:
+ with session.begin_nested():
+ session.execute(sa.insert(AssetTag).from_select(["asset_id", "tag_name", "origin", "added_at"], unlinked))
+ except IntegrityError:
+ # A concurrent writer linked one of them first; settle each link the race-safe way.
+ for record_id in session.scalars(sa.select(Asset.id).where(Asset.content_id.in_(live))):
+ ensure_tag_link(session, asset_id=record_id, tag_name="missing", origin="automatic")
+ session.flush()
+ return live
+
+
def unset_content_missing(session: Session, content_id: str) -> None:
content = session.get(AssetContent, content_id)
if content is None:
diff --git a/app/assets/scanner.py b/app/assets/scanner.py
index 9ed26336c..193f515c4 100644
--- a/app/assets/scanner.py
+++ b/app/assets/scanner.py
@@ -11,6 +11,7 @@ so the rows it never reached are selected again when the scan resumes.
import enum
import logging
import os
+import time
from dataclasses import dataclass
from itertools import islice
from pathlib import Path
@@ -25,7 +26,7 @@ from app.assets.event_log import emit, error_kind, error_type
from app.assets.database.queries import (
create_content_reporting_insert,
is_live_path_conflict,
- mark_content_missing,
+ mark_contents_missing,
create_record,
)
from app.assets.database.models import Asset, AssetContent
@@ -134,6 +135,46 @@ class _ReferenceObservation(NamedTuple):
stat_result: os.stat_result | None
+# Before each write batch: blocks while the scan is paused, returns True to stop.
+ShouldStop = Callable[[], bool]
+
+
+def _never_stop() -> bool:
+ return False
+
+
+# The marking steps write in batches of WRITE_BATCH_ROWS, each its own transaction, so
+# the write lock is never held for a whole pass and a foreground write (an output being
+# registered) waits at most one batch. After each commit the thread sleeps about as
+# long as it held the lock: a writer in SQLite's busy handler polls with growing sleeps
+# (up to 100 ms), so taking the lock straight back would keep winning it.
+WRITE_BATCH_ROWS = 256
+WRITE_YIELD_MIN_SECONDS = 0.02
+WRITE_YIELD_MAX_SECONDS = 0.1
+
+
+def _write_in_batches(
+ items: list,
+ write_batch: Callable[[Session, list], int],
+ should_stop: ShouldStop,
+ committed: list[int],
+) -> None:
+ """Apply ``write_batch`` to ``items`` a batch per transaction, appending the count
+ each batch reports to ``committed`` once it commits. A failure leaves the batches
+ before it committed, and ``committed`` says how much they wrote."""
+ for start in range(0, len(items), WRITE_BATCH_ROWS):
+ if should_stop():
+ break
+ opened = time.perf_counter()
+ with create_write_session() as session:
+ count = write_batch(session, items[start : start + WRITE_BATCH_ROWS])
+ session.commit()
+ committed.append(count)
+ held = time.perf_counter() - opened
+ if start + WRITE_BATCH_ROWS < len(items):
+ time.sleep(min(max(held, WRITE_YIELD_MIN_SECONDS), WRITE_YIELD_MAX_SECONDS))
+
+
def _log_scan_error(phase: str, error: OSError) -> None:
error_type = (
"permission_denied" if isinstance(error, PermissionError) else "os_error"
@@ -241,7 +282,9 @@ def apply_reference_observations(
session: Session, observations: list[_ReferenceObservation]
) -> int:
"""Apply the observations; returns how many rows were marked missing."""
- marked = 0
+ # One query loads the rows, so the session.get calls below never go to the database.
+ list(session.scalars(sa.select(AssetContent).where(AssetContent.id.in_([o.content_id for o in observations]))))
+ gone: list[str] = []
for observation in observations:
content = session.get(AssetContent, observation.content_id)
# Skip a row another writer changed since it was observed; the next scan sees it afresh.
@@ -253,8 +296,7 @@ def apply_reference_observations(
):
continue
if observation.stat_result is None:
- mark_content_missing(session, content.id)
- marked += 1
+ gone.append(content.id)
continue
detect_content_change(
session,
@@ -262,35 +304,38 @@ def apply_reference_observations(
observation.stat_result,
hashing_is_enabled=mode.hashing_enabled(),
)
- return marked
+ return len(mark_contents_missing(session, gone))
-def _sync_prefixes_in_write_txn(
- prefixes: list[str], progress: _ScanProgress | None
-) -> tuple[set[str], int]:
- """Returns the surviving paths and how many rows were marked missing."""
+def _sync_prefixes(
+ prefixes: list[str],
+ progress: _ScanProgress | None,
+ should_stop: ShouldStop,
+ marked: list[int],
+) -> set[str]:
+ """Returns the surviving paths; ``marked`` gets each committed batch's count of rows
+ marked missing, which a caller still has if a later batch raises."""
with create_session() as session:
observations, survivors = observe_references_on_filesystem(
session, prefixes, progress
)
- marked = 0
- if observations:
- with create_write_session() as session:
- marked = apply_reference_observations(session, observations)
- session.commit()
- return survivors, marked
+ _write_in_batches(observations, apply_reference_observations, should_stop, marked)
+ return survivors
def sync_root_safely(
- root: RootType, progress: _ScanProgress | None = None
+ root: RootType,
+ progress: _ScanProgress | None = None,
+ should_stop: ShouldStop = _never_stop,
) -> set[str]:
"""Sync a single root's references with the filesystem.
Returns survivors (existing paths) or empty set on failure.
"""
+ marked: list[int] = []
try:
- survivors, marked = _sync_prefixes_in_write_txn(
- get_scan_prefixes_for_root(root), progress
+ survivors = _sync_prefixes(
+ get_scan_prefixes_for_root(root), progress, should_stop, marked
)
except Exception as exc:
logging.exception("fast DB scan failed for %s: %s", root, exc)
@@ -300,18 +345,19 @@ def sync_root_safely(
error_type=error_type(exc),
error_kind=error_kind(exc),
)
- return set()
+ survivors = set()
if progress is not None:
- progress.missing_marked += marked
+ progress.missing_marked += sum(marked)
return survivors
def sync_temp_references_safely(
progress: _ScanProgress | None = None,
+ should_stop: ShouldStop = _never_stop,
) -> None:
"""Retire temp references whose file is gone; temp is never scanned, so nothing else stats them."""
try:
- _sync_prefixes_in_write_txn(get_temp_prefixes(), progress)
+ _sync_prefixes(get_temp_prefixes(), progress, should_stop, [])
except Exception as exc:
logging.exception("temp reference sync failed: %s", exc)
emit(
@@ -322,19 +368,27 @@ def sync_temp_references_safely(
)
-def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int | None:
+def mark_missing_outside_prefixes_safely(
+ prefixes: list[str], should_stop: ShouldStop = _never_stop
+) -> int | None:
"""Mark references as missing when outside the given prefixes.
This is a non-destructive soft-delete. Returns the count marked, or None when
- the operation fails.
+ the operation fails; batches committed before a failure stay committed.
"""
+ marked: list[int] = []
try:
with create_session() as sess:
- count = mark_contents_missing_outside_prefixes(sess, prefixes)
- sess.commit()
- return count
+ content_ids = content_ids_outside_prefixes(sess, prefixes)
+ _write_in_batches(
+ content_ids,
+ lambda session, batch: len(mark_contents_missing(session, batch)),
+ should_stop,
+ marked,
+ )
+ return sum(marked)
except Exception as exc:
- logging.exception("marking missing assets failed: %s", exc)
+ logging.exception("marking missing assets failed after marking %d: %s", sum(marked), exc)
emit(
"scanner.mark_missing_failed",
error_type=error_type(exc),
@@ -343,19 +397,16 @@ def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int | None:
return None
-def mark_contents_missing_outside_prefixes(
- session: Session, prefixes: list[str]
-) -> int:
- contents = session.scalars(
- sa.select(AssetContent)
- .where(AssetContent.is_missing.is_(False))
+def content_ids_outside_prefixes(session: Session, prefixes: list[str]) -> list[str]:
+ """The live rows outside every prefix. A read: the marking re-checks each is still
+ live inside its own write transaction."""
+ is_owned = path_prefix_matcher(prefixes)
+ rows = session.execute(
+ sa.select(AssetContent.id, AssetContent.path)
+ .where(AssetContent.is_missing == sa.false())
.execution_options(yield_per=500)
)
- is_owned = path_prefix_matcher(prefixes)
- missing = [content for content in contents if not is_owned(content.path)]
- for content in missing:
- mark_content_missing(session, content.id)
- return len(missing)
+ return [content_id for content_id, path in rows if not is_owned(path)]
def collect_paths_for_roots(
@@ -509,15 +560,13 @@ def mark_unlisted_references_missing_safely(
root: RootType,
observations: list[_ReferenceObservation],
progress: _ScanProgress | None = None,
+ should_stop: ShouldStop = _never_stop,
) -> None:
"""Retire rows whose file the listing lacks, through the same guarded write
sync_root applies to a row whose file has vanished."""
- if not observations:
- return
+ marked: list[int] = []
try:
- with create_write_session() as session:
- marked = apply_reference_observations(session, observations)
- session.commit()
+ _write_in_batches(observations, apply_reference_observations, should_stop, marked)
except Exception as exc:
logging.exception("fast DB scan failed for %s: %s", root, exc)
emit(
@@ -526,9 +575,8 @@ def mark_unlisted_references_missing_safely(
error_type=error_type(exc),
error_kind=error_kind(exc),
)
- return
if progress is not None:
- progress.missing_marked += marked
+ progress.missing_marked += sum(marked)
def list_output_for_rescan() -> ListingWalk:
diff --git a/app/assets/seeder.py b/app/assets/seeder.py
index 3476b90d2..29b444f7d 100644
--- a/app/assets/seeder.py
+++ b/app/assets/seeder.py
@@ -43,6 +43,15 @@ class ScanInProgressError(Exception):
"""Raised when an operation cannot proceed because a scan is running."""
+class PruneCancelledError(Exception):
+ """A standalone prune stopped by a cancel. The batches before it stay committed;
+ ``marked`` counts them."""
+
+ def __init__(self, marked: int) -> None:
+ super().__init__(f"prune cancelled after marking {marked}")
+ self.marked = marked
+
+
class State(Enum):
"""Seeder state machine states."""
@@ -172,6 +181,11 @@ class _AssetSeeder:
self._cancel_event = threading.Event()
self._run_gate = threading.Event()
self._run_gate.set() # Start unpaused (set = running, clear = paused)
+ # Clear while a standalone prune holds the seeder (set = no prune running).
+ self._prune_idle = threading.Event()
+ self._prune_idle.set()
+ # Set by shutdown(): a standalone prune that has not started by then does not.
+ self._shutting_down = False
self._roots: tuple[RootType, ...] = ()
self._phase: ScanPhase = ScanPhase.FULL
self._compute_hashes: bool = False
@@ -447,8 +461,14 @@ class _AssetSeeder:
Returns:
True if the scan thread joined cleanly; False on timeout.
"""
+ with self._lock:
+ self._shutting_down = True
self.cancel()
+ # A standalone prune stops at its next batch once cancelled. One deadline
+ # covers both waits.
+ deadline = time.monotonic() + timeout
joined = self.wait(timeout=timeout)
+ joined = self.wait_for_standalone_prune(max(0.0, deadline - time.monotonic())) and joined
if not joined:
logging.warning(
"Asset seeder thread did not exit within %ss",
@@ -480,13 +500,18 @@ class _AssetSeeder:
Raises:
ScanInProgressError: If a scan is currently running
+ PruneCancelledError: If a cancel stopped it part way
"""
with self._lock:
if self._state != State.IDLE:
raise ScanInProgressError(
"Cannot mark missing assets while scan is running"
)
+ if self._shutting_down:
+ raise PruneCancelledError(0)
self._state = State.RUNNING
+ self._cancel_event.clear()
+ self._prune_idle.clear()
try:
if not dependencies_available():
@@ -496,7 +521,16 @@ class _AssetSeeder:
return 0
all_prefixes = get_owned_prefixes()
- marked = mark_missing_outside_prefixes_safely(all_prefixes)
+ # Not pausable (the API waits on it), but a cancel or shutdown stops it
+ # between batches: it runs on a worker thread that exit would wait for.
+ stopped = False
+
+ def should_stop() -> bool:
+ nonlocal stopped
+ stopped = self._cancel_event.is_set()
+ return stopped
+
+ marked = mark_missing_outside_prefixes_safely(all_prefixes, should_stop)
if marked is None:
return None
emit(
@@ -504,12 +538,32 @@ class _AssetSeeder:
count=marked,
stage=_ScanStage.MARK_MISSING.value,
)
+ if stopped:
+ logging.info("Marking missing assets cancelled after marking %d", marked)
+ raise PruneCancelledError(marked)
if marked > 0:
logging.info("Marked %d references as missing", marked)
return marked
finally:
+ # The API runs this off the event loop, so a prompt can finish meanwhile
+ # and queue its output rescan; start it now. Not after a cancel: shutdown
+ # cancels, and a scan started here would run on into teardown. It stays
+ # queued for the next scan to start.
with self._lock:
- self._reset_to_idle()
+ try:
+ if self._cancel_event.is_set():
+ self._reset_to_idle()
+ else:
+ self._finish_and_start_pending()
+ finally:
+ self._prune_idle.set()
+
+ def standalone_prune_running(self) -> bool:
+ return not self._prune_idle.is_set()
+
+ def wait_for_standalone_prune(self, timeout: float | None = None) -> bool:
+ """Block until no standalone prune holds the seeder. True unless it timed out."""
+ return self._prune_idle.wait(timeout)
def _reset_to_idle(self) -> None:
"""Reset state to IDLE, preserving last progress. Caller must hold _lock."""
@@ -655,11 +709,13 @@ class _AssetSeeder:
if self._prune_first:
all_prefixes = get_owned_prefixes()
- marked = mark_missing_outside_prefixes_safely(all_prefixes)
+ marked = mark_missing_outside_prefixes_safely(
+ all_prefixes, lambda: self._check_pause_and_cancel(_ScanStage.PRUNING)
+ )
marked_count = 0 if marked is None else marked
if marked is None:
self._add_error(
- "Marking missing assets failed; scan continued without pruning"
+ "Marking missing assets failed; scan continued with the prune incomplete"
)
else:
emit(
@@ -671,7 +727,9 @@ class _AssetSeeder:
logging.info(
"Marked %d refs as missing before scan", marked_count
)
- sync_temp_references_safely(scan_state)
+ sync_temp_references_safely(
+ scan_state, lambda: self._check_pause_and_cancel(_ScanStage.PRUNING)
+ )
if self._check_pause_and_cancel(_ScanStage.PRUNING):
logging.info("Asset scan cancelled after pruning phase")
@@ -801,23 +859,28 @@ class _AssetSeeder:
)
finally:
with self._lock:
- start_paused = self._state is State.PAUSED
- self._reset_to_idle()
- pending = self._pending_scan
- if pending is not None:
- self._pending_scan = None
- if not self.start(
- roots=pending["roots"],
- phase=pending["phase"],
- prune_first=False,
- compute_hashes=pending["compute_hashes"],
- _start_paused=start_paused,
- ):
- logging.warning(
- "Pending scan could not start (roots=%s, phase=%s)",
- pending["roots"],
- pending["phase"].value,
- )
+ self._finish_and_start_pending()
+
+ def _finish_and_start_pending(self) -> None:
+ """Reset to IDLE, then start the scan queued while this run held the seeder,
+ paused if this run was. Caller must hold _lock."""
+ start_paused = self._state is State.PAUSED
+ self._reset_to_idle()
+ pending = self._pending_scan
+ if pending is not None:
+ self._pending_scan = None
+ if not self.start(
+ roots=pending["roots"],
+ phase=pending["phase"],
+ prune_first=False,
+ compute_hashes=pending["compute_hashes"],
+ _start_paused=start_paused,
+ ):
+ logging.warning(
+ "Pending scan could not start (roots=%s, phase=%s)",
+ pending["roots"],
+ pending["phase"].value,
+ )
@staticmethod
def _emit_marked_missing(root: RootType, marked: int) -> None:
@@ -854,7 +917,11 @@ class _AssetSeeder:
existing_paths.update(live_references)
else:
marked_before = scan_state.missing_marked
- existing_paths.update(sync_root_safely(r, scan_state))
+ existing_paths.update(
+ sync_root_safely(
+ r, scan_state, lambda: self._check_pause_and_cancel(_ScanStage.FAST_SCAN)
+ )
+ )
self._emit_marked_missing(r, scan_state.missing_marked - marked_before)
logging.debug(
"Fast scan: sync_root phase took %.3fs (%d existing paths)",
@@ -877,7 +944,12 @@ class _AssetSeeder:
scan_state.dirs_listed += walk.dirs_listed
vanished, unlisted = unlisted_references(live_references, walk.listings, scan_state)
marked_before = scan_state.missing_marked
- mark_unlisted_references_missing_safely("output", vanished, scan_state)
+ mark_unlisted_references_missing_safely(
+ "output",
+ vanished,
+ scan_state,
+ lambda: self._check_pause_and_cancel(_ScanStage.FAST_SCAN),
+ )
self._emit_marked_missing("output", scan_state.missing_marked - marked_before)
logging.debug(
"Fast scan: output listing: %d dirs listed, %d rows retired, "
diff --git a/openapi.yaml b/openapi.yaml
index d2288354e..d1146ffe7 100644
--- a/openapi.yaml
+++ b/openapi.yaml
@@ -2402,6 +2402,7 @@ paths:
description: Number of assets marked as missing
type: integer
status:
+ description: '"completed", or "cancelled" when a cancel stopped the prune part way; the assets it marked before that stay marked'
type: string
type: object
description: Prune result
diff --git a/tests-unit/assets_test/services/test_batched_marking.py b/tests-unit/assets_test/services/test_batched_marking.py
new file mode 100644
index 000000000..7f77c4168
--- /dev/null
+++ b/tests-unit/assets_test/services/test_batched_marking.py
@@ -0,0 +1,722 @@
+"""The prune and the offline marking write in short batches rather than one long
+transaction, so a foreground write gets the lock between batches. These tests pin the
+batch boundaries, what a failure or a stop leaves behind, the pause between batches,
+what a batch does with rows another writer changed, and the set-based mark."""
+
+import asyncio
+import json
+import os
+import sqlite3
+import threading
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import sqlalchemy as sa
+from aiohttp.test_utils import make_mocked_request
+from sqlalchemy import event
+from sqlalchemy.orm import Session as SASession, sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.assets import scanner, seeder as seeder_module
+from app.assets.api import routes
+from app.assets.database.models import Asset, AssetContent, AssetTag, Base, Tag
+from app.assets.database.queries.records import (
+ create_content,
+ create_record,
+ ensure_tag,
+ ensure_tag_link,
+ mark_content_missing,
+ mark_contents_missing,
+)
+from app.assets.scanner_admission import _WATCH_LIST
+from app.assets.seeder import State
+
+
+def db_engine():
+ """One in-memory database every thread shares, for the tests that prune on a worker."""
+ engine = sa.create_engine("sqlite://", poolclass=StaticPool, connect_args={"check_same_thread": False})
+ Base.metadata.create_all(engine)
+ return engine
+
+
[email protected](autouse=True)
+def no_yield(monkeypatch):
+ monkeypatch.setattr(scanner, "WRITE_YIELD_MIN_SECONDS", 0.0)
+ monkeypatch.setattr(scanner, "WRITE_YIELD_MAX_SECONDS", 0.0)
+
+
+def catalog(db_engine):
+ """Routes the scanner's sessions to the test engine and counts write transactions."""
+ opened: list[int] = []
+ write_factory = sessionmaker(bind=db_engine)
+
+ @contextmanager
+ def _create_session():
+ with SASession(db_engine) as sess:
+ yield sess
+
+ def _create_write_session():
+ opened.append(1)
+ return write_factory()
+
+ _WATCH_LIST.clear()
+ with patch("app.assets.scanner.create_session", _create_session), \
+ patch("app.assets.seeder.create_session", _create_session), \
+ patch("app.assets.scanner.create_write_session", _create_write_session):
+ yield opened
+ _WATCH_LIST.clear()
+
+
+def _rows(session, directory: Path, count: int, *, files: bool = False) -> list[str]:
+ ids = []
+ for i in range(count):
+ path = directory / f"f{i:05d}.png"
+ if files:
+ path.write_bytes(f"bytes-{i}".encode())
+ stat = path.stat() if files else None
+ content = create_content(
+ session,
+ path=str(path),
+ size_bytes=stat.st_size if stat else 1,
+ mtime_ns=stat.st_mtime_ns if stat else 1,
+ )
+ create_record(session, content_id=content.id, name=path.name, tags=["output"])
+ ids.append(content.id)
+ session.commit()
+ return ids
+
+
+def _live_ids(session) -> set[str]:
+ session.expire_all()
+ return set(session.scalars(sa.select(AssetContent.id).where(AssetContent.is_missing == sa.false())))
+
+
+def _prune(owned: list[str]) -> int | None:
+ return scanner.mark_missing_outside_prefixes_safely(owned)
+
+
[email protected]("count, batches", [(0, 0), (1, 1), (256, 1), (257, 2), (600, 3)])
+def test_prune_commits_one_batch_per_256_rows(session, catalog, temp_dir, count, batches):
+ _rows(session, temp_dir, count)
+
+ assert _prune([]) == count
+ assert len(catalog) == batches
+ assert _live_ids(session) == set()
+
+
+def test_rows_under_an_owned_prefix_are_not_pruned(session, catalog, temp_dir):
+ owned, gone = temp_dir / "owned", temp_dir / "gone"
+ owned.mkdir()
+ gone.mkdir()
+ kept = set(_rows(session, owned, 5))
+ _rows(session, gone, 5)
+
+ assert _prune([str(owned)]) == 5
+ assert _live_ids(session) == kept
+
+
+def test_a_row_re_registered_between_batches_is_left_to_its_new_owner(session, catalog, temp_dir):
+ """register_executed_output retires the live row at a path and inserts a new one.
+ Doing that between two batches must leave exactly the new row live."""
+ ids = _rows(session, temp_dir, 300)
+ last = session.get(AssetContent, ids[-1])
+ replacement: list[str] = []
+
+ def between_batches() -> bool:
+ if len(catalog) == 1:
+ mark_content_missing(session, last.id)
+ new = create_content(session, path=last.path, size_bytes=2, mtime_ns=2)
+ session.commit()
+ replacement.append(new.id)
+ return False
+
+ marked = scanner.mark_missing_outside_prefixes_safely([], between_batches)
+
+ # The replacement was never a candidate; the retired row is not counted twice.
+ assert marked == len(ids) - 1
+ assert _live_ids(session) == set(replacement)
+
+
+def test_a_foreground_write_gets_the_lock_between_batches(tmp_path, monkeypatch):
+ """On a real file database with the production write-session setup, another
+ connection can take the write lock at every point between two batches."""
+ db = tmp_path / "catalog.db"
+ read_engine = sa.create_engine(f"sqlite:///{db}")
+ write_engine = sa.create_engine(f"sqlite:///{db}")
+
+ @event.listens_for(write_engine, "connect")
+ def _connect(dbapi_connection, _record):
+ dbapi_connection.isolation_level = None
+
+ @event.listens_for(write_engine, "begin")
+ def _begin(connection):
+ connection.exec_driver_sql("BEGIN IMMEDIATE")
+
+ with read_engine.connect() as conn:
+ conn.exec_driver_sql("PRAGMA journal_mode=WAL")
+ Base.metadata.create_all(read_engine)
+ with SASession(read_engine) as session:
+ _rows(session, tmp_path, 600)
+
+ foreground: list[bool] = []
+
+ def between_batches() -> bool:
+ other = sqlite3.connect(db, timeout=0, isolation_level=None)
+ try:
+ other.execute("BEGIN IMMEDIATE")
+ other.execute("INSERT INTO tags (name) VALUES (?)", (f"fg-{len(foreground)}",))
+ other.execute("COMMIT")
+ foreground.append(True)
+ except sqlite3.OperationalError:
+ foreground.append(False)
+ finally:
+ other.close()
+ return False
+
+ monkeypatch.setattr(scanner, "create_session", lambda: SASession(read_engine))
+ monkeypatch.setattr(scanner, "create_write_session", sessionmaker(bind=write_engine))
+ assert scanner.mark_missing_outside_prefixes_safely([], between_batches) == 600
+
+ assert foreground == [True, True, True]
+
+
+def test_a_failed_batch_keeps_the_batches_before_it(session, catalog, temp_dir):
+ _rows(session, temp_dir, 600)
+ real = scanner.mark_contents_missing
+
+ def fail_in_the_second_batch(sess, ids):
+ if len(catalog) == 2:
+ raise RuntimeError("disk I/O error")
+ return real(sess, ids)
+
+ with patch("app.assets.scanner.mark_contents_missing", fail_in_the_second_batch):
+ assert _prune([]) is None
+
+ assert len(_live_ids(session)) == 600 - scanner.WRITE_BATCH_ROWS
+
+
+def test_sync_root_counts_the_batches_committed_before_a_failure(session, catalog, temp_dir, monkeypatch):
+ output = temp_dir / "output"
+ output.mkdir()
+ monkeypatch.setattr("folder_paths.get_output_directory", lambda: str(output))
+ _rows(session, output, 300)
+ real = scanner.mark_contents_missing
+
+ def fail_in_the_second_batch(sess, ids):
+ if len(catalog) == 2:
+ raise RuntimeError("disk I/O error")
+ return real(sess, ids)
+
+ progress = seeder_module._ScanState()
+ with patch("app.assets.scanner.mark_contents_missing", fail_in_the_second_batch):
+ assert scanner.sync_root_safely("output", progress) == set()
+
+ assert progress.missing_marked == scanner.WRITE_BATCH_ROWS
+
+
+def _gone_observations(session, temp_dir: Path, count: int) -> list[scanner._ReferenceObservation]:
+ ids = _rows(session, temp_dir, count, files=True)
+ observations = []
+ for content_id in ids:
+ content = session.get(AssetContent, content_id)
+ os.remove(content.path)
+ observations.append(
+ scanner._ReferenceObservation(content.id, content.size_bytes, content.mtime_ns, None)
+ )
+ return observations
+
+
+def test_stop_between_batches_leaves_the_rest_live(session, catalog, temp_dir):
+ observations = _gone_observations(session, temp_dir, 600)
+
+ committed: list[int] = []
+ scanner._write_in_batches(observations, scanner.apply_reference_observations, lambda: bool(committed), committed)
+
+ assert committed == [scanner.WRITE_BATCH_ROWS]
+ assert len(_live_ids(session)) == 600 - scanner.WRITE_BATCH_ROWS
+
+
+def test_a_scan_waits_between_batches_while_a_prompt_runs(session, catalog, temp_dir, monkeypatch):
+ output = temp_dir / "output"
+ output.mkdir()
+ monkeypatch.setattr("folder_paths.get_output_directory", lambda: str(output))
+ _rows(session, output, 600)
+ instance = seeder_module._AssetSeeder()
+ instance._state = State.RUNNING
+ instance._scan_state = seeder_module._ScanState()
+ instance._run_gate.set()
+ real = scanner.mark_contents_missing
+ first_batch = threading.Event()
+
+ def mark(sess, ids):
+ if not first_batch.is_set():
+ assert instance.pause() # a prompt starts during the first batch
+ first_batch.set()
+ return real(sess, ids)
+
+ def sync():
+ scanner.sync_root_safely(
+ "output",
+ instance._scan_state,
+ lambda: instance._check_pause_and_cancel(seeder_module._ScanStage.FAST_SCAN),
+ )
+
+ with patch("app.assets.scanner.mark_contents_missing", mark):
+ worker = threading.Thread(target=sync)
+ worker.start()
+ assert first_batch.wait(5)
+ time.sleep(0.2)
+ assert len(catalog) == 1 # no batch while paused
+ assert instance.resume()
+ worker.join(5)
+
+ assert len(catalog) == 3
+ assert instance._scan_state.missing_marked == 600
+
+
+def test_offline_rows_retired_across_batches_recover_when_the_drive_returns(
+ session, catalog, temp_dir, monkeypatch
+):
+ """#16646's hashing-off recovery with the marking split over several batches: files
+ that come back while the marking is part way through are retired by the batches
+ that follow, then recovered by the walk that follows in the same scan, every record
+ keeping its id."""
+ output = temp_dir / "output"
+ (temp_dir / "input").mkdir()
+ output.mkdir()
+ monkeypatch.setattr("folder_paths.get_output_directory", lambda: str(output))
+ monkeypatch.setattr("folder_paths.get_input_directory", lambda: str(temp_dir / "input"))
+ monkeypatch.setattr(scanner, "get_comfy_models_folders", lambda: [])
+ ids = _rows(session, output, 700, files=True)
+ records = {record.id: record.content_id for record in session.scalars(sa.select(Asset))}
+ parked = temp_dir / "parked"
+ output.rename(parked)
+ output.mkdir()
+
+ instance = seeder_module._AssetSeeder()
+ instance._scan_state = seeder_module._ScanState()
+ instance._phase = seeder_module.ScanPhase.FAST
+ instance._run_gate.set()
+ real = scanner.mark_contents_missing
+
+ def mark(sess, content_ids):
+ marked = real(sess, content_ids)
+ if len(catalog) == 1:
+ # The drive comes back after the first batch.
+ for name in os.listdir(parked):
+ os.rename(parked / name, output / name)
+ return marked
+
+ with patch("app.assets.scanner.mark_contents_missing", mark):
+ instance._run_fast_phase(("input", "output"))
+
+ session.expire_all()
+ assert instance._scan_state.missing_marked == 700
+ assert instance._scan_state.recovered == 700
+ assert _live_ids(session) == set(ids)
+ assert {record.id: record.content_id for record in session.scalars(sa.select(Asset))} == records
+ live_paths = session.scalars(sa.select(AssetContent.path).where(AssetContent.is_missing == sa.false())).all()
+ assert len(live_paths) == len(set(live_paths)) == 700
+
+
+def _link_state(session) -> tuple[dict[str, bool], set[tuple[str, str, str]]]:
+ session.expire_all()
+ contents = {c.path: c.is_missing for c in session.scalars(sa.select(AssetContent))}
+ links = {
+ (record.name, link.tag_name, link.origin)
+ for record in session.scalars(sa.select(Asset))
+ for link in session.scalars(sa.select(AssetTag).where(AssetTag.asset_id == record.id))
+ }
+ return contents, links
+
+
+def _equivalence_fixture(session) -> list[str]:
+ """Rows covering each case the mark handles: no record, one, two, a record already
+ tagged missing by hand, an already-missing row, and an id that does not exist."""
+ none = create_content(session, path="/c/none.png")
+ one = create_content(session, path="/c/one.png")
+ create_record(session, content_id=one.id, name="one", tags=["output"])
+ two = create_content(session, path="/c/two.png")
+ create_record(session, content_id=two.id, name="two-a")
+ create_record(session, content_id=two.id, name="two-b", tags=["input"])
+ tagged = create_content(session, path="/c/tagged.png")
+ record = create_record(session, content_id=tagged.id, name="tagged")
+ ensure_tag(session, "missing")
+ ensure_tag_link(session, asset_id=record.id, tag_name="missing", origin="manual")
+ already = create_content(session, path="/c/already.png")
+ create_record(session, content_id=already.id, name="already")
+ mark_content_missing(session, already.id)
+ session.commit()
+ return [none.id, one.id, two.id, tagged.id, already.id, "no-such-id"]
+
+
+def test_the_set_mark_matches_marking_row_by_row(session):
+ engine = sa.create_engine("sqlite:///:memory:")
+ Base.metadata.create_all(engine)
+ with SASession(engine) as per_row:
+ # Same fixture in both catalogs, then the ids made to match by path.
+ ids = _equivalence_fixture(session)
+ _equivalence_fixture(per_row)
+ by_path = {c.path: c.id for c in per_row.scalars(sa.select(AssetContent))}
+ paths = {c.id: c.path for c in session.scalars(sa.select(AssetContent))}
+
+ marked = mark_contents_missing(session, ids)
+ session.commit()
+ for content_id in ids:
+ path = paths.get(content_id)
+ content = per_row.get(AssetContent, by_path[path]) if path else None
+ if content is not None and not content.is_missing:
+ mark_content_missing(per_row, content.id)
+ per_row.commit()
+
+ # create_content stores os.path.abspath(path), so compare in that form (a drive letter on Windows).
+ expected = sorted(os.path.abspath(f"/c/{name}.png") for name in ("none", "one", "tagged", "two"))
+ assert sorted(paths[i] for i in marked) == expected
+ assert _link_state(session) == _link_state(per_row)
+ assert session.get(Tag, "missing") is not None
+
+
+def test_the_set_mark_settles_a_link_race_row_by_row(session, monkeypatch):
+ content = create_content(session, path="/c/raced.png")
+ record = create_record(session, content_id=content.id, name="raced")
+ session.commit()
+ real_execute = session.execute
+
+ def insert_loses_the_race(statement, *args, **kwargs):
+ if isinstance(statement, sa.Insert) and statement.table is AssetTag.__table__:
+ raise sa.exc.IntegrityError("INSERT", {}, Exception("UNIQUE constraint failed"))
+ return real_execute(statement, *args, **kwargs)
+
+ monkeypatch.setattr(session, "execute", insert_loses_the_race)
+ assert mark_contents_missing(session, [content.id]) == [content.id]
+ session.commit()
+
+ link = session.get(AssetTag, (record.id, "missing"))
+ assert link is not None and link.origin == "automatic"
+
+
+def test_the_batched_writes_run_no_table_scan_inside_their_transactions(session, db_engine, temp_dir, monkeypatch):
+ """Each statement a batch runs while it holds the write lock is looked up by key or
+ index, so the lock window grows with the batch, not the catalog. (The prune's
+ candidate read is a full read by design, and runs before any write transaction.)"""
+ in_write: list[bool] = []
+
+ class _Tracked(SASession):
+ def __enter__(self):
+ in_write.append(True)
+ return super().__enter__()
+
+ def __exit__(self, *exc_info):
+ in_write.clear()
+ return super().__exit__(*exc_info)
+
+ monkeypatch.setattr(scanner, "create_write_session", sessionmaker(bind=db_engine, class_=_Tracked))
+ monkeypatch.setattr(scanner, "create_session", lambda: SASession(db_engine))
+ # A size change splits the row, and the new record's tags come from its root.
+ monkeypatch.setattr("folder_paths.get_output_directory", lambda: str(temp_dir))
+ gone_dir, changed_dir, pruned_dir = (temp_dir / name for name in ("gone", "changed", "pruned"))
+ for directory in (gone_dir, changed_dir, pruned_dir):
+ directory.mkdir()
+ observations = _gone_observations(session, gone_dir, 50)
+ for content_id in _rows(session, changed_dir, 2, files=True):
+ content = session.get(AssetContent, content_id)
+ if content.path.endswith("0.png"):
+ Path(content.path).write_bytes(b"a different size")
+ os.utime(content.path, ns=(10**18, 10**18))
+ observations.append(
+ scanner._ReferenceObservation(content.id, content.size_bytes, content.mtime_ns, os.stat(content.path))
+ )
+ statements: list[tuple[str, object]] = []
+
+ def capture(conn, cursor, statement, parameters, context, executemany):
+ if in_write and statement.lstrip().upper().startswith(("SELECT", "UPDATE", "INSERT", "DELETE")):
+ statements.append((statement, parameters))
+
+ event.listen(db_engine, "before_cursor_execute", capture)
+ try:
+ scanner._write_in_batches(observations, scanner.apply_reference_observations, lambda: False, [])
+ _rows(session, pruned_dir, 50)
+ assert scanner.mark_missing_outside_prefixes_safely([str(gone_dir), str(changed_dir)]) == 50
+ finally:
+ event.remove(db_engine, "before_cursor_execute", capture)
+
+ kinds = {statement.split()[0].upper() for statement, _ in statements}
+ assert {"SELECT", "UPDATE", "INSERT"} <= kinds
+ with db_engine.connect() as conn:
+ for statement, parameters in statements:
+ plan = conn.exec_driver_sql(f"EXPLAIN QUERY PLAN {statement}", parameters).all()
+ scans = [row[-1] for row in plan if row[-1].startswith("SCAN")]
+ assert not scans, (statement, plan)
+
+
+async def test_the_prune_endpoint_keeps_the_event_loop_serving(monkeypatch):
+ def slow_prune() -> int:
+ time.sleep(0.5)
+ return 3
+
+ monkeypatch.setattr(routes.asset_seeder, "mark_missing_outside_prefixes", slow_prune)
+ ticks = 0
+
+ async def ticker():
+ nonlocal ticks
+ while True:
+ await asyncio.sleep(0.01)
+ ticks += 1
+
+ ticking = asyncio.create_task(ticker())
+ response = await routes.mark_missing_assets.__wrapped__(make_mocked_request("POST", "/api/assets/prune"))
+ ticking.cancel()
+
+ assert json.loads(response.body) == {"status": "completed", "marked": 3}
+ assert ticks >= 20
+
+
+
+def test_the_standalone_prune_starts_the_scan_queued_while_it_ran(session, catalog, temp_dir, monkeypatch):
+ """The API runs the prune off the event loop, so a prompt can finish meanwhile and
+ queue its output rescan, which cannot start while the prune holds the seeder."""
+ _rows(session, temp_dir, 10)
+ instance = seeder_module._AssetSeeder()
+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)
+ monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: [])
+ started: list[dict] = []
+
+ def start(**kwargs) -> bool:
+ if instance._state is not State.IDLE:
+ return False
+ started.append(kwargs)
+ return True
+
+ monkeypatch.setattr(instance, "start", start)
+ real = scanner.mark_contents_missing
+
+ def mark(sess, ids):
+ assert instance.enqueue_scan(roots=("output",), phase=seeder_module.ScanPhase.FULL) is False
+ return real(sess, ids)
+
+ with patch("app.assets.scanner.mark_contents_missing", mark):
+ assert instance.mark_missing_outside_prefixes() == 10
+
+ assert [kwargs["roots"] for kwargs in started] == [("output",)]
+ assert instance._pending_scan is None
+
+
+def _seeder_with_recorded_starts(monkeypatch) -> tuple[seeder_module._AssetSeeder, list[tuple]]:
+ instance = seeder_module._AssetSeeder()
+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)
+ monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: [])
+ started: list[tuple] = []
+
+ def start(roots=("models", "input", "output"), **kwargs) -> bool:
+ if instance._state is not State.IDLE:
+ return False
+ started.append(tuple(roots))
+ return True
+
+ monkeypatch.setattr(instance, "start", start)
+ monkeypatch.setattr(routes, "asset_seeder", instance)
+ monkeypatch.setattr(routes, "_ASSETS_ENABLED", True)
+ return instance, started
+
+
+async def test_a_seed_request_during_an_api_prune_waits_for_it_then_starts(monkeypatch):
+ """The prune runs off the event loop now, so a seed request can arrive while it
+ holds the seeder. A 409 there would tell the client a scan is coming when none is."""
+ instance, started = _seeder_with_recorded_starts(monkeypatch)
+ release = threading.Event()
+ pruning = threading.Event()
+
+ def blocking_prune(prefixes, should_stop=None):
+ pruning.set()
+ assert release.wait(5)
+ return 0
+
+ monkeypatch.setattr(seeder_module, "mark_missing_outside_prefixes_safely", blocking_prune)
+ monkeypatch.setattr(routes, "_PRUNE_POLL_SECONDS", 0.01)
+ prune = asyncio.create_task(asyncio.to_thread(instance.mark_missing_outside_prefixes))
+ assert await asyncio.to_thread(pruning.wait, 5)
+
+ def must_not_block_a_thread(timeout=None):
+ raise AssertionError("the seed route held an executor thread for the prune")
+
+ # The route waits on the loop; a blocking wait would hold an executor thread per request.
+ monkeypatch.setattr(instance, "wait_for_standalone_prune", must_not_block_a_thread)
+ seed = asyncio.create_task(routes.seed_assets.__wrapped__(make_mocked_request("POST", "/api/assets/seed")))
+ await asyncio.sleep(0.2)
+ assert not seed.done() # waiting out the prune, not answering 409
+ release.set()
+ response = await seed
+ await prune
+
+ assert response.status == 202
+ assert started == [("models", "input", "output")]
+
+
+async def test_a_seed_request_during_a_scan_still_gets_409(monkeypatch):
+ instance, started = _seeder_with_recorded_starts(monkeypatch)
+ instance._state = State.RUNNING
+
+ response = await routes.seed_assets.__wrapped__(make_mocked_request("POST", "/api/assets/seed"))
+
+ assert response.status == 409
+ assert started == []
+
+
+def test_a_cancel_stops_a_standalone_prune_and_shutdown_waits_for_it(session, catalog, temp_dir, monkeypatch):
+ """The API prune runs on a worker thread that interpreter exit joins, so shutdown's
+ cancel must stop it between batches rather than let it run to the end."""
+ _rows(session, temp_dir, 600)
+ instance = seeder_module._AssetSeeder()
+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)
+ monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: [])
+ real = scanner.mark_contents_missing
+ first_batch = threading.Event()
+ shut_down = threading.Event()
+
+ def mark(sess, ids):
+ marked = real(sess, ids)
+ first_batch.set()
+ assert shut_down.wait(5) # hold the first batch until shutdown has cancelled
+ return marked
+
+ result: list[object] = []
+
+ def prune() -> None:
+ try:
+ result.append(instance.mark_missing_outside_prefixes())
+ except seeder_module.PruneCancelledError as cancelled:
+ result.append(cancelled)
+
+ with patch("app.assets.scanner.mark_contents_missing", mark):
+ worker = threading.Thread(target=prune)
+ worker.start()
+ assert first_batch.wait(5)
+ threading.Timer(0.1, shut_down.set).start()
+ assert instance.shutdown(timeout=5)
+ worker.join(5)
+
+ assert len(catalog) == 1
+ assert len(result) == 1 and isinstance(result[0], seeder_module.PruneCancelledError)
+ assert result[0].marked == scanner.WRITE_BATCH_ROWS
+ assert not instance.standalone_prune_running()
+
+
+async def test_a_cancelled_api_prune_is_not_reported_as_completed(monkeypatch):
+ def cancelled_prune() -> int:
+ raise seeder_module.PruneCancelledError(256)
+
+ monkeypatch.setattr(routes.asset_seeder, "mark_missing_outside_prefixes", cancelled_prune)
+ response = await routes.mark_missing_assets.__wrapped__(make_mocked_request("POST", "/api/assets/prune"))
+
+ assert response.status == 200
+ assert json.loads(response.body) == {"status": "cancelled", "marked": 256}
+
+
+def test_a_prune_that_finishes_before_a_late_cancel_reports_completed(session, catalog, temp_dir, monkeypatch):
+ """The cancel only counts if it stopped a batch from running."""
+ _rows(session, temp_dir, 10)
+ instance = seeder_module._AssetSeeder()
+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)
+ monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: [])
+ real = scanner.mark_contents_missing
+
+ def mark(sess, ids):
+ marked = real(sess, ids)
+ instance.cancel() # arrives during the last (only) batch
+ return marked
+
+ with patch("app.assets.scanner.mark_contents_missing", mark):
+ assert instance.mark_missing_outside_prefixes() == 10
+
+
+def test_shutdown_during_a_prune_does_not_start_the_scan_a_prompt_queued(session, catalog, temp_dir, monkeypatch):
+ """A scan started after shutdown cancelled the prune would run on into teardown."""
+ _rows(session, temp_dir, 600)
+ instance = seeder_module._AssetSeeder()
+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)
+ monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: [])
+ started: list[tuple] = []
+
+ def start(roots=("models", "input", "output"), **kwargs) -> bool:
+ if instance._state is not State.IDLE:
+ return False
+ started.append(tuple(roots))
+ return True
+
+ monkeypatch.setattr(instance, "start", start)
+ real = scanner.mark_contents_missing
+ first_batch = threading.Event()
+ shut_down = threading.Event()
+
+ def mark(sess, ids):
+ marked = real(sess, ids)
+ # A prompt finishes and queues its output rescan; the prune holds the seeder.
+ assert instance.enqueue_scan(roots=("output",), phase=seeder_module.ScanPhase.FULL) is False
+ first_batch.set()
+ assert shut_down.wait(5)
+ return marked
+
+ outcome: list[object] = []
+
+ def prune() -> None:
+ try:
+ outcome.append(instance.mark_missing_outside_prefixes())
+ except seeder_module.PruneCancelledError as cancelled:
+ outcome.append(cancelled)
+
+ with patch("app.assets.scanner.mark_contents_missing", mark):
+ worker = threading.Thread(target=prune)
+ worker.start()
+ assert first_batch.wait(5)
+ threading.Timer(0.1, shut_down.set).start()
+ assert instance.shutdown(timeout=5)
+ worker.join(5)
+
+ # Asserted here, not in the worker: a failure there would only surface as a warning.
+ assert len(outcome) == 1 and isinstance(outcome[0], seeder_module.PruneCancelledError)
+ assert started == []
+ assert instance._state is State.IDLE
+
+
+def test_shutdown_before_a_prune_starts_keeps_it_from_starting(session, catalog, temp_dir, monkeypatch):
+ """The API hands the prune to a worker thread; a shutdown that lands before it takes
+ the seeder must still keep it from running into teardown."""
+ _rows(session, temp_dir, 10)
+ instance = seeder_module._AssetSeeder()
+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)
+ monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: [])
+
+ assert instance.shutdown(timeout=1)
+ with pytest.raises(seeder_module.PruneCancelledError) as cancelled:
+ instance.mark_missing_outside_prefixes()
+
+ assert cancelled.value.marked == 0
+ assert len(catalog) == 0
+ assert len(_live_ids(session)) == 10
+ assert not instance.standalone_prune_running()
+
+
+def test_the_prune_flag_clears_even_if_its_cleanup_raises(session, catalog, temp_dir, monkeypatch):
+ _rows(session, temp_dir, 10)
+ instance = seeder_module._AssetSeeder()
+ monkeypatch.setattr(seeder_module, "dependencies_available", lambda: True)
+ monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: [])
+
+ def start_fails():
+ raise RuntimeError("can't start new thread")
+
+ monkeypatch.setattr(instance, "_finish_and_start_pending", start_fails)
+ with pytest.raises(RuntimeError):
+ instance.mark_missing_outside_prefixes()
+
+ assert not instance.standalone_prune_running()
+ assert instance.wait_for_standalone_prune(0)
diff --git a/tests-unit/assets_test/services/test_scanner_b.py b/tests-unit/assets_test/services/test_scanner_b.py
index ad0828a9d..729d6d41a 100644
--- a/tests-unit/assets_test/services/test_scanner_b.py
+++ b/tests-unit/assets_test/services/test_scanner_b.py
@@ -5,14 +5,13 @@ from pathlib import Path
from unittest.mock import patch
from sqlalchemy import select
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, sessionmaker
from app.assets.database.models import Asset, AssetContent, AssetTag
from app.assets.helpers import to_stored_hash
from app.assets.scanner import (
build_asset_specs,
enrich_asset,
- mark_contents_missing_outside_prefixes,
mark_missing_outside_prefixes_safely,
seed_asset_specs,
apply_reference_observations,
@@ -151,7 +150,7 @@ def test_seed_creates_content_and_record(session, temp_dir: Path):
]
-def test_prune_marks_missing_not_deletes(session, temp_dir: Path):
+def test_prune_marks_missing_not_deletes(session, db_engine, temp_dir: Path):
input_root = temp_dir / "input"
input_root.mkdir()
file_path = input_root / "removed-from-registry.png"
@@ -161,8 +160,11 @@ def test_prune_marks_missing_not_deletes(session, temp_dir: Path):
seed_asset_specs(session, _build_seed_specs(input_root))
session.commit()
- marked = mark_contents_missing_outside_prefixes(session, prefixes=[])
- session.commit()
+ with patch("app.assets.scanner.create_session", sessionmaker(bind=db_engine)), \
+ patch("app.database.db.WriteSession", sessionmaker(bind=db_engine)), \
+ patch("app.assets.scanner.get_owned_prefixes", return_value=[]):
+ marked = mark_missing_outside_prefixes_safely([])
+ session.expire_all()
content = session.scalar(select(AssetContent))
record = session.scalar(select(Asset))
diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py
index a403170d7..6b1c12cc3 100644
--- a/tests-unit/seeder_test/test_seeder.py
+++ b/tests-unit/seeder_test/test_seeder.py
@@ -93,7 +93,7 @@ def _configure_fast_phase(
specs: list[SeedAssetSpec],
) -> None:
monkeypatch.setattr(
- seeder_module, "sync_root_safely", lambda _root, _progress: set()
+ seeder_module, "sync_root_safely", lambda _root, _progress, _should_stop=None: set()
)
monkeypatch.setattr(
seeder_module,
@@ -435,7 +435,7 @@ def test_idle_reset_survives_a_raising_cancellation_emit(
monkeypatch.setattr(scan_seeder, "_check_pause_and_cancel", cancel_at_pruning)
monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ())
monkeypatch.setattr(
- seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes: 0
+ seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes, _should_stop=None: 0
)
original_emit = seeder_module.emit
@@ -462,7 +462,7 @@ def test_scan_paused_after_its_last_phase_still_completes(
scan_seeder._phase = ScanPhase.ENRICH
monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ())
monkeypatch.setattr(
- seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes: 0
+ seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes, _should_stop=None: 0
)
def pause_while_finishing(roots) -> tuple[bool, int]:
@@ -507,10 +507,10 @@ def test_prune_before_scan_emits_marked_missing_with_pruning_stage(
scan_seeder._phase = ScanPhase.FAST
monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ())
monkeypatch.setattr(
- seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes: 5
+ seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes, _should_stop=None: 5
)
monkeypatch.setattr(
- seeder_module, "sync_temp_references_safely", lambda _progress: None
+ seeder_module, "sync_temp_references_safely", lambda _progress, _should_stop=None: None
)
monkeypatch.setattr(scan_seeder, "_run_fast_phase", lambda roots: (0, 0, 0))
@@ -530,7 +530,7 @@ def test_standalone_mark_missing_emits_count_with_mark_missing_stage(
scan_seeder._state = State.IDLE
monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: ())
monkeypatch.setattr(
- seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes: 7
+ seeder_module, "mark_missing_outside_prefixes_safely", lambda prefixes, _should_stop=None: 7
)
with caplog.at_level(logging.INFO):
@@ -575,10 +575,10 @@ def test_scan_prune_failure_is_reported_and_the_scan_still_runs(
fast_phase_roots: list[tuple[str, ...]] = []
monkeypatch.setattr(seeder_module, "get_owned_prefixes", lambda: [])
monkeypatch.setattr(
- seeder_module, "mark_missing_outside_prefixes_safely", lambda _prefixes: None
+ seeder_module, "mark_missing_outside_prefixes_safely", lambda _prefixes, _should_stop=None: None
)
monkeypatch.setattr(
- seeder_module, "sync_temp_references_safely", lambda _progress: None
+ seeder_module, "sync_temp_references_safely", lambda _progress, _should_stop=None: None
)
def run_fast_phase(roots: tuple[str, ...]) -> tuple[int, int, int]:
@@ -591,7 +591,7 @@ def test_scan_prune_failure_is_reported_and_the_scan_still_runs(
scan_seeder._run_scan()
assert scan_seeder._errors == [
- "Marking missing assets failed; scan continued without pruning"
+ "Marking missing assets failed; scan continued with the prune incomplete"
]
assert fast_phase_roots == [("models", "input")]
assert events_named(caplog, "seeder.marked_missing") == []
@@ -604,7 +604,7 @@ def test_batch_insert_failure_emits_only_the_exception_type(
) -> None:
session = Mock()
monkeypatch.setattr(
- seeder_module, "sync_root_safely", lambda _root, _progress: set()
+ seeder_module, "sync_root_safely", lambda _root, _progress, _should_stop=None: set()
)
monkeypatch.setattr(
seeder_module,