The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Keep temp-directory assets visible while their files exist (#15510)
Simon Pinfoldcc0fc21fea7a
diff --git a/app/assets/scanner.py b/app/assets/scanner.pyindex 42c4c1e9d..0db7588de 100644--- a/app/assets/scanner.py+++ b/app/assets/scanner.py@@ -57,10 +57,11 @@ class _AssetAccumulator(TypedDict):refs: list[_RefInfo]+# Temp is deliberately absent: it is wiped before every scan, so walking it finds nothing.RootType = Literal["models", "input", "output"]-def get_prefixes_for_root(root: RootType) -> list[str]:+def get_scan_prefixes_for_root(root: RootType) -> list[str]:if root == "models":bases: list[str] = []for _bucket, paths, _exts in get_comfy_models_folders():@@ -73,10 +74,15 @@ def get_prefixes_for_root(root: RootType) -> list[str]:return []-def get_all_known_prefixes() -> list[str]:- """Get all known asset prefixes across all root types."""- all_roots: tuple[RootType, ...] = ("models", "input", "output")- return [p for root in all_roots for p in get_prefixes_for_root(root)]+def get_owned_prefixes() -> list[str]:+ """Every directory an asset may live in; references outside these are marked missing."""+ scan_roots: tuple[RootType, ...] = ("models", "input", "output")+ prefixes = [p for root in scan_roots for p in get_scan_prefixes_for_root(root)]+ return prefixes + get_temp_prefixes()+++def get_temp_prefixes() -> list[str]:+ return [os.path.abspath(folder_paths.get_temp_directory())]def collect_models_files() -> list[str]:@@ -107,7 +113,21 @@ def sync_references_with_filesystem(collect_existing_paths: bool = False,update_missing_tags: bool = False,) -> set[str] | None:- """Reconcile asset references with filesystem for a root.+ return sync_prefixes_with_filesystem(+ session,+ get_scan_prefixes_for_root(root),+ collect_existing_paths=collect_existing_paths,+ update_missing_tags=update_missing_tags,+ )+++def sync_prefixes_with_filesystem(+ session,+ prefixes: list[str],+ collect_existing_paths: bool = False,+ update_missing_tags: bool = False,+) -> set[str] | None:+ """Reconcile asset references with filesystem under the given prefixes.- Toggle needs_verify per reference using mtime/size stat check- For hashed assets with at least one stat-unchanged ref: delete stale missing refs@@ -117,14 +137,13 @@ def sync_references_with_filesystem(Args:session: Database session- root: Root type to scan+ prefixes: Absolute directory prefixes whose references to reconcilecollect_existing_paths: If True, return set of surviving file pathsupdate_missing_tags: If True, update 'missing' tags based on file statusReturns:Set of surviving absolute paths if collect_existing_paths=True, else None"""- prefixes = get_prefixes_for_root(root)if not prefixes:return set() if collect_existing_paths else None@@ -251,6 +270,16 @@ def sync_root_safely(root: RootType) -> set[str]:return set()+def sync_temp_references_safely() -> None:+ """Retire temp references whose file is gone; temp is never scanned, so nothing else stats them."""+ try:+ with create_session() as sess:+ sync_prefixes_with_filesystem(sess, get_temp_prefixes())+ sess.commit()+ except Exception as e:+ logging.exception("temp reference sync failed: %s", e)++def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int:"""Mark references as missing when outside the given prefixes.@@ -384,7 +413,7 @@ def get_unenriched_assets_for_roots("""prefixes: list[str] = []for root in roots:- prefixes.extend(get_prefixes_for_root(root))+ prefixes.extend(get_scan_prefixes_for_root(root))if not prefixes:return []diff --git a/app/assets/seeder.py b/app/assets/seeder.pyindex 2262928e5..134fc98a8 100644--- a/app/assets/seeder.py+++ b/app/assets/seeder.py@@ -15,12 +15,13 @@ from app.assets.scanner import (build_asset_specs,collect_paths_for_roots,enrich_assets_batch,- get_all_known_prefixes,- get_prefixes_for_root,+ get_owned_prefixes,+ get_scan_prefixes_for_root,get_unenriched_assets_for_roots,insert_asset_specs,mark_missing_outside_prefixes_safely,sync_root_safely,+ sync_temp_references_safely,)from app.database.db import dependencies_available@@ -413,7 +414,7 @@ class _AssetSeeder:)return 0- all_prefixes = get_all_known_prefixes()+ all_prefixes = get_owned_prefixes()marked = mark_missing_outside_prefixes_safely(all_prefixes)if marked > 0:logging.info("Marked %d references as missing", marked)@@ -523,7 +524,7 @@ class _AssetSeeder:os.path.abspath(folder_paths.models_dir),)else:- prefixes = get_prefixes_for_root(root)+ prefixes = get_scan_prefixes_for_root(root)if prefixes:logging.info("Asset scan [%s] directories: %s", root, prefixes)@@ -548,10 +549,11 @@ class _AssetSeeder:returnif self._prune_first:- all_prefixes = get_all_known_prefixes()+ all_prefixes = get_owned_prefixes()marked = mark_missing_outside_prefixes_safely(all_prefixes)if marked > 0:logging.info("Marked %d refs as missing before scan", marked)+ sync_temp_references_safely()if self._check_pause_and_cancel():logging.info("Asset scan cancelled after pruning phase")diff --git a/tests-unit/assets_test/test_sync_references.py b/tests-unit/assets_test/test_sync_references.pyindex 2e85076e0..39c19ed8a 100644--- a/tests-unit/assets_test/test_sync_references.py+++ b/tests-unit/assets_test/test_sync_references.py@@ -150,7 +150,7 @@ def test_needs_verify_toggling(session, temp_dir, case):)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models")session.commit()@@ -185,7 +185,7 @@ def test_is_missing_flag(session, temp_dir, case):_make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models")session.commit()@@ -200,7 +200,7 @@ def test_seed_asset_all_missing_deletes_asset(session, temp_dir):_make_asset(session, "seed1", fp, "r1", asset_hash=None, mtime_ns=999)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models")session.commit()@@ -215,7 +215,7 @@ def test_seed_asset_some_exist_returns_survivors(session, temp_dir):_make_asset(session, "seed1", fp, "r1", asset_hash=None, mtime_ns=mtime)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):survivors = sync_references_with_filesystem(session, "models", collect_existing_paths=True,)@@ -240,7 +240,7 @@ def test_hashed_asset_prunes_missing_refs_when_one_is_ok(session, temp_dir):session.add(ref_gone)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models")session.commit()@@ -255,7 +255,7 @@ def test_hashed_asset_all_missing_keeps_refs(session, temp_dir):_make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models")session.commit()@@ -272,7 +272,7 @@ def test_missing_tag_added_when_all_refs_gone(session, temp_dir):_make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models", update_missing_tags=True,)@@ -295,7 +295,7 @@ def test_missing_tag_removed_when_ref_ok(session, temp_dir):))session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models", update_missing_tags=True,)@@ -313,7 +313,7 @@ def test_missing_tags_not_touched_when_flag_false(session, temp_dir):_make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models", update_missing_tags=False,)@@ -329,7 +329,7 @@ def test_returns_none_when_collect_false(session, temp_dir):_make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):result = sync_references_with_filesystem(session, "models", collect_existing_paths=False,)@@ -338,7 +338,7 @@ def test_returns_none_when_collect_false(session, temp_dir):def test_returns_empty_set_for_no_prefixes(session):- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[]):result = sync_references_with_filesystem(session, "models", collect_existing_paths=True,)@@ -348,7 +348,7 @@ def test_returns_empty_set_for_no_prefixes(session):def test_no_references_is_noop(session, temp_dir):"""No crash and no side effects when there are no references."""- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):survivors = sync_references_with_filesystem(session, "models", collect_existing_paths=True,)@@ -388,7 +388,7 @@ def test_sync_does_not_resurrect_soft_deleted_ref(session, temp_dir):_soft_delete_ref(session, "r1")session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models")session.commit()@@ -472,7 +472,7 @@ def test_sync_ignores_soft_deleted_seed_asset(session, temp_dir):_soft_delete_ref(session, "r1")session.commit()- with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ with patch("app.assets.scanner.get_scan_prefixes_for_root", return_value=[str(temp_dir)]):sync_references_with_filesystem(session, "models")session.commit()diff --git a/tests-unit/assets_test/test_temp_assets.py b/tests-unit/assets_test/test_temp_assets.pynew file mode 100644index 000000000..8a5f77d52--- /dev/null+++ b/tests-unit/assets_test/test_temp_assets.py@@ -0,0 +1,180 @@++import os+import tempfile+from pathlib import Path+from unittest.mock import patch++import pytest+from sqlalchemy import create_engine+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference, Base+from app.assets.database.queries.asset_reference import (+ mark_references_missing_outside_prefixes,+)+from app.assets.scanner import (+ collect_paths_for_roots,+ get_owned_prefixes,+ get_temp_prefixes,+ sync_prefixes_with_filesystem,+)+from app.assets.services.file_utils import get_mtime_ns++[email protected](autouse=True)+def autoclean_unit_test_assets():+ """Override parent autouse fixture - temp asset tests don't need server cleanup."""+ yield+++def session():+ engine = create_engine("sqlite:///:memory:")+ Base.metadata.create_all(engine)+ with Session(engine) as sess:+ yield sess+++def comfy_dirs():+ with tempfile.TemporaryDirectory() as base:+ dirs = {+ name: Path(base) / name+ for name in ("models", "input", "output", "temp", "elsewhere")+ }+ for d in dirs.values():+ d.mkdir()+ with (+ patch("folder_paths.get_input_directory", return_value=str(dirs["input"])),+ patch("folder_paths.get_output_directory", return_value=str(dirs["output"])),+ patch("folder_paths.get_temp_directory", return_value=str(dirs["temp"])),+ patch(+ "app.assets.scanner.get_comfy_models_folders",+ return_value=[("checkpoints", [str(dirs["models"])], set())],+ ),+ ):+ yield dirs+++def _write(directory: Path, name: str) -> str:+ p = directory / name+ p.write_bytes(b"\x00" * 100)+ return str(p)+++def _register(+ session: Session,+ file_path: str,+ ref_id: str,+ *,+ mtime_ns: int,+ asset_hash: str | None = "",+) -> None:+ if asset_hash == "":+ asset_hash = f"blake3:{ref_id}"+ session.add(Asset(id=f"asset-{ref_id}", hash=asset_hash, size_bytes=100))+ session.flush()+ session.add(+ AssetReference(+ id=ref_id,+ asset_id=f"asset-{ref_id}",+ name=os.path.basename(file_path),+ owner_id="",+ file_path=file_path,+ mtime_ns=mtime_ns,+ )+ )+ session.flush()+++def _mtime(path: str) -> int:+ return get_mtime_ns(os.stat(path, follow_symlinks=True))+++def test_owned_prefixes_include_temp(comfy_dirs):+ owned = get_owned_prefixes()+ assert str(comfy_dirs["temp"]) in owned, (+ "temp must be owned, or the prune disowns assets whose files are present"+ )+ for name in ("models", "input", "output"):+ assert str(comfy_dirs[name]) in owned, f"{name} must stay owned"+++def test_discovery_does_not_walk_temp(comfy_dirs):+ temp_file = _write(comfy_dirs["temp"], "preview.png")+ output_file = _write(comfy_dirs["output"], "render.png")++ with patch("app.assets.scanner.collect_models_files", return_value=[]):+ paths = collect_paths_for_roots(("models", "input", "output"))++ assert output_file in paths, "scan roots must still be walked"+ assert temp_file not in paths, (+ "temp is wiped before every scan, so walking it only ever finds nothing"+ )+++def test_prune_keeps_live_temp_reference(session, comfy_dirs):+ temp_file = _write(comfy_dirs["temp"], "preview.png")+ stray_file = _write(comfy_dirs["elsewhere"], "stray.png")+ _register(session, temp_file, "temp-ref", mtime_ns=_mtime(temp_file))+ _register(session, stray_file, "stray-ref", mtime_ns=_mtime(stray_file))+ session.commit()++ marked = mark_references_missing_outside_prefixes(session, get_owned_prefixes())+ session.commit()++ session.expire_all()+ assert marked == 1, "only the reference outside every owned directory is disowned"+ assert session.get(AssetReference, "temp-ref").is_missing is False, (+ "a temp file on disk is not missing, however often the prune runs"+ )+ assert session.get(AssetReference, "stray-ref").is_missing is True, (+ "owning temp must not stop the prune disowning files elsewhere"+ )+++def test_temp_sync_marks_deleted_file_missing(session, comfy_dirs):+ temp_file = _write(comfy_dirs["temp"], "preview.png")+ _register(session, temp_file, "temp-ref", mtime_ns=_mtime(temp_file))+ session.commit()+ os.remove(temp_file)++ sync_prefixes_with_filesystem(session, get_temp_prefixes())+ session.commit()++ session.expire_all()+ assert session.get(AssetReference, "temp-ref").is_missing is True, (+ "nothing else stats temp, so this pass is what retires a wiped file"+ )+++def test_temp_sync_drops_unhashed_asset_whose_file_is_gone(session, comfy_dirs):+ temp_file = _write(comfy_dirs["temp"], "preview.png")+ _register(session, temp_file, "temp-ref", mtime_ns=_mtime(temp_file), asset_hash=None)+ session.commit()+ os.remove(temp_file)++ sync_prefixes_with_filesystem(session, get_temp_prefixes())+ session.commit()++ session.expire_all()+ assert session.get(AssetReference, "temp-ref") is None, (+ "an unhashed asset with no surviving reference is retired, not kept as missing"+ )+ assert session.get(Asset, "asset-temp-ref") is None, (+ "the orphaned asset row goes with its last reference"+ )+++def test_temp_sync_keeps_live_file(session, comfy_dirs):+ temp_file = _write(comfy_dirs["temp"], "preview.png")+ _register(session, temp_file, "temp-ref", mtime_ns=_mtime(temp_file))+ session.commit()++ sync_prefixes_with_filesystem(session, get_temp_prefixes())+ session.commit()++ session.expire_all()+ ref = session.get(AssetReference, "temp-ref")+ assert ref.is_missing is False, "the file is still there"+ assert ref.needs_verify is False, "an unchanged file needs no re-verification"diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.pyindex 6aed6d6f3..55f813585 100644--- a/tests-unit/seeder_test/test_seeder.py+++ b/tests-unit/seeder_test/test_seeder.py@@ -399,7 +399,7 @@ class TestSeederMarkMissing:with (patch("app.assets.seeder.dependencies_available", return_value=True),patch(- "app.assets.seeder.get_all_known_prefixes",+ "app.assets.seeder.get_owned_prefixes",return_value=["/models", "/input", "/output"],),patch(@@ -454,8 +454,9 @@ class TestSeederMarkMissing:with (patch("app.assets.seeder.dependencies_available", return_value=True),- patch("app.assets.seeder.get_all_known_prefixes", return_value=["/models"]),+ patch("app.assets.seeder.get_owned_prefixes", return_value=["/models"]),patch("app.assets.seeder.mark_missing_outside_prefixes_safely", side_effect=track_mark),+ patch("app.assets.seeder.sync_temp_references_safely"),patch("app.assets.seeder.sync_root_safely", side_effect=track_sync),patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),@@ -469,6 +470,29 @@ class TestSeederMarkMissing:assert call_order[0] == "mark_missing"assert "sync_models" in call_order+ def test_prune_first_flag_reconciles_temp_references(+ self, fresh_seeder: _AssetSeeder+ ):+ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.get_owned_prefixes", return_value=["/models"]),+ patch("app.assets.seeder.mark_missing_outside_prefixes_safely", return_value=0),+ patch("app.assets.seeder.sync_temp_references_safely") as sync_temp,+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start(roots=("models",), prune_first=True)+ fresh_seeder.wait(timeout=5.0)++ assert sync_temp.called, (+ "temp is not a scan root, so the scan must reconcile it explicitly "+ "or files wiped at startup stay listed"+ )+class TestSeederPhases:"""Test phased scanning behavior."""diff --git a/tests/test_asset_seeder.py b/tests/test_asset_seeder.pyindex 4274dab8e..2c6727abe 100644--- a/tests/test_asset_seeder.py+++ b/tests/test_asset_seeder.py@@ -162,7 +162,7 @@ class TestPendingEnrichDrain:"""Verify that _run_scan drains _pending_enrich via start_enrich."""@patch("app.assets.seeder.dependencies_available", return_value=True)- @patch("app.assets.seeder.get_all_known_prefixes", return_value=[])+ @patch("app.assets.seeder.get_owned_prefixes", return_value=[])@patch("app.assets.seeder.sync_root_safely", return_value=set())@patch("app.assets.seeder.collect_paths_for_roots", return_value=[])@patch("app.assets.seeder.build_asset_specs", return_value=([], {}, 0))@@ -187,7 +187,7 @@ class TestPendingEnrichDrain:assert seeder._pending_enrich is None@patch("app.assets.seeder.dependencies_available", return_value=True)- @patch("app.assets.seeder.get_all_known_prefixes", return_value=[])+ @patch("app.assets.seeder.get_owned_prefixes", return_value=[])@patch("app.assets.seeder.sync_root_safely", return_value=set())@patch("app.assets.seeder.collect_paths_for_roots", return_value=[])@patch("app.assets.seeder.build_asset_specs", return_value=([], {}, 0))@@ -206,7 +206,7 @@ class TestPendingEnrichDrain:assert seeder._pending_enrich is None@patch("app.assets.seeder.dependencies_available", return_value=True)- @patch("app.assets.seeder.get_all_known_prefixes", return_value=[])+ @patch("app.assets.seeder.get_owned_prefixes", return_value=[])@patch("app.assets.seeder.sync_root_safely", return_value=set())@patch("app.assets.seeder.collect_paths_for_roots", return_value=[])@patch("app.assets.seeder.build_asset_specs", return_value=([], {}, 0))