The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
perf(assets): replace pathlib prefix checks in the asset scan (#16543)
Simon Pinfold7897b4ee3527
diff --git a/app/assets/helpers.py b/app/assets/helpers.pyindex b65eb3b78..05876972a 100644--- a/app/assets/helpers.py+++ b/app/assets/helpers.py@@ -1,10 +1,13 @@"""Small conversions the asset system needs in more than one layer: canonical-stored-hash strings, UTC timestamps, tag normalization, and a SQL predicate for-path containment. That predicate is deliberately case-sensitive and-component-bounded, because callers use it to choose rows for hard deletion.+stored-hash strings, UTC timestamps, tag normalization, and path containment+checks. The SQL predicate is deliberately case-sensitive and component-bounded,+because callers use it to choose rows for hard deletion; the Python matcher+follows ``Path.is_relative_to`` instead, including its platform case rules."""+import functoolsimport os+from collections.abc import Callable, Iterablefrom datetime import datetime, timezoneimport sqlalchemy as sa@@ -42,6 +45,53 @@ def sql_path_under_prefix()+def path_prefix_matcher(prefixes: Iterable[str]) -> Callable[[str], bool]:+ """Return ``path -> Path(path).is_relative_to(<any prefix>)``, with the prefixes+ normalized once.++ The startup prune tests every catalogued row against every owned prefix, and+ ``Path.is_relative_to`` walks the path's parents on each call, so a pathlib+ check there costs rows x prefixes x depth. A normcase'd, separator-bounded+ string prefix keeps its component bounds and platform case rules.+ """+ # abspath keeps exactly two leading separators, and pathlib treats that "//" as an+ # anchor of its own: "//server/f" is not under "/". Such paths are only matched+ # against prefixes with the same anchor.+ double = os.sep * 2+ exact: dict[bool, set[str]] = {False: set(), True: set()}+ stems: dict[bool, list[str]] = {False: [], True: []}+ for prefix in prefixes:+ base = os.path.normcase(os.path.abspath(prefix))+ is_double = base.startswith(double)+ exact[is_double].add(base)+ stems[is_double].append(base if base.endswith(os.sep) else base + os.sep)+ stem_tuples = {key: tuple(value) for key, value in stems.items()}++ def matches(path: str) -> bool:+ candidate = os.path.normcase(os.path.abspath(path))+ is_double = candidate.startswith(double)+ return candidate in exact[is_double] or candidate.startswith(stem_tuples[is_double])++ return matches++[email protected]_cache(maxsize=None)+def cached_prefix_matcher(prefixes: tuple[str, ...]) -> Callable[[str], bool]:+ """path_prefix_matcher, built once per distinct prefix tuple, for callers that check+ every scanned file against the same folders.++ Keyed on the raw prefixes: normalizing them in the key would bring back the per-call+ cost this avoids. A folder-config change is just a new key.++ Precondition: the prefixes are absolute. That is not checked. main.py and+ extra_model_paths make their folders absolute, but folder_paths' setters and+ add_model_folder_path store whatever they are given, so a custom node can register a+ relative one. A relative prefix is resolved against the working directory once, on+ first use, and that resolution is then frozen for as long as the key is unchanged.+ """+ return path_prefix_matcher(prefixes)++def escape_sql_like_string(s: str, escape: str = "!") -> tuple[str, str]:"""Escapes %, _ and the escape char in a LIKE prefix.diff --git a/app/assets/scanner.py b/app/assets/scanner.pyindex 7fa7b7f41..cd3083ddf 100644--- a/app/assets/scanner.py+++ b/app/assets/scanner.py@@ -27,13 +27,12 @@ from app.assets.database.queries import (create_record,)from app.assets.database.models import Asset, AssetContent-from app.assets.helpers import sql_path_under_prefix, to_stored_hash+from app.assets.helpers import path_prefix_matcher, sql_path_under_prefix, to_stored_hashfrom app.assets.lifecycle import get_excluded_scan_rootsfrom app.assets.scanner_changes import (clear_pending_verifications,detect_content_change,drain_pending_verifications,- is_path_under_prefixes,live_contents_under_prefixes,pending_recovery_count,recover_missing_content,@@ -240,10 +239,6 @@ def _sync_prefixes_in_write_txn(return survivors-def _is_under_prefixes(path: str, prefixes: list[str]) -> bool:- return is_path_under_prefixes(path, prefixes)--def sync_root_safely(root: RootType, progress: _ScanProgress | None = None) -> set[str]:@@ -304,7 +299,8 @@ def mark_contents_missing_outside_prefixes(contents = session.scalars(sa.select(AssetContent).where(AssetContent.is_missing.is_(False)))- missing = [content for content in contents if not _is_under_prefixes(content.path, prefixes)]+ is_owned = path_prefix_matcher(prefixes)+ missing = [content for content in contents if not is_owned(content.path)]for content in missing:mark_content_missing(session, content.id)return len(missing)diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.pyindex 5cfee7d83..19b60b5ff 100644--- a/app/assets/scanner_changes.py+++ b/app/assets/scanner_changes.py@@ -8,7 +8,6 @@ so a restored file can never leave two live rows describing one location.from __future__ import annotationsimport os-from pathlib import Pathfrom typing import Literalimport sqlalchemy as sa@@ -21,7 +20,7 @@ from app.assets.database.queries.records import (mark_content_missing,unset_content_missing,)-from app.assets.helpers import sql_path_under_prefix, to_stored_hash+from app.assets.helpers import path_prefix_matcher, sql_path_under_prefix, to_stored_hashfrom app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_pathfrom app.assets.services.snapshot_hash import snapshot_hash@@ -108,8 +107,7 @@ def recover_missing_content(def is_path_under_prefixes(path: str, prefixes: list[str]) -> bool:- candidate = Path(os.path.abspath(path))- return any(candidate.is_relative_to(os.path.abspath(prefix)) for prefix in prefixes)+ return path_prefix_matcher(prefixes)(path)def split_content(session: Session, content: AssetContent, stat_result: os.stat_result, hash_value: str | None) -> AssetContent:diff --git a/app/assets/services/path_utils.py b/app/assets/services/path_utils.pyindex 7c27c8878..6b1975aeb 100644--- a/app/assets/services/path_utils.py+++ b/app/assets/services/path_utils.py@@ -3,6 +3,7 @@ from pathlib import Pathfrom typing import Literalimport folder_paths+from app.assets.helpers import cached_prefix_matcher_NON_MODEL_FOLDER_NAMES = frozenset({"configs", "custom_nodes"})@@ -250,7 +251,6 @@ def get_backend_system_tags_from_path(path: str) -> list[str]:category still get the ``models`` tag."""fp_abs = os.path.abspath(path)- fp_path = Path(fp_abs)tags: list[str] = []def _add(tag: str) -> None:@@ -262,21 +262,19 @@ def get_backend_system_tags_from_path(path: str) -> list[str]:("output", folder_paths.get_output_directory()),("temp", folder_paths.get_temp_directory()),):- if fp_path.is_relative_to(os.path.abspath(base)):+ if cached_prefix_matcher((base,))(fp_abs):_add(role)ext = os.path.splitext(fp_abs)[1].lower()model_types: list[str] = []under_models_base = Falsefor folder_name, bases, extensions in get_comfy_models_folders():- for base in bases:- if fp_path.is_relative_to(os.path.abspath(base)):- under_models_base = True- # Empty set accepts any extension, matching- # folder_paths.filter_files_extensions semantics.- if not extensions or ext in extensions:- model_types.append(folder_name)- break+ if cached_prefix_matcher(tuple(bases))(fp_abs):+ under_models_base = True+ # Empty set accepts any extension, matching+ # folder_paths.filter_files_extensions semantics.+ if not extensions or ext in extensions:+ model_types.append(folder_name)if under_models_base:_add("models")@@ -306,7 +304,7 @@ def get_known_input_subfolder_tags_from_path(path: str) -> list[str]:"""fp_abs = os.path.abspath(path)input_base = os.path.abspath(folder_paths.get_input_directory())- if not Path(fp_abs).is_relative_to(input_base):+ if not cached_prefix_matcher((input_base,))(fp_abs):return []rel = os.path.relpath(fp_abs, input_base)diff --git a/tests-unit/assets_test/services/path_prefix_cases.py b/tests-unit/assets_test/services/path_prefix_cases.pyindex f5586d9e8..789842f9b 100644--- a/tests-unit/assets_test/services/path_prefix_cases.py+++ b/tests-unit/assets_test/services/path_prefix_cases.py@@ -33,3 +33,19 @@ def prefix_case_paths(root: str) -> list[tuple[str, bool]]:def expected_prefix_case_paths(root: str) -> set[str]:return {os.path.abspath(path) for path, expected in prefix_case_paths(root) if expected}+++def anchor_case_paths() -> list[tuple[str, str, bool]]:+ """(path, prefix, Path(path).is_relative_to(prefix)) across POSIX anchors.++ abspath keeps exactly two leading slashes, and pathlib treats "//" as its own+ anchor, so "//server/..." is not under "/". Three or more collapse to "/".+ """+ return [+ ("//server/file", "/", False),+ ("//server/file", "//server", True),+ ("//server", "//", True),+ ("/x", "//", False),+ ("///x/y", "/", True),+ ("/x/y", "/", True),+ ]diff --git a/tests-unit/assets_test/services/test_path_prefix_matcher.py b/tests-unit/assets_test/services/test_path_prefix_matcher.pynew file mode 100644index 000000000..0721b4b63--- /dev/null+++ b/tests-unit/assets_test/services/test_path_prefix_matcher.py@@ -0,0 +1,38 @@+import os+from pathlib import Path++import pytest++from app.assets.helpers import path_prefix_matcher++from .path_prefix_cases import anchor_case_paths, prefix_case_paths+++def _is_relative_to_any(path: str, prefixes: list[str]) -> bool:+ candidate = Path(os.path.abspath(path))+ return any(candidate.is_relative_to(os.path.abspath(prefix)) for prefix in prefixes)+++def test_matches_path_is_relative_to(tmp_path):+ root = os.path.abspath(str(tmp_path / "root"))+ other = os.path.abspath(str(tmp_path / "models" / "checkpoints"))+ for prefixes in ([root], [other, root], [root + os.sep]):+ matches = path_prefix_matcher(prefixes)+ for path, _ in prefix_case_paths(root):+ assert matches(path) == _is_relative_to_any(path, prefixes), (path, prefixes)+++def test_no_prefixes_matches_nothing(tmp_path):+ assert path_prefix_matcher([])(str(tmp_path)) is False+++def test_filesystem_root_contains_everything(tmp_path):+ path = os.path.abspath(str(tmp_path))+ assert path_prefix_matcher([Path(path).anchor])(path) is True++[email protected](os.sep != "/", reason="POSIX anchors; Windows drives and UNC shares normalize differently")+def test_matches_path_is_relative_to_across_posix_anchors():+ for path, prefix, expected in anchor_case_paths():+ assert _is_relative_to_any(path, [prefix]) is expected, (path, prefix)+ assert path_prefix_matcher([prefix])(path) is expected, (path, prefix)diff --git a/tests-unit/assets_test/services/test_path_utils.py b/tests-unit/assets_test/services/test_path_utils.pyindex ddf23c676..97f92e1df 100644--- a/tests-unit/assets_test/services/test_path_utils.py+++ b/tests-unit/assets_test/services/test_path_utils.py@@ -6,11 +6,13 @@ from unittest.mock import patchimport pytest+from app.assets.helpers import cached_prefix_matcherfrom app.assets.services.path_utils import (compute_display_name,compute_loader_path,compute_logical_path,get_asset_category_and_relative_path,+ get_backend_system_tags_from_path,get_known_input_subfolder_tags_from_path,get_known_subfolder_tags,get_name_and_tags_from_asset_path,@@ -623,3 +625,35 @@ class TestResolveDestinationFromTags:resolve_destination_from_tags(["models", f"model_type:{folder_name}"])+++class TestCachedPrefixMatchers:+ def test_unchanged_config_reuses_matchers_across_files(self, fake_dirs):+ files = [fake_dirs["output"] / f"f{i}.png" for i in range(5)]+ assert get_backend_system_tags_from_path(str(files[0])) == ["output"]+ hits = cached_prefix_matcher.cache_info().hits++ for f in files[1:]:+ assert get_backend_system_tags_from_path(str(f)) == ["output"]++ # input, output, temp and the checkpoints category, per file.+ assert cached_prefix_matcher.cache_info().hits - hits >= 4 * 4++ def test_changed_folder_config_builds_a_correct_new_matcher(self, fake_dirs, tmp_path):+ old = fake_dirs["output"] / "old.png"+ assert get_backend_system_tags_from_path(str(old)) == ["output"]+ moved = tmp_path / "moved-output"+ moved.mkdir()+ new = moved / "new.png"++ with patch("app.assets.services.path_utils.folder_paths") as mock_fp:+ mock_fp.get_input_directory.return_value = str(fake_dirs["input"])+ mock_fp.get_output_directory.return_value = str(moved)+ mock_fp.get_temp_directory.return_value = str(fake_dirs["temp"])+ misses = cached_prefix_matcher.cache_info().misses++ assert get_backend_system_tags_from_path(str(new)) == ["output"]+ with pytest.raises(ValueError):+ get_backend_system_tags_from_path(str(old))++ assert cached_prefix_matcher.cache_info().misses > misses