Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

perf(assets): replace pathlib prefix checks in the asset scan (#16543)

Simon Pinfold7897b4ee3527
diff --git a/app/assets/helpers.py b/app/assets/helpers.py
index b65eb3b78..05876972a 100644
--- a/app/assets/helpers.py
+++ b/app/assets/helpers.py
@@ -1,10 +1,13 @@
"""Small conversions the asset system needs in more than one layer: canonical
-stored-hash strings, UTC timestamps, tag normalization, and a SQL predicate for
-path containment. That predicate is deliberately case-sensitive and
-component-bounded, because callers use it to choose rows for hard deletion.
+stored-hash strings, UTC timestamps, tag normalization, and path containment
+checks. The SQL predicate is deliberately case-sensitive and component-bounded,
+because callers use it to choose rows for hard deletion; the Python matcher
+follows ``Path.is_relative_to`` instead, including its platform case rules.
"""
+import functools
import os
+from collections.abc import Callable, Iterable
from datetime import datetime, timezone
import sqlalchemy as sa
@@ -42,6 +45,53 @@ def sql_path_under_prefix(
)
+def path_prefix_matcher(prefixes: Iterable[str]) -> Callable[[str], bool]:
+ """Return ``path -> Path(path).is_relative_to(<any prefix>)``, with the prefixes
+ normalized once.
+
+ The startup prune tests every catalogued row against every owned prefix, and
+ ``Path.is_relative_to`` walks the path's parents on each call, so a pathlib
+ check there costs rows x prefixes x depth. A normcase'd, separator-bounded
+ string prefix keeps its component bounds and platform case rules.
+ """
+ # abspath keeps exactly two leading separators, and pathlib treats that "//" as an
+ # anchor of its own: "//server/f" is not under "/". Such paths are only matched
+ # against prefixes with the same anchor.
+ double = os.sep * 2
+ exact: dict[bool, set[str]] = {False: set(), True: set()}
+ stems: dict[bool, list[str]] = {False: [], True: []}
+ for prefix in prefixes:
+ base = os.path.normcase(os.path.abspath(prefix))
+ is_double = base.startswith(double)
+ exact[is_double].add(base)
+ stems[is_double].append(base if base.endswith(os.sep) else base + os.sep)
+ stem_tuples = {key: tuple(value) for key, value in stems.items()}
+
+ def matches(path: str) -> bool:
+ candidate = os.path.normcase(os.path.abspath(path))
+ is_double = candidate.startswith(double)
+ return candidate in exact[is_double] or candidate.startswith(stem_tuples[is_double])
+
+ return matches
+
+
[email protected]_cache(maxsize=None)
+def cached_prefix_matcher(prefixes: tuple[str, ...]) -> Callable[[str], bool]:
+ """path_prefix_matcher, built once per distinct prefix tuple, for callers that check
+ every scanned file against the same folders.
+
+ Keyed on the raw prefixes: normalizing them in the key would bring back the per-call
+ cost this avoids. A folder-config change is just a new key.
+
+ Precondition: the prefixes are absolute. That is not checked. main.py and
+ extra_model_paths make their folders absolute, but folder_paths' setters and
+ add_model_folder_path store whatever they are given, so a custom node can register a
+ relative one. A relative prefix is resolved against the working directory once, on
+ first use, and that resolution is then frozen for as long as the key is unchanged.
+ """
+ return path_prefix_matcher(prefixes)
+
+
def escape_sql_like_string(s: str, escape: str = "!") -> tuple[str, str]:
"""Escapes %, _ and the escape char in a LIKE prefix.
diff --git a/app/assets/scanner.py b/app/assets/scanner.py
index 7fa7b7f41..cd3083ddf 100644
--- a/app/assets/scanner.py
+++ b/app/assets/scanner.py
@@ -27,13 +27,12 @@ from app.assets.database.queries import (
create_record,
)
from app.assets.database.models import Asset, AssetContent
-from app.assets.helpers import sql_path_under_prefix, to_stored_hash
+from app.assets.helpers import path_prefix_matcher, sql_path_under_prefix, to_stored_hash
from app.assets.lifecycle import get_excluded_scan_roots
from app.assets.scanner_changes import (
clear_pending_verifications,
detect_content_change,
drain_pending_verifications,
- is_path_under_prefixes,
live_contents_under_prefixes,
pending_recovery_count,
recover_missing_content,
@@ -240,10 +239,6 @@ def _sync_prefixes_in_write_txn(
return survivors
-def _is_under_prefixes(path: str, prefixes: list[str]) -> bool:
- return is_path_under_prefixes(path, prefixes)
-
-
def sync_root_safely(
root: RootType, progress: _ScanProgress | None = None
) -> set[str]:
@@ -304,7 +299,8 @@ def mark_contents_missing_outside_prefixes(
contents = session.scalars(
sa.select(AssetContent).where(AssetContent.is_missing.is_(False))
)
- missing = [content for content in contents if not _is_under_prefixes(content.path, prefixes)]
+ is_owned = path_prefix_matcher(prefixes)
+ missing = [content for content in contents if not is_owned(content.path)]
for content in missing:
mark_content_missing(session, content.id)
return len(missing)
diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py
index 5cfee7d83..19b60b5ff 100644
--- a/app/assets/scanner_changes.py
+++ b/app/assets/scanner_changes.py
@@ -8,7 +8,6 @@ so a restored file can never leave two live rows describing one location.
from __future__ import annotations
import os
-from pathlib import Path
from typing import Literal
import sqlalchemy as sa
@@ -21,7 +20,7 @@ from app.assets.database.queries.records import (
mark_content_missing,
unset_content_missing,
)
-from app.assets.helpers import sql_path_under_prefix, to_stored_hash
+from app.assets.helpers import path_prefix_matcher, sql_path_under_prefix, to_stored_hash
from app.assets.services.path_utils import compute_loader_path, get_name_and_tags_from_asset_path
from app.assets.services.snapshot_hash import snapshot_hash
@@ -108,8 +107,7 @@ def recover_missing_content(
def is_path_under_prefixes(path: str, prefixes: list[str]) -> bool:
- candidate = Path(os.path.abspath(path))
- return any(candidate.is_relative_to(os.path.abspath(prefix)) for prefix in prefixes)
+ return path_prefix_matcher(prefixes)(path)
def split_content(session: Session, content: AssetContent, stat_result: os.stat_result, hash_value: str | None) -> AssetContent:
diff --git a/app/assets/services/path_utils.py b/app/assets/services/path_utils.py
index 7c27c8878..6b1975aeb 100644
--- a/app/assets/services/path_utils.py
+++ b/app/assets/services/path_utils.py
@@ -3,6 +3,7 @@ from pathlib import Path
from typing import Literal
import folder_paths
+from app.assets.helpers import cached_prefix_matcher
_NON_MODEL_FOLDER_NAMES = frozenset({"configs", "custom_nodes"})
@@ -250,7 +251,6 @@ def get_backend_system_tags_from_path(path: str) -> list[str]:
category still get the ``models`` tag.
"""
fp_abs = os.path.abspath(path)
- fp_path = Path(fp_abs)
tags: list[str] = []
def _add(tag: str) -> None:
@@ -262,21 +262,19 @@ def get_backend_system_tags_from_path(path: str) -> list[str]:
("output", folder_paths.get_output_directory()),
("temp", folder_paths.get_temp_directory()),
):
- if fp_path.is_relative_to(os.path.abspath(base)):
+ if cached_prefix_matcher((base,))(fp_abs):
_add(role)
ext = os.path.splitext(fp_abs)[1].lower()
model_types: list[str] = []
under_models_base = False
for folder_name, bases, extensions in get_comfy_models_folders():
- for base in bases:
- if fp_path.is_relative_to(os.path.abspath(base)):
- under_models_base = True
- # Empty set accepts any extension, matching
- # folder_paths.filter_files_extensions semantics.
- if not extensions or ext in extensions:
- model_types.append(folder_name)
- break
+ if cached_prefix_matcher(tuple(bases))(fp_abs):
+ under_models_base = True
+ # Empty set accepts any extension, matching
+ # folder_paths.filter_files_extensions semantics.
+ if not extensions or ext in extensions:
+ model_types.append(folder_name)
if under_models_base:
_add("models")
@@ -306,7 +304,7 @@ def get_known_input_subfolder_tags_from_path(path: str) -> list[str]:
"""
fp_abs = os.path.abspath(path)
input_base = os.path.abspath(folder_paths.get_input_directory())
- if not Path(fp_abs).is_relative_to(input_base):
+ if not cached_prefix_matcher((input_base,))(fp_abs):
return []
rel = os.path.relpath(fp_abs, input_base)
diff --git a/tests-unit/assets_test/services/path_prefix_cases.py b/tests-unit/assets_test/services/path_prefix_cases.py
index f5586d9e8..789842f9b 100644
--- a/tests-unit/assets_test/services/path_prefix_cases.py
+++ b/tests-unit/assets_test/services/path_prefix_cases.py
@@ -33,3 +33,19 @@ def prefix_case_paths(root: str) -> list[tuple[str, bool]]:
def expected_prefix_case_paths(root: str) -> set[str]:
return {os.path.abspath(path) for path, expected in prefix_case_paths(root) if expected}
+
+
+def anchor_case_paths() -> list[tuple[str, str, bool]]:
+ """(path, prefix, Path(path).is_relative_to(prefix)) across POSIX anchors.
+
+ abspath keeps exactly two leading slashes, and pathlib treats "//" as its own
+ anchor, so "//server/..." is not under "/". Three or more collapse to "/".
+ """
+ return [
+ ("//server/file", "/", False),
+ ("//server/file", "//server", True),
+ ("//server", "//", True),
+ ("/x", "//", False),
+ ("///x/y", "/", True),
+ ("/x/y", "/", True),
+ ]
diff --git a/tests-unit/assets_test/services/test_path_prefix_matcher.py b/tests-unit/assets_test/services/test_path_prefix_matcher.py
new file mode 100644
index 000000000..0721b4b63
--- /dev/null
+++ b/tests-unit/assets_test/services/test_path_prefix_matcher.py
@@ -0,0 +1,38 @@
+import os
+from pathlib import Path
+
+import pytest
+
+from app.assets.helpers import path_prefix_matcher
+
+from .path_prefix_cases import anchor_case_paths, prefix_case_paths
+
+
+def _is_relative_to_any(path: str, prefixes: list[str]) -> bool:
+ candidate = Path(os.path.abspath(path))
+ return any(candidate.is_relative_to(os.path.abspath(prefix)) for prefix in prefixes)
+
+
+def test_matches_path_is_relative_to(tmp_path):
+ root = os.path.abspath(str(tmp_path / "root"))
+ other = os.path.abspath(str(tmp_path / "models" / "checkpoints"))
+ for prefixes in ([root], [other, root], [root + os.sep]):
+ matches = path_prefix_matcher(prefixes)
+ for path, _ in prefix_case_paths(root):
+ assert matches(path) == _is_relative_to_any(path, prefixes), (path, prefixes)
+
+
+def test_no_prefixes_matches_nothing(tmp_path):
+ assert path_prefix_matcher([])(str(tmp_path)) is False
+
+
+def test_filesystem_root_contains_everything(tmp_path):
+ path = os.path.abspath(str(tmp_path))
+ assert path_prefix_matcher([Path(path).anchor])(path) is True
+
+
[email protected](os.sep != "/", reason="POSIX anchors; Windows drives and UNC shares normalize differently")
+def test_matches_path_is_relative_to_across_posix_anchors():
+ for path, prefix, expected in anchor_case_paths():
+ assert _is_relative_to_any(path, [prefix]) is expected, (path, prefix)
+ assert path_prefix_matcher([prefix])(path) is expected, (path, prefix)
diff --git a/tests-unit/assets_test/services/test_path_utils.py b/tests-unit/assets_test/services/test_path_utils.py
index ddf23c676..97f92e1df 100644
--- a/tests-unit/assets_test/services/test_path_utils.py
+++ b/tests-unit/assets_test/services/test_path_utils.py
@@ -6,11 +6,13 @@ from unittest.mock import patch
import pytest
+from app.assets.helpers import cached_prefix_matcher
from app.assets.services.path_utils import (
compute_display_name,
compute_loader_path,
compute_logical_path,
get_asset_category_and_relative_path,
+ get_backend_system_tags_from_path,
get_known_input_subfolder_tags_from_path,
get_known_subfolder_tags,
get_name_and_tags_from_asset_path,
@@ -623,3 +625,35 @@ class TestResolveDestinationFromTags:
resolve_destination_from_tags(
["models", f"model_type:{folder_name}"]
)
+
+
+class TestCachedPrefixMatchers:
+ def test_unchanged_config_reuses_matchers_across_files(self, fake_dirs):
+ files = [fake_dirs["output"] / f"f{i}.png" for i in range(5)]
+ assert get_backend_system_tags_from_path(str(files[0])) == ["output"]
+ hits = cached_prefix_matcher.cache_info().hits
+
+ for f in files[1:]:
+ assert get_backend_system_tags_from_path(str(f)) == ["output"]
+
+ # input, output, temp and the checkpoints category, per file.
+ assert cached_prefix_matcher.cache_info().hits - hits >= 4 * 4
+
+ def test_changed_folder_config_builds_a_correct_new_matcher(self, fake_dirs, tmp_path):
+ old = fake_dirs["output"] / "old.png"
+ assert get_backend_system_tags_from_path(str(old)) == ["output"]
+ moved = tmp_path / "moved-output"
+ moved.mkdir()
+ new = moved / "new.png"
+
+ with patch("app.assets.services.path_utils.folder_paths") as mock_fp:
+ mock_fp.get_input_directory.return_value = str(fake_dirs["input"])
+ mock_fp.get_output_directory.return_value = str(moved)
+ mock_fp.get_temp_directory.return_value = str(fake_dirs["temp"])
+ misses = cached_prefix_matcher.cache_info().misses
+
+ assert get_backend_system_tags_from_path(str(new)) == ["output"]
+ with pytest.raises(ValueError):
+ get_backend_system_tags_from_path(str(old))
+
+ assert cached_prefix_matcher.cache_info().misses > misses