Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

perf(assets): let the partial live-path index serve live-row-at-path lookups (#16659)

Simon Pinfold30c4c3aa756f
diff --git a/app/assets/database/models.py b/app/assets/database/models.py
index 2c261ed04..f2855a523 100644
--- a/app/assets/database/models.py
+++ b/app/assets/database/models.py
@@ -58,6 +58,8 @@ class AssetContent(Base):
records: Mapped[list[Asset]] = relationship(back_populates="content")
__table_args__ = (
+ # Query live rows with is_missing == false() ("= 0"): SQLite won't use this partial
+ # index for is_(False) ("IS 0"), so a path lookup written that way scans the table.
Index(
"uq_asset_contents_path_live",
"path",
diff --git a/app/assets/database/queries/records.py b/app/assets/database/queries/records.py
index 60b7f8231..bb12e2315 100644
--- a/app/assets/database/queries/records.py
+++ b/app/assets/database/queries/records.py
@@ -68,7 +68,7 @@ def create_content_reporting_insert(session: Session, path: str, hash: str | Non
except IntegrityError as error:
if not is_live_path_conflict(error):
raise
- winner = session.execute(sa.select(AssetContent).where(AssetContent.path == path, AssetContent.is_missing.is_(False))).scalar_one()
+ winner = session.execute(sa.select(AssetContent).where(AssetContent.path == path, AssetContent.is_missing == sa.false())).scalar_one()
return winner, False
@@ -141,7 +141,7 @@ def get_record_by_path_or_none(session: Session, path: str) -> Asset | None:
return session.scalar(
sa.select(Asset)
.join(AssetContent, Asset.content_id == AssetContent.id)
- .where(AssetContent.path == path, AssetContent.is_missing.is_(False))
+ .where(AssetContent.path == path, AssetContent.is_missing == sa.false())
.order_by(Asset.created_at.desc(), Asset.id.desc())
.limit(1)
)
diff --git a/app/assets/scanner_changes.py b/app/assets/scanner_changes.py
index 9f5ddd452..e1d84c6b1 100644
--- a/app/assets/scanner_changes.py
+++ b/app/assets/scanner_changes.py
@@ -57,7 +57,7 @@ def recover_missing_content(
return "no_match"
occupied = session.scalar(
sa.select(AssetContent.id)
- .where(AssetContent.path == path, AssetContent.is_missing.is_(False))
+ .where(AssetContent.path == path, AssetContent.is_missing == sa.false())
.limit(1)
)
if occupied is not None:
diff --git a/app/assets/services/hash_mode_state.py b/app/assets/services/hash_mode_state.py
index ff7f77179..b1b5d6abd 100644
--- a/app/assets/services/hash_mode_state.py
+++ b/app/assets/services/hash_mode_state.py
@@ -14,7 +14,7 @@ from collections import deque
from dataclasses import dataclass
from typing import Final
-from sqlalchemy import select
+from sqlalchemy import false, select
from sqlalchemy.orm import Session
from app.assets import mode as _mode
@@ -102,7 +102,7 @@ def _retry_or_retire(session: Session, entry: _PendingEntry) -> None:
return
content = session.scalars(
select(AssetContent).where(
- AssetContent.path == entry.path, AssetContent.is_missing.is_(False)
+ AssetContent.path == entry.path, AssetContent.is_missing == false()
)
).first()
if content is not None:
@@ -138,7 +138,7 @@ def drain_transition_queue(session: Session) -> None:
except FileNotFoundError:
gone = session.scalars(
select(AssetContent).where(
- AssetContent.path == path, AssetContent.is_missing.is_(False)
+ AssetContent.path == path, AssetContent.is_missing == false()
)
).first()
if gone is not None:
@@ -152,7 +152,7 @@ def drain_transition_queue(session: Session) -> None:
stored_hash = to_stored_hash(digest)
content = session.scalars(
select(AssetContent).where(
- AssetContent.path == path, AssetContent.is_missing.is_(False)
+ AssetContent.path == path, AssetContent.is_missing == false()
)
).first()
if content is None:
diff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.py
index 52bcde4c0..c20cf9431 100644
--- a/app/assets/services/ingest.py
+++ b/app/assets/services/ingest.py
@@ -13,7 +13,7 @@ import mimetypes
import os
from typing import Any, NamedTuple, Sequence
-from sqlalchemy import func, select
+from sqlalchemy import false, func, select
from sqlalchemy.orm import Session
from app.assets import mode
@@ -310,7 +310,7 @@ def _reconcile_live_content_at_path(
existing = session.scalars(
select(AssetContent).where(
AssetContent.path == locator,
- AssetContent.is_missing.is_(False),
+ AssetContent.is_missing == false(),
)
).first()
if existing is None:
@@ -365,7 +365,7 @@ def _settle_destination_before_write(session: Session, dest_abs: str) -> None:
existing = session.scalars(
select(AssetContent).where(
AssetContent.path == dest_abs,
- AssetContent.is_missing.is_(False),
+ AssetContent.is_missing == false(),
)
).first()
if existing is None:
@@ -661,7 +661,7 @@ def register_cached_output(
with create_session() as session:
existing = session.scalars(
select(AssetContent).where(
- AssetContent.path == locator, AssetContent.is_missing.is_(False)
+ AssetContent.path == locator, AssetContent.is_missing == false()
)
).first()
if existing is None:
@@ -740,7 +740,7 @@ def register_executed_output(
existing = session.scalars(
select(AssetContent).where(
AssetContent.path == locator,
- AssetContent.is_missing.is_(False),
+ AssetContent.is_missing == false(),
)
).first()
if existing is not None:
diff --git a/tests-unit/assets_test/services/test_live_path_index.py b/tests-unit/assets_test/services/test_live_path_index.py
new file mode 100644
index 000000000..420d5a5e7
--- /dev/null
+++ b/tests-unit/assets_test/services/test_live_path_index.py
@@ -0,0 +1,163 @@
+"""Every "live row at this path" lookup must be served by the partial index
+uq_asset_contents_path_live. Written as ``is_missing IS 0`` it is not, and the lookup scans
+asset_contents, which inside a write transaction holds the lock for as long as the scan runs."""
+
+import os
+import re
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import sqlalchemy as sa
+
+import folder_paths
+from app.assets.database.models import AssetContent
+from app.assets.database.queries.records import (
+ create_content,
+ create_content_reporting_insert,
+ get_record_by_path_or_none,
+)
+from app.assets.scanner_changes import recover_missing_content
+from app.assets.services import ingest
+from app.assets.services.hash_mode_state import (
+ clear_transition_queue,
+ drain_transition_queue,
+ enqueue_transition_work,
+)
+
+# SQLite before 3.36 words it "SCAN TABLE asset_contents".
+_CONTENT_SCAN = re.compile(r"\bSCAN (TABLE )?asset_contents\b")
+
+
+@contextmanager
+def no_content_scans(db_engine):
+ """Fail if any statement run inside the block scans asset_contents."""
+ statements: list[tuple[str, object]] = []
+
+ def capture(conn, cursor, statement, parameters, context, executemany):
+ if statement.lstrip().upper().startswith(("SELECT", "UPDATE", "DELETE")):
+ statements.append((statement, parameters))
+
+ sa.event.listen(db_engine, "before_cursor_execute", capture)
+ try:
+ yield
+ finally:
+ sa.event.remove(db_engine, "before_cursor_execute", capture)
+
+ path_lookups = [s for s in statements if "asset_contents.path = ?" in s[0]]
+ assert path_lookups, "the block ran no live-path lookup"
+ with db_engine.connect() as conn:
+ for statement, parameters in statements:
+ plan = conn.exec_driver_sql(f"EXPLAIN QUERY PLAN {statement}", parameters).all()
+ assert not any(_CONTENT_SCAN.search(row[-1]) for row in plan), (statement, plan)
+
+
+def _output_file(name: str) -> str:
+ path = Path(folder_paths.get_output_directory()) / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(b"output bytes")
+ return str(path)
+
+
+def output_file():
+ paths: list[str] = []
+
+ def make(name: str) -> str:
+ paths.append(_output_file(name))
+ return paths[-1]
+
+ yield make
+ for path in paths:
+ Path(path).unlink(missing_ok=True)
+
+
+def test_register_executed_output(mock_create_session, db_engine, output_file):
+ path = output_file("live-index-executed.png")
+ ingest.register_executed_output(path, job_id=None)
+ with no_content_scans(db_engine):
+ assert ingest.register_executed_output(path, job_id=None) is not None
+
+
+def test_register_cached_output(mock_create_session, db_engine, output_file):
+ path = output_file("live-index-cached.png")
+ ingest.register_executed_output(path, job_id=None)
+ with no_content_scans(db_engine):
+ assert ingest.register_cached_output(path, job_id=None) is not None
+
+
+def test_register_file_in_place(mock_create_session, db_engine, output_file):
+ path = output_file("live-index-in-place.png")
+ ingest.register_file_in_place(path, "in-place.png", ["output"])
+ with no_content_scans(db_engine):
+ ingest.register_file_in_place(path, "in-place.png", ["output"])
+
+
+def test_upload_onto_an_existing_destination(mock_create_session, db_engine):
+ def upload():
+ # Staged where the upload route stages it: the move into input/ is an os.replace,
+ # which fails across drives. The upload removes this directory along with the file.
+ uploads = Path(folder_paths.get_temp_directory()) / "uploads"
+ uploads.mkdir(parents=True, exist_ok=True)
+ temp = Path(tempfile.mkdtemp(dir=uploads)) / "upload.tmp"
+ temp.write_bytes(b"upload bytes")
+ return ingest.upload_from_temp_path(str(temp), name="live-index-upload.png", tags=["input"])
+
+ result = upload()
+ try:
+ with no_content_scans(db_engine):
+ upload()
+ finally:
+ with mock_create_session() as session:
+ for content in session.scalars(sa.select(AssetContent)):
+ Path(content.path).unlink(missing_ok=True)
+ assert result is not None
+
+
+def test_insert_race_fallback(session, db_engine, temp_dir):
+ path = str(temp_dir / "raced.png")
+ create_content(session, path)
+ with no_content_scans(db_engine):
+ _, inserted = create_content_reporting_insert(session, path)
+ assert inserted is False
+
+
+def test_get_record_by_path_or_none(session, db_engine, temp_dir):
+ with no_content_scans(db_engine):
+ get_record_by_path_or_none(session, str(temp_dir / "absent.png"))
+
+
+def test_hash_on_recovery_occupied_check(session, db_engine, temp_dir):
+ path = str(temp_dir / "returning.png")
+ with no_content_scans(db_engine):
+ recover_missing_content(session, path, None, hashing_is_enabled=True)
+
+
[email protected]("outcome", ["hashed", "vanished", "retired"])
+def test_off_to_on_transition_drain(session, db_engine, temp_dir, outcome):
+ path = temp_dir / "transition.png"
+ path.write_bytes(b"bytes")
+ create_content(session, str(path))
+ session.commit()
+ enqueue_transition_work(session, "off_to_on")
+ if outcome == "vanished":
+ path.unlink()
+ try:
+ with no_content_scans(db_engine):
+ if outcome == "retired":
+ with patch("app.assets.services.hash_mode_state.snapshot_hash", side_effect=OSError):
+ for _ in range(3):
+ drain_transition_queue(session)
+ else:
+ drain_transition_queue(session)
+ session.commit()
+ finally:
+ clear_transition_queue()
+ content = session.scalars(sa.select(AssetContent).where(AssetContent.path == os.path.abspath(path))).one()
+ assert (content.hash is not None, content.is_missing) == {
+ "hashed": (True, False),
+ "vanished": (False, True),
+ "retired": (False, False),
+ }[outcome]