The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
fix(assets): date scanned assets by their file's mtime (#16810)
Simon Pinfold0df64eb242b7
diff --git a/alembic_db/versions/0009_created_at_from_mtime.py b/alembic_db/versions/0009_created_at_from_mtime.pynew file mode 100644index 000000000..a4f6943b2--- /dev/null+++ b/alembic_db/versions/0009_created_at_from_mtime.py@@ -0,0 +1,42 @@+"""+Date each scanned record by its file's mtime instead of the scan time.++Revision ID: 0009_created_at_from_mtime+Revises: 0008_drop_asset_meta+Create Date: 2026-10-05+"""++from datetime import datetime, timedelta++from alembic import op+import sqlalchemy as sa++revision = "0009_created_at_from_mtime"+down_revision = "0008_drop_asset_meta"+branch_labels = None+depends_on = None+++def upgrade() -> None:+ # The scanner writes a jobless record in the same transaction as its content; a record+ # added to existing content later (a re-upload, from-hash) is not the scan's, and uploads,+ # which also create both together, carry the "uploaded" tag.+ rows = op.get_bind().execute(sa.text(+ "SELECT a.id, a.created_at, c.mtime_ns FROM assets AS a JOIN asset_contents AS c"+ " ON c.id = a.content_id WHERE a.job_id IS NULL AND c.mtime_ns IS NOT NULL"+ " AND abs(julianday(a.created_at) - julianday(c.created_at)) * 86400 < 1"+ " AND NOT EXISTS (SELECT 1 FROM asset_tags AS t WHERE t.asset_id = a.id AND t.tag_name = 'uploaded')"+ ).columns(created_at=sa.DateTime)).all()+ # Only ever earlier: an mtime changed after cataloguing must not move a record above newer work.+ # The conversion is inlined, not app.assets.helpers', so this migration can't change if that helper does.+ params = [+ {"id": record_id, "created": min(datetime(1970, 1, 1) + timedelta(microseconds=mtime_ns // 1000), created_at)}+ for record_id, created_at, mtime_ns in rows+ ]+ if params:+ op.get_bind().execute(sa.text("UPDATE assets SET created_at = :created WHERE id = :id").bindparams(+ sa.bindparam("created", type_=sa.DateTime)), params)+++def downgrade() -> None:+ passdiff --git a/app/assets/database/queries/records.py b/app/assets/database/queries/records.pyindex 19dec16cf..a41317468 100644--- a/app/assets/database/queries/records.py+++ b/app/assets/database/queries/records.py@@ -104,8 +104,10 @@ def ensure_tag_link(session: Session, *, asset_id: str, tag_name: str, origin: sreturn True-def create_record(session: Session, content_id: str, name: str, mime_type: str | None = None, job_id: str | None = None, loader_path: str | None = None, tags: Sequence[str] | None = None, *, system_metadata: dict[str, Any] | None = None) -> Asset:+def create_record(session: Session, content_id: str, name: str, mime_type: str | None = None, job_id: str | None = None, loader_path: str | None = None, tags: Sequence[str] | None = None, *, system_metadata: dict[str, Any] | None = None, created_at: datetime | None = None) -> Asset:record = Asset(content_id=content_id, name=name, mime_type=mime_type, job_id=job_id, loader_path=loader_path, system_metadata=system_metadata)+ if created_at is not None:+ record.created_at = created_atsession.add(record)session.flush()for tag_name in dict.fromkeys(tags or ()):diff --git a/app/assets/helpers.py b/app/assets/helpers.pyindex b83a6855a..4b98d5288 100644--- a/app/assets/helpers.py+++ b/app/assets/helpers.py@@ -8,7 +8,7 @@ follows ``Path.is_relative_to`` instead, including its platform case rules.import functoolsimport osfrom collections.abc import Callable, Iterable-from datetime import datetime, timezone+from datetime import datetime, timedelta, timezoneimport sqlalchemy as safrom sqlalchemy.sql import ColumnElement@@ -144,6 +144,17 @@ def get_utc_now() -> datetime:return datetime.now(timezone.utc).replace(tzinfo=None)+def mtime_ns_to_utc(mtime_ns: int, ctime_ns: int) -> datetime:+ """Naive UTC time of a file mtime. A future mtime falls back to the ctime, capped at now: the+ creation time on Windows, the last metadata change elsewhere."""+ now = get_utc_now()+ made = datetime(1970, 1, 1) + timedelta(microseconds=mtime_ns // 1000)+ if made <= now:+ return made+ # A ctime before 1970 (an unset Windows creation time reads as 1601) can't be paged by cursor.+ return min(datetime(1970, 1, 1) + timedelta(microseconds=ctime_ns // 1000), now) if ctime_ns >= 0 else now++def normalize_tags(tags: list[str] | None) -> list[str]:"""Normalize a list of tags by:diff --git a/app/assets/scanner.py b/app/assets/scanner.pyindex 8c441c044..da7963efe 100644--- a/app/assets/scanner.py+++ b/app/assets/scanner.py@@ -32,6 +32,7 @@ from app.assets.database.queries import (from app.assets.database.models import Asset, AssetContentfrom app.assets.helpers import (PREFIX_BATCH_SIZE,+ mtime_ns_to_utc,path_prefix_matcher,sql_path_under_prefix,sql_path_under_prefix_batches,@@ -795,6 +796,8 @@ def seed_asset_specs(job_id=spec["job_id"],loader_path=spec["fname"],tags=spec["tags"],+ # Sorts files already on disk by when they were made, not by walk order.+ created_at=mtime_ns_to_utc(get_mtime_ns(stat_result), stat_result.st_ctime_ns),)created += 1except IntegrityError as error:diff --git a/tests-unit/app_test/test_migration_0009.py b/tests-unit/app_test/test_migration_0009.pynew file mode 100644index 000000000..add28ef86--- /dev/null+++ b/tests-unit/app_test/test_migration_0009.py@@ -0,0 +1,137 @@+import os+import sqlite3+from datetime import datetime, timedelta++import pytest+from alembic import command+from alembic.config import Config++_BASELINE_0008 = "0008_drop_asset_meta"+_REVISION_0009 = "0009_created_at_from_mtime"++# Content and record take separate clock reads in one scan transaction, so they differ by µs.+_CONTENT_TIME = "2026-10-01 12:00:00.000000"+_SCAN_TIME = "2026-10-01 12:00:00.000004"+_UPDATED_TIME = "2026-10-01 12:00:00.000007"+_HALF_SECOND_LATER = "2026-10-01 12:00:00.500000"+_OUTSIDE_WINDOW = "2026-10-01 12:00:01.100000"+_LATER = "2026-10-03 09:00:00.000000"+_MTIME = datetime(2025, 3, 4, 5, 6, 7, 123456)+_MTIME_TEXT = "2025-03-04 05:06:07.123456"+++def _ns(when: datetime) -> int:+ return (when - datetime(1970, 1, 1)) // timedelta(microseconds=1) * 1000+++def _make_config(db_path: str) -> Config:+ root = os.path.join(os.path.dirname(__file__), "../..")+ cfg = Config(os.path.abspath(os.path.join(root, "alembic.ini")))+ cfg.set_main_option("script_location", os.path.abspath(os.path.join(root, "alembic_db")))+ cfg.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}")+ return cfg+++def _add_content(conn: sqlite3.Connection, content_id: str, mtime_ns: int | None) -> None:+ conn.execute(+ "INSERT INTO asset_contents (id, size_bytes, path, mtime_ns, is_missing, created_at) "+ "VALUES (?, 1, ?, ?, 0, ?)",+ (content_id, f"/out/{content_id}.png", mtime_ns, _CONTENT_TIME),+ )+++def _add_record(+ conn: sqlite3.Connection,+ record_id: str,+ content_id: str,+ created_at: str = _SCAN_TIME,+ job_id: str | None = None,+) -> None:+ conn.execute(+ "INSERT INTO assets (id, content_id, name, job_id, created_at, updated_at) "+ "VALUES (?, ?, ?, ?, ?, ?)",+ (record_id, content_id, record_id, job_id, created_at, _UPDATED_TIME),+ )+++def _created(db_path: str) -> dict[str, str]:+ with sqlite3.connect(db_path) as conn:+ return dict(conn.execute("SELECT id, created_at FROM assets"))+++def db_at_0008(tmp_path):+ db_path = str(tmp_path / "test.db")+ cfg = _make_config(db_path)+ command.upgrade(cfg, _BASELINE_0008)+ with sqlite3.connect(db_path) as conn:+ mtimes = {+ "scanned": _ns(_MTIME) + 789, # sub-µs part is floored+ "generated": _ns(_MTIME),+ "shared": _ns(_MTIME),+ "reuploaded": _ns(_MTIME),+ "uploaded": _ns(_MTIME),+ "half-second": _ns(_MTIME),+ "outside-window": _ns(_MTIME),+ "touched": _ns(datetime(2026, 10, 2)), # mtime refreshed after the scan+ "future": _ns(datetime(2200, 1, 1)),+ "no-mtime": None,+ }+ for content_id, mtime_ns in mtimes.items():+ _add_content(conn, content_id, mtime_ns)+ _add_record(conn, "scanned", "scanned")+ _add_record(conn, "generated", "generated", job_id="job-1")+ _add_record(conn, "shared-job", "shared", job_id="job-2")+ _add_record(conn, "shared-later", "shared", created_at=_LATER)+ # Its scanned sibling was deleted before the upgrade; this one came later.+ _add_record(conn, "reuploaded", "reuploaded", created_at=_LATER)+ # Uploads create content and record together too; an /upload/image duplicate of an+ # older file keeps that file's mtime.+ _add_record(conn, "uploaded", "uploaded")+ conn.execute("INSERT INTO tags (name) VALUES ('uploaded')")+ conn.execute(+ "INSERT INTO asset_tags (asset_id, tag_name, origin, added_at) VALUES (?, ?, ?, ?)",+ ("uploaded", "uploaded", "manual", _SCAN_TIME),+ )+ _add_record(conn, "half-second", "half-second", created_at=_HALF_SECOND_LATER)+ _add_record(conn, "outside-window", "outside-window", created_at=_OUTSIDE_WINDOW)+ _add_record(conn, "touched", "touched")+ _add_record(conn, "future", "future")+ _add_record(conn, "no-mtime", "no-mtime")+ conn.commit()+ yield cfg, db_path+++def test_0009_dates_scanned_records_by_mtime(db_at_0008):+ cfg, db_path = db_at_0008++ command.upgrade(cfg, _REVISION_0009)+ created = _created(db_path)++ assert created.pop("scanned") == _MTIME_TEXT+ assert created.pop("half-second") == _MTIME_TEXT+ assert created == {+ "generated": _SCAN_TIME,+ "shared-job": _SCAN_TIME,+ "shared-later": _LATER,+ "reuploaded": _LATER,+ "uploaded": _SCAN_TIME,+ "outside-window": _OUTSIDE_WINDOW,+ "touched": _SCAN_TIME,+ "future": _SCAN_TIME,+ "no-mtime": _SCAN_TIME,+ }+ with sqlite3.connect(db_path) as conn:+ assert {row[0] for row in conn.execute("SELECT updated_at FROM assets")} == {_UPDATED_TIME}+++def test_0009_rerun_sets_the_same_dates(db_at_0008):+ cfg, db_path = db_at_0008+ command.upgrade(cfg, _REVISION_0009)+ first = _created(db_path)++ command.downgrade(cfg, _BASELINE_0008)+ assert _created(db_path) == first, "downgrade leaves the dates"+ command.upgrade(cfg, _REVISION_0009)++ assert _created(db_path) == firstdiff --git a/tests-unit/assets_test/services/test_created_at_from_mtime.py b/tests-unit/assets_test/services/test_created_at_from_mtime.pynew file mode 100644index 000000000..f077e68b6--- /dev/null+++ b/tests-unit/assets_test/services/test_created_at_from_mtime.py@@ -0,0 +1,193 @@+import json+import os+import time+from contextlib import contextmanager+from datetime import datetime, timedelta+from pathlib import Path+from urllib.parse import urlencode++import pytest+from aiohttp import web+from aiohttp.test_utils import make_mocked_request+from sqlalchemy.orm import Session++import folder_paths+from app.assets.api import routes+from app.assets.database.models import Asset+from app.assets.helpers import mtime_ns_to_utc+from app.assets.scanner import SeedAssetSpec, insert_asset_specs+from app.assets.services.ingest import register_executed_output, register_file_in_place++_EPOCH = datetime(1970, 1, 1)+++def output_dir(temp_dir: Path, monkeypatch: pytest.MonkeyPatch):+ monkeypatch.setattr(folder_paths, "output_directory", str(temp_dir))+ return temp_dir+++def one_database(db_engine, mock_create_session, monkeypatch: pytest.MonkeyPatch):+ @contextmanager+ def _create_session():+ with Session(db_engine) as session:+ yield session++ monkeypatch.setattr("app.assets.scanner.create_session", _create_session)+ monkeypatch.setattr("app.assets.scanner.create_write_session", _create_session)+ monkeypatch.setattr(routes, "create_session", lambda: Session(db_engine))+ monkeypatch.setattr(routes, "_ASSETS_ENABLED", True)+ return db_engine+++def _write(directory: Path, name: str, when: datetime | None = None, extra_ns: int = 0) -> Path:+ path = directory / name+ path.write_bytes(name.encode())+ if when is not None:+ mtime_ns = (when - _EPOCH) // timedelta(microseconds=1) * 1000 + extra_ns+ os.utime(path, ns=(mtime_ns, mtime_ns))+ return path+++def _ns(when: datetime) -> int:+ return (when - _EPOCH) // timedelta(microseconds=1) * 1000+++def _spec(path: Path) -> SeedAssetSpec:+ stat_result = path.stat()+ return {+ "abs_path": str(path),+ "size_bytes": stat_result.st_size,+ "mtime_ns": stat_result.st_mtime_ns,+ "info_name": path.name,+ "tags": ["output"],+ "fname": path.name,+ "metadata": None,+ "mime_type": "image/png",+ "job_id": None,+ }+++def _scan(*paths: Path) -> None:+ created, error = insert_asset_specs([_spec(path) for path in paths], set())+ assert (created, error) == (len(paths), None)+++async def _list(params: dict[str, str]) -> dict:+ response = await routes.list_assets_route(+ make_mocked_request("GET", f"/api/assets?{urlencode(params)}")+ )+ assert isinstance(response, web.Response)+ return json.loads(response.body)+++async def _listed_names(**params: str) -> list[str]:+ body = await _list(params)+ return [asset["name"] for asset in body["assets"]]+++def _created_at(engine, name: str) -> datetime:+ with Session(engine) as session:+ return session.query(Asset).filter(Asset.name == name).one().created_at+++async def test_scanned_files_list_newest_mtime_first(one_database, output_dir: Path):+ old = _write(output_dir, "old.png", datetime(2024, 1, 1))+ middle = _write(output_dir, "middle.png", datetime(2025, 1, 1, 0, 0, 0, 123456), extra_ns=789)+ recent = _write(output_dir, "recent.png", datetime(2026, 1, 1))++ # Newest first, so stamping the scan time would list them the other way round.+ _scan(recent, middle, old)++ assert await _listed_names() == ["recent.png", "middle.png", "old.png"]+ assert _created_at(one_database, "middle.png") == datetime(2025, 1, 1, 0, 0, 0, 123456) # floored+++async def test_new_generation_stays_on_top_while_the_first_scan_is_still_inserting(+ one_database, output_dir: Path+):+ old = [_write(output_dir, f"old-{i}.png", datetime(2025, 1, 1 + i)) for i in range(3)]+ generated = _write(output_dir, "generated.png")+ assert register_executed_output(str(generated), job_id="job-1") is not None++ # The scan reaches the older files only after the prompt registered its output.+ _scan(*old)++ assert (await _listed_names())[0] == "generated.png"+++async def test_upload_after_a_scan_lists_first(one_database, output_dir: Path):+ _scan(_write(output_dir, "old.png", datetime(2025, 1, 1)))+ uploaded = _write(output_dir, "uploaded.png", datetime(2020, 1, 1))++ register_file_in_place(str(uploaded), "uploaded.png", ["output"])++ assert (await _listed_names())[0] == "uploaded.png"+++async def test_future_mtime_file_dates_from_its_arrival_when_scanned_after_a_generation(+ one_database, output_dir: Path+):+ future = _write(output_dir, "future.png", datetime(2200, 1, 1))+ time.sleep(0.05) # past a coarse clock tick (Windows)+ generated = _write(output_dir, "generated.png")+ assert register_executed_output(str(generated), job_id="job-1") is not None++ # A scan reaches the file only after the prompt registered its output.+ _scan(future)++ assert (await _listed_names())[0] == "generated.png"+ arrived = _EPOCH + timedelta(microseconds=future.stat().st_ctime_ns // 1000)+ assert _created_at(one_database, "future.png") == arrived+++async def test_cursor_walks_files_sharing_an_mtime_once_each_in_order(+ one_database, output_dir: Path+):+ shared = datetime(2025, 6, 1, 12, 0, 0, 654321)+ paths = [_write(output_dir, f"copy-{i}.png", shared) for i in range(5)]+ paths.append(_write(output_dir, "older.png", datetime(2025, 1, 1)))+ _scan(*paths)+ expected = await _listed_names(limit="50")++ walked: list[str] = []+ params = {"limit": "1"}+ for _ in range(len(paths)):+ body = await _list(params)+ walked += [asset["name"] for asset in body["assets"]]+ if not body["has_more"]:+ break+ params = {"limit": "1", "after": body["next_cursor"]}+ else:+ pytest.fail(f"cursor walk did not finish in {len(paths)} pages: {walked}")++ assert walked == expected+ assert sorted(walked) == sorted(path.name for path in paths)+ assert walked[-1] == "older.png"+++def test_ctime_only_dates_a_file_whose_mtime_is_in_the_future():+ created, modified = datetime(2025, 1, 1), datetime(2025, 9, 1)+ # On Windows ctime is the creation time, earlier than any later edit.+ assert mtime_ns_to_utc(_ns(modified), _ns(created)) == modified+ assert mtime_ns_to_utc(_ns(datetime(2200, 1, 1)), _ns(created)) == created+++def test_future_ctime_is_capped_at_now(monkeypatch: pytest.MonkeyPatch):+ now = datetime(2026, 10, 1)+ monkeypatch.setattr("app.assets.helpers.get_utc_now", lambda: now)+ future = _ns(datetime(2200, 1, 1))+ assert mtime_ns_to_utc(future, future) == now+++def test_future_mtime_with_a_pre_1970_ctime_is_dated_now(monkeypatch: pytest.MonkeyPatch):+ # An unset Windows creation time reads as 1601, which a cursor can't encode.+ now = datetime(2026, 10, 1)+ monkeypatch.setattr("app.assets.helpers.get_utc_now", lambda: now)+ assert mtime_ns_to_utc(_ns(datetime(2200, 1, 1)), -11_644_473_600 * 10**9) == nowdiff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.pyindex 7a2d10e10..5e02c034d 100644--- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py+++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py@@ -1,5 +1,6 @@import loggingimport os+from datetime import datetimefrom collections.abc import Callablefrom contextlib import contextmanagerfrom pathlib import Path@@ -202,6 +203,7 @@ def test_seed_propagates_unrelated_integrity_error(job_id: str | None,loader_path: str | None,tags: list[str],+ created_at: datetime,) -> Asset:raise unrelated_error@@ -231,6 +233,7 @@ def test_seed_raises_memory_error_instead_of_attempting_later_specs(job_id: str | None,loader_path: str | None,tags: list[str],+ created_at: datetime,) -> Asset:attempted.append(name)raise MemoryError("out of memory")@@ -265,6 +268,7 @@ def test_seed_attempts_remaining_specs_before_propagating_integrity_error(job_id: str | None,loader_path: str | None,tags: list[str],+ created_at: datetime,) -> Asset:attempted.append(name)if name == "broken.bin":@@ -277,6 +281,7 @@ def test_seed_attempts_remaining_specs_before_propagating_integrity_error(job_id=job_id,loader_path=loader_path,tags=tags,+ created_at=created_at,)monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise)@@ -314,6 +319,7 @@ def test_insert_commits_successful_specs_before_propagating_batch_fault(job_id: str | None,loader_path: str | None,tags: list[str],+ created_at: datetime,) -> Asset:if name == "broken.bin":raise RuntimeError("forced record creation failure")@@ -325,6 +331,7 @@ def test_insert_commits_successful_specs_before_propagating_batch_fault(job_id=job_id,loader_path=loader_path,tags=tags,+ created_at=created_at,)monkeypatch.setattr("app.assets.scanner.create_session", _create_session)diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.pyindex 7c40b0653..c7f1f982a 100644--- a/tests-unit/seeder_test/test_seeder.py+++ b/tests-unit/seeder_test/test_seeder.py@@ -2,6 +2,7 @@ import loggingimport reimport threadingimport time+from datetime import datetimefrom contextlib import contextmanager, nullcontextfrom pathlib import Pathfrom unittest.mock import Mock@@ -143,6 +144,7 @@ def _run_faulting_fast_phase(job_id: str | None,loader_path: str | None,tags: list[str],+ created_at: datetime,) -> Asset:if name == "broken.bin":raise original_fault@@ -154,6 +156,7 @@ def _run_faulting_fast_phase(job_id=job_id,loader_path=loader_path,tags=tags,+ created_at=created_at,)monkeypatch.setattr(scanner_module, "create_session", database_session)