Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

fix(assets): date scanned assets by their file's mtime (#16810)

Simon Pinfold0df64eb242b7
diff --git a/alembic_db/versions/0009_created_at_from_mtime.py b/alembic_db/versions/0009_created_at_from_mtime.py
new file mode 100644
index 000000000..a4f6943b2
--- /dev/null
+++ b/alembic_db/versions/0009_created_at_from_mtime.py
@@ -0,0 +1,42 @@
+"""
+Date each scanned record by its file's mtime instead of the scan time.
+
+Revision ID: 0009_created_at_from_mtime
+Revises: 0008_drop_asset_meta
+Create Date: 2026-10-05
+"""
+
+from datetime import datetime, timedelta
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0009_created_at_from_mtime"
+down_revision = "0008_drop_asset_meta"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ # The scanner writes a jobless record in the same transaction as its content; a record
+ # added to existing content later (a re-upload, from-hash) is not the scan's, and uploads,
+ # which also create both together, carry the "uploaded" tag.
+ rows = op.get_bind().execute(sa.text(
+ "SELECT a.id, a.created_at, c.mtime_ns FROM assets AS a JOIN asset_contents AS c"
+ " ON c.id = a.content_id WHERE a.job_id IS NULL AND c.mtime_ns IS NOT NULL"
+ " AND abs(julianday(a.created_at) - julianday(c.created_at)) * 86400 < 1"
+ " AND NOT EXISTS (SELECT 1 FROM asset_tags AS t WHERE t.asset_id = a.id AND t.tag_name = 'uploaded')"
+ ).columns(created_at=sa.DateTime)).all()
+ # Only ever earlier: an mtime changed after cataloguing must not move a record above newer work.
+ # The conversion is inlined, not app.assets.helpers', so this migration can't change if that helper does.
+ params = [
+ {"id": record_id, "created": min(datetime(1970, 1, 1) + timedelta(microseconds=mtime_ns // 1000), created_at)}
+ for record_id, created_at, mtime_ns in rows
+ ]
+ if params:
+ op.get_bind().execute(sa.text("UPDATE assets SET created_at = :created WHERE id = :id").bindparams(
+ sa.bindparam("created", type_=sa.DateTime)), params)
+
+
+def downgrade() -> None:
+ pass
diff --git a/app/assets/database/queries/records.py b/app/assets/database/queries/records.py
index 19dec16cf..a41317468 100644
--- a/app/assets/database/queries/records.py
+++ b/app/assets/database/queries/records.py
@@ -104,8 +104,10 @@ def ensure_tag_link(session: Session, *, asset_id: str, tag_name: str, origin: s
return True
-def create_record(session: Session, content_id: str, name: str, mime_type: str | None = None, job_id: str | None = None, loader_path: str | None = None, tags: Sequence[str] | None = None, *, system_metadata: dict[str, Any] | None = None) -> Asset:
+def create_record(session: Session, content_id: str, name: str, mime_type: str | None = None, job_id: str | None = None, loader_path: str | None = None, tags: Sequence[str] | None = None, *, system_metadata: dict[str, Any] | None = None, created_at: datetime | None = None) -> Asset:
record = Asset(content_id=content_id, name=name, mime_type=mime_type, job_id=job_id, loader_path=loader_path, system_metadata=system_metadata)
+ if created_at is not None:
+ record.created_at = created_at
session.add(record)
session.flush()
for tag_name in dict.fromkeys(tags or ()):
diff --git a/app/assets/helpers.py b/app/assets/helpers.py
index b83a6855a..4b98d5288 100644
--- a/app/assets/helpers.py
+++ b/app/assets/helpers.py
@@ -8,7 +8,7 @@ follows ``Path.is_relative_to`` instead, including its platform case rules.
import functools
import os
from collections.abc import Callable, Iterable
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
import sqlalchemy as sa
from sqlalchemy.sql import ColumnElement
@@ -144,6 +144,17 @@ def get_utc_now() -> datetime:
return datetime.now(timezone.utc).replace(tzinfo=None)
+def mtime_ns_to_utc(mtime_ns: int, ctime_ns: int) -> datetime:
+ """Naive UTC time of a file mtime. A future mtime falls back to the ctime, capped at now: the
+ creation time on Windows, the last metadata change elsewhere."""
+ now = get_utc_now()
+ made = datetime(1970, 1, 1) + timedelta(microseconds=mtime_ns // 1000)
+ if made <= now:
+ return made
+ # A ctime before 1970 (an unset Windows creation time reads as 1601) can't be paged by cursor.
+ return min(datetime(1970, 1, 1) + timedelta(microseconds=ctime_ns // 1000), now) if ctime_ns >= 0 else now
+
+
def normalize_tags(tags: list[str] | None) -> list[str]:
"""
Normalize a list of tags by:
diff --git a/app/assets/scanner.py b/app/assets/scanner.py
index 8c441c044..da7963efe 100644
--- a/app/assets/scanner.py
+++ b/app/assets/scanner.py
@@ -32,6 +32,7 @@ from app.assets.database.queries import (
from app.assets.database.models import Asset, AssetContent
from app.assets.helpers import (
PREFIX_BATCH_SIZE,
+ mtime_ns_to_utc,
path_prefix_matcher,
sql_path_under_prefix,
sql_path_under_prefix_batches,
@@ -795,6 +796,8 @@ def seed_asset_specs(
job_id=spec["job_id"],
loader_path=spec["fname"],
tags=spec["tags"],
+ # Sorts files already on disk by when they were made, not by walk order.
+ created_at=mtime_ns_to_utc(get_mtime_ns(stat_result), stat_result.st_ctime_ns),
)
created += 1
except IntegrityError as error:
diff --git a/tests-unit/app_test/test_migration_0009.py b/tests-unit/app_test/test_migration_0009.py
new file mode 100644
index 000000000..add28ef86
--- /dev/null
+++ b/tests-unit/app_test/test_migration_0009.py
@@ -0,0 +1,137 @@
+import os
+import sqlite3
+from datetime import datetime, timedelta
+
+import pytest
+from alembic import command
+from alembic.config import Config
+
+_BASELINE_0008 = "0008_drop_asset_meta"
+_REVISION_0009 = "0009_created_at_from_mtime"
+
+# Content and record take separate clock reads in one scan transaction, so they differ by µs.
+_CONTENT_TIME = "2026-10-01 12:00:00.000000"
+_SCAN_TIME = "2026-10-01 12:00:00.000004"
+_UPDATED_TIME = "2026-10-01 12:00:00.000007"
+_HALF_SECOND_LATER = "2026-10-01 12:00:00.500000"
+_OUTSIDE_WINDOW = "2026-10-01 12:00:01.100000"
+_LATER = "2026-10-03 09:00:00.000000"
+_MTIME = datetime(2025, 3, 4, 5, 6, 7, 123456)
+_MTIME_TEXT = "2025-03-04 05:06:07.123456"
+
+
+def _ns(when: datetime) -> int:
+ return (when - datetime(1970, 1, 1)) // timedelta(microseconds=1) * 1000
+
+
+def _make_config(db_path: str) -> Config:
+ root = os.path.join(os.path.dirname(__file__), "../..")
+ cfg = Config(os.path.abspath(os.path.join(root, "alembic.ini")))
+ cfg.set_main_option("script_location", os.path.abspath(os.path.join(root, "alembic_db")))
+ cfg.set_main_option("sqlalchemy.url", f"sqlite:///{db_path}")
+ return cfg
+
+
+def _add_content(conn: sqlite3.Connection, content_id: str, mtime_ns: int | None) -> None:
+ conn.execute(
+ "INSERT INTO asset_contents (id, size_bytes, path, mtime_ns, is_missing, created_at) "
+ "VALUES (?, 1, ?, ?, 0, ?)",
+ (content_id, f"/out/{content_id}.png", mtime_ns, _CONTENT_TIME),
+ )
+
+
+def _add_record(
+ conn: sqlite3.Connection,
+ record_id: str,
+ content_id: str,
+ created_at: str = _SCAN_TIME,
+ job_id: str | None = None,
+) -> None:
+ conn.execute(
+ "INSERT INTO assets (id, content_id, name, job_id, created_at, updated_at) "
+ "VALUES (?, ?, ?, ?, ?, ?)",
+ (record_id, content_id, record_id, job_id, created_at, _UPDATED_TIME),
+ )
+
+
+def _created(db_path: str) -> dict[str, str]:
+ with sqlite3.connect(db_path) as conn:
+ return dict(conn.execute("SELECT id, created_at FROM assets"))
+
+
+def db_at_0008(tmp_path):
+ db_path = str(tmp_path / "test.db")
+ cfg = _make_config(db_path)
+ command.upgrade(cfg, _BASELINE_0008)
+ with sqlite3.connect(db_path) as conn:
+ mtimes = {
+ "scanned": _ns(_MTIME) + 789, # sub-µs part is floored
+ "generated": _ns(_MTIME),
+ "shared": _ns(_MTIME),
+ "reuploaded": _ns(_MTIME),
+ "uploaded": _ns(_MTIME),
+ "half-second": _ns(_MTIME),
+ "outside-window": _ns(_MTIME),
+ "touched": _ns(datetime(2026, 10, 2)), # mtime refreshed after the scan
+ "future": _ns(datetime(2200, 1, 1)),
+ "no-mtime": None,
+ }
+ for content_id, mtime_ns in mtimes.items():
+ _add_content(conn, content_id, mtime_ns)
+ _add_record(conn, "scanned", "scanned")
+ _add_record(conn, "generated", "generated", job_id="job-1")
+ _add_record(conn, "shared-job", "shared", job_id="job-2")
+ _add_record(conn, "shared-later", "shared", created_at=_LATER)
+ # Its scanned sibling was deleted before the upgrade; this one came later.
+ _add_record(conn, "reuploaded", "reuploaded", created_at=_LATER)
+ # Uploads create content and record together too; an /upload/image duplicate of an
+ # older file keeps that file's mtime.
+ _add_record(conn, "uploaded", "uploaded")
+ conn.execute("INSERT INTO tags (name) VALUES ('uploaded')")
+ conn.execute(
+ "INSERT INTO asset_tags (asset_id, tag_name, origin, added_at) VALUES (?, ?, ?, ?)",
+ ("uploaded", "uploaded", "manual", _SCAN_TIME),
+ )
+ _add_record(conn, "half-second", "half-second", created_at=_HALF_SECOND_LATER)
+ _add_record(conn, "outside-window", "outside-window", created_at=_OUTSIDE_WINDOW)
+ _add_record(conn, "touched", "touched")
+ _add_record(conn, "future", "future")
+ _add_record(conn, "no-mtime", "no-mtime")
+ conn.commit()
+ yield cfg, db_path
+
+
+def test_0009_dates_scanned_records_by_mtime(db_at_0008):
+ cfg, db_path = db_at_0008
+
+ command.upgrade(cfg, _REVISION_0009)
+ created = _created(db_path)
+
+ assert created.pop("scanned") == _MTIME_TEXT
+ assert created.pop("half-second") == _MTIME_TEXT
+ assert created == {
+ "generated": _SCAN_TIME,
+ "shared-job": _SCAN_TIME,
+ "shared-later": _LATER,
+ "reuploaded": _LATER,
+ "uploaded": _SCAN_TIME,
+ "outside-window": _OUTSIDE_WINDOW,
+ "touched": _SCAN_TIME,
+ "future": _SCAN_TIME,
+ "no-mtime": _SCAN_TIME,
+ }
+ with sqlite3.connect(db_path) as conn:
+ assert {row[0] for row in conn.execute("SELECT updated_at FROM assets")} == {_UPDATED_TIME}
+
+
+def test_0009_rerun_sets_the_same_dates(db_at_0008):
+ cfg, db_path = db_at_0008
+ command.upgrade(cfg, _REVISION_0009)
+ first = _created(db_path)
+
+ command.downgrade(cfg, _BASELINE_0008)
+ assert _created(db_path) == first, "downgrade leaves the dates"
+ command.upgrade(cfg, _REVISION_0009)
+
+ assert _created(db_path) == first
diff --git a/tests-unit/assets_test/services/test_created_at_from_mtime.py b/tests-unit/assets_test/services/test_created_at_from_mtime.py
new file mode 100644
index 000000000..f077e68b6
--- /dev/null
+++ b/tests-unit/assets_test/services/test_created_at_from_mtime.py
@@ -0,0 +1,193 @@
+import json
+import os
+import time
+from contextlib import contextmanager
+from datetime import datetime, timedelta
+from pathlib import Path
+from urllib.parse import urlencode
+
+import pytest
+from aiohttp import web
+from aiohttp.test_utils import make_mocked_request
+from sqlalchemy.orm import Session
+
+import folder_paths
+from app.assets.api import routes
+from app.assets.database.models import Asset
+from app.assets.helpers import mtime_ns_to_utc
+from app.assets.scanner import SeedAssetSpec, insert_asset_specs
+from app.assets.services.ingest import register_executed_output, register_file_in_place
+
+_EPOCH = datetime(1970, 1, 1)
+
+
+def output_dir(temp_dir: Path, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(folder_paths, "output_directory", str(temp_dir))
+ return temp_dir
+
+
+def one_database(db_engine, mock_create_session, monkeypatch: pytest.MonkeyPatch):
+ @contextmanager
+ def _create_session():
+ with Session(db_engine) as session:
+ yield session
+
+ monkeypatch.setattr("app.assets.scanner.create_session", _create_session)
+ monkeypatch.setattr("app.assets.scanner.create_write_session", _create_session)
+ monkeypatch.setattr(routes, "create_session", lambda: Session(db_engine))
+ monkeypatch.setattr(routes, "_ASSETS_ENABLED", True)
+ return db_engine
+
+
+def _write(directory: Path, name: str, when: datetime | None = None, extra_ns: int = 0) -> Path:
+ path = directory / name
+ path.write_bytes(name.encode())
+ if when is not None:
+ mtime_ns = (when - _EPOCH) // timedelta(microseconds=1) * 1000 + extra_ns
+ os.utime(path, ns=(mtime_ns, mtime_ns))
+ return path
+
+
+def _ns(when: datetime) -> int:
+ return (when - _EPOCH) // timedelta(microseconds=1) * 1000
+
+
+def _spec(path: Path) -> SeedAssetSpec:
+ stat_result = path.stat()
+ return {
+ "abs_path": str(path),
+ "size_bytes": stat_result.st_size,
+ "mtime_ns": stat_result.st_mtime_ns,
+ "info_name": path.name,
+ "tags": ["output"],
+ "fname": path.name,
+ "metadata": None,
+ "mime_type": "image/png",
+ "job_id": None,
+ }
+
+
+def _scan(*paths: Path) -> None:
+ created, error = insert_asset_specs([_spec(path) for path in paths], set())
+ assert (created, error) == (len(paths), None)
+
+
+async def _list(params: dict[str, str]) -> dict:
+ response = await routes.list_assets_route(
+ make_mocked_request("GET", f"/api/assets?{urlencode(params)}")
+ )
+ assert isinstance(response, web.Response)
+ return json.loads(response.body)
+
+
+async def _listed_names(**params: str) -> list[str]:
+ body = await _list(params)
+ return [asset["name"] for asset in body["assets"]]
+
+
+def _created_at(engine, name: str) -> datetime:
+ with Session(engine) as session:
+ return session.query(Asset).filter(Asset.name == name).one().created_at
+
+
+async def test_scanned_files_list_newest_mtime_first(one_database, output_dir: Path):
+ old = _write(output_dir, "old.png", datetime(2024, 1, 1))
+ middle = _write(output_dir, "middle.png", datetime(2025, 1, 1, 0, 0, 0, 123456), extra_ns=789)
+ recent = _write(output_dir, "recent.png", datetime(2026, 1, 1))
+
+ # Newest first, so stamping the scan time would list them the other way round.
+ _scan(recent, middle, old)
+
+ assert await _listed_names() == ["recent.png", "middle.png", "old.png"]
+ assert _created_at(one_database, "middle.png") == datetime(2025, 1, 1, 0, 0, 0, 123456) # floored
+
+
+async def test_new_generation_stays_on_top_while_the_first_scan_is_still_inserting(
+ one_database, output_dir: Path
+):
+ old = [_write(output_dir, f"old-{i}.png", datetime(2025, 1, 1 + i)) for i in range(3)]
+ generated = _write(output_dir, "generated.png")
+ assert register_executed_output(str(generated), job_id="job-1") is not None
+
+ # The scan reaches the older files only after the prompt registered its output.
+ _scan(*old)
+
+ assert (await _listed_names())[0] == "generated.png"
+
+
+async def test_upload_after_a_scan_lists_first(one_database, output_dir: Path):
+ _scan(_write(output_dir, "old.png", datetime(2025, 1, 1)))
+ uploaded = _write(output_dir, "uploaded.png", datetime(2020, 1, 1))
+
+ register_file_in_place(str(uploaded), "uploaded.png", ["output"])
+
+ assert (await _listed_names())[0] == "uploaded.png"
+
+
+async def test_future_mtime_file_dates_from_its_arrival_when_scanned_after_a_generation(
+ one_database, output_dir: Path
+):
+ future = _write(output_dir, "future.png", datetime(2200, 1, 1))
+ time.sleep(0.05) # past a coarse clock tick (Windows)
+ generated = _write(output_dir, "generated.png")
+ assert register_executed_output(str(generated), job_id="job-1") is not None
+
+ # A scan reaches the file only after the prompt registered its output.
+ _scan(future)
+
+ assert (await _listed_names())[0] == "generated.png"
+ arrived = _EPOCH + timedelta(microseconds=future.stat().st_ctime_ns // 1000)
+ assert _created_at(one_database, "future.png") == arrived
+
+
+async def test_cursor_walks_files_sharing_an_mtime_once_each_in_order(
+ one_database, output_dir: Path
+):
+ shared = datetime(2025, 6, 1, 12, 0, 0, 654321)
+ paths = [_write(output_dir, f"copy-{i}.png", shared) for i in range(5)]
+ paths.append(_write(output_dir, "older.png", datetime(2025, 1, 1)))
+ _scan(*paths)
+ expected = await _listed_names(limit="50")
+
+ walked: list[str] = []
+ params = {"limit": "1"}
+ for _ in range(len(paths)):
+ body = await _list(params)
+ walked += [asset["name"] for asset in body["assets"]]
+ if not body["has_more"]:
+ break
+ params = {"limit": "1", "after": body["next_cursor"]}
+ else:
+ pytest.fail(f"cursor walk did not finish in {len(paths)} pages: {walked}")
+
+ assert walked == expected
+ assert sorted(walked) == sorted(path.name for path in paths)
+ assert walked[-1] == "older.png"
+
+
+def test_ctime_only_dates_a_file_whose_mtime_is_in_the_future():
+ created, modified = datetime(2025, 1, 1), datetime(2025, 9, 1)
+ # On Windows ctime is the creation time, earlier than any later edit.
+ assert mtime_ns_to_utc(_ns(modified), _ns(created)) == modified
+ assert mtime_ns_to_utc(_ns(datetime(2200, 1, 1)), _ns(created)) == created
+
+
+def test_future_ctime_is_capped_at_now(monkeypatch: pytest.MonkeyPatch):
+ now = datetime(2026, 10, 1)
+ monkeypatch.setattr("app.assets.helpers.get_utc_now", lambda: now)
+ future = _ns(datetime(2200, 1, 1))
+ assert mtime_ns_to_utc(future, future) == now
+
+
+def test_future_mtime_with_a_pre_1970_ctime_is_dated_now(monkeypatch: pytest.MonkeyPatch):
+ # An unset Windows creation time reads as 1601, which a cursor can't encode.
+ now = datetime(2026, 10, 1)
+ monkeypatch.setattr("app.assets.helpers.get_utc_now", lambda: now)
+ assert mtime_ns_to_utc(_ns(datetime(2200, 1, 1)), -11_644_473_600 * 10**9) == now
diff --git a/tests-unit/assets_test/services/test_scanner_seed_resilience.py b/tests-unit/assets_test/services/test_scanner_seed_resilience.py
index 7a2d10e10..5e02c034d 100644
--- a/tests-unit/assets_test/services/test_scanner_seed_resilience.py
+++ b/tests-unit/assets_test/services/test_scanner_seed_resilience.py
@@ -1,5 +1,6 @@
import logging
import os
+from datetime import datetime
from collections.abc import Callable
from contextlib import contextmanager
from pathlib import Path
@@ -202,6 +203,7 @@ def test_seed_propagates_unrelated_integrity_error(
job_id: str | None,
loader_path: str | None,
tags: list[str],
+ created_at: datetime,
) -> Asset:
raise unrelated_error
@@ -231,6 +233,7 @@ def test_seed_raises_memory_error_instead_of_attempting_later_specs(
job_id: str | None,
loader_path: str | None,
tags: list[str],
+ created_at: datetime,
) -> Asset:
attempted.append(name)
raise MemoryError("out of memory")
@@ -265,6 +268,7 @@ def test_seed_attempts_remaining_specs_before_propagating_integrity_error(
job_id: str | None,
loader_path: str | None,
tags: list[str],
+ created_at: datetime,
) -> Asset:
attempted.append(name)
if name == "broken.bin":
@@ -277,6 +281,7 @@ def test_seed_attempts_remaining_specs_before_propagating_integrity_error(
job_id=job_id,
loader_path=loader_path,
tags=tags,
+ created_at=created_at,
)
monkeypatch.setattr("app.assets.scanner.create_record", _create_record_or_raise)
@@ -314,6 +319,7 @@ def test_insert_commits_successful_specs_before_propagating_batch_fault(
job_id: str | None,
loader_path: str | None,
tags: list[str],
+ created_at: datetime,
) -> Asset:
if name == "broken.bin":
raise RuntimeError("forced record creation failure")
@@ -325,6 +331,7 @@ def test_insert_commits_successful_specs_before_propagating_batch_fault(
job_id=job_id,
loader_path=loader_path,
tags=tags,
+ created_at=created_at,
)
monkeypatch.setattr("app.assets.scanner.create_session", _create_session)
diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.py
index 7c40b0653..c7f1f982a 100644
--- a/tests-unit/seeder_test/test_seeder.py
+++ b/tests-unit/seeder_test/test_seeder.py
@@ -2,6 +2,7 @@ import logging
import re
import threading
import time
+from datetime import datetime
from contextlib import contextmanager, nullcontext
from pathlib import Path
from unittest.mock import Mock
@@ -143,6 +144,7 @@ def _run_faulting_fast_phase(
job_id: str | None,
loader_path: str | None,
tags: list[str],
+ created_at: datetime,
) -> Asset:
if name == "broken.bin":
raise original_fault
@@ -154,6 +156,7 @@ def _run_faulting_fast_phase(
job_id=job_id,
loader_path=loader_path,
tags=tags,
+ created_at=created_at,
)
monkeypatch.setattr(scanner_module, "create_session", database_session)