The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
refactor(assets): modular architecture + async two-phase scanner & background seeder (#12621)
Luke Mino-Altherr29b24cb5177e
diff --git a/alembic_db/versions/0002_merge_to_asset_references.py b/alembic_db/versions/0002_merge_to_asset_references.pynew file mode 100644index 000000000..1ac1b980c--- /dev/null+++ b/alembic_db/versions/0002_merge_to_asset_references.py@@ -0,0 +1,267 @@+"""+Merge AssetInfo and AssetCacheState into unified asset_references table.++This migration drops old tables and creates the new unified schema.+All existing data is discarded.++Revision ID: 0002_merge_to_asset_references+Revises: 0001_assets+Create Date: 2025-02-11+"""++from alembic import op+import sqlalchemy as sa++revision = "0002_merge_to_asset_references"+down_revision = "0001_assets"+branch_labels = None+depends_on = None+++def upgrade() -> None:+ # Drop old tables (order matters due to FK constraints)+ op.drop_index("ix_asset_info_meta_key_val_bool", table_name="asset_info_meta")+ op.drop_index("ix_asset_info_meta_key_val_num", table_name="asset_info_meta")+ op.drop_index("ix_asset_info_meta_key_val_str", table_name="asset_info_meta")+ op.drop_index("ix_asset_info_meta_key", table_name="asset_info_meta")+ op.drop_table("asset_info_meta")++ op.drop_index("ix_asset_info_tags_asset_info_id", table_name="asset_info_tags")+ op.drop_index("ix_asset_info_tags_tag_name", table_name="asset_info_tags")+ op.drop_table("asset_info_tags")++ op.drop_index("ix_asset_cache_state_asset_id", table_name="asset_cache_state")+ op.drop_index("ix_asset_cache_state_file_path", table_name="asset_cache_state")+ op.drop_table("asset_cache_state")++ op.drop_index("ix_assets_info_owner_name", table_name="assets_info")+ op.drop_index("ix_assets_info_last_access_time", table_name="assets_info")+ op.drop_index("ix_assets_info_created_at", table_name="assets_info")+ op.drop_index("ix_assets_info_name", table_name="assets_info")+ op.drop_index("ix_assets_info_asset_id", table_name="assets_info")+ op.drop_index("ix_assets_info_owner_id", table_name="assets_info")+ op.drop_table("assets_info")++ # Truncate assets table (cascades handled by dropping dependent tables first)+ op.execute("DELETE FROM assets")++ # Create asset_references table+ op.create_table(+ "asset_references",+ sa.Column("id", sa.String(length=36), primary_key=True),+ sa.Column(+ "asset_id",+ sa.String(length=36),+ sa.ForeignKey("assets.id", ondelete="CASCADE"),+ nullable=False,+ ),+ sa.Column("file_path", sa.Text(), nullable=True),+ sa.Column("mtime_ns", sa.BigInteger(), nullable=True),+ sa.Column(+ "needs_verify",+ sa.Boolean(),+ nullable=False,+ server_default=sa.text("false"),+ ),+ sa.Column(+ "is_missing", sa.Boolean(), nullable=False, server_default=sa.text("false")+ ),+ sa.Column("enrichment_level", sa.Integer(), nullable=False, server_default="0"),+ sa.Column("owner_id", sa.String(length=128), nullable=False, server_default=""),+ sa.Column("name", sa.String(length=512), nullable=False),+ sa.Column(+ "preview_id",+ sa.String(length=36),+ sa.ForeignKey("assets.id", ondelete="SET NULL"),+ nullable=True,+ ),+ sa.Column("user_metadata", sa.JSON(), nullable=True),+ sa.Column("created_at", sa.DateTime(timezone=False), nullable=False),+ sa.Column("updated_at", sa.DateTime(timezone=False), nullable=False),+ sa.Column("last_access_time", sa.DateTime(timezone=False), nullable=False),+ sa.Column("deleted_at", sa.DateTime(timezone=False), nullable=True),+ sa.CheckConstraint(+ "(mtime_ns IS NULL) OR (mtime_ns >= 0)", name="ck_ar_mtime_nonneg"+ ),+ sa.CheckConstraint(+ "enrichment_level >= 0 AND enrichment_level <= 2",+ name="ck_ar_enrichment_level_range",+ ),+ )+ op.create_index(+ "uq_asset_references_file_path", "asset_references", ["file_path"], unique=True+ )+ op.create_index("ix_asset_references_asset_id", "asset_references", ["asset_id"])+ op.create_index("ix_asset_references_owner_id", "asset_references", ["owner_id"])+ op.create_index("ix_asset_references_name", "asset_references", ["name"])+ op.create_index("ix_asset_references_is_missing", "asset_references", ["is_missing"])+ op.create_index(+ "ix_asset_references_enrichment_level", "asset_references", ["enrichment_level"]+ )+ op.create_index("ix_asset_references_created_at", "asset_references", ["created_at"])+ op.create_index(+ "ix_asset_references_last_access_time", "asset_references", ["last_access_time"]+ )+ op.create_index(+ "ix_asset_references_owner_name", "asset_references", ["owner_id", "name"]+ )+ op.create_index("ix_asset_references_deleted_at", "asset_references", ["deleted_at"])++ # Create asset_reference_tags table+ op.create_table(+ "asset_reference_tags",+ sa.Column(+ "asset_reference_id",+ sa.String(length=36),+ sa.ForeignKey("asset_references.id", ondelete="CASCADE"),+ nullable=False,+ ),+ sa.Column(+ "tag_name",+ sa.String(length=512),+ sa.ForeignKey("tags.name", ondelete="RESTRICT"),+ nullable=False,+ ),+ sa.Column(+ "origin", sa.String(length=32), nullable=False, server_default="manual"+ ),+ sa.Column("added_at", sa.DateTime(timezone=False), nullable=False),+ sa.PrimaryKeyConstraint(+ "asset_reference_id", "tag_name", name="pk_asset_reference_tags"+ ),+ )+ op.create_index(+ "ix_asset_reference_tags_tag_name", "asset_reference_tags", ["tag_name"]+ )+ op.create_index(+ "ix_asset_reference_tags_asset_reference_id",+ "asset_reference_tags",+ ["asset_reference_id"],+ )++ # Create asset_reference_meta table+ op.create_table(+ "asset_reference_meta",+ sa.Column(+ "asset_reference_id",+ sa.String(length=36),+ sa.ForeignKey("asset_references.id", ondelete="CASCADE"),+ nullable=False,+ ),+ sa.Column("key", sa.String(length=256), nullable=False),+ sa.Column("ordinal", sa.Integer(), nullable=False, server_default="0"),+ sa.Column("val_str", sa.String(length=2048), nullable=True),+ sa.Column("val_num", sa.Numeric(38, 10), nullable=True),+ sa.Column("val_bool", sa.Boolean(), nullable=True),+ sa.Column("val_json", sa.JSON(), nullable=True),+ sa.PrimaryKeyConstraint(+ "asset_reference_id", "key", "ordinal", name="pk_asset_reference_meta"+ ),+ )+ op.create_index("ix_asset_reference_meta_key", "asset_reference_meta", ["key"])+ op.create_index(+ "ix_asset_reference_meta_key_val_str", "asset_reference_meta", ["key", "val_str"]+ )+ op.create_index(+ "ix_asset_reference_meta_key_val_num", "asset_reference_meta", ["key", "val_num"]+ )+ op.create_index(+ "ix_asset_reference_meta_key_val_bool",+ "asset_reference_meta",+ ["key", "val_bool"],+ )+++def downgrade() -> None:+ """Reverse 0002_merge_to_asset_references: drop new tables, recreate old schema.++ NOTE: Data is not recoverable. The upgrade discards all rows from the old+ tables and truncates assets. After downgrade the old schema will be empty.+ A filesystem rescan will repopulate data once the older code is running.+ """+ # Drop new tables (order matters due to FK constraints)+ op.drop_index("ix_asset_reference_meta_key_val_bool", table_name="asset_reference_meta")+ op.drop_index("ix_asset_reference_meta_key_val_num", table_name="asset_reference_meta")+ op.drop_index("ix_asset_reference_meta_key_val_str", table_name="asset_reference_meta")+ op.drop_index("ix_asset_reference_meta_key", table_name="asset_reference_meta")+ op.drop_table("asset_reference_meta")++ op.drop_index("ix_asset_reference_tags_asset_reference_id", table_name="asset_reference_tags")+ op.drop_index("ix_asset_reference_tags_tag_name", table_name="asset_reference_tags")+ op.drop_table("asset_reference_tags")++ op.drop_index("ix_asset_references_deleted_at", table_name="asset_references")+ op.drop_index("ix_asset_references_owner_name", table_name="asset_references")+ op.drop_index("ix_asset_references_last_access_time", table_name="asset_references")+ op.drop_index("ix_asset_references_created_at", table_name="asset_references")+ op.drop_index("ix_asset_references_enrichment_level", table_name="asset_references")+ op.drop_index("ix_asset_references_is_missing", table_name="asset_references")+ op.drop_index("ix_asset_references_name", table_name="asset_references")+ op.drop_index("ix_asset_references_owner_id", table_name="asset_references")+ op.drop_index("ix_asset_references_asset_id", table_name="asset_references")+ op.drop_index("uq_asset_references_file_path", table_name="asset_references")+ op.drop_table("asset_references")++ # Truncate assets (upgrade deleted all rows; downgrade starts fresh too)+ op.execute("DELETE FROM assets")++ # Recreate old tables from 0001_assets schema+ op.create_table(+ "assets_info",+ sa.Column("id", sa.String(length=36), primary_key=True),+ sa.Column("owner_id", sa.String(length=128), nullable=False, server_default=""),+ sa.Column("name", sa.String(length=512), nullable=False),+ sa.Column("asset_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="RESTRICT"), nullable=False),+ sa.Column("preview_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="SET NULL"), nullable=True),+ sa.Column("user_metadata", sa.JSON(), nullable=True),+ sa.Column("created_at", sa.DateTime(timezone=False), nullable=False),+ sa.Column("updated_at", sa.DateTime(timezone=False), nullable=False),+ sa.Column("last_access_time", sa.DateTime(timezone=False), nullable=False),+ sa.UniqueConstraint("asset_id", "owner_id", "name", name="uq_assets_info_asset_owner_name"),+ )+ op.create_index("ix_assets_info_owner_id", "assets_info", ["owner_id"])+ op.create_index("ix_assets_info_asset_id", "assets_info", ["asset_id"])+ op.create_index("ix_assets_info_name", "assets_info", ["name"])+ op.create_index("ix_assets_info_created_at", "assets_info", ["created_at"])+ op.create_index("ix_assets_info_last_access_time", "assets_info", ["last_access_time"])+ op.create_index("ix_assets_info_owner_name", "assets_info", ["owner_id", "name"])++ op.create_table(+ "asset_cache_state",+ sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),+ sa.Column("asset_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="CASCADE"), nullable=False),+ sa.Column("file_path", sa.Text(), nullable=False),+ sa.Column("mtime_ns", sa.BigInteger(), nullable=True),+ sa.Column("needs_verify", sa.Boolean(), nullable=False, server_default=sa.text("false")),+ sa.CheckConstraint("(mtime_ns IS NULL) OR (mtime_ns >= 0)", name="ck_acs_mtime_nonneg"),+ sa.UniqueConstraint("file_path", name="uq_asset_cache_state_file_path"),+ )+ op.create_index("ix_asset_cache_state_file_path", "asset_cache_state", ["file_path"])+ op.create_index("ix_asset_cache_state_asset_id", "asset_cache_state", ["asset_id"])++ op.create_table(+ "asset_info_tags",+ sa.Column("asset_info_id", sa.String(length=36), sa.ForeignKey("assets_info.id", ondelete="CASCADE"), nullable=False),+ sa.Column("tag_name", sa.String(length=512), sa.ForeignKey("tags.name", ondelete="RESTRICT"), nullable=False),+ sa.Column("origin", sa.String(length=32), nullable=False, server_default="manual"),+ sa.Column("added_at", sa.DateTime(timezone=False), nullable=False),+ sa.PrimaryKeyConstraint("asset_info_id", "tag_name", name="pk_asset_info_tags"),+ )+ op.create_index("ix_asset_info_tags_tag_name", "asset_info_tags", ["tag_name"])+ op.create_index("ix_asset_info_tags_asset_info_id", "asset_info_tags", ["asset_info_id"])++ op.create_table(+ "asset_info_meta",+ sa.Column("asset_info_id", sa.String(length=36), sa.ForeignKey("assets_info.id", ondelete="CASCADE"), nullable=False),+ sa.Column("key", sa.String(length=256), nullable=False),+ sa.Column("ordinal", sa.Integer(), nullable=False, server_default="0"),+ sa.Column("val_str", sa.String(length=2048), nullable=True),+ sa.Column("val_num", sa.Numeric(38, 10), nullable=True),+ sa.Column("val_bool", sa.Boolean(), nullable=True),+ sa.Column("val_json", sa.JSON(), nullable=True),+ sa.PrimaryKeyConstraint("asset_info_id", "key", "ordinal", name="pk_asset_info_meta"),+ )+ op.create_index("ix_asset_info_meta_key", "asset_info_meta", ["key"])+ op.create_index("ix_asset_info_meta_key_val_str", "asset_info_meta", ["key", "val_str"])+ op.create_index("ix_asset_info_meta_key_val_num", "asset_info_meta", ["key", "val_num"])+ op.create_index("ix_asset_info_meta_key_val_bool", "asset_info_meta", ["key", "val_bool"])diff --git a/app/assets/api/routes.py b/app/assets/api/routes.pyindex 7676e50b4..40dee9f46 100644--- a/app/assets/api/routes.py+++ b/app/assets/api/routes.py@@ -1,56 +1,144 @@+import asyncio+import functools+import jsonimport logging-import uuid-import urllib.parseimport os-import contextlib-from aiohttp import web+import urllib.parse+import uuid+from typing import Any+from aiohttp import webfrom pydantic import ValidationError-import app.assets.manager as manager-from app import user_manager-from app.assets.api import schemas_in-from app.assets.helpers import get_query_dict-from app.assets.scanner import seed_assets-import folder_paths+from app import user_manager+from app.assets.api import schemas_in, schemas_out+from app.assets.api.schemas_in import (+ AssetValidationError,+ UploadError,+)+from app.assets.helpers import validate_blake3_hash+from app.assets.api.upload import (+ delete_temp_file_if_exists,+ parse_multipart_upload,+)+from app.assets.seeder import ScanInProgressError, asset_seeder+from app.assets.services import (+ DependencyMissingError,+ HashMismatchError,+ apply_tags,+ asset_exists,+ create_from_hash,+ delete_asset_reference,+ get_asset_detail,+ list_assets_page,+ list_tags,+ remove_tags,+ resolve_asset_for_download,+ update_asset_metadata,+ upload_from_temp_path,+)ROUTES = web.RouteTableDef()USER_MANAGER: user_manager.UserManager | None = None+_ASSETS_ENABLED = False+++def _require_assets_feature_enabled(handler):+ @functools.wraps(handler)+ async def wrapper(request: web.Request) -> web.Response:+ if not _ASSETS_ENABLED:+ return _build_error_response(+ 503,+ "SERVICE_DISABLED",+ "Assets system is disabled. Start the server with --enable-assets to use this feature.",+ )+ return await handler(request)++ return wrapper+# UUID regex (canonical hyphenated form, case-insensitive)UUID_RE = r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"-# Note to any custom node developers reading this code:-# The assets system is not yet fully implemented, do not rely on the code in /app/assets remaining the same.-def register_assets_system(app: web.Application, user_manager_instance: user_manager.UserManager) -> None:- global USER_MANAGER- USER_MANAGER = user_manager_instance+def get_query_dict(request: web.Request) -> dict[str, Any]:+ """Gets a dictionary of query parameters from the request.++ request.query is a MultiMapping[str], needs to be converted to a dict+ to be validated by Pydantic.+ """+ query_dict = {+ key: request.query.getall(key)+ if len(request.query.getall(key)) > 1+ else request.query.get(key)+ for key in request.query.keys()+ }+ return query_dict+++# Note to any custom node developers reading this code:+# The assets system is not yet fully implemented,+# do not rely on the code in /app/assets remaining the same.+++def register_assets_routes(+ app: web.Application,+ user_manager_instance: user_manager.UserManager | None = None,+) -> None:+ global USER_MANAGER, _ASSETS_ENABLED+ if user_manager_instance is not None:+ USER_MANAGER = user_manager_instance+ _ASSETS_ENABLED = Trueapp.add_routes(ROUTES)-def _error_response(status: int, code: str, message: str, details: dict | None = None) -> web.Response:- return web.json_response({"error": {"code": code, "message": message, "details": details or {}}}, status=status)+def disable_assets_routes() -> None:+ """Disable asset routes at runtime (e.g. after DB init failure)."""+ global _ASSETS_ENABLED+ _ASSETS_ENABLED = False-def _validation_error_response(code: str, ve: ValidationError) -> web.Response:- return _error_response(400, code, "Validation failed.", {"errors": ve.json()})++def _build_error_response(+ status: int, code: str, message: str, details: dict | None = None+) -> web.Response:+ return web.json_response(+ {"error": {"code": code, "message": message, "details": details or {}}},+ status=status,+ )+++def _build_validation_error_response(code: str, ve: ValidationError) -> web.Response:+ errors = json.loads(ve.json())+ return _build_error_response(400, code, "Validation failed.", {"errors": errors})+++def _validate_sort_field(requested: str | None) -> str:+ if not requested:+ return "created_at"+ v = requested.lower()+ if v in {"name", "created_at", "updated_at", "size", "last_access_time"}:+ return v+ return "created_at"@ROUTES.head("/api/assets/hash/{hash}")+@_require_assets_feature_enabledasync def head_asset_by_hash(request: web.Request) -> web.Response:hash_str = request.match_info.get("hash", "").strip().lower()- if not hash_str or ":" not in hash_str:- return _error_response(400, "INVALID_HASH", "hash must be like 'blake3:<hex>'")- algo, digest = hash_str.split(":", 1)- if algo != "blake3" or not digest or any(c for c in digest if c not in "0123456789abcdef"):- return _error_response(400, "INVALID_HASH", "hash must be like 'blake3:<hex>'")- exists = manager.asset_exists(asset_hash=hash_str)+ try:+ hash_str = validate_blake3_hash(hash_str)+ except ValueError:+ return _build_error_response(+ 400, "INVALID_HASH", "hash must be like 'blake3:<hex>'"+ )+ exists = asset_exists(hash_str)return web.Response(status=200 if exists else 404)@ROUTES.get("/api/assets")-async def list_assets(request: web.Request) -> web.Response:+@_require_assets_feature_enabled+async def list_assets_route(request: web.Request) -> web.Response:"""GET request to list assets."""@@ -58,78 +146,140 @@ async def list_assets(request: web.Request) -> web.Response:try:q = schemas_in.ListAssetsQuery.model_validate(query_dict)except ValidationError as ve:- return _validation_error_response("INVALID_QUERY", ve)+ return _build_validation_error_response("INVALID_QUERY", ve)++ sort = _validate_sort_field(q.sort)+ order_candidate = (q.order or "desc").lower()+ order = order_candidate if order_candidate in {"asc", "desc"} else "desc"- payload = manager.list_assets(+ result = list_assets_page(+ owner_id=USER_MANAGER.get_request_user_id(request),include_tags=q.include_tags,exclude_tags=q.exclude_tags,name_contains=q.name_contains,metadata_filter=q.metadata_filter,limit=q.limit,offset=q.offset,- sort=q.sort,- order=q.order,- owner_id=USER_MANAGER.get_request_user_id(request),+ sort=sort,+ order=order,+ )++ summaries = [+ schemas_out.AssetSummary(+ id=item.ref.id,+ name=item.ref.name,+ asset_hash=item.asset.hash if item.asset else None,+ size=int(item.asset.size_bytes) if item.asset else None,+ mime_type=item.asset.mime_type if item.asset else None,+ tags=item.tags,+ created_at=item.ref.created_at,+ updated_at=item.ref.updated_at,+ last_access_time=item.ref.last_access_time,+ )+ for item in result.items+ ]++ payload = schemas_out.AssetsList(+ assets=summaries,+ total=result.total,+ has_more=(q.offset + len(summaries)) < result.total,)return web.json_response(payload.model_dump(mode="json", exclude_none=True))@ROUTES.get(f"/api/assets/{{id:{UUID_RE}}}")-async def get_asset(request: web.Request) -> web.Response:+@_require_assets_feature_enabled+async def get_asset_route(request: web.Request) -> web.Response:"""GET request to get an asset's info as JSON."""- asset_info_id = str(uuid.UUID(request.match_info["id"]))+ reference_id = str(uuid.UUID(request.match_info["id"]))try:- result = manager.get_asset(- asset_info_id=asset_info_id,+ result = get_asset_detail(+ reference_id=reference_id,owner_id=USER_MANAGER.get_request_user_id(request),)+ if not result:+ return _build_error_response(+ 404,+ "ASSET_NOT_FOUND",+ f"AssetReference {reference_id} not found",+ {"id": reference_id},+ )++ payload = schemas_out.AssetDetail(+ id=result.ref.id,+ name=result.ref.name,+ asset_hash=result.asset.hash if result.asset else None,+ size=int(result.asset.size_bytes) if result.asset else None,+ mime_type=result.asset.mime_type if result.asset else None,+ tags=result.tags,+ user_metadata=result.ref.user_metadata or {},+ preview_id=result.ref.preview_id,+ created_at=result.ref.created_at,+ last_access_time=result.ref.last_access_time,+ )except ValueError as e:- return _error_response(404, "ASSET_NOT_FOUND", str(e), {"id": asset_info_id})+ return _build_error_response(+ 404, "ASSET_NOT_FOUND", str(e), {"id": reference_id}+ )except Exception:logging.exception(- "get_asset failed for asset_info_id=%s, owner_id=%s",- asset_info_id,+ "get_asset failed for reference_id=%s, owner_id=%s",+ reference_id,USER_MANAGER.get_request_user_id(request),)- return _error_response(500, "INTERNAL", "Unexpected server error.")- return web.json_response(result.model_dump(mode="json"), status=200)+ return _build_error_response(500, "INTERNAL", "Unexpected server error.")+ return web.json_response(payload.model_dump(mode="json"), status=200)@ROUTES.get(f"/api/assets/{{id:{UUID_RE}}}/content")+@_require_assets_feature_enabledasync def download_asset_content(request: web.Request) -> web.Response:- # question: do we need disposition? could we just stick with one of these?disposition = request.query.get("disposition", "attachment").lower().strip()if disposition not in {"inline", "attachment"}:disposition = "attachment"try:- abs_path, content_type, filename = manager.resolve_asset_content_for_download(- asset_info_id=str(uuid.UUID(request.match_info["id"])),+ result = resolve_asset_for_download(+ reference_id=str(uuid.UUID(request.match_info["id"])),owner_id=USER_MANAGER.get_request_user_id(request),)+ abs_path = result.abs_path+ content_type = result.content_type+ filename = result.download_nameexcept ValueError as ve:- return _error_response(404, "ASSET_NOT_FOUND", str(ve))+ return _build_error_response(404, "ASSET_NOT_FOUND", str(ve))except NotImplementedError as nie:- return _error_response(501, "BACKEND_UNSUPPORTED", str(nie))+ return _build_error_response(501, "BACKEND_UNSUPPORTED", str(nie))except FileNotFoundError:- return _error_response(404, "FILE_NOT_FOUND", "Underlying file not found on disk.")+ return _build_error_response(+ 404, "FILE_NOT_FOUND", "Underlying file not found on disk."+ )- quoted = (filename or "").replace("\r", "").replace("\n", "").replace('"', "'")- cd = f'{disposition}; filename="{quoted}"; filename*=UTF-8\'\'{urllib.parse.quote(filename)}'+ _DANGEROUS_MIME_TYPES = {+ "text/html", "text/html-sandboxed", "application/xhtml+xml",+ "text/javascript", "text/css",+ }+ if content_type in _DANGEROUS_MIME_TYPES:+ content_type = "application/octet-stream"++ safe_name = (filename or "").replace("\r", "").replace("\n", "")+ encoded = urllib.parse.quote(safe_name)+ cd = f"{disposition}; filename*=UTF-8''{encoded}"file_size = os.path.getsize(abs_path)+ size_mb = file_size / (1024 * 1024)logging.info(- "download_asset_content: path=%s, size=%d bytes (%.2f MB), content_type=%s, filename=%s",+ "download_asset_content: path=%s, size=%d bytes (%.2f MB), type=%s, name=%s",abs_path,file_size,- file_size / (1024 * 1024),+ size_mb,content_type,filename,)- async def file_sender():+ async def stream_file_chunks():chunk_size = 64 * 1024with open(abs_path, "rb") as f:while True:@@ -139,26 +289,30 @@ async def download_asset_content(request: web.Request) -> web.Response:yield chunkreturn web.Response(- body=file_sender(),+ body=stream_file_chunks(),content_type=content_type,headers={"Content-Disposition": cd,"Content-Length": str(file_size),+ "X-Content-Type-Options": "nosniff",},)@ROUTES.post("/api/assets/from-hash")-async def create_asset_from_hash(request: web.Request) -> web.Response:+@_require_assets_feature_enabled+async def create_asset_from_hash_route(request: web.Request) -> web.Response:try:payload = await request.json()body = schemas_in.CreateFromHashBody.model_validate(payload)except ValidationError as ve:- return _validation_error_response("INVALID_BODY", ve)+ return _build_validation_error_response("INVALID_BODY", ve)except Exception:- return _error_response(400, "INVALID_JSON", "Request body must be valid JSON.")+ return _build_error_response(+ 400, "INVALID_JSON", "Request body must be valid JSON."+ )- result = manager.create_asset_from_hash(+ result = create_from_hash(hash_str=body.hash,name=body.name,tags=body.tags,@@ -166,246 +320,209 @@ async def create_asset_from_hash(request: web.Request) -> web.Response:owner_id=USER_MANAGER.get_request_user_id(request),)if result is None:- return _error_response(404, "ASSET_NOT_FOUND", f"Asset content {body.hash} does not exist")- return web.json_response(result.model_dump(mode="json"), status=201)+ return _build_error_response(+ 404, "ASSET_NOT_FOUND", f"Asset content {body.hash} does not exist"+ )++ payload_out = schemas_out.AssetCreated(+ id=result.ref.id,+ name=result.ref.name,+ asset_hash=result.asset.hash,+ size=int(result.asset.size_bytes),+ mime_type=result.asset.mime_type,+ tags=result.tags,+ user_metadata=result.ref.user_metadata or {},+ preview_id=result.ref.preview_id,+ created_at=result.ref.created_at,+ last_access_time=result.ref.last_access_time,+ created_new=result.created_new,+ )+ return web.json_response(payload_out.model_dump(mode="json"), status=201)@ROUTES.post("/api/assets")+@_require_assets_feature_enabledasync def upload_asset(request: web.Request) -> web.Response:"""Multipart/form-data endpoint for Asset uploads."""- if not (request.content_type or "").lower().startswith("multipart/"):- return _error_response(415, "UNSUPPORTED_MEDIA_TYPE", "Use multipart/form-data for uploads.")-- reader = await request.multipart()-- file_present = False- file_client_name: str | None = None- tags_raw: list[str] = []- provided_name: str | None = None- user_metadata_raw: str | None = None- provided_hash: str | None = None- provided_hash_exists: bool | None = None-- file_written = 0- tmp_path: str | None = None- while True:- field = await reader.next()- if field is None:- break-- fname = getattr(field, "name", "") or ""-- if fname == "hash":- try:- s = ((await field.text()) or "").strip().lower()- except Exception:- return _error_response(400, "INVALID_HASH", "hash must be like 'blake3:<hex>'")-- if s:- if ":" not in s:- return _error_response(400, "INVALID_HASH", "hash must be like 'blake3:<hex>'")- algo, digest = s.split(":", 1)- if algo != "blake3" or not digest or any(c for c in digest if c not in "0123456789abcdef"):- return _error_response(400, "INVALID_HASH", "hash must be like 'blake3:<hex>'")- provided_hash = f"{algo}:{digest}"- try:- provided_hash_exists = manager.asset_exists(asset_hash=provided_hash)- except Exception:- provided_hash_exists = None # do not fail the whole request here-- elif fname == "file":- file_present = True- file_client_name = (field.filename or "").strip()-- if provided_hash and provided_hash_exists is True:- # If client supplied a hash that we know exists, drain but do not write to disk- try:- while True:- chunk = await field.read_chunk(8 * 1024 * 1024)- if not chunk:- break- file_written += len(chunk)- except Exception:- return _error_response(500, "UPLOAD_IO_ERROR", "Failed to receive uploaded file.")- continue # Do not create temp file; we will create AssetInfo from the existing content-- # Otherwise, store to temp for hashing/ingest- uploads_root = os.path.join(folder_paths.get_temp_directory(), "uploads")- unique_dir = os.path.join(uploads_root, uuid.uuid4().hex)- os.makedirs(unique_dir, exist_ok=True)- tmp_path = os.path.join(unique_dir, ".upload.part")-- try:- with open(tmp_path, "wb") as f:- while True:- chunk = await field.read_chunk(8 * 1024 * 1024)- if not chunk:- break- f.write(chunk)- file_written += len(chunk)- except Exception:- try:- if os.path.exists(tmp_path or ""):- os.remove(tmp_path)- finally:- return _error_response(500, "UPLOAD_IO_ERROR", "Failed to receive and store uploaded file.")- elif fname == "tags":- tags_raw.append((await field.text()) or "")- elif fname == "name":- provided_name = (await field.text()) or None- elif fname == "user_metadata":- user_metadata_raw = (await field.text()) or None-- # If client did not send file, and we are not doing a from-hash fast path -> error- if not file_present and not (provided_hash and provided_hash_exists):- return _error_response(400, "MISSING_FILE", "Form must include a 'file' part or a known 'hash'.")-- if file_present and file_written == 0 and not (provided_hash and provided_hash_exists):- # Empty upload is only acceptable if we are fast-pathing from existing hash- try:- if tmp_path and os.path.exists(tmp_path):- os.remove(tmp_path)- finally:- return _error_response(400, "EMPTY_UPLOAD", "Uploaded file is empty.")+ try:+ parsed = await parse_multipart_upload(request, check_hash_exists=asset_exists)+ except UploadError as e:+ return _build_error_response(e.status, e.code, e.message)++ owner_id = USER_MANAGER.get_request_user_id(request)try:- spec = schemas_in.UploadAssetSpec.model_validate({- "tags": tags_raw,- "name": provided_name,- "user_metadata": user_metadata_raw,- "hash": provided_hash,- })+ spec = schemas_in.UploadAssetSpec.model_validate(+ {+ "tags": parsed.tags_raw,+ "name": parsed.provided_name,+ "user_metadata": parsed.user_metadata_raw,+ "hash": parsed.provided_hash,+ }+ )except ValidationError as ve:- try:- if tmp_path and os.path.exists(tmp_path):- os.remove(tmp_path)- finally:- return _validation_error_response("INVALID_BODY", ve)+ delete_temp_file_if_exists(parsed.tmp_path)+ return _build_error_response(+ 400, "INVALID_BODY", f"Validation failed: {ve.json()}"+ )- # Validate models category against configured folders (consistent with previous behavior)if spec.tags and spec.tags[0] == "models":- if len(spec.tags) < 2 or spec.tags[1] not in folder_paths.folder_names_and_paths:- if tmp_path and os.path.exists(tmp_path):- os.remove(tmp_path)- return _error_response(- 400, "INVALID_BODY", f"unknown models category '{spec.tags[1] if len(spec.tags) >= 2 else ''}'"+ if (+ len(spec.tags) < 2+ or spec.tags[1] not in folder_paths.folder_names_and_paths+ ):+ delete_temp_file_if_exists(parsed.tmp_path)+ category = spec.tags[1] if len(spec.tags) >= 2 else ""+ return _build_error_response(+ 400, "INVALID_BODY", f"unknown models category '{category}'")- owner_id = USER_MANAGER.get_request_user_id(request)-- # Fast path: if a valid provided hash exists, create AssetInfo without writing anything- if spec.hash and provided_hash_exists is True:- try:- result = manager.create_asset_from_hash(+ try:+ # Fast path: hash exists, create AssetReference without writing anything+ if spec.hash and parsed.provided_hash_exists is True:+ result = create_from_hash(hash_str=spec.hash,name=spec.name or (spec.hash.split(":", 1)[1]),tags=spec.tags,user_metadata=spec.user_metadata or {},owner_id=owner_id,)- except Exception:- logging.exception("create_asset_from_hash failed for hash=%s, owner_id=%s", spec.hash, owner_id)- return _error_response(500, "INTERNAL", "Unexpected server error.")-- if result is None:- return _error_response(404, "ASSET_NOT_FOUND", f"Asset content {spec.hash} does not exist")-- # Drain temp if we accidentally saved (e.g., hash field came after file)- if tmp_path and os.path.exists(tmp_path):- with contextlib.suppress(Exception):- os.remove(tmp_path)-- status = 200 if (not result.created_new) else 201- return web.json_response(result.model_dump(mode="json"), status=status)-- # Otherwise, we must have a temp file path to ingest- if not tmp_path or not os.path.exists(tmp_path):- # The only case we reach here without a temp file is: client sent a hash that does not exist and no file- return _error_response(404, "ASSET_NOT_FOUND", "Provided hash not found and no file uploaded.")-- try:- created = manager.upload_asset_from_temp_path(- spec,- temp_path=tmp_path,- client_filename=file_client_name,- owner_id=owner_id,- expected_asset_hash=spec.hash,- )- status = 201 if created.created_new else 200- return web.json_response(created.model_dump(mode="json"), status=status)- except ValueError as e:- if tmp_path and os.path.exists(tmp_path):- os.remove(tmp_path)- msg = str(e)- if "HASH_MISMATCH" in msg or msg.strip().upper() == "HASH_MISMATCH":- return _error_response(- 400,- "HASH_MISMATCH",- "Uploaded file hash does not match provided hash.",+ if result is None:+ delete_temp_file_if_exists(parsed.tmp_path)+ return _build_error_response(+ 404, "ASSET_NOT_FOUND", f"Asset content {spec.hash} does not exist"+ )+ delete_temp_file_if_exists(parsed.tmp_path)+ else:+ # Otherwise, we must have a temp file path to ingest+ if not parsed.tmp_path or not os.path.exists(parsed.tmp_path):+ return _build_error_response(+ 400,+ "MISSING_INPUT",+ "Provided hash not found and no file uploaded.",+ )++ result = upload_from_temp_path(+ temp_path=parsed.tmp_path,+ name=spec.name,+ tags=spec.tags,+ user_metadata=spec.user_metadata or {},+ client_filename=parsed.file_client_name,+ owner_id=owner_id,+ expected_hash=spec.hash,)- return _error_response(400, "BAD_REQUEST", "Invalid inputs.")+ except AssetValidationError as e:+ delete_temp_file_if_exists(parsed.tmp_path)+ return _build_error_response(400, e.code, str(e))+ except ValueError as e:+ delete_temp_file_if_exists(parsed.tmp_path)+ return _build_error_response(400, "BAD_REQUEST", str(e))+ except HashMismatchError as e:+ delete_temp_file_if_exists(parsed.tmp_path)+ return _build_error_response(400, "HASH_MISMATCH", str(e))+ except DependencyMissingError as e:+ delete_temp_file_if_exists(parsed.tmp_path)+ return _build_error_response(503, "DEPENDENCY_MISSING", e.message)except Exception:- if tmp_path and os.path.exists(tmp_path):- os.remove(tmp_path)- logging.exception("upload_asset_from_temp_path failed for tmp_path=%s, owner_id=%s", tmp_path, owner_id)- return _error_response(500, "INTERNAL", "Unexpected server error.")+ delete_temp_file_if_exists(parsed.tmp_path)+ logging.exception("upload_asset failed for owner_id=%s", owner_id)+ return _build_error_response(500, "INTERNAL", "Unexpected server error.")++ payload = schemas_out.AssetCreated(+ id=result.ref.id,+ name=result.ref.name,+ asset_hash=result.asset.hash,+ size=int(result.asset.size_bytes),+ mime_type=result.asset.mime_type,+ tags=result.tags,+ user_metadata=result.ref.user_metadata or {},+ preview_id=result.ref.preview_id,+ created_at=result.ref.created_at,+ last_access_time=result.ref.last_access_time,+ created_new=result.created_new,+ )+ status = 201 if result.created_new else 200+ return web.json_response(payload.model_dump(mode="json"), status=status)@ROUTES.put(f"/api/assets/{{id:{UUID_RE}}}")-async def update_asset(request: web.Request) -> web.Response:- asset_info_id = str(uuid.UUID(request.match_info["id"]))+@_require_assets_feature_enabled+async def update_asset_route(request: web.Request) -> web.Response:+ reference_id = str(uuid.UUID(request.match_info["id"]))try:body = schemas_in.UpdateAssetBody.model_validate(await request.json())except ValidationError as ve:- return _validation_error_response("INVALID_BODY", ve)+ return _build_validation_error_response("INVALID_BODY", ve)except Exception:- return _error_response(400, "INVALID_JSON", "Request body must be valid JSON.")+ return _build_error_response(+ 400, "INVALID_JSON", "Request body must be valid JSON."+ )try:- result = manager.update_asset(- asset_info_id=asset_info_id,+ result = update_asset_metadata(+ reference_id=reference_id,name=body.name,user_metadata=body.user_metadata,owner_id=USER_MANAGER.get_request_user_id(request),)- except (ValueError, PermissionError) as ve:- return _error_response(404, "ASSET_NOT_FOUND", str(ve), {"id": asset_info_id})+ payload = schemas_out.AssetUpdated(+ id=result.ref.id,+ name=result.ref.name,+ asset_hash=result.asset.hash if result.asset else None,+ tags=result.tags,+ user_metadata=result.ref.user_metadata or {},+ updated_at=result.ref.updated_at,+ )+ except PermissionError as pe:+ return _build_error_response(403, "FORBIDDEN", str(pe), {"id": reference_id})+ except ValueError as ve:+ return _build_error_response(+ 404, "ASSET_NOT_FOUND", str(ve), {"id": reference_id}+ )except Exception:logging.exception(- "update_asset failed for asset_info_id=%s, owner_id=%s",- asset_info_id,+ "update_asset failed for reference_id=%s, owner_id=%s",+ reference_id,USER_MANAGER.get_request_user_id(request),)- return _error_response(500, "INTERNAL", "Unexpected server error.")- return web.json_response(result.model_dump(mode="json"), status=200)+ return _build_error_response(500, "INTERNAL", "Unexpected server error.")+ return web.json_response(payload.model_dump(mode="json"), status=200)@ROUTES.delete(f"/api/assets/{{id:{UUID_RE}}}")-async def delete_asset(request: web.Request) -> web.Response:- asset_info_id = str(uuid.UUID(request.match_info["id"]))- delete_content = request.query.get("delete_content")- delete_content = True if delete_content is None else delete_content.lower() not in {"0", "false", "no"}+@_require_assets_feature_enabled+async def delete_asset_route(request: web.Request) -> web.Response:+ reference_id = str(uuid.UUID(request.match_info["id"]))+ delete_content_param = request.query.get("delete_content")+ delete_content = (+ False+ if delete_content_param is None+ else delete_content_param.lower() not in {"0", "false", "no"}+ )try:- deleted = manager.delete_asset_reference(- asset_info_id=asset_info_id,+ deleted = delete_asset_reference(+ reference_id=reference_id,owner_id=USER_MANAGER.get_request_user_id(request),delete_content_if_orphan=delete_content,)except Exception:logging.exception(- "delete_asset_reference failed for asset_info_id=%s, owner_id=%s",- asset_info_id,+ "delete_asset_reference failed for reference_id=%s, owner_id=%s",+ reference_id,USER_MANAGER.get_request_user_id(request),)- return _error_response(500, "INTERNAL", "Unexpected server error.")+ return _build_error_response(500, "INTERNAL", "Unexpected server error.")if not deleted:- return _error_response(404, "ASSET_NOT_FOUND", f"AssetInfo {asset_info_id} not found.")+ return _build_error_response(+ 404, "ASSET_NOT_FOUND", f"AssetReference {reference_id} not found."+ )return web.Response(status=204)@ROUTES.get("/api/tags")+@_require_assets_feature_enabledasync def get_tags(request: web.Request) -> web.Response:"""GET request to list all tags based on query parameters.@@ -415,12 +532,14 @@ async def get_tags(request: web.Request) -> web.Response:try:query = schemas_in.TagsListQuery.model_validate(query_map)except ValidationError as e:- return web.json_response(- {"error": {"code": "INVALID_QUERY", "message": "Invalid query parameters", "details": e.errors()}},- status=400,+ return _build_error_response(+ 400,+ "INVALID_QUERY",+ "Invalid query parameters",+ {"errors": json.loads(e.json())},)- result = manager.list_tags(+ rows, total = list_tags(prefix=query.prefix,limit=query.limit,offset=query.offset,@@ -428,87 +547,212 @@ async def get_tags(request: web.Request) -> web.Response:include_zero=query.include_zero,owner_id=USER_MANAGER.get_request_user_id(request),)- return web.json_response(result.model_dump(mode="json"))++ tags = [+ schemas_out.TagUsage(name=name, count=count, type=tag_type)+ for (name, tag_type, count) in rows+ ]+ payload = schemas_out.TagsList(+ tags=tags, total=total, has_more=(query.offset + len(tags)) < total+ )+ return web.json_response(payload.model_dump(mode="json"))@ROUTES.post(f"/api/assets/{{id:{UUID_RE}}}/tags")+@_require_assets_feature_enabledasync def add_asset_tags(request: web.Request) -> web.Response:- asset_info_id = str(uuid.UUID(request.match_info["id"]))+ reference_id = str(uuid.UUID(request.match_info["id"]))try:- payload = await request.json()- data = schemas_in.TagsAdd.model_validate(payload)+ json_payload = await request.json()+ data = schemas_in.TagsAdd.model_validate(json_payload)except ValidationError as ve:- return _error_response(400, "INVALID_BODY", "Invalid JSON body for tags add.", {"errors": ve.errors()})+ return _build_error_response(+ 400,+ "INVALID_BODY",+ "Invalid JSON body for tags add.",+ {"errors": ve.errors()},+ )except Exception:- return _error_response(400, "INVALID_JSON", "Request body must be valid JSON.")+ return _build_error_response(+ 400, "INVALID_JSON", "Request body must be valid JSON."+ )try:- result = manager.add_tags_to_asset(- asset_info_id=asset_info_id,+ result = apply_tags(+ reference_id=reference_id,tags=data.tags,origin="manual",owner_id=USER_MANAGER.get_request_user_id(request),)- except (ValueError, PermissionError) as ve:- return _error_response(404, "ASSET_NOT_FOUND", str(ve), {"id": asset_info_id})+ payload = schemas_out.TagsAdd(+ added=result.added,+ already_present=result.already_present,+ total_tags=result.total_tags,+ )+ except PermissionError as pe:+ return _build_error_response(403, "FORBIDDEN", str(pe), {"id": reference_id})+ except ValueError as ve:+ return _build_error_response(+ 404, "ASSET_NOT_FOUND", str(ve), {"id": reference_id}+ )except Exception:logging.exception(- "add_tags_to_asset failed for asset_info_id=%s, owner_id=%s",- asset_info_id,+ "add_tags_to_asset failed for reference_id=%s, owner_id=%s",+ reference_id,USER_MANAGER.get_request_user_id(request),)- return _error_response(500, "INTERNAL", "Unexpected server error.")+ return _build_error_response(500, "INTERNAL", "Unexpected server error.")- return web.json_response(result.model_dump(mode="json"), status=200)+ return web.json_response(payload.model_dump(mode="json"), status=200)@ROUTES.delete(f"/api/assets/{{id:{UUID_RE}}}/tags")+@_require_assets_feature_enabledasync def delete_asset_tags(request: web.Request) -> web.Response:- asset_info_id = str(uuid.UUID(request.match_info["id"]))+ reference_id = str(uuid.UUID(request.match_info["id"]))try:- payload = await request.json()- data = schemas_in.TagsRemove.model_validate(payload)+ json_payload = await request.json()+ data = schemas_in.TagsRemove.model_validate(json_payload)except ValidationError as ve:- return _error_response(400, "INVALID_BODY", "Invalid JSON body for tags remove.", {"errors": ve.errors()})+ return _build_error_response(+ 400,+ "INVALID_BODY",+ "Invalid JSON body for tags remove.",+ {"errors": ve.errors()},+ )except Exception:- return _error_response(400, "INVALID_JSON", "Request body must be valid JSON.")+ return _build_error_response(+ 400, "INVALID_JSON", "Request body must be valid JSON."+ )try:- result = manager.remove_tags_from_asset(- asset_info_id=asset_info_id,+ result = remove_tags(+ reference_id=reference_id,tags=data.tags,owner_id=USER_MANAGER.get_request_user_id(request),)+ payload = schemas_out.TagsRemove(+ removed=result.removed,+ not_present=result.not_present,+ total_tags=result.total_tags,+ )+ except PermissionError as pe:+ return _build_error_response(403, "FORBIDDEN", str(pe), {"id": reference_id})except ValueError as ve:- return _error_response(404, "ASSET_NOT_FOUND", str(ve), {"id": asset_info_id})+ return _build_error_response(+ 404, "ASSET_NOT_FOUND", str(ve), {"id": reference_id}+ )except Exception:logging.exception(- "remove_tags_from_asset failed for asset_info_id=%s, owner_id=%s",- asset_info_id,+ "remove_tags_from_asset failed for reference_id=%s, owner_id=%s",+ reference_id,USER_MANAGER.get_request_user_id(request),)- return _error_response(500, "INTERNAL", "Unexpected server error.")+ return _build_error_response(500, "INTERNAL", "Unexpected server error.")- return web.json_response(result.model_dump(mode="json"), status=200)+ return web.json_response(payload.model_dump(mode="json"), status=200)@ROUTES.post("/api/assets/seed")-async def seed_assets_endpoint(request: web.Request) -> web.Response:- """Trigger asset seeding for specified roots (models, input, output)."""+@_require_assets_feature_enabled+async def seed_assets(request: web.Request) -> web.Response:+ """Trigger asset seeding for specified roots (models, input, output).++ Query params:+ wait: If "true", block until scan completes (synchronous behavior for tests)++ Returns:+ 202 Accepted if scan started+ 409 Conflict if scan already running+ 200 OK with final stats if wait=true+ """try:payload = await request.json()roots = payload.get("roots", ["models", "input", "output"])except Exception:roots = ["models", "input", "output"]- valid_roots = [r for r in roots if r in ("models", "input", "output")]+ valid_roots = tuple(r for r in roots if r in ("models", "input", "output"))if not valid_roots:- return _error_response(400, "INVALID_BODY", "No valid roots specified")+ return _build_error_response(400, "INVALID_BODY", "No valid roots specified")++ wait_param = request.query.get("wait", "").lower()+ should_wait = wait_param in ("true", "1", "yes")++ started = asset_seeder.start(roots=valid_roots)+ if not started:+ return web.json_response({"status": "already_running"}, status=409)++ if should_wait:+ await asyncio.to_thread(asset_seeder.wait)+ status = asset_seeder.get_status()+ return web.json_response(+ {+ "status": "completed",+ "progress": {+ "scanned": status.progress.scanned if status.progress else 0,+ "total": status.progress.total if status.progress else 0,+ "created": status.progress.created if status.progress else 0,+ "skipped": status.progress.skipped if status.progress else 0,+ },+ "errors": status.errors,+ },+ status=200,+ )++ return web.json_response({"status": "started"}, status=202)++[email protected]("/api/assets/seed/status")+@_require_assets_feature_enabled+async def get_seed_status(request: web.Request) -> web.Response:+ """Get current scan status and progress."""+ status = asset_seeder.get_status()+ return web.json_response(+ {+ "state": status.state.value,+ "progress": {+ "scanned": status.progress.scanned,+ "total": status.progress.total,+ "created": status.progress.created,+ "skipped": status.progress.skipped,+ }+ if status.progress+ else None,+ "errors": status.errors,+ },+ status=200,+ )++[email protected]("/api/assets/seed/cancel")+@_require_assets_feature_enabled+async def cancel_seed(request: web.Request) -> web.Response:+ """Request cancellation of in-progress scan."""+ cancelled = asset_seeder.cancel()+ if cancelled:+ return web.json_response({"status": "cancelling"}, status=200)+ return web.json_response({"status": "idle"}, status=200)- try:- seed_assets(tuple(valid_roots))- except Exception:- logging.exception("seed_assets failed for roots=%s", valid_roots)- return _error_response(500, "INTERNAL", "Seed operation failed")- return web.json_response({"seeded": valid_roots}, status=200)[email protected]("/api/assets/prune")+@_require_assets_feature_enabled+async def mark_missing_assets(request: web.Request) -> web.Response:+ """Mark assets as missing when outside all known root prefixes.++ This is a non-destructive soft-delete operation. Assets and metadata+ are preserved, but references are flagged as missing. They can be+ restored if the file reappears in a future scan.++ Returns:+ 200 OK with count of marked assets+ 409 Conflict if a scan is currently running+ """+ try:+ marked = asset_seeder.mark_missing_outside_prefixes()+ except ScanInProgressError:+ return web.json_response(+ {"status": "scan_running", "marked": 0},+ status=409,+ )+ return web.json_response({"status": "completed", "marked": marked}, status=200)diff --git a/app/assets/api/schemas_in.py b/app/assets/api/schemas_in.pyindex 6707ffb0c..d255c938e 100644--- a/app/assets/api/schemas_in.py+++ b/app/assets/api/schemas_in.py@@ -1,6 +1,8 @@import json+from dataclasses import dataclassfrom typing import Any, Literal+from app.assets.helpers import validate_blake3_hashfrom pydantic import (BaseModel,ConfigDict,@@ -10,6 +12,41 @@ from pydantic import (model_validator,)++class UploadError(Exception):+ """Error during upload parsing with HTTP status and code."""++ def __init__(self, status: int, code: str, message: str):+ super().__init__(message)+ self.status = status+ self.code = code+ self.message = message+++class AssetValidationError(Exception):+ """Validation error in asset processing (invalid tags, metadata, etc.)."""++ def __init__(self, code: str, message: str):+ super().__init__(message)+ self.code = code+ self.message = message+++@dataclass+class ParsedUpload:+ """Result of parsing a multipart upload request."""++ file_present: bool+ file_written: int+ file_client_name: str | None+ tmp_path: str | None+ tags_raw: list[str]+ provided_name: str | None+ user_metadata_raw: str | None+ provided_hash: str | None+ provided_hash_exists: bool | None++class ListAssetsQuery(BaseModel):include_tags: list[str] = Field(default_factory=list)exclude_tags: list[str] = Field(default_factory=list)@@ -21,7 +58,9 @@ class ListAssetsQuery(BaseModel):limit: conint(ge=1, le=500) = 20offset: conint(ge=0) = 0- sort: Literal["name", "created_at", "updated_at", "size", "last_access_time"] = "created_at"+ sort: Literal["name", "created_at", "updated_at", "size", "last_access_time"] = (+ "created_at"+ )order: Literal["asc", "desc"] = "desc"@field_validator("include_tags", "exclude_tags", mode="before")@@ -61,7 +100,7 @@ class UpdateAssetBody(BaseModel):user_metadata: dict[str, Any] | None = None@model_validator(mode="after")- def _at_least_one(self):+ def _validate_at_least_one_field(self):if self.name is None and self.user_metadata is None:raise ValueError("Provide at least one of: name, user_metadata.")return self@@ -78,19 +117,11 @@ class CreateFromHashBody(BaseModel):@field_validator("hash")@classmethoddef _require_blake3(cls, v):- s = (v or "").strip().lower()- if ":" not in s:- raise ValueError("hash must be 'blake3:<hex>'")- algo, digest = s.split(":", 1)- if algo != "blake3":- raise ValueError("only canonical 'blake3:<hex>' is accepted here")- if not digest or any(c for c in digest if c not in "0123456789abcdef"):- raise ValueError("hash digest must be lowercase hex")- return s+ return validate_blake3_hash(v or "")@field_validator("tags", mode="before")@classmethod- def _tags_norm(cls, v):+ def _normalize_tags_field(cls, v):if v is None:return []if isinstance(v, list):@@ -154,15 +185,16 @@ class TagsRemove(TagsAdd):class UploadAssetSpec(BaseModel):"""Upload Asset operation.+- tags: ordered; first is root ('models'|'input'|'output');- if root == 'models', second must be a valid category from folder_paths.folder_names_and_paths+ if root == 'models', second must be a valid category- name: display name- user_metadata: arbitrary JSON object (optional)- - hash: optional canonical 'blake3:<hex>' provided by the client for validation / fast-path+ - hash: optional canonical 'blake3:<hex>' for validation / fast-path- Files created via this endpoint are stored on disk using the **content hash** as the filename stem- and the original extension is preserved when available.+ Files are stored using the content hash as filename stem."""+model_config = ConfigDict(extra="ignore", str_strip_whitespace=True)tags: list[str] = Field(..., min_length=1)@@ -175,17 +207,10 @@ class UploadAssetSpec(BaseModel):def _parse_hash(cls, v):if v is None:return None- s = str(v).strip().lower()+ s = str(v).strip()if not s:return None- if ":" not in s:- raise ValueError("hash must be 'blake3:<hex>'")- algo, digest = s.split(":", 1)- if algo != "blake3":- raise ValueError("only canonical 'blake3:<hex>' is accepted here")- if not digest or any(c for c in digest if c not in "0123456789abcdef"):- raise ValueError("hash digest must be lowercase hex")- return f"{algo}:{digest}"+ return validate_blake3_hash(s)@field_validator("tags", mode="before")@classmethod@@ -260,5 +285,7 @@ class UploadAssetSpec(BaseModel):raise ValueError("first tag must be one of: models, input, output")if root == "models":if len(self.tags) < 2:- raise ValueError("models uploads require a category tag as the second tag")+ raise ValueError(+ "models uploads require a category tag as the second tag"+ )return selfdiff --git a/app/assets/api/schemas_out.py b/app/assets/api/schemas_out.pyindex b6fb3da0c..f36447856 100644--- a/app/assets/api/schemas_out.py+++ b/app/assets/api/schemas_out.py@@ -19,7 +19,7 @@ class AssetSummary(BaseModel):model_config = ConfigDict(from_attributes=True)@field_serializer("created_at", "updated_at", "last_access_time")- def _ser_dt(self, v: datetime | None, _info):+ def _serialize_datetime(self, v: datetime | None, _info):return v.isoformat() if v else None@@ -40,7 +40,7 @@ class AssetUpdated(BaseModel):model_config = ConfigDict(from_attributes=True)@field_serializer("updated_at")- def _ser_updated(self, v: datetime | None, _info):+ def _serialize_updated_at(self, v: datetime | None, _info):return v.isoformat() if v else None@@ -59,7 +59,7 @@ class AssetDetail(BaseModel):model_config = ConfigDict(from_attributes=True)@field_serializer("created_at", "last_access_time")- def _ser_dt(self, v: datetime | None, _info):+ def _serialize_datetime(self, v: datetime | None, _info):return v.isoformat() if v else Nonediff --git a/app/assets/api/upload.py b/app/assets/api/upload.pynew file mode 100644index 000000000..721c12f4d--- /dev/null+++ b/app/assets/api/upload.py@@ -0,0 +1,171 @@+import logging+import os+import uuid+from typing import Callable++from aiohttp import web++import folder_paths+from app.assets.api.schemas_in import ParsedUpload, UploadError+from app.assets.helpers import validate_blake3_hash+++def normalize_and_validate_hash(s: str) -> str:+ """Validate and normalize a hash string.++ Returns canonical 'blake3:<hex>' or raises UploadError.+ """+ try:+ return validate_blake3_hash(s)+ except ValueError:+ raise UploadError(400, "INVALID_HASH", "hash must be like 'blake3:<hex>'")+++async def parse_multipart_upload(+ request: web.Request,+ check_hash_exists: Callable[[str], bool],+) -> ParsedUpload:+ """+ Parse a multipart/form-data upload request.++ Args:+ request: The aiohttp request+ check_hash_exists: Callable(hash_str) -> bool to check if a hash exists++ Returns:+ ParsedUpload with parsed fields and temp file path++ Raises:+ UploadError: On validation or I/O errors+ """+ if not (request.content_type or "").lower().startswith("multipart/"):+ raise UploadError(+ 415, "UNSUPPORTED_MEDIA_TYPE", "Use multipart/form-data for uploads."+ )++ reader = await request.multipart()++ file_present = False+ file_client_name: str | None = None+ tags_raw: list[str] = []+ provided_name: str | None = None+ user_metadata_raw: str | None = None+ provided_hash: str | None = None+ provided_hash_exists: bool | None = None++ file_written = 0+ tmp_path: str | None = None++ while True:+ field = await reader.next()+ if field is None:+ break++ fname = getattr(field, "name", "") or ""++ if fname == "hash":+ try:+ s = ((await field.text()) or "").strip().lower()+ except Exception:+ raise UploadError(+ 400, "INVALID_HASH", "hash must be like 'blake3:<hex>'"+ )++ if s:+ provided_hash = normalize_and_validate_hash(s)+ try:+ provided_hash_exists = check_hash_exists(provided_hash)+ except Exception as e:+ logging.exception(+ "check_hash_exists failed for hash=%s: %s", provided_hash, e+ )+ raise UploadError(+ 500,+ "HASH_CHECK_FAILED",+ "Backend error while checking asset hash.",+ )++ elif fname == "file":+ file_present = True+ file_client_name = (field.filename or "").strip()++ if provided_hash and provided_hash_exists is True:+ # Hash exists - drain file but don't write to disk+ try:+ while True:+ chunk = await field.read_chunk(8 * 1024 * 1024)+ if not chunk:+ break+ file_written += len(chunk)+ except Exception:+ raise UploadError(+ 500, "UPLOAD_IO_ERROR", "Failed to receive uploaded file."+ )+ continue++ uploads_root = os.path.join(folder_paths.get_temp_directory(), "uploads")+ unique_dir = os.path.join(uploads_root, uuid.uuid4().hex)+ os.makedirs(unique_dir, exist_ok=True)+ tmp_path = os.path.join(unique_dir, ".upload.part")++ try:+ with open(tmp_path, "wb") as f:+ while True:+ chunk = await field.read_chunk(8 * 1024 * 1024)+ if not chunk:+ break+ f.write(chunk)+ file_written += len(chunk)+ except Exception:+ delete_temp_file_if_exists(tmp_path)+ raise UploadError(+ 500, "UPLOAD_IO_ERROR", "Failed to receive and store uploaded file."+ )++ elif fname == "tags":+ tags_raw.append((await field.text()) or "")+ elif fname == "name":+ provided_name = (await field.text()) or None+ elif fname == "user_metadata":+ user_metadata_raw = (await field.text()) or None++ if not file_present and not (provided_hash and provided_hash_exists):+ raise UploadError(+ 400, "MISSING_FILE", "Form must include a 'file' part or a known 'hash'."+ )++ if (+ file_present+ and file_written == 0+ and not (provided_hash and provided_hash_exists)+ ):+ delete_temp_file_if_exists(tmp_path)+ raise UploadError(400, "EMPTY_UPLOAD", "Uploaded file is empty.")++ return ParsedUpload(+ file_present=file_present,+ file_written=file_written,+ file_client_name=file_client_name,+ tmp_path=tmp_path,+ tags_raw=tags_raw,+ provided_name=provided_name,+ user_metadata_raw=user_metadata_raw,+ provided_hash=provided_hash,+ provided_hash_exists=provided_hash_exists,+ )+++def delete_temp_file_if_exists(tmp_path: str | None) -> None:+ """Safely remove a temp file and its parent directory if empty."""+ if tmp_path:+ try:+ if os.path.exists(tmp_path):+ os.remove(tmp_path)+ except OSError as e:+ logging.debug("Failed to delete temp file %s: %s", tmp_path, e)+ try:+ parent = os.path.dirname(tmp_path)+ if parent and os.path.isdir(parent):+ os.rmdir(parent) # only succeeds if empty+ except OSError:+ passdiff --git a/app/assets/database/bulk_ops.py b/app/assets/database/bulk_ops.pydeleted file mode 100644index c7b75290a..000000000--- a/app/assets/database/bulk_ops.py+++ /dev/null@@ -1,204 +0,0 @@-import os-import uuid-import sqlalchemy-from typing import Iterable-from sqlalchemy.orm import Session-from sqlalchemy.dialects import sqlite--from app.assets.helpers import utcnow-from app.assets.database.models import Asset, AssetCacheState, AssetInfo, AssetInfoTag, AssetInfoMeta--MAX_BIND_PARAMS = 800--def _chunk_rows(rows: list[dict], cols_per_row: int, max_bind_params: int) -> Iterable[list[dict]]:- if not rows:- return []- rows_per_stmt = max(1, max_bind_params // max(1, cols_per_row))- for i in range(0, len(rows), rows_per_stmt):- yield rows[i:i + rows_per_stmt]--def _iter_chunks(seq, n: int):- for i in range(0, len(seq), n):- yield seq[i:i + n]--def _rows_per_stmt(cols: int) -> int:- return max(1, MAX_BIND_PARAMS // max(1, cols))---def seed_from_paths_batch(- session: Session,- *,- specs: list[dict],- owner_id: str = "",-) -> dict:- """Each spec is a dict with keys:- - abs_path: str- - size_bytes: int- - mtime_ns: int- - info_name: str- - tags: list[str]- - fname: Optional[str]- """- if not specs:- return {"inserted_infos": 0, "won_states": 0, "lost_states": 0}-- now = utcnow()- asset_rows: list[dict] = []- state_rows: list[dict] = []- path_to_asset: dict[str, str] = {}- asset_to_info: dict[str, dict] = {} # asset_id -> prepared info row- path_list: list[str] = []-- for sp in specs:- ap = os.path.abspath(sp["abs_path"])- aid = str(uuid.uuid4())- iid = str(uuid.uuid4())- path_list.append(ap)- path_to_asset[ap] = aid-- asset_rows.append(- {- "id": aid,- "hash": None,- "size_bytes": sp["size_bytes"],- "mime_type": None,- "created_at": now,- }- )- state_rows.append(- {- "asset_id": aid,- "file_path": ap,- "mtime_ns": sp["mtime_ns"],- }- )- asset_to_info[aid] = {- "id": iid,- "owner_id": owner_id,- "name": sp["info_name"],- "asset_id": aid,- "preview_id": None,- "user_metadata": {"filename": sp["fname"]} if sp["fname"] else None,- "created_at": now,- "updated_at": now,- "last_access_time": now,- "_tags": sp["tags"],- "_filename": sp["fname"],- }-- # insert all seed Assets (hash=NULL)- ins_asset = sqlite.insert(Asset)- for chunk in _iter_chunks(asset_rows, _rows_per_stmt(5)):- session.execute(ins_asset, chunk)-- # try to claim AssetCacheState (file_path)- # Insert with ON CONFLICT DO NOTHING, then query to find which paths were actually inserted- ins_state = (- sqlite.insert(AssetCacheState)- .on_conflict_do_nothing(index_elements=[AssetCacheState.file_path])- )- for chunk in _iter_chunks(state_rows, _rows_per_stmt(3)):- session.execute(ins_state, chunk)-- # Query to find which of our paths won (were actually inserted)- winners_by_path: set[str] = set()- for chunk in _iter_chunks(path_list, MAX_BIND_PARAMS):- result = session.execute(- sqlalchemy.select(AssetCacheState.file_path)- .where(AssetCacheState.file_path.in_(chunk))- .where(AssetCacheState.asset_id.in_([path_to_asset[p] for p in chunk]))- )- winners_by_path.update(result.scalars().all())-- all_paths_set = set(path_list)- losers_by_path = all_paths_set - winners_by_path- lost_assets = [path_to_asset[p] for p in losers_by_path]- if lost_assets: # losers get their Asset removed- for id_chunk in _iter_chunks(lost_assets, MAX_BIND_PARAMS):- session.execute(sqlalchemy.delete(Asset).where(Asset.id.in_(id_chunk)))-- if not winners_by_path:- return {"inserted_infos": 0, "won_states": 0, "lost_states": len(losers_by_path)}-- # insert AssetInfo only for winners- # Insert with ON CONFLICT DO NOTHING, then query to find which were actually inserted- winner_info_rows = [asset_to_info[path_to_asset[p]] for p in winners_by_path]- ins_info = (- sqlite.insert(AssetInfo)- .on_conflict_do_nothing(index_elements=[AssetInfo.asset_id, AssetInfo.owner_id, AssetInfo.name])- )- for chunk in _iter_chunks(winner_info_rows, _rows_per_stmt(9)):- session.execute(ins_info, chunk)-- # Query to find which info rows were actually inserted (by matching our generated IDs)- all_info_ids = [row["id"] for row in winner_info_rows]- inserted_info_ids: set[str] = set()- for chunk in _iter_chunks(all_info_ids, MAX_BIND_PARAMS):- result = session.execute(- sqlalchemy.select(AssetInfo.id).where(AssetInfo.id.in_(chunk))- )- inserted_info_ids.update(result.scalars().all())-- # build and insert tag + meta rows for the AssetInfo- tag_rows: list[dict] = []- meta_rows: list[dict] = []- if inserted_info_ids:- for row in winner_info_rows:- iid = row["id"]- if iid not in inserted_info_ids:- continue- for t in row["_tags"]:- tag_rows.append({- "asset_info_id": iid,- "tag_name": t,- "origin": "automatic",- "added_at": now,- })- if row["_filename"]:- meta_rows.append(- {- "asset_info_id": iid,- "key": "filename",- "ordinal": 0,- "val_str": row["_filename"],- "val_num": None,- "val_bool": None,- "val_json": None,- }- )-- bulk_insert_tags_and_meta(session, tag_rows=tag_rows, meta_rows=meta_rows, max_bind_params=MAX_BIND_PARAMS)- return {- "inserted_infos": len(inserted_info_ids),- "won_states": len(winners_by_path),- "lost_states": len(losers_by_path),- }---def bulk_insert_tags_and_meta(- session: Session,- *,- tag_rows: list[dict],- meta_rows: list[dict],- max_bind_params: int,-) -> None:- """Batch insert into asset_info_tags and asset_info_meta with ON CONFLICT DO NOTHING.- - tag_rows keys: asset_info_id, tag_name, origin, added_at- - meta_rows keys: asset_info_id, key, ordinal, val_str, val_num, val_bool, val_json- """- if tag_rows:- ins_links = (- sqlite.insert(AssetInfoTag)- .on_conflict_do_nothing(index_elements=[AssetInfoTag.asset_info_id, AssetInfoTag.tag_name])- )- for chunk in _chunk_rows(tag_rows, cols_per_row=4, max_bind_params=max_bind_params):- session.execute(ins_links, chunk)- if meta_rows:- ins_meta = (- sqlite.insert(AssetInfoMeta)- .on_conflict_do_nothing(- index_elements=[AssetInfoMeta.asset_info_id, AssetInfoMeta.key, AssetInfoMeta.ordinal]- )- )- for chunk in _chunk_rows(meta_rows, cols_per_row=7, max_bind_params=max_bind_params):- session.execute(ins_meta, chunk)diff --git a/app/assets/database/models.py b/app/assets/database/models.pyindex 3cd28f68b..03c1c1707 100644--- a/app/assets/database/models.py+++ b/app/assets/database/models.py@@ -2,8 +2,8 @@ from __future__ import annotationsimport uuidfrom datetime import datetime-from typing import Any+from sqlalchemy import (JSON,BigInteger,@@ -16,102 +16,102 @@ from sqlalchemy import (Numeric,String,Text,- UniqueConstraint,)from sqlalchemy.orm import Mapped, foreign, mapped_column, relationship-from app.assets.helpers import utcnow-from app.database.models import to_dict, Base+from app.assets.helpers import get_utc_now+from app.database.models import Baseclass Asset(Base):__tablename__ = "assets"- id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))+ id: Mapped[str] = mapped_column(+ String(36), primary_key=True, default=lambda: str(uuid.uuid4())+ )hash: Mapped[str | None] = mapped_column(String(256), nullable=True)size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)mime_type: Mapped[str | None] = mapped_column(String(255))created_at: Mapped[datetime] = mapped_column(- DateTime(timezone=False), nullable=False, default=utcnow+ DateTime(timezone=False), nullable=False, default=get_utc_now)- infos: Mapped[list[AssetInfo]] = relationship(- "AssetInfo",+ references: Mapped[list[AssetReference]] = relationship(+ "AssetReference",back_populates="asset",- primaryjoin=lambda: Asset.id == foreign(AssetInfo.asset_id),- foreign_keys=lambda: [AssetInfo.asset_id],+ primaryjoin=lambda: Asset.id == foreign(AssetReference.asset_id),+ foreign_keys=lambda: [AssetReference.asset_id],cascade="all,delete-orphan",passive_deletes=True,)- preview_of: Mapped[list[AssetInfo]] = relationship(- "AssetInfo",+ preview_of: Mapped[list[AssetReference]] = relationship(+ "AssetReference",back_populates="preview_asset",- primaryjoin=lambda: Asset.id == foreign(AssetInfo.preview_id),- foreign_keys=lambda: [AssetInfo.preview_id],+ primaryjoin=lambda: Asset.id == foreign(AssetReference.preview_id),+ foreign_keys=lambda: [AssetReference.preview_id],viewonly=True,)- cache_states: Mapped[list[AssetCacheState]] = relationship(- back_populates="asset",- cascade="all, delete-orphan",- passive_deletes=True,- )-__table_args__ = (Index("uq_assets_hash", "hash", unique=True),Index("ix_assets_mime_type", "mime_type"),CheckConstraint("size_bytes >= 0", name="ck_assets_size_nonneg"),)- def to_dict(self, include_none: bool = False) -> dict[str, Any]:- return to_dict(self, include_none=include_none)-def __repr__(self) -> str:return f"<Asset id={self.id} hash={(self.hash or '')[:12]}>"-class AssetCacheState(Base):- __tablename__ = "asset_cache_state"+class AssetReference(Base):+ """Unified model combining file cache state and user-facing metadata.- id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)- asset_id: Mapped[str] = mapped_column(String(36), ForeignKey("assets.id", ondelete="CASCADE"), nullable=False)- file_path: Mapped[str] = mapped_column(Text, nullable=False)- mtime_ns: Mapped[int | None] = mapped_column(BigInteger, nullable=True)- needs_verify: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)+ Each row represents either:+ - A filesystem reference (file_path is set) with cache state+ - An API-created reference (file_path is NULL) without cache state+ """- asset: Mapped[Asset] = relationship(back_populates="cache_states")+ __tablename__ = "asset_references"- __table_args__ = (- Index("ix_asset_cache_state_file_path", "file_path"),- Index("ix_asset_cache_state_asset_id", "asset_id"),- CheckConstraint("(mtime_ns IS NULL) OR (mtime_ns >= 0)", name="ck_acs_mtime_nonneg"),- UniqueConstraint("file_path", name="uq_asset_cache_state_file_path"),+ id: Mapped[str] = mapped_column(+ String(36), primary_key=True, default=lambda: str(uuid.uuid4())+ )+ asset_id: Mapped[str] = mapped_column(+ String(36), ForeignKey("assets.id", ondelete="CASCADE"), nullable=False)- def to_dict(self, include_none: bool = False) -> dict[str, Any]:- return to_dict(self, include_none=include_none)-- def __repr__(self) -> str:- return f"<AssetCacheState id={self.id} asset_id={self.asset_id} path={self.file_path!r}>"---class AssetInfo(Base):- __tablename__ = "assets_info"+ # Cache state fields (from former AssetCacheState)+ file_path: Mapped[str | None] = mapped_column(Text, nullable=True)+ mtime_ns: Mapped[int | None] = mapped_column(BigInteger, nullable=True)+ needs_verify: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)+ is_missing: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)+ enrichment_level: Mapped[int] = mapped_column(Integer, nullable=False, default=0)- id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))+ # Info fields (from former AssetInfo)owner_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")name: Mapped[str] = mapped_column(String(512), nullable=False)- asset_id: Mapped[str] = mapped_column(String(36), ForeignKey("assets.id", ondelete="RESTRICT"), nullable=False)- preview_id: Mapped[str | None] = mapped_column(String(36), ForeignKey("assets.id", ondelete="SET NULL"))- user_metadata: Mapped[dict[str, Any] | None] = mapped_column(JSON(none_as_null=True))- created_at: Mapped[datetime] = mapped_column(DateTime(timezone=False), nullable=False, default=utcnow)- updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=False), nullable=False, default=utcnow)- last_access_time: Mapped[datetime] = mapped_column(DateTime(timezone=False), nullable=False, default=utcnow)+ preview_id: Mapped[str | None] = mapped_column(+ String(36), ForeignKey("assets.id", ondelete="SET NULL")+ )+ user_metadata: Mapped[dict[str, Any] | None] = mapped_column(+ JSON(none_as_null=True)+ )+ created_at: Mapped[datetime] = mapped_column(+ DateTime(timezone=False), nullable=False, default=get_utc_now+ )+ updated_at: Mapped[datetime] = mapped_column(+ DateTime(timezone=False), nullable=False, default=get_utc_now+ )+ last_access_time: Mapped[datetime] = mapped_column(+ DateTime(timezone=False), nullable=False, default=get_utc_now+ )+ deleted_at: Mapped[datetime | None] = mapped_column(+ DateTime(timezone=False), nullable=True, default=None+ )asset: Mapped[Asset] = relationship("Asset",- back_populates="infos",+ back_populates="references",foreign_keys=[asset_id],lazy="selectin",)@@ -121,51 +121,59 @@ class AssetInfo(Base):foreign_keys=[preview_id],)- metadata_entries: Mapped[list[AssetInfoMeta]] = relationship(- back_populates="asset_info",+ metadata_entries: Mapped[list[AssetReferenceMeta]] = relationship(+ back_populates="asset_reference",cascade="all,delete-orphan",passive_deletes=True,)- tag_links: Mapped[list[AssetInfoTag]] = relationship(- back_populates="asset_info",+ tag_links: Mapped[list[AssetReferenceTag]] = relationship(+ back_populates="asset_reference",cascade="all,delete-orphan",passive_deletes=True,- overlaps="tags,asset_infos",+ overlaps="tags,asset_references",)tags: Mapped[list[Tag]] = relationship(- secondary="asset_info_tags",- back_populates="asset_infos",+ secondary="asset_reference_tags",+ back_populates="asset_references",lazy="selectin",viewonly=True,- overlaps="tag_links,asset_info_links,asset_infos,tag",+ overlaps="tag_links,asset_reference_links,asset_references,tag",)__table_args__ = (- UniqueConstraint("asset_id", "owner_id", "name", name="uq_assets_info_asset_owner_name"),- Index("ix_assets_info_owner_name", "owner_id", "name"),- Index("ix_assets_info_owner_id", "owner_id"),- Index("ix_assets_info_asset_id", "asset_id"),- Index("ix_assets_info_name", "name"),- Index("ix_assets_info_created_at", "created_at"),- Index("ix_assets_info_last_access_time", "last_access_time"),+ Index("uq_asset_references_file_path", "file_path", unique=True),+ Index("ix_asset_references_asset_id", "asset_id"),+ Index("ix_asset_references_owner_id", "owner_id"),+ Index("ix_asset_references_name", "name"),+ Index("ix_asset_references_is_missing", "is_missing"),+ Index("ix_asset_references_enrichment_level", "enrichment_level"),+ Index("ix_asset_references_created_at", "created_at"),+ Index("ix_asset_references_last_access_time", "last_access_time"),+ Index("ix_asset_references_deleted_at", "deleted_at"),+ Index("ix_asset_references_owner_name", "owner_id", "name"),+ CheckConstraint(+ "(mtime_ns IS NULL) OR (mtime_ns >= 0)", name="ck_ar_mtime_nonneg"+ ),+ CheckConstraint(+ "enrichment_level >= 0 AND enrichment_level <= 2",+ name="ck_ar_enrichment_level_range",+ ),)- def to_dict(self, include_none: bool = False) -> dict[str, Any]:- data = to_dict(self, include_none=include_none)- data["tags"] = [t.name for t in self.tags]- return data-def __repr__(self) -> str:- return f"<AssetInfo id={self.id} name={self.name!r} asset_id={self.asset_id}>"+ path_part = f" path={self.file_path!r}" if self.file_path else ""+ return f"<AssetReference id={self.id} name={self.name!r}{path_part}>"-class AssetInfoMeta(Base):- __tablename__ = "asset_info_meta"+class AssetReferenceMeta(Base):+ __tablename__ = "asset_reference_meta"- asset_info_id: Mapped[str] = mapped_column(- String(36), ForeignKey("assets_info.id", ondelete="CASCADE"), primary_key=True+ asset_reference_id: Mapped[str] = mapped_column(+ String(36),+ ForeignKey("asset_references.id", ondelete="CASCADE"),+ primary_key=True,)key: Mapped[str] = mapped_column(String(256), primary_key=True)ordinal: Mapped[int] = mapped_column(Integer, primary_key=True, default=0)@@ -175,36 +183,40 @@ class AssetInfoMeta(Base):val_bool: Mapped[bool | None] = mapped_column(Boolean, nullable=True)val_json: Mapped[Any | None] = mapped_column(JSON(none_as_null=True), nullable=True)- asset_info: Mapped[AssetInfo] = relationship(back_populates="metadata_entries")+ asset_reference: Mapped[AssetReference] = relationship(+ back_populates="metadata_entries"+ )__table_args__ = (- Index("ix_asset_info_meta_key", "key"),- Index("ix_asset_info_meta_key_val_str", "key", "val_str"),- Index("ix_asset_info_meta_key_val_num", "key", "val_num"),- Index("ix_asset_info_meta_key_val_bool", "key", "val_bool"),+ Index("ix_asset_reference_meta_key", "key"),+ Index("ix_asset_reference_meta_key_val_str", "key", "val_str"),+ Index("ix_asset_reference_meta_key_val_num", "key", "val_num"),+ Index("ix_asset_reference_meta_key_val_bool", "key", "val_bool"),)-class AssetInfoTag(Base):- __tablename__ = "asset_info_tags"+class AssetReferenceTag(Base):+ __tablename__ = "asset_reference_tags"- asset_info_id: Mapped[str] = mapped_column(- String(36), ForeignKey("assets_info.id", ondelete="CASCADE"), primary_key=True+ asset_reference_id: Mapped[str] = mapped_column(+ String(36),+ ForeignKey("asset_references.id", ondelete="CASCADE"),+ primary_key=True,)tag_name: Mapped[str] = mapped_column(String(512), ForeignKey("tags.name", ondelete="RESTRICT"), primary_key=True)origin: Mapped[str] = mapped_column(String(32), nullable=False, default="manual")added_at: Mapped[datetime] = mapped_column(- DateTime(timezone=False), nullable=False, default=utcnow+ DateTime(timezone=False), nullable=False, default=get_utc_now)- asset_info: Mapped[AssetInfo] = relationship(back_populates="tag_links")- tag: Mapped[Tag] = relationship(back_populates="asset_info_links")+ asset_reference: Mapped[AssetReference] = relationship(back_populates="tag_links")+ tag: Mapped[Tag] = relationship(back_populates="asset_reference_links")__table_args__ = (- Index("ix_asset_info_tags_tag_name", "tag_name"),- Index("ix_asset_info_tags_asset_info_id", "asset_info_id"),+ Index("ix_asset_reference_tags_tag_name", "tag_name"),+ Index("ix_asset_reference_tags_asset_reference_id", "asset_reference_id"),)@@ -214,20 +226,18 @@ class Tag(Base):name: Mapped[str] = mapped_column(String(512), primary_key=True)tag_type: Mapped[str] = mapped_column(String(32), nullable=False, default="user")- asset_info_links: Mapped[list[AssetInfoTag]] = relationship(+ asset_reference_links: Mapped[list[AssetReferenceTag]] = relationship(back_populates="tag",- overlaps="asset_infos,tags",+ overlaps="asset_references,tags",)- asset_infos: Mapped[list[AssetInfo]] = relationship(- secondary="asset_info_tags",+ asset_references: Mapped[list[AssetReference]] = relationship(+ secondary="asset_reference_tags",back_populates="tags",viewonly=True,- overlaps="asset_info_links,tag_links,tags,asset_info",+ overlaps="asset_reference_links,tag_links,tags,asset_reference",)- __table_args__ = (- Index("ix_tags_tag_type", "tag_type"),- )+ __table_args__ = (Index("ix_tags_tag_type", "tag_type"),)def __repr__(self) -> str:return f"<Tag {self.name}>"diff --git a/app/assets/database/queries.py b/app/assets/database/queries.pydeleted file mode 100644index d6b33ec7b..000000000--- a/app/assets/database/queries.py+++ /dev/null@@ -1,976 +0,0 @@-import os-import logging-import sqlalchemy as sa-from collections import defaultdict-from datetime import datetime-from typing import Iterable, Any-from sqlalchemy import select, delete, exists, func-from sqlalchemy.dialects import sqlite-from sqlalchemy.exc import IntegrityError-from sqlalchemy.orm import Session, contains_eager, noload-from app.assets.database.models import Asset, AssetInfo, AssetCacheState, AssetInfoMeta, AssetInfoTag, Tag-from app.assets.helpers import (- compute_relative_filename, escape_like_prefix, normalize_tags, project_kv, utcnow-)-from typing import Sequence---def visible_owner_clause(owner_id: str) -> sa.sql.ClauseElement:- """Build owner visibility predicate for reads. Owner-less rows are visible to everyone."""- owner_id = (owner_id or "").strip()- if owner_id == "":- return AssetInfo.owner_id == ""- return AssetInfo.owner_id.in_(["", owner_id])---def pick_best_live_path(states: Sequence[AssetCacheState]) -> str:- """- Return the best on-disk path among cache states:- 1) Prefer a path that exists with needs_verify == False (already verified).- 2) Otherwise, pick the first path that exists.- 3) Otherwise return empty string.- """- alive = [s for s in states if getattr(s, "file_path", None) and os.path.isfile(s.file_path)]- if not alive:- return ""- for s in alive:- if not getattr(s, "needs_verify", False):- return s.file_path- return alive[0].file_path---def apply_tag_filters(- stmt: sa.sql.Select,- include_tags: Sequence[str] | None = None,- exclude_tags: Sequence[str] | None = None,-) -> sa.sql.Select:- """include_tags: every tag must be present; exclude_tags: none may be present."""- include_tags = normalize_tags(include_tags)- exclude_tags = normalize_tags(exclude_tags)-- if include_tags:- for tag_name in include_tags:- stmt = stmt.where(- exists().where(- (AssetInfoTag.asset_info_id == AssetInfo.id)- & (AssetInfoTag.tag_name == tag_name)- )- )-- if exclude_tags:- stmt = stmt.where(- ~exists().where(- (AssetInfoTag.asset_info_id == AssetInfo.id)- & (AssetInfoTag.tag_name.in_(exclude_tags))- )- )- return stmt---def apply_metadata_filter(- stmt: sa.sql.Select,- metadata_filter: dict | None = None,-) -> sa.sql.Select:- """Apply filters using asset_info_meta projection table."""- if not metadata_filter:- return stmt-- def _exists_for_pred(key: str, *preds) -> sa.sql.ClauseElement:- return sa.exists().where(- AssetInfoMeta.asset_info_id == AssetInfo.id,- AssetInfoMeta.key == key,- *preds,- )-- def _exists_clause_for_value(key: str, value) -> sa.sql.ClauseElement:- if value is None:- no_row_for_key = sa.not_(- sa.exists().where(- AssetInfoMeta.asset_info_id == AssetInfo.id,- AssetInfoMeta.key == key,- )- )- null_row = _exists_for_pred(- key,- AssetInfoMeta.val_json.is_(None),- AssetInfoMeta.val_str.is_(None),- AssetInfoMeta.val_num.is_(None),- AssetInfoMeta.val_bool.is_(None),- )- return sa.or_(no_row_for_key, null_row)-- if isinstance(value, bool):- return _exists_for_pred(key, AssetInfoMeta.val_bool == bool(value))- if isinstance(value, (int, float)):- from decimal import Decimal- num = value if isinstance(value, Decimal) else Decimal(str(value))- return _exists_for_pred(key, AssetInfoMeta.val_num == num)- if isinstance(value, str):- return _exists_for_pred(key, AssetInfoMeta.val_str == value)- return _exists_for_pred(key, AssetInfoMeta.val_json == value)-- for k, v in metadata_filter.items():- if isinstance(v, list):- ors = [_exists_clause_for_value(k, elem) for elem in v]- if ors:- stmt = stmt.where(sa.or_(*ors))- else:- stmt = stmt.where(_exists_clause_for_value(k, v))- return stmt---def asset_exists_by_hash(- session: Session,- *,- asset_hash: str,-) -> bool:- """- Check if an asset with a given hash exists in database.- """- row = (- session.execute(- select(sa.literal(True)).select_from(Asset).where(Asset.hash == asset_hash).limit(1)- )- ).first()- return row is not None---def asset_info_exists_for_asset_id(- session: Session,- *,- asset_id: str,-) -> bool:- q = (- select(sa.literal(True))- .select_from(AssetInfo)- .where(AssetInfo.asset_id == asset_id)- .limit(1)- )- return (session.execute(q)).first() is not None---def get_asset_by_hash(- session: Session,- *,- asset_hash: str,-) -> Asset | None:- return (- session.execute(select(Asset).where(Asset.hash == asset_hash).limit(1))- ).scalars().first()---def get_asset_info_by_id(- session: Session,- *,- asset_info_id: str,-) -> AssetInfo | None:- return session.get(AssetInfo, asset_info_id)---def list_asset_infos_page(- session: Session,- owner_id: str = "",- include_tags: Sequence[str] | None = None,- exclude_tags: Sequence[str] | None = None,- name_contains: str | None = None,- metadata_filter: dict | None = None,- limit: int = 20,- offset: int = 0,- sort: str = "created_at",- order: str = "desc",-) -> tuple[list[AssetInfo], dict[str, list[str]], int]:- base = (- select(AssetInfo)- .join(Asset, Asset.id == AssetInfo.asset_id)- .options(contains_eager(AssetInfo.asset), noload(AssetInfo.tags))- .where(visible_owner_clause(owner_id))- )-- if name_contains:- escaped, esc = escape_like_prefix(name_contains)- base = base.where(AssetInfo.name.ilike(f"%{escaped}%", escape=esc))-- base = apply_tag_filters(base, include_tags, exclude_tags)- base = apply_metadata_filter(base, metadata_filter)-- sort = (sort or "created_at").lower()- order = (order or "desc").lower()- sort_map = {- "name": AssetInfo.name,- "created_at": AssetInfo.created_at,- "updated_at": AssetInfo.updated_at,- "last_access_time": AssetInfo.last_access_time,- "size": Asset.size_bytes,- }- sort_col = sort_map.get(sort, AssetInfo.created_at)- sort_exp = sort_col.desc() if order == "desc" else sort_col.asc()-- base = base.order_by(sort_exp).limit(limit).offset(offset)-- count_stmt = (- select(sa.func.count())- .select_from(AssetInfo)- .join(Asset, Asset.id == AssetInfo.asset_id)- .where(visible_owner_clause(owner_id))- )- if name_contains:- escaped, esc = escape_like_prefix(name_contains)- count_stmt = count_stmt.where(AssetInfo.name.ilike(f"%{escaped}%", escape=esc))- count_stmt = apply_tag_filters(count_stmt, include_tags, exclude_tags)- count_stmt = apply_metadata_filter(count_stmt, metadata_filter)-- total = int((session.execute(count_stmt)).scalar_one() or 0)-- infos = (session.execute(base)).unique().scalars().all()-- id_list: list[str] = [i.id for i in infos]- tag_map: dict[str, list[str]] = defaultdict(list)- if id_list:- rows = session.execute(- select(AssetInfoTag.asset_info_id, Tag.name)- .join(Tag, Tag.name == AssetInfoTag.tag_name)- .where(AssetInfoTag.asset_info_id.in_(id_list))- .order_by(AssetInfoTag.added_at)- )- for aid, tag_name in rows.all():- tag_map[aid].append(tag_name)-- return infos, tag_map, total---def fetch_asset_info_asset_and_tags(- session: Session,- asset_info_id: str,- owner_id: str = "",-) -> tuple[AssetInfo, Asset, list[str]] | None:- stmt = (- select(AssetInfo, Asset, Tag.name)- .join(Asset, Asset.id == AssetInfo.asset_id)- .join(AssetInfoTag, AssetInfoTag.asset_info_id == AssetInfo.id, isouter=True)- .join(Tag, Tag.name == AssetInfoTag.tag_name, isouter=True)- .where(- AssetInfo.id == asset_info_id,- visible_owner_clause(owner_id),- )- .options(noload(AssetInfo.tags))- .order_by(Tag.name.asc())- )-- rows = (session.execute(stmt)).all()- if not rows:- return None-- first_info, first_asset, _ = rows[0]- tags: list[str] = []- seen: set[str] = set()- for _info, _asset, tag_name in rows:- if tag_name and tag_name not in seen:- seen.add(tag_name)- tags.append(tag_name)- return first_info, first_asset, tags---def fetch_asset_info_and_asset(- session: Session,- *,- asset_info_id: str,- owner_id: str = "",-) -> tuple[AssetInfo, Asset] | None:- stmt = (- select(AssetInfo, Asset)- .join(Asset, Asset.id == AssetInfo.asset_id)- .where(- AssetInfo.id == asset_info_id,- visible_owner_clause(owner_id),- )- .limit(1)- .options(noload(AssetInfo.tags))- )- row = session.execute(stmt)- pair = row.first()- if not pair:- return None- return pair[0], pair[1]--def list_cache_states_by_asset_id(- session: Session, *, asset_id: str-) -> Sequence[AssetCacheState]:- return (- session.execute(- select(AssetCacheState)- .where(AssetCacheState.asset_id == asset_id)- .order_by(AssetCacheState.id.asc())- )- ).scalars().all()---def touch_asset_info_by_id(- session: Session,- *,- asset_info_id: str,- ts: datetime | None = None,- only_if_newer: bool = True,-) -> None:- ts = ts or utcnow()- stmt = sa.update(AssetInfo).where(AssetInfo.id == asset_info_id)- if only_if_newer:- stmt = stmt.where(- sa.or_(AssetInfo.last_access_time.is_(None), AssetInfo.last_access_time < ts)- )- session.execute(stmt.values(last_access_time=ts))---def create_asset_info_for_existing_asset(- session: Session,- *,- asset_hash: str,- name: str,- user_metadata: dict | None = None,- tags: Sequence[str] | None = None,- tag_origin: str = "manual",- owner_id: str = "",-) -> AssetInfo:- """Create or return an existing AssetInfo for an Asset identified by asset_hash."""- now = utcnow()- asset = get_asset_by_hash(session, asset_hash=asset_hash)- if not asset:- raise ValueError(f"Unknown asset hash {asset_hash}")-- info = AssetInfo(- owner_id=owner_id,- name=name,- asset_id=asset.id,- preview_id=None,- created_at=now,- updated_at=now,- last_access_time=now,- )- try:- with session.begin_nested():- session.add(info)- session.flush()- except IntegrityError:- existing = (- session.execute(- select(AssetInfo)- .options(noload(AssetInfo.tags))- .where(- AssetInfo.asset_id == asset.id,- AssetInfo.name == name,- AssetInfo.owner_id == owner_id,- )- .limit(1)- )- ).unique().scalars().first()- if not existing:- raise RuntimeError("AssetInfo upsert failed to find existing row after conflict.")- return existing-- # metadata["filename"] hack- new_meta = dict(user_metadata or {})- computed_filename = None- try:- p = pick_best_live_path(list_cache_states_by_asset_id(session, asset_id=asset.id))- if p:- computed_filename = compute_relative_filename(p)- except Exception:- computed_filename = None- if computed_filename:- new_meta["filename"] = computed_filename- if new_meta:- replace_asset_info_metadata_projection(- session,- asset_info_id=info.id,- user_metadata=new_meta,- )-- if tags is not None:- set_asset_info_tags(- session,- asset_info_id=info.id,- tags=tags,- origin=tag_origin,- )- return info---def set_asset_info_tags(- session: Session,- *,- asset_info_id: str,- tags: Sequence[str],- origin: str = "manual",-) -> dict:- desired = normalize_tags(tags)-- current = set(- tag_name for (tag_name,) in (- session.execute(select(AssetInfoTag.tag_name).where(AssetInfoTag.asset_info_id == asset_info_id))- ).all()- )-- to_add = [t for t in desired if t not in current]- to_remove = [t for t in current if t not in desired]-- if to_add:- ensure_tags_exist(session, to_add, tag_type="user")- session.add_all([- AssetInfoTag(asset_info_id=asset_info_id, tag_name=t, origin=origin, added_at=utcnow())- for t in to_add- ])- session.flush()-- if to_remove:- session.execute(- delete(AssetInfoTag)- .where(AssetInfoTag.asset_info_id == asset_info_id, AssetInfoTag.tag_name.in_(to_remove))- )- session.flush()-- return {"added": to_add, "removed": to_remove, "total": desired}---def replace_asset_info_metadata_projection(- session: Session,- *,- asset_info_id: str,- user_metadata: dict | None = None,-) -> None:- info = session.get(AssetInfo, asset_info_id)- if not info:- raise ValueError(f"AssetInfo {asset_info_id} not found")-- info.user_metadata = user_metadata or {}- info.updated_at = utcnow()- session.flush()-- session.execute(delete(AssetInfoMeta).where(AssetInfoMeta.asset_info_id == asset_info_id))- session.flush()-- if not user_metadata:- return-- rows: list[AssetInfoMeta] = []- for k, v in user_metadata.items():- for r in project_kv(k, v):- rows.append(- AssetInfoMeta(- asset_info_id=asset_info_id,- key=r["key"],- ordinal=int(r["ordinal"]),- val_str=r.get("val_str"),- val_num=r.get("val_num"),- val_bool=r.get("val_bool"),- val_json=r.get("val_json"),- )- )- if rows:- session.add_all(rows)- session.flush()---def ingest_fs_asset(- session: Session,- *,- asset_hash: str,- abs_path: str,- size_bytes: int,- mtime_ns: int,- mime_type: str | None = None,- info_name: str | None = None,- owner_id: str = "",- preview_id: str | None = None,- user_metadata: dict | None = None,- tags: Sequence[str] = (),- tag_origin: str = "manual",- require_existing_tags: bool = False,-) -> dict:- """- Idempotently upsert:- - Asset by content hash (create if missing)- - AssetCacheState(file_path) pointing to asset_id- - Optionally AssetInfo + tag links and metadata projection- Returns flags and ids.- """- locator = os.path.abspath(abs_path)- now = utcnow()-- if preview_id:- if not session.get(Asset, preview_id):- preview_id = None-- out: dict[str, Any] = {- "asset_created": False,- "asset_updated": False,- "state_created": False,- "state_updated": False,- "asset_info_id": None,- }-- # 1) Asset by hash- asset = (- session.execute(select(Asset).where(Asset.hash == asset_hash).limit(1))- ).scalars().first()- if not asset:- vals = {- "hash": asset_hash,- "size_bytes": int(size_bytes),- "mime_type": mime_type,- "created_at": now,- }- res = session.execute(- sqlite.insert(Asset)- .values(**vals)- .on_conflict_do_nothing(index_elements=[Asset.hash])- )- if int(res.rowcount or 0) > 0:- out["asset_created"] = True- asset = (- session.execute(- select(Asset).where(Asset.hash == asset_hash).limit(1)- )- ).scalars().first()- if not asset:- raise RuntimeError("Asset row not found after upsert.")- else:- changed = False- if asset.size_bytes != int(size_bytes) and int(size_bytes) > 0:- asset.size_bytes = int(size_bytes)- changed = True- if mime_type and asset.mime_type != mime_type:- asset.mime_type = mime_type- changed = True- if changed:- out["asset_updated"] = True-- # 2) AssetCacheState upsert by file_path (unique)- vals = {- "asset_id": asset.id,- "file_path": locator,- "mtime_ns": int(mtime_ns),- }- ins = (- sqlite.insert(AssetCacheState)- .values(**vals)- .on_conflict_do_nothing(index_elements=[AssetCacheState.file_path])- )-- res = session.execute(ins)- if int(res.rowcount or 0) > 0:- out["state_created"] = True- else:- upd = (- sa.update(AssetCacheState)- .where(AssetCacheState.file_path == locator)- .where(- sa.or_(- AssetCacheState.asset_id != asset.id,- AssetCacheState.mtime_ns.is_(None),- AssetCacheState.mtime_ns != int(mtime_ns),- )- )- .values(asset_id=asset.id, mtime_ns=int(mtime_ns))- )- res2 = session.execute(upd)- if int(res2.rowcount or 0) > 0:- out["state_updated"] = True-- # 3) Optional AssetInfo + tags + metadata- if info_name:- try:- with session.begin_nested():- info = AssetInfo(- owner_id=owner_id,- name=info_name,- asset_id=asset.id,- preview_id=preview_id,- created_at=now,- updated_at=now,- last_access_time=now,- )- session.add(info)- session.flush()- out["asset_info_id"] = info.id- except IntegrityError:- pass-- existing_info = (- session.execute(- select(AssetInfo)- .where(- AssetInfo.asset_id == asset.id,- AssetInfo.name == info_name,- (AssetInfo.owner_id == owner_id),- )- .limit(1)- )- ).unique().scalar_one_or_none()- if not existing_info:- raise RuntimeError("Failed to update or insert AssetInfo.")-- if preview_id and existing_info.preview_id != preview_id:- existing_info.preview_id = preview_id-- existing_info.updated_at = now- if existing_info.last_access_time < now:- existing_info.last_access_time = now- session.flush()- out["asset_info_id"] = existing_info.id-- norm = [t.strip().lower() for t in (tags or []) if (t or "").strip()]- if norm and out["asset_info_id"] is not None:- if not require_existing_tags:- ensure_tags_exist(session, norm, tag_type="user")-- existing_tag_names = set(- name for (name,) in (session.execute(select(Tag.name).where(Tag.name.in_(norm)))).all()- )- missing = [t for t in norm if t not in existing_tag_names]- if missing and require_existing_tags:- raise ValueError(f"Unknown tags: {missing}")-- existing_links = set(- tag_name- for (tag_name,) in (- session.execute(- select(AssetInfoTag.tag_name).where(AssetInfoTag.asset_info_id == out["asset_info_id"])- )- ).all()- )- to_add = [t for t in norm if t in existing_tag_names and t not in existing_links]- if to_add:- session.add_all(- [- AssetInfoTag(- asset_info_id=out["asset_info_id"],- tag_name=t,- origin=tag_origin,- added_at=now,- )- for t in to_add- ]- )- session.flush()-- # metadata["filename"] hack- if out["asset_info_id"] is not None:- primary_path = pick_best_live_path(list_cache_states_by_asset_id(session, asset_id=asset.id))- computed_filename = compute_relative_filename(primary_path) if primary_path else None-- current_meta = existing_info.user_metadata or {}- new_meta = dict(current_meta)- if user_metadata is not None:- for k, v in user_metadata.items():- new_meta[k] = v- if computed_filename:- new_meta["filename"] = computed_filename-- if new_meta != current_meta:- replace_asset_info_metadata_projection(- session,- asset_info_id=out["asset_info_id"],- user_metadata=new_meta,- )-- try:- remove_missing_tag_for_asset_id(session, asset_id=asset.id)- except Exception:- logging.exception("Failed to clear 'missing' tag for asset %s", asset.id)- return out---def update_asset_info_full(- session: Session,- *,- asset_info_id: str,- name: str | None = None,- tags: Sequence[str] | None = None,- user_metadata: dict | None = None,- tag_origin: str = "manual",- asset_info_row: Any = None,-) -> AssetInfo:- if not asset_info_row:- info = session.get(AssetInfo, asset_info_id)- if not info:- raise ValueError(f"AssetInfo {asset_info_id} not found")- else:- info = asset_info_row-- touched = False- if name is not None and name != info.name:- info.name = name- touched = True-- computed_filename = None- try:- p = pick_best_live_path(list_cache_states_by_asset_id(session, asset_id=info.asset_id))- if p:- computed_filename = compute_relative_filename(p)- except Exception:- computed_filename = None-- if user_metadata is not None:- new_meta = dict(user_metadata)- if computed_filename:- new_meta["filename"] = computed_filename- replace_asset_info_metadata_projection(- session, asset_info_id=asset_info_id, user_metadata=new_meta- )- touched = True- else:- if computed_filename:- current_meta = info.user_metadata or {}- if current_meta.get("filename") != computed_filename:- new_meta = dict(current_meta)- new_meta["filename"] = computed_filename- replace_asset_info_metadata_projection(- session, asset_info_id=asset_info_id, user_metadata=new_meta- )- touched = True-- if tags is not None:- set_asset_info_tags(- session,- asset_info_id=asset_info_id,- tags=tags,- origin=tag_origin,- )- touched = True-- if touched and user_metadata is None:- info.updated_at = utcnow()- session.flush()-- return info---def delete_asset_info_by_id(- session: Session,- *,- asset_info_id: str,- owner_id: str,-) -> bool:- stmt = sa.delete(AssetInfo).where(- AssetInfo.id == asset_info_id,- visible_owner_clause(owner_id),- )- return int((session.execute(stmt)).rowcount or 0) > 0---def list_tags_with_usage(- session: Session,- prefix: str | None = None,- limit: int = 100,- offset: int = 0,- include_zero: bool = True,- order: str = "count_desc",- owner_id: str = "",-) -> tuple[list[tuple[str, str, int]], int]:- counts_sq = (- select(- AssetInfoTag.tag_name.label("tag_name"),- func.count(AssetInfoTag.asset_info_id).label("cnt"),- )- .select_from(AssetInfoTag)- .join(AssetInfo, AssetInfo.id == AssetInfoTag.asset_info_id)- .where(visible_owner_clause(owner_id))- .group_by(AssetInfoTag.tag_name)- .subquery()- )-- q = (- select(- Tag.name,- Tag.tag_type,- func.coalesce(counts_sq.c.cnt, 0).label("count"),- )- .select_from(Tag)- .join(counts_sq, counts_sq.c.tag_name == Tag.name, isouter=True)- )-- if prefix:- escaped, esc = escape_like_prefix(prefix.strip().lower())- q = q.where(Tag.name.like(escaped + "%", escape=esc))-- if not include_zero:- q = q.where(func.coalesce(counts_sq.c.cnt, 0) > 0)-- if order == "name_asc":- q = q.order_by(Tag.name.asc())- else:- q = q.order_by(func.coalesce(counts_sq.c.cnt, 0).desc(), Tag.name.asc())-- total_q = select(func.count()).select_from(Tag)- if prefix:- escaped, esc = escape_like_prefix(prefix.strip().lower())- total_q = total_q.where(Tag.name.like(escaped + "%", escape=esc))- if not include_zero:- total_q = total_q.where(- Tag.name.in_(select(AssetInfoTag.tag_name).group_by(AssetInfoTag.tag_name))- )-- rows = (session.execute(q.limit(limit).offset(offset))).all()- total = (session.execute(total_q)).scalar_one()-- rows_norm = [(name, ttype, int(count or 0)) for (name, ttype, count) in rows]- return rows_norm, int(total or 0)---def ensure_tags_exist(session: Session, names: Iterable[str], tag_type: str = "user") -> None:- wanted = normalize_tags(list(names))- if not wanted:- return- rows = [{"name": n, "tag_type": tag_type} for n in list(dict.fromkeys(wanted))]- ins = (- sqlite.insert(Tag)- .values(rows)- .on_conflict_do_nothing(index_elements=[Tag.name])- )- session.execute(ins)---def get_asset_tags(session: Session, *, asset_info_id: str) -> list[str]:- return [- tag_name for (tag_name,) in (- session.execute(- select(AssetInfoTag.tag_name).where(AssetInfoTag.asset_info_id == asset_info_id)- )- ).all()- ]---def add_tags_to_asset_info(- session: Session,- *,- asset_info_id: str,- tags: Sequence[str],- origin: str = "manual",- create_if_missing: bool = True,- asset_info_row: Any = None,-) -> dict:- if not asset_info_row:- info = session.get(AssetInfo, asset_info_id)- if not info:- raise ValueError(f"AssetInfo {asset_info_id} not found")-- norm = normalize_tags(tags)- if not norm:- total = get_asset_tags(session, asset_info_id=asset_info_id)- return {"added": [], "already_present": [], "total_tags": total}-- if create_if_missing:- ensure_tags_exist(session, norm, tag_type="user")-- current = {- tag_name- for (tag_name,) in (- session.execute(- sa.select(AssetInfoTag.tag_name).where(AssetInfoTag.asset_info_id == asset_info_id)- )- ).all()- }-- want = set(norm)- to_add = sorted(want - current)-- if to_add:- with session.begin_nested() as nested:- try:- session.add_all(- [- AssetInfoTag(- asset_info_id=asset_info_id,- tag_name=t,- origin=origin,- added_at=utcnow(),- )- for t in to_add- ]- )- session.flush()- except IntegrityError:- nested.rollback()-- after = set(get_asset_tags(session, asset_info_id=asset_info_id))- return {- "added": sorted(((after - current) & want)),- "already_present": sorted(want & current),- "total_tags": sorted(after),- }---def remove_tags_from_asset_info(- session: Session,- *,- asset_info_id: str,- tags: Sequence[str],-) -> dict:- info = session.get(AssetInfo, asset_info_id)- if not info:- raise ValueError(f"AssetInfo {asset_info_id} not found")-- norm = normalize_tags(tags)- if not norm:- total = get_asset_tags(session, asset_info_id=asset_info_id)- return {"removed": [], "not_present": [], "total_tags": total}-- existing = {- tag_name- for (tag_name,) in (- session.execute(- sa.select(AssetInfoTag.tag_name).where(AssetInfoTag.asset_info_id == asset_info_id)- )- ).all()- }-- to_remove = sorted(set(t for t in norm if t in existing))- not_present = sorted(set(t for t in norm if t not in existing))-- if to_remove:- session.execute(- delete(AssetInfoTag)- .where(- AssetInfoTag.asset_info_id == asset_info_id,- AssetInfoTag.tag_name.in_(to_remove),- )- )- session.flush()-- total = get_asset_tags(session, asset_info_id=asset_info_id)- return {"removed": to_remove, "not_present": not_present, "total_tags": total}---def remove_missing_tag_for_asset_id(- session: Session,- *,- asset_id: str,-) -> None:- session.execute(- sa.delete(AssetInfoTag).where(- AssetInfoTag.asset_info_id.in_(sa.select(AssetInfo.id).where(AssetInfo.asset_id == asset_id)),- AssetInfoTag.tag_name == "missing",- )- )---def set_asset_info_preview(- session: Session,- *,- asset_info_id: str,- preview_asset_id: str | None = None,-) -> None:- """Set or clear preview_id and bump updated_at. Raises on unknown IDs."""- info = session.get(AssetInfo, asset_info_id)- if not info:- raise ValueError(f"AssetInfo {asset_info_id} not found")-- if preview_asset_id is None:- info.preview_id = None- else:- # validate preview asset exists- if not session.get(Asset, preview_asset_id):- raise ValueError(f"Preview Asset {preview_asset_id} not found")- info.preview_id = preview_asset_id-- info.updated_at = utcnow()- session.flush()diff --git a/app/assets/database/queries/__init__.py b/app/assets/database/queries/__init__.pynew file mode 100644index 000000000..7888d0645--- /dev/null+++ b/app/assets/database/queries/__init__.py@@ -0,0 +1,121 @@+from app.assets.database.queries.asset import (+ asset_exists_by_hash,+ bulk_insert_assets,+ get_asset_by_hash,+ get_existing_asset_ids,+ reassign_asset_references,+ update_asset_hash_and_mime,+ upsert_asset,+)+from app.assets.database.queries.asset_reference import (+ CacheStateRow,+ UnenrichedReferenceRow,+ bulk_insert_references_ignore_conflicts,+ bulk_update_enrichment_level,+ bulk_update_is_missing,+ bulk_update_needs_verify,+ convert_metadata_to_rows,+ delete_assets_by_ids,+ delete_orphaned_seed_asset,+ delete_reference_by_id,+ delete_references_by_ids,+ fetch_reference_and_asset,+ fetch_reference_asset_and_tags,+ get_or_create_reference,+ get_reference_by_file_path,+ get_reference_by_id,+ get_reference_with_owner_check,+ get_reference_ids_by_ids,+ get_references_by_paths_and_asset_ids,+ get_references_for_prefixes,+ get_unenriched_references,+ get_unreferenced_unhashed_asset_ids,+ insert_reference,+ list_references_by_asset_id,+ list_references_page,+ mark_references_missing_outside_prefixes,+ reference_exists_for_asset_id,+ restore_references_by_paths,+ set_reference_metadata,+ set_reference_preview,+ soft_delete_reference_by_id,+ update_reference_access_time,+ update_reference_name,+ update_reference_timestamps,+ update_reference_updated_at,+ upsert_reference,+)+from app.assets.database.queries.tags import (+ AddTagsResult,+ RemoveTagsResult,+ SetTagsResult,+ add_missing_tag_for_asset_id,+ add_tags_to_reference,+ bulk_insert_tags_and_meta,+ ensure_tags_exist,+ get_reference_tags,+ list_tags_with_usage,+ remove_missing_tag_for_asset_id,+ remove_tags_from_reference,+ set_reference_tags,+ validate_tags_exist,+)++__all__ = [+ "AddTagsResult",+ "CacheStateRow",+ "RemoveTagsResult",+ "SetTagsResult",+ "UnenrichedReferenceRow",+ "add_missing_tag_for_asset_id",+ "add_tags_to_reference",+ "asset_exists_by_hash",+ "bulk_insert_assets",+ "bulk_insert_references_ignore_conflicts",+ "bulk_insert_tags_and_meta",+ "bulk_update_enrichment_level",+ "bulk_update_is_missing",+ "bulk_update_needs_verify",+ "convert_metadata_to_rows",+ "delete_assets_by_ids",+ "delete_orphaned_seed_asset",+ "delete_reference_by_id",+ "delete_references_by_ids",+ "ensure_tags_exist",+ "fetch_reference_and_asset",+ "fetch_reference_asset_and_tags",+ "get_asset_by_hash",+ "get_existing_asset_ids",+ "get_or_create_reference",+ "get_reference_by_file_path",+ "get_reference_by_id",+ "get_reference_with_owner_check",+ "get_reference_ids_by_ids",+ "get_reference_tags",+ "get_references_by_paths_and_asset_ids",+ "get_references_for_prefixes",+ "get_unenriched_references",+ "get_unreferenced_unhashed_asset_ids",+ "insert_reference",+ "list_references_by_asset_id",+ "list_references_page",+ "list_tags_with_usage",+ "mark_references_missing_outside_prefixes",+ "reassign_asset_references",+ "reference_exists_for_asset_id",+ "remove_missing_tag_for_asset_id",+ "remove_tags_from_reference",+ "restore_references_by_paths",+ "set_reference_metadata",+ "set_reference_preview",+ "soft_delete_reference_by_id",+ "set_reference_tags",+ "update_asset_hash_and_mime",+ "update_reference_access_time",+ "update_reference_name",+ "update_reference_timestamps",+ "update_reference_updated_at",+ "upsert_asset",+ "upsert_reference",+ "validate_tags_exist",+]diff --git a/app/assets/database/queries/asset.py b/app/assets/database/queries/asset.pynew file mode 100644index 000000000..a21f5b68f--- /dev/null+++ b/app/assets/database/queries/asset.py@@ -0,0 +1,140 @@+import sqlalchemy as sa+from sqlalchemy import select+from sqlalchemy.dialects import sqlite+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference+from app.assets.database.queries.common import MAX_BIND_PARAMS, calculate_rows_per_statement, iter_chunks+++def asset_exists_by_hash(+ session: Session,+ asset_hash: str,+) -> bool:+ """+ Check if an asset with a given hash exists in database.+ """+ row = (+ session.execute(+ select(sa.literal(True))+ .select_from(Asset)+ .where(Asset.hash == asset_hash)+ .limit(1)+ )+ ).first()+ return row is not None+++def get_asset_by_hash(+ session: Session,+ asset_hash: str,+) -> Asset | None:+ return (+ (session.execute(select(Asset).where(Asset.hash == asset_hash).limit(1)))+ .scalars()+ .first()+ )+++def upsert_asset(+ session: Session,+ asset_hash: str,+ size_bytes: int,+ mime_type: str | None = None,+) -> tuple[Asset, bool, bool]:+ """Upsert an Asset by hash. Returns (asset, created, updated)."""+ vals = {"hash": asset_hash, "size_bytes": int(size_bytes)}+ if mime_type:+ vals["mime_type"] = mime_type++ ins = (+ sqlite.insert(Asset)+ .values(**vals)+ .on_conflict_do_nothing(index_elements=[Asset.hash])+ )+ res = session.execute(ins)+ created = int(res.rowcount or 0) > 0++ asset = (+ session.execute(select(Asset).where(Asset.hash == asset_hash).limit(1))+ .scalars()+ .first()+ )+ if not asset:+ raise RuntimeError("Asset row not found after upsert.")++ updated = False+ if not created:+ changed = False+ if asset.size_bytes != int(size_bytes) and int(size_bytes) > 0:+ asset.size_bytes = int(size_bytes)+ changed = True+ if mime_type and asset.mime_type != mime_type:+ asset.mime_type = mime_type+ changed = True+ if changed:+ updated = True++ return asset, created, updated+++def bulk_insert_assets(+ session: Session,+ rows: list[dict],+) -> None:+ """Bulk insert Asset rows with ON CONFLICT DO NOTHING on hash."""+ if not rows:+ return+ ins = sqlite.insert(Asset).on_conflict_do_nothing(index_elements=[Asset.hash])+ for chunk in iter_chunks(rows, calculate_rows_per_statement(5)):+ session.execute(ins, chunk)+++def get_existing_asset_ids(+ session: Session,+ asset_ids: list[str],+) -> set[str]:+ """Return the subset of asset_ids that exist in the database."""+ if not asset_ids:+ return set()+ found: set[str] = set()+ for chunk in iter_chunks(asset_ids, MAX_BIND_PARAMS):+ rows = session.execute(+ select(Asset.id).where(Asset.id.in_(chunk))+ ).fetchall()+ found.update(row[0] for row in rows)+ return found+++def update_asset_hash_and_mime(+ session: Session,+ asset_id: str,+ asset_hash: str | None = None,+ mime_type: str | None = None,+) -> bool:+ """Update asset hash and/or mime_type. Returns True if asset was found."""+ asset = session.get(Asset, asset_id)+ if not asset:+ return False+ if asset_hash is not None:+ asset.hash = asset_hash+ if mime_type is not None:+ asset.mime_type = mime_type+ return True+++def reassign_asset_references(+ session: Session,+ from_asset_id: str,+ to_asset_id: str,+ reference_id: str,+) -> None:+ """Reassign a reference from one asset to another.++ Used when merging a stub asset into an existing asset with the same hash.+ """+ ref = session.get(AssetReference, reference_id)+ if ref and ref.asset_id == from_asset_id:+ ref.asset_id = to_asset_id++ session.flush()diff --git a/app/assets/database/queries/asset_reference.py b/app/assets/database/queries/asset_reference.pynew file mode 100644index 000000000..6524791cc--- /dev/null+++ b/app/assets/database/queries/asset_reference.py@@ -0,0 +1,1033 @@+"""Query functions for the unified AssetReference table.++This module replaces the separate asset_info.py and cache_state.py query modules,+providing a unified interface for the merged asset_references table.+"""++from collections import defaultdict+from datetime import datetime+from decimal import Decimal+from typing import NamedTuple, Sequence++import sqlalchemy as sa+from sqlalchemy import delete, exists, select+from sqlalchemy.dialects import sqlite+from sqlalchemy.exc import IntegrityError+from sqlalchemy.orm import Session, noload++from app.assets.database.models import (+ Asset,+ AssetReference,+ AssetReferenceMeta,+ AssetReferenceTag,+ Tag,+)+from app.assets.database.queries.common import (+ MAX_BIND_PARAMS,+ build_prefix_like_conditions,+ build_visible_owner_clause,+ calculate_rows_per_statement,+ iter_chunks,+)+from app.assets.helpers import escape_sql_like_string, get_utc_now, normalize_tags+++def _check_is_scalar(v):+ if v is None:+ return True+ if isinstance(v, bool):+ return True+ if isinstance(v, (int, float, Decimal, str)):+ return True+ return False+++def _scalar_to_row(key: str, ordinal: int, value) -> dict:+ """Convert a scalar value to a typed projection row."""+ if value is None:+ return {+ "key": key,+ "ordinal": ordinal,+ "val_str": None,+ "val_num": None,+ "val_bool": None,+ "val_json": None,+ }+ if isinstance(value, bool):+ return {"key": key, "ordinal": ordinal, "val_bool": bool(value)}+ if isinstance(value, (int, float, Decimal)):+ num = value if isinstance(value, Decimal) else Decimal(str(value))+ return {"key": key, "ordinal": ordinal, "val_num": num}+ if isinstance(value, str):+ return {"key": key, "ordinal": ordinal, "val_str": value}+ return {"key": key, "ordinal": ordinal, "val_json": value}+++def convert_metadata_to_rows(key: str, value) -> list[dict]:+ """Turn a metadata key/value into typed projection rows."""+ if value is None:+ return [_scalar_to_row(key, 0, None)]++ if _check_is_scalar(value):+ return [_scalar_to_row(key, 0, value)]++ if isinstance(value, list):+ if all(_check_is_scalar(x) for x in value):+ return [_scalar_to_row(key, i, x) for i, x in enumerate(value)]+ return [{"key": key, "ordinal": i, "val_json": x} for i, x in enumerate(value)]++ return [{"key": key, "ordinal": 0, "val_json": value}]+++def _apply_tag_filters(+ stmt: sa.sql.Select,+ include_tags: Sequence[str] | None = None,+ exclude_tags: Sequence[str] | None = None,+) -> sa.sql.Select:+ """include_tags: every tag must be present; exclude_tags: none may be present."""+ include_tags = normalize_tags(include_tags)+ exclude_tags = normalize_tags(exclude_tags)++ if include_tags:+ for tag_name in include_tags:+ stmt = stmt.where(+ exists().where(+ (AssetReferenceTag.asset_reference_id == AssetReference.id)+ & (AssetReferenceTag.tag_name == tag_name)+ )+ )++ if exclude_tags:+ stmt = stmt.where(+ ~exists().where(+ (AssetReferenceTag.asset_reference_id == AssetReference.id)+ & (AssetReferenceTag.tag_name.in_(exclude_tags))+ )+ )+ return stmt+++def _apply_metadata_filter(+ stmt: sa.sql.Select,+ metadata_filter: dict | None = None,+) -> sa.sql.Select:+ """Apply filters using asset_reference_meta projection table."""+ if not metadata_filter:+ return stmt++ def _exists_for_pred(key: str, *preds) -> sa.sql.ClauseElement:+ return sa.exists().where(+ AssetReferenceMeta.asset_reference_id == AssetReference.id,+ AssetReferenceMeta.key == key,+ *preds,+ )++ def _exists_clause_for_value(key: str, value) -> sa.sql.ClauseElement:+ if value is None:+ no_row_for_key = sa.not_(+ sa.exists().where(+ AssetReferenceMeta.asset_reference_id == AssetReference.id,+ AssetReferenceMeta.key == key,+ )+ )+ null_row = _exists_for_pred(+ key,+ AssetReferenceMeta.val_json.is_(None),+ AssetReferenceMeta.val_str.is_(None),+ AssetReferenceMeta.val_num.is_(None),+ AssetReferenceMeta.val_bool.is_(None),+ )+ return sa.or_(no_row_for_key, null_row)++ if isinstance(value, bool):+ return _exists_for_pred(key, AssetReferenceMeta.val_bool == bool(value))+ if isinstance(value, (int, float, Decimal)):+ num = value if isinstance(value, Decimal) else Decimal(str(value))+ return _exists_for_pred(key, AssetReferenceMeta.val_num == num)+ if isinstance(value, str):+ return _exists_for_pred(key, AssetReferenceMeta.val_str == value)+ return _exists_for_pred(key, AssetReferenceMeta.val_json == value)++ for k, v in metadata_filter.items():+ if isinstance(v, list):+ ors = [_exists_clause_for_value(k, elem) for elem in v]+ if ors:+ stmt = stmt.where(sa.or_(*ors))+ else:+ stmt = stmt.where(_exists_clause_for_value(k, v))+ return stmt+++def get_reference_by_id(+ session: Session,+ reference_id: str,+) -> AssetReference | None:+ return session.get(AssetReference, reference_id)+++def get_reference_with_owner_check(+ session: Session,+ reference_id: str,+ owner_id: str,+) -> AssetReference:+ """Fetch a reference and verify ownership.++ Raises:+ ValueError: if reference not found or soft-deleted+ PermissionError: if owner_id doesn't match+ """+ ref = get_reference_by_id(session, reference_id=reference_id)+ if not ref or ref.deleted_at is not None:+ raise ValueError(f"AssetReference {reference_id} not found")+ if ref.owner_id and ref.owner_id != owner_id:+ raise PermissionError("not owner")+ return ref+++def get_reference_by_file_path(+ session: Session,+ file_path: str,+) -> AssetReference | None:+ """Get a reference by its file path."""+ return (+ session.execute(+ select(AssetReference).where(AssetReference.file_path == file_path).limit(1)+ )+ .scalars()+ .first()+ )+++def reference_exists_for_asset_id(+ session: Session,+ asset_id: str,+) -> bool:+ q = (+ select(sa.literal(True))+ .select_from(AssetReference)+ .where(AssetReference.asset_id == asset_id)+ .where(AssetReference.deleted_at.is_(None))+ .limit(1)+ )+ return session.execute(q).first() is not None+++def insert_reference(+ session: Session,+ asset_id: str,+ name: str,+ owner_id: str = "",+ file_path: str | None = None,+ mtime_ns: int | None = None,+ preview_id: str | None = None,+) -> AssetReference | None:+ """Insert a new AssetReference. Returns None if unique constraint violated."""+ now = get_utc_now()+ try:+ with session.begin_nested():+ ref = AssetReference(+ asset_id=asset_id,+ name=name,+ owner_id=owner_id,+ file_path=file_path,+ mtime_ns=mtime_ns,+ preview_id=preview_id,+ created_at=now,+ updated_at=now,+ last_access_time=now,+ )+ session.add(ref)+ session.flush()+ return ref+ except IntegrityError:+ return None+++def get_or_create_reference(+ session: Session,+ asset_id: str,+ name: str,+ owner_id: str = "",+ file_path: str | None = None,+ mtime_ns: int | None = None,+ preview_id: str | None = None,+) -> tuple[AssetReference, bool]:+ """Get existing or create new AssetReference.++ For filesystem references (file_path is set), uniqueness is by file_path.+ For API references (file_path is None), we look for matching+ asset_id + owner_id + name.++ Returns (reference, created).+ """+ ref = insert_reference(+ session,+ asset_id=asset_id,+ name=name,+ owner_id=owner_id,+ file_path=file_path,+ mtime_ns=mtime_ns,+ preview_id=preview_id,+ )+ if ref:+ return ref, True++ # Find existing - priority to file_path match, then name match+ if file_path:+ existing = get_reference_by_file_path(session, file_path)+ else:+ existing = (+ session.execute(+ select(AssetReference)+ .where(+ AssetReference.asset_id == asset_id,+ AssetReference.name == name,+ AssetReference.owner_id == owner_id,+ AssetReference.file_path.is_(None),+ )+ .limit(1)+ )+ .unique()+ .scalar_one_or_none()+ )+ if not existing:+ raise RuntimeError("Failed to find AssetReference after insert conflict.")+ return existing, False+++def update_reference_timestamps(+ session: Session,+ reference: AssetReference,+ preview_id: str | None = None,+) -> None:+ """Update timestamps and optionally preview_id on existing AssetReference."""+ now = get_utc_now()+ if preview_id and reference.preview_id != preview_id:+ reference.preview_id = preview_id+ reference.updated_at = now+++def list_references_page(+ session: Session,+ owner_id: str = "",+ limit: int = 100,+ offset: int = 0,+ name_contains: str | None = None,+ include_tags: Sequence[str] | None = None,+ exclude_tags: Sequence[str] | None = None,+ metadata_filter: dict | None = None,+ sort: str | None = None,+ order: str | None = None,+) -> tuple[list[AssetReference], dict[str, list[str]], int]:+ """List references with pagination, filtering, and sorting.++ Returns (references, tag_map, total_count).+ """+ base = (+ select(AssetReference)+ .join(Asset, Asset.id == AssetReference.asset_id)+ .where(build_visible_owner_clause(owner_id))+ .where(AssetReference.is_missing == False) # noqa: E712+ .where(AssetReference.deleted_at.is_(None))+ .options(noload(AssetReference.tags))+ )++ if name_contains:+ escaped, esc = escape_sql_like_string(name_contains)+ base = base.where(AssetReference.name.ilike(f"%{escaped}%", escape=esc))++ base = _apply_tag_filters(base, include_tags, exclude_tags)+ base = _apply_metadata_filter(base, metadata_filter)++ sort = (sort or "created_at").lower()+ order = (order or "desc").lower()+ sort_map = {+ "name": AssetReference.name,+ "created_at": AssetReference.created_at,+ "updated_at": AssetReference.updated_at,+ "last_access_time": AssetReference.last_access_time,+ "size": Asset.size_bytes,+ }+ sort_col = sort_map.get(sort, AssetReference.created_at)+ sort_exp = sort_col.desc() if order == "desc" else sort_col.asc()++ base = base.order_by(sort_exp).limit(limit).offset(offset)++ count_stmt = (+ select(sa.func.count())+ .select_from(AssetReference)+ .join(Asset, Asset.id == AssetReference.asset_id)+ .where(build_visible_owner_clause(owner_id))+ .where(AssetReference.is_missing == False) # noqa: E712+ .where(AssetReference.deleted_at.is_(None))+ )+ if name_contains:+ escaped, esc = escape_sql_like_string(name_contains)+ count_stmt = count_stmt.where(+ AssetReference.name.ilike(f"%{escaped}%", escape=esc)+ )+ count_stmt = _apply_tag_filters(count_stmt, include_tags, exclude_tags)+ count_stmt = _apply_metadata_filter(count_stmt, metadata_filter)++ total = int(session.execute(count_stmt).scalar_one() or 0)+ refs = session.execute(base).unique().scalars().all()++ id_list: list[str] = [r.id for r in refs]+ tag_map: dict[str, list[str]] = defaultdict(list)+ if id_list:+ rows = session.execute(+ select(AssetReferenceTag.asset_reference_id, Tag.name)+ .join(Tag, Tag.name == AssetReferenceTag.tag_name)+ .where(AssetReferenceTag.asset_reference_id.in_(id_list))+ .order_by(AssetReferenceTag.added_at)+ )+ for ref_id, tag_name in rows.all():+ tag_map[ref_id].append(tag_name)++ return list(refs), tag_map, total+++def fetch_reference_asset_and_tags(+ session: Session,+ reference_id: str,+ owner_id: str = "",+) -> tuple[AssetReference, Asset, list[str]] | None:+ stmt = (+ select(AssetReference, Asset, Tag.name)+ .join(Asset, Asset.id == AssetReference.asset_id)+ .join(+ AssetReferenceTag,+ AssetReferenceTag.asset_reference_id == AssetReference.id,+ isouter=True,+ )+ .join(Tag, Tag.name == AssetReferenceTag.tag_name, isouter=True)+ .where(+ AssetReference.id == reference_id,+ AssetReference.deleted_at.is_(None),+ build_visible_owner_clause(owner_id),+ )+ .options(noload(AssetReference.tags))+ .order_by(Tag.name.asc())+ )++ rows = session.execute(stmt).all()+ if not rows:+ return None++ first_ref, first_asset, _ = rows[0]+ tags: list[str] = []+ seen: set[str] = set()+ for _ref, _asset, tag_name in rows:+ if tag_name and tag_name not in seen:+ seen.add(tag_name)+ tags.append(tag_name)+ return first_ref, first_asset, tags+++def fetch_reference_and_asset(+ session: Session,+ reference_id: str,+ owner_id: str = "",+) -> tuple[AssetReference, Asset] | None:+ stmt = (+ select(AssetReference, Asset)+ .join(Asset, Asset.id == AssetReference.asset_id)+ .where(+ AssetReference.id == reference_id,+ AssetReference.deleted_at.is_(None),+ build_visible_owner_clause(owner_id),+ )+ .limit(1)+ .options(noload(AssetReference.tags))+ )+ pair = session.execute(stmt).first()+ if not pair:+ return None+ return pair[0], pair[1]+++def update_reference_access_time(+ session: Session,+ reference_id: str,+ ts: datetime | None = None,+ only_if_newer: bool = True,+) -> None:+ ts = ts or get_utc_now()+ stmt = sa.update(AssetReference).where(AssetReference.id == reference_id)+ if only_if_newer:+ stmt = stmt.where(+ sa.or_(+ AssetReference.last_access_time.is_(None),+ AssetReference.last_access_time < ts,+ )+ )+ session.execute(stmt.values(last_access_time=ts))+++def update_reference_name(+ session: Session,+ reference_id: str,+ name: str,+) -> None:+ """Update the name of an AssetReference."""+ now = get_utc_now()+ session.execute(+ sa.update(AssetReference)+ .where(AssetReference.id == reference_id)+ .values(name=name, updated_at=now)+ )+++def update_reference_updated_at(+ session: Session,+ reference_id: str,+ ts: datetime | None = None,+) -> None:+ """Update the updated_at timestamp of an AssetReference."""+ ts = ts or get_utc_now()+ session.execute(+ sa.update(AssetReference)+ .where(AssetReference.id == reference_id)+ .values(updated_at=ts)+ )+++def set_reference_metadata(+ session: Session,+ reference_id: str,+ user_metadata: dict | None = None,+) -> None:+ ref = session.get(AssetReference, reference_id)+ if not ref:+ raise ValueError(f"AssetReference {reference_id} not found")++ ref.user_metadata = user_metadata or {}+ ref.updated_at = get_utc_now()+ session.flush()++ session.execute(+ delete(AssetReferenceMeta).where(+ AssetReferenceMeta.asset_reference_id == reference_id+ )+ )+ session.flush()++ if not user_metadata:+ return++ rows: list[AssetReferenceMeta] = []+ for k, v in user_metadata.items():+ for r in convert_metadata_to_rows(k, v):+ rows.append(+ AssetReferenceMeta(+ asset_reference_id=reference_id,+ key=r["key"],+ ordinal=int(r["ordinal"]),+ val_str=r.get("val_str"),+ val_num=r.get("val_num"),+ val_bool=r.get("val_bool"),+ val_json=r.get("val_json"),+ )+ )+ if rows:+ session.add_all(rows)+ session.flush()+++def delete_reference_by_id(+ session: Session,+ reference_id: str,+ owner_id: str,+) -> bool:+ stmt = sa.delete(AssetReference).where(+ AssetReference.id == reference_id,+ build_visible_owner_clause(owner_id),+ )+ return int(session.execute(stmt).rowcount or 0) > 0+++def soft_delete_reference_by_id(+ session: Session,+ reference_id: str,+ owner_id: str,+) -> bool:+ """Mark a reference as soft-deleted by setting deleted_at timestamp.++ Returns True if the reference was found and marked deleted.+ """+ now = get_utc_now()+ stmt = (+ sa.update(AssetReference)+ .where(+ AssetReference.id == reference_id,+ AssetReference.deleted_at.is_(None),+ build_visible_owner_clause(owner_id),+ )+ .values(deleted_at=now)+ )+ return int(session.execute(stmt).rowcount or 0) > 0+++def set_reference_preview(+ session: Session,+ reference_id: str,+ preview_asset_id: str | None = None,+) -> None:+ """Set or clear preview_id and bump updated_at. Raises on unknown IDs."""+ ref = session.get(AssetReference, reference_id)+ if not ref:+ raise ValueError(f"AssetReference {reference_id} not found")++ if preview_asset_id is None:+ ref.preview_id = None+ else:+ if not session.get(Asset, preview_asset_id):+ raise ValueError(f"Preview Asset {preview_asset_id} not found")+ ref.preview_id = preview_asset_id++ ref.updated_at = get_utc_now()+ session.flush()+++class CacheStateRow(NamedTuple):+ """Row from reference query with cache state data."""++ reference_id: str+ file_path: str+ mtime_ns: int | None+ needs_verify: bool+ asset_id: str+ asset_hash: str | None+ size_bytes: int | None+++def list_references_by_asset_id(+ session: Session,+ asset_id: str,+) -> Sequence[AssetReference]:+ return (+ session.execute(+ select(AssetReference)+ .where(AssetReference.asset_id == asset_id)+ .order_by(AssetReference.id.asc())+ )+ .scalars()+ .all()+ )+++def upsert_reference(+ session: Session,+ asset_id: str,+ file_path: str,+ name: str,+ mtime_ns: int,+ owner_id: str = "",+) -> tuple[bool, bool]:+ """Upsert a reference by file_path. Returns (created, updated).++ Also restores references that were previously marked as missing.+ """+ now = get_utc_now()+ vals = {+ "asset_id": asset_id,+ "file_path": file_path,+ "name": name,+ "owner_id": owner_id,+ "mtime_ns": int(mtime_ns),+ "is_missing": False,+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ }+ ins = (+ sqlite.insert(AssetReference)+ .values(**vals)+ .on_conflict_do_nothing(index_elements=[AssetReference.file_path])+ )+ res = session.execute(ins)+ created = int(res.rowcount or 0) > 0++ if created:+ return True, False++ upd = (+ sa.update(AssetReference)+ .where(AssetReference.file_path == file_path)+ .where(+ sa.or_(+ AssetReference.asset_id != asset_id,+ AssetReference.mtime_ns.is_(None),+ AssetReference.mtime_ns != int(mtime_ns),+ AssetReference.is_missing == True, # noqa: E712+ AssetReference.deleted_at.isnot(None),+ )+ )+ .values(+ asset_id=asset_id, mtime_ns=int(mtime_ns), is_missing=False,+ deleted_at=None, updated_at=now,+ )+ )+ res2 = session.execute(upd)+ updated = int(res2.rowcount or 0) > 0+ return False, updated+++def mark_references_missing_outside_prefixes(+ session: Session,+ valid_prefixes: list[str],+) -> int:+ """Mark references as missing when file_path doesn't match any valid prefix.++ Returns number of references marked as missing.+ """+ if not valid_prefixes:+ return 0++ conds = build_prefix_like_conditions(valid_prefixes)+ matches_valid_prefix = sa.or_(*conds)+ result = session.execute(+ sa.update(AssetReference)+ .where(AssetReference.file_path.isnot(None))+ .where(AssetReference.deleted_at.is_(None))+ .where(~matches_valid_prefix)+ .where(AssetReference.is_missing == False) # noqa: E712+ .values(is_missing=True)+ )+ return result.rowcount+++def restore_references_by_paths(session: Session, file_paths: list[str]) -> int:+ """Restore references that were previously marked as missing.++ Returns number of references restored.+ """+ if not file_paths:+ return 0++ total = 0+ for chunk in iter_chunks(file_paths, MAX_BIND_PARAMS):+ result = session.execute(+ sa.update(AssetReference)+ .where(AssetReference.file_path.in_(chunk))+ .where(AssetReference.is_missing == True) # noqa: E712+ .where(AssetReference.deleted_at.is_(None))+ .values(is_missing=False)+ )+ total += result.rowcount+ return total+++def get_unreferenced_unhashed_asset_ids(session: Session) -> list[str]:+ """Get IDs of unhashed assets (hash=None) with no active references.++ An asset is considered unreferenced if it has no references,+ or all its references are marked as missing.++ Returns list of asset IDs that are unreferenced.+ """+ active_ref_exists = (+ sa.select(sa.literal(1))+ .where(AssetReference.asset_id == Asset.id)+ .where(AssetReference.is_missing == False) # noqa: E712+ .where(AssetReference.deleted_at.is_(None))+ .correlate(Asset)+ .exists()+ )+ unreferenced_subq = sa.select(Asset.id).where(+ Asset.hash.is_(None), ~active_ref_exists+ )+ return [row[0] for row in session.execute(unreferenced_subq).all()]+++def delete_assets_by_ids(session: Session, asset_ids: list[str]) -> int:+ """Delete assets and their references by ID.++ Returns number of assets deleted.+ """+ if not asset_ids:+ return 0+ total = 0+ for chunk in iter_chunks(asset_ids, MAX_BIND_PARAMS):+ session.execute(+ sa.delete(AssetReference).where(AssetReference.asset_id.in_(chunk))+ )+ result = session.execute(sa.delete(Asset).where(Asset.id.in_(chunk)))+ total += result.rowcount+ return total+++def get_references_for_prefixes(+ session: Session,+ prefixes: list[str],+ *,+ include_missing: bool = False,+) -> list[CacheStateRow]:+ """Get all references with file paths matching any of the given prefixes.++ Args:+ session: Database session+ prefixes: List of absolute directory prefixes to match+ include_missing: If False (default), exclude references marked as missing++ Returns:+ List of cache state rows with joined asset data+ """+ if not prefixes:+ return []++ conds = build_prefix_like_conditions(prefixes)++ query = (+ sa.select(+ AssetReference.id,+ AssetReference.file_path,+ AssetReference.mtime_ns,+ AssetReference.needs_verify,+ AssetReference.asset_id,+ Asset.hash,+ Asset.size_bytes,+ )+ .join(Asset, Asset.id == AssetReference.asset_id)+ .where(AssetReference.file_path.isnot(None))+ .where(AssetReference.deleted_at.is_(None))+ .where(sa.or_(*conds))+ )++ if not include_missing:+ query = query.where(AssetReference.is_missing == False) # noqa: E712++ rows = session.execute(+ query.order_by(AssetReference.asset_id.asc(), AssetReference.id.asc())+ ).all()++ return [+ CacheStateRow(+ reference_id=row[0],+ file_path=row[1],+ mtime_ns=row[2],+ needs_verify=row[3],+ asset_id=row[4],+ asset_hash=row[5],+ size_bytes=int(row[6]) if row[6] is not None else None,+ )+ for row in rows+ ]+++def bulk_update_needs_verify(+ session: Session, reference_ids: list[str], value: bool+) -> int:+ """Set needs_verify flag for multiple references.++ Returns: Number of rows updated+ """+ if not reference_ids:+ return 0+ total = 0+ for chunk in iter_chunks(reference_ids, MAX_BIND_PARAMS):+ result = session.execute(+ sa.update(AssetReference)+ .where(AssetReference.id.in_(chunk))+ .values(needs_verify=value)+ )+ total += result.rowcount+ return total+++def bulk_update_is_missing(+ session: Session, reference_ids: list[str], value: bool+) -> int:+ """Set is_missing flag for multiple references.++ Returns: Number of rows updated+ """+ if not reference_ids:+ return 0+ total = 0+ for chunk in iter_chunks(reference_ids, MAX_BIND_PARAMS):+ result = session.execute(+ sa.update(AssetReference)+ .where(AssetReference.id.in_(chunk))+ .values(is_missing=value)+ )+ total += result.rowcount+ return total+++def delete_references_by_ids(session: Session, reference_ids: list[str]) -> int:+ """Delete references by their IDs.++ Returns: Number of rows deleted+ """+ if not reference_ids:+ return 0+ total = 0+ for chunk in iter_chunks(reference_ids, MAX_BIND_PARAMS):+ result = session.execute(+ sa.delete(AssetReference).where(AssetReference.id.in_(chunk))+ )+ total += result.rowcount+ return total+++def delete_orphaned_seed_asset(session: Session, asset_id: str) -> bool:+ """Delete a seed asset (hash is None) and its references.++ Returns: True if asset was deleted, False if not found or has a hash+ """+ asset = session.get(Asset, asset_id)+ if not asset:+ return False+ if asset.hash is not None:+ return False+ session.execute(+ sa.delete(AssetReference).where(AssetReference.asset_id == asset_id)+ )+ session.delete(asset)+ return True+++class UnenrichedReferenceRow(NamedTuple):+ """Row for references needing enrichment."""++ reference_id: str+ asset_id: str+ file_path: str+ enrichment_level: int+++def get_unenriched_references(+ session: Session,+ prefixes: list[str],+ max_level: int = 0,+ limit: int = 1000,+) -> list[UnenrichedReferenceRow]:+ """Get references that need enrichment (enrichment_level <= max_level).++ Args:+ session: Database session+ prefixes: List of absolute directory prefixes to scan+ max_level: Maximum enrichment level to include (0=stubs, 1=metadata done)+ limit: Maximum number of rows to return++ Returns:+ List of unenriched reference rows with file paths+ """+ if not prefixes:+ return []++ conds = build_prefix_like_conditions(prefixes)++ query = (+ sa.select(+ AssetReference.id,+ AssetReference.asset_id,+ AssetReference.file_path,+ AssetReference.enrichment_level,+ )+ .where(AssetReference.file_path.isnot(None))+ .where(AssetReference.deleted_at.is_(None))+ .where(sa.or_(*conds))+ .where(AssetReference.is_missing == False) # noqa: E712+ .where(AssetReference.enrichment_level <= max_level)+ .order_by(AssetReference.id.asc())+ .limit(limit)+ )++ rows = session.execute(query).all()+ return [+ UnenrichedReferenceRow(+ reference_id=row[0],+ asset_id=row[1],+ file_path=row[2],+ enrichment_level=row[3],+ )+ for row in rows+ ]+++def bulk_update_enrichment_level(+ session: Session,+ reference_ids: list[str],+ level: int,+) -> int:+ """Update enrichment level for multiple references.++ Returns: Number of rows updated+ """+ if not reference_ids:+ return 0+ result = session.execute(+ sa.update(AssetReference)+ .where(AssetReference.id.in_(reference_ids))+ .values(enrichment_level=level)+ )+ return result.rowcount+++def bulk_insert_references_ignore_conflicts(+ session: Session,+ rows: list[dict],+) -> None:+ """Bulk insert reference rows with ON CONFLICT DO NOTHING on file_path.++ Each dict should have: id, asset_id, file_path, name, owner_id, mtime_ns, etc.+ The is_missing field is automatically set to False for new inserts.+ """+ if not rows:+ return+ enriched_rows = [{**row, "is_missing": False} for row in rows]+ ins = sqlite.insert(AssetReference).on_conflict_do_nothing(+ index_elements=[AssetReference.file_path]+ )+ for chunk in iter_chunks(enriched_rows, calculate_rows_per_statement(14)):+ session.execute(ins, chunk)+++def get_references_by_paths_and_asset_ids(+ session: Session,+ path_to_asset: dict[str, str],+) -> set[str]:+ """Query references to find paths where our asset_id won the insert.++ Args:+ path_to_asset: Mapping of file_path -> asset_id we tried to insert++ Returns:+ Set of file_paths where our asset_id is present+ """+ if not path_to_asset:+ return set()++ pairs = list(path_to_asset.items())+ winners: set[str] = set()++ # Each pair uses 2 bind params, so chunk at MAX_BIND_PARAMS // 2+ for chunk in iter_chunks(pairs, MAX_BIND_PARAMS // 2):+ pairwise = sa.tuple_(AssetReference.file_path, AssetReference.asset_id).in_(+ chunk+ )+ result = session.execute(+ select(AssetReference.file_path).where(pairwise)+ )+ winners.update(result.scalars().all())++ return winners+++def get_reference_ids_by_ids(+ session: Session,+ reference_ids: list[str],+) -> set[str]:+ """Query to find which reference IDs exist in the database."""+ if not reference_ids:+ return set()++ found: set[str] = set()+ for chunk in iter_chunks(reference_ids, MAX_BIND_PARAMS):+ result = session.execute(+ select(AssetReference.id).where(AssetReference.id.in_(chunk))+ )+ found.update(result.scalars().all())+ return founddiff --git a/app/assets/database/queries/common.py b/app/assets/database/queries/common.pynew file mode 100644index 000000000..194c39a1e--- /dev/null+++ b/app/assets/database/queries/common.py@@ -0,0 +1,54 @@+"""Shared utilities for database query modules."""++import os+from typing import Iterable++import sqlalchemy as sa++from app.assets.database.models import AssetReference+from app.assets.helpers import escape_sql_like_string++MAX_BIND_PARAMS = 800+++def calculate_rows_per_statement(cols: int) -> int:+ """Calculate how many rows can fit in one statement given column count."""+ return max(1, MAX_BIND_PARAMS // max(1, cols))+++def iter_chunks(seq, n: int):+ """Yield successive n-sized chunks from seq."""+ for i in range(0, len(seq), n):+ yield seq[i : i + n]+++def iter_row_chunks(rows: list[dict], cols_per_row: int) -> Iterable[list[dict]]:+ """Yield chunks of rows sized to fit within bind param limits."""+ if not rows:+ return+ yield from iter_chunks(rows, calculate_rows_per_statement(cols_per_row))+++def build_visible_owner_clause(owner_id: str) -> sa.sql.ClauseElement:+ """Build owner visibility predicate for reads.++ Owner-less rows are visible to everyone.+ """+ owner_id = (owner_id or "").strip()+ if owner_id == "":+ return AssetReference.owner_id == ""+ return AssetReference.owner_id.in_(["", owner_id])+++def build_prefix_like_conditions(+ prefixes: list[str],+) -> list[sa.sql.ColumnElement]:+ """Build LIKE conditions for matching file paths under directory prefixes."""+ conds = []+ for p in prefixes:+ base = os.path.abspath(p)+ if not base.endswith(os.sep):+ base += os.sep+ escaped, esc = escape_sql_like_string(base)+ conds.append(AssetReference.file_path.like(escaped + "%", escape=esc))+ return condsdiff --git a/app/assets/database/queries/tags.py b/app/assets/database/queries/tags.pynew file mode 100644index 000000000..8b25fee67--- /dev/null+++ b/app/assets/database/queries/tags.py@@ -0,0 +1,356 @@+from dataclasses import dataclass+from typing import Iterable, Sequence++import sqlalchemy as sa+from sqlalchemy import delete, func, select+from sqlalchemy.dialects import sqlite+from sqlalchemy.exc import IntegrityError+from sqlalchemy.orm import Session++from app.assets.database.models import (+ AssetReference,+ AssetReferenceMeta,+ AssetReferenceTag,+ Tag,+)+from app.assets.database.queries.common import (+ build_visible_owner_clause,+ iter_row_chunks,+)+from app.assets.helpers import escape_sql_like_string, get_utc_now, normalize_tags+++@dataclass(frozen=True)+class AddTagsResult:+ added: list[str]+ already_present: list[str]+ total_tags: list[str]+++@dataclass(frozen=True)+class RemoveTagsResult:+ removed: list[str]+ not_present: list[str]+ total_tags: list[str]+++@dataclass(frozen=True)+class SetTagsResult:+ added: list[str]+ removed: list[str]+ total: list[str]+++def validate_tags_exist(session: Session, tags: list[str]) -> None:+ """Raise ValueError if any of the given tag names do not exist."""+ existing_tag_names = set(+ name+ for (name,) in session.execute(select(Tag.name).where(Tag.name.in_(tags))).all()+ )+ missing = [t for t in tags if t not in existing_tag_names]+ if missing:+ raise ValueError(f"Unknown tags: {missing}")+++def ensure_tags_exist(+ session: Session, names: Iterable[str], tag_type: str = "user"+) -> None:+ wanted = normalize_tags(list(names))+ if not wanted:+ return+ rows = [{"name": n, "tag_type": tag_type} for n in list(dict.fromkeys(wanted))]+ ins = (+ sqlite.insert(Tag)+ .values(rows)+ .on_conflict_do_nothing(index_elements=[Tag.name])+ )+ session.execute(ins)+++def get_reference_tags(session: Session, reference_id: str) -> list[str]:+ return [+ tag_name+ for (tag_name,) in (+ session.execute(+ select(AssetReferenceTag.tag_name).where(+ AssetReferenceTag.asset_reference_id == reference_id+ )+ )+ ).all()+ ]+++def set_reference_tags(+ session: Session,+ reference_id: str,+ tags: Sequence[str],+ origin: str = "manual",+) -> SetTagsResult:+ desired = normalize_tags(tags)++ current = set(get_reference_tags(session, reference_id))++ to_add = [t for t in desired if t not in current]+ to_remove = [t for t in current if t not in desired]++ if to_add:+ ensure_tags_exist(session, to_add, tag_type="user")+ session.add_all(+ [+ AssetReferenceTag(+ asset_reference_id=reference_id,+ tag_name=t,+ origin=origin,+ added_at=get_utc_now(),+ )+ for t in to_add+ ]+ )+ session.flush()++ if to_remove:+ session.execute(+ delete(AssetReferenceTag).where(+ AssetReferenceTag.asset_reference_id == reference_id,+ AssetReferenceTag.tag_name.in_(to_remove),+ )+ )+ session.flush()++ return SetTagsResult(added=to_add, removed=to_remove, total=desired)+++def add_tags_to_reference(+ session: Session,+ reference_id: str,+ tags: Sequence[str],+ origin: str = "manual",+ create_if_missing: bool = True,+ reference_row: AssetReference | None = None,+) -> AddTagsResult:+ if not reference_row:+ ref = session.get(AssetReference, reference_id)+ if not ref:+ raise ValueError(f"AssetReference {reference_id} not found")++ norm = normalize_tags(tags)+ if not norm:+ total = get_reference_tags(session, reference_id=reference_id)+ return AddTagsResult(added=[], already_present=[], total_tags=total)++ if create_if_missing:+ ensure_tags_exist(session, norm, tag_type="user")++ current = set(get_reference_tags(session, reference_id))++ want = set(norm)+ to_add = sorted(want - current)++ if to_add:+ with session.begin_nested() as nested:+ try:+ session.add_all(+ [+ AssetReferenceTag(+ asset_reference_id=reference_id,+ tag_name=t,+ origin=origin,+ added_at=get_utc_now(),+ )+ for t in to_add+ ]+ )+ session.flush()+ except IntegrityError:+ nested.rollback()++ after = set(get_reference_tags(session, reference_id=reference_id))+ return AddTagsResult(+ added=sorted(((after - current) & want)),+ already_present=sorted(want & current),+ total_tags=sorted(after),+ )+++def remove_tags_from_reference(+ session: Session,+ reference_id: str,+ tags: Sequence[str],+) -> RemoveTagsResult:+ ref = session.get(AssetReference, reference_id)+ if not ref:+ raise ValueError(f"AssetReference {reference_id} not found")++ norm = normalize_tags(tags)+ if not norm:+ total = get_reference_tags(session, reference_id=reference_id)+ return RemoveTagsResult(removed=[], not_present=[], total_tags=total)++ existing = set(get_reference_tags(session, reference_id))++ to_remove = sorted(set(t for t in norm if t in existing))+ not_present = sorted(set(t for t in norm if t not in existing))++ if to_remove:+ session.execute(+ delete(AssetReferenceTag).where(+ AssetReferenceTag.asset_reference_id == reference_id,+ AssetReferenceTag.tag_name.in_(to_remove),+ )+ )+ session.flush()++ total = get_reference_tags(session, reference_id=reference_id)+ return RemoveTagsResult(removed=to_remove, not_present=not_present, total_tags=total)+++def add_missing_tag_for_asset_id(+ session: Session,+ asset_id: str,+ origin: str = "automatic",+) -> None:+ select_rows = (+ sa.select(+ AssetReference.id.label("asset_reference_id"),+ sa.literal("missing").label("tag_name"),+ sa.literal(origin).label("origin"),+ sa.literal(get_utc_now()).label("added_at"),+ )+ .where(AssetReference.asset_id == asset_id)+ .where(+ sa.not_(+ sa.exists().where(+ (AssetReferenceTag.asset_reference_id == AssetReference.id)+ & (AssetReferenceTag.tag_name == "missing")+ )+ )+ )+ )+ session.execute(+ sqlite.insert(AssetReferenceTag)+ .from_select(+ ["asset_reference_id", "tag_name", "origin", "added_at"],+ select_rows,+ )+ .on_conflict_do_nothing(+ index_elements=[+ AssetReferenceTag.asset_reference_id,+ AssetReferenceTag.tag_name,+ ]+ )+ )+++def remove_missing_tag_for_asset_id(+ session: Session,+ asset_id: str,+) -> None:+ session.execute(+ sa.delete(AssetReferenceTag).where(+ AssetReferenceTag.asset_reference_id.in_(+ sa.select(AssetReference.id).where(AssetReference.asset_id == asset_id)+ ),+ AssetReferenceTag.tag_name == "missing",+ )+ )+++def list_tags_with_usage(+ session: Session,+ prefix: str | None = None,+ limit: int = 100,+ offset: int = 0,+ include_zero: bool = True,+ order: str = "count_desc",+ owner_id: str = "",+) -> tuple[list[tuple[str, str, int]], int]:+ counts_sq = (+ select(+ AssetReferenceTag.tag_name.label("tag_name"),+ func.count(AssetReferenceTag.asset_reference_id).label("cnt"),+ )+ .select_from(AssetReferenceTag)+ .join(AssetReference, AssetReference.id == AssetReferenceTag.asset_reference_id)+ .where(build_visible_owner_clause(owner_id))+ .where(AssetReference.deleted_at.is_(None))+ .group_by(AssetReferenceTag.tag_name)+ .subquery()+ )++ q = (+ select(+ Tag.name,+ Tag.tag_type,+ func.coalesce(counts_sq.c.cnt, 0).label("count"),+ )+ .select_from(Tag)+ .join(counts_sq, counts_sq.c.tag_name == Tag.name, isouter=True)+ )++ if prefix:+ escaped, esc = escape_sql_like_string(prefix.strip().lower())+ q = q.where(Tag.name.like(escaped + "%", escape=esc))++ if not include_zero:+ q = q.where(func.coalesce(counts_sq.c.cnt, 0) > 0)++ if order == "name_asc":+ q = q.order_by(Tag.name.asc())+ else:+ q = q.order_by(func.coalesce(counts_sq.c.cnt, 0).desc(), Tag.name.asc())++ total_q = select(func.count()).select_from(Tag)+ if prefix:+ escaped, esc = escape_sql_like_string(prefix.strip().lower())+ total_q = total_q.where(Tag.name.like(escaped + "%", escape=esc))+ if not include_zero:+ visible_tags_sq = (+ select(AssetReferenceTag.tag_name)+ .join(AssetReference, AssetReference.id == AssetReferenceTag.asset_reference_id)+ .where(build_visible_owner_clause(owner_id))+ .where(AssetReference.deleted_at.is_(None))+ .group_by(AssetReferenceTag.tag_name)+ )+ total_q = total_q.where(Tag.name.in_(visible_tags_sq))++ rows = (session.execute(q.limit(limit).offset(offset))).all()+ total = (session.execute(total_q)).scalar_one()++ rows_norm = [(name, ttype, int(count or 0)) for (name, ttype, count) in rows]+ return rows_norm, int(total or 0)+++def bulk_insert_tags_and_meta(+ session: Session,+ tag_rows: list[dict],+ meta_rows: list[dict],+) -> None:+ """Batch insert into asset_reference_tags and asset_reference_meta.++ Uses ON CONFLICT DO NOTHING.++ Args:+ session: Database session+ tag_rows: Dicts with: asset_reference_id, tag_name, origin, added_at+ meta_rows: Dicts with: asset_reference_id, key, ordinal, val_*+ """+ if tag_rows:+ ins_tags = sqlite.insert(AssetReferenceTag).on_conflict_do_nothing(+ index_elements=[+ AssetReferenceTag.asset_reference_id,+ AssetReferenceTag.tag_name,+ ]+ )+ for chunk in iter_row_chunks(tag_rows, cols_per_row=4):+ session.execute(ins_tags, chunk)++ if meta_rows:+ ins_meta = sqlite.insert(AssetReferenceMeta).on_conflict_do_nothing(+ index_elements=[+ AssetReferenceMeta.asset_reference_id,+ AssetReferenceMeta.key,+ AssetReferenceMeta.ordinal,+ ]+ )+ for chunk in iter_row_chunks(meta_rows, cols_per_row=7):+ session.execute(ins_meta, chunk)diff --git a/app/assets/database/tags.py b/app/assets/database/tags.pydeleted file mode 100644index 3ab6497c2..000000000--- a/app/assets/database/tags.py+++ /dev/null@@ -1,62 +0,0 @@-from typing import Iterable--import sqlalchemy-from sqlalchemy.orm import Session-from sqlalchemy.dialects import sqlite--from app.assets.helpers import normalize_tags, utcnow-from app.assets.database.models import Tag, AssetInfoTag, AssetInfo---def ensure_tags_exist(session: Session, names: Iterable[str], tag_type: str = "user") -> None:- wanted = normalize_tags(list(names))- if not wanted:- return- rows = [{"name": n, "tag_type": tag_type} for n in list(dict.fromkeys(wanted))]- ins = (- sqlite.insert(Tag)- .values(rows)- .on_conflict_do_nothing(index_elements=[Tag.name])- )- return session.execute(ins)--def add_missing_tag_for_asset_id(- session: Session,- *,- asset_id: str,- origin: str = "automatic",-) -> None:- select_rows = (- sqlalchemy.select(- AssetInfo.id.label("asset_info_id"),- sqlalchemy.literal("missing").label("tag_name"),- sqlalchemy.literal(origin).label("origin"),- sqlalchemy.literal(utcnow()).label("added_at"),- )- .where(AssetInfo.asset_id == asset_id)- .where(- sqlalchemy.not_(- sqlalchemy.exists().where((AssetInfoTag.asset_info_id == AssetInfo.id) & (AssetInfoTag.tag_name == "missing"))- )- )- )- session.execute(- sqlite.insert(AssetInfoTag)- .from_select(- ["asset_info_id", "tag_name", "origin", "added_at"],- select_rows,- )- .on_conflict_do_nothing(index_elements=[AssetInfoTag.asset_info_id, AssetInfoTag.tag_name])- )--def remove_missing_tag_for_asset_id(- session: Session,- *,- asset_id: str,-) -> None:- session.execute(- sqlalchemy.delete(AssetInfoTag).where(- AssetInfoTag.asset_info_id.in_(sqlalchemy.select(AssetInfo.id).where(AssetInfo.asset_id == asset_id)),- AssetInfoTag.tag_name == "missing",- )- )diff --git a/app/assets/hashing.py b/app/assets/hashing.pydeleted file mode 100644index 4b72084b9..000000000--- a/app/assets/hashing.py+++ /dev/null@@ -1,75 +0,0 @@-from blake3 import blake3-from typing import IO-import os-import asyncio---DEFAULT_CHUNK = 8 * 1024 *1024 # 8MB--# NOTE: this allows hashing different representations of a file-like object-def blake3_hash(- fp: str | IO[bytes],- chunk_size: int = DEFAULT_CHUNK,-) -> str:- """- Returns a BLAKE3 hex digest for ``fp``, which may be:- - a filename (str/bytes) or PathLike- - an open binary file object- If ``fp`` is a file object, it must be opened in **binary** mode and support- ``read``, ``seek``, and ``tell``. The function will seek to the start before- reading and will attempt to restore the original position afterward.- """- # duck typing to check if input is a file-like object- if hasattr(fp, "read"):- return _hash_file_obj(fp, chunk_size)-- with open(os.fspath(fp), "rb") as f:- return _hash_file_obj(f, chunk_size)---async def blake3_hash_async(- fp: str | IO[bytes],- chunk_size: int = DEFAULT_CHUNK,-) -> str:- """Async wrapper for ``blake3_hash_sync``.- Uses a worker thread so the event loop remains responsive.- """- # If it is a path, open inside the worker thread to keep I/O off the loop.- if hasattr(fp, "read"):- return await asyncio.to_thread(blake3_hash, fp, chunk_size)-- def _worker() -> str:- with open(os.fspath(fp), "rb") as f:- return _hash_file_obj(f, chunk_size)-- return await asyncio.to_thread(_worker)---def _hash_file_obj(file_obj: IO, chunk_size: int = DEFAULT_CHUNK) -> str:- """- Hash an already-open binary file object by streaming in chunks.- - Seeks to the beginning before reading (if supported).- - Restores the original position afterward (if tell/seek are supported).- """- if chunk_size <= 0:- chunk_size = DEFAULT_CHUNK-- # in case file object is already open and not at the beginning, track so can be restored after hashing- orig_pos = file_obj.tell()-- try:- # seek to the beginning before reading- if orig_pos != 0:- file_obj.seek(0)-- h = blake3()- while True:- chunk = file_obj.read(chunk_size)- if not chunk:- break- h.update(chunk)- return h.hexdigest()- finally:- # restore original position in file object, if needed- if orig_pos != 0:- file_obj.seek(orig_pos)diff --git a/app/assets/helpers.py b/app/assets/helpers.pyindex 5030b123a..3798f3933 100644--- a/app/assets/helpers.py+++ b/app/assets/helpers.py@@ -1,226 +1,42 @@-import contextlibimport os-from decimal import Decimal-from aiohttp import webfrom datetime import datetime, timezone-from pathlib import Path-from typing import Literal, Any+from typing import Sequence-import folder_paths--RootType = Literal["models", "input", "output"]-ALLOWED_ROOTS: tuple[RootType, ...] = ("models", "input", "output")--def get_query_dict(request: web.Request) -> dict[str, Any]:+def select_best_live_path(states: Sequence) -> str:"""- Gets a dictionary of query parameters from the request.-- 'request.query' is a MultiMapping[str], needs to be converted to a dictionary to be validated by Pydantic.+ Return the best on-disk path among cache states:+ 1) Prefer a path that exists with needs_verify == False (already verified).+ 2) Otherwise, pick the first path that exists.+ 3) Otherwise return empty string."""- query_dict = {- key: request.query.getall(key) if len(request.query.getall(key)) > 1 else request.query.get(key)- for key in request.query.keys()- }- return query_dict--def list_tree(base_dir: str) -> list[str]:- out: list[str] = []- base_abs = os.path.abspath(base_dir)- if not os.path.isdir(base_abs):- return out- for dirpath, _subdirs, filenames in os.walk(base_abs, topdown=True, followlinks=False):- for name in filenames:- out.append(os.path.abspath(os.path.join(dirpath, name)))- return out--def prefixes_for_root(root: RootType) -> list[str]:- if root == "models":- bases: list[str] = []- for _bucket, paths in get_comfy_models_folders():- bases.extend(paths)- return [os.path.abspath(p) for p in bases]- if root == "input":- return [os.path.abspath(folder_paths.get_input_directory())]- if root == "output":- return [os.path.abspath(folder_paths.get_output_directory())]- return []--def escape_like_prefix(s: str, escape: str = "!") -> tuple[str, str]:- """Escapes %, _ and the escape char itself in a LIKE prefix.- Returns (escaped_prefix, escape_char). Caller should append '%' and pass escape=escape_char to .like().+ alive = [+ s+ for s in states+ if getattr(s, "file_path", None) and os.path.isfile(s.file_path)+ ]+ if not alive:+ return ""+ for s in alive:+ if not getattr(s, "needs_verify", False):+ return s.file_path+ return alive[0].file_path+++def escape_sql_like_string(s: str, escape: str = "!") -> tuple[str, str]:+ """Escapes %, _ and the escape char in a LIKE prefix.++ Returns (escaped_prefix, escape_char)."""s = s.replace(escape, escape + escape) # escape the escape char firsts = s.replace("%", escape + "%").replace("_", escape + "_") # escape LIKE wildcardsreturn s, escape-def fast_asset_file_check(- *,- mtime_db: int | None,- size_db: int | None,- stat_result: os.stat_result,-) -> bool:- if mtime_db is None:- return False- actual_mtime_ns = getattr(stat_result, "st_mtime_ns", int(stat_result.st_mtime * 1_000_000_000))- if int(mtime_db) != int(actual_mtime_ns):- return False- sz = int(size_db or 0)- if sz > 0:- return int(stat_result.st_size) == sz- return True-def utcnow() -> datetime:+def get_utc_now() -> datetime:"""Naive UTC timestamp (no tzinfo). We always treat DB datetimes as UTC."""return datetime.now(timezone.utc).replace(tzinfo=None)-def get_comfy_models_folders() -> list[tuple[str, list[str]]]:- """Build a list of (folder_name, base_paths[]) categories that are configured for model locations.-- We trust `folder_paths.folder_names_and_paths` and include a category if- *any* of its base paths lies under the Comfy `models_dir`.- """- targets: list[tuple[str, list[str]]] = []- models_root = os.path.abspath(folder_paths.models_dir)- for name, values in folder_paths.folder_names_and_paths.items():- paths, _exts = values[0], values[1] # NOTE: this prevents nodepacks that hackily edit folder_... from breaking ComfyUI- if any(os.path.abspath(p).startswith(models_root + os.sep) for p in paths):- targets.append((name, paths))- return targets--def resolve_destination_from_tags(tags: list[str]) -> tuple[str, list[str]]:- """Validates and maps tags -> (base_dir, subdirs_for_fs)"""- root = tags[0]- if root == "models":- if len(tags) < 2:- raise ValueError("at least two tags required for model asset")- try:- bases = folder_paths.folder_names_and_paths[tags[1]][0]- except KeyError:- raise ValueError(f"unknown model category '{tags[1]}'")- if not bases:- raise ValueError(f"no base path configured for category '{tags[1]}'")- base_dir = os.path.abspath(bases[0])- raw_subdirs = tags[2:]- else:- base_dir = os.path.abspath(- folder_paths.get_input_directory() if root == "input" else folder_paths.get_output_directory()- )- raw_subdirs = tags[1:]- for i in raw_subdirs:- if i in (".", ".."):- raise ValueError("invalid path component in tags")-- return base_dir, raw_subdirs if raw_subdirs else []--def ensure_within_base(candidate: str, base: str) -> None:- cand_abs = os.path.abspath(candidate)- base_abs = os.path.abspath(base)- try:- if os.path.commonpath([cand_abs, base_abs]) != base_abs:- raise ValueError("destination escapes base directory")- except Exception:- raise ValueError("invalid destination path")--def compute_relative_filename(file_path: str) -> str | None:- """- Return the model's path relative to the last well-known folder (the model category),- using forward slashes, eg:- /.../models/checkpoints/flux/123/flux.safetensors -> "flux/123/flux.safetensors"- /.../models/text_encoders/clip_g.safetensors -> "clip_g.safetensors"-- For non-model paths, returns None.- NOTE: this is a temporary helper, used only for initializing metadata["filename"] field.- """- try:- root_category, rel_path = get_relative_to_root_category_path_of_asset(file_path)- except ValueError:- return None-- p = Path(rel_path)- parts = [seg for seg in p.parts if seg not in (".", "..", p.anchor)]- if not parts:- return None-- if root_category == "models":- # parts[0] is the category ("checkpoints", "vae", etc) – drop it- inside = parts[1:] if len(parts) > 1 else [parts[0]]- return "/".join(inside)- return "/".join(parts) # input/output: keep all parts--def get_relative_to_root_category_path_of_asset(file_path: str) -> tuple[Literal["input", "output", "models"], str]:- """Given an absolute or relative file path, determine which root category the path belongs to:- - 'input' if the file resides under `folder_paths.get_input_directory()`- - 'output' if the file resides under `folder_paths.get_output_directory()`- - 'models' if the file resides under any base path of categories returned by `get_comfy_models_folders()`-- Returns:- (root_category, relative_path_inside_that_root)- For 'models', the relative path is prefixed with the category name:- e.g. ('models', 'vae/test/sub/ae.safetensors')-- Raises:- ValueError: if the path does not belong to input, output, or configured model bases.- """- fp_abs = os.path.abspath(file_path)-- def _is_within(child: str, parent: str) -> bool:- try:- return os.path.commonpath([child, parent]) == parent- except Exception:- return False-- def _rel(child: str, parent: str) -> str:- return os.path.relpath(os.path.join(os.sep, os.path.relpath(child, parent)), os.sep)-- # 1) input- input_base = os.path.abspath(folder_paths.get_input_directory())- if _is_within(fp_abs, input_base):- return "input", _rel(fp_abs, input_base)-- # 2) output- output_base = os.path.abspath(folder_paths.get_output_directory())- if _is_within(fp_abs, output_base):- return "output", _rel(fp_abs, output_base)-- # 3) models (check deepest matching base to avoid ambiguity)- best: tuple[int, str, str] | None = None # (base_len, bucket, rel_inside_bucket)- for bucket, bases in get_comfy_models_folders():- for b in bases:- base_abs = os.path.abspath(b)- if not _is_within(fp_abs, base_abs):- continue- cand = (len(base_abs), bucket, _rel(fp_abs, base_abs))- if best is None or cand[0] > best[0]:- best = cand-- if best is not None:- _, bucket, rel_inside = best- combined = os.path.join(bucket, rel_inside)- return "models", os.path.relpath(os.path.join(os.sep, combined), os.sep)-- raise ValueError(f"Path is not within input, output, or configured model bases: {file_path}")--def get_name_and_tags_from_asset_path(file_path: str) -> tuple[str, list[str]]:- """Return a tuple (name, tags) derived from a filesystem path.-- Semantics:- - Root category is determined by `get_relative_to_root_category_path_of_asset`.- - The returned `name` is the base filename with extension from the relative path.- - The returned `tags` are:- [root_category] + parent folders of the relative path (in order)- For 'models', this means:- file '/.../ModelsDir/vae/test_tag/ae.safetensors'- -> root_category='models', some_path='vae/test_tag/ae.safetensors'- -> name='ae.safetensors', tags=['models', 'vae', 'test_tag']-- Raises:- ValueError: if the path does not belong to input, output, or configured model bases.- """- root_category, some_path = get_relative_to_root_category_path_of_asset(file_path)- p = Path(some_path)- parent_parts = [part for part in p.parent.parts if part not in (".", "..", p.anchor)]- return p.name, list(dict.fromkeys(normalize_tags([root_category, *parent_parts])))def normalize_tags(tags: list[str] | None) -> list[str]:"""@@ -228,85 +44,22 @@ def normalize_tags(tags: list[str] | None) -> list[str]:- Stripping whitespace and converting to lowercase.- Removing duplicates."""- return [t.strip().lower() for t in (tags or []) if (t or "").strip()]+ return list(dict.fromkeys(t.strip().lower() for t in (tags or []) if (t or "").strip()))-def collect_models_files() -> list[str]:- out: list[str] = []- for folder_name, bases in get_comfy_models_folders():- rel_files = folder_paths.get_filename_list(folder_name) or []- for rel_path in rel_files:- abs_path = folder_paths.get_full_path(folder_name, rel_path)- if not abs_path:- continue- abs_path = os.path.abspath(abs_path)- allowed = False- for b in bases:- base_abs = os.path.abspath(b)- with contextlib.suppress(Exception):- if os.path.commonpath([abs_path, base_abs]) == base_abs:- allowed = True- break- if allowed:- out.append(abs_path)- return out-def is_scalar(v):- if v is None:- return True- if isinstance(v, bool):- return True- if isinstance(v, (int, float, Decimal, str)):- return True- return False+def validate_blake3_hash(s: str) -> str:+ """Validate and normalize a blake3 hash string.-def project_kv(key: str, value):+ Returns canonical 'blake3:<hex>' or raises ValueError."""- Turn a metadata key/value into typed projection rows.- Returns list[dict] with keys:- key, ordinal, and one of val_str / val_num / val_bool / val_json (others None)- """- rows: list[dict] = []-- def _null_row(ordinal: int) -> dict:- return {- "key": key, "ordinal": ordinal,- "val_str": None, "val_num": None, "val_bool": None, "val_json": None- }-- if value is None:- rows.append(_null_row(0))- return rows-- if is_scalar(value):- if isinstance(value, bool):- rows.append({"key": key, "ordinal": 0, "val_bool": bool(value)})- elif isinstance(value, (int, float, Decimal)):- num = value if isinstance(value, Decimal) else Decimal(str(value))- rows.append({"key": key, "ordinal": 0, "val_num": num})- elif isinstance(value, str):- rows.append({"key": key, "ordinal": 0, "val_str": value})- else:- rows.append({"key": key, "ordinal": 0, "val_json": value})- return rows-- if isinstance(value, list):- if all(is_scalar(x) for x in value):- for i, x in enumerate(value):- if x is None:- rows.append(_null_row(i))- elif isinstance(x, bool):- rows.append({"key": key, "ordinal": i, "val_bool": bool(x)})- elif isinstance(x, (int, float, Decimal)):- num = x if isinstance(x, Decimal) else Decimal(str(x))- rows.append({"key": key, "ordinal": i, "val_num": num})- elif isinstance(x, str):- rows.append({"key": key, "ordinal": i, "val_str": x})- else:- rows.append({"key": key, "ordinal": i, "val_json": x})- return rows- for i, x in enumerate(value):- rows.append({"key": key, "ordinal": i, "val_json": x})- return rows-- rows.append({"key": key, "ordinal": 0, "val_json": value})- return rows+ s = s.strip().lower()+ if not s or ":" not in s:+ raise ValueError("hash must be 'blake3:<hex>'")+ algo, digest = s.split(":", 1)+ if (+ algo != "blake3"+ or len(digest) != 64+ or any(c for c in digest if c not in "0123456789abcdef")+ ):+ raise ValueError("hash must be 'blake3:<hex>'")+ return f"{algo}:{digest}"diff --git a/app/assets/manager.py b/app/assets/manager.pydeleted file mode 100644index a68c8c8ae..000000000--- a/app/assets/manager.py+++ /dev/null@@ -1,516 +0,0 @@-import os-import mimetypes-import contextlib-from typing import Sequence--from app.database.db import create_session-from app.assets.api import schemas_out, schemas_in-from app.assets.database.queries import (- asset_exists_by_hash,- asset_info_exists_for_asset_id,- get_asset_by_hash,- get_asset_info_by_id,- fetch_asset_info_asset_and_tags,- fetch_asset_info_and_asset,- create_asset_info_for_existing_asset,- touch_asset_info_by_id,- update_asset_info_full,- delete_asset_info_by_id,- list_cache_states_by_asset_id,- list_asset_infos_page,- list_tags_with_usage,- get_asset_tags,- add_tags_to_asset_info,- remove_tags_from_asset_info,- pick_best_live_path,- ingest_fs_asset,- set_asset_info_preview,-)-from app.assets.helpers import resolve_destination_from_tags, ensure_within_base-from app.assets.database.models import Asset---def _safe_sort_field(requested: str | None) -> str:- if not requested:- return "created_at"- v = requested.lower()- if v in {"name", "created_at", "updated_at", "size", "last_access_time"}:- return v- return "created_at"---def _get_size_mtime_ns(path: str) -> tuple[int, int]:- st = os.stat(path, follow_symlinks=True)- return st.st_size, getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000))---def _safe_filename(name: str | None, fallback: str) -> str:- n = os.path.basename((name or "").strip() or fallback)- if n:- return n- return fallback---def asset_exists(*, asset_hash: str) -> bool:- """- Check if an asset with a given hash exists in database.- """- with create_session() as session:- return asset_exists_by_hash(session, asset_hash=asset_hash)---def list_assets(- *,- include_tags: Sequence[str] | None = None,- exclude_tags: Sequence[str] | None = None,- name_contains: str | None = None,- metadata_filter: dict | None = None,- limit: int = 20,- offset: int = 0,- sort: str = "created_at",- order: str = "desc",- owner_id: str = "",-) -> schemas_out.AssetsList:- sort = _safe_sort_field(sort)- order = "desc" if (order or "desc").lower() not in {"asc", "desc"} else order.lower()-- with create_session() as session:- infos, tag_map, total = list_asset_infos_page(- session,- owner_id=owner_id,- include_tags=include_tags,- exclude_tags=exclude_tags,- name_contains=name_contains,- metadata_filter=metadata_filter,- limit=limit,- offset=offset,- sort=sort,- order=order,- )-- summaries: list[schemas_out.AssetSummary] = []- for info in infos:- asset = info.asset- tags = tag_map.get(info.id, [])- summaries.append(- schemas_out.AssetSummary(- id=info.id,- name=info.name,- asset_hash=asset.hash if asset else None,- size=int(asset.size_bytes) if asset else None,- mime_type=asset.mime_type if asset else None,- tags=tags,- created_at=info.created_at,- updated_at=info.updated_at,- last_access_time=info.last_access_time,- )- )-- return schemas_out.AssetsList(- assets=summaries,- total=total,- has_more=(offset + len(summaries)) < total,- )---def get_asset(- *,- asset_info_id: str,- owner_id: str = "",-) -> schemas_out.AssetDetail:- with create_session() as session:- res = fetch_asset_info_asset_and_tags(session, asset_info_id=asset_info_id, owner_id=owner_id)- if not res:- raise ValueError(f"AssetInfo {asset_info_id} not found")- info, asset, tag_names = res- preview_id = info.preview_id-- return schemas_out.AssetDetail(- id=info.id,- name=info.name,- asset_hash=asset.hash if asset else None,- size=int(asset.size_bytes) if asset and asset.size_bytes is not None else None,- mime_type=asset.mime_type if asset else None,- tags=tag_names,- user_metadata=info.user_metadata or {},- preview_id=preview_id,- created_at=info.created_at,- last_access_time=info.last_access_time,- )---def resolve_asset_content_for_download(- *,- asset_info_id: str,- owner_id: str = "",-) -> tuple[str, str, str]:- with create_session() as session:- pair = fetch_asset_info_and_asset(session, asset_info_id=asset_info_id, owner_id=owner_id)- if not pair:- raise ValueError(f"AssetInfo {asset_info_id} not found")-- info, asset = pair- states = list_cache_states_by_asset_id(session, asset_id=asset.id)- abs_path = pick_best_live_path(states)- if not abs_path:- raise FileNotFoundError-- touch_asset_info_by_id(session, asset_info_id=asset_info_id)- session.commit()-- ctype = asset.mime_type or mimetypes.guess_type(info.name or abs_path)[0] or "application/octet-stream"- download_name = info.name or os.path.basename(abs_path)- return abs_path, ctype, download_name---def upload_asset_from_temp_path(- spec: schemas_in.UploadAssetSpec,- *,- temp_path: str,- client_filename: str | None = None,- owner_id: str = "",- expected_asset_hash: str | None = None,-) -> schemas_out.AssetCreated:- """- Create new asset or update existing asset from a temporary file path.- """- try:- # NOTE: blake3 is not required right now, so this will fail if blake3 is not installed in local environment- import app.assets.hashing as hashing- digest = hashing.blake3_hash(temp_path)- except Exception as e:- raise RuntimeError(f"failed to hash uploaded file: {e}")- asset_hash = "blake3:" + digest-- if expected_asset_hash and asset_hash != expected_asset_hash.strip().lower():- raise ValueError("HASH_MISMATCH")-- with create_session() as session:- existing = get_asset_by_hash(session, asset_hash=asset_hash)- if existing is not None:- with contextlib.suppress(Exception):- if temp_path and os.path.exists(temp_path):- os.remove(temp_path)-- display_name = _safe_filename(spec.name or (client_filename or ""), fallback=digest)- info = create_asset_info_for_existing_asset(- session,- asset_hash=asset_hash,- name=display_name,- user_metadata=spec.user_metadata or {},- tags=spec.tags or [],- tag_origin="manual",- owner_id=owner_id,- )- tag_names = get_asset_tags(session, asset_info_id=info.id)- session.commit()-- return schemas_out.AssetCreated(- id=info.id,- name=info.name,- asset_hash=existing.hash,- size=int(existing.size_bytes) if existing.size_bytes is not None else None,- mime_type=existing.mime_type,- tags=tag_names,- user_metadata=info.user_metadata or {},- preview_id=info.preview_id,- created_at=info.created_at,- last_access_time=info.last_access_time,- created_new=False,- )-- base_dir, subdirs = resolve_destination_from_tags(spec.tags)- dest_dir = os.path.join(base_dir, *subdirs) if subdirs else base_dir- os.makedirs(dest_dir, exist_ok=True)-- src_for_ext = (client_filename or spec.name or "").strip()- _ext = os.path.splitext(os.path.basename(src_for_ext))[1] if src_for_ext else ""- ext = _ext if 0 < len(_ext) <= 16 else ""- hashed_basename = f"{digest}{ext}"- dest_abs = os.path.abspath(os.path.join(dest_dir, hashed_basename))- ensure_within_base(dest_abs, base_dir)-- content_type = (- mimetypes.guess_type(os.path.basename(src_for_ext), strict=False)[0]- or mimetypes.guess_type(hashed_basename, strict=False)[0]- or "application/octet-stream"- )-- try:- os.replace(temp_path, dest_abs)- except Exception as e:- raise RuntimeError(f"failed to move uploaded file into place: {e}")-- try:- size_bytes, mtime_ns = _get_size_mtime_ns(dest_abs)- except OSError as e:- raise RuntimeError(f"failed to stat destination file: {e}")-- with create_session() as session:- result = ingest_fs_asset(- session,- asset_hash=asset_hash,- abs_path=dest_abs,- size_bytes=size_bytes,- mtime_ns=mtime_ns,- mime_type=content_type,- info_name=_safe_filename(spec.name or (client_filename or ""), fallback=digest),- owner_id=owner_id,- preview_id=None,- user_metadata=spec.user_metadata or {},- tags=spec.tags,- tag_origin="manual",- require_existing_tags=False,- )- info_id = result["asset_info_id"]- if not info_id:- raise RuntimeError("failed to create asset metadata")-- pair = fetch_asset_info_and_asset(session, asset_info_id=info_id, owner_id=owner_id)- if not pair:- raise RuntimeError("inconsistent DB state after ingest")- info, asset = pair- tag_names = get_asset_tags(session, asset_info_id=info.id)- created_result = schemas_out.AssetCreated(- id=info.id,- name=info.name,- asset_hash=asset.hash,- size=int(asset.size_bytes),- mime_type=asset.mime_type,- tags=tag_names,- user_metadata=info.user_metadata or {},- preview_id=info.preview_id,- created_at=info.created_at,- last_access_time=info.last_access_time,- created_new=result["asset_created"],- )- session.commit()-- return created_result---def update_asset(- *,- asset_info_id: str,- name: str | None = None,- tags: list[str] | None = None,- user_metadata: dict | None = None,- owner_id: str = "",-) -> schemas_out.AssetUpdated:- with create_session() as session:- info_row = get_asset_info_by_id(session, asset_info_id=asset_info_id)- if not info_row:- raise ValueError(f"AssetInfo {asset_info_id} not found")- if info_row.owner_id and info_row.owner_id != owner_id:- raise PermissionError("not owner")-- info = update_asset_info_full(- session,- asset_info_id=asset_info_id,- name=name,- tags=tags,- user_metadata=user_metadata,- tag_origin="manual",- asset_info_row=info_row,- )-- tag_names = get_asset_tags(session, asset_info_id=asset_info_id)- result = schemas_out.AssetUpdated(- id=info.id,- name=info.name,- asset_hash=info.asset.hash if info.asset else None,- tags=tag_names,- user_metadata=info.user_metadata or {},- updated_at=info.updated_at,- )- session.commit()-- return result---def set_asset_preview(- *,- asset_info_id: str,- preview_asset_id: str | None = None,- owner_id: str = "",-) -> schemas_out.AssetDetail:- with create_session() as session:- info_row = get_asset_info_by_id(session, asset_info_id=asset_info_id)- if not info_row:- raise ValueError(f"AssetInfo {asset_info_id} not found")- if info_row.owner_id and info_row.owner_id != owner_id:- raise PermissionError("not owner")-- set_asset_info_preview(- session,- asset_info_id=asset_info_id,- preview_asset_id=preview_asset_id,- )-- res = fetch_asset_info_asset_and_tags(session, asset_info_id=asset_info_id, owner_id=owner_id)- if not res:- raise RuntimeError("State changed during preview update")- info, asset, tags = res- result = schemas_out.AssetDetail(- id=info.id,- name=info.name,- asset_hash=asset.hash if asset else None,- size=int(asset.size_bytes) if asset and asset.size_bytes is not None else None,- mime_type=asset.mime_type if asset else None,- tags=tags,- user_metadata=info.user_metadata or {},- preview_id=info.preview_id,- created_at=info.created_at,- last_access_time=info.last_access_time,- )- session.commit()-- return result---def delete_asset_reference(*, asset_info_id: str, owner_id: str, delete_content_if_orphan: bool = True) -> bool:- with create_session() as session:- info_row = get_asset_info_by_id(session, asset_info_id=asset_info_id)- asset_id = info_row.asset_id if info_row else None- deleted = delete_asset_info_by_id(session, asset_info_id=asset_info_id, owner_id=owner_id)- if not deleted:- session.commit()- return False-- if not delete_content_if_orphan or not asset_id:- session.commit()- return True-- still_exists = asset_info_exists_for_asset_id(session, asset_id=asset_id)- if still_exists:- session.commit()- return True-- states = list_cache_states_by_asset_id(session, asset_id=asset_id)- file_paths = [s.file_path for s in (states or []) if getattr(s, "file_path", None)]-- asset_row = session.get(Asset, asset_id)- if asset_row is not None:- session.delete(asset_row)-- session.commit()- for p in file_paths:- with contextlib.suppress(Exception):- if p and os.path.isfile(p):- os.remove(p)- return True---def create_asset_from_hash(- *,- hash_str: str,- name: str,- tags: list[str] | None = None,- user_metadata: dict | None = None,- owner_id: str = "",-) -> schemas_out.AssetCreated | None:- canonical = hash_str.strip().lower()- with create_session() as session:- asset = get_asset_by_hash(session, asset_hash=canonical)- if not asset:- return None-- info = create_asset_info_for_existing_asset(- session,- asset_hash=canonical,- name=_safe_filename(name, fallback=canonical.split(":", 1)[1]),- user_metadata=user_metadata or {},- tags=tags or [],- tag_origin="manual",- owner_id=owner_id,- )- tag_names = get_asset_tags(session, asset_info_id=info.id)- result = schemas_out.AssetCreated(- id=info.id,- name=info.name,- asset_hash=asset.hash,- size=int(asset.size_bytes),- mime_type=asset.mime_type,- tags=tag_names,- user_metadata=info.user_metadata or {},- preview_id=info.preview_id,- created_at=info.created_at,- last_access_time=info.last_access_time,- created_new=False,- )- session.commit()-- return result---def add_tags_to_asset(- *,- asset_info_id: str,- tags: list[str],- origin: str = "manual",- owner_id: str = "",-) -> schemas_out.TagsAdd:- with create_session() as session:- info_row = get_asset_info_by_id(session, asset_info_id=asset_info_id)- if not info_row:- raise ValueError(f"AssetInfo {asset_info_id} not found")- if info_row.owner_id and info_row.owner_id != owner_id:- raise PermissionError("not owner")- data = add_tags_to_asset_info(- session,- asset_info_id=asset_info_id,- tags=tags,- origin=origin,- create_if_missing=True,- asset_info_row=info_row,- )- session.commit()- return schemas_out.TagsAdd(**data)---def remove_tags_from_asset(- *,- asset_info_id: str,- tags: list[str],- owner_id: str = "",-) -> schemas_out.TagsRemove:- with create_session() as session:- info_row = get_asset_info_by_id(session, asset_info_id=asset_info_id)- if not info_row:- raise ValueError(f"AssetInfo {asset_info_id} not found")- if info_row.owner_id and info_row.owner_id != owner_id:- raise PermissionError("not owner")-- data = remove_tags_from_asset_info(- session,- asset_info_id=asset_info_id,- tags=tags,- )- session.commit()- return schemas_out.TagsRemove(**data)---def list_tags(- prefix: str | None = None,- limit: int = 100,- offset: int = 0,- order: str = "count_desc",- include_zero: bool = True,- owner_id: str = "",-) -> schemas_out.TagsList:- limit = max(1, min(1000, limit))- offset = max(0, offset)-- with create_session() as session:- rows, total = list_tags_with_usage(- session,- prefix=prefix,- limit=limit,- offset=offset,- include_zero=include_zero,- order=order,- owner_id=owner_id,- )-- tags = [schemas_out.TagUsage(name=name, count=count, type=tag_type) for (name, tag_type, count) in rows]- return schemas_out.TagsList(tags=tags, total=total, has_more=(offset + len(tags)) < total)diff --git a/app/assets/scanner.py b/app/assets/scanner.pyindex 0172a5c2f..e27ea5123 100644--- a/app/assets/scanner.py+++ b/app/assets/scanner.py@@ -1,263 +1,567 @@-import contextlib-import timeimport loggingimport os-import sqlalchemy+from pathlib import Path+from typing import Callable, Literal, TypedDictimport folder_paths-from app.database.db import create_session, dependencies_available-from app.assets.helpers import (- collect_models_files, compute_relative_filename, fast_asset_file_check, get_name_and_tags_from_asset_path,- list_tree,prefixes_for_root, escape_like_prefix,- RootType+from app.assets.database.queries import (+ add_missing_tag_for_asset_id,+ bulk_update_enrichment_level,+ bulk_update_is_missing,+ bulk_update_needs_verify,+ delete_orphaned_seed_asset,+ delete_references_by_ids,+ ensure_tags_exist,+ get_asset_by_hash,+ get_references_for_prefixes,+ get_unenriched_references,+ mark_references_missing_outside_prefixes,+ reassign_asset_references,+ remove_missing_tag_for_asset_id,+ set_reference_metadata,+ update_asset_hash_and_mime,)-from app.assets.database.tags import add_missing_tag_for_asset_id, ensure_tags_exist, remove_missing_tag_for_asset_id-from app.assets.database.bulk_ops import seed_from_paths_batch-from app.assets.database.models import Asset, AssetCacheState, AssetInfo+from app.assets.services.bulk_ingest import (+ SeedAssetSpec,+ batch_insert_seed_assets,+)+from app.assets.services.file_utils import (+ get_mtime_ns,+ is_visible,+ list_files_recursively,+ verify_file_unchanged,+)+from app.assets.services.hashing import HashCheckpoint, compute_blake3_hash+from app.assets.services.metadata_extract import extract_file_metadata+from app.assets.services.path_utils import (+ compute_relative_filename,+ get_comfy_models_folders,+ get_name_and_tags_from_asset_path,+)+from app.database.db import create_session-def seed_assets(roots: tuple[RootType, ...], enable_logging: bool = False) -> None:- """- Scan the given roots and seed the assets into the database.+class _RefInfo(TypedDict):+ ref_id: str+ file_path: str+ exists: bool+ stat_unchanged: bool+ needs_verify: bool+++class _AssetAccumulator(TypedDict):+ hash: str | None+ size_db: int+ refs: list[_RefInfo]+++RootType = Literal["models", "input", "output"]+++def get_prefixes_for_root(root: RootType) -> list[str]:+ if root == "models":+ bases: list[str] = []+ for _bucket, paths in get_comfy_models_folders():+ bases.extend(paths)+ return [os.path.abspath(p) for p in bases]+ if root == "input":+ return [os.path.abspath(folder_paths.get_input_directory())]+ if root == "output":+ return [os.path.abspath(folder_paths.get_output_directory())]+ return []+++def get_all_known_prefixes() -> list[str]:+ """Get all known asset prefixes across all root types."""+ all_roots: tuple[RootType, ...] = ("models", "input", "output")+ return [p for root in all_roots for p in get_prefixes_for_root(root)]+++def collect_models_files() -> list[str]:+ out: list[str] = []+ for folder_name, bases in get_comfy_models_folders():+ rel_files = folder_paths.get_filename_list(folder_name) or []+ for rel_path in rel_files:+ if not all(is_visible(part) for part in Path(rel_path).parts):+ continue+ abs_path = folder_paths.get_full_path(folder_name, rel_path)+ if not abs_path:+ continue+ abs_path = os.path.abspath(abs_path)+ allowed = False+ abs_p = Path(abs_path)+ for b in bases:+ if abs_p.is_relative_to(os.path.abspath(b)):+ allowed = True+ break+ if allowed:+ out.append(abs_path)+ return out+++def sync_references_with_filesystem(+ session,+ root: RootType,+ collect_existing_paths: bool = False,+ update_missing_tags: bool = False,+) -> set[str] | None:+ """Reconcile asset references with filesystem for a root.++ - Toggle needs_verify per reference using mtime/size stat check+ - For hashed assets with at least one stat-unchanged ref: delete stale missing refs+ - For seed assets with all refs missing: delete Asset and its references+ - Optionally add/remove 'missing' tags based on stat check in this root+ - Optionally return surviving absolute paths++ Args:+ session: Database session+ root: Root type to scan+ collect_existing_paths: If True, return set of surviving file paths+ update_missing_tags: If True, update 'missing' tags based on file status++ Returns:+ Set of surviving absolute paths if collect_existing_paths=True, else None"""- if not dependencies_available():- if enable_logging:- logging.warning("Database dependencies not available, skipping assets scan")- return- t_start = time.perf_counter()- created = 0- skipped_existing = 0- orphans_pruned = 0- paths: list[str] = []- try:- existing_paths: set[str] = set()- for r in roots:- try:- survivors: set[str] = _fast_db_consistency_pass(r, collect_existing_paths=True, update_missing_tags=True)- if survivors:- existing_paths.update(survivors)- except Exception as e:- logging.exception("fast DB scan failed for %s: %s", r, e)+ prefixes = get_prefixes_for_root(root)+ if not prefixes:+ return set() if collect_existing_paths else None+ rows = get_references_for_prefixes(+ session, prefixes, include_missing=update_missing_tags+ )++ by_asset: dict[str, _AssetAccumulator] = {}+ for row in rows:+ acc = by_asset.get(row.asset_id)+ if acc is None:+ acc = {"hash": row.asset_hash, "size_db": row.size_bytes, "refs": []}+ by_asset[row.asset_id] = acc++ stat_unchanged = Falsetry:- orphans_pruned = _prune_orphaned_assets(roots)- except Exception as e:- logging.exception("orphan pruning failed: %s", e)-- if "models" in roots:- paths.extend(collect_models_files())- if "input" in roots:- paths.extend(list_tree(folder_paths.get_input_directory()))- if "output" in roots:- paths.extend(list_tree(folder_paths.get_output_directory()))-- specs: list[dict] = []- tag_pool: set[str] = set()- for p in paths:- abs_p = os.path.abspath(p)- if abs_p in existing_paths:- skipped_existing += 1+ exists = True+ stat_unchanged = verify_file_unchanged(+ mtime_db=row.mtime_ns,+ size_db=acc["size_db"],+ stat_result=os.stat(row.file_path, follow_symlinks=True),+ )+ except FileNotFoundError:+ exists = False+ except PermissionError:+ exists = True+ logging.debug("Permission denied accessing %s", row.file_path)+ except OSError as e:+ exists = False+ logging.debug("OSError checking %s: %s", row.file_path, e)++ acc["refs"].append(+ {+ "ref_id": row.reference_id,+ "file_path": row.file_path,+ "exists": exists,+ "stat_unchanged": stat_unchanged,+ "needs_verify": row.needs_verify,+ }+ )++ to_set_verify: list[str] = []+ to_clear_verify: list[str] = []+ stale_ref_ids: list[str] = []+ to_mark_missing: list[str] = []+ to_clear_missing: list[str] = []+ survivors: set[str] = set()++ for aid, acc in by_asset.items():+ a_hash = acc["hash"]+ refs = acc["refs"]+ any_unchanged = any(r["stat_unchanged"] for r in refs)+ all_missing = all(not r["exists"] for r in refs)++ for r in refs:+ if not r["exists"]:+ to_mark_missing.append(r["ref_id"])continue+ if r["stat_unchanged"]:+ to_clear_missing.append(r["ref_id"])+ if r["needs_verify"]:+ to_clear_verify.append(r["ref_id"])+ if not r["stat_unchanged"] and not r["needs_verify"]:+ to_set_verify.append(r["ref_id"])++ if a_hash is None:+ if refs and all_missing:+ delete_orphaned_seed_asset(session, aid)+ else:+ for r in refs:+ if r["exists"]:+ survivors.add(os.path.abspath(r["file_path"]))+ continue++ if any_unchanged:+ for r in refs:+ if not r["exists"]:+ stale_ref_ids.append(r["ref_id"])+ if update_missing_tags:+ try:+ remove_missing_tag_for_asset_id(session, asset_id=aid)+ except Exception as e:+ logging.warning(+ "Failed to remove missing tag for asset %s: %s", aid, e+ )+ elif update_missing_tags:try:- stat_p = os.stat(abs_p, follow_symlinks=False)- except OSError:- continue- # skip empty files- if not stat_p.st_size:- continue- name, tags = get_name_and_tags_from_asset_path(abs_p)- specs.append(- {- "abs_path": abs_p,- "size_bytes": stat_p.st_size,- "mtime_ns": getattr(stat_p, "st_mtime_ns", int(stat_p.st_mtime * 1_000_000_000)),- "info_name": name,- "tags": tags,- "fname": compute_relative_filename(abs_p),- }- )- for t in tags:- tag_pool.add(t)- # if no file specs, nothing to do- if not specs:- return+ add_missing_tag_for_asset_id(session, asset_id=aid, origin="automatic")+ except Exception as e:+ logging.warning("Failed to add missing tag for asset %s: %s", aid, e)++ for r in refs:+ if r["exists"]:+ survivors.add(os.path.abspath(r["file_path"]))++ delete_references_by_ids(session, stale_ref_ids)+ stale_set = set(stale_ref_ids)+ to_mark_missing = [ref_id for ref_id in to_mark_missing if ref_id not in stale_set]+ bulk_update_is_missing(session, to_mark_missing, value=True)+ bulk_update_is_missing(session, to_clear_missing, value=False)+ bulk_update_needs_verify(session, to_set_verify, value=True)+ bulk_update_needs_verify(session, to_clear_verify, value=False)++ return survivors if collect_existing_paths else None+++def sync_root_safely(root: RootType) -> set[str]:+ """Sync a single root's references with the filesystem.++ Returns survivors (existing paths) or empty set on failure.+ """+ try:with create_session() as sess:- if tag_pool:- ensure_tags_exist(sess, tag_pool, tag_type="user")+ survivors = sync_references_with_filesystem(+ sess,+ root,+ collect_existing_paths=True,+ update_missing_tags=True,+ )+ sess.commit()+ return survivors or set()+ except Exception as e:+ logging.exception("fast DB scan failed for %s: %s", root, e)+ return set()+++def mark_missing_outside_prefixes_safely(prefixes: list[str]) -> int:+ """Mark references as missing when outside the given prefixes.- result = seed_from_paths_batch(sess, specs=specs, owner_id="")- created += result["inserted_infos"]+ This is a non-destructive soft-delete. Returns count marked or 0 on failure.+ """+ try:+ with create_session() as sess:+ count = mark_references_missing_outside_prefixes(sess, prefixes)sess.commit()- finally:- if enable_logging:- logging.info(- "Assets scan(roots=%s) completed in %.3fs (created=%d, skipped_existing=%d, orphans_pruned=%d, total_seen=%d)",- roots,- time.perf_counter() - t_start,- created,- skipped_existing,- orphans_pruned,- len(paths),+ return count+ except Exception as e:+ logging.exception("marking missing assets failed: %s", e)+ return 0+++def collect_paths_for_roots(roots: tuple[RootType, ...]) -> list[str]:+ """Collect all file paths for the given roots."""+ paths: list[str] = []+ if "models" in roots:+ paths.extend(collect_models_files())+ if "input" in roots:+ paths.extend(list_files_recursively(folder_paths.get_input_directory()))+ if "output" in roots:+ paths.extend(list_files_recursively(folder_paths.get_output_directory()))+ return paths+++def build_asset_specs(+ paths: list[str],+ existing_paths: set[str],+ enable_metadata_extraction: bool = True,+ compute_hashes: bool = False,+) -> tuple[list[SeedAssetSpec], set[str], int]:+ """Build asset specs from paths, returning (specs, tag_pool, skipped_count).++ Args:+ paths: List of file paths to process+ existing_paths: Set of paths that already exist in the database+ enable_metadata_extraction: If True, extract tier 1 & 2 metadata+ compute_hashes: If True, compute blake3 hashes (slow for large files)+ """+ specs: list[SeedAssetSpec] = []+ tag_pool: set[str] = set()+ skipped = 0++ for p in paths:+ abs_p = os.path.abspath(p)+ if abs_p in existing_paths:+ skipped += 1+ continue+ try:+ stat_p = os.stat(abs_p, follow_symlinks=True)+ except OSError:+ continue+ if not stat_p.st_size:+ continue+ name, tags = get_name_and_tags_from_asset_path(abs_p)+ rel_fname = compute_relative_filename(abs_p)++ # Extract metadata (tier 1: filesystem, tier 2: safetensors header)+ metadata = None+ if enable_metadata_extraction:+ metadata = extract_file_metadata(+ abs_p,+ stat_result=stat_p,+ relative_filename=rel_fname,)+ # Compute hash if requested+ asset_hash: str | None = None+ if compute_hashes:+ try:+ digest, _ = compute_blake3_hash(abs_p)+ asset_hash = "blake3:" + digest+ except Exception as e:+ logging.warning("Failed to hash %s: %s", abs_p, e)-def _prune_orphaned_assets(roots: tuple[RootType, ...]) -> int:- """Prune cache states outside configured prefixes, then delete orphaned seed assets."""- all_prefixes = [os.path.abspath(p) for r in roots for p in prefixes_for_root(r)]- if not all_prefixes:- return 0+ mime_type = metadata.content_type if metadata else None+ specs.append(+ {+ "abs_path": abs_p,+ "size_bytes": stat_p.st_size,+ "mtime_ns": get_mtime_ns(stat_p),+ "info_name": name,+ "tags": tags,+ "fname": rel_fname,+ "metadata": metadata,+ "hash": asset_hash,+ "mime_type": mime_type,+ }+ )+ tag_pool.update(tags)- def make_prefix_condition(prefix: str):- base = prefix if prefix.endswith(os.sep) else prefix + os.sep- escaped, esc = escape_like_prefix(base)- return AssetCacheState.file_path.like(escaped + "%", escape=esc)+ return specs, tag_pool, skipped- matches_valid_prefix = sqlalchemy.or_(*[make_prefix_condition(p) for p in all_prefixes])- orphan_subq = (- sqlalchemy.select(Asset.id)- .outerjoin(AssetCacheState, AssetCacheState.asset_id == Asset.id)- .where(Asset.hash.is_(None), AssetCacheState.id.is_(None))- ).scalar_subquery()+def insert_asset_specs(specs: list[SeedAssetSpec], tag_pool: set[str]) -> int:+ """Insert asset specs into database, returning count of created refs."""+ if not specs:+ return 0with create_session() as sess:- sess.execute(sqlalchemy.delete(AssetCacheState).where(~matches_valid_prefix))- sess.execute(sqlalchemy.delete(AssetInfo).where(AssetInfo.asset_id.in_(orphan_subq)))- result = sess.execute(sqlalchemy.delete(Asset).where(Asset.id.in_(orphan_subq)))+ if tag_pool:+ ensure_tags_exist(sess, tag_pool, tag_type="user")+ result = batch_insert_seed_assets(sess, specs=specs, owner_id="")sess.commit()- return result.rowcount+ return result.inserted_refs-def _fast_db_consistency_pass(- root: RootType,- *,- collect_existing_paths: bool = False,- update_missing_tags: bool = False,-) -> set[str] | None:- """Fast DB+FS pass for a root:- - Toggle needs_verify per state using fast check- - For hashed assets with at least one fast-ok state in this root: delete stale missing states- - For seed assets with all states missing: delete Asset and its AssetInfos- - Optionally add/remove 'missing' tags based on fast-ok in this root- - Optionally return surviving absolute paths+# Enrichment level constants+ENRICHMENT_STUB = 0 # Fast scan: path, size, mtime only+ENRICHMENT_METADATA = 1 # Metadata extracted (safetensors header, mime type)+ENRICHMENT_HASHED = 2 # Hash computed (blake3)+++def get_unenriched_assets_for_roots(+ roots: tuple[RootType, ...],+ max_level: int = ENRICHMENT_STUB,+ limit: int = 1000,+) -> list:+ """Get assets that need enrichment for the given roots.++ Args:+ roots: Tuple of root types to scan+ max_level: Maximum enrichment level to include+ limit: Maximum number of rows to return++ Returns:+ List of UnenrichedReferenceRow"""- prefixes = prefixes_for_root(root)- if not prefixes:- return set() if collect_existing_paths else None+ prefixes: list[str] = []+ for root in roots:+ prefixes.extend(get_prefixes_for_root(root))- conds = []- for p in prefixes:- base = os.path.abspath(p)- if not base.endswith(os.sep):- base += os.sep- escaped, esc = escape_like_prefix(base)- conds.append(AssetCacheState.file_path.like(escaped + "%", escape=esc))+ if not prefixes:+ return []with create_session() as sess:- rows = (- sess.execute(- sqlalchemy.select(- AssetCacheState.id,- AssetCacheState.file_path,- AssetCacheState.mtime_ns,- AssetCacheState.needs_verify,- AssetCacheState.asset_id,- Asset.hash,- Asset.size_bytes,- )- .join(Asset, Asset.id == AssetCacheState.asset_id)- .where(sqlalchemy.or_(*conds))- .order_by(AssetCacheState.asset_id.asc(), AssetCacheState.id.asc())+ return get_unenriched_references(+ sess, prefixes, max_level=max_level, limit=limit+ )+++def enrich_asset(+ session,+ file_path: str,+ reference_id: str,+ asset_id: str,+ extract_metadata: bool = True,+ compute_hash: bool = False,+ interrupt_check: Callable[[], bool] | None = None,+ hash_checkpoints: dict[str, HashCheckpoint] | None = None,+) -> int:+ """Enrich a single asset with metadata and/or hash.++ Args:+ session: Database session (caller manages lifecycle)+ file_path: Absolute path to the file+ reference_id: ID of the reference to update+ asset_id: ID of the asset to update (for mime_type and hash)+ extract_metadata: If True, extract safetensors header and mime type+ compute_hash: If True, compute blake3 hash+ interrupt_check: Optional non-blocking callable that returns True if+ the operation should be interrupted (e.g. paused or cancelled)+ hash_checkpoints: Optional dict for saving/restoring hash progress+ across interruptions, keyed by file path++ Returns:+ New enrichment level achieved+ """+ new_level = ENRICHMENT_STUB++ try:+ stat_p = os.stat(file_path, follow_symlinks=True)+ except OSError:+ return new_level++ rel_fname = compute_relative_filename(file_path)+ mime_type: str | None = None+ metadata = None++ if extract_metadata:+ metadata = extract_file_metadata(+ file_path,+ stat_result=stat_p,+ relative_filename=rel_fname,+ )+ if metadata:+ mime_type = metadata.content_type+ new_level = ENRICHMENT_METADATA++ full_hash: str | None = None+ if compute_hash:+ try:+ mtime_before = get_mtime_ns(stat_p)+ size_before = stat_p.st_size++ # Restore checkpoint if available and file unchanged+ checkpoint = None+ if hash_checkpoints is not None:+ checkpoint = hash_checkpoints.get(file_path)+ if checkpoint is not None:+ cur_stat = os.stat(file_path, follow_symlinks=True)+ if (checkpoint.mtime_ns != get_mtime_ns(cur_stat)+ or checkpoint.file_size != cur_stat.st_size):+ checkpoint = None+ hash_checkpoints.pop(file_path, None)+ else:+ mtime_before = get_mtime_ns(cur_stat)++ digest, new_checkpoint = compute_blake3_hash(+ file_path,+ interrupt_check=interrupt_check,+ checkpoint=checkpoint,)- ).all()- by_asset: dict[str, dict] = {}- for sid, fp, mtime_db, needs_verify, aid, a_hash, a_size in rows:- acc = by_asset.get(aid)- if acc is None:- acc = {"hash": a_hash, "size_db": int(a_size or 0), "states": []}- by_asset[aid] = acc+ if digest is None:+ # Interrupted — save checkpoint for later resumption+ if hash_checkpoints is not None and new_checkpoint is not None:+ new_checkpoint.mtime_ns = mtime_before+ new_checkpoint.file_size = size_before+ hash_checkpoints[file_path] = new_checkpoint+ return new_level++ # Completed — clear any saved checkpoint+ if hash_checkpoints is not None:+ hash_checkpoints.pop(file_path, None)++ stat_after = os.stat(file_path, follow_symlinks=True)+ mtime_after = get_mtime_ns(stat_after)+ if mtime_before != mtime_after:+ logging.warning("File modified during hashing, discarding hash: %s", file_path)+ else:+ full_hash = f"blake3:{digest}"+ metadata_ok = not extract_metadata or metadata is not None+ if metadata_ok:+ new_level = ENRICHMENT_HASHED+ except Exception as e:+ logging.warning("Failed to hash %s: %s", file_path, e)++ if extract_metadata and metadata:+ user_metadata = metadata.to_user_metadata()+ set_reference_metadata(session, reference_id, user_metadata)++ if full_hash:+ existing = get_asset_by_hash(session, full_hash)+ if existing and existing.id != asset_id:+ reassign_asset_references(session, asset_id, existing.id, reference_id)+ delete_orphaned_seed_asset(session, asset_id)+ if mime_type:+ update_asset_hash_and_mime(session, existing.id, mime_type=mime_type)+ else:+ update_asset_hash_and_mime(session, asset_id, full_hash, mime_type)+ elif mime_type:+ update_asset_hash_and_mime(session, asset_id, mime_type=mime_type)++ bulk_update_enrichment_level(session, [reference_id], new_level)+ session.commit()++ return new_level+++def enrich_assets_batch(+ rows: list,+ extract_metadata: bool = True,+ compute_hash: bool = False,+ interrupt_check: Callable[[], bool] | None = None,+ hash_checkpoints: dict[str, HashCheckpoint] | None = None,+) -> tuple[int, list[str]]:+ """Enrich a batch of assets.++ Uses a single DB session for the entire batch, committing after each+ individual asset to avoid long-held transactions while eliminating+ per-asset session creation overhead.++ Args:+ rows: List of UnenrichedReferenceRow from get_unenriched_assets_for_roots+ extract_metadata: If True, extract metadata for each asset+ compute_hash: If True, compute hash for each asset+ interrupt_check: Optional non-blocking callable that returns True if+ the operation should be interrupted (e.g. paused or cancelled)+ hash_checkpoints: Optional dict for saving/restoring hash progress+ across interruptions, keyed by file path++ Returns:+ Tuple of (enriched_count, failed_reference_ids)+ """+ enriched = 0+ failed_ids: list[str] = []++ with create_session() as sess:+ for row in rows:+ if interrupt_check is not None and interrupt_check():+ break- fast_ok = Falsetry:- exists = True- fast_ok = fast_asset_file_check(- mtime_db=mtime_db,- size_db=acc["size_db"],- stat_result=os.stat(fp, follow_symlinks=True),+ new_level = enrich_asset(+ sess,+ file_path=row.file_path,+ reference_id=row.reference_id,+ asset_id=row.asset_id,+ extract_metadata=extract_metadata,+ compute_hash=compute_hash,+ interrupt_check=interrupt_check,+ hash_checkpoints=hash_checkpoints,)- except FileNotFoundError:- exists = False- except OSError:- exists = False-- acc["states"].append({- "sid": sid,- "fp": fp,- "exists": exists,- "fast_ok": fast_ok,- "needs_verify": bool(needs_verify),- })-- to_set_verify: list[int] = []- to_clear_verify: list[int] = []- stale_state_ids: list[int] = []- survivors: set[str] = set()-- for aid, acc in by_asset.items():- a_hash = acc["hash"]- states = acc["states"]- any_fast_ok = any(s["fast_ok"] for s in states)- all_missing = all(not s["exists"] for s in states)-- for s in states:- if not s["exists"]:- continue- if s["fast_ok"] and s["needs_verify"]:- to_clear_verify.append(s["sid"])- if not s["fast_ok"] and not s["needs_verify"]:- to_set_verify.append(s["sid"])-- if a_hash is None:- if states and all_missing: # remove seed Asset completely, if no valid AssetCache exists- sess.execute(sqlalchemy.delete(AssetInfo).where(AssetInfo.asset_id == aid))- asset = sess.get(Asset, aid)- if asset:- sess.delete(asset)+ if new_level > row.enrichment_level:+ enriched += 1else:- for s in states:- if s["exists"]:- survivors.add(os.path.abspath(s["fp"]))- continue+ failed_ids.append(row.reference_id)+ except Exception as e:+ logging.warning("Failed to enrich %s: %s", row.file_path, e)+ sess.rollback()+ failed_ids.append(row.reference_id)- if any_fast_ok: # if Asset has at least one valid AssetCache record, remove any invalid AssetCache records- for s in states:- if not s["exists"]:- stale_state_ids.append(s["sid"])- if update_missing_tags:- with contextlib.suppress(Exception):- remove_missing_tag_for_asset_id(sess, asset_id=aid)- elif update_missing_tags:- with contextlib.suppress(Exception):- add_missing_tag_for_asset_id(sess, asset_id=aid, origin="automatic")-- for s in states:- if s["exists"]:- survivors.add(os.path.abspath(s["fp"]))-- if stale_state_ids:- sess.execute(sqlalchemy.delete(AssetCacheState).where(AssetCacheState.id.in_(stale_state_ids)))- if to_set_verify:- sess.execute(- sqlalchemy.update(AssetCacheState)- .where(AssetCacheState.id.in_(to_set_verify))- .values(needs_verify=True)- )- if to_clear_verify:- sess.execute(- sqlalchemy.update(AssetCacheState)- .where(AssetCacheState.id.in_(to_clear_verify))- .values(needs_verify=False)- )- sess.commit()- return survivors if collect_existing_paths else None+ return enriched, failed_idsdiff --git a/app/assets/seeder.py b/app/assets/seeder.pynew file mode 100644index 000000000..029448464--- /dev/null+++ b/app/assets/seeder.py@@ -0,0 +1,794 @@+"""Background asset seeder with thread management and cancellation support."""++import logging+import os+import threading+import time+from dataclasses import dataclass, field+from enum import Enum+from typing import Callable++from app.assets.scanner import (+ ENRICHMENT_METADATA,+ ENRICHMENT_STUB,+ RootType,+ build_asset_specs,+ collect_paths_for_roots,+ enrich_assets_batch,+ get_all_known_prefixes,+ get_prefixes_for_root,+ get_unenriched_assets_for_roots,+ insert_asset_specs,+ mark_missing_outside_prefixes_safely,+ sync_root_safely,+)+from app.database.db import dependencies_available+++class ScanInProgressError(Exception):+ """Raised when an operation cannot proceed because a scan is running."""+++class State(Enum):+ """Seeder state machine states."""++ IDLE = "IDLE"+ RUNNING = "RUNNING"+ PAUSED = "PAUSED"+ CANCELLING = "CANCELLING"+++class ScanPhase(Enum):+ """Scan phase options."""++ FAST = "fast" # Phase 1: filesystem only (stubs)+ ENRICH = "enrich" # Phase 2: metadata + hash+ FULL = "full" # Both phases sequentially+++@dataclass+class Progress:+ """Progress information for a scan operation."""++ scanned: int = 0+ total: int = 0+ created: int = 0+ skipped: int = 0+++@dataclass+class ScanStatus:+ """Current status of the asset seeder."""++ state: State+ progress: Progress | None+ errors: list[str] = field(default_factory=list)+++ProgressCallback = Callable[[Progress], None]+++class _AssetSeeder:+ """Background asset scanning manager.++ Spawns ephemeral daemon threads for scanning.+ Each scan creates a new thread that exits when complete.+ Use the module-level ``asset_seeder`` instance.+ """++ def __init__(self) -> None:+ self._lock = threading.Lock()+ self._state = State.IDLE+ self._progress: Progress | None = None+ self._last_progress: Progress | None = None+ self._errors: list[str] = []+ self._thread: threading.Thread | None = None+ self._cancel_event = threading.Event()+ self._run_gate = threading.Event()+ self._run_gate.set() # Start unpaused (set = running, clear = paused)+ self._roots: tuple[RootType, ...] = ()+ self._phase: ScanPhase = ScanPhase.FULL+ self._compute_hashes: bool = False+ self._prune_first: bool = False+ self._progress_callback: ProgressCallback | None = None+ self._disabled: bool = False++ def disable(self) -> None:+ """Disable the asset seeder, preventing any scans from starting."""+ self._disabled = True+ logging.info("Asset seeder disabled")++ def is_disabled(self) -> bool:+ """Check if the asset seeder is disabled."""+ return self._disabled++ def start(+ self,+ roots: tuple[RootType, ...] = ("models", "input", "output"),+ phase: ScanPhase = ScanPhase.FULL,+ progress_callback: ProgressCallback | None = None,+ prune_first: bool = False,+ compute_hashes: bool = False,+ ) -> bool:+ """Start a background scan for the given roots.++ Args:+ roots: Tuple of root types to scan (models, input, output)+ phase: Scan phase to run (FAST, ENRICH, or FULL for both)+ progress_callback: Optional callback called with progress updates+ prune_first: If True, prune orphaned assets before scanning+ compute_hashes: If True, compute blake3 hashes (slow)++ Returns:+ True if scan was started, False if already running+ """+ if self._disabled:+ logging.debug("Asset seeder is disabled, skipping start")+ return False+ logging.info("Seeder start (roots=%s, phase=%s)", roots, phase.value)+ with self._lock:+ if self._state != State.IDLE:+ logging.info("Asset seeder already running, skipping start")+ return False+ self._state = State.RUNNING+ self._progress = Progress()+ self._errors = []+ self._roots = roots+ self._phase = phase+ self._prune_first = prune_first+ self._compute_hashes = compute_hashes+ self._progress_callback = progress_callback+ self._cancel_event.clear()+ self._run_gate.set() # Ensure unpaused when starting+ self._thread = threading.Thread(+ target=self._run_scan,+ name="_AssetSeeder",+ daemon=True,+ )+ self._thread.start()+ return True++ def start_fast(+ self,+ roots: tuple[RootType, ...] = ("models", "input", "output"),+ progress_callback: ProgressCallback | None = None,+ prune_first: bool = False,+ ) -> bool:+ """Start a fast scan (phase 1 only) - creates stub records.++ Args:+ roots: Tuple of root types to scan+ progress_callback: Optional callback for progress updates+ prune_first: If True, prune orphaned assets before scanning++ Returns:+ True if scan was started, False if already running+ """+ return self.start(+ roots=roots,+ phase=ScanPhase.FAST,+ progress_callback=progress_callback,+ prune_first=prune_first,+ compute_hashes=False,+ )++ def start_enrich(+ self,+ roots: tuple[RootType, ...] = ("models", "input", "output"),+ progress_callback: ProgressCallback | None = None,+ compute_hashes: bool = False,+ ) -> bool:+ """Start an enrichment scan (phase 2 only) - extracts metadata and hashes.++ Args:+ roots: Tuple of root types to scan+ progress_callback: Optional callback for progress updates+ compute_hashes: If True, compute blake3 hashes++ Returns:+ True if scan was started, False if already running+ """+ return self.start(+ roots=roots,+ phase=ScanPhase.ENRICH,+ progress_callback=progress_callback,+ prune_first=False,+ compute_hashes=compute_hashes,+ )++ def cancel(self) -> bool:+ """Request cancellation of the current scan.++ Returns:+ True if cancellation was requested, False if not running or paused+ """+ with self._lock:+ if self._state not in (State.RUNNING, State.PAUSED):+ return False+ logging.info("Asset seeder cancelling (was %s)", self._state.value)+ self._state = State.CANCELLING+ self._cancel_event.set()+ self._run_gate.set() # Unblock if paused so thread can exit+ return True++ def stop(self) -> bool:+ """Stop the current scan (alias for cancel).++ Returns:+ True if stop was requested, False if not running+ """+ return self.cancel()++ def pause(self) -> bool:+ """Pause the current scan.++ The scan will complete its current batch before pausing.++ Returns:+ True if pause was requested, False if not running+ """+ with self._lock:+ if self._state != State.RUNNING:+ return False+ logging.info("Asset seeder pausing")+ self._state = State.PAUSED+ self._run_gate.clear()+ return True++ def resume(self) -> bool:+ """Resume a paused scan.++ This is a noop if the scan is not in the PAUSED state++ Returns:+ True if resumed, False if not paused+ """+ with self._lock:+ if self._state != State.PAUSED:+ return False+ logging.info("Asset seeder resuming")+ self._state = State.RUNNING+ self._run_gate.set()+ self._emit_event("assets.seed.resumed", {})+ return True++ def restart(+ self,+ roots: tuple[RootType, ...] | None = None,+ phase: ScanPhase | None = None,+ progress_callback: ProgressCallback | None = None,+ prune_first: bool | None = None,+ compute_hashes: bool | None = None,+ timeout: float = 5.0,+ ) -> bool:+ """Cancel any running scan and start a new one.++ Args:+ roots: Roots to scan (defaults to previous roots)+ phase: Scan phase (defaults to previous phase)+ progress_callback: Progress callback (defaults to previous)+ prune_first: Prune before scan (defaults to previous)+ compute_hashes: Compute hashes (defaults to previous)+ timeout: Max seconds to wait for current scan to stop++ Returns:+ True if new scan was started, False if failed to stop previous+ """+ logging.info("Asset seeder restart requested")+ with self._lock:+ prev_roots = self._roots+ prev_phase = self._phase+ prev_callback = self._progress_callback+ prev_prune = self._prune_first+ prev_hashes = self._compute_hashes++ self.cancel()+ if not self.wait(timeout=timeout):+ return False++ cb = progress_callback if progress_callback is not None else prev_callback+ return self.start(+ roots=roots if roots is not None else prev_roots,+ phase=phase if phase is not None else prev_phase,+ progress_callback=cb,+ prune_first=prune_first if prune_first is not None else prev_prune,+ compute_hashes=(+ compute_hashes if compute_hashes is not None else prev_hashes+ ),+ )++ def wait(self, timeout: float | None = None) -> bool:+ """Wait for the current scan to complete.++ Args:+ timeout: Maximum seconds to wait, or None for no timeout++ Returns:+ True if scan completed, False if timeout expired or no scan running+ """+ with self._lock:+ thread = self._thread+ if thread is None:+ return True+ thread.join(timeout=timeout)+ return not thread.is_alive()++ def get_status(self) -> ScanStatus:+ """Get the current status and progress of the seeder."""+ with self._lock:+ src = self._progress or self._last_progress+ return ScanStatus(+ state=self._state,+ progress=Progress(+ scanned=src.scanned,+ total=src.total,+ created=src.created,+ skipped=src.skipped,+ )+ if src+ else None,+ errors=list(self._errors),+ )++ def shutdown(self, timeout: float = 5.0) -> None:+ """Gracefully shutdown: cancel any running scan and wait for thread.++ Args:+ timeout: Maximum seconds to wait for thread to exit+ """+ self.cancel()+ self.wait(timeout=timeout)+ with self._lock:+ self._thread = None++ def mark_missing_outside_prefixes(self) -> int:+ """Mark references as missing when outside all known root prefixes.++ This is a non-destructive soft-delete operation. Assets and their+ metadata are preserved, but references are flagged as missing.+ They can be restored if the file reappears in a future scan.++ This operation is decoupled from scanning to prevent partial scans+ from accidentally marking assets belonging to other roots.++ Should be called explicitly when cleanup is desired, typically after+ a full scan of all roots or during maintenance.++ Returns:+ Number of references marked as missing++ Raises:+ ScanInProgressError: If a scan is currently running+ """+ with self._lock:+ if self._state != State.IDLE:+ raise ScanInProgressError(+ "Cannot mark missing assets while scan is running"+ )+ self._state = State.RUNNING++ try:+ if not dependencies_available():+ logging.warning(+ "Database dependencies not available, skipping mark missing"+ )+ return 0++ all_prefixes = get_all_known_prefixes()+ marked = mark_missing_outside_prefixes_safely(all_prefixes)+ if marked > 0:+ logging.info("Marked %d references as missing", marked)+ return marked+ finally:+ with self._lock:+ self._last_progress = self._progress+ self._state = State.IDLE+ self._progress = None++ def _is_cancelled(self) -> bool:+ """Check if cancellation has been requested."""+ return self._cancel_event.is_set()++ def _is_paused_or_cancelled(self) -> bool:+ """Non-blocking check: True if paused or cancelled.++ Use as interrupt_check for I/O-bound work (e.g. hashing) so that+ file handles are released immediately on pause rather than held+ open while blocked. The caller is responsible for blocking on+ _check_pause_and_cancel() afterward.+ """+ return not self._run_gate.is_set() or self._cancel_event.is_set()++ def _check_pause_and_cancel(self) -> bool:+ """Block while paused, then check if cancelled.++ Call this at checkpoint locations in scan loops. It will:+ 1. Block indefinitely while paused (until resume or cancel)+ 2. Return True if cancelled, False to continue++ Returns:+ True if scan should stop, False to continue+ """+ if not self._run_gate.is_set():+ self._emit_event("assets.seed.paused", {})+ self._run_gate.wait() # Blocks if paused+ return self._is_cancelled()++ def _emit_event(self, event_type: str, data: dict) -> None:+ """Emit a WebSocket event if server is available."""+ try:+ from server import PromptServer++ if hasattr(PromptServer, "instance") and PromptServer.instance:+ PromptServer.instance.send_sync(event_type, data)+ except Exception:+ pass++ def _update_progress(+ self,+ scanned: int | None = None,+ total: int | None = None,+ created: int | None = None,+ skipped: int | None = None,+ ) -> None:+ """Update progress counters (thread-safe)."""+ callback: ProgressCallback | None = None+ progress: Progress | None = None++ with self._lock:+ if self._progress is None:+ return+ if scanned is not None:+ self._progress.scanned = scanned+ if total is not None:+ self._progress.total = total+ if created is not None:+ self._progress.created = created+ if skipped is not None:+ self._progress.skipped = skipped+ if self._progress_callback:+ callback = self._progress_callback+ progress = Progress(+ scanned=self._progress.scanned,+ total=self._progress.total,+ created=self._progress.created,+ skipped=self._progress.skipped,+ )++ if callback and progress:+ try:+ callback(progress)+ except Exception:+ pass++ _MAX_ERRORS = 200++ def _add_error(self, message: str) -> None:+ """Add an error message (thread-safe), capped at _MAX_ERRORS."""+ with self._lock:+ if len(self._errors) < self._MAX_ERRORS:+ self._errors.append(message)++ def _log_scan_config(self, roots: tuple[RootType, ...]) -> None:+ """Log the directories that will be scanned."""+ import folder_paths++ for root in roots:+ if root == "models":+ logging.info(+ "Asset scan [models] directory: %s",+ os.path.abspath(folder_paths.models_dir),+ )+ else:+ prefixes = get_prefixes_for_root(root)+ if prefixes:+ logging.info("Asset scan [%s] directories: %s", root, prefixes)++ def _run_scan(self) -> None:+ """Main scan loop running in background thread."""+ t_start = time.perf_counter()+ roots = self._roots+ phase = self._phase+ cancelled = False+ total_created = 0+ total_enriched = 0+ skipped_existing = 0+ total_paths = 0++ try:+ if not dependencies_available():+ self._add_error("Database dependencies not available")+ self._emit_event(+ "assets.seed.error",+ {"message": "Database dependencies not available"},+ )+ return++ if self._prune_first:+ all_prefixes = get_all_known_prefixes()+ marked = mark_missing_outside_prefixes_safely(all_prefixes)+ if marked > 0:+ logging.info("Marked %d refs as missing before scan", marked)++ if self._check_pause_and_cancel():+ logging.info("Asset scan cancelled after pruning phase")+ cancelled = True+ return++ self._log_scan_config(roots)++ # Phase 1: Fast scan (stub records)+ if phase in (ScanPhase.FAST, ScanPhase.FULL):+ created, skipped, paths = self._run_fast_phase(roots)+ total_created, skipped_existing, total_paths = created, skipped, paths++ if self._check_pause_and_cancel():+ cancelled = True+ return++ self._emit_event(+ "assets.seed.fast_complete",+ {+ "roots": list(roots),+ "created": total_created,+ "skipped": skipped_existing,+ "total": total_paths,+ },+ )++ # Phase 2: Enrichment scan (metadata + hashes)+ if phase in (ScanPhase.ENRICH, ScanPhase.FULL):+ if self._check_pause_and_cancel():+ cancelled = True+ return++ enrich_cancelled, total_enriched = self._run_enrich_phase(roots)++ if enrich_cancelled:+ cancelled = True+ return++ self._emit_event(+ "assets.seed.enrich_complete",+ {+ "roots": list(roots),+ "enriched": total_enriched,+ },+ )++ elapsed = time.perf_counter() - t_start+ logging.info(+ "Scan(%s, %s) done %.3fs: created=%d enriched=%d skipped=%d",+ roots,+ phase.value,+ elapsed,+ total_created,+ total_enriched,+ skipped_existing,+ )++ self._emit_event(+ "assets.seed.completed",+ {+ "phase": phase.value,+ "total": total_paths,+ "created": total_created,+ "enriched": total_enriched,+ "skipped": skipped_existing,+ "elapsed": round(elapsed, 3),+ },+ )++ except Exception as e:+ self._add_error(f"Scan failed: {e}")+ logging.exception("Asset scan failed")+ self._emit_event("assets.seed.error", {"message": str(e)})+ finally:+ if cancelled:+ self._emit_event(+ "assets.seed.cancelled",+ {+ "scanned": self._progress.scanned if self._progress else 0,+ "total": total_paths,+ "created": total_created,+ },+ )+ with self._lock:+ self._last_progress = self._progress+ self._state = State.IDLE+ self._progress = None++ def _run_fast_phase(self, roots: tuple[RootType, ...]) -> tuple[int, int, int]:+ """Run phase 1: fast scan to create stub records.++ Returns:+ Tuple of (total_created, skipped_existing, total_paths)+ """+ t_fast_start = time.perf_counter()+ total_created = 0+ skipped_existing = 0++ existing_paths: set[str] = set()+ t_sync = time.perf_counter()+ for r in roots:+ if self._check_pause_and_cancel():+ return total_created, skipped_existing, 0+ existing_paths.update(sync_root_safely(r))+ logging.debug(+ "Fast scan: sync_root phase took %.3fs (%d existing paths)",+ time.perf_counter() - t_sync,+ len(existing_paths),+ )++ if self._check_pause_and_cancel():+ return total_created, skipped_existing, 0++ t_collect = time.perf_counter()+ paths = collect_paths_for_roots(roots)+ logging.debug(+ "Fast scan: collect_paths took %.3fs (%d paths found)",+ time.perf_counter() - t_collect,+ len(paths),+ )+ total_paths = len(paths)+ self._update_progress(total=total_paths)++ self._emit_event(+ "assets.seed.started",+ {"roots": list(roots), "total": total_paths, "phase": "fast"},+ )++ # Use stub specs (no metadata extraction, no hashing)+ t_specs = time.perf_counter()+ specs, tag_pool, skipped_existing = build_asset_specs(+ paths,+ existing_paths,+ enable_metadata_extraction=False,+ compute_hashes=False,+ )+ logging.debug(+ "Fast scan: build_asset_specs took %.3fs (%d specs, %d skipped)",+ time.perf_counter() - t_specs,+ len(specs),+ skipped_existing,+ )+ self._update_progress(skipped=skipped_existing)++ if self._check_pause_and_cancel():+ return total_created, skipped_existing, total_paths++ batch_size = 500+ last_progress_time = time.perf_counter()+ progress_interval = 1.0++ for i in range(0, len(specs), batch_size):+ if self._check_pause_and_cancel():+ logging.info(+ "Fast scan cancelled after %d/%d files (created=%d)",+ i,+ len(specs),+ total_created,+ )+ return total_created, skipped_existing, total_paths++ batch = specs[i : i + batch_size]+ batch_tags = {t for spec in batch for t in spec["tags"]}+ try:+ created = insert_asset_specs(batch, batch_tags)+ total_created += created+ except Exception as e:+ self._add_error(f"Batch insert failed at offset {i}: {e}")+ logging.exception("Batch insert failed at offset %d", i)++ scanned = i + len(batch)+ now = time.perf_counter()+ self._update_progress(scanned=scanned, created=total_created)++ if now - last_progress_time >= progress_interval:+ self._emit_event(+ "assets.seed.progress",+ {+ "phase": "fast",+ "scanned": scanned,+ "total": len(specs),+ "created": total_created,+ },+ )+ last_progress_time = now++ self._update_progress(scanned=len(specs), created=total_created)+ logging.info(+ "Fast scan complete: %.3fs total (created=%d, skipped=%d, total_paths=%d)",+ time.perf_counter() - t_fast_start,+ total_created,+ skipped_existing,+ total_paths,+ )+ return total_created, skipped_existing, total_paths++ def _run_enrich_phase(self, roots: tuple[RootType, ...]) -> tuple[bool, int]:+ """Run phase 2: enrich existing records with metadata and hashes.++ Returns:+ Tuple of (cancelled, total_enriched)+ """+ total_enriched = 0+ batch_size = 100+ last_progress_time = time.perf_counter()+ progress_interval = 1.0++ # Get the target enrichment level based on compute_hashes+ if not self._compute_hashes:+ target_max_level = ENRICHMENT_STUB+ else:+ target_max_level = ENRICHMENT_METADATA++ self._emit_event(+ "assets.seed.started",+ {"roots": list(roots), "phase": "enrich"},+ )++ skip_ids: set[str] = set()+ consecutive_empty = 0+ max_consecutive_empty = 3++ # Hash checkpoints survive across batches so interrupted hashes+ # can be resumed without re-reading the entire file.+ hash_checkpoints: dict[str, object] = {}++ while True:+ if self._check_pause_and_cancel():+ logging.info("Enrich scan cancelled after %d assets", total_enriched)+ return True, total_enriched++ # Fetch next batch of unenriched assets+ unenriched = get_unenriched_assets_for_roots(+ roots,+ max_level=target_max_level,+ limit=batch_size,+ )++ # Filter out previously failed references+ if skip_ids:+ unenriched = [r for r in unenriched if r.reference_id not in skip_ids]++ if not unenriched:+ break++ enriched, failed_ids = enrich_assets_batch(+ unenriched,+ extract_metadata=True,+ compute_hash=self._compute_hashes,+ interrupt_check=self._is_paused_or_cancelled,+ hash_checkpoints=hash_checkpoints,+ )+ total_enriched += enriched+ skip_ids.update(failed_ids)++ if enriched == 0:+ consecutive_empty += 1+ if consecutive_empty >= max_consecutive_empty:+ logging.warning(+ "Enrich phase stopping: %d consecutive batches with no progress (%d skipped)",+ consecutive_empty,+ len(skip_ids),+ )+ break+ else:+ consecutive_empty = 0++ now = time.perf_counter()+ if now - last_progress_time >= progress_interval:+ self._emit_event(+ "assets.seed.progress",+ {+ "phase": "enrich",+ "enriched": total_enriched,+ },+ )+ last_progress_time = now++ return False, total_enriched+++asset_seeder = _AssetSeeder()diff --git a/app/assets/services/__init__.py b/app/assets/services/__init__.pynew file mode 100644index 000000000..11fcb4122--- /dev/null+++ b/app/assets/services/__init__.py@@ -0,0 +1,87 @@+from app.assets.services.asset_management import (+ asset_exists,+ delete_asset_reference,+ get_asset_by_hash,+ get_asset_detail,+ list_assets_page,+ resolve_asset_for_download,+ set_asset_preview,+ update_asset_metadata,+)+from app.assets.services.bulk_ingest import (+ BulkInsertResult,+ batch_insert_seed_assets,+ cleanup_unreferenced_assets,+)+from app.assets.services.file_utils import (+ get_mtime_ns,+ get_size_and_mtime_ns,+ list_files_recursively,+ verify_file_unchanged,+)+from app.assets.services.ingest import (+ DependencyMissingError,+ HashMismatchError,+ create_from_hash,+ upload_from_temp_path,+)+from app.assets.database.queries import (+ AddTagsResult,+ RemoveTagsResult,+)+from app.assets.services.schemas import (+ AssetData,+ AssetDetailResult,+ AssetSummaryData,+ DownloadResolutionResult,+ IngestResult,+ ListAssetsResult,+ ReferenceData,+ RegisterAssetResult,+ TagUsage,+ UploadResult,+ UserMetadata,+)+from app.assets.services.tagging import (+ apply_tags,+ list_tags,+ remove_tags,+)++__all__ = [+ "AddTagsResult",+ "AssetData",+ "AssetDetailResult",+ "AssetSummaryData",+ "ReferenceData",+ "BulkInsertResult",+ "DependencyMissingError",+ "DownloadResolutionResult",+ "HashMismatchError",+ "IngestResult",+ "ListAssetsResult",+ "RegisterAssetResult",+ "RemoveTagsResult",+ "TagUsage",+ "UploadResult",+ "UserMetadata",+ "apply_tags",+ "asset_exists",+ "batch_insert_seed_assets",+ "create_from_hash",+ "delete_asset_reference",+ "get_asset_by_hash",+ "get_asset_detail",+ "get_mtime_ns",+ "get_size_and_mtime_ns",+ "list_assets_page",+ "list_files_recursively",+ "list_tags",+ "cleanup_unreferenced_assets",+ "remove_tags",+ "resolve_asset_for_download",+ "set_asset_preview",+ "update_asset_metadata",+ "upload_from_temp_path",+ "verify_file_unchanged",+]diff --git a/app/assets/services/asset_management.py b/app/assets/services/asset_management.pynew file mode 100644index 000000000..3fe7115c8--- /dev/null+++ b/app/assets/services/asset_management.py@@ -0,0 +1,309 @@+import contextlib+import mimetypes+import os+from typing import Sequence+++from app.assets.database.models import Asset+from app.assets.database.queries import (+ asset_exists_by_hash,+ reference_exists_for_asset_id,+ delete_reference_by_id,+ fetch_reference_and_asset,+ soft_delete_reference_by_id,+ fetch_reference_asset_and_tags,+ get_asset_by_hash as queries_get_asset_by_hash,+ get_reference_by_id,+ get_reference_with_owner_check,+ list_references_page,+ list_references_by_asset_id,+ set_reference_metadata,+ set_reference_preview,+ set_reference_tags,+ update_reference_access_time,+ update_reference_name,+ update_reference_updated_at,+)+from app.assets.helpers import select_best_live_path+from app.assets.services.path_utils import compute_relative_filename+from app.assets.services.schemas import (+ AssetData,+ AssetDetailResult,+ AssetSummaryData,+ DownloadResolutionResult,+ ListAssetsResult,+ UserMetadata,+ extract_asset_data,+ extract_reference_data,+)+from app.database.db import create_session+++def get_asset_detail(+ reference_id: str,+ owner_id: str = "",+) -> AssetDetailResult | None:+ with create_session() as session:+ result = fetch_reference_asset_and_tags(+ session,+ reference_id=reference_id,+ owner_id=owner_id,+ )+ if not result:+ return None++ ref, asset, tags = result+ return AssetDetailResult(+ ref=extract_reference_data(ref),+ asset=extract_asset_data(asset),+ tags=tags,+ )+++def update_asset_metadata(+ reference_id: str,+ name: str | None = None,+ tags: Sequence[str] | None = None,+ user_metadata: UserMetadata = None,+ tag_origin: str = "manual",+ owner_id: str = "",+) -> AssetDetailResult:+ with create_session() as session:+ ref = get_reference_with_owner_check(session, reference_id, owner_id)++ touched = False+ if name is not None and name != ref.name:+ update_reference_name(session, reference_id=reference_id, name=name)+ touched = True++ computed_filename = compute_relative_filename(ref.file_path) if ref.file_path else None++ new_meta: dict | None = None+ if user_metadata is not None:+ new_meta = dict(user_metadata)+ elif computed_filename:+ current_meta = ref.user_metadata or {}+ if current_meta.get("filename") != computed_filename:+ new_meta = dict(current_meta)++ if new_meta is not None:+ if computed_filename:+ new_meta["filename"] = computed_filename+ set_reference_metadata(+ session, reference_id=reference_id, user_metadata=new_meta+ )+ touched = True++ if tags is not None:+ set_reference_tags(+ session,+ reference_id=reference_id,+ tags=tags,+ origin=tag_origin,+ )+ touched = True++ if touched and user_metadata is None:+ update_reference_updated_at(session, reference_id=reference_id)++ result = fetch_reference_asset_and_tags(+ session,+ reference_id=reference_id,+ owner_id=owner_id,+ )+ if not result:+ raise RuntimeError("State changed during update")++ ref, asset, tag_list = result+ detail = AssetDetailResult(+ ref=extract_reference_data(ref),+ asset=extract_asset_data(asset),+ tags=tag_list,+ )+ session.commit()++ return detail+++def delete_asset_reference(+ reference_id: str,+ owner_id: str,+ delete_content_if_orphan: bool = True,+) -> bool:+ with create_session() as session:+ if not delete_content_if_orphan:+ # Soft delete: mark the reference as deleted but keep everything+ deleted = soft_delete_reference_by_id(+ session, reference_id=reference_id, owner_id=owner_id+ )+ session.commit()+ return deleted++ ref_row = get_reference_by_id(session, reference_id=reference_id)+ asset_id = ref_row.asset_id if ref_row else None+ file_path = ref_row.file_path if ref_row else None++ deleted = delete_reference_by_id(+ session, reference_id=reference_id, owner_id=owner_id+ )+ if not deleted:+ session.commit()+ return False++ if not asset_id:+ session.commit()+ return True++ still_exists = reference_exists_for_asset_id(session, asset_id=asset_id)+ if still_exists:+ session.commit()+ return True++ # Orphaned asset - delete it and its files+ refs = list_references_by_asset_id(session, asset_id=asset_id)+ file_paths = [+ r.file_path for r in (refs or []) if getattr(r, "file_path", None)+ ]+ # Also include the just-deleted file path+ if file_path:+ file_paths.append(file_path)++ asset_row = session.get(Asset, asset_id)+ if asset_row is not None:+ session.delete(asset_row)++ session.commit()++ # Delete files after commit+ for p in file_paths:+ with contextlib.suppress(Exception):+ if p and os.path.isfile(p):+ os.remove(p)++ return True+++def set_asset_preview(+ reference_id: str,+ preview_asset_id: str | None = None,+ owner_id: str = "",+) -> AssetDetailResult:+ with create_session() as session:+ get_reference_with_owner_check(session, reference_id, owner_id)++ set_reference_preview(+ session,+ reference_id=reference_id,+ preview_asset_id=preview_asset_id,+ )++ result = fetch_reference_asset_and_tags(+ session, reference_id=reference_id, owner_id=owner_id+ )+ if not result:+ raise RuntimeError("State changed during preview update")++ ref, asset, tags = result+ detail = AssetDetailResult(+ ref=extract_reference_data(ref),+ asset=extract_asset_data(asset),+ tags=tags,+ )+ session.commit()++ return detail+++def asset_exists(asset_hash: str) -> bool:+ with create_session() as session:+ return asset_exists_by_hash(session, asset_hash=asset_hash)+++def get_asset_by_hash(asset_hash: str) -> AssetData | None:+ with create_session() as session:+ asset = queries_get_asset_by_hash(session, asset_hash=asset_hash)+ return extract_asset_data(asset)+++def list_assets_page(+ owner_id: str = "",+ include_tags: Sequence[str] | None = None,+ exclude_tags: Sequence[str] | None = None,+ name_contains: str | None = None,+ metadata_filter: dict | None = None,+ limit: int = 20,+ offset: int = 0,+ sort: str = "created_at",+ order: str = "desc",+) -> ListAssetsResult:+ with create_session() as session:+ refs, tag_map, total = list_references_page(+ session,+ owner_id=owner_id,+ include_tags=include_tags,+ exclude_tags=exclude_tags,+ name_contains=name_contains,+ metadata_filter=metadata_filter,+ limit=limit,+ offset=offset,+ sort=sort,+ order=order,+ )++ items: list[AssetSummaryData] = []+ for ref in refs:+ items.append(+ AssetSummaryData(+ ref=extract_reference_data(ref),+ asset=extract_asset_data(ref.asset),+ tags=tag_map.get(ref.id, []),+ )+ )++ return ListAssetsResult(items=items, total=total)+++def resolve_asset_for_download(+ reference_id: str,+ owner_id: str = "",+) -> DownloadResolutionResult:+ with create_session() as session:+ pair = fetch_reference_and_asset(+ session, reference_id=reference_id, owner_id=owner_id+ )+ if not pair:+ raise ValueError(f"AssetReference {reference_id} not found")++ ref, asset = pair++ # For references with file_path, use that directly+ if ref.file_path and os.path.isfile(ref.file_path):+ abs_path = ref.file_path+ else:+ # For API-created refs without file_path, find a path from other refs+ refs = list_references_by_asset_id(session, asset_id=asset.id)+ abs_path = select_best_live_path(refs)+ if not abs_path:+ raise FileNotFoundError(+ f"No live path for AssetReference {reference_id} "+ f"(asset id={asset.id}, name={ref.name})"+ )++ # Capture ORM attributes before commit (commit expires loaded objects)+ ref_name = ref.name+ asset_mime = asset.mime_type++ update_reference_access_time(session, reference_id=reference_id)+ session.commit()++ ctype = (+ asset_mime+ or mimetypes.guess_type(ref_name or abs_path)[0]+ or "application/octet-stream"+ )+ download_name = ref_name or os.path.basename(abs_path)+ return DownloadResolutionResult(+ abs_path=abs_path,+ content_type=ctype,+ download_name=download_name,+ )diff --git a/app/assets/services/bulk_ingest.py b/app/assets/services/bulk_ingest.pynew file mode 100644index 000000000..54e72730c--- /dev/null+++ b/app/assets/services/bulk_ingest.py@@ -0,0 +1,280 @@+from __future__ import annotations++import os+import uuid+from dataclasses import dataclass+from datetime import datetime+from typing import TYPE_CHECKING, Any, TypedDict++from sqlalchemy.orm import Session++from app.assets.database.queries import (+ bulk_insert_assets,+ bulk_insert_references_ignore_conflicts,+ bulk_insert_tags_and_meta,+ delete_assets_by_ids,+ get_existing_asset_ids,+ get_reference_ids_by_ids,+ get_references_by_paths_and_asset_ids,+ get_unreferenced_unhashed_asset_ids,+ restore_references_by_paths,+)+from app.assets.helpers import get_utc_now++if TYPE_CHECKING:+ from app.assets.services.metadata_extract import ExtractedMetadata+++class SeedAssetSpec(TypedDict):+ """Spec for seeding an asset from filesystem."""++ abs_path: str+ size_bytes: int+ mtime_ns: int+ info_name: str+ tags: list[str]+ fname: str+ metadata: ExtractedMetadata | None+ hash: str | None+ mime_type: str | None+++class AssetRow(TypedDict):+ """Row data for inserting an Asset."""++ id: str+ hash: str | None+ size_bytes: int+ mime_type: str | None+ created_at: datetime+++class ReferenceRow(TypedDict):+ """Row data for inserting an AssetReference."""++ id: str+ asset_id: str+ file_path: str+ mtime_ns: int+ owner_id: str+ name: str+ preview_id: str | None+ user_metadata: dict[str, Any] | None+ created_at: datetime+ updated_at: datetime+ last_access_time: datetime+++class TagRow(TypedDict):+ """Row data for inserting a Tag."""++ asset_reference_id: str+ tag_name: str+ origin: str+ added_at: datetime+++class MetadataRow(TypedDict):+ """Row data for inserting asset metadata."""++ asset_reference_id: str+ key: str+ ordinal: int+ val_str: str | None+ val_num: float | None+ val_bool: bool | None+ val_json: dict[str, Any] | None+++@dataclass+class BulkInsertResult:+ """Result of bulk asset insertion."""++ inserted_refs: int+ won_paths: int+ lost_paths: int+++def batch_insert_seed_assets(+ session: Session,+ specs: list[SeedAssetSpec],+ owner_id: str = "",+) -> BulkInsertResult:+ """Seed assets from filesystem specs in batch.++ Each spec is a dict with keys:+ - abs_path: str+ - size_bytes: int+ - mtime_ns: int+ - info_name: str+ - tags: list[str]+ - fname: Optional[str]++ This function orchestrates:+ 1. Insert seed Assets (hash=NULL)+ 2. Claim references with ON CONFLICT DO NOTHING on file_path+ 3. Query to find winners (paths where our asset_id was inserted)+ 4. Delete Assets for losers (path already claimed by another asset)+ 5. Insert tags and metadata for successfully inserted references++ Returns:+ BulkInsertResult with inserted_refs, won_paths, lost_paths+ """+ if not specs:+ return BulkInsertResult(inserted_refs=0, won_paths=0, lost_paths=0)++ current_time = get_utc_now()+ asset_rows: list[AssetRow] = []+ reference_rows: list[ReferenceRow] = []+ path_to_asset_id: dict[str, str] = {}+ asset_id_to_ref_data: dict[str, dict] = {}+ absolute_path_list: list[str] = []++ for spec in specs:+ absolute_path = os.path.abspath(spec["abs_path"])+ asset_id = str(uuid.uuid4())+ reference_id = str(uuid.uuid4())+ absolute_path_list.append(absolute_path)+ path_to_asset_id[absolute_path] = asset_id++ mime_type = spec.get("mime_type")+ asset_rows.append(+ {+ "id": asset_id,+ "hash": spec.get("hash"),+ "size_bytes": spec["size_bytes"],+ "mime_type": mime_type,+ "created_at": current_time,+ }+ )++ # Build user_metadata from extracted metadata or fallback to filename+ extracted_metadata = spec.get("metadata")+ if extracted_metadata:+ user_metadata: dict[str, Any] | None = extracted_metadata.to_user_metadata()+ elif spec["fname"]:+ user_metadata = {"filename": spec["fname"]}+ else:+ user_metadata = None++ reference_rows.append(+ {+ "id": reference_id,+ "asset_id": asset_id,+ "file_path": absolute_path,+ "mtime_ns": spec["mtime_ns"],+ "owner_id": owner_id,+ "name": spec["info_name"],+ "preview_id": None,+ "user_metadata": user_metadata,+ "created_at": current_time,+ "updated_at": current_time,+ "last_access_time": current_time,+ }+ )++ asset_id_to_ref_data[asset_id] = {+ "reference_id": reference_id,+ "tags": spec["tags"],+ "filename": spec["fname"],+ "extracted_metadata": extracted_metadata,+ }++ bulk_insert_assets(session, asset_rows)++ # Filter reference rows to only those whose assets were actually inserted+ # (assets with duplicate hashes are silently dropped by ON CONFLICT DO NOTHING)+ inserted_asset_ids = get_existing_asset_ids(+ session, [r["asset_id"] for r in reference_rows]+ )+ reference_rows = [r for r in reference_rows if r["asset_id"] in inserted_asset_ids]++ bulk_insert_references_ignore_conflicts(session, reference_rows)+ restore_references_by_paths(session, absolute_path_list)+ winning_paths = get_references_by_paths_and_asset_ids(session, path_to_asset_id)++ inserted_paths = {+ path+ for path in absolute_path_list+ if path_to_asset_id[path] in inserted_asset_ids+ }+ losing_paths = inserted_paths - winning_paths+ lost_asset_ids = [path_to_asset_id[path] for path in losing_paths]++ if lost_asset_ids:+ delete_assets_by_ids(session, lost_asset_ids)++ if not winning_paths:+ return BulkInsertResult(+ inserted_refs=0,+ won_paths=0,+ lost_paths=len(losing_paths),+ )++ # Get reference IDs for winners+ winning_ref_ids = [+ asset_id_to_ref_data[path_to_asset_id[path]]["reference_id"]+ for path in winning_paths+ ]+ inserted_ref_ids = get_reference_ids_by_ids(session, winning_ref_ids)++ tag_rows: list[TagRow] = []+ metadata_rows: list[MetadataRow] = []++ if inserted_ref_ids:+ for path in winning_paths:+ asset_id = path_to_asset_id[path]+ ref_data = asset_id_to_ref_data[asset_id]+ ref_id = ref_data["reference_id"]++ if ref_id not in inserted_ref_ids:+ continue++ for tag in ref_data["tags"]:+ tag_rows.append(+ {+ "asset_reference_id": ref_id,+ "tag_name": tag,+ "origin": "automatic",+ "added_at": current_time,+ }+ )++ # Use extracted metadata for meta rows if available+ extracted_metadata = ref_data.get("extracted_metadata")+ if extracted_metadata:+ metadata_rows.extend(extracted_metadata.to_meta_rows(ref_id))+ elif ref_data["filename"]:+ # Fallback: just store filename+ metadata_rows.append(+ {+ "asset_reference_id": ref_id,+ "key": "filename",+ "ordinal": 0,+ "val_str": ref_data["filename"],+ "val_num": None,+ "val_bool": None,+ "val_json": None,+ }+ )++ bulk_insert_tags_and_meta(session, tag_rows=tag_rows, meta_rows=metadata_rows)++ return BulkInsertResult(+ inserted_refs=len(inserted_ref_ids),+ won_paths=len(winning_paths),+ lost_paths=len(losing_paths),+ )+++def cleanup_unreferenced_assets(session: Session) -> int:+ """Hard-delete unhashed assets with no active references.++ This is a destructive operation intended for explicit cleanup.+ Only deletes assets where hash=None and all references are missing.++ Returns:+ Number of assets deleted+ """+ unreferenced_ids = get_unreferenced_unhashed_asset_ids(session)+ return delete_assets_by_ids(session, unreferenced_ids)diff --git a/app/assets/services/file_utils.py b/app/assets/services/file_utils.pynew file mode 100644index 000000000..c47ebe460--- /dev/null+++ b/app/assets/services/file_utils.py@@ -0,0 +1,70 @@+import os+++def get_mtime_ns(stat_result: os.stat_result) -> int:+ """Extract mtime in nanoseconds from a stat result."""+ return getattr(+ stat_result, "st_mtime_ns", int(stat_result.st_mtime * 1_000_000_000)+ )+++def get_size_and_mtime_ns(path: str, follow_symlinks: bool = True) -> tuple[int, int]:+ """Get file size in bytes and mtime in nanoseconds."""+ st = os.stat(path, follow_symlinks=follow_symlinks)+ return st.st_size, get_mtime_ns(st)+++def verify_file_unchanged(+ mtime_db: int | None,+ size_db: int | None,+ stat_result: os.stat_result,+) -> bool:+ """Check if a file is unchanged based on mtime and size.++ Returns True if the file's mtime and size match the database values.+ Returns False if mtime_db is None or values don't match.++ size_db=None means don't check size; 0 is a valid recorded size.+ """+ if mtime_db is None:+ return False+ actual_mtime_ns = get_mtime_ns(stat_result)+ if int(mtime_db) != int(actual_mtime_ns):+ return False+ if size_db is not None:+ return int(stat_result.st_size) == int(size_db)+ return True+++def is_visible(name: str) -> bool:+ """Return True if a file or directory name is visible (not hidden)."""+ return not name.startswith(".")+++def list_files_recursively(base_dir: str) -> list[str]:+ """Recursively list all files in a directory, following symlinks."""+ out: list[str] = []+ base_abs = os.path.abspath(base_dir)+ if not os.path.isdir(base_abs):+ return out+ # Track seen real directory identities to prevent circular symlink loops+ seen_dirs: set[tuple[int, int]] = set()+ for dirpath, subdirs, filenames in os.walk(+ base_abs, topdown=True, followlinks=True+ ):+ try:+ st = os.stat(dirpath)+ dir_id = (st.st_dev, st.st_ino)+ except OSError:+ subdirs.clear()+ continue+ if dir_id in seen_dirs:+ subdirs.clear()+ continue+ seen_dirs.add(dir_id)+ subdirs[:] = [d for d in subdirs if is_visible(d)]+ for name in filenames:+ if not is_visible(name):+ continue+ out.append(os.path.abspath(os.path.join(dirpath, name)))+ return outdiff --git a/app/assets/services/hashing.py b/app/assets/services/hashing.pynew file mode 100644index 000000000..92aee6402--- /dev/null+++ b/app/assets/services/hashing.py@@ -0,0 +1,95 @@+import io+import os+from contextlib import contextmanager+from dataclasses import dataclass+from typing import IO, Any, Callable, Iterator++from blake3 import blake3++DEFAULT_CHUNK = 8 * 1024 * 1024++InterruptCheck = Callable[[], bool]+++@dataclass+class HashCheckpoint:+ """Saved state for resuming an interrupted hash computation."""++ bytes_processed: int+ hasher: Any # blake3 hasher instance+ mtime_ns: int = 0+ file_size: int = 0+++@contextmanager+def _open_for_hashing(fp: str | IO[bytes]) -> Iterator[tuple[IO[bytes], bool]]:+ """Yield (file_object, is_path) with appropriate setup/teardown."""+ if hasattr(fp, "read"):+ seekable = getattr(fp, "seekable", lambda: False)()+ orig_pos = None+ if seekable:+ try:+ orig_pos = fp.tell()+ if orig_pos != 0:+ fp.seek(0)+ except io.UnsupportedOperation:+ orig_pos = None+ try:+ yield fp, False+ finally:+ if orig_pos is not None:+ fp.seek(orig_pos)+ else:+ with open(os.fspath(fp), "rb") as f:+ yield f, True+++def compute_blake3_hash(+ fp: str | IO[bytes],+ chunk_size: int = DEFAULT_CHUNK,+ interrupt_check: InterruptCheck | None = None,+ checkpoint: HashCheckpoint | None = None,+) -> tuple[str | None, HashCheckpoint | None]:+ """Compute BLAKE3 hash of a file, with optional checkpoint support.++ Args:+ fp: File path or file-like object+ chunk_size: Size of chunks to read at a time+ interrupt_check: Optional callable that returns True if the operation+ should be interrupted (e.g. paused or cancelled). Must be+ non-blocking so file handles are released immediately. Checked+ between chunk reads.+ checkpoint: Optional checkpoint to resume from (file paths only)++ Returns:+ Tuple of (hex_digest, None) on completion, or+ (None, checkpoint) on interruption (file paths only), or+ (None, None) on interruption of a file object+ """+ if chunk_size <= 0:+ chunk_size = DEFAULT_CHUNK++ with _open_for_hashing(fp) as (f, is_path):+ if checkpoint is not None and is_path:+ f.seek(checkpoint.bytes_processed)+ h = checkpoint.hasher+ bytes_processed = checkpoint.bytes_processed+ else:+ h = blake3()+ bytes_processed = 0++ while True:+ if interrupt_check is not None and interrupt_check():+ if is_path:+ return None, HashCheckpoint(+ bytes_processed=bytes_processed,+ hasher=h,+ )+ return None, None+ chunk = f.read(chunk_size)+ if not chunk:+ break+ h.update(chunk)+ bytes_processed += len(chunk)++ return h.hexdigest(), Nonediff --git a/app/assets/services/ingest.py b/app/assets/services/ingest.pynew file mode 100644index 000000000..44d7aef36--- /dev/null+++ b/app/assets/services/ingest.py@@ -0,0 +1,375 @@+import contextlib+import logging+import mimetypes+import os+from typing import Any, Sequence++from sqlalchemy.orm import Session++import app.assets.services.hashing as hashing+from app.assets.database.queries import (+ add_tags_to_reference,+ fetch_reference_and_asset,+ get_asset_by_hash,+ get_existing_asset_ids,+ get_reference_by_file_path,+ get_reference_tags,+ get_or_create_reference,+ remove_missing_tag_for_asset_id,+ set_reference_metadata,+ set_reference_tags,+ upsert_asset,+ upsert_reference,+ validate_tags_exist,+)+from app.assets.helpers import normalize_tags+from app.assets.services.file_utils import get_size_and_mtime_ns+from app.assets.services.path_utils import (+ compute_relative_filename,+ resolve_destination_from_tags,+ validate_path_within_base,+)+from app.assets.services.schemas import (+ IngestResult,+ RegisterAssetResult,+ UploadResult,+ UserMetadata,+ extract_asset_data,+ extract_reference_data,+)+from app.database.db import create_session+++def _ingest_file_from_path(+ abs_path: str,+ asset_hash: str,+ size_bytes: int,+ mtime_ns: int,+ mime_type: str | None = None,+ info_name: str | None = None,+ owner_id: str = "",+ preview_id: str | None = None,+ user_metadata: UserMetadata = None,+ tags: Sequence[str] = (),+ tag_origin: str = "manual",+ require_existing_tags: bool = False,+) -> IngestResult:+ locator = os.path.abspath(abs_path)+ user_metadata = user_metadata or {}++ asset_created = False+ asset_updated = False+ ref_created = False+ ref_updated = False+ reference_id: str | None = None++ with create_session() as session:+ if preview_id:+ if preview_id not in get_existing_asset_ids(session, [preview_id]):+ preview_id = None++ asset, asset_created, asset_updated = upsert_asset(+ session,+ asset_hash=asset_hash,+ size_bytes=size_bytes,+ mime_type=mime_type,+ )++ ref_created, ref_updated = upsert_reference(+ session,+ asset_id=asset.id,+ file_path=locator,+ name=info_name or os.path.basename(locator),+ mtime_ns=mtime_ns,+ owner_id=owner_id,+ )++ # Get the reference we just created/updated+ ref = get_reference_by_file_path(session, locator)+ if ref:+ reference_id = ref.id++ if preview_id and ref.preview_id != preview_id:+ ref.preview_id = preview_id++ norm = normalize_tags(list(tags))+ if norm:+ if require_existing_tags:+ validate_tags_exist(session, norm)+ add_tags_to_reference(+ session,+ reference_id=reference_id,+ tags=norm,+ origin=tag_origin,+ create_if_missing=not require_existing_tags,+ )++ _update_metadata_with_filename(+ session,+ reference_id=reference_id,+ file_path=ref.file_path,+ current_metadata=ref.user_metadata,+ user_metadata=user_metadata,+ )++ try:+ remove_missing_tag_for_asset_id(session, asset_id=asset.id)+ except Exception:+ logging.exception("Failed to clear 'missing' tag for asset %s", asset.id)++ session.commit()++ return IngestResult(+ asset_created=asset_created,+ asset_updated=asset_updated,+ ref_created=ref_created,+ ref_updated=ref_updated,+ reference_id=reference_id,+ )+++def _register_existing_asset(+ asset_hash: str,+ name: str,+ user_metadata: UserMetadata = None,+ tags: list[str] | None = None,+ tag_origin: str = "manual",+ owner_id: str = "",+) -> RegisterAssetResult:+ user_metadata = user_metadata or {}++ with create_session() as session:+ asset = get_asset_by_hash(session, asset_hash=asset_hash)+ if not asset:+ raise ValueError(f"No asset with hash {asset_hash}")++ ref, ref_created = get_or_create_reference(+ session,+ asset_id=asset.id,+ owner_id=owner_id,+ name=name,+ )++ if not ref_created:+ tag_names = get_reference_tags(session, reference_id=ref.id)+ result = RegisterAssetResult(+ ref=extract_reference_data(ref),+ asset=extract_asset_data(asset),+ tags=tag_names,+ created=False,+ )+ session.commit()+ return result++ new_meta = dict(user_metadata)+ computed_filename = compute_relative_filename(ref.file_path) if ref.file_path else None+ if computed_filename:+ new_meta["filename"] = computed_filename++ if new_meta:+ set_reference_metadata(+ session,+ reference_id=ref.id,+ user_metadata=new_meta,+ )++ if tags is not None:+ set_reference_tags(+ session,+ reference_id=ref.id,+ tags=tags,+ origin=tag_origin,+ )++ tag_names = get_reference_tags(session, reference_id=ref.id)+ session.refresh(ref)+ result = RegisterAssetResult(+ ref=extract_reference_data(ref),+ asset=extract_asset_data(asset),+ tags=tag_names,+ created=True,+ )+ session.commit()++ return result++++def _update_metadata_with_filename(+ session: Session,+ reference_id: str,+ file_path: str | None,+ current_metadata: dict | None,+ user_metadata: dict[str, Any],+) -> None:+ computed_filename = compute_relative_filename(file_path) if file_path else None++ current_meta = current_metadata or {}+ new_meta = dict(current_meta)+ for k, v in user_metadata.items():+ new_meta[k] = v+ if computed_filename:+ new_meta["filename"] = computed_filename++ if new_meta != current_meta:+ set_reference_metadata(+ session,+ reference_id=reference_id,+ user_metadata=new_meta,+ )+++def _sanitize_filename(name: str | None, fallback: str) -> str:+ n = os.path.basename((name or "").strip() or fallback)+ return n if n else fallback+++class HashMismatchError(Exception):+ pass+++class DependencyMissingError(Exception):+ def __init__(self, message: str):+ self.message = message+ super().__init__(message)+++def upload_from_temp_path(+ temp_path: str,+ name: str | None = None,+ tags: list[str] | None = None,+ user_metadata: dict | None = None,+ client_filename: str | None = None,+ owner_id: str = "",+ expected_hash: str | None = None,+) -> UploadResult:+ try:+ digest, _ = hashing.compute_blake3_hash(temp_path)+ except ImportError as e:+ raise DependencyMissingError(str(e))+ except Exception as e:+ raise RuntimeError(f"failed to hash uploaded file: {e}")+ asset_hash = "blake3:" + digest++ if expected_hash and asset_hash != expected_hash.strip().lower():+ raise HashMismatchError("Uploaded file hash does not match provided hash.")++ with create_session() as session:+ existing = get_asset_by_hash(session, asset_hash=asset_hash)++ if existing is not None:+ with contextlib.suppress(Exception):+ if temp_path and os.path.exists(temp_path):+ os.remove(temp_path)++ display_name = _sanitize_filename(name or client_filename, fallback=digest)+ result = _register_existing_asset(+ asset_hash=asset_hash,+ name=display_name,+ user_metadata=user_metadata or {},+ tags=tags or [],+ tag_origin="manual",+ owner_id=owner_id,+ )+ return UploadResult(+ ref=result.ref,+ asset=result.asset,+ tags=result.tags,+ created_new=False,+ )++ if not tags:+ raise ValueError("tags are required for new asset uploads")+ base_dir, subdirs = resolve_destination_from_tags(tags)+ dest_dir = os.path.join(base_dir, *subdirs) if subdirs else base_dir+ os.makedirs(dest_dir, exist_ok=True)++ src_for_ext = (client_filename or name or "").strip()+ _ext = os.path.splitext(os.path.basename(src_for_ext))[1] if src_for_ext else ""+ ext = _ext if 0 < len(_ext) <= 16 else ""+ hashed_basename = f"{digest}{ext}"+ dest_abs = os.path.abspath(os.path.join(dest_dir, hashed_basename))+ validate_path_within_base(dest_abs, base_dir)++ content_type = (+ mimetypes.guess_type(os.path.basename(src_for_ext), strict=False)[0]+ or mimetypes.guess_type(hashed_basename, strict=False)[0]+ or "application/octet-stream"+ )++ try:+ os.replace(temp_path, dest_abs)+ except Exception as e:+ raise RuntimeError(f"failed to move uploaded file into place: {e}")++ try:+ size_bytes, mtime_ns = get_size_and_mtime_ns(dest_abs)+ except OSError as e:+ raise RuntimeError(f"failed to stat destination file: {e}")++ ingest_result = _ingest_file_from_path(+ asset_hash=asset_hash,+ abs_path=dest_abs,+ size_bytes=size_bytes,+ mtime_ns=mtime_ns,+ mime_type=content_type,+ info_name=_sanitize_filename(name or client_filename, fallback=digest),+ owner_id=owner_id,+ preview_id=None,+ user_metadata=user_metadata or {},+ tags=tags,+ tag_origin="manual",+ require_existing_tags=False,+ )+ reference_id = ingest_result.reference_id+ if not reference_id:+ raise RuntimeError("failed to create asset reference")++ with create_session() as session:+ pair = fetch_reference_and_asset(+ session, reference_id=reference_id, owner_id=owner_id+ )+ if not pair:+ raise RuntimeError("inconsistent DB state after ingest")+ ref, asset = pair+ tag_names = get_reference_tags(session, reference_id=ref.id)++ return UploadResult(+ ref=extract_reference_data(ref),+ asset=extract_asset_data(asset),+ tags=tag_names,+ created_new=ingest_result.asset_created,+ )+++def create_from_hash(+ hash_str: str,+ name: str,+ tags: list[str] | None = None,+ user_metadata: dict | None = None,+ owner_id: str = "",+) -> UploadResult | None:+ canonical = hash_str.strip().lower()++ with create_session() as session:+ asset = get_asset_by_hash(session, asset_hash=canonical)+ if not asset:+ return None++ result = _register_existing_asset(+ asset_hash=canonical,+ name=_sanitize_filename(+ name, fallback=canonical.split(":", 1)[1] if ":" in canonical else canonical+ ),+ user_metadata=user_metadata or {},+ tags=tags or [],+ tag_origin="manual",+ owner_id=owner_id,+ )++ return UploadResult(+ ref=result.ref,+ asset=result.asset,+ tags=result.tags,+ created_new=False,+ )diff --git a/app/assets/services/metadata_extract.py b/app/assets/services/metadata_extract.pynew file mode 100644index 000000000..a004929bc--- /dev/null+++ b/app/assets/services/metadata_extract.py@@ -0,0 +1,327 @@+"""Metadata extraction for asset scanning.++Tier 1: Filesystem metadata (zero parsing)+Tier 2: Safetensors header metadata (fast JSON read only)+"""++from __future__ import annotations++import json+import logging+import mimetypes+import os+import struct+from dataclasses import dataclass+from typing import Any++from utils.mime_types import init_mime_types++init_mime_types()++# Supported safetensors extensions+SAFETENSORS_EXTENSIONS = frozenset({".safetensors", ".sft"})++# Maximum safetensors header size to read (8MB)+MAX_SAFETENSORS_HEADER_SIZE = 8 * 1024 * 1024+++@dataclass+class ExtractedMetadata:+ """Metadata extracted from a file during scanning."""++ # Tier 1: Filesystem (always available)+ filename: str = ""+ file_path: str = "" # Full absolute path to the file+ content_length: int = 0+ content_type: str | None = None+ format: str = "" # file extension without dot++ # Tier 2: Safetensors header (if available)+ base_model: str | None = None+ trained_words: list[str] | None = None+ air: str | None = None # CivitAI AIR identifier+ has_preview_images: bool = False++ # Source provenance (populated if embedded in safetensors)+ source_url: str | None = None+ source_arn: str | None = None+ repo_url: str | None = None+ preview_url: str | None = None+ source_hash: str | None = None++ # HuggingFace specific+ repo_id: str | None = None+ revision: str | None = None+ filepath: str | None = None+ resolve_url: str | None = None++ def to_user_metadata(self) -> dict[str, Any]:+ """Convert to user_metadata dict for AssetReference.user_metadata JSON field."""+ data: dict[str, Any] = {+ "filename": self.filename,+ "content_length": self.content_length,+ "format": self.format,+ }+ if self.file_path:+ data["file_path"] = self.file_path+ if self.content_type:+ data["content_type"] = self.content_type++ # Tier 2 fields+ if self.base_model:+ data["base_model"] = self.base_model+ if self.trained_words:+ data["trained_words"] = self.trained_words+ if self.air:+ data["air"] = self.air+ if self.has_preview_images:+ data["has_preview_images"] = True++ # Source provenance+ if self.source_url:+ data["source_url"] = self.source_url+ if self.source_arn:+ data["source_arn"] = self.source_arn+ if self.repo_url:+ data["repo_url"] = self.repo_url+ if self.preview_url:+ data["preview_url"] = self.preview_url+ if self.source_hash:+ data["source_hash"] = self.source_hash++ # HuggingFace+ if self.repo_id:+ data["repo_id"] = self.repo_id+ if self.revision:+ data["revision"] = self.revision+ if self.filepath:+ data["filepath"] = self.filepath+ if self.resolve_url:+ data["resolve_url"] = self.resolve_url++ return data++ def to_meta_rows(self, reference_id: str) -> list[dict]:+ """Convert to asset_reference_meta rows for typed/indexed querying."""+ rows: list[dict] = []++ def add_str(key: str, val: str | None, ordinal: int = 0) -> None:+ if val:+ rows.append({+ "asset_reference_id": reference_id,+ "key": key,+ "ordinal": ordinal,+ "val_str": val[:2048] if len(val) > 2048 else val,+ "val_num": None,+ "val_bool": None,+ "val_json": None,+ })++ def add_num(key: str, val: int | float | None) -> None:+ if val is not None:+ rows.append({+ "asset_reference_id": reference_id,+ "key": key,+ "ordinal": 0,+ "val_str": None,+ "val_num": val,+ "val_bool": None,+ "val_json": None,+ })++ def add_bool(key: str, val: bool | None) -> None:+ if val is not None:+ rows.append({+ "asset_reference_id": reference_id,+ "key": key,+ "ordinal": 0,+ "val_str": None,+ "val_num": None,+ "val_bool": val,+ "val_json": None,+ })++ # Tier 1+ add_str("filename", self.filename)+ add_num("content_length", self.content_length)+ add_str("content_type", self.content_type)+ add_str("format", self.format)++ # Tier 2+ add_str("base_model", self.base_model)+ add_str("air", self.air)+ has_previews = self.has_preview_images if self.has_preview_images else None+ add_bool("has_preview_images", has_previews)++ # trained_words as multiple rows with ordinals+ if self.trained_words:+ for i, word in enumerate(self.trained_words[:100]): # limit to 100 words+ add_str("trained_words", word, ordinal=i)++ # Source provenance+ add_str("source_url", self.source_url)+ add_str("source_arn", self.source_arn)+ add_str("repo_url", self.repo_url)+ add_str("preview_url", self.preview_url)+ add_str("source_hash", self.source_hash)++ # HuggingFace+ add_str("repo_id", self.repo_id)+ add_str("revision", self.revision)+ add_str("filepath", self.filepath)+ add_str("resolve_url", self.resolve_url)++ return rows+++def _read_safetensors_header(+ path: str, max_size: int = MAX_SAFETENSORS_HEADER_SIZE+) -> dict[str, Any] | None:+ """Read only the JSON header from a safetensors file.++ This is very fast - reads 8 bytes for header length, then the JSON header.+ No tensor data is loaded.++ Args:+ path: Absolute path to safetensors file+ max_size: Maximum header size to read (default 8MB)++ Returns:+ Parsed header dict or None if failed+ """+ try:+ with open(path, "rb") as f:+ header_bytes = f.read(8)+ if len(header_bytes) < 8:+ return None+ length_of_header = struct.unpack("<Q", header_bytes)[0]+ if length_of_header > max_size:+ return None+ header_data = f.read(length_of_header)+ if len(header_data) < length_of_header:+ return None+ return json.loads(header_data.decode("utf-8"))+ except (OSError, json.JSONDecodeError, UnicodeDecodeError, struct.error):+ return None+++def _extract_safetensors_metadata(+ header: dict[str, Any], meta: ExtractedMetadata+) -> None:+ """Extract metadata from safetensors header __metadata__ section.++ Modifies meta in-place.+ """+ st_meta = header.get("__metadata__", {})+ if not isinstance(st_meta, dict):+ return++ # Common model metadata+ meta.base_model = (+ st_meta.get("ss_base_model_version")+ or st_meta.get("modelspec.base_model")+ or st_meta.get("base_model")+ )++ # Trained words / trigger words+ trained_words = st_meta.get("ss_tag_frequency")+ if trained_words and isinstance(trained_words, str):+ try:+ tag_freq = json.loads(trained_words)+ # Extract unique tags from all datasets+ all_tags: set[str] = set()+ for dataset_tags in tag_freq.values():+ if isinstance(dataset_tags, dict):+ all_tags.update(dataset_tags.keys())+ if all_tags:+ meta.trained_words = sorted(all_tags)[:100]+ except json.JSONDecodeError:+ pass++ # Direct trained_words field (some formats)+ if not meta.trained_words:+ tw = st_meta.get("trained_words")+ if isinstance(tw, str):+ try:+ parsed = json.loads(tw)+ if isinstance(parsed, list):+ meta.trained_words = [str(x) for x in parsed]+ else:+ meta.trained_words = [w.strip() for w in tw.split(",") if w.strip()]+ except json.JSONDecodeError:+ meta.trained_words = [w.strip() for w in tw.split(",") if w.strip()]+ elif isinstance(tw, list):+ meta.trained_words = [str(x) for x in tw]++ # CivitAI AIR+ meta.air = st_meta.get("air") or st_meta.get("modelspec.air")++ # Preview images (ssmd_cover_images)+ cover_images = st_meta.get("ssmd_cover_images")+ if cover_images:+ meta.has_preview_images = True++ # Source provenance fields+ meta.source_url = st_meta.get("source_url")+ meta.source_arn = st_meta.get("source_arn")+ meta.repo_url = st_meta.get("repo_url")+ meta.preview_url = st_meta.get("preview_url")+ meta.source_hash = st_meta.get("source_hash") or st_meta.get("sshs_model_hash")++ # HuggingFace fields+ meta.repo_id = st_meta.get("repo_id") or st_meta.get("hf_repo_id")+ meta.revision = st_meta.get("revision") or st_meta.get("hf_revision")+ meta.filepath = st_meta.get("filepath") or st_meta.get("hf_filepath")+ meta.resolve_url = st_meta.get("resolve_url") or st_meta.get("hf_url")+++def extract_file_metadata(+ abs_path: str,+ stat_result: os.stat_result | None = None,+ relative_filename: str | None = None,+) -> ExtractedMetadata:+ """Extract metadata from a file using tier 1 and tier 2 methods.++ Tier 1: Filesystem metadata from path and stat+ Tier 2: Safetensors header parsing if applicable++ Args:+ abs_path: Absolute path to the file+ stat_result: Optional pre-fetched stat result (saves a syscall)+ relative_filename: Optional relative filename to use instead of basename+ (e.g., "flux/123/model.safetensors" for model paths)++ Returns:+ ExtractedMetadata with all available fields populated+ """+ meta = ExtractedMetadata()++ # Tier 1: Filesystem metadata+ meta.filename = relative_filename or os.path.basename(abs_path)+ meta.file_path = abs_path+ _, ext = os.path.splitext(abs_path)+ meta.format = ext.lstrip(".").lower() if ext else ""++ mime_type, _ = mimetypes.guess_type(abs_path)+ meta.content_type = mime_type++ # Size from stat+ if stat_result is None:+ try:+ stat_result = os.stat(abs_path, follow_symlinks=True)+ except OSError:+ pass++ if stat_result:+ meta.content_length = stat_result.st_size++ # Tier 2: Safetensors header (if applicable and enabled)+ if ext.lower() in SAFETENSORS_EXTENSIONS:+ header = _read_safetensors_header(abs_path)+ if header:+ try:+ _extract_safetensors_metadata(header, meta)+ except Exception as e:+ logging.debug("Safetensors meta extract failed %s: %s", abs_path, e)++ return metadiff --git a/app/assets/services/path_utils.py b/app/assets/services/path_utils.pynew file mode 100644index 000000000..f5dd7f7fd--- /dev/null+++ b/app/assets/services/path_utils.py@@ -0,0 +1,167 @@+import os+from pathlib import Path+from typing import Literal++import folder_paths+from app.assets.helpers import normalize_tags+++_NON_MODEL_FOLDER_NAMES = frozenset({"custom_nodes"})+++def get_comfy_models_folders() -> list[tuple[str, list[str]]]:+ """Build list of (folder_name, base_paths[]) for all model locations.++ Includes every category registered in folder_names_and_paths,+ regardless of whether its paths are under the main models_dir,+ but excludes non-model entries like custom_nodes.+ """+ targets: list[tuple[str, list[str]]] = []+ for name, values in folder_paths.folder_names_and_paths.items():+ if name in _NON_MODEL_FOLDER_NAMES:+ continue+ paths, _exts = values[0], values[1]+ if paths:+ targets.append((name, paths))+ return targets+++def resolve_destination_from_tags(tags: list[str]) -> tuple[str, list[str]]:+ """Validates and maps tags -> (base_dir, subdirs_for_fs)"""+ if not tags:+ raise ValueError("tags must not be empty")+ root = tags[0].lower()+ if root == "models":+ if len(tags) < 2:+ raise ValueError("at least two tags required for model asset")+ try:+ bases = folder_paths.folder_names_and_paths[tags[1]][0]+ except KeyError:+ raise ValueError(f"unknown model category '{tags[1]}'")+ if not bases:+ raise ValueError(f"no base path configured for category '{tags[1]}'")+ base_dir = os.path.abspath(bases[0])+ raw_subdirs = tags[2:]+ elif root == "input":+ base_dir = os.path.abspath(folder_paths.get_input_directory())+ raw_subdirs = tags[1:]+ elif root == "output":+ base_dir = os.path.abspath(folder_paths.get_output_directory())+ raw_subdirs = tags[1:]+ else:+ raise ValueError(f"unknown root tag '{tags[0]}'; expected 'models', 'input', or 'output'")+ _sep_chars = frozenset(("/", "\\", os.sep))+ for i in raw_subdirs:+ if i in (".", "..") or _sep_chars & set(i):+ raise ValueError("invalid path component in tags")++ return base_dir, raw_subdirs if raw_subdirs else []+++def validate_path_within_base(candidate: str, base: str) -> None:+ cand_abs = Path(os.path.abspath(candidate))+ base_abs = Path(os.path.abspath(base))+ if not cand_abs.is_relative_to(base_abs):+ raise ValueError("destination escapes base directory")+++def compute_relative_filename(file_path: str) -> str | None:+ """+ Return the model's path relative to the last well-known folder (the model category),+ using forward slashes, eg:+ /.../models/checkpoints/flux/123/flux.safetensors -> "flux/123/flux.safetensors"+ /.../models/text_encoders/clip_g.safetensors -> "clip_g.safetensors"++ For non-model paths, returns None.+ """+ try:+ root_category, rel_path = get_asset_category_and_relative_path(file_path)+ except ValueError:+ return None++ p = Path(rel_path)+ parts = [seg for seg in p.parts if seg not in (".", "..", p.anchor)]+ if not parts:+ return None++ if root_category == "models":+ # parts[0] is the category ("checkpoints", "vae", etc) – drop it+ inside = parts[1:] if len(parts) > 1 else [parts[0]]+ return "/".join(inside)+ return "/".join(parts) # input/output: keep all parts+++def get_asset_category_and_relative_path(+ file_path: str,+) -> tuple[Literal["input", "output", "models"], str]:+ """Determine which root category a file path belongs to.++ Categories:+ - 'input': under folder_paths.get_input_directory()+ - 'output': under folder_paths.get_output_directory()+ - 'models': under any base path from get_comfy_models_folders()++ Returns:+ (root_category, relative_path_inside_that_root)++ Raises:+ ValueError: path does not belong to any known root.+ """+ fp_abs = os.path.abspath(file_path)++ def _check_is_within(child: str, parent: str) -> bool:+ return Path(child).is_relative_to(parent)++ def _compute_relative(child: str, parent: str) -> str:+ # Normalize relative path, stripping any leading ".." components+ # by anchoring to root (os.sep) then computing relpath back from it.+ return os.path.relpath(+ os.path.join(os.sep, os.path.relpath(child, parent)), os.sep+ )++ # 1) input+ input_base = os.path.abspath(folder_paths.get_input_directory())+ if _check_is_within(fp_abs, input_base):+ return "input", _compute_relative(fp_abs, input_base)++ # 2) output+ output_base = os.path.abspath(folder_paths.get_output_directory())+ if _check_is_within(fp_abs, output_base):+ return "output", _compute_relative(fp_abs, output_base)++ # 3) models (check deepest matching base to avoid ambiguity)+ best: tuple[int, str, str] | None = None # (base_len, bucket, rel_inside_bucket)+ for bucket, bases in get_comfy_models_folders():+ for b in bases:+ base_abs = os.path.abspath(b)+ if not _check_is_within(fp_abs, base_abs):+ continue+ cand = (len(base_abs), bucket, _compute_relative(fp_abs, base_abs))+ if best is None or cand[0] > best[0]:+ best = cand++ if best is not None:+ _, bucket, rel_inside = best+ combined = os.path.join(bucket, rel_inside)+ return "models", os.path.relpath(os.path.join(os.sep, combined), os.sep)++ raise ValueError(+ f"Path is not within input, output, or configured model bases: {file_path}"+ )+++def get_name_and_tags_from_asset_path(file_path: str) -> tuple[str, list[str]]:+ """Return (name, tags) derived from a filesystem path.++ - name: base filename with extension+ - tags: [root_category] + parent folder names in order++ Raises:+ ValueError: path does not belong to any known root.+ """+ root_category, some_path = get_asset_category_and_relative_path(file_path)+ p = Path(some_path)+ parent_parts = [+ part for part in p.parent.parts if part not in (".", "..", p.anchor)+ ]+ return p.name, list(dict.fromkeys(normalize_tags([root_category, *parent_parts])))diff --git a/app/assets/services/schemas.py b/app/assets/services/schemas.pynew file mode 100644index 000000000..8b1f1f4dc--- /dev/null+++ b/app/assets/services/schemas.py@@ -0,0 +1,109 @@+from dataclasses import dataclass+from datetime import datetime+from typing import Any, NamedTuple++from app.assets.database.models import Asset, AssetReference++UserMetadata = dict[str, Any] | None+++@dataclass(frozen=True)+class AssetData:+ hash: str | None+ size_bytes: int | None+ mime_type: str | None+++@dataclass(frozen=True)+class ReferenceData:+ """Data transfer object for AssetReference."""++ id: str+ name: str+ file_path: str | None+ user_metadata: UserMetadata+ preview_id: str | None+ created_at: datetime+ updated_at: datetime+ last_access_time: datetime | None+++@dataclass(frozen=True)+class AssetDetailResult:+ ref: ReferenceData+ asset: AssetData | None+ tags: list[str]+++@dataclass(frozen=True)+class RegisterAssetResult:+ ref: ReferenceData+ asset: AssetData+ tags: list[str]+ created: bool+++@dataclass(frozen=True)+class IngestResult:+ asset_created: bool+ asset_updated: bool+ ref_created: bool+ ref_updated: bool+ reference_id: str | None+++class TagUsage(NamedTuple):+ name: str+ tag_type: str+ count: int+++@dataclass(frozen=True)+class AssetSummaryData:+ ref: ReferenceData+ asset: AssetData | None+ tags: list[str]+++@dataclass(frozen=True)+class ListAssetsResult:+ items: list[AssetSummaryData]+ total: int+++@dataclass(frozen=True)+class DownloadResolutionResult:+ abs_path: str+ content_type: str+ download_name: str+++@dataclass(frozen=True)+class UploadResult:+ ref: ReferenceData+ asset: AssetData+ tags: list[str]+ created_new: bool+++def extract_reference_data(ref: AssetReference) -> ReferenceData:+ return ReferenceData(+ id=ref.id,+ name=ref.name,+ file_path=ref.file_path,+ user_metadata=ref.user_metadata,+ preview_id=ref.preview_id,+ created_at=ref.created_at,+ updated_at=ref.updated_at,+ last_access_time=ref.last_access_time,+ )+++def extract_asset_data(asset: Asset | None) -> AssetData | None:+ if asset is None:+ return None+ return AssetData(+ hash=asset.hash,+ size_bytes=asset.size_bytes,+ mime_type=asset.mime_type,+ )diff --git a/app/assets/services/tagging.py b/app/assets/services/tagging.pynew file mode 100644index 000000000..28900464d--- /dev/null+++ b/app/assets/services/tagging.py@@ -0,0 +1,75 @@+from app.assets.database.queries import (+ AddTagsResult,+ RemoveTagsResult,+ add_tags_to_reference,+ get_reference_with_owner_check,+ list_tags_with_usage,+ remove_tags_from_reference,+)+from app.assets.services.schemas import TagUsage+from app.database.db import create_session+++def apply_tags(+ reference_id: str,+ tags: list[str],+ origin: str = "manual",+ owner_id: str = "",+) -> AddTagsResult:+ with create_session() as session:+ ref_row = get_reference_with_owner_check(session, reference_id, owner_id)++ result = add_tags_to_reference(+ session,+ reference_id=reference_id,+ tags=tags,+ origin=origin,+ create_if_missing=True,+ reference_row=ref_row,+ )+ session.commit()++ return result+++def remove_tags(+ reference_id: str,+ tags: list[str],+ owner_id: str = "",+) -> RemoveTagsResult:+ with create_session() as session:+ get_reference_with_owner_check(session, reference_id, owner_id)++ result = remove_tags_from_reference(+ session,+ reference_id=reference_id,+ tags=tags,+ )+ session.commit()++ return result+++def list_tags(+ prefix: str | None = None,+ limit: int = 100,+ offset: int = 0,+ order: str = "count_desc",+ include_zero: bool = True,+ owner_id: str = "",+) -> tuple[list[TagUsage], int]:+ limit = max(1, min(1000, limit))+ offset = max(0, offset)++ with create_session() as session:+ rows, total = list_tags_with_usage(+ session,+ prefix=prefix,+ limit=limit,+ offset=offset,+ include_zero=include_zero,+ order=order,+ owner_id=owner_id,+ )++ return [TagUsage(name, tag_type, count) for name, tag_type, count in rows], totaldiff --git a/app/database/db.py b/app/database/db.pyindex 1de8b80ed..0aab09a49 100644--- a/app/database/db.py+++ b/app/database/db.py@@ -3,6 +3,7 @@ import osimport shutilfrom app.logger import log_startup_warningfrom utils.install_util import get_missing_requirements_message+from filelock import FileLock, Timeoutfrom comfy.cli_args import args_DB_AVAILABLE = False@@ -14,8 +15,12 @@ try:from alembic.config import Configfrom alembic.runtime.migration import MigrationContextfrom alembic.script import ScriptDirectory- from sqlalchemy import create_engine+ from sqlalchemy import create_engine, eventfrom sqlalchemy.orm import sessionmaker+ from sqlalchemy.pool import StaticPool++ from app.database.models import Base+ import app.assets.database.models # noqa: F401 — register models with Base.metadata_DB_AVAILABLE = Trueexcept ImportError as e:@@ -65,9 +70,69 @@ def get_db_path():raise ValueError(f"Unsupported database URL '{url}'.")+_db_lock = None++def _acquire_file_lock(db_path):+ """Acquire an OS-level file lock to prevent multi-process access.++ Uses filelock for cross-platform support (macOS, Linux, Windows).+ The OS automatically releases the lock when the process exits, even on crashes.+ """+ global _db_lock+ lock_path = db_path + ".lock"+ _db_lock = FileLock(lock_path)+ try:+ _db_lock.acquire(timeout=0)+ except Timeout:+ raise RuntimeError(+ f"Could not acquire lock on database '{db_path}'. "+ "Another ComfyUI process may already be using it. "+ "Use --database-url to specify a separate database file."+ )+++def _is_memory_db(db_url):+ """Check if the database URL refers to an in-memory SQLite database."""+ return db_url in ("sqlite:///:memory:", "sqlite://")++def init_db():db_url = args.database_urllogging.debug(f"Database URL: {db_url}")++ if _is_memory_db(db_url):+ _init_memory_db(db_url)+ else:+ _init_file_db(db_url)+++def _init_memory_db(db_url):+ """Initialize an in-memory SQLite database using metadata.create_all.++ Alembic migrations don't work with in-memory SQLite because each+ connection gets its own separate database — tables created by Alembic's+ internal connection are lost immediately.+ """+ engine = create_engine(+ db_url,+ poolclass=StaticPool,+ connect_args={"check_same_thread": False},+ )++ @event.listens_for(engine, "connect")+ def set_sqlite_pragma(dbapi_connection, connection_record):+ cursor = dbapi_connection.cursor()+ cursor.execute("PRAGMA foreign_keys=ON")+ cursor.close()++ Base.metadata.create_all(engine)++ global Session+ Session = sessionmaker(bind=engine)+++def _init_file_db(db_url):+ """Initialize a file-backed SQLite database using Alembic migrations."""db_path = get_db_path()db_exists = os.path.exists(db_path)@@ -75,6 +140,14 @@ def init_db():# Check if we need to upgradeengine = create_engine(db_url)++ # Enable foreign key enforcement for SQLite+ @event.listens_for(engine, "connect")+ def set_sqlite_pragma(dbapi_connection, connection_record):+ cursor = dbapi_connection.cursor()+ cursor.execute("PRAGMA foreign_keys=ON")+ cursor.close()+conn = engine.connect()context = MigrationContext.configure(conn)@@ -104,6 +177,12 @@ def init_db():logging.exception("Error upgrading database: ")raise e+ # Acquire an OS-level file lock after migrations are complete.+ # Alembic uses its own connection, so we must wait until it's done+ # before locking — otherwise our own lock blocks the migration.+ conn.close()+ _acquire_file_lock(db_path)+global SessionSession = sessionmaker(bind=engine)diff --git a/comfy/cli_args.py b/comfy/cli_args.pyindex 13079c7bc..e9832acaf 100644--- a/comfy/cli_args.py+++ b/comfy/cli_args.py@@ -232,7 +232,7 @@ database_default_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "user", "comfyui.db"))parser.add_argument("--database-url", type=str, default=f"sqlite:///{database_default_path}", help="Specify the database URL, e.g. for an in-memory database you can use 'sqlite:///:memory:'.")-parser.add_argument("--disable-assets-autoscan", action="store_true", help="Disable asset scanning on startup for database synchronization.")+parser.add_argument("--enable-assets", action="store_true", help="Enable the assets system (API routes, database synchronization, and background scanning).")if comfy.options.args_parsing:args = parser.parse_args()diff --git a/comfy_api/feature_flags.py b/comfy_api/feature_flags.pyindex a90a5ca40..9f6918315 100644--- a/comfy_api/feature_flags.py+++ b/comfy_api/feature_flags.py@@ -15,6 +15,7 @@ SERVER_FEATURE_FLAGS: dict[str, Any] = {"max_upload_size": args.max_upload_size * 1024 * 1024, # Convert MB to bytes"extension": {"manager": {"supports_v4": True}},"node_replacements": True,+ "assets": args.enable_assets,}diff --git a/main.py b/main.pyindex 0f58d57b8..a8fc1a28d 100644--- a/main.py+++ b/main.py@@ -7,14 +7,16 @@ import folder_pathsimport timefrom comfy.cli_args import args, enables_dynamic_vramfrom app.logger import setup_logger-from app.assets.scanner import seed_assets+from app.assets.seeder import asset_seederimport itertoolsimport utils.extra_config+from utils.mime_types import init_mime_typesimport loggingimport sysfrom comfy_execution.progress import get_progress_statefrom comfy_execution.utils import get_executing_contextfrom comfy_api import feature_flags+from app.database.db import init_db, dependencies_availableif __name__ == "__main__":#NOTE: These do not do anything on core ComfyUI, they are for custom nodes.@@ -161,6 +163,7 @@ def execute_prestartup_script():logging.info("")apply_custom_paths()+init_mime_types()if args.enable_manager:comfyui_manager.prestartup()@@ -258,6 +261,7 @@ def prompt_worker(q, server_instance):for k in sensitive:extra_data[k] = sensitive[k]+ asset_seeder.pause()e.execute(item[2], prompt_id, extra_data, item[4])need_gc = True@@ -302,6 +306,7 @@ def prompt_worker(q, server_instance):last_gc_collect = current_timeneed_gc = Falsehook_breaker_ac10a0.restore_functions()+ asset_seeder.resume()async def run(server_instance, address='', port=8188, verbose=True, call_on_start=None):@@ -352,12 +357,29 @@ def cleanup_temp():def setup_database():try:- from app.database.db import init_db, dependencies_availableif dependencies_available():init_db()- if not args.disable_assets_autoscan:- seed_assets(["models"], enable_logging=True)+ if args.enable_assets:+ if asset_seeder.start(roots=("models", "input", "output"), prune_first=True, compute_hashes=True):+ logging.info("Background asset scan initiated for models, input, output")except Exception as e:+ if "database is locked" in str(e):+ logging.error(+ "Database is locked. Another ComfyUI process is already using this database.\n"+ "To resolve this, specify a separate database file for this instance:\n"+ " --database-url sqlite:///path/to/another.db"+ )+ sys.exit(1)+ if args.enable_assets:+ logging.error(+ f"Failed to initialize database: {e}\n"+ "The --enable-assets flag requires a working database connection.\n"+ "To resolve this, try one of the following:\n"+ " 1. Install the latest requirements: pip install -r requirements.txt\n"+ " 2. Specify an alternative database URL: --database-url sqlite:///path/to/your.db\n"+ " 3. Use an in-memory database: --database-url sqlite:///:memory:"+ )+ sys.exit(1)logging.error(f"Failed to initialize database. Please ensure you have installed the latest requirements. If the error persists, please report this as in future the database will be required: {e}")@@ -440,5 +462,6 @@ if __name__ == "__main__":event_loop.run_until_complete(x)except KeyboardInterrupt:logging.info("\nStopped server")-- cleanup_temp()+ finally:+ asset_seeder.shutdown()+ cleanup_temp()diff --git a/requirements.txt b/requirements.txtindex dc9a9ded0..9527135ec 100644--- a/requirements.txt+++ b/requirements.txt@@ -20,11 +20,13 @@ tqdmpsutilalembicSQLAlchemy+filelockav>=14.2.0comfy-kitchen>=0.2.7comfy-aimdo>=0.2.7requestssimpleeval>=1.0.0+blake3#non essential dependencies:kornia>=0.7.1diff --git a/server.py b/server.pyindex 275bce5a7..76904ebc9 100644--- a/server.py+++ b/server.py@@ -33,8 +33,8 @@ import node_helpersfrom comfyui_version import __version__from app.frontend_management import FrontendManager, parse_versionfrom comfy_api.internal import _ComfyNodeInternal-from app.assets.scanner import seed_assets-from app.assets.api.routes import register_assets_system+from app.assets.seeder import asset_seeder+from app.assets.api.routes import register_assets_routesfrom app.user_manager import UserManagerfrom app.model_manager import ModelFileManager@@ -197,10 +197,6 @@ class PromptServer():def __init__(self, loop):PromptServer.instance = self- mimetypes.init()- mimetypes.add_type('application/javascript; charset=utf-8', '.js')- mimetypes.add_type('image/webp', '.webp')-self.user_manager = UserManager()self.model_file_manager = ModelFileManager()self.custom_node_manager = CustomNodeManager()@@ -239,7 +235,11 @@ class PromptServer():else args.front_end_root)logging.info(f"[Prompt Server] web root: {self.web_root}")- register_assets_system(self.app, self.user_manager)+ if args.enable_assets:+ register_assets_routes(self.app, self.user_manager)+ else:+ register_assets_routes(self.app)+ asset_seeder.disable()routes = web.RouteTableDef()self.routes = routesself.last_node_id = None@@ -697,10 +697,7 @@ class PromptServer():@routes.get("/object_info")async def get_object_info(request):- try:- seed_assets(["models"])- except Exception as e:- logging.error(f"Failed to seed assets: {e}")+ asset_seeder.start(roots=("models", "input", "output"))with folder_paths.cache_helper:out = {}for x in nodes.NODE_CLASS_MAPPINGS:diff --git a/tests-unit/assets_test/conftest.py b/tests-unit/assets_test/conftest.pyindex 0a57dd7b5..6c5c56113 100644--- a/tests-unit/assets_test/conftest.py+++ b/tests-unit/assets_test/conftest.py@@ -108,7 +108,7 @@ def comfy_url_and_proc(comfy_tmp_base_dir: Path, request: pytest.FixtureRequest)"main.py",f"--base-directory={str(comfy_tmp_base_dir)}",f"--database-url={db_url}",- "--disable-assets-autoscan",+ "--enable-assets","--listen","127.0.0.1","--port",@@ -212,7 +212,7 @@ def asset_factory(http: requests.Session, api_base: str):for aid in created:with contextlib.suppress(Exception):- http.delete(f"{api_base}/api/assets/{aid}", timeout=30)+ http.delete(f"{api_base}/api/assets/{aid}?delete_content=true", timeout=30)@pytest.fixture@@ -258,14 +258,4 @@ def autoclean_unit_test_assets(http: requests.Session, api_base: str):breakfor aid in ids:with contextlib.suppress(Exception):- http.delete(f"{api_base}/api/assets/{aid}", timeout=30)---def trigger_sync_seed_assets(session: requests.Session, base_url: str) -> None:- """Force a fast sync/seed pass by calling the seed endpoint."""- session.post(base_url + "/api/assets/seed", json={"roots": ["models", "input", "output"]}, timeout=30)- time.sleep(0.2)---def get_asset_filename(asset_hash: str, extension: str) -> str:- return asset_hash.removeprefix("blake3:") + extension+ http.delete(f"{api_base}/api/assets/{aid}?delete_content=true", timeout=30)diff --git a/tests-unit/assets_test/helpers.py b/tests-unit/assets_test/helpers.pynew file mode 100644index 000000000..770e011f4--- /dev/null+++ b/tests-unit/assets_test/helpers.py@@ -0,0 +1,28 @@+"""Helper functions for assets integration tests."""+import time++import requests+++def trigger_sync_seed_assets(session: requests.Session, base_url: str) -> None:+ """Force a synchronous sync/seed pass by calling the seed endpoint with wait=true.++ Retries on 409 (already running) until the previous scan finishes.+ """+ deadline = time.monotonic() + 60+ while True:+ r = session.post(+ base_url + "/api/assets/seed?wait=true",+ json={"roots": ["models", "input", "output"]},+ timeout=60,+ )+ if r.status_code != 409:+ assert r.status_code == 200, f"seed endpoint returned {r.status_code}: {r.text}"+ return+ if time.monotonic() > deadline:+ raise TimeoutError("seed endpoint stuck in 409 (already running)")+ time.sleep(0.25)+++def get_asset_filename(asset_hash: str, extension: str) -> str:+ return asset_hash.removeprefix("blake3:") + extensiondiff --git a/tests-unit/assets_test/queries/conftest.py b/tests-unit/assets_test/queries/conftest.pynew file mode 100644index 000000000..4ca0e86a9--- /dev/null+++ b/tests-unit/assets_test/queries/conftest.py@@ -0,0 +1,20 @@+import pytest+from sqlalchemy import create_engine+from sqlalchemy.orm import Session++from app.assets.database.models import Base+++def session():+ """In-memory SQLite session for fast unit tests."""+ engine = create_engine("sqlite:///:memory:")+ Base.metadata.create_all(engine)+ with Session(engine) as sess:+ yield sess++[email protected](autouse=True)+def autoclean_unit_test_assets():+ """Override parent autouse fixture - query tests don't need server cleanup."""+ yielddiff --git a/tests-unit/assets_test/queries/test_asset.py b/tests-unit/assets_test/queries/test_asset.pynew file mode 100644index 000000000..08f84cd11--- /dev/null+++ b/tests-unit/assets_test/queries/test_asset.py@@ -0,0 +1,144 @@+import uuid++import pytest+from sqlalchemy.orm import Session++from app.assets.helpers import get_utc_now+from app.assets.database.models import Asset+from app.assets.database.queries import (+ asset_exists_by_hash,+ get_asset_by_hash,+ upsert_asset,+ bulk_insert_assets,+)+++class TestAssetExistsByHash:+ @pytest.mark.parametrize(+ "setup_hash,query_hash,expected",+ [+ (None, "nonexistent", False), # No asset exists+ ("blake3:abc123", "blake3:abc123", True), # Asset exists with matching hash+ (None, "", False), # Null hash in DB doesn't match empty string+ ],+ ids=["nonexistent", "existing", "null_hash_no_match"],+ )+ def test_exists_by_hash(self, session: Session, setup_hash, query_hash, expected):+ if setup_hash is not None or query_hash == "":+ asset = Asset(hash=setup_hash, size_bytes=100)+ session.add(asset)+ session.commit()++ assert asset_exists_by_hash(session, asset_hash=query_hash) is expected+++class TestGetAssetByHash:+ @pytest.mark.parametrize(+ "setup_hash,query_hash,should_find",+ [+ (None, "nonexistent", False),+ ("blake3:def456", "blake3:def456", True),+ ],+ ids=["nonexistent", "existing"],+ )+ def test_get_by_hash(self, session: Session, setup_hash, query_hash, should_find):+ if setup_hash is not None:+ asset = Asset(hash=setup_hash, size_bytes=200, mime_type="image/png")+ session.add(asset)+ session.commit()++ result = get_asset_by_hash(session, asset_hash=query_hash)+ if should_find:+ assert result is not None+ assert result.size_bytes == 200+ assert result.mime_type == "image/png"+ else:+ assert result is None+++class TestUpsertAsset:+ @pytest.mark.parametrize(+ "first_size,first_mime,second_size,second_mime,expect_created,expect_updated,final_size,final_mime",+ [+ # New asset creation+ (None, None, 1024, "application/octet-stream", True, False, 1024, "application/octet-stream"),+ # Existing asset, same values - no update+ (500, "text/plain", 500, "text/plain", False, False, 500, "text/plain"),+ # Existing asset with size 0, update with new values+ (0, None, 2048, "image/png", False, True, 2048, "image/png"),+ # Existing asset, second call with size 0 - no update+ (1000, None, 0, None, False, False, 1000, None),+ ],+ ids=["new_asset", "existing_no_change", "update_from_zero", "zero_size_no_update"],+ )+ def test_upsert_scenarios(+ self,+ session: Session,+ first_size,+ first_mime,+ second_size,+ second_mime,+ expect_created,+ expect_updated,+ final_size,+ final_mime,+ ):+ asset_hash = f"blake3:test_{first_size}_{second_size}"++ # First upsert (if first_size is not None, we're testing the second call)+ if first_size is not None:+ upsert_asset(+ session,+ asset_hash=asset_hash,+ size_bytes=first_size,+ mime_type=first_mime,+ )+ session.commit()++ # The upsert call we're testing+ asset, created, updated = upsert_asset(+ session,+ asset_hash=asset_hash,+ size_bytes=second_size,+ mime_type=second_mime,+ )+ session.commit()++ assert created is expect_created+ assert updated is expect_updated+ assert asset.size_bytes == final_size+ assert asset.mime_type == final_mime+++class TestBulkInsertAssets:+ def test_inserts_multiple_assets(self, session: Session):+ now = get_utc_now()+ rows = [+ {"id": str(uuid.uuid4()), "hash": "blake3:bulk1", "size_bytes": 100, "mime_type": "text/plain", "created_at": now},+ {"id": str(uuid.uuid4()), "hash": "blake3:bulk2", "size_bytes": 200, "mime_type": "image/png", "created_at": now},+ {"id": str(uuid.uuid4()), "hash": "blake3:bulk3", "size_bytes": 300, "mime_type": None, "created_at": now},+ ]+ bulk_insert_assets(session, rows)+ session.commit()++ assets = session.query(Asset).all()+ assert len(assets) == 3+ hashes = {a.hash for a in assets}+ assert hashes == {"blake3:bulk1", "blake3:bulk2", "blake3:bulk3"}++ def test_empty_list_is_noop(self, session: Session):+ bulk_insert_assets(session, [])+ session.commit()+ assert session.query(Asset).count() == 0++ def test_handles_large_batch(self, session: Session):+ """Test chunking logic with more rows than MAX_BIND_PARAMS allows."""+ now = get_utc_now()+ rows = [+ {"id": str(uuid.uuid4()), "hash": f"blake3:large{i}", "size_bytes": i, "mime_type": None, "created_at": now}+ for i in range(200)+ ]+ bulk_insert_assets(session, rows)+ session.commit()++ assert session.query(Asset).count() == 200diff --git a/tests-unit/assets_test/queries/test_asset_info.py b/tests-unit/assets_test/queries/test_asset_info.pynew file mode 100644index 000000000..8f6c7fcdb--- /dev/null+++ b/tests-unit/assets_test/queries/test_asset_info.py@@ -0,0 +1,517 @@+import time+import uuid+import pytest+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference, AssetReferenceMeta+from app.assets.database.queries import (+ reference_exists_for_asset_id,+ get_reference_by_id,+ insert_reference,+ get_or_create_reference,+ update_reference_timestamps,+ list_references_page,+ fetch_reference_asset_and_tags,+ fetch_reference_and_asset,+ update_reference_access_time,+ set_reference_metadata,+ delete_reference_by_id,+ set_reference_preview,+ bulk_insert_references_ignore_conflicts,+ get_reference_ids_by_ids,+ ensure_tags_exist,+ add_tags_to_reference,+)+from app.assets.helpers import get_utc_now+++def _make_asset(session: Session, hash_val: str | None = None, size: int = 1024) -> Asset:+ asset = Asset(hash=hash_val, size_bytes=size, mime_type="application/octet-stream")+ session.add(asset)+ session.flush()+ return asset+++def _make_reference(+ session: Session,+ asset: Asset,+ name: str = "test",+ owner_id: str = "",+) -> AssetReference:+ now = get_utc_now()+ ref = AssetReference(+ owner_id=owner_id,+ name=name,+ asset_id=asset.id,+ created_at=now,+ updated_at=now,+ last_access_time=now,+ )+ session.add(ref)+ session.flush()+ return ref+++class TestReferenceExistsForAssetId:+ def test_returns_false_when_no_reference(self, session: Session):+ asset = _make_asset(session, "hash1")+ assert reference_exists_for_asset_id(session, asset_id=asset.id) is False++ def test_returns_true_when_reference_exists(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset)+ assert reference_exists_for_asset_id(session, asset_id=asset.id) is True+++class TestGetReferenceById:+ def test_returns_none_for_nonexistent(self, session: Session):+ assert get_reference_by_id(session, reference_id="nonexistent") is None++ def test_returns_reference(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset, name="myfile.txt")++ result = get_reference_by_id(session, reference_id=ref.id)+ assert result is not None+ assert result.name == "myfile.txt"+++class TestListReferencesPage:+ def test_empty_db(self, session: Session):+ refs, tag_map, total = list_references_page(session)+ assert refs == []+ assert tag_map == {}+ assert total == 0++ def test_returns_references_with_tags(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset, name="test.bin")+ ensure_tags_exist(session, ["alpha", "beta"])+ add_tags_to_reference(session, reference_id=ref.id, tags=["alpha", "beta"])+ session.commit()++ refs, tag_map, total = list_references_page(session)+ assert len(refs) == 1+ assert refs[0].id == ref.id+ assert set(tag_map[ref.id]) == {"alpha", "beta"}+ assert total == 1++ def test_name_contains_filter(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, name="model_v1.safetensors")+ _make_reference(session, asset, name="config.json")+ session.commit()++ refs, _, total = list_references_page(session, name_contains="model")+ assert total == 1+ assert refs[0].name == "model_v1.safetensors"++ def test_owner_visibility(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, name="public", owner_id="")+ _make_reference(session, asset, name="private", owner_id="user1")+ session.commit()++ # Empty owner sees only public+ refs, _, total = list_references_page(session, owner_id="")+ assert total == 1+ assert refs[0].name == "public"++ # Owner sees both+ refs, _, total = list_references_page(session, owner_id="user1")+ assert total == 2++ def test_include_tags_filter(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref1 = _make_reference(session, asset, name="tagged")+ _make_reference(session, asset, name="untagged")+ ensure_tags_exist(session, ["wanted"])+ add_tags_to_reference(session, reference_id=ref1.id, tags=["wanted"])+ session.commit()++ refs, _, total = list_references_page(session, include_tags=["wanted"])+ assert total == 1+ assert refs[0].name == "tagged"++ def test_exclude_tags_filter(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, name="keep")+ ref_exclude = _make_reference(session, asset, name="exclude")+ ensure_tags_exist(session, ["bad"])+ add_tags_to_reference(session, reference_id=ref_exclude.id, tags=["bad"])+ session.commit()++ refs, _, total = list_references_page(session, exclude_tags=["bad"])+ assert total == 1+ assert refs[0].name == "keep"++ def test_sorting(self, session: Session):+ asset = _make_asset(session, "hash1", size=100)+ asset2 = _make_asset(session, "hash2", size=500)+ _make_reference(session, asset, name="small")+ _make_reference(session, asset2, name="large")+ session.commit()++ refs, _, _ = list_references_page(session, sort="size", order="desc")+ assert refs[0].name == "large"++ refs, _, _ = list_references_page(session, sort="name", order="asc")+ assert refs[0].name == "large"+++class TestFetchReferenceAssetAndTags:+ def test_returns_none_for_nonexistent(self, session: Session):+ result = fetch_reference_asset_and_tags(session, "nonexistent")+ assert result is None++ def test_returns_tuple(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset, name="test.bin")+ ensure_tags_exist(session, ["tag1"])+ add_tags_to_reference(session, reference_id=ref.id, tags=["tag1"])+ session.commit()++ result = fetch_reference_asset_and_tags(session, ref.id)+ assert result is not None+ ret_ref, ret_asset, ret_tags = result+ assert ret_ref.id == ref.id+ assert ret_asset.id == asset.id+ assert ret_tags == ["tag1"]+++class TestFetchReferenceAndAsset:+ def test_returns_none_for_nonexistent(self, session: Session):+ result = fetch_reference_and_asset(session, reference_id="nonexistent")+ assert result is None++ def test_returns_tuple(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ session.commit()++ result = fetch_reference_and_asset(session, reference_id=ref.id)+ assert result is not None+ ret_ref, ret_asset = result+ assert ret_ref.id == ref.id+ assert ret_asset.id == asset.id+++class TestUpdateReferenceAccessTime:+ def test_updates_last_access_time(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ original_time = ref.last_access_time+ session.commit()++ import time+ time.sleep(0.01)++ update_reference_access_time(session, reference_id=ref.id)+ session.commit()++ session.refresh(ref)+ assert ref.last_access_time > original_time+++class TestDeleteReferenceById:+ def test_deletes_existing(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ session.commit()++ result = delete_reference_by_id(session, reference_id=ref.id, owner_id="")+ assert result is True+ assert get_reference_by_id(session, reference_id=ref.id) is None++ def test_returns_false_for_nonexistent(self, session: Session):+ result = delete_reference_by_id(session, reference_id="nonexistent", owner_id="")+ assert result is False++ def test_respects_owner_visibility(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset, owner_id="user1")+ session.commit()++ result = delete_reference_by_id(session, reference_id=ref.id, owner_id="user2")+ assert result is False+ assert get_reference_by_id(session, reference_id=ref.id) is not None+++class TestSetReferencePreview:+ def test_sets_preview(self, session: Session):+ asset = _make_asset(session, "hash1")+ preview_asset = _make_asset(session, "preview_hash")+ ref = _make_reference(session, asset)+ session.commit()++ set_reference_preview(session, reference_id=ref.id, preview_asset_id=preview_asset.id)+ session.commit()++ session.refresh(ref)+ assert ref.preview_id == preview_asset.id++ def test_clears_preview(self, session: Session):+ asset = _make_asset(session, "hash1")+ preview_asset = _make_asset(session, "preview_hash")+ ref = _make_reference(session, asset)+ ref.preview_id = preview_asset.id+ session.commit()++ set_reference_preview(session, reference_id=ref.id, preview_asset_id=None)+ session.commit()++ session.refresh(ref)+ assert ref.preview_id is None++ def test_raises_for_nonexistent_reference(self, session: Session):+ with pytest.raises(ValueError, match="not found"):+ set_reference_preview(session, reference_id="nonexistent", preview_asset_id=None)++ def test_raises_for_nonexistent_preview(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ session.commit()++ with pytest.raises(ValueError, match="Preview Asset"):+ set_reference_preview(session, reference_id=ref.id, preview_asset_id="nonexistent")+++class TestInsertReference:+ def test_creates_new_reference(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = insert_reference(+ session, asset_id=asset.id, owner_id="user1", name="test.bin"+ )+ session.commit()++ assert ref is not None+ assert ref.name == "test.bin"+ assert ref.owner_id == "user1"++ def test_allows_duplicate_names(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref1 = insert_reference(session, asset_id=asset.id, owner_id="user1", name="dup.bin")+ session.commit()++ # Duplicate names are now allowed+ ref2 = insert_reference(+ session, asset_id=asset.id, owner_id="user1", name="dup.bin"+ )+ session.commit()++ assert ref1 is not None+ assert ref2 is not None+ assert ref1.id != ref2.id+++class TestGetOrCreateReference:+ def test_creates_new_reference(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref, created = get_or_create_reference(+ session, asset_id=asset.id, owner_id="user1", name="new.bin"+ )+ session.commit()++ assert created is True+ assert ref.name == "new.bin"++ def test_always_creates_new_reference(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref1, created1 = get_or_create_reference(+ session, asset_id=asset.id, owner_id="user1", name="existing.bin"+ )+ session.commit()++ # Duplicate names are allowed, so always creates new+ ref2, created2 = get_or_create_reference(+ session, asset_id=asset.id, owner_id="user1", name="existing.bin"+ )+ session.commit()++ assert created1 is True+ assert created2 is True+ assert ref1.id != ref2.id+++class TestUpdateReferenceTimestamps:+ def test_updates_timestamps(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ original_updated_at = ref.updated_at+ session.commit()++ time.sleep(0.01)+ update_reference_timestamps(session, ref)+ session.commit()++ session.refresh(ref)+ assert ref.updated_at > original_updated_at++ def test_updates_preview_id(self, session: Session):+ asset = _make_asset(session, "hash1")+ preview_asset = _make_asset(session, "preview_hash")+ ref = _make_reference(session, asset)+ session.commit()++ update_reference_timestamps(session, ref, preview_id=preview_asset.id)+ session.commit()++ session.refresh(ref)+ assert ref.preview_id == preview_asset.id+++class TestSetReferenceMetadata:+ def test_sets_metadata(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ session.commit()++ set_reference_metadata(+ session, reference_id=ref.id, user_metadata={"key": "value"}+ )+ session.commit()++ session.refresh(ref)+ assert ref.user_metadata == {"key": "value"}+ # Check metadata table+ meta = session.query(AssetReferenceMeta).filter_by(asset_reference_id=ref.id).all()+ assert len(meta) == 1+ assert meta[0].key == "key"+ assert meta[0].val_str == "value"++ def test_replaces_existing_metadata(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ session.commit()++ set_reference_metadata(+ session, reference_id=ref.id, user_metadata={"old": "data"}+ )+ session.commit()++ set_reference_metadata(+ session, reference_id=ref.id, user_metadata={"new": "data"}+ )+ session.commit()++ meta = session.query(AssetReferenceMeta).filter_by(asset_reference_id=ref.id).all()+ assert len(meta) == 1+ assert meta[0].key == "new"++ def test_clears_metadata_with_empty_dict(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ session.commit()++ set_reference_metadata(+ session, reference_id=ref.id, user_metadata={"key": "value"}+ )+ session.commit()++ set_reference_metadata(+ session, reference_id=ref.id, user_metadata={}+ )+ session.commit()++ session.refresh(ref)+ assert ref.user_metadata == {}+ meta = session.query(AssetReferenceMeta).filter_by(asset_reference_id=ref.id).all()+ assert len(meta) == 0++ def test_raises_for_nonexistent(self, session: Session):+ with pytest.raises(ValueError, match="not found"):+ set_reference_metadata(+ session, reference_id="nonexistent", user_metadata={"key": "value"}+ )+++class TestBulkInsertReferencesIgnoreConflicts:+ def test_inserts_multiple_references(self, session: Session):+ asset = _make_asset(session, "hash1")+ now = get_utc_now()+ rows = [+ {+ "id": str(uuid.uuid4()),+ "owner_id": "",+ "name": "bulk1.bin",+ "asset_id": asset.id,+ "preview_id": None,+ "user_metadata": {},+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ },+ {+ "id": str(uuid.uuid4()),+ "owner_id": "",+ "name": "bulk2.bin",+ "asset_id": asset.id,+ "preview_id": None,+ "user_metadata": {},+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ },+ ]+ bulk_insert_references_ignore_conflicts(session, rows)+ session.commit()++ refs = session.query(AssetReference).all()+ assert len(refs) == 2++ def test_allows_duplicate_names(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, name="existing.bin", owner_id="")+ session.commit()++ now = get_utc_now()+ rows = [+ {+ "id": str(uuid.uuid4()),+ "owner_id": "",+ "name": "existing.bin",+ "asset_id": asset.id,+ "preview_id": None,+ "user_metadata": {},+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ },+ {+ "id": str(uuid.uuid4()),+ "owner_id": "",+ "name": "new.bin",+ "asset_id": asset.id,+ "preview_id": None,+ "user_metadata": {},+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ },+ ]+ bulk_insert_references_ignore_conflicts(session, rows)+ session.commit()++ # Duplicate names allowed, so all 3 rows exist+ refs = session.query(AssetReference).all()+ assert len(refs) == 3++ def test_empty_list_is_noop(self, session: Session):+ bulk_insert_references_ignore_conflicts(session, [])+ assert session.query(AssetReference).count() == 0+++class TestGetReferenceIdsByIds:+ def test_returns_existing_ids(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref1 = _make_reference(session, asset, name="a.bin")+ ref2 = _make_reference(session, asset, name="b.bin")+ session.commit()++ found = get_reference_ids_by_ids(session, [ref1.id, ref2.id, "nonexistent"])++ assert found == {ref1.id, ref2.id}++ def test_empty_list_returns_empty(self, session: Session):+ found = get_reference_ids_by_ids(session, [])+ assert found == set()diff --git a/tests-unit/assets_test/queries/test_cache_state.py b/tests-unit/assets_test/queries/test_cache_state.pynew file mode 100644index 000000000..ead60e570--- /dev/null+++ b/tests-unit/assets_test/queries/test_cache_state.py@@ -0,0 +1,499 @@+"""Tests for cache_state (AssetReference file path) query functions."""+import pytest+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference+from app.assets.database.queries import (+ list_references_by_asset_id,+ upsert_reference,+ get_unreferenced_unhashed_asset_ids,+ delete_assets_by_ids,+ get_references_for_prefixes,+ bulk_update_needs_verify,+ delete_references_by_ids,+ delete_orphaned_seed_asset,+ bulk_insert_references_ignore_conflicts,+ get_references_by_paths_and_asset_ids,+ mark_references_missing_outside_prefixes,+ restore_references_by_paths,+)+from app.assets.helpers import select_best_live_path, get_utc_now+++def _make_asset(session: Session, hash_val: str | None = None, size: int = 1024) -> Asset:+ asset = Asset(hash=hash_val, size_bytes=size)+ session.add(asset)+ session.flush()+ return asset+++def _make_reference(+ session: Session,+ asset: Asset,+ file_path: str,+ name: str = "test",+ mtime_ns: int | None = None,+ needs_verify: bool = False,+) -> AssetReference:+ now = get_utc_now()+ ref = AssetReference(+ asset_id=asset.id,+ file_path=file_path,+ name=name,+ mtime_ns=mtime_ns,+ needs_verify=needs_verify,+ created_at=now,+ updated_at=now,+ last_access_time=now,+ )+ session.add(ref)+ session.flush()+ return ref+++class TestListReferencesByAssetId:+ def test_returns_empty_for_no_references(self, session: Session):+ asset = _make_asset(session, "hash1")+ refs = list_references_by_asset_id(session, asset_id=asset.id)+ assert list(refs) == []++ def test_returns_references_for_asset(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "/path/a.bin", name="a")+ _make_reference(session, asset, "/path/b.bin", name="b")+ session.commit()++ refs = list_references_by_asset_id(session, asset_id=asset.id)+ paths = [r.file_path for r in refs]+ assert set(paths) == {"/path/a.bin", "/path/b.bin"}++ def test_does_not_return_other_assets_references(self, session: Session):+ asset1 = _make_asset(session, "hash1")+ asset2 = _make_asset(session, "hash2")+ _make_reference(session, asset1, "/path/asset1.bin", name="a1")+ _make_reference(session, asset2, "/path/asset2.bin", name="a2")+ session.commit()++ refs = list_references_by_asset_id(session, asset_id=asset1.id)+ paths = [r.file_path for r in refs]+ assert paths == ["/path/asset1.bin"]+++class TestSelectBestLivePath:+ def test_returns_empty_for_empty_list(self):+ result = select_best_live_path([])+ assert result == ""++ def test_returns_empty_when_no_files_exist(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset, "/nonexistent/path.bin")+ session.commit()++ result = select_best_live_path([ref])+ assert result == ""++ def test_prefers_verified_path(self, session: Session, tmp_path):+ """needs_verify=False should be preferred."""+ asset = _make_asset(session, "hash1")++ verified_file = tmp_path / "verified.bin"+ verified_file.write_bytes(b"data")++ unverified_file = tmp_path / "unverified.bin"+ unverified_file.write_bytes(b"data")++ ref_verified = _make_reference(+ session, asset, str(verified_file), name="verified", needs_verify=False+ )+ ref_unverified = _make_reference(+ session, asset, str(unverified_file), name="unverified", needs_verify=True+ )+ session.commit()++ refs = [ref_unverified, ref_verified]+ result = select_best_live_path(refs)+ assert result == str(verified_file)++ def test_falls_back_to_existing_unverified(self, session: Session, tmp_path):+ """If all references need verification, return first existing path."""+ asset = _make_asset(session, "hash1")++ existing_file = tmp_path / "exists.bin"+ existing_file.write_bytes(b"data")++ ref = _make_reference(session, asset, str(existing_file), needs_verify=True)+ session.commit()++ result = select_best_live_path([ref])+ assert result == str(existing_file)+++class TestSelectBestLivePathWithMocking:+ def test_handles_missing_file_path_attr(self):+ """Gracefully handle references with None file_path."""++ class MockRef:+ file_path = None+ needs_verify = False++ result = select_best_live_path([MockRef()])+ assert result == ""+++class TestUpsertReference:+ @pytest.mark.parametrize(+ "initial_mtime,second_mtime,expect_created,expect_updated,final_mtime",+ [+ # New reference creation+ (None, 12345, True, False, 12345),+ # Existing reference, same mtime - no update+ (100, 100, False, False, 100),+ # Existing reference, different mtime - update+ (100, 200, False, True, 200),+ ],+ ids=["new_reference", "existing_no_change", "existing_update_mtime"],+ )+ def test_upsert_scenarios(+ self, session: Session, initial_mtime, second_mtime, expect_created, expect_updated, final_mtime+ ):+ asset = _make_asset(session, "hash1")+ file_path = f"/path_{initial_mtime}_{second_mtime}.bin"+ name = f"file_{initial_mtime}_{second_mtime}"++ # Create initial reference if needed+ if initial_mtime is not None:+ upsert_reference(session, asset_id=asset.id, file_path=file_path, name=name, mtime_ns=initial_mtime)+ session.commit()++ # The upsert call we're testing+ created, updated = upsert_reference(+ session, asset_id=asset.id, file_path=file_path, name=name, mtime_ns=second_mtime+ )+ session.commit()++ assert created is expect_created+ assert updated is expect_updated+ ref = session.query(AssetReference).filter_by(file_path=file_path).one()+ assert ref.mtime_ns == final_mtime++ def test_upsert_restores_missing_reference(self, session: Session):+ """Upserting a reference that was marked missing should restore it."""+ asset = _make_asset(session, "hash1")+ file_path = "/restored/file.bin"++ ref = _make_reference(session, asset, file_path, mtime_ns=100)+ ref.is_missing = True+ session.commit()++ created, updated = upsert_reference(+ session, asset_id=asset.id, file_path=file_path, name="restored", mtime_ns=100+ )+ session.commit()++ assert created is False+ assert updated is True+ restored_ref = session.query(AssetReference).filter_by(file_path=file_path).one()+ assert restored_ref.is_missing is False+++class TestRestoreReferencesByPaths:+ def test_restores_missing_references(self, session: Session):+ asset = _make_asset(session, "hash1")+ missing_path = "/missing/file.bin"+ active_path = "/active/file.bin"++ missing_ref = _make_reference(session, asset, missing_path, name="missing")+ missing_ref.is_missing = True+ _make_reference(session, asset, active_path, name="active")+ session.commit()++ restored = restore_references_by_paths(session, [missing_path])+ session.commit()++ assert restored == 1+ ref = session.query(AssetReference).filter_by(file_path=missing_path).one()+ assert ref.is_missing is False++ def test_empty_list_restores_nothing(self, session: Session):+ restored = restore_references_by_paths(session, [])+ assert restored == 0+++class TestMarkReferencesMissingOutsidePrefixes:+ def test_marks_references_missing_outside_prefixes(self, session: Session, tmp_path):+ asset = _make_asset(session, "hash1")+ valid_dir = tmp_path / "valid"+ valid_dir.mkdir()+ invalid_dir = tmp_path / "invalid"+ invalid_dir.mkdir()++ valid_path = str(valid_dir / "file.bin")+ invalid_path = str(invalid_dir / "file.bin")++ _make_reference(session, asset, valid_path, name="valid")+ _make_reference(session, asset, invalid_path, name="invalid")+ session.commit()++ marked = mark_references_missing_outside_prefixes(session, [str(valid_dir)])+ session.commit()++ assert marked == 1+ all_refs = session.query(AssetReference).all()+ assert len(all_refs) == 2++ valid_ref = next(r for r in all_refs if r.file_path == valid_path)+ invalid_ref = next(r for r in all_refs if r.file_path == invalid_path)+ assert valid_ref.is_missing is False+ assert invalid_ref.is_missing is True++ def test_empty_prefixes_marks_nothing(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "/some/path.bin")+ session.commit()++ marked = mark_references_missing_outside_prefixes(session, [])++ assert marked == 0+++class TestGetUnreferencedUnhashedAssetIds:+ def test_returns_unreferenced_unhashed_assets(self, session: Session):+ # Unhashed asset (hash=None) with no references (no file_path)+ no_refs = _make_asset(session, hash_val=None)+ # Unhashed asset with active reference (not unreferenced)+ with_active_ref = _make_asset(session, hash_val=None)+ _make_reference(session, with_active_ref, "/has/ref.bin", name="has_ref")+ # Unhashed asset with only missing reference (should be unreferenced)+ with_missing_ref = _make_asset(session, hash_val=None)+ missing_ref = _make_reference(session, with_missing_ref, "/missing/ref.bin", name="missing_ref")+ missing_ref.is_missing = True+ # Regular asset (hash not None) - should not be returned+ _make_asset(session, hash_val="blake3:regular")+ session.commit()++ unreferenced = get_unreferenced_unhashed_asset_ids(session)++ assert no_refs.id in unreferenced+ assert with_missing_ref.id in unreferenced+ assert with_active_ref.id not in unreferenced+++class TestDeleteAssetsByIds:+ def test_deletes_assets_and_references(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "/test/path.bin", name="test")+ session.commit()++ deleted = delete_assets_by_ids(session, [asset.id])+ session.commit()++ assert deleted == 1+ assert session.query(Asset).count() == 0+ assert session.query(AssetReference).count() == 0++ def test_empty_list_deletes_nothing(self, session: Session):+ _make_asset(session, "hash1")+ session.commit()++ deleted = delete_assets_by_ids(session, [])++ assert deleted == 0+ assert session.query(Asset).count() == 1+++class TestGetReferencesForPrefixes:+ def test_returns_references_matching_prefix(self, session: Session, tmp_path):+ asset = _make_asset(session, "hash1")+ dir1 = tmp_path / "dir1"+ dir1.mkdir()+ dir2 = tmp_path / "dir2"+ dir2.mkdir()++ path1 = str(dir1 / "file.bin")+ path2 = str(dir2 / "file.bin")++ _make_reference(session, asset, path1, name="file1", mtime_ns=100)+ _make_reference(session, asset, path2, name="file2", mtime_ns=200)+ session.commit()++ rows = get_references_for_prefixes(session, [str(dir1)])++ assert len(rows) == 1+ assert rows[0].file_path == path1++ def test_empty_prefixes_returns_empty(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "/some/path.bin")+ session.commit()++ rows = get_references_for_prefixes(session, [])++ assert rows == []+++class TestBulkSetNeedsVerify:+ def test_sets_needs_verify_flag(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref1 = _make_reference(session, asset, "/path1.bin", needs_verify=False)+ ref2 = _make_reference(session, asset, "/path2.bin", needs_verify=False)+ session.commit()++ updated = bulk_update_needs_verify(session, [ref1.id, ref2.id], True)+ session.commit()++ assert updated == 2+ session.refresh(ref1)+ session.refresh(ref2)+ assert ref1.needs_verify is True+ assert ref2.needs_verify is True++ def test_empty_list_updates_nothing(self, session: Session):+ updated = bulk_update_needs_verify(session, [], True)+ assert updated == 0+++class TestDeleteReferencesByIds:+ def test_deletes_references_by_id(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref1 = _make_reference(session, asset, "/path1.bin")+ _make_reference(session, asset, "/path2.bin")+ session.commit()++ deleted = delete_references_by_ids(session, [ref1.id])+ session.commit()++ assert deleted == 1+ assert session.query(AssetReference).count() == 1++ def test_empty_list_deletes_nothing(self, session: Session):+ deleted = delete_references_by_ids(session, [])+ assert deleted == 0+++class TestDeleteOrphanedSeedAsset:+ @pytest.mark.parametrize(+ "create_asset,expected_deleted,expected_count",+ [+ (True, True, 0), # Existing asset gets deleted+ (False, False, 0), # Nonexistent returns False+ ],+ ids=["deletes_existing", "nonexistent_returns_false"],+ )+ def test_delete_orphaned_seed_asset(+ self, session: Session, create_asset, expected_deleted, expected_count+ ):+ asset_id = "nonexistent-id"+ if create_asset:+ asset = _make_asset(session, hash_val=None)+ asset_id = asset.id+ _make_reference(session, asset, "/test/path.bin", name="test")+ session.commit()++ deleted = delete_orphaned_seed_asset(session, asset_id)+ if create_asset:+ session.commit()++ assert deleted is expected_deleted+ assert session.query(Asset).count() == expected_count+++class TestBulkInsertReferencesIgnoreConflicts:+ def test_inserts_multiple_references(self, session: Session):+ asset = _make_asset(session, "hash1")+ now = get_utc_now()+ rows = [+ {+ "asset_id": asset.id,+ "file_path": "/bulk1.bin",+ "name": "bulk1",+ "mtime_ns": 100,+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ },+ {+ "asset_id": asset.id,+ "file_path": "/bulk2.bin",+ "name": "bulk2",+ "mtime_ns": 200,+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ },+ ]+ bulk_insert_references_ignore_conflicts(session, rows)+ session.commit()++ assert session.query(AssetReference).count() == 2++ def test_ignores_conflicts(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "/existing.bin", mtime_ns=100)+ session.commit()++ now = get_utc_now()+ rows = [+ {+ "asset_id": asset.id,+ "file_path": "/existing.bin",+ "name": "existing",+ "mtime_ns": 999,+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ },+ {+ "asset_id": asset.id,+ "file_path": "/new.bin",+ "name": "new",+ "mtime_ns": 200,+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ },+ ]+ bulk_insert_references_ignore_conflicts(session, rows)+ session.commit()++ assert session.query(AssetReference).count() == 2+ existing = session.query(AssetReference).filter_by(file_path="/existing.bin").one()+ assert existing.mtime_ns == 100 # Original value preserved++ def test_empty_list_is_noop(self, session: Session):+ bulk_insert_references_ignore_conflicts(session, [])+ assert session.query(AssetReference).count() == 0+++class TestGetReferencesByPathsAndAssetIds:+ def test_returns_matching_paths(self, session: Session):+ asset1 = _make_asset(session, "hash1")+ asset2 = _make_asset(session, "hash2")++ _make_reference(session, asset1, "/path1.bin")+ _make_reference(session, asset2, "/path2.bin")+ session.commit()++ path_to_asset = {+ "/path1.bin": asset1.id,+ "/path2.bin": asset2.id,+ }+ winners = get_references_by_paths_and_asset_ids(session, path_to_asset)++ assert winners == {"/path1.bin", "/path2.bin"}++ def test_excludes_non_matching_asset_ids(self, session: Session):+ asset1 = _make_asset(session, "hash1")+ asset2 = _make_asset(session, "hash2")++ _make_reference(session, asset1, "/path1.bin")+ session.commit()++ # Path exists but with different asset_id+ path_to_asset = {"/path1.bin": asset2.id}+ winners = get_references_by_paths_and_asset_ids(session, path_to_asset)++ assert winners == set()++ def test_empty_dict_returns_empty(self, session: Session):+ winners = get_references_by_paths_and_asset_ids(session, {})+ assert winners == set()diff --git a/tests-unit/assets_test/queries/test_metadata.py b/tests-unit/assets_test/queries/test_metadata.pynew file mode 100644index 000000000..6a545e819--- /dev/null+++ b/tests-unit/assets_test/queries/test_metadata.py@@ -0,0 +1,184 @@+"""Tests for metadata filtering logic in asset_reference queries."""+import pytest+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference, AssetReferenceMeta+from app.assets.database.queries import list_references_page+from app.assets.database.queries.asset_reference import convert_metadata_to_rows+from app.assets.helpers import get_utc_now+++def _make_asset(session: Session, hash_val: str) -> Asset:+ asset = Asset(hash=hash_val, size_bytes=1024)+ session.add(asset)+ session.flush()+ return asset+++def _make_reference(+ session: Session,+ asset: Asset,+ name: str,+ metadata: dict | None = None,+) -> AssetReference:+ now = get_utc_now()+ ref = AssetReference(+ owner_id="",+ name=name,+ asset_id=asset.id,+ user_metadata=metadata,+ created_at=now,+ updated_at=now,+ last_access_time=now,+ )+ session.add(ref)+ session.flush()++ if metadata:+ for key, val in metadata.items():+ for row in convert_metadata_to_rows(key, val):+ meta_row = AssetReferenceMeta(+ asset_reference_id=ref.id,+ key=row["key"],+ ordinal=row.get("ordinal", 0),+ val_str=row.get("val_str"),+ val_num=row.get("val_num"),+ val_bool=row.get("val_bool"),+ val_json=row.get("val_json"),+ )+ session.add(meta_row)+ session.flush()++ return ref+++class TestMetadataFilterByType:+ """Table-driven tests for metadata filtering by different value types."""++ @pytest.mark.parametrize(+ "match_meta,nomatch_meta,filter_key,filter_val",+ [+ # String matching+ ({"category": "models"}, {"category": "images"}, "category", "models"),+ # Integer matching+ ({"epoch": 5}, {"epoch": 10}, "epoch", 5),+ # Float matching+ ({"score": 0.95}, {"score": 0.5}, "score", 0.95),+ # Boolean True matching+ ({"enabled": True}, {"enabled": False}, "enabled", True),+ # Boolean False matching+ ({"enabled": False}, {"enabled": True}, "enabled", False),+ ],+ ids=["string", "int", "float", "bool_true", "bool_false"],+ )+ def test_filter_matches_correct_value(+ self, session: Session, match_meta, nomatch_meta, filter_key, filter_val+ ):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "match", match_meta)+ _make_reference(session, asset, "nomatch", nomatch_meta)+ session.commit()++ refs, _, total = list_references_page(+ session, metadata_filter={filter_key: filter_val}+ )+ assert total == 1+ assert refs[0].name == "match"++ @pytest.mark.parametrize(+ "stored_meta,filter_key,filter_val",+ [+ # String no match+ ({"category": "models"}, "category", "other"),+ # Int no match+ ({"epoch": 5}, "epoch", 99),+ # Float no match+ ({"score": 0.5}, "score", 0.99),+ ],+ ids=["string_no_match", "int_no_match", "float_no_match"],+ )+ def test_filter_returns_empty_when_no_match(+ self, session: Session, stored_meta, filter_key, filter_val+ ):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "item", stored_meta)+ session.commit()++ refs, _, total = list_references_page(+ session, metadata_filter={filter_key: filter_val}+ )+ assert total == 0+++class TestMetadataFilterNull:+ """Tests for null/missing key filtering."""++ @pytest.mark.parametrize(+ "match_name,match_meta,nomatch_name,nomatch_meta,filter_key",+ [+ # Null matches missing key+ ("missing_key", {}, "has_key", {"optional": "value"}, "optional"),+ # Null matches explicit null+ ("explicit_null", {"nullable": None}, "has_value", {"nullable": "present"}, "nullable"),+ ],+ ids=["missing_key", "explicit_null"],+ )+ def test_null_filter_matches(+ self, session: Session, match_name, match_meta, nomatch_name, nomatch_meta, filter_key+ ):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, match_name, match_meta)+ _make_reference(session, asset, nomatch_name, nomatch_meta)+ session.commit()++ refs, _, total = list_references_page(session, metadata_filter={filter_key: None})+ assert total == 1+ assert refs[0].name == match_name+++class TestMetadataFilterList:+ """Tests for list-based (OR) filtering."""++ def test_filter_by_list_matches_any(self, session: Session):+ """List values should match ANY of the values (OR)."""+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "cat_a", {"category": "a"})+ _make_reference(session, asset, "cat_b", {"category": "b"})+ _make_reference(session, asset, "cat_c", {"category": "c"})+ session.commit()++ refs, _, total = list_references_page(session, metadata_filter={"category": ["a", "b"]})+ assert total == 2+ names = {r.name for r in refs}+ assert names == {"cat_a", "cat_b"}+++class TestMetadataFilterMultipleKeys:+ """Tests for multiple filter keys (AND semantics)."""++ def test_multiple_keys_must_all_match(self, session: Session):+ """Multiple keys should ALL match (AND)."""+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "match", {"type": "model", "version": 2})+ _make_reference(session, asset, "wrong_type", {"type": "config", "version": 2})+ _make_reference(session, asset, "wrong_version", {"type": "model", "version": 1})+ session.commit()++ refs, _, total = list_references_page(+ session, metadata_filter={"type": "model", "version": 2}+ )+ assert total == 1+ assert refs[0].name == "match"+++class TestMetadataFilterEmptyDict:+ """Tests for empty filter behavior."""++ def test_empty_filter_returns_all(self, session: Session):+ asset = _make_asset(session, "hash1")+ _make_reference(session, asset, "a", {"key": "val"})+ _make_reference(session, asset, "b", {})+ session.commit()++ refs, _, total = list_references_page(session, metadata_filter={})+ assert total == 2diff --git a/tests-unit/assets_test/queries/test_tags.py b/tests-unit/assets_test/queries/test_tags.pynew file mode 100644index 000000000..4ed99aa37--- /dev/null+++ b/tests-unit/assets_test/queries/test_tags.py@@ -0,0 +1,366 @@+import pytest+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference, AssetReferenceTag, AssetReferenceMeta, Tag+from app.assets.database.queries import (+ ensure_tags_exist,+ get_reference_tags,+ set_reference_tags,+ add_tags_to_reference,+ remove_tags_from_reference,+ add_missing_tag_for_asset_id,+ remove_missing_tag_for_asset_id,+ list_tags_with_usage,+ bulk_insert_tags_and_meta,+)+from app.assets.helpers import get_utc_now+++def _make_asset(session: Session, hash_val: str | None = None) -> Asset:+ asset = Asset(hash=hash_val, size_bytes=1024)+ session.add(asset)+ session.flush()+ return asset+++def _make_reference(session: Session, asset: Asset, name: str = "test", owner_id: str = "") -> AssetReference:+ now = get_utc_now()+ ref = AssetReference(+ owner_id=owner_id,+ name=name,+ asset_id=asset.id,+ created_at=now,+ updated_at=now,+ last_access_time=now,+ )+ session.add(ref)+ session.flush()+ return ref+++class TestEnsureTagsExist:+ def test_creates_new_tags(self, session: Session):+ ensure_tags_exist(session, ["alpha", "beta"], tag_type="user")+ session.commit()++ tags = session.query(Tag).all()+ assert {t.name for t in tags} == {"alpha", "beta"}++ def test_is_idempotent(self, session: Session):+ ensure_tags_exist(session, ["alpha"], tag_type="user")+ ensure_tags_exist(session, ["alpha"], tag_type="user")+ session.commit()++ assert session.query(Tag).count() == 1++ def test_normalizes_tags(self, session: Session):+ ensure_tags_exist(session, [" ALPHA ", "Beta", "alpha"])+ session.commit()++ tags = session.query(Tag).all()+ assert {t.name for t in tags} == {"alpha", "beta"}++ def test_empty_list_is_noop(self, session: Session):+ ensure_tags_exist(session, [])+ session.commit()+ assert session.query(Tag).count() == 0++ def test_tag_type_is_set(self, session: Session):+ ensure_tags_exist(session, ["system-tag"], tag_type="system")+ session.commit()++ tag = session.query(Tag).filter_by(name="system-tag").one()+ assert tag.tag_type == "system"+++class TestGetReferenceTags:+ def test_returns_empty_for_no_tags(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ tags = get_reference_tags(session, reference_id=ref.id)+ assert tags == []++ def test_returns_tags_for_reference(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ ensure_tags_exist(session, ["tag1", "tag2"])+ session.add_all([+ AssetReferenceTag(asset_reference_id=ref.id, tag_name="tag1", origin="manual", added_at=get_utc_now()),+ AssetReferenceTag(asset_reference_id=ref.id, tag_name="tag2", origin="manual", added_at=get_utc_now()),+ ])+ session.flush()++ tags = get_reference_tags(session, reference_id=ref.id)+ assert set(tags) == {"tag1", "tag2"}+++class TestSetReferenceTags:+ def test_adds_new_tags(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ result = set_reference_tags(session, reference_id=ref.id, tags=["a", "b"])+ session.commit()++ assert set(result.added) == {"a", "b"}+ assert result.removed == []+ assert set(result.total) == {"a", "b"}++ def test_removes_old_tags(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ set_reference_tags(session, reference_id=ref.id, tags=["a", "b", "c"])+ result = set_reference_tags(session, reference_id=ref.id, tags=["a"])+ session.commit()++ assert result.added == []+ assert set(result.removed) == {"b", "c"}+ assert result.total == ["a"]++ def test_replaces_tags(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ set_reference_tags(session, reference_id=ref.id, tags=["a", "b"])+ result = set_reference_tags(session, reference_id=ref.id, tags=["b", "c"])+ session.commit()++ assert result.added == ["c"]+ assert result.removed == ["a"]+ assert set(result.total) == {"b", "c"}+++class TestAddTagsToReference:+ def test_adds_tags(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ result = add_tags_to_reference(session, reference_id=ref.id, tags=["x", "y"])+ session.commit()++ assert set(result.added) == {"x", "y"}+ assert result.already_present == []++ def test_reports_already_present(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ add_tags_to_reference(session, reference_id=ref.id, tags=["x"])+ result = add_tags_to_reference(session, reference_id=ref.id, tags=["x", "y"])+ session.commit()++ assert result.added == ["y"]+ assert result.already_present == ["x"]++ def test_raises_for_missing_reference(self, session: Session):+ with pytest.raises(ValueError, match="not found"):+ add_tags_to_reference(session, reference_id="nonexistent", tags=["x"])+++class TestRemoveTagsFromReference:+ def test_removes_tags(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ add_tags_to_reference(session, reference_id=ref.id, tags=["a", "b", "c"])+ result = remove_tags_from_reference(session, reference_id=ref.id, tags=["a", "b"])+ session.commit()++ assert set(result.removed) == {"a", "b"}+ assert result.not_present == []+ assert result.total_tags == ["c"]++ def test_reports_not_present(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)++ add_tags_to_reference(session, reference_id=ref.id, tags=["a"])+ result = remove_tags_from_reference(session, reference_id=ref.id, tags=["a", "x"])+ session.commit()++ assert result.removed == ["a"]+ assert result.not_present == ["x"]++ def test_raises_for_missing_reference(self, session: Session):+ with pytest.raises(ValueError, match="not found"):+ remove_tags_from_reference(session, reference_id="nonexistent", tags=["x"])+++class TestMissingTagFunctions:+ def test_add_missing_tag_for_asset_id(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["missing"], tag_type="system")++ add_missing_tag_for_asset_id(session, asset_id=asset.id)+ session.commit()++ tags = get_reference_tags(session, reference_id=ref.id)+ assert "missing" in tags++ def test_add_missing_tag_is_idempotent(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["missing"], tag_type="system")++ add_missing_tag_for_asset_id(session, asset_id=asset.id)+ add_missing_tag_for_asset_id(session, asset_id=asset.id)+ session.commit()++ links = session.query(AssetReferenceTag).filter_by(asset_reference_id=ref.id, tag_name="missing").all()+ assert len(links) == 1++ def test_remove_missing_tag_for_asset_id(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["missing"], tag_type="system")+ add_missing_tag_for_asset_id(session, asset_id=asset.id)++ remove_missing_tag_for_asset_id(session, asset_id=asset.id)+ session.commit()++ tags = get_reference_tags(session, reference_id=ref.id)+ assert "missing" not in tags+++class TestListTagsWithUsage:+ def test_returns_tags_with_counts(self, session: Session):+ ensure_tags_exist(session, ["used", "unused"])++ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ add_tags_to_reference(session, reference_id=ref.id, tags=["used"])+ session.commit()++ rows, total = list_tags_with_usage(session)++ tag_dict = {name: count for name, _, count in rows}+ assert tag_dict["used"] == 1+ assert tag_dict["unused"] == 0+ assert total == 2++ def test_exclude_zero_counts(self, session: Session):+ ensure_tags_exist(session, ["used", "unused"])++ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ add_tags_to_reference(session, reference_id=ref.id, tags=["used"])+ session.commit()++ rows, total = list_tags_with_usage(session, include_zero=False)++ tag_names = {name for name, _, _ in rows}+ assert "used" in tag_names+ assert "unused" not in tag_names++ def test_prefix_filter(self, session: Session):+ ensure_tags_exist(session, ["alpha", "beta", "alphabet"])+ session.commit()++ rows, total = list_tags_with_usage(session, prefix="alph")++ tag_names = {name for name, _, _ in rows}+ assert tag_names == {"alpha", "alphabet"}++ def test_order_by_name(self, session: Session):+ ensure_tags_exist(session, ["zebra", "alpha", "middle"])+ session.commit()++ rows, _ = list_tags_with_usage(session, order="name_asc")++ names = [name for name, _, _ in rows]+ assert names == ["alpha", "middle", "zebra"]++ def test_owner_visibility(self, session: Session):+ ensure_tags_exist(session, ["shared-tag", "owner-tag"])++ asset = _make_asset(session, "hash1")+ shared_ref = _make_reference(session, asset, name="shared", owner_id="")+ owner_ref = _make_reference(session, asset, name="owned", owner_id="user1")++ add_tags_to_reference(session, reference_id=shared_ref.id, tags=["shared-tag"])+ add_tags_to_reference(session, reference_id=owner_ref.id, tags=["owner-tag"])+ session.commit()++ # Empty owner sees only shared+ rows, _ = list_tags_with_usage(session, owner_id="", include_zero=False)+ tag_dict = {name: count for name, _, count in rows}+ assert tag_dict.get("shared-tag", 0) == 1+ assert tag_dict.get("owner-tag", 0) == 0++ # User1 sees both+ rows, _ = list_tags_with_usage(session, owner_id="user1", include_zero=False)+ tag_dict = {name: count for name, _, count in rows}+ assert tag_dict.get("shared-tag", 0) == 1+ assert tag_dict.get("owner-tag", 0) == 1+++class TestBulkInsertTagsAndMeta:+ def test_inserts_tags(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["bulk-tag1", "bulk-tag2"])+ session.commit()++ now = get_utc_now()+ tag_rows = [+ {"asset_reference_id": ref.id, "tag_name": "bulk-tag1", "origin": "manual", "added_at": now},+ {"asset_reference_id": ref.id, "tag_name": "bulk-tag2", "origin": "manual", "added_at": now},+ ]+ bulk_insert_tags_and_meta(session, tag_rows=tag_rows, meta_rows=[])+ session.commit()++ tags = get_reference_tags(session, reference_id=ref.id)+ assert set(tags) == {"bulk-tag1", "bulk-tag2"}++ def test_inserts_meta(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ session.commit()++ meta_rows = [+ {+ "asset_reference_id": ref.id,+ "key": "meta-key",+ "ordinal": 0,+ "val_str": "meta-value",+ "val_num": None,+ "val_bool": None,+ "val_json": None,+ },+ ]+ bulk_insert_tags_and_meta(session, tag_rows=[], meta_rows=meta_rows)+ session.commit()++ meta = session.query(AssetReferenceMeta).filter_by(asset_reference_id=ref.id).all()+ assert len(meta) == 1+ assert meta[0].key == "meta-key"+ assert meta[0].val_str == "meta-value"++ def test_ignores_conflicts(self, session: Session):+ asset = _make_asset(session, "hash1")+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["existing-tag"])+ add_tags_to_reference(session, reference_id=ref.id, tags=["existing-tag"])+ session.commit()++ now = get_utc_now()+ tag_rows = [+ {"asset_reference_id": ref.id, "tag_name": "existing-tag", "origin": "duplicate", "added_at": now},+ ]+ bulk_insert_tags_and_meta(session, tag_rows=tag_rows, meta_rows=[])+ session.commit()++ # Should still have only one tag link+ links = session.query(AssetReferenceTag).filter_by(asset_reference_id=ref.id, tag_name="existing-tag").all()+ assert len(links) == 1+ # Origin should be original, not overwritten+ assert links[0].origin == "manual"++ def test_empty_lists_is_noop(self, session: Session):+ bulk_insert_tags_and_meta(session, tag_rows=[], meta_rows=[])+ assert session.query(AssetReferenceTag).count() == 0+ assert session.query(AssetReferenceMeta).count() == 0diff --git a/tests-unit/assets_test/services/__init__.py b/tests-unit/assets_test/services/__init__.pynew file mode 100644index 000000000..d0213422e--- /dev/null+++ b/tests-unit/assets_test/services/__init__.py@@ -0,0 +1 @@+# Service layer testsdiff --git a/tests-unit/assets_test/services/conftest.py b/tests-unit/assets_test/services/conftest.pynew file mode 100644index 000000000..31c763d48--- /dev/null+++ b/tests-unit/assets_test/services/conftest.py@@ -0,0 +1,54 @@+import tempfile+from pathlib import Path+from unittest.mock import patch++import pytest+from sqlalchemy import create_engine+from sqlalchemy.orm import Session++from app.assets.database.models import Base++[email protected](autouse=True)+def autoclean_unit_test_assets():+ """Override parent autouse fixture - service unit tests don't need server cleanup."""+ yield+++def db_engine():+ """In-memory SQLite engine for fast unit tests."""+ engine = create_engine("sqlite:///:memory:")+ Base.metadata.create_all(engine)+ return engine+++def session(db_engine):+ """Session fixture for tests that need direct DB access."""+ with Session(db_engine) as sess:+ yield sess+++def mock_create_session(db_engine):+ """Patch create_session to use our in-memory database."""+ from contextlib import contextmanager+ from sqlalchemy.orm import Session as SASession++ @contextmanager+ def _create_session():+ with SASession(db_engine) as sess:+ yield sess++ with patch("app.assets.services.ingest.create_session", _create_session), \+ patch("app.assets.services.asset_management.create_session", _create_session), \+ patch("app.assets.services.tagging.create_session", _create_session):+ yield _create_session+++def temp_dir():+ """Temporary directory for file operations."""+ with tempfile.TemporaryDirectory() as tmpdir:+ yield Path(tmpdir)diff --git a/tests-unit/assets_test/services/test_asset_management.py b/tests-unit/assets_test/services/test_asset_management.pynew file mode 100644index 000000000..101ef7292--- /dev/null+++ b/tests-unit/assets_test/services/test_asset_management.py@@ -0,0 +1,268 @@+"""Tests for asset_management services."""+import pytest+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference+from app.assets.database.queries import ensure_tags_exist, add_tags_to_reference+from app.assets.helpers import get_utc_now+from app.assets.services import (+ get_asset_detail,+ update_asset_metadata,+ delete_asset_reference,+ set_asset_preview,+)+++def _make_asset(session: Session, hash_val: str = "blake3:test", size: int = 1024) -> Asset:+ asset = Asset(hash=hash_val, size_bytes=size, mime_type="application/octet-stream")+ session.add(asset)+ session.flush()+ return asset+++def _make_reference(+ session: Session,+ asset: Asset,+ name: str = "test",+ owner_id: str = "",+) -> AssetReference:+ now = get_utc_now()+ ref = AssetReference(+ owner_id=owner_id,+ name=name,+ asset_id=asset.id,+ created_at=now,+ updated_at=now,+ last_access_time=now,+ )+ session.add(ref)+ session.flush()+ return ref+++class TestGetAssetDetail:+ def test_returns_none_for_nonexistent(self, mock_create_session):+ result = get_asset_detail(reference_id="nonexistent")+ assert result is None++ def test_returns_asset_with_tags(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset, name="test.bin")+ ensure_tags_exist(session, ["alpha", "beta"])+ add_tags_to_reference(session, reference_id=ref.id, tags=["alpha", "beta"])+ session.commit()++ result = get_asset_detail(reference_id=ref.id)++ assert result is not None+ assert result.ref.id == ref.id+ assert result.asset.hash == asset.hash+ assert set(result.tags) == {"alpha", "beta"}++ def test_respects_owner_visibility(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset, owner_id="user1")+ session.commit()++ # Wrong owner cannot see+ result = get_asset_detail(reference_id=ref.id, owner_id="user2")+ assert result is None++ # Correct owner can see+ result = get_asset_detail(reference_id=ref.id, owner_id="user1")+ assert result is not None+++class TestUpdateAssetMetadata:+ def test_updates_name(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset, name="old_name.bin")+ ref_id = ref.id+ session.commit()++ update_asset_metadata(+ reference_id=ref_id,+ name="new_name.bin",+ )++ # Verify by re-fetching from DB+ session.expire_all()+ updated_ref = session.get(AssetReference, ref_id)+ assert updated_ref.name == "new_name.bin"++ def test_updates_tags(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["old"])+ add_tags_to_reference(session, reference_id=ref.id, tags=["old"])+ session.commit()++ result = update_asset_metadata(+ reference_id=ref.id,+ tags=["new1", "new2"],+ )++ assert set(result.tags) == {"new1", "new2"}+ assert "old" not in result.tags++ def test_updates_user_metadata(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ ref_id = ref.id+ session.commit()++ update_asset_metadata(+ reference_id=ref_id,+ user_metadata={"key": "value", "num": 42},+ )++ # Verify by re-fetching from DB+ session.expire_all()+ updated_ref = session.get(AssetReference, ref_id)+ assert updated_ref.user_metadata["key"] == "value"+ assert updated_ref.user_metadata["num"] == 42++ def test_raises_for_nonexistent(self, mock_create_session):+ with pytest.raises(ValueError, match="not found"):+ update_asset_metadata(reference_id="nonexistent", name="fail")++ def test_raises_for_wrong_owner(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset, owner_id="user1")+ session.commit()++ with pytest.raises(PermissionError, match="not owner"):+ update_asset_metadata(+ reference_id=ref.id,+ name="new",+ owner_id="user2",+ )+++class TestDeleteAssetReference:+ def test_soft_deletes_reference(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ ref_id = ref.id+ session.commit()++ result = delete_asset_reference(+ reference_id=ref_id,+ owner_id="",+ delete_content_if_orphan=False,+ )++ assert result is True+ # Row still exists but is marked as soft-deleted+ session.expire_all()+ row = session.get(AssetReference, ref_id)+ assert row is not None+ assert row.deleted_at is not None++ def test_returns_false_for_nonexistent(self, mock_create_session):+ result = delete_asset_reference(+ reference_id="nonexistent",+ owner_id="",+ )+ assert result is False++ def test_returns_false_for_wrong_owner(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset, owner_id="user1")+ ref_id = ref.id+ session.commit()++ result = delete_asset_reference(+ reference_id=ref_id,+ owner_id="user2",+ )++ assert result is False+ assert session.get(AssetReference, ref_id) is not None++ def test_keeps_asset_if_other_references_exist(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref1 = _make_reference(session, asset, name="ref1")+ _make_reference(session, asset, name="ref2") # Second ref keeps asset alive+ asset_id = asset.id+ session.commit()++ delete_asset_reference(+ reference_id=ref1.id,+ owner_id="",+ delete_content_if_orphan=True,+ )++ # Asset should still exist+ assert session.get(Asset, asset_id) is not None++ def test_deletes_orphaned_asset(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ asset_id = asset.id+ ref_id = ref.id+ session.commit()++ delete_asset_reference(+ reference_id=ref_id,+ owner_id="",+ delete_content_if_orphan=True,+ )++ # Both ref and asset should be gone+ assert session.get(AssetReference, ref_id) is None+ assert session.get(Asset, asset_id) is None+++class TestSetAssetPreview:+ def test_sets_preview(self, mock_create_session, session: Session):+ asset = _make_asset(session, hash_val="blake3:main")+ preview_asset = _make_asset(session, hash_val="blake3:preview")+ ref = _make_reference(session, asset)+ ref_id = ref.id+ preview_id = preview_asset.id+ session.commit()++ set_asset_preview(+ reference_id=ref_id,+ preview_asset_id=preview_id,+ )++ # Verify by re-fetching from DB+ session.expire_all()+ updated_ref = session.get(AssetReference, ref_id)+ assert updated_ref.preview_id == preview_id++ def test_clears_preview(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ preview_asset = _make_asset(session, hash_val="blake3:preview")+ ref = _make_reference(session, asset)+ ref.preview_id = preview_asset.id+ ref_id = ref.id+ session.commit()++ set_asset_preview(+ reference_id=ref_id,+ preview_asset_id=None,+ )++ # Verify by re-fetching from DB+ session.expire_all()+ updated_ref = session.get(AssetReference, ref_id)+ assert updated_ref.preview_id is None++ def test_raises_for_nonexistent_ref(self, mock_create_session):+ with pytest.raises(ValueError, match="not found"):+ set_asset_preview(reference_id="nonexistent")++ def test_raises_for_wrong_owner(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset, owner_id="user1")+ session.commit()++ with pytest.raises(PermissionError, match="not owner"):+ set_asset_preview(+ reference_id=ref.id,+ preview_asset_id=None,+ owner_id="user2",+ )diff --git a/tests-unit/assets_test/services/test_bulk_ingest.py b/tests-unit/assets_test/services/test_bulk_ingest.pynew file mode 100644index 000000000..26e22a01d--- /dev/null+++ b/tests-unit/assets_test/services/test_bulk_ingest.py@@ -0,0 +1,137 @@+"""Tests for bulk ingest services."""++from pathlib import Path++from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference+from app.assets.services.bulk_ingest import SeedAssetSpec, batch_insert_seed_assets+++class TestBatchInsertSeedAssets:+ def test_populates_mime_type_for_model_files(self, session: Session, temp_dir: Path):+ """Verify mime_type is stored in the Asset table for model files."""+ file_path = temp_dir / "model.safetensors"+ file_path.write_bytes(b"fake safetensors content")++ specs: list[SeedAssetSpec] = [+ {+ "abs_path": str(file_path),+ "size_bytes": 24,+ "mtime_ns": 1234567890000000000,+ "info_name": "Test Model",+ "tags": ["models"],+ "fname": "model.safetensors",+ "metadata": None,+ "hash": None,+ "mime_type": "application/safetensors",+ }+ ]++ result = batch_insert_seed_assets(session, specs=specs, owner_id="")++ assert result.inserted_refs == 1++ # Verify Asset has mime_type populated+ assets = session.query(Asset).all()+ assert len(assets) == 1+ assert assets[0].mime_type == "application/safetensors"++ def test_mime_type_none_when_not_provided(self, session: Session, temp_dir: Path):+ """Verify mime_type is None when not provided in spec."""+ file_path = temp_dir / "unknown.bin"+ file_path.write_bytes(b"binary data")++ specs: list[SeedAssetSpec] = [+ {+ "abs_path": str(file_path),+ "size_bytes": 11,+ "mtime_ns": 1234567890000000000,+ "info_name": "Unknown File",+ "tags": [],+ "fname": "unknown.bin",+ "metadata": None,+ "hash": None,+ "mime_type": None,+ }+ ]++ result = batch_insert_seed_assets(session, specs=specs, owner_id="")++ assert result.inserted_refs == 1++ assets = session.query(Asset).all()+ assert len(assets) == 1+ assert assets[0].mime_type is None++ def test_various_model_mime_types(self, session: Session, temp_dir: Path):+ """Verify various model file types get correct mime_type."""+ test_cases = [+ ("model.safetensors", "application/safetensors"),+ ("model.pt", "application/pytorch"),+ ("model.ckpt", "application/pickle"),+ ("model.gguf", "application/gguf"),+ ]++ specs: list[SeedAssetSpec] = []+ for filename, mime_type in test_cases:+ file_path = temp_dir / filename+ file_path.write_bytes(b"content")+ specs.append(+ {+ "abs_path": str(file_path),+ "size_bytes": 7,+ "mtime_ns": 1234567890000000000,+ "info_name": filename,+ "tags": [],+ "fname": filename,+ "metadata": None,+ "hash": None,+ "mime_type": mime_type,+ }+ )++ result = batch_insert_seed_assets(session, specs=specs, owner_id="")++ assert result.inserted_refs == len(test_cases)++ for filename, expected_mime in test_cases:+ ref = session.query(AssetReference).filter_by(name=filename).first()+ assert ref is not None+ asset = session.query(Asset).filter_by(id=ref.asset_id).first()+ assert asset.mime_type == expected_mime, f"Expected {expected_mime} for {filename}, got {asset.mime_type}"+++class TestMetadataExtraction:+ def test_extracts_mime_type_for_model_files(self, temp_dir: Path):+ """Verify metadata extraction returns correct mime_type for model files."""+ from app.assets.services.metadata_extract import extract_file_metadata++ file_path = temp_dir / "model.safetensors"+ file_path.write_bytes(b"fake safetensors content")++ meta = extract_file_metadata(str(file_path))++ assert meta.content_type == "application/safetensors"++ def test_mime_type_for_various_model_formats(self, temp_dir: Path):+ """Verify various model file types get correct mime_type from metadata."""+ from app.assets.services.metadata_extract import extract_file_metadata++ test_cases = [+ ("model.safetensors", "application/safetensors"),+ ("model.sft", "application/safetensors"),+ ("model.pt", "application/pytorch"),+ ("model.pth", "application/pytorch"),+ ("model.ckpt", "application/pickle"),+ ("model.pkl", "application/pickle"),+ ("model.gguf", "application/gguf"),+ ]++ for filename, expected_mime in test_cases:+ file_path = temp_dir / filename+ file_path.write_bytes(b"content")++ meta = extract_file_metadata(str(file_path))++ assert meta.content_type == expected_mime, f"Expected {expected_mime} for {filename}, got {meta.content_type}"diff --git a/tests-unit/assets_test/services/test_enrich.py b/tests-unit/assets_test/services/test_enrich.pynew file mode 100644index 000000000..2bd79a01a--- /dev/null+++ b/tests-unit/assets_test/services/test_enrich.py@@ -0,0 +1,207 @@+"""Tests for asset enrichment (mime_type and hash population)."""+from pathlib import Path++from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference+from app.assets.scanner import (+ ENRICHMENT_HASHED,+ ENRICHMENT_METADATA,+ ENRICHMENT_STUB,+ enrich_asset,+)+++def _create_stub_asset(+ session: Session,+ file_path: str,+ asset_id: str = "test-asset-id",+ reference_id: str = "test-ref-id",+ name: str | None = None,+) -> tuple[Asset, AssetReference]:+ """Create a stub asset with reference for testing enrichment."""+ asset = Asset(+ id=asset_id,+ hash=None,+ size_bytes=100,+ mime_type=None,+ )+ session.add(asset)+ session.flush()++ ref = AssetReference(+ id=reference_id,+ asset_id=asset_id,+ name=name or f"test-asset-{asset_id}",+ owner_id="system",+ file_path=file_path,+ mtime_ns=1234567890000000000,+ enrichment_level=ENRICHMENT_STUB,+ )+ session.add(ref)+ session.flush()++ return asset, ref+++class TestEnrichAsset:+ def test_extracts_mime_type_and_updates_asset(+ self, db_engine, temp_dir: Path, session: Session+ ):+ """Verify mime_type is written to the Asset table during enrichment."""+ file_path = temp_dir / "model.safetensors"+ file_path.write_bytes(b"\x00" * 100)++ asset, ref = _create_stub_asset(+ session, str(file_path), "asset-1", "ref-1"+ )+ session.commit()++ new_level = enrich_asset(+ session,+ file_path=str(file_path),+ reference_id=ref.id,+ asset_id=asset.id,+ extract_metadata=True,+ compute_hash=False,+ )++ assert new_level == ENRICHMENT_METADATA++ session.expire_all()+ updated_asset = session.get(Asset, "asset-1")+ assert updated_asset is not None+ assert updated_asset.mime_type == "application/safetensors"++ def test_computes_hash_and_updates_asset(+ self, db_engine, temp_dir: Path, session: Session+ ):+ """Verify hash is written to the Asset table during enrichment."""+ file_path = temp_dir / "data.bin"+ file_path.write_bytes(b"test content for hashing")++ asset, ref = _create_stub_asset(+ session, str(file_path), "asset-2", "ref-2"+ )+ session.commit()++ new_level = enrich_asset(+ session,+ file_path=str(file_path),+ reference_id=ref.id,+ asset_id=asset.id,+ extract_metadata=True,+ compute_hash=True,+ )++ assert new_level == ENRICHMENT_HASHED++ session.expire_all()+ updated_asset = session.get(Asset, "asset-2")+ assert updated_asset is not None+ assert updated_asset.hash is not None+ assert updated_asset.hash.startswith("blake3:")++ def test_enrichment_updates_both_mime_and_hash(+ self, db_engine, temp_dir: Path, session: Session+ ):+ """Verify both mime_type and hash are set when full enrichment runs."""+ file_path = temp_dir / "model.safetensors"+ file_path.write_bytes(b"\x00" * 50)++ asset, ref = _create_stub_asset(+ session, str(file_path), "asset-3", "ref-3"+ )+ session.commit()++ enrich_asset(+ session,+ file_path=str(file_path),+ reference_id=ref.id,+ asset_id=asset.id,+ extract_metadata=True,+ compute_hash=True,+ )++ session.expire_all()+ updated_asset = session.get(Asset, "asset-3")+ assert updated_asset is not None+ assert updated_asset.mime_type == "application/safetensors"+ assert updated_asset.hash is not None+ assert updated_asset.hash.startswith("blake3:")++ def test_missing_file_returns_stub_level(+ self, db_engine, temp_dir: Path, session: Session+ ):+ """Verify missing files don't cause errors and return STUB level."""+ file_path = temp_dir / "nonexistent.bin"++ asset, ref = _create_stub_asset(+ session, str(file_path), "asset-4", "ref-4"+ )+ session.commit()++ new_level = enrich_asset(+ session,+ file_path=str(file_path),+ reference_id=ref.id,+ asset_id=asset.id,+ extract_metadata=True,+ compute_hash=True,+ )++ assert new_level == ENRICHMENT_STUB++ session.expire_all()+ updated_asset = session.get(Asset, "asset-4")+ assert updated_asset.mime_type is None+ assert updated_asset.hash is None++ def test_duplicate_hash_merges_into_existing_asset(+ self, db_engine, temp_dir: Path, session: Session+ ):+ """Verify duplicate files merge into existing asset instead of failing."""+ file_path_1 = temp_dir / "file1.bin"+ file_path_2 = temp_dir / "file2.bin"+ content = b"identical content"+ file_path_1.write_bytes(content)+ file_path_2.write_bytes(content)++ asset1, ref1 = _create_stub_asset(+ session, str(file_path_1), "asset-dup-1", "ref-dup-1"+ )+ asset2, ref2 = _create_stub_asset(+ session, str(file_path_2), "asset-dup-2", "ref-dup-2"+ )+ session.commit()++ enrich_asset(+ session,+ file_path=str(file_path_1),+ reference_id=ref1.id,+ asset_id=asset1.id,+ extract_metadata=True,+ compute_hash=True,+ )++ enrich_asset(+ session,+ file_path=str(file_path_2),+ reference_id=ref2.id,+ asset_id=asset2.id,+ extract_metadata=True,+ compute_hash=True,+ )++ session.expire_all()++ updated_asset1 = session.get(Asset, "asset-dup-1")+ assert updated_asset1 is not None+ assert updated_asset1.hash is not None++ updated_asset2 = session.get(Asset, "asset-dup-2")+ assert updated_asset2 is None++ updated_ref2 = session.get(AssetReference, "ref-dup-2")+ assert updated_ref2 is not None+ assert updated_ref2.asset_id == "asset-dup-1"diff --git a/tests-unit/assets_test/services/test_ingest.py b/tests-unit/assets_test/services/test_ingest.pynew file mode 100644index 000000000..367bc7721--- /dev/null+++ b/tests-unit/assets_test/services/test_ingest.py@@ -0,0 +1,229 @@+"""Tests for ingest services."""+from pathlib import Path++import pytest+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference, Tag+from app.assets.database.queries import get_reference_tags+from app.assets.services.ingest import _ingest_file_from_path, _register_existing_asset+++class TestIngestFileFromPath:+ def test_creates_asset_and_reference(self, mock_create_session, temp_dir: Path, session: Session):+ file_path = temp_dir / "test_file.bin"+ file_path.write_bytes(b"test content")++ result = _ingest_file_from_path(+ abs_path=str(file_path),+ asset_hash="blake3:abc123",+ size_bytes=12,+ mtime_ns=1234567890000000000,+ mime_type="application/octet-stream",+ )++ assert result.asset_created is True+ assert result.ref_created is True+ assert result.reference_id is not None++ # Verify DB state+ assets = session.query(Asset).all()+ assert len(assets) == 1+ assert assets[0].hash == "blake3:abc123"++ refs = session.query(AssetReference).all()+ assert len(refs) == 1+ assert refs[0].file_path == str(file_path)++ def test_creates_reference_when_name_provided(self, mock_create_session, temp_dir: Path, session: Session):+ file_path = temp_dir / "model.safetensors"+ file_path.write_bytes(b"model data")++ result = _ingest_file_from_path(+ abs_path=str(file_path),+ asset_hash="blake3:def456",+ size_bytes=10,+ mtime_ns=1234567890000000000,+ mime_type="application/octet-stream",+ info_name="My Model",+ owner_id="user1",+ )++ assert result.asset_created is True+ assert result.reference_id is not None++ ref = session.query(AssetReference).first()+ assert ref is not None+ assert ref.name == "My Model"+ assert ref.owner_id == "user1"++ def test_creates_tags_when_provided(self, mock_create_session, temp_dir: Path, session: Session):+ file_path = temp_dir / "tagged.bin"+ file_path.write_bytes(b"data")++ result = _ingest_file_from_path(+ abs_path=str(file_path),+ asset_hash="blake3:ghi789",+ size_bytes=4,+ mtime_ns=1234567890000000000,+ info_name="Tagged Asset",+ tags=["models", "checkpoints"],+ )++ assert result.reference_id is not None++ # Verify tags were created and linked+ tags = session.query(Tag).all()+ tag_names = {t.name for t in tags}+ assert "models" in tag_names+ assert "checkpoints" in tag_names++ ref_tags = get_reference_tags(session, reference_id=result.reference_id)+ assert set(ref_tags) == {"models", "checkpoints"}++ def test_idempotent_upsert(self, mock_create_session, temp_dir: Path, session: Session):+ file_path = temp_dir / "dup.bin"+ file_path.write_bytes(b"content")++ # First ingest+ r1 = _ingest_file_from_path(+ abs_path=str(file_path),+ asset_hash="blake3:repeat",+ size_bytes=7,+ mtime_ns=1234567890000000000,+ )+ assert r1.asset_created is True++ # Second ingest with same hash - should update, not create+ r2 = _ingest_file_from_path(+ abs_path=str(file_path),+ asset_hash="blake3:repeat",+ size_bytes=7,+ mtime_ns=1234567890000000001, # different mtime+ )+ assert r2.asset_created is False+ assert r2.ref_created is False+ assert r2.ref_updated is True++ # Still only one asset+ assets = session.query(Asset).all()+ assert len(assets) == 1++ def test_validates_preview_id(self, mock_create_session, temp_dir: Path, session: Session):+ file_path = temp_dir / "with_preview.bin"+ file_path.write_bytes(b"data")++ # Create a preview asset first+ preview_asset = Asset(hash="blake3:preview", size_bytes=100)+ session.add(preview_asset)+ session.commit()+ preview_id = preview_asset.id++ result = _ingest_file_from_path(+ abs_path=str(file_path),+ asset_hash="blake3:main",+ size_bytes=4,+ mtime_ns=1234567890000000000,+ info_name="With Preview",+ preview_id=preview_id,+ )++ assert result.reference_id is not None+ ref = session.query(AssetReference).filter_by(id=result.reference_id).first()+ assert ref.preview_id == preview_id++ def test_invalid_preview_id_is_cleared(self, mock_create_session, temp_dir: Path, session: Session):+ file_path = temp_dir / "bad_preview.bin"+ file_path.write_bytes(b"data")++ result = _ingest_file_from_path(+ abs_path=str(file_path),+ asset_hash="blake3:badpreview",+ size_bytes=4,+ mtime_ns=1234567890000000000,+ info_name="Bad Preview",+ preview_id="nonexistent-uuid",+ )++ assert result.reference_id is not None+ ref = session.query(AssetReference).filter_by(id=result.reference_id).first()+ assert ref.preview_id is None+++class TestRegisterExistingAsset:+ def test_creates_reference_for_existing_asset(self, mock_create_session, session: Session):+ # Create existing asset+ asset = Asset(hash="blake3:existing", size_bytes=1024, mime_type="image/png")+ session.add(asset)+ session.commit()++ result = _register_existing_asset(+ asset_hash="blake3:existing",+ name="Registered Asset",+ user_metadata={"key": "value"},+ tags=["models"],+ )++ assert result.created is True+ assert "models" in result.tags++ # Verify by re-fetching from DB+ session.expire_all()+ refs = session.query(AssetReference).filter_by(name="Registered Asset").all()+ assert len(refs) == 1++ def test_creates_new_reference_even_with_same_name(self, mock_create_session, session: Session):+ # Create asset and reference+ asset = Asset(hash="blake3:withref", size_bytes=512)+ session.add(asset)+ session.flush()++ from app.assets.helpers import get_utc_now+ ref = AssetReference(+ owner_id="",+ name="Existing Ref",+ asset_id=asset.id,+ created_at=get_utc_now(),+ updated_at=get_utc_now(),+ last_access_time=get_utc_now(),+ )+ session.add(ref)+ session.flush()+ ref_id = ref.id+ session.commit()++ result = _register_existing_asset(+ asset_hash="blake3:withref",+ name="Existing Ref",+ owner_id="",+ )++ # Multiple files with same name are allowed+ assert result.created is True++ # Verify two AssetReferences exist for this name+ session.expire_all()+ refs = session.query(AssetReference).filter_by(name="Existing Ref").all()+ assert len(refs) == 2+ assert ref_id in [r.id for r in refs]++ def test_raises_for_nonexistent_hash(self, mock_create_session):+ with pytest.raises(ValueError, match="No asset with hash"):+ _register_existing_asset(+ asset_hash="blake3:doesnotexist",+ name="Fail",+ )++ def test_applies_tags_to_new_reference(self, mock_create_session, session: Session):+ asset = Asset(hash="blake3:tagged", size_bytes=256)+ session.add(asset)+ session.commit()++ result = _register_existing_asset(+ asset_hash="blake3:tagged",+ name="Tagged Ref",+ tags=["alpha", "beta"],+ )++ assert result.created is True+ assert set(result.tags) == {"alpha", "beta"}diff --git a/tests-unit/assets_test/services/test_tagging.py b/tests-unit/assets_test/services/test_tagging.pynew file mode 100644index 000000000..ab69e5dc1--- /dev/null+++ b/tests-unit/assets_test/services/test_tagging.py@@ -0,0 +1,197 @@+"""Tests for tagging services."""+import pytest+from sqlalchemy.orm import Session++from app.assets.database.models import Asset, AssetReference+from app.assets.database.queries import ensure_tags_exist, add_tags_to_reference+from app.assets.helpers import get_utc_now+from app.assets.services import apply_tags, remove_tags, list_tags+++def _make_asset(session: Session, hash_val: str = "blake3:test") -> Asset:+ asset = Asset(hash=hash_val, size_bytes=1024)+ session.add(asset)+ session.flush()+ return asset+++def _make_reference(+ session: Session,+ asset: Asset,+ name: str = "test",+ owner_id: str = "",+) -> AssetReference:+ now = get_utc_now()+ ref = AssetReference(+ owner_id=owner_id,+ name=name,+ asset_id=asset.id,+ created_at=now,+ updated_at=now,+ last_access_time=now,+ )+ session.add(ref)+ session.flush()+ return ref+++class TestApplyTags:+ def test_adds_new_tags(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ session.commit()++ result = apply_tags(+ reference_id=ref.id,+ tags=["alpha", "beta"],+ )++ assert set(result.added) == {"alpha", "beta"}+ assert result.already_present == []+ assert set(result.total_tags) == {"alpha", "beta"}++ def test_reports_already_present(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["existing"])+ add_tags_to_reference(session, reference_id=ref.id, tags=["existing"])+ session.commit()++ result = apply_tags(+ reference_id=ref.id,+ tags=["existing", "new"],+ )++ assert result.added == ["new"]+ assert result.already_present == ["existing"]++ def test_raises_for_nonexistent_ref(self, mock_create_session):+ with pytest.raises(ValueError, match="not found"):+ apply_tags(reference_id="nonexistent", tags=["x"])++ def test_raises_for_wrong_owner(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset, owner_id="user1")+ session.commit()++ with pytest.raises(PermissionError, match="not owner"):+ apply_tags(+ reference_id=ref.id,+ tags=["new"],+ owner_id="user2",+ )+++class TestRemoveTags:+ def test_removes_tags(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["a", "b", "c"])+ add_tags_to_reference(session, reference_id=ref.id, tags=["a", "b", "c"])+ session.commit()++ result = remove_tags(+ reference_id=ref.id,+ tags=["a", "b"],+ )++ assert set(result.removed) == {"a", "b"}+ assert result.not_present == []+ assert result.total_tags == ["c"]++ def test_reports_not_present(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ ensure_tags_exist(session, ["present"])+ add_tags_to_reference(session, reference_id=ref.id, tags=["present"])+ session.commit()++ result = remove_tags(+ reference_id=ref.id,+ tags=["present", "absent"],+ )++ assert result.removed == ["present"]+ assert result.not_present == ["absent"]++ def test_raises_for_nonexistent_ref(self, mock_create_session):+ with pytest.raises(ValueError, match="not found"):+ remove_tags(reference_id="nonexistent", tags=["x"])++ def test_raises_for_wrong_owner(self, mock_create_session, session: Session):+ asset = _make_asset(session)+ ref = _make_reference(session, asset, owner_id="user1")+ session.commit()++ with pytest.raises(PermissionError, match="not owner"):+ remove_tags(+ reference_id=ref.id,+ tags=["x"],+ owner_id="user2",+ )+++class TestListTags:+ def test_returns_tags_with_counts(self, mock_create_session, session: Session):+ ensure_tags_exist(session, ["used", "unused"])+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ add_tags_to_reference(session, reference_id=ref.id, tags=["used"])+ session.commit()++ rows, total = list_tags()++ tag_dict = {name: count for name, _, count in rows}+ assert tag_dict["used"] == 1+ assert tag_dict["unused"] == 0+ assert total == 2++ def test_excludes_zero_counts(self, mock_create_session, session: Session):+ ensure_tags_exist(session, ["used", "unused"])+ asset = _make_asset(session)+ ref = _make_reference(session, asset)+ add_tags_to_reference(session, reference_id=ref.id, tags=["used"])+ session.commit()++ rows, total = list_tags(include_zero=False)++ tag_names = {name for name, _, _ in rows}+ assert "used" in tag_names+ assert "unused" not in tag_names++ def test_prefix_filter(self, mock_create_session, session: Session):+ ensure_tags_exist(session, ["alpha", "beta", "alphabet"])+ session.commit()++ rows, _ = list_tags(prefix="alph")++ tag_names = {name for name, _, _ in rows}+ assert tag_names == {"alpha", "alphabet"}++ def test_order_by_name(self, mock_create_session, session: Session):+ ensure_tags_exist(session, ["zebra", "alpha", "middle"])+ session.commit()++ rows, _ = list_tags(order="name_asc")++ names = [name for name, _, _ in rows]+ assert names == ["alpha", "middle", "zebra"]++ def test_pagination(self, mock_create_session, session: Session):+ ensure_tags_exist(session, ["a", "b", "c", "d", "e"])+ session.commit()++ rows, total = list_tags(limit=2, offset=1, order="name_asc")++ assert total == 5+ assert len(rows) == 2+ names = [name for name, _, _ in rows]+ assert names == ["b", "c"]++ def test_clamps_limit(self, mock_create_session, session: Session):+ ensure_tags_exist(session, ["a"])+ session.commit()++ # Service should clamp limit to max 1000+ rows, _ = list_tags(limit=2000)+ assert len(rows) <= 1000diff --git a/tests-unit/assets_test/test_assets_missing_sync.py b/tests-unit/assets_test/test_assets_missing_sync.pyindex 78fa7b404..47dc130cb 100644--- a/tests-unit/assets_test/test_assets_missing_sync.py+++ b/tests-unit/assets_test/test_assets_missing_sync.py@@ -4,7 +4,7 @@ from pathlib import Pathimport pytestimport requests-from conftest import get_asset_filename, trigger_sync_seed_assets+from helpers import get_asset_filename, trigger_sync_seed_assetsdiff --git a/tests-unit/assets_test/test_crud.py b/tests-unit/assets_test/test_crud.pyindex d2b69f475..07310223e 100644--- a/tests-unit/assets_test/test_crud.py+++ b/tests-unit/assets_test/test_crud.py@@ -4,7 +4,7 @@ from pathlib import Pathimport pytestimport requests-from conftest import get_asset_filename, trigger_sync_seed_assets+from helpers import get_asset_filename, trigger_sync_seed_assetsdef test_create_from_hash_success(@@ -24,11 +24,11 @@ def test_create_from_hash_success(assert b1["created_new"] is Falseaid = b1["id"]- # Calling again with the same name should return the same AssetInfo id+ # Calling again with the same name creates a new AssetReference (duplicates allowed)r2 = http.post(f"{api_base}/api/assets/from-hash", json=payload, timeout=120)b2 = r2.json()assert r2.status_code == 201, b2- assert b2["id"] == aid+ assert b2["id"] != aid # new reference, not the same onedef test_get_and_delete_asset(http: requests.Session, api_base: str, seeded_asset: dict):@@ -42,8 +42,8 @@ def test_get_and_delete_asset(http: requests.Session, api_base: str, seeded_asseassert "user_metadata" in detailassert "filename" in detail["user_metadata"]- # DELETE- rd = http.delete(f"{api_base}/api/assets/{aid}", timeout=120)+ # DELETE (hard delete to also remove underlying asset and file)+ rd = http.delete(f"{api_base}/api/assets/{aid}?delete_content=true", timeout=120)assert rd.status_code == 204# GET again -> 404@@ -53,6 +53,35 @@ def test_get_and_delete_asset(http: requests.Session, api_base: str, seeded_asseassert body["error"]["code"] == "ASSET_NOT_FOUND"+def test_soft_delete_hides_from_get(http: requests.Session, api_base: str, seeded_asset: dict):+ aid = seeded_asset["id"]+ asset_hash = seeded_asset["asset_hash"]++ # Soft-delete (default, no delete_content param)+ rd = http.delete(f"{api_base}/api/assets/{aid}", timeout=120)+ assert rd.status_code == 204++ # GET by reference ID -> 404 (soft-deleted references are hidden)+ rg = http.get(f"{api_base}/api/assets/{aid}", timeout=120)+ assert rg.status_code == 404++ # Asset identity is preserved (underlying content still exists)+ rh = http.head(f"{api_base}/api/assets/hash/{asset_hash}", timeout=120)+ assert rh.status_code == 200++ # Soft-deleted reference should not appear in listings+ rl = http.get(+ f"{api_base}/api/assets",+ params={"include_tags": "unit-tests", "limit": "500"},+ timeout=120,+ )+ ids = [a["id"] for a in rl.json().get("assets", [])]+ assert aid not in ids++ # Clean up: hard-delete the soft-deleted reference and orphaned asset+ http.delete(f"{api_base}/api/assets/{aid}?delete_content=true", timeout=120)++def test_delete_upon_reference_count(http: requests.Session, api_base: str, seeded_asset: dict):@@ -70,21 +99,32 @@ def test_delete_upon_reference_count(assert copy["asset_hash"] == src_hashassert copy["created_new"] is False- # Delete original reference -> asset identity must remain+ # Soft-delete original reference (default) -> asset identity must remainaid1 = seeded_asset["id"]rd1 = http.delete(f"{api_base}/api/assets/{aid1}", timeout=120)assert rd1.status_code == 204rh1 = http.head(f"{api_base}/api/assets/hash/{src_hash}", timeout=120)- assert rh1.status_code == 200 # identity still present+ assert rh1.status_code == 200 # identity still present (second ref exists)- # Delete the last reference with default semantics -> identity and cached files removed+ # Soft-delete the last reference -> asset identity preserved (no hard delete)aid2 = copy["id"]rd2 = http.delete(f"{api_base}/api/assets/{aid2}", timeout=120)assert rd2.status_code == 204rh2 = http.head(f"{api_base}/api/assets/hash/{src_hash}", timeout=120)- assert rh2.status_code == 404 # orphan content removed+ assert rh2.status_code == 200 # asset identity preserved (soft delete)++ # Re-associate via from-hash, then hard-delete -> orphan content removed+ r3 = http.post(f"{api_base}/api/assets/from-hash", json=payload, timeout=120)+ assert r3.status_code == 201, r3.json()+ aid3 = r3.json()["id"]++ rd3 = http.delete(f"{api_base}/api/assets/{aid3}?delete_content=true", timeout=120)+ assert rd3.status_code == 204++ rh3 = http.head(f"{api_base}/api/assets/hash/{src_hash}", timeout=120)+ assert rh3.status_code == 404 # orphan content removeddef test_update_asset_fields(http: requests.Session, api_base: str, seeded_asset: dict):@@ -126,42 +166,52 @@ def test_head_asset_bad_hash_returns_400_and_no_body(http: requests.Session, apiassert body == b""-def test_delete_nonexistent_returns_404(http: requests.Session, api_base: str):- bogus = str(uuid.uuid4())- r = http.delete(f"{api_base}/api/assets/{bogus}", timeout=120)+ "method,endpoint_template,payload,expected_status,error_code",+ [+ # Delete nonexistent asset+ ("delete", "/api/assets/{uuid}", None, 404, "ASSET_NOT_FOUND"),+ # Bad hash algorithm in from-hash+ (+ "post",+ "/api/assets/from-hash",+ {"hash": "sha256:" + "0" * 64, "name": "x.bin", "tags": ["models", "checkpoints", "unit-tests"]},+ 400,+ "INVALID_BODY",+ ),+ # Get with bad UUID format+ ("get", "/api/assets/not-a-uuid", None, 404, None),+ # Get content with bad UUID format+ ("get", "/api/assets/not-a-uuid/content", None, 404, None),+ ],+ ids=["delete_nonexistent", "bad_hash_algorithm", "get_bad_uuid", "content_bad_uuid"],+)+def test_error_responses(+ http: requests.Session, api_base: str, method, endpoint_template, payload, expected_status, error_code+):+ # Replace {uuid} placeholder with a random UUID for delete test+ endpoint = endpoint_template.replace("{uuid}", str(uuid.uuid4()))+ url = f"{api_base}{endpoint}"++ if method == "get":+ r = http.get(url, timeout=120)+ elif method == "post":+ r = http.post(url, json=payload, timeout=120)+ elif method == "delete":+ r = http.delete(url, timeout=120)++ assert r.status_code == expected_status+ if error_code:+ body = r.json()+ assert body["error"]["code"] == error_code+++def test_create_from_hash_invalid_json(http: requests.Session, api_base: str):+ """Invalid JSON body requires special handling (data= instead of json=)."""+ r = http.post(f"{api_base}/api/assets/from-hash", data=b"{not json}", timeout=120)body = r.json()- assert r.status_code == 404- assert body["error"]["code"] == "ASSET_NOT_FOUND"---def test_create_from_hash_invalids(http: requests.Session, api_base: str):- # Bad hash algorithm- bad = {- "hash": "sha256:" + "0" * 64,- "name": "x.bin",- "tags": ["models", "checkpoints", "unit-tests"],- }- r1 = http.post(f"{api_base}/api/assets/from-hash", json=bad, timeout=120)- b1 = r1.json()- assert r1.status_code == 400- assert b1["error"]["code"] == "INVALID_BODY"-- # Invalid JSON body- r2 = http.post(f"{api_base}/api/assets/from-hash", data=b"{not json}", timeout=120)- b2 = r2.json()- assert r2.status_code == 400- assert b2["error"]["code"] == "INVALID_JSON"---def test_get_update_download_bad_ids(http: requests.Session, api_base: str):- # All endpoints should be not found, as we UUID regex directly in the route definition.- bad_id = "not-a-uuid"-- r1 = http.get(f"{api_base}/api/assets/{bad_id}", timeout=120)- assert r1.status_code == 404-- r3 = http.get(f"{api_base}/api/assets/{bad_id}/content", timeout=120)- assert r3.status_code == 404+ assert r.status_code == 400+ assert body["error"]["code"] == "INVALID_JSON"def test_update_requires_at_least_one_field(http: requests.Session, api_base: str, seeded_asset: dict):diff --git a/tests-unit/assets_test/test_downloads.py b/tests-unit/assets_test/test_downloads.pyindex cdebf9082..672ba9728 100644--- a/tests-unit/assets_test/test_downloads.py+++ b/tests-unit/assets_test/test_downloads.py@@ -6,7 +6,7 @@ from typing import Optionalimport pytestimport requests-from conftest import get_asset_filename, trigger_sync_seed_assets+from helpers import get_asset_filename, trigger_sync_seed_assetsdef test_download_attachment_and_inline(http: requests.Session, api_base: str, seeded_asset: dict):@@ -117,7 +117,7 @@ def test_download_missing_file_returns_404(assert body["error"]["code"] == "FILE_NOT_FOUND"finally:# We created asset without the "unit-tests" tag(see `autoclean_unit_test_assets`), we need to clear it manually.- dr = http.delete(f"{api_base}/api/assets/{aid}", timeout=120)+ dr = http.delete(f"{api_base}/api/assets/{aid}?delete_content=true", timeout=120)dr.contentdiff --git a/tests-unit/assets_test/test_file_utils.py b/tests-unit/assets_test/test_file_utils.pynew file mode 100644index 000000000..e3591d49b--- /dev/null+++ b/tests-unit/assets_test/test_file_utils.py@@ -0,0 +1,121 @@+import os+import sys++import pytest++from app.assets.services.file_utils import is_visible, list_files_recursively+++class TestIsVisible:+ def test_visible_file(self):+ assert is_visible("file.txt") is True++ def test_hidden_file(self):+ assert is_visible(".hidden") is False++ def test_hidden_directory(self):+ assert is_visible(".git") is False++ def test_visible_directory(self):+ assert is_visible("src") is True++ def test_dotdot_is_hidden(self):+ assert is_visible("..") is False++ def test_dot_is_hidden(self):+ assert is_visible(".") is False+++class TestListFilesRecursively:+ def test_skips_hidden_files(self, tmp_path):+ (tmp_path / "visible.txt").write_text("a")+ (tmp_path / ".hidden").write_text("b")++ result = list_files_recursively(str(tmp_path))++ assert len(result) == 1+ assert result[0].endswith("visible.txt")++ def test_skips_hidden_directories(self, tmp_path):+ hidden_dir = tmp_path / ".hidden_dir"+ hidden_dir.mkdir()+ (hidden_dir / "file.txt").write_text("a")++ visible_dir = tmp_path / "visible_dir"+ visible_dir.mkdir()+ (visible_dir / "file.txt").write_text("b")++ result = list_files_recursively(str(tmp_path))++ assert len(result) == 1+ assert "visible_dir" in result[0]+ assert ".hidden_dir" not in result[0]++ def test_empty_directory(self, tmp_path):+ result = list_files_recursively(str(tmp_path))+ assert result == []++ def test_nonexistent_directory(self, tmp_path):+ result = list_files_recursively(str(tmp_path / "nonexistent"))+ assert result == []++ @pytest.mark.skipif(sys.platform == "win32", reason="symlinks need privileges on Windows")+ def test_follows_symlinked_directories(self, tmp_path):+ target = tmp_path / "real_dir"+ target.mkdir()+ (target / "model.safetensors").write_text("data")++ root = tmp_path / "root"+ root.mkdir()+ (root / "link").symlink_to(target)++ result = list_files_recursively(str(root))++ assert len(result) == 1+ assert result[0].endswith("model.safetensors")+ assert "link" in result[0]++ @pytest.mark.skipif(sys.platform == "win32", reason="symlinks need privileges on Windows")+ def test_follows_symlinked_files(self, tmp_path):+ real_file = tmp_path / "real.txt"+ real_file.write_text("content")++ root = tmp_path / "root"+ root.mkdir()+ (root / "link.txt").symlink_to(real_file)++ result = list_files_recursively(str(root))++ assert len(result) == 1+ assert result[0].endswith("link.txt")++ @pytest.mark.skipif(sys.platform == "win32", reason="symlinks need privileges on Windows")+ def test_circular_symlinks_do_not_loop(self, tmp_path):+ dir_a = tmp_path / "a"+ dir_a.mkdir()+ (dir_a / "file.txt").write_text("a")+ # a/b -> a (circular)+ (dir_a / "b").symlink_to(dir_a)++ result = list_files_recursively(str(dir_a))++ assert len(result) == 1+ assert result[0].endswith("file.txt")++ @pytest.mark.skipif(sys.platform == "win32", reason="symlinks need privileges on Windows")+ def test_mutual_circular_symlinks(self, tmp_path):+ dir_a = tmp_path / "a"+ dir_b = tmp_path / "b"+ dir_a.mkdir()+ dir_b.mkdir()+ (dir_a / "file_a.txt").write_text("a")+ (dir_b / "file_b.txt").write_text("b")+ # a/link_b -> b and b/link_a -> a+ (dir_a / "link_b").symlink_to(dir_b)+ (dir_b / "link_a").symlink_to(dir_a)++ result = list_files_recursively(str(dir_a))+ basenames = sorted(os.path.basename(p) for p in result)++ assert "file_a.txt" in basenames+ assert "file_b.txt" in basenamesdiff --git a/tests-unit/assets_test/test_list_filter.py b/tests-unit/assets_test/test_list_filter.pyindex 82e109832..dcb7a73ca 100644--- a/tests-unit/assets_test/test_list_filter.py+++ b/tests-unit/assets_test/test_list_filter.py@@ -1,6 +1,7 @@import timeimport uuid+import pytestimport requests@@ -283,30 +284,21 @@ def test_list_assets_offset_beyond_total_and_limit_boundary(http, api_base, asseassert b2["has_more"] is False-def test_list_assets_offset_negative_and_limit_nonint_rejected(http, api_base):- r1 = http.get(api_base + "/api/assets", params={"offset": "-1"}, timeout=120)- b1 = r1.json()- assert r1.status_code == 400- assert b1["error"]["code"] == "INVALID_QUERY"-- r2 = http.get(api_base + "/api/assets", params={"limit": "abc"}, timeout=120)- b2 = r2.json()- assert r2.status_code == 400- assert b2["error"]["code"] == "INVALID_QUERY"---def test_list_assets_invalid_query_rejected(http: requests.Session, api_base: str):- # limit too small- r1 = http.get(api_base + "/api/assets", params={"limit": "0"}, timeout=120)- b1 = r1.json()- assert r1.status_code == 400- assert b1["error"]["code"] == "INVALID_QUERY"-- # bad metadata JSON- r2 = http.get(api_base + "/api/assets", params={"metadata_filter": "{not json"}, timeout=120)- b2 = r2.json()- assert r2.status_code == 400- assert b2["error"]["code"] == "INVALID_QUERY"+ "params,error_code",+ [+ ({"offset": "-1"}, "INVALID_QUERY"),+ ({"limit": "abc"}, "INVALID_QUERY"),+ ({"limit": "0"}, "INVALID_QUERY"),+ ({"metadata_filter": "{not json"}, "INVALID_QUERY"),+ ],+ ids=["negative_offset", "non_int_limit", "zero_limit", "invalid_metadata_json"],+)+def test_list_assets_invalid_query_rejected(http: requests.Session, api_base: str, params, error_code):+ r = http.get(api_base + "/api/assets", params=params, timeout=120)+ body = r.json()+ assert r.status_code == 400+ assert body["error"]["code"] == error_codedef test_list_assets_name_contains_literal_underscore(diff --git a/tests-unit/assets_test/test_prune_orphaned_assets.py b/tests-unit/assets_test/test_prune_orphaned_assets.pyindex f602e5a77..1fbd4d4e2 100644--- a/tests-unit/assets_test/test_prune_orphaned_assets.py+++ b/tests-unit/assets_test/test_prune_orphaned_assets.py@@ -3,7 +3,7 @@ from pathlib import Pathimport pytestimport requests-from conftest import get_asset_filename, trigger_sync_seed_assets+from helpers import get_asset_filename, trigger_sync_seed_assets@pytest.fixturediff --git a/tests-unit/assets_test/test_sync_references.py b/tests-unit/assets_test/test_sync_references.pynew file mode 100644index 000000000..94cc255bc--- /dev/null+++ b/tests-unit/assets_test/test_sync_references.py@@ -0,0 +1,482 @@+"""Tests for sync_references_with_filesystem in scanner.py."""++import os+import tempfile+from datetime import datetime+from pathlib import Path+from unittest.mock import patch++import pytest+from sqlalchemy import create_engine+from sqlalchemy.orm import Session++from app.assets.database.models import (+ Asset,+ AssetReference,+ AssetReferenceTag,+ Base,+ Tag,+)+from app.assets.database.queries.asset_reference import (+ bulk_insert_references_ignore_conflicts,+ get_references_for_prefixes,+ get_unenriched_references,+ restore_references_by_paths,+)+from app.assets.scanner import sync_references_with_filesystem+from app.assets.services.file_utils import get_mtime_ns+++def db_engine():+ engine = create_engine("sqlite:///:memory:")+ Base.metadata.create_all(engine)+ return engine+++def session(db_engine):+ with Session(db_engine) as sess:+ yield sess+++def temp_dir():+ with tempfile.TemporaryDirectory() as tmpdir:+ yield Path(tmpdir)+++def _create_file(temp_dir: Path, name: str, content: bytes = b"\x00" * 100) -> str:+ """Create a file and return its absolute path (no symlink resolution)."""+ p = temp_dir / name+ p.parent.mkdir(parents=True, exist_ok=True)+ p.write_bytes(content)+ return os.path.abspath(str(p))+++def _stat_mtime_ns(path: str) -> int:+ return get_mtime_ns(os.stat(path, follow_symlinks=True))+++def _make_asset(+ session: Session,+ asset_id: str,+ file_path: str,+ ref_id: str,+ *,+ asset_hash: str | None = None,+ size_bytes: int = 100,+ mtime_ns: int | None = None,+ needs_verify: bool = False,+ is_missing: bool = False,+) -> tuple[Asset, AssetReference]:+ """Insert an Asset + AssetReference and flush."""+ asset = session.get(Asset, asset_id)+ if asset is None:+ asset = Asset(id=asset_id, hash=asset_hash, size_bytes=size_bytes)+ session.add(asset)+ session.flush()++ ref = AssetReference(+ id=ref_id,+ asset_id=asset_id,+ name=f"test-{ref_id}",+ owner_id="system",+ file_path=file_path,+ mtime_ns=mtime_ns,+ needs_verify=needs_verify,+ is_missing=is_missing,+ )+ session.add(ref)+ session.flush()+ return asset, ref+++def _ensure_missing_tag(session: Session):+ """Ensure the 'missing' tag exists."""+ if not session.get(Tag, "missing"):+ session.add(Tag(name="missing", tag_type="system"))+ session.flush()+++class _VerifyCase:+ def __init__(self, id, stat_unchanged, needs_verify_before, expect_needs_verify):+ self.id = id+ self.stat_unchanged = stat_unchanged+ self.needs_verify_before = needs_verify_before+ self.expect_needs_verify = expect_needs_verify+++VERIFY_CASES = [+ _VerifyCase(+ id="unchanged_clears_verify",+ stat_unchanged=True,+ needs_verify_before=True,+ expect_needs_verify=False,+ ),+ _VerifyCase(+ id="unchanged_keeps_clear",+ stat_unchanged=True,+ needs_verify_before=False,+ expect_needs_verify=False,+ ),+ _VerifyCase(+ id="changed_sets_verify",+ stat_unchanged=False,+ needs_verify_before=False,+ expect_needs_verify=True,+ ),+ _VerifyCase(+ id="changed_keeps_verify",+ stat_unchanged=False,+ needs_verify_before=True,+ expect_needs_verify=True,+ ),+]++[email protected]("case", VERIFY_CASES, ids=lambda c: c.id)+def test_needs_verify_toggling(session, temp_dir, case):+ """needs_verify is set/cleared based on mtime+size match."""+ fp = _create_file(temp_dir, "model.bin")+ real_mtime = _stat_mtime_ns(fp)++ mtime_for_db = real_mtime if case.stat_unchanged else real_mtime + 1+ _make_asset(+ session, "a1", fp, "r1",+ asset_hash="blake3:abc",+ mtime_ns=mtime_for_db,+ needs_verify=case.needs_verify_before,+ )+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(session, "models")+ session.commit()++ session.expire_all()+ ref = session.get(AssetReference, "r1")+ assert ref.needs_verify is case.expect_needs_verify+++class _MissingCase:+ def __init__(self, id, file_exists, expect_is_missing):+ self.id = id+ self.file_exists = file_exists+ self.expect_is_missing = expect_is_missing+++MISSING_CASES = [+ _MissingCase(id="existing_file_not_missing", file_exists=True, expect_is_missing=False),+ _MissingCase(id="missing_file_marked_missing", file_exists=False, expect_is_missing=True),+]++[email protected]("case", MISSING_CASES, ids=lambda c: c.id)+def test_is_missing_flag(session, temp_dir, case):+ """is_missing reflects whether the file exists on disk."""+ if case.file_exists:+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ else:+ fp = str(temp_dir / "gone.bin")+ mtime = 999++ _make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(session, "models")+ session.commit()++ session.expire_all()+ ref = session.get(AssetReference, "r1")+ assert ref.is_missing is case.expect_is_missing+++def test_seed_asset_all_missing_deletes_asset(session, temp_dir):+ """Seed asset with all refs missing gets deleted entirely."""+ fp = str(temp_dir / "gone.bin")+ _make_asset(session, "seed1", fp, "r1", asset_hash=None, mtime_ns=999)+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(session, "models")+ session.commit()++ assert session.get(Asset, "seed1") is None+ assert session.get(AssetReference, "r1") is None+++def test_seed_asset_some_exist_returns_survivors(session, temp_dir):+ """Seed asset with at least one existing ref survives and is returned."""+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ _make_asset(session, "seed1", fp, "r1", asset_hash=None, mtime_ns=mtime)+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ survivors = sync_references_with_filesystem(+ session, "models", collect_existing_paths=True,+ )+ session.commit()++ assert session.get(Asset, "seed1") is not None+ assert os.path.abspath(fp) in survivors+++def test_hashed_asset_prunes_missing_refs_when_one_is_ok(session, temp_dir):+ """Hashed asset with one stat-unchanged ref deletes missing refs."""+ fp_ok = _create_file(temp_dir, "good.bin")+ fp_gone = str(temp_dir / "gone.bin")+ mtime = _stat_mtime_ns(fp_ok)++ _make_asset(session, "h1", fp_ok, "r_ok", asset_hash="blake3:aaa", mtime_ns=mtime)+ # Second ref on same asset, file missing+ ref_gone = AssetReference(+ id="r_gone", asset_id="h1", name="gone",+ owner_id="system", file_path=fp_gone, mtime_ns=999,+ )+ session.add(ref_gone)+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(session, "models")+ session.commit()++ session.expire_all()+ assert session.get(AssetReference, "r_ok") is not None+ assert session.get(AssetReference, "r_gone") is None+++def test_hashed_asset_all_missing_keeps_refs(session, temp_dir):+ """Hashed asset with all refs missing keeps refs (no pruning)."""+ fp = str(temp_dir / "gone.bin")+ _make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(session, "models")+ session.commit()++ session.expire_all()+ assert session.get(AssetReference, "r1") is not None+ ref = session.get(AssetReference, "r1")+ assert ref.is_missing is True+++def test_missing_tag_added_when_all_refs_gone(session, temp_dir):+ """Missing tag is added to hashed asset when all refs are missing."""+ _ensure_missing_tag(session)+ fp = str(temp_dir / "gone.bin")+ _make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(+ session, "models", update_missing_tags=True,+ )+ session.commit()++ session.expire_all()+ tag_link = session.get(AssetReferenceTag, ("r1", "missing"))+ assert tag_link is not None+++def test_missing_tag_removed_when_ref_ok(session, temp_dir):+ """Missing tag is removed from hashed asset when a ref is stat-unchanged."""+ _ensure_missing_tag(session)+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ _make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=mtime)+ # Pre-add a stale missing tag+ session.add(AssetReferenceTag(+ asset_reference_id="r1", tag_name="missing", origin="automatic",+ ))+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(+ session, "models", update_missing_tags=True,+ )+ session.commit()++ session.expire_all()+ tag_link = session.get(AssetReferenceTag, ("r1", "missing"))+ assert tag_link is None+++def test_missing_tags_not_touched_when_flag_false(session, temp_dir):+ """Missing tags are not modified when update_missing_tags=False."""+ _ensure_missing_tag(session)+ fp = str(temp_dir / "gone.bin")+ _make_asset(session, "h1", fp, "r1", asset_hash="blake3:aaa", mtime_ns=999)+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(+ session, "models", update_missing_tags=False,+ )+ session.commit()++ tag_link = session.get(AssetReferenceTag, ("r1", "missing"))+ assert tag_link is None # tag was never added+++def test_returns_none_when_collect_false(session, temp_dir):+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ _make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ result = sync_references_with_filesystem(+ session, "models", collect_existing_paths=False,+ )++ assert result is None+++def test_returns_empty_set_for_no_prefixes(session):+ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[]):+ result = sync_references_with_filesystem(+ session, "models", collect_existing_paths=True,+ )++ assert result == set()+++def test_no_references_is_noop(session, temp_dir):+ """No crash and no side effects when there are no references."""+ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ survivors = sync_references_with_filesystem(+ session, "models", collect_existing_paths=True,+ )+ session.commit()++ assert survivors == set()+++# ---------------------------------------------------------------------------+# Soft-delete persistence across scanner operations+# ---------------------------------------------------------------------------++def _soft_delete_ref(session: Session, ref_id: str) -> None:+ """Mark a reference as soft-deleted (mimics the API DELETE behaviour)."""+ ref = session.get(AssetReference, ref_id)+ ref.deleted_at = datetime(2025, 1, 1)+ session.flush()+++def test_soft_deleted_ref_excluded_from_get_references_for_prefixes(session, temp_dir):+ """get_references_for_prefixes skips soft-deleted references."""+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ _make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)+ _soft_delete_ref(session, "r1")+ session.commit()++ rows = get_references_for_prefixes(session, [str(temp_dir)], include_missing=True)+ assert len(rows) == 0+++def test_sync_does_not_resurrect_soft_deleted_ref(session, temp_dir):+ """Scanner sync leaves soft-deleted refs untouched even when file exists on disk."""+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ _make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)+ _soft_delete_ref(session, "r1")+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(session, "models")+ session.commit()++ session.expire_all()+ ref = session.get(AssetReference, "r1")+ assert ref.deleted_at is not None, "soft-deleted ref must stay deleted after sync"+++def test_bulk_insert_does_not_overwrite_soft_deleted_ref(session, temp_dir):+ """bulk_insert_references_ignore_conflicts cannot replace a soft-deleted row."""+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ _make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)+ _soft_delete_ref(session, "r1")+ session.commit()++ now = datetime.now(tz=None)+ bulk_insert_references_ignore_conflicts(session, [+ {+ "id": "r_new",+ "asset_id": "a1",+ "file_path": fp,+ "name": "model.bin",+ "owner_id": "",+ "mtime_ns": mtime,+ "preview_id": None,+ "user_metadata": None,+ "created_at": now,+ "updated_at": now,+ "last_access_time": now,+ }+ ])+ session.commit()++ session.expire_all()+ # Original row is still the soft-deleted one+ ref = session.get(AssetReference, "r1")+ assert ref is not None+ assert ref.deleted_at is not None+ # The new row was not inserted (conflict on file_path)+ assert session.get(AssetReference, "r_new") is None+++def test_restore_references_by_paths_skips_soft_deleted(session, temp_dir):+ """restore_references_by_paths does not clear is_missing on soft-deleted refs."""+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ _make_asset(+ session, "a1", fp, "r1",+ asset_hash="blake3:abc", mtime_ns=mtime, is_missing=True,+ )+ _soft_delete_ref(session, "r1")+ session.commit()++ restored = restore_references_by_paths(session, [fp])+ session.commit()++ assert restored == 0+ session.expire_all()+ ref = session.get(AssetReference, "r1")+ assert ref.is_missing is True, "is_missing must not be cleared on soft-deleted ref"+ assert ref.deleted_at is not None+++def test_get_unenriched_references_excludes_soft_deleted(session, temp_dir):+ """Enrichment queries do not pick up soft-deleted references."""+ fp = _create_file(temp_dir, "model.bin")+ mtime = _stat_mtime_ns(fp)+ _make_asset(session, "a1", fp, "r1", asset_hash="blake3:abc", mtime_ns=mtime)+ _soft_delete_ref(session, "r1")+ session.commit()++ rows = get_unenriched_references(session, [str(temp_dir)], max_level=2)+ assert len(rows) == 0+++def test_sync_ignores_soft_deleted_seed_asset(session, temp_dir):+ """Soft-deleted seed ref is not garbage-collected even when file is missing."""+ fp = str(temp_dir / "gone.bin") # file does not exist+ _make_asset(session, "seed1", fp, "r1", asset_hash=None, mtime_ns=999)+ _soft_delete_ref(session, "r1")+ session.commit()++ with patch("app.assets.scanner.get_prefixes_for_root", return_value=[str(temp_dir)]):+ sync_references_with_filesystem(session, "models")+ session.commit()++ session.expire_all()+ # Asset and ref must still exist — scanner did not see the soft-deleted row+ assert session.get(Asset, "seed1") is not None+ assert session.get(AssetReference, "r1") is not Nonediff --git a/tests-unit/assets_test/test_tags.py b/tests-unit/assets_test/test_tags_api.pysimilarity index 98%rename from tests-unit/assets_test/test_tags.pyrename to tests-unit/assets_test/test_tags_api.pyindex 6b1047802..595bf29c6 100644--- a/tests-unit/assets_test/test_tags.py+++ b/tests-unit/assets_test/test_tags_api.py@@ -69,8 +69,8 @@ def test_tags_empty_usage(http: requests.Session, api_base: str, asset_factory,used_names = [t["name"] for t in body2["tags"]]assert custom_tag in used_names- # Delete the asset so the tag usage drops to zero- rd = http.delete(f"{api_base}/api/assets/{_asset['id']}", timeout=120)+ # Hard-delete the asset so the tag usage drops to zero+ rd = http.delete(f"{api_base}/api/assets/{_asset['id']}?delete_content=true", timeout=120)assert rd.status_code == 204# Now the custom tag must not be returned when include_zero=falsediff --git a/tests-unit/assets_test/test_uploads.py b/tests-unit/assets_test/test_uploads.pyindex 137d7391a..d68e5b5d7 100644--- a/tests-unit/assets_test/test_uploads.py+++ b/tests-unit/assets_test/test_uploads.py@@ -18,25 +18,25 @@ def test_upload_ok_duplicate_reference(http: requests.Session, api_base: str, maassert r1.status_code == 201, a1assert a1["created_new"] is True- # Second upload with the same data and name should return created_new == False and the same asset+ # Second upload with the same data and name creates a new AssetReference (duplicates allowed)+ # Returns 200 because Asset already exists, but a new AssetReference is createdfiles = {"file": (name, data, "application/octet-stream")}form = {"tags": json.dumps(tags), "name": name, "user_metadata": json.dumps(meta)}r2 = http.post(api_base + "/api/assets", data=form, files=files, timeout=120)a2 = r2.json()- assert r2.status_code == 200, a2- assert a2["created_new"] is False+ assert r2.status_code in (200, 201), a2assert a2["asset_hash"] == a1["asset_hash"]- assert a2["id"] == a1["id"] # old reference+ assert a2["id"] != a1["id"] # new reference with same content- # Third upload with the same data but new name should return created_new == False and the new AssetReference+ # Third upload with the same data but different name also creates new AssetReferencefiles = {"file": (name, data, "application/octet-stream")}form = {"tags": json.dumps(tags), "name": name + "_d", "user_metadata": json.dumps(meta)}- r2 = http.post(api_base + "/api/assets", data=form, files=files, timeout=120)- a3 = r2.json()- assert r2.status_code == 200, a3- assert a3["created_new"] is False+ r3 = http.post(api_base + "/api/assets", data=form, files=files, timeout=120)+ a3 = r3.json()+ assert r3.status_code in (200, 201), a3assert a3["asset_hash"] == a1["asset_hash"]- assert a3["id"] != a1["id"] # old reference+ assert a3["id"] != a1["id"]+ assert a3["id"] != a2["id"]def test_upload_fastpath_from_existing_hash_no_file(http: requests.Session, api_base: str):@@ -116,7 +116,7 @@ def test_concurrent_upload_identical_bytes_different_names():"""Two concurrent uploads of identical bytes but different names.- Expect a single Asset (same hash), two AssetInfo rows, and exactly one created_new=True.+ Expect a single Asset (same hash), two AssetReference rows, and exactly one created_new=True."""scope = f"concupload-{uuid.uuid4().hex[:6]}"name1, name2 = "cu_a.bin", "cu_b.bin"diff --git a/tests-unit/requirements.txt b/tests-unit/requirements.txtindex 2355b8000..3a6790ee0 100644--- a/tests-unit/requirements.txt+++ b/tests-unit/requirements.txt@@ -2,4 +2,3 @@ pytest>=7.8.0pytest-aiohttppytest-asynciowebsocket-client-blake3diff --git a/tests-unit/seeder_test/test_seeder.py b/tests-unit/seeder_test/test_seeder.pynew file mode 100644index 000000000..db3795e48--- /dev/null+++ b/tests-unit/seeder_test/test_seeder.py@@ -0,0 +1,900 @@+"""Unit tests for the _AssetSeeder background scanning class."""++import threading+from unittest.mock import patch++import pytest++from app.assets.database.queries.asset_reference import UnenrichedReferenceRow+from app.assets.seeder import _AssetSeeder, Progress, ScanInProgressError, ScanPhase, State+++def fresh_seeder():+ """Create a fresh _AssetSeeder instance for testing."""+ seeder = _AssetSeeder()+ yield seeder+ seeder.shutdown(timeout=1.0)+++def mock_dependencies():+ """Mock all external dependencies for isolated testing."""+ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ yield+++class TestSeederStateTransitions:+ """Test state machine transitions."""++ def test_initial_state_is_idle(self, fresh_seeder: _AssetSeeder):+ assert fresh_seeder.get_status().state == State.IDLE++ def test_start_transitions_to_running(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ started = fresh_seeder.start(roots=("models",))+ assert started is True+ assert reached.wait(timeout=2.0)+ assert fresh_seeder.get_status().state == State.RUNNING++ barrier.set()++ def test_start_while_running_returns_false(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ second_start = fresh_seeder.start(roots=("models",))+ assert second_start is False++ barrier.set()++ def test_cancel_transitions_to_cancelling(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ cancelled = fresh_seeder.cancel()+ assert cancelled is True+ assert fresh_seeder.get_status().state == State.CANCELLING++ barrier.set()++ def test_cancel_when_idle_returns_false(self, fresh_seeder: _AssetSeeder):+ cancelled = fresh_seeder.cancel()+ assert cancelled is False++ def test_state_returns_to_idle_after_completion(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ fresh_seeder.start(roots=("models",))+ completed = fresh_seeder.wait(timeout=5.0)+ assert completed is True+ assert fresh_seeder.get_status().state == State.IDLE+++class TestSeederWait:+ """Test wait() behavior."""++ def test_wait_blocks_until_complete(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ fresh_seeder.start(roots=("models",))+ completed = fresh_seeder.wait(timeout=5.0)+ assert completed is True+ assert fresh_seeder.get_status().state == State.IDLE++ def test_wait_returns_false_on_timeout(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()++ def slow_collect(*args):+ barrier.wait(timeout=10.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ completed = fresh_seeder.wait(timeout=0.1)+ assert completed is False++ barrier.set()++ def test_wait_when_idle_returns_true(self, fresh_seeder: _AssetSeeder):+ completed = fresh_seeder.wait(timeout=1.0)+ assert completed is True+++class TestSeederProgress:+ """Test progress tracking."""++ def test_get_status_returns_progress_during_scan(+ self, fresh_seeder: _AssetSeeder+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_build(*args, **kwargs):+ reached.set()+ barrier.wait(timeout=5.0)+ return ([], set(), 0)++ paths = ["/path/file1.safetensors", "/path/file2.safetensors"]++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=paths),+ patch("app.assets.seeder.build_asset_specs", side_effect=slow_build),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ status = fresh_seeder.get_status()+ assert status.state == State.RUNNING+ assert status.progress is not None+ assert status.progress.total == 2++ barrier.set()++ def test_progress_callback_is_invoked(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ progress_updates: list[Progress] = []++ def callback(p: Progress):+ progress_updates.append(p)++ with patch(+ "app.assets.seeder.collect_paths_for_roots",+ return_value=[f"/path/file{i}.safetensors" for i in range(10)],+ ):+ fresh_seeder.start(roots=("models",), progress_callback=callback)+ fresh_seeder.wait(timeout=5.0)++ assert len(progress_updates) > 0+++class TestSeederCancellation:+ """Test cancellation behavior."""++ def test_scan_commits_partial_progress_on_cancellation(+ self, fresh_seeder: _AssetSeeder+ ):+ insert_count = 0+ barrier = threading.Event()+ first_insert_done = threading.Event()++ def slow_insert(specs, tags):+ nonlocal insert_count+ insert_count += 1+ if insert_count == 1:+ first_insert_done.set()+ if insert_count >= 2:+ barrier.wait(timeout=5.0)+ return len(specs)++ paths = [f"/path/file{i}.safetensors" for i in range(1500)]+ specs = [+ {+ "abs_path": p,+ "size_bytes": 100,+ "mtime_ns": 0,+ "info_name": f"file{i}",+ "tags": [],+ "fname": f"file{i}",+ "metadata": None,+ "hash": None,+ "mime_type": None,+ }+ for i, p in enumerate(paths)+ ]++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=paths),+ patch(+ "app.assets.seeder.build_asset_specs", return_value=(specs, set(), 0)+ ),+ patch("app.assets.seeder.insert_asset_specs", side_effect=slow_insert),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start(roots=("models",))+ assert first_insert_done.wait(timeout=2.0)++ fresh_seeder.cancel()+ barrier.set()+ fresh_seeder.wait(timeout=5.0)++ assert 1 <= insert_count < 3 # 1500 paths / 500 batch = 3; cancel stopped early+++class TestSeederErrorHandling:+ """Test error handling behavior."""++ def test_database_errors_captured_in_status(self, fresh_seeder: _AssetSeeder):+ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch(+ "app.assets.seeder.collect_paths_for_roots",+ return_value=["/path/file.safetensors"],+ ),+ patch(+ "app.assets.seeder.build_asset_specs",+ return_value=(+ [+ {+ "abs_path": "/path/file.safetensors",+ "size_bytes": 100,+ "mtime_ns": 0,+ "info_name": "file",+ "tags": [],+ "fname": "file",+ "metadata": None,+ "hash": None,+ "mime_type": None,+ }+ ],+ set(),+ 0,+ ),+ ),+ patch(+ "app.assets.seeder.insert_asset_specs",+ side_effect=Exception("DB connection failed"),+ ),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start(roots=("models",))+ fresh_seeder.wait(timeout=5.0)++ status = fresh_seeder.get_status()+ assert len(status.errors) > 0+ assert "DB connection failed" in status.errors[0]++ def test_dependencies_unavailable_captured_in_errors(+ self, fresh_seeder: _AssetSeeder+ ):+ with patch("app.assets.seeder.dependencies_available", return_value=False):+ fresh_seeder.start(roots=("models",))+ fresh_seeder.wait(timeout=5.0)++ status = fresh_seeder.get_status()+ assert len(status.errors) > 0+ assert "dependencies" in status.errors[0].lower()++ def test_thread_crash_resets_state_to_idle(self, fresh_seeder: _AssetSeeder):+ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch(+ "app.assets.seeder.sync_root_safely",+ side_effect=RuntimeError("Unexpected crash"),+ ),+ ):+ fresh_seeder.start(roots=("models",))+ fresh_seeder.wait(timeout=5.0)++ status = fresh_seeder.get_status()+ assert status.state == State.IDLE+ assert len(status.errors) > 0+++class TestSeederThreadSafety:+ """Test thread safety of concurrent operations."""++ def test_concurrent_start_calls_spawn_only_one_thread(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()++ def slow_collect(*args):+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ results = []++ def try_start():+ results.append(fresh_seeder.start(roots=("models",)))++ threads = [threading.Thread(target=try_start) for _ in range(10)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()++ barrier.set()++ assert sum(results) == 1++ def test_get_status_safe_during_scan(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ statuses = []+ for _ in range(100):+ statuses.append(fresh_seeder.get_status())++ barrier.set()++ assert all(+ s.state in (State.RUNNING, State.IDLE, State.CANCELLING)+ for s in statuses+ )+++class TestSeederMarkMissing:+ """Test mark_missing_outside_prefixes behavior."""++ def test_mark_missing_when_idle(self, fresh_seeder: _AssetSeeder):+ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch(+ "app.assets.seeder.get_all_known_prefixes",+ return_value=["/models", "/input", "/output"],+ ),+ patch(+ "app.assets.seeder.mark_missing_outside_prefixes_safely", return_value=5+ ) as mock_mark,+ ):+ result = fresh_seeder.mark_missing_outside_prefixes()+ assert result == 5+ mock_mark.assert_called_once_with(["/models", "/input", "/output"])++ def test_mark_missing_raises_when_running(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ with pytest.raises(ScanInProgressError):+ fresh_seeder.mark_missing_outside_prefixes()++ barrier.set()++ def test_mark_missing_returns_zero_when_dependencies_unavailable(+ self, fresh_seeder: _AssetSeeder+ ):+ with patch("app.assets.seeder.dependencies_available", return_value=False):+ result = fresh_seeder.mark_missing_outside_prefixes()+ assert result == 0++ def test_prune_first_flag_triggers_mark_missing_before_scan(+ self, fresh_seeder: _AssetSeeder+ ):+ call_order = []++ def track_mark(prefixes):+ call_order.append("mark_missing")+ return 3++ def track_sync(root):+ call_order.append(f"sync_{root}")+ return set()++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.get_all_known_prefixes", return_value=["/models"]),+ patch("app.assets.seeder.mark_missing_outside_prefixes_safely", side_effect=track_mark),+ patch("app.assets.seeder.sync_root_safely", side_effect=track_sync),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start(roots=("models",), prune_first=True)+ fresh_seeder.wait(timeout=5.0)++ assert call_order[0] == "mark_missing"+ assert "sync_models" in call_order+++class TestSeederPhases:+ """Test phased scanning behavior."""++ def test_start_fast_only_runs_fast_phase(self, fresh_seeder: _AssetSeeder):+ """Verify start_fast only runs the fast phase."""+ fast_called = []+ enrich_called = []++ def track_fast(*args, **kwargs):+ fast_called.append(True)+ return ([], set(), 0)++ def track_enrich(*args, **kwargs):+ enrich_called.append(True)+ return []++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", side_effect=track_fast),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", side_effect=track_enrich),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start_fast(roots=("models",))+ fresh_seeder.wait(timeout=5.0)++ assert len(fast_called) == 1+ assert len(enrich_called) == 0++ def test_start_enrich_only_runs_enrich_phase(self, fresh_seeder: _AssetSeeder):+ """Verify start_enrich only runs the enrich phase."""+ fast_called = []+ enrich_called = []++ def track_fast(*args, **kwargs):+ fast_called.append(True)+ return ([], set(), 0)++ def track_enrich(*args, **kwargs):+ enrich_called.append(True)+ return []++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", side_effect=track_fast),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", side_effect=track_enrich),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start_enrich(roots=("models",))+ fresh_seeder.wait(timeout=5.0)++ assert len(fast_called) == 0+ assert len(enrich_called) == 1++ def test_full_scan_runs_both_phases(self, fresh_seeder: _AssetSeeder):+ """Verify full scan runs both fast and enrich phases."""+ fast_called = []+ enrich_called = []++ def track_fast(*args, **kwargs):+ fast_called.append(True)+ return ([], set(), 0)++ def track_enrich(*args, **kwargs):+ enrich_called.append(True)+ return []++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", side_effect=track_fast),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", side_effect=track_enrich),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start(roots=("models",), phase=ScanPhase.FULL)+ fresh_seeder.wait(timeout=5.0)++ assert len(fast_called) == 1+ assert len(enrich_called) == 1+++class TestSeederPauseResume:+ """Test pause/resume behavior."""++ def test_pause_transitions_to_paused(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ paused = fresh_seeder.pause()+ assert paused is True+ assert fresh_seeder.get_status().state == State.PAUSED++ barrier.set()++ def test_pause_when_idle_returns_false(self, fresh_seeder: _AssetSeeder):+ paused = fresh_seeder.pause()+ assert paused is False++ def test_resume_returns_to_running(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ fresh_seeder.pause()+ assert fresh_seeder.get_status().state == State.PAUSED++ resumed = fresh_seeder.resume()+ assert resumed is True+ assert fresh_seeder.get_status().state == State.RUNNING++ barrier.set()++ def test_resume_when_not_paused_returns_false(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ resumed = fresh_seeder.resume()+ assert resumed is False++ barrier.set()++ def test_cancel_while_paused_works(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached_checkpoint = threading.Event()++ def slow_collect(*args):+ reached_checkpoint.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached_checkpoint.wait(timeout=2.0)++ fresh_seeder.pause()+ assert fresh_seeder.get_status().state == State.PAUSED++ cancelled = fresh_seeder.cancel()+ assert cancelled is True++ barrier.set()+ fresh_seeder.wait(timeout=5.0)+ assert fresh_seeder.get_status().state == State.IDLE++class TestSeederStopRestart:+ """Test stop and restart behavior."""++ def test_stop_is_alias_for_cancel(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()++ def slow_collect(*args):+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ stopped = fresh_seeder.stop()+ assert stopped is True+ assert fresh_seeder.get_status().state == State.CANCELLING++ barrier.set()++ def test_restart_cancels_and_starts_new_scan(+ self, fresh_seeder: _AssetSeeder, mock_dependencies+ ):+ barrier = threading.Event()+ reached = threading.Event()+ start_count = 0++ def slow_collect(*args):+ nonlocal start_count+ start_count += 1+ if start_count == 1:+ reached.set()+ barrier.wait(timeout=5.0)+ return []++ with patch(+ "app.assets.seeder.collect_paths_for_roots", side_effect=slow_collect+ ):+ fresh_seeder.start(roots=("models",))+ assert reached.wait(timeout=2.0)++ barrier.set()+ restarted = fresh_seeder.restart()+ assert restarted is True++ fresh_seeder.wait(timeout=5.0)+ assert start_count == 2++ def test_restart_preserves_previous_params(self, fresh_seeder: _AssetSeeder):+ """Verify restart uses previous params when not overridden."""+ collected_roots = []++ def track_collect(roots):+ collected_roots.append(roots)+ return []++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", side_effect=track_collect),+ patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start(roots=("input", "output"))+ fresh_seeder.wait(timeout=5.0)++ fresh_seeder.restart()+ fresh_seeder.wait(timeout=5.0)++ assert len(collected_roots) == 2+ assert collected_roots[0] == ("input", "output")+ assert collected_roots[1] == ("input", "output")++ def test_restart_can_override_params(self, fresh_seeder: _AssetSeeder):+ """Verify restart can override previous params."""+ collected_roots = []++ def track_collect(roots):+ collected_roots.append(roots)+ return []++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", side_effect=track_collect),+ patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),+ patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),+ ):+ fresh_seeder.start(roots=("models",))+ fresh_seeder.wait(timeout=5.0)++ fresh_seeder.restart(roots=("input",))+ fresh_seeder.wait(timeout=5.0)++ assert len(collected_roots) == 2+ assert collected_roots[0] == ("models",)+ assert collected_roots[1] == ("input",)+++def _make_row(ref_id: str, asset_id: str = "a1") -> UnenrichedReferenceRow:+ return UnenrichedReferenceRow(+ reference_id=ref_id, asset_id=asset_id,+ file_path=f"/fake/{ref_id}.bin", enrichment_level=0,+ )+++class TestEnrichPhaseDefensiveLogic:+ """Test skip_ids filtering and consecutive_empty termination."""++ def test_failed_refs_are_skipped_on_subsequent_batches(+ self, fresh_seeder: _AssetSeeder,+ ):+ """References that fail enrichment are filtered out of future batches."""+ row_a = _make_row("r1")+ row_b = _make_row("r2")+ call_count = 0++ def fake_get_unenriched(*args, **kwargs):+ nonlocal call_count+ call_count += 1+ if call_count <= 2:+ return [row_a, row_b]+ return []++ enriched_refs: list[list[str]] = []++ def fake_enrich(rows, **kwargs):+ ref_ids = [r.reference_id for r in rows]+ enriched_refs.append(ref_ids)+ # r1 always fails, r2 succeeds+ failed = [r.reference_id for r in rows if r.reference_id == "r1"]+ enriched = len(rows) - len(failed)+ return enriched, failed++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", side_effect=fake_get_unenriched),+ patch("app.assets.seeder.enrich_assets_batch", side_effect=fake_enrich),+ ):+ fresh_seeder.start(roots=("models",), phase=ScanPhase.ENRICH)+ fresh_seeder.wait(timeout=5.0)++ # First batch: both refs attempted+ assert "r1" in enriched_refs[0]+ assert "r2" in enriched_refs[0]+ # Second batch: r1 filtered out+ assert "r1" not in enriched_refs[1]+ assert "r2" in enriched_refs[1]++ def test_stops_after_consecutive_empty_batches(+ self, fresh_seeder: _AssetSeeder,+ ):+ """Enrich phase terminates after 3 consecutive batches with zero progress."""+ row = _make_row("r1")+ batch_count = 0++ def fake_get_unenriched(*args, **kwargs):+ nonlocal batch_count+ batch_count += 1+ # Always return the same row (simulating a permanently failing ref)+ return [row]++ def fake_enrich(rows, **kwargs):+ # Always fail — zero enriched, all failed+ return 0, [r.reference_id for r in rows]++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", side_effect=fake_get_unenriched),+ patch("app.assets.seeder.enrich_assets_batch", side_effect=fake_enrich),+ ):+ fresh_seeder.start(roots=("models",), phase=ScanPhase.ENRICH)+ fresh_seeder.wait(timeout=5.0)++ # Should stop after exactly 3 consecutive empty batches+ # Batch 1: returns row, enrich fails → filtered out in batch 2++ # But get_unenriched keeps returning it, filter removes it → empty → break+ # Actually: batch 1 has row, fails. Batch 2 get_unenriched returns [row],+ # skip_ids filters it → empty list → breaks via `if not unenriched: break`+ # So it terminates in 2 calls to get_unenriched.+ assert batch_count == 2++ def test_consecutive_empty_counter_resets_on_success(+ self, fresh_seeder: _AssetSeeder,+ ):+ """A successful batch resets the consecutive empty counter."""+ call_count = 0++ def fake_get_unenriched(*args, **kwargs):+ nonlocal call_count+ call_count += 1+ if call_count <= 6:+ return [_make_row(f"r{call_count}", f"a{call_count}")]+ return []++ def fake_enrich(rows, **kwargs):+ ref_id = rows[0].reference_id+ # Fail batches 1-2, succeed batch 3, fail batches 4-5, succeed batch 6+ if ref_id in ("r1", "r2", "r4", "r5"):+ return 0, [ref_id]+ return 1, []++ with (+ patch("app.assets.seeder.dependencies_available", return_value=True),+ patch("app.assets.seeder.sync_root_safely", return_value=set()),+ patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),+ patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),+ patch("app.assets.seeder.insert_asset_specs", return_value=0),+ patch("app.assets.seeder.get_unenriched_assets_for_roots", side_effect=fake_get_unenriched),+ patch("app.assets.seeder.enrich_assets_batch", side_effect=fake_enrich),+ ):+ fresh_seeder.start(roots=("models",), phase=ScanPhase.ENRICH)+ fresh_seeder.wait(timeout=5.0)++ # All 6 batches should run + 1 final call returning empty+ assert call_count == 7+ status = fresh_seeder.get_status()+ assert status.state == State.IDLEdiff --git a/utils/mime_types.py b/utils/mime_types.pynew file mode 100644index 000000000..916e963c5--- /dev/null+++ b/utils/mime_types.py@@ -0,0 +1,37 @@+"""Centralized MIME type initialization.++Call init_mime_types() once at startup to initialize the MIME type database+and register all custom types used across ComfyUI.+"""++import mimetypes++_initialized = False+++def init_mime_types():+ """Initialize the MIME type database and register all custom types.++ Safe to call multiple times; only runs once.+ """+ global _initialized+ if _initialized:+ return+ _initialized = True++ mimetypes.init()++ # Web types (used by server.py for static file serving)+ mimetypes.add_type('application/javascript; charset=utf-8', '.js')+ mimetypes.add_type('image/webp', '.webp')++ # Model and data file types (used by asset scanning / metadata extraction)+ mimetypes.add_type("application/safetensors", ".safetensors")+ mimetypes.add_type("application/safetensors", ".sft")+ mimetypes.add_type("application/pytorch", ".pt")+ mimetypes.add_type("application/pytorch", ".pth")+ mimetypes.add_type("application/pickle", ".ckpt")+ mimetypes.add_type("application/pickle", ".pkl")+ mimetypes.add_type("application/gguf", ".gguf")+ mimetypes.add_type("application/yaml", ".yaml")+ mimetypes.add_type("application/yaml", ".yml")