Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Lists Studio files by id prefix: upload:<id>, attachment:<message_id>:<attachment_id>,
image:<id>, sandbox:<session_id>:<relative path>, video:<id>, audio:<id>,
model:<training|exported>:<path>; message and session ids are URL-encoded; bytes come from each
source's fileUrl."""
from __future__ import annotations
import contextlib
import errno
import filecmp
import functools
import hashlib
import importlib
import io
import json
import mimetypes
import os
import platform
import re
import shutil
import stat
import tempfile
import threading
import time
import uuid
import zlib
from collections import OrderedDict
from datetime import datetime
from pathlib import Path
from types import ModuleType
from typing import BinaryIO, Callable, NamedTuple, Optional, Union
from urllib.parse import quote, unquote
from core.inference.gallery_projects import UNSAFE_NAME_CHARS, _bad_name
from loggers import get_logger
from storage import library_db
from utils.paths import relocations, storage_roots
from utils.paths.path_utils import is_path_within, same_path
from utils.paths.storage_roots import account_path
from utils.scan_memo import in_scan_memo
logger = get_logger(__name__)
_UPLOAD_ID_RE = re.compile(r"^[a-f0-9]{32}$")
_PROJECT_SESSION_PREFIX = "project-"
def _to_ms(value: object) -> int:
"""Sources store seconds or milliseconds; the Library speaks milliseconds."""
try:
number = float(value) # type: ignore[arg-type]
except (TypeError, ValueError):
# Video and audio records keep ISO-8601 ("2026-09-25T10:00:00Z").
try:
return int(datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() * 1000)
except ValueError:
return 0
return int(number if number > 1e12 else number * 1000)
def _named_for_type(name: str, content_type: str) -> str:
"""A pasted image or clip is named "Chat image"; the download needs its format's suffix."""
if Path(name).suffix or not content_type.startswith(("image/", "audio/", "video/")):
return name
suffix = next(
(ext for ext, known in content_types().items() if known == content_type), None
) or mimetypes.guess_extension(content_type)
return f"{name}{suffix}" if suffix else name
def _item(
item_id: str,
*,
name: str,
source: str,
content_type: str,
size_bytes: Optional[int],
created_at: int,
file_url: str,
thread_id: Optional[str] = None,
thread_title: Optional[str] = None,
text_only: bool = False,
model: Optional[dict] = None,
archived: bool = False,
fingerprint: Optional[str] = None,
pair_id: Optional[str] = None,
project_id: Optional[str] = None,
run_id: Optional[str] = None,
audio: Optional[dict] = None,
) -> dict:
return {
"id": item_id,
"name": name,
# The source's own name, which a rename leaves alone, so the type never follows it.
"fileName": name,
"source": source,
"contentType": content_type,
"sizeBytes": size_bytes,
"createdAt": created_at,
"updatedAt": created_at,
"fileUrl": file_url,
"threadId": thread_id,
"threadTitle": thread_title,
"pairId": pair_id,
"projectId": project_id,
"runId": run_id,
"textOnly": text_only,
"model": model,
"audio": audio,
"archived": archived,
"_fingerprint": fingerprint,
}
def _fingerprint(info: os.stat_result) -> str:
"""Inode (file id on Windows) plus birth time where the OS keeps one, not device (remounts change it); on Linux a recreated file can pass for the old."""
birth = getattr(info, "st_birthtime", None)
if birth is None and os.name == "nt":
birth = info.st_ctime
return str(info.st_ino) if birth is None else f"{info.st_ino}:{birth!r}"
def uploads_dir() -> Path:
return relocations.location_dir("uploads", account_path("library"))
_SOURCE_ROOTS: dict[str, Callable[[], Path]] = {
"upload": lambda: uploads_dir(),
"attachment": lambda: storage_roots.studio_db_path().parent,
"image": lambda: _gallery("image").module.gallery_dir(),
"video": lambda: _gallery("video").module.gallery_dir(),
"audio": lambda: _gallery("audio").module.gallery_dir(),
"model:training": lambda: storage_roots.outputs_root(),
"model:exported": lambda: storage_roots.exports_root(),
"sandbox": lambda: Path(importlib.import_module("core.inference.tools").sandbox_root()),
}
def _nearest_existing(path) -> Path:
path = Path(path)
return next((candidate for candidate in (path, *path.parents) if candidate.exists()), path)
def _device(path) -> Optional[int]:
"""The filesystem holding ``path``, or its nearest existing parent."""
try:
return os.stat(_nearest_existing(path)).st_dev
except OSError:
return None
def _disk(path) -> Optional[dict]:
try:
usage = shutil.disk_usage(_nearest_existing(path))
except OSError:
return None
return {"totalBytes": usage.total, "freeBytes": usage.free}
_reported_unavailable: set[str] = set()
def _log_unavailable(exc: Exception) -> None:
"""Once per folder per process: every listing asks again while a drive is unplugged."""
if str(exc) not in _reported_unavailable:
_reported_unavailable.add(str(exc))
logger.info("library.location_unavailable: %s", exc)
def disk_usage() -> Optional[dict]:
"""The disk the Library's own files live on (Studio's while it is unplugged), which need not
be the system disk, and the sources on it, so the bar only counts bytes that disk holds. A
source that cannot say where it lives is left out rather than failing the listing."""
try:
home = uploads_dir()
except relocations.LocationUnavailable as exc:
_log_unavailable(exc)
home = storage_roots.studio_root()
except OSError:
return None
disk, device = _disk(home), _device(home)
if disk is None:
return None
sources = []
for source, root in _SOURCE_ROOTS.items():
try:
if _device(root()) == device:
sources.append(source)
except relocations.LocationUnavailable as exc:
_log_unavailable(exc)
except Exception:
logger.warning("library.source_root_failed: %s", source, exc_info = True)
return {**disk, "sources": sources}
def upload_path(upload_id: str) -> Optional[Path]:
if not _UPLOAD_ID_RE.match(upload_id):
return None
return uploads_dir() / upload_id
def save_upload(name: str, content_type: str, chunks) -> dict:
"""Stream ``chunks`` to disk and record the upload."""
upload_id = uuid.uuid4().hex
tmp_path = uploads_dir() / f".{upload_id}.tmp"
final_path = tmp_path.with_name(upload_id)
size = 0
try:
with open(tmp_path, "wb") as handle:
for chunk in chunks:
size += len(chunk)
handle.write(chunk)
# Under the move lock, so the upload lands wherever a concurrent move left uploads.
with _move_lock:
final_path = uploads_dir() / upload_id
if final_path.parent == tmp_path.parent:
os.replace(tmp_path, final_path)
else:
shutil.move(str(tmp_path), str(final_path))
return library_db.insert_upload(upload_id, name, content_type, size)
except BaseException:
tmp_path.unlink(missing_ok = True)
final_path.unlink(missing_ok = True)
raise
def _verify_native(lease: str, *, consume: bool):
from utils.native_path_leases import verify_native_path_lease
grant = verify_native_path_lease(
lease,
operation = "attach",
expected_kind = "attachment",
expected_path_type = "file",
consume = consume,
)
account_path(str(grant.canonical_path))
return grant
def check_native_upload(lease: str) -> tuple[str, int]:
"""(name, size) of a desktop drop, its grant checked but not spent, so a batch can check all."""
grant = _verify_native(lease, consume = False)
return grant.canonical_path.name, os.stat(grant.canonical_path).st_size
def open_native_upload(lease: str):
"""(name, content type, binary handle) of a desktop drop: the app signs the path the OS handed
it, never the webview, and the grant is spent here before a byte is read."""
grant = _verify_native(lease, consume = True)
name = grant.canonical_path.name
# Open once: the name could be swapped for a link outside the workspace after the grant check.
try:
handle = _open_regular(str(grant.canonical_path))
except LookupError:
raise ValueError("The dropped file changed after it was checked.") from None
info = os.fstat(handle.fileno())
identity = (grant.device_id, grant.file_id)
if (None not in identity and (info.st_dev, info.st_ino) != identity) or (
grant.size_bytes is not None and info.st_size != grant.size_bytes
):
handle.close()
raise ValueError("The dropped file changed after it was checked.")
return name, _guess_type(name), handle
_upload_lock = threading.Lock()
def _swap_in(path: Path, data: bytes) -> None:
"""Replace the file whole: staged beside it and renamed over, so a failed write leaves it be."""
tmp_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
try:
tmp_path.write_bytes(data)
os.replace(tmp_path, path)
except BaseException:
tmp_path.unlink(missing_ok = True)
raise
def write_upload_text(
upload_id: str,
text: str,
encoding: str = "utf-8",
) -> bool:
"""Write an edited note back in `encoding`. A BOM, if the note had one, is the text's first
character, which each of these codecs writes as that encoding's own BOM."""
if upload_path(upload_id) is None:
return False
data = text.encode(encoding)
# Under the move lock, or a concurrent move would leave this write in the old folder.
with _move_lock, _upload_lock:
path = upload_path(upload_id)
if library_db.get_upload(upload_id) is None:
return False
# Old file kept on disk under a second name for rollback (512 MiB uploads); in memory where links are unsupported.
kept, previous = None, None
if path.exists():
kept = path.with_name(f".{upload_id}.{uuid.uuid4().hex}.tmp")
try:
os.link(path, kept)
except OSError:
kept, previous = None, path.read_bytes()
try:
_swap_in(path, data)
library_db.touch_upload(upload_id, len(data))
except BaseException:
if kept is not None:
os.replace(kept, path)
elif previous is not None:
_swap_in(path, previous)
raise
finally:
if kept is not None:
kept.unlink(missing_ok = True)
return True
_LEFTOVER_AGE_SECONDS = 3600
_swept_at: dict[str, float] = {}
def _sweep_leftovers() -> None:
"""Clears crash leftovers in uploads, at most once an hour; a delete cut short puts its file back."""
directory = uploads_dir()
now = time.time()
if now - _swept_at.get(str(directory), 0) < _LEFTOVER_AGE_SECONDS:
return
_swept_at[str(directory)] = now
with _upload_lock:
with os.scandir(directory) as entries:
leftovers = [
entry
for entry in entries
if entry.name.startswith(".") and entry.name.endswith((".tmp", ".deleting"))
]
for entry in leftovers:
try:
info = entry.stat(follow_symlinks = False)
if not stat.S_ISREG(info.st_mode) or now - info.st_mtime < _LEFTOVER_AGE_SECONDS:
continue
upload_id = entry.name[1 : -len(".deleting")]
if (
entry.name.endswith(".deleting")
and _UPLOAD_ID_RE.match(upload_id)
and library_db.get_upload(upload_id) is not None
and not (directory / upload_id).exists()
):
os.replace(entry.path, directory / upload_id)
else:
os.unlink(entry.path)
except OSError:
logger.debug("library.leftover_sweep_failed: %s", entry.name, exc_info = True)
def _upload_items() -> list[dict]:
try:
_sweep_leftovers()
except relocations.LocationUnavailable:
pass # Its drive is unplugged: the rows still list, and disk_usage reports it once.
except OSError:
logger.debug("library.leftover_sweep_failed", exc_info = True)
# Rows whose file is absent from the folder in use are kept, not listed or counted until restored.
try:
present = set(os.listdir(uploads_dir()))
except OSError:
present = None
items = []
for upload in library_db.list_uploads():
if present is not None and upload["id"] not in present:
continue
item = _item(
f"upload:{upload['id']}",
name = upload["name"],
source = "uploaded",
content_type = upload["contentType"],
size_bytes = upload["sizeBytes"],
created_at = upload["createdAt"],
file_url = f"/api/library/uploads/{upload['id']}/file",
)
item["updatedAt"] = upload["updatedAt"]
items.append(item)
return items
def _attachment_id(message_id: str, attachment_id: str) -> str:
"""Message ids are quoted, so an attachment id is whatever follows the first colon."""
return f"attachment:{quote(message_id, safe = '')}:{attachment_id}"
def _attachment_ref(ref: str) -> tuple[str, str]:
"""The message and attachment ids an ``attachment:`` item id's ref names."""
message, _, attachment = ref.partition(":")
return unquote(message), attachment
def _attachment_items() -> list[dict]:
from storage.studio_db import list_chat_attachments
items = []
counted: set[str] = set()
for attachment in list_chat_attachments():
content_type = str(attachment.get("contentType") or "").split(";", 1)[0].strip().lower()
has_bytes = (
attachment.get("type") in ("image", "audio")
or content_type.startswith(("image/", "audio/", "video/"))
or bool(attachment.get("hasOriginal"))
)
message_id, attachment_id = attachment["messageId"], attachment["id"]
items.append(
_item(
_attachment_id(message_id, attachment_id),
name = _named_for_type(attachment.get("name") or "Attachment", content_type),
source = "uploaded",
content_type = content_type or "application/octet-stream",
size_bytes = attachment.get("sizeBytes"),
created_at = _to_ms(attachment.get("createdAt")),
file_url = f"/api/chat/attachments/{quote(message_id, safe = '')}/{quote(attachment_id, safe = '')}/file",
thread_id = attachment.get("threadId"),
thread_title = attachment.get("threadTitle"),
text_only = not has_bytes,
pair_id = attachment.get("pairId"),
)
)
sha256 = attachment.get("originalSha256")
if sha256:
text_bytes = attachment.get("textBytes") or 0
first = attachment.get("hasOriginal") and sha256 not in counted
items[-1]["storageBytes"] = text_bytes + (
(attachment.get("sizeBytes") or 0) if first else 0
)
counted.add(sha256)
return items
_NAME_BREAK_RE = re.compile(f"[{UNSAFE_NAME_CHARS}\\s]+")
def _prompt_name(
prompt: str,
fallback: str,
extension: str,
suffix: str = "",
) -> str:
cleaned = _NAME_BREAK_RE.sub(" ", prompt).strip()
if len(cleaned) > 60:
cleaned = cleaned[:60].rsplit(" ", 1)[0] or cleaned[:60]
return f"{cleaned or fallback}{suffix}.{extension}"
_AUDIO_EXTENSION_RE = re.compile(r"\.(?:wav|mp3|flac|ogg|opus|m4a|aac|webm)$", re.IGNORECASE)
def _audio_name(record: dict) -> str:
"""``song - Vocals.wav``, ``prompt (2).wav``: a run's clips share one prompt."""
prompt = str(record.get("prompt") or "").rstrip(" .")
role = record.get("role")
settings = record.get("settings") if isinstance(record.get("settings"), dict) else {}
variation = settings.get("variation")
if record.get("workflow") == "separate" and isinstance(role, str) and role:
prompt = _AUDIO_EXTENSION_RE.sub("", prompt.strip())
stem = _NAME_BREAK_RE.sub(" ", role.replace("_", " ").replace("-", " ")).strip()
suffix = f" - {stem[:1].upper()}{stem[1:]}" if stem else ""
else:
tags = [
*(["edit"] if role == "edit" else []),
*([str(variation)] if type(variation) is int else []),
]
suffix = f" ({' '.join(tags)})" if tags else ""
return _prompt_name(prompt, "Audio", "wav", suffix)
def _audio_details(record: dict) -> dict:
settings = record.get("settings") if isinstance(record.get("settings"), dict) else {}
music = record.get("workflow") == "music"
variation = settings.get("variation")
return {
"workflow": record.get("workflow"),
"role": record.get("role"),
"groupId": record.get("group_id"),
"durationS": record.get("duration_s"),
"model": record.get("model"),
"mode": settings.get("mode") if music else None,
"variation": variation if music and type(variation) is int else None,
}
class _Gallery(NamedTuple):
module: ModuleType
records: Callable
resolve: Callable
label: str # what an item with no prompt is called
extension: str
content_type: str
folder: str # the project folder Add to project copies into
_GALLERIES = {
"image": ("image_gallery", "list_images", "image_path", "Image", "png", "image/png", "images"),
"video": ("video_gallery", "list_videos", "video_path", "Video", "mp4", "video/mp4", "videos"),
"audio": ("audio_gallery", "list_audio", "audio_path", "Audio", "wav", "audio/wav", "audio"),
}
def _gallery(kind: str) -> _Gallery:
module_name, records, resolve, *rest = _GALLERIES[kind]
module = importlib.import_module(f"core.inference.{module_name}")
return _Gallery(module, getattr(module, records), getattr(module, resolve), *rest)
def _listed_file(gallery: _Gallery, root: Path, record_id: str) -> Optional[Path]:
"""``gallery.resolve`` for an id its own listing gave, with one lstat: a plain file in the
folder cannot lead out of it, a symlink still takes the full check."""
if not gallery.module._ID_RE.match(record_id):
return None
path = root / f"{record_id}.{gallery.extension}"
try:
mode = path.lstat().st_mode
except OSError:
return None
if stat.S_ISREG(mode):
return path
return gallery.resolve(record_id) if stat.S_ISLNK(mode) else None
def _gallery_items(kind: str) -> list[dict]:
gallery = _gallery(kind)
items = []
for archived in (False, True):
records = gallery.records(archived = archived)
# Looked up after the records, and only when some exist, so an unreadable folder lists nothing.
root = gallery.module.gallery_dir() if records else None
for record in records:
# An Edit's hidden Original: deleting it here would break that Edit.
if kind == "audio" and record.get("role") == "source":
continue
path = _listed_file(gallery, root, record["id"])
try:
size = path.stat().st_size if path is not None else None
except OSError:
size = None
sidecar = None
if size is not None:
try:
sidecar = path.with_suffix(".json").stat().st_size
except OSError:
pass
# A conversion keeps the recording it converted as {id}.source.wav beside its clip.
if sidecar is not None and kind == "audio":
try:
sidecar += path.with_name(f"{path.stem}.source.wav").stat().st_size
except OSError:
pass
items.append(
_item(
f"{kind}:{record['id']}",
name = _audio_name(record)
if kind == "audio"
else _prompt_name(
str(record.get("prompt") or ""), gallery.label, gallery.extension
),
source = "generated",
content_type = gallery.content_type,
size_bytes = size,
created_at = _to_ms(record.get("created_at")),
file_url = record["url"],
archived = archived,
audio = _audio_details(record) if kind == "audio" else None,
)
)
if sidecar is not None:
items[-1]["storageBytes"] = size + sidecar
return items
def _image_items() -> list[dict]:
return _gallery_items("image")
def _video_items() -> list[dict]:
return _gallery_items("video")
def _audio_items() -> list[dict]:
return _gallery_items("audio")
def unlisted_bytes(items: list[dict]) -> dict[str, int]:
"""Bytes no item lists (voices, recordings, transcripts, hidden Edit originals), per source."""
from core.inference import transcript_gallery
try:
roots = (_gallery("audio").module.gallery_dir(), transcript_gallery.gallery_dir())
except relocations.LocationUnavailable as exc:
_log_unavailable(exc)
return {}
except OSError:
return {}
total = 0
for root in roots:
try:
total += _tree_stats(root)[0]
except OSError:
pass
listed = sum(
item.get("storageBytes") or item.get("sizeBytes") or 0
for item in items
if item["id"].startswith("audio:")
)
return {"audio": max(0, total - listed)}
MODEL_CONTENT_TYPE = "application/x-unsloth-model"
def _tree_stats(path: Path) -> tuple[int, float, os.stat_result]:
"""(total bytes, newest mtime, own stat) of a model file or dir; symlinks not followed."""
info = path.stat()
if stat.S_ISREG(info.st_mode):
return info.st_size, info.st_mtime, info
total, newest = 0, info.st_mtime
for root, _dirs, files in os.walk(path):
for name in files:
try:
entry = os.lstat(os.path.join(root, name))
except OSError:
continue
total += entry.st_size
newest = max(newest, entry.st_mtime)
return total, newest, info
_EXPORT_SUFFIX_RE = re.compile(r"[-_](gguf|adapter|merged|finetune)$", re.IGNORECASE)
def _training_runs_by_dir() -> dict[str, str]:
"""Folder name under outputs/ to the id of the newest training run that wrote it. A run that wrote
anywhere else is left out, so an external folder of the same name never claims a managed model.
A finished run outranks a running or failed one reusing the folder, which may have written nothing."""
from storage.studio_db import get_connection
from utils.paths.storage_roots import outputs_root, resolve_output_dir
conn = get_connection()
try:
rows = conn.execute(
"SELECT id, output_dir FROM training_runs WHERE output_dir IS NOT NULL"
" ORDER BY status IN ('completed', 'stopped'), started_at"
).fetchall()
except Exception:
logger.debug("library.training_runs_unavailable", exc_info = True)
return {}
finally:
conn.close()
root = Path(outputs_root()).resolve()
runs: dict[str, str] = {}
for row in rows:
try:
path = Path(resolve_output_dir(row["output_dir"])).resolve()
except (ValueError, OSError):
continue
if path.parent == root:
runs[path.name] = row["id"]
return runs
_MAX_EXPORT_METADATA_BYTES = 1024 * 1024
def _export_metadata(path: str, root: Path) -> Optional[dict]:
"""The export_metadata.json Studio writes beside an export, from the model up to its top folder."""
here = Path(path)
if not here.is_dir():
here = here.parent
while True:
meta = here / "export_metadata.json"
if meta.is_file():
try:
with open(meta, "rb") as handle:
raw = handle.read(_MAX_EXPORT_METADATA_BYTES + 1)
if len(raw) > _MAX_EXPORT_METADATA_BYTES:
return None
data = json.loads(raw.decode("utf-8-sig"))
except (OSError, ValueError):
return None
return data if isinstance(data, dict) else None
if here.parent == root or here == root or here.parent == here:
return None
here = here.parent
def _model_run_id(path: str, origin: str, runs: dict[str, str]) -> Optional[str]:
"""A folder with no metadata has no known origin. An export uses its recorded checkpoint, else its folder name (run/checkpoint, run-GGUF)."""
from utils.paths.storage_roots import exports_root, outputs_root
try:
outputs = Path(outputs_root()).resolve()
if origin == "training":
return runs.get(Path(path).resolve().relative_to(outputs).parts[0])
root = Path(exports_root()).resolve()
top = Path(path).resolve().relative_to(root).parts[0]
meta = _export_metadata(path, root)
if meta is None:
return None
source = meta.get("source_checkpoint")
if isinstance(source, str) and source:
return runs.get(Path(source).resolve().relative_to(outputs).parts[0])
except (ValueError, IndexError, OSError):
return None
while top and top not in runs:
stripped = _EXPORT_SUFFIX_RE.sub("", top)
if stripped == top:
break
top = stripped
return runs.get(top)
def _model_items() -> list[dict]:
from utils.models.model_config import (
get_base_model_from_checkpoint,
scan_exported_models,
scan_trained_models,
)
from utils.paths.storage_roots import exports_root, outputs_root
found = [
(name, path, "training", model_type, None)
for name, path, model_type in scan_trained_models(str(outputs_root()))
]
found.extend(
(name, path, "exported", export_type, base_model)
for name, path, export_type, base_model in scan_exported_models(str(exports_root()))
)
runs = _training_runs_by_dir() if found else {}
items = []
for name, path, origin, model_type, base_model in found:
stats_path = Path(path)
if model_type == "gguf" and stats_path.is_file():
stats_path = stats_path.parent
try:
size, modified, info = _tree_stats(stats_path)
if stats_path != Path(path):
info = os.stat(path)
except OSError:
continue
if base_model is None and origin == "training":
try:
base_model = get_base_model_from_checkpoint(path)
except Exception:
base_model = None
items.append(
_item(
f"model:{origin}:{path}",
name = name,
source = "generated",
content_type = MODEL_CONTENT_TYPE,
size_bytes = size,
created_at = _to_ms(modified),
file_url = "",
model = {
"path": path,
"origin": origin,
"exportType": model_type,
"baseModel": base_model,
},
fingerprint = _fingerprint(info),
run_id = _model_run_id(path, origin, runs),
)
)
return items
class _SandboxSession(NamedTuple):
session_id: str
thread_id: Optional[str]
title: Optional[str]
pair_id: Optional[str] = None
project_id: Optional[str] = None
def _sandbox_sessions() -> list[_SandboxSession]:
from storage.studio_db import get_connection
conn = get_connection()
try:
threads = conn.execute("SELECT id, title, pair_id FROM chat_threads").fetchall()
projects = conn.execute(
"SELECT id, name, root_path FROM chat_projects WHERE COALESCE(working_folder, '') = ''"
).fetchall()
finally:
conn.close()
thread_ids = {row["id"] for row in threads}
return [
_SandboxSession(row["id"], row["id"], row["title"], pair_id = row["pair_id"])
for row in threads
] + [
_SandboxSession(
f"{_PROJECT_SESSION_PREFIX}{row['id']}", None, row["name"], project_id = row["id"]
)
for row in projects
# A chat with this id owns the directory (_get_project_workdir): listing both walks it twice.
if f"{_PROJECT_SESSION_PREFIX}{row['id']}" not in thread_ids
and _studio_project_root(row["root_path"])
]
def _studio_project_root(root_path: Optional[str]) -> bool:
"""Whether a project works in Studio's own workspace (every project is given a folder there),
not a folder of the user's own, whose files are not Studio's."""
from utils.paths.storage_roots import project_workspaces_root
if not root_path:
return True
return is_path_within(
os.path.realpath(os.path.expanduser(root_path)),
os.path.realpath(project_workspaces_root()),
)
def _sandbox_session_eligible(session_id: str) -> bool:
"""Whether one session id is among ``_sandbox_sessions``, asked of that row alone."""
from storage.studio_db import get_connection
conn = get_connection()
try:
if conn.execute("SELECT 1 FROM chat_threads WHERE id = ?", (session_id,)).fetchone():
return True
if not session_id.startswith(_PROJECT_SESSION_PREFIX):
return False
row = conn.execute(
"SELECT root_path FROM chat_projects WHERE id = ? AND COALESCE(working_folder, '') = ''",
(session_id[len(_PROJECT_SESSION_PREFIX) :],),
).fetchone()
finally:
conn.close()
return row is not None and _studio_project_root(row["root_path"])
def _sandbox_names(directory: str) -> frozenset:
"""File names of one sandbox listing, cached so a grid of cards walks each sandbox once."""
from routes.inference import _sandbox_listing_names
return _LISTING.get(
("names", directory),
lambda: frozenset(_sandbox_listing_names(directory) if os.path.isdir(directory) else []),
_SANDBOX_TTL_SECONDS,
)
def _sandbox_path(ref: str) -> str:
"""Resolves a sandbox: id by the listing walk's rules, for this one file only; else LookupError."""
from core.inference.tools import (
_MAX_SANDBOX_PATH_SEGMENTS,
_servable_segment,
_user_path_parts,
resolve_sandbox_workdir,
)
session, _, relative = ref.partition(":")
session_id = unquote(session)
parts = relative.split("/")
if not relative or any(part.startswith(".") or not _servable_segment(part) for part in parts):
raise LookupError(ref)
if not _sandbox_session_eligible(session_id):
raise LookupError(ref)
directory = os.path.realpath(resolve_sandbox_workdir(session_id))
if len(_user_path_parts(parts, directory)) > _MAX_SANDBOX_PATH_SEGMENTS:
raise LookupError(ref)
path = os.path.join(directory, *parts)
if not same_path(os.path.realpath(path), path) or not is_path_within(path, directory):
raise LookupError(ref)
# Past the listing's cap, or skipped by its walk: never listed, so not reachable by id either.
if relative not in _sandbox_names(directory):
raise LookupError(ref)
return path
def _open_regular(path: str) -> BinaryIO:
"""LookupError unless a regular file; O_NOFOLLOW where available, and the path must still resolve to this file, refusing swapped parent links too."""
flags = (
os.O_RDONLY
| getattr(os, "O_NOFOLLOW", 0)
| getattr(os, "O_BINARY", 0)
| getattr(os, "O_NONBLOCK", 0)
)
try:
fd = os.open(path, flags)
except OSError:
raise LookupError(path) from None
try:
info = os.fstat(fd)
checked = os.stat(path, follow_symlinks = False)
if (
not stat.S_ISREG(info.st_mode)
or not same_path(os.path.realpath(path), path)
or (checked.st_dev, checked.st_ino) != (info.st_dev, info.st_ino)
):
raise LookupError(path)
except OSError:
os.close(fd)
raise LookupError(path) from None
except BaseException:
os.close(fd)
raise
return os.fdopen(fd, "rb")
def _open_sandbox_file(ref: str) -> tuple[BinaryIO, str]:
"""Opens a sandbox: id's file once and checks it by descriptor, since tool code runs here."""
path = _sandbox_path(ref)
try:
return _open_regular(path), path
except LookupError:
raise LookupError(ref) from None
def _delete_sandbox_file(ref: str, expected: Optional[str] = None) -> bool:
"""Unlink the file an id names, only while its name is still the file checked, and with
`expected`, only while it is the file listed with that fingerprint."""
from core.inference.gallery_projects import _DIR_FLAGS, _USE_DIR_FD
try:
handle, path = _open_sandbox_file(ref)
except LookupError:
return False
parent, name = os.path.split(path)
with handle:
info = os.fstat(handle.fileno())
if expected is not None and _fingerprint(info) != expected:
raise ItemChanged("This file changed since the Library listed it.")
if _USE_DIR_FD:
dir_fd = os.open(parent, _DIR_FLAGS)
try:
entry = os.stat(name, dir_fd = dir_fd, follow_symlinks = False)
if (entry.st_dev, entry.st_ino) != (info.st_dev, info.st_ino):
return False
os.unlink(name, dir_fd = dir_fd)
finally:
os.close(dir_fd)
return True
os.unlink(path)
return True
def _sandbox_items() -> list[dict]:
from core.inference.tools import resolve_sandbox_workdir
from routes.inference import _sandbox_listing_names
items = []
generation = _LISTING.generation
for session_id, thread_id, title, pair_id, project_id in _sandbox_sessions():
try:
directory = os.path.realpath(resolve_sandbox_workdir(session_id))
names = _sandbox_listing_names(directory) if os.path.isdir(directory) else []
except Exception:
logger.debug("library.sandbox_listing_failed", exc_info = True)
continue
_LISTING.put(("names", directory), frozenset(names), generation)
for relative in names:
if os.path.basename(relative).startswith("."):
continue
try:
info = os.stat(os.path.join(directory, relative))
except OSError:
continue
items.append(
_item(
f"sandbox:{quote(session_id, safe = '')}:{relative}",
name = os.path.basename(relative),
source = "generated",
content_type = _guess_type(relative),
size_bytes = info.st_size,
created_at = info.st_mtime_ns // 1_000_000,
file_url = f"/api/inference/sandbox/{quote(session_id, safe = '')}/{quote(relative)}",
thread_id = thread_id,
thread_title = title,
fingerprint = _fingerprint(info),
pair_id = pair_id,
project_id = project_id,
)
)
return items
_CONTENT_TYPES = {
f".{extension}": content_type
for content_type, extensions in (
("image/svg+xml", "svg"),
("image/tiff", "tif tiff"),
("image/x-icon", "ico"),
("image/heic", "heic"),
("image/heif", "heif"),
("audio/mpeg", "mp3"),
("audio/wav", "wav"),
("audio/ogg", "ogg oga opus"),
("audio/flac", "flac"),
("audio/mp4", "m4a"),
("audio/aac", "aac"),
("audio/webm", "weba"),
("video/mp4", "mp4 m4v"),
("video/quicktime", "mov"),
("video/webm", "webm"),
("video/x-matroska", "mkv"),
("video/x-msvideo", "avi"),
("video/ogg", "ogv"),
("application/pdf", "pdf"),
("text/plain", "txt log toml ini sql"),
("text/markdown", "md markdown"),
("text/csv", "csv"),
("text/tab-separated-values", "tsv"),
("application/json", "json"),
("application/jsonl", "jsonl"),
("application/xml", "xml"),
("text/yaml", "yaml yml"),
("text/html", "html htm"),
("text/css", "css"),
("text/javascript", "js mjs cjs jsx"),
("text/typescript", "ts tsx mts cts"),
("text/x-python", "py"),
("application/x-ipynb+json", "ipynb"),
("text/x-shellscript", "sh"),
("application/rtf", "rtf"),
("application/msword", "doc"),
("application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"),
("application/vnd.ms-excel", "xls"),
("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"),
("application/vnd.ms-powerpoint", "ppt"),
("application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx"),
("application/vnd.oasis.opendocument.text", "odt"),
("application/vnd.oasis.opendocument.spreadsheet", "ods"),
("application/vnd.oasis.opendocument.presentation", "odp"),
("application/epub+zip", "epub"),
("application/zip", "zip"),
("application/gzip", "gz"),
("application/x-tar", "tar"),
("application/vnd.apache.parquet", "parquet"),
("application/octet-stream", "safetensors gguf"),
)
for extension in extensions.split()
}
_MEDIA_TYPE_RE = re.compile(r"[a-z0-9][a-z0-9!#$&^_.+-]*/[a-z0-9][a-z0-9!#$&^_.+-]*")
_ACTIVE_TYPES = frozenset(
"text/html application/xhtml+xml image/svg+xml application/pdf text/xml application/xml"
" text/javascript application/javascript application/x-shockwave-flash".split()
)
@functools.lru_cache(maxsize = None)
def content_types() -> dict[str, str]:
from routes.inference import _SANDBOX_MEDIA_TYPES
return {**_CONTENT_TYPES, **_SANDBOX_MEDIA_TYPES}
def media_type(value: object) -> Optional[str]:
"""The one media type ``value`` names, lower case and without parameters; None when it names
none, or several (``audio/x, text/html``)."""
essence = str(value or "").split(";", 1)[0].strip().lower()
return essence if _MEDIA_TYPE_RE.fullmatch(essence) else None
def _guess_type(name: str) -> str:
return content_types().get(os.path.splitext(name)[1].lower(), "application/octet-stream")
def upload_content_type(name: str, declared: Optional[str]) -> str:
"""What an upload is stored as: its extension's type when the map knows it, else the type the
client declared, as long as it is one type and nothing a browser would run."""
known = content_types().get(os.path.splitext(name)[1].lower())
if known:
return known
declared_type = media_type(declared)
if declared_type is None or declared_type in _ACTIVE_TYPES:
return "application/octet-stream"
return declared_type
def item_type(item_id: str) -> str:
"""The media type an item's bytes are, by the Library's fixed map; "" for an item it has none
for. LookupError for an upload the account does not have."""
kind, _, ref = item_id.partition(":")
if kind in _GALLERIES:
return _gallery(kind).content_type
if kind == "upload":
record = library_db.get_upload(ref)
if record is None:
raise LookupError(item_id)
return media_type(record["contentType"]) or ""
return _guess_type(ref) if kind == "sandbox" else ""
def _account_key() -> str:
"""Which account's stores a cached answer came from: each has a studio.db of its own."""
from utils.paths import studio_db_path
return str(studio_db_path())
class _Memo:
"""Answers kept per account and key, each for ``ttl`` seconds while its ``stamp`` agrees, and
only the ``size`` most recent when that is set. ``forget`` drops them all, and an answer worked
out across a forget is not kept: it can predate the write that forgot."""
def __init__(self, size: int = 0):
self.size = size
self.generation = 0
self._entries: OrderedDict[tuple, tuple[float, object, object]] = OrderedDict()
self._lock = threading.Lock()
def forget(self) -> None:
with self._lock:
self.generation += 1
self._entries.clear()
def get(
self,
key: tuple,
compute: Callable,
ttl: float = float("inf"),
stamp = None,
):
full, now = (_account_key(), *key), time.monotonic()
with self._lock:
generation, hit = self.generation, self._entries.get(full)
if hit is not None and now - hit[0] < ttl and hit[1] == stamp:
self._entries.move_to_end(full)
return hit[2]
value = compute()
self.put(key, value, generation, stamp)
return value
def put(
self,
key: tuple,
value: object,
generation: int,
stamp = None,
) -> None:
with self._lock:
if generation == self.generation:
self._entries[(_account_key(), *key)] = (time.monotonic(), stamp, value)
while self.size and len(self._entries) > self.size:
self._entries.popitem(last = False)
_SANDBOX_TTL_SECONDS = 5.0
_MODEL_TTL_SECONDS = 60.0
# Bounded: sandboxes of chats and accounts nobody lists again would otherwise stay forever.
_LISTING = _Memo(size = 1024)
invalidate_listing = _LISTING.forget
def _model_stamp() -> tuple:
"""Fine-tune root and folder mtimes; adding, removing or renaming a run changes one."""
from utils.paths.storage_roots import exports_root, outputs_root
stamp = []
for root in (outputs_root(), exports_root()):
try:
with os.scandir(root) as entries:
stamp.append((str(root), os.stat(root).st_mtime_ns))
stamp.extend(
(entry.name, entry.stat(follow_symlinks = False).st_mtime_ns)
for entry in entries
if entry.is_dir(follow_symlinks = False)
)
except OSError:
stamp.append((str(root), None))
return tuple(sorted(stamp, key = repr))
def _remembered(
name: str,
ttl: float,
stamp: Optional[Callable] = None,
) -> Callable:
"""The source ``name``, remembered. Found by name, so a test that swaps the source swaps it."""
def remembered() -> list[dict]:
items = _LISTING.get((name,), globals()[name], ttl, stamp() if stamp else None)
return [dict(item) for item in items]
remembered.__name__ = name
return remembered
_SOURCES = (
_upload_items,
_attachment_items,
_image_items,
_video_items,
_audio_items,
_remembered("_model_items", _MODEL_TTL_SECONDS, _model_stamp),
_remembered("_sandbox_items", _SANDBOX_TTL_SECONDS),
)
@in_scan_memo
def list_items() -> list[dict]:
"""Every item with its overlay applied, newest activity first. A failing source is skipped so
one broken store cannot empty the whole Library."""
generation = _LISTING.generation
overlay = library_db.list_entries()
items: list[dict] = []
for source in _SOURCES:
try:
items.extend(source())
except relocations.LocationUnavailable as exc:
_log_unavailable(exc)
except Exception:
logger.warning("library.source_failed: %s", source.__name__, exc_info = True)
adopt: list[tuple[str, str]] = []
stale: list[tuple[str, str]] = []
for item in items:
entry = overlay.get(item["id"])
fingerprint = item.pop("_fingerprint", None)
if fingerprint is not None:
# Sent back with a delete, so a stale card never deletes a file made at its path since.
item["fingerprint"] = fingerprint
if entry and fingerprint is not None and entry["fingerprint"] != fingerprint:
if entry["fingerprint"] is None:
adopt.append((item["id"], fingerprint))
else:
stale.append((item["id"], entry["fingerprint"]))
entry = None
item["favorite"] = bool(entry and entry["favorite"])
item["folderId"] = entry["folderId"] if entry else None
item["openedAt"] = entry["openedAt"] if entry else None
if entry and entry["name"]:
item["name"] = entry["name"]
try:
if adopt or stale:
with _replace_lock:
# A listing begun before a replace_file swap may have seen its new inode before the carry: not stale.
if _LISTING.generation != generation:
stale = []
library_db.reconcile_entries(adopt, stale)
except Exception:
logger.warning("library.overlay_reconcile_failed", exc_info = True)
items.sort(key = lambda item: item["updatedAt"], reverse = True)
return items
def path_derived(item_id: str) -> bool:
"""Whether the id names a path, which another file can take later."""
return item_id.partition(":")[0] in ("sandbox", "model")
def fingerprint(item_id: str) -> Optional[str]:
"""The fingerprint of the file a path-derived id names now, for its overlay row; None for other
ids (never given to another file) and for a file that is gone."""
kind, _, ref = item_id.partition(":")
if kind not in ("sandbox", "model"):
return None
try:
path = _sandbox_path(ref) if kind == "sandbox" else local_path(item_id)
return _fingerprint(os.stat(path))
except (LookupError, ValueError, OSError):
return None
_replace_lock = threading.Lock()
@contextlib.contextmanager
def overlay_write():
"""Held across a fingerprint check and the overlay write it guards, so an edit_file swap cannot
land between them and leave the write keyed to the file it replaced."""
with _replace_lock:
yield
def replace_file(
tmp: str,
path: str,
dir_fd: "int | None" = None,
) -> None:
"""``os.replace(tmp, path)``, carrying the overlay row of the file it replaces over to the new
inode, which the listing would otherwise drop as a different file. With ``dir_fd``, ``tmp`` and
the basename of ``path`` are names in that open folder."""
leaf = path if dir_fd is None else os.path.basename(path)
with _replace_lock:
before = os.stat(leaf, dir_fd = dir_fd)
staged = os.stat(tmp, dir_fd = dir_fd)
os.replace(tmp, leaf, src_dir_fd = dir_fd, dst_dir_fd = dir_fd)
_carry_overlay(path, before, staged)
def _carry_overlay(path: str, before: os.stat_result, staged: os.stat_result) -> None:
old = _fingerprint(before)
try:
after = os.stat(path)
# A path made again since the swap is a different file, which starts fresh.
if not os.path.samestat(staged, after):
return
for item_id, entry in library_db.list_entries().items():
# NULL: a legacy row not yet adopted, which a listing racing this swap would adopt as the old inode.
if entry["fingerprint"] not in (old, None) or not item_id.startswith("sandbox:"):
continue
try:
listed = os.stat(_sandbox_path(item_id.partition(":")[2]))
except (LookupError, OSError):
continue
if os.path.samestat(listed, after):
library_db.carry_fingerprint(item_id, entry["fingerprint"], _fingerprint(after))
invalidate_listing()
except Exception:
logger.warning("library.overlay_carry_failed", exc_info = True)
def safe_file_name(
name: str,
fallback: str = "file",
item_id: Optional[str] = None,
) -> str:
"""Last path segment; no Windows-refused chars, leading/trailing dot or space, or CON-style device name. With item_id, hashed per item: re-adding is a no-op."""
stem, ext = os.path.splitext(re.split(r"[\\/]", name or "")[-1])
if not ext and stem.startswith("."):
stem, ext = os.path.splitext(stem.lstrip("."))
stem = _NAME_BREAK_RE.sub(" ", stem).strip(" .") or fallback
ext = _utf8_prefix(re.sub(f"[{UNSAFE_NAME_CHARS}]", "", ext).rstrip(" ."), 16)
ext = ext if len(ext) > 1 else ""
stem = _utf8_prefix(stem, 160 if item_id else 200).rstrip(" .") or fallback
if _bad_name(stem):
stem = f"_{stem}"
if item_id:
return f"{stem}-{hashlib.sha1(item_id.encode()).hexdigest()[:8]}{ext}"
return f"{stem}{ext}"
def _utf8_prefix(text: str, limit: int) -> str:
"""The longest start of ``text`` at most ``limit`` bytes long in UTF-8, never half a character."""
return text.encode("utf-8", "ignore")[:limit].decode("utf-8", "ignore")
class ItemFile:
"""An item's file, open. Every consumer reads this one descriptor, so what was checked is what
is read, and nothing reopens the name."""
def __init__(self, handle: BinaryIO, name: str, folder: str, project_name: str):
self.handle = handle
info = os.fstat(handle.fileno())
self.size = info.st_size
self.modified_ns = info.st_mtime_ns
self.name = name
self.folder = folder
self.project_name = project_name
def close(self) -> None:
self.handle.close()
def __enter__(self) -> "ItemFile":
return self
def __exit__(self, *_exc) -> None:
self.close()
def _gallery_file(kind: str, ref: str) -> tuple[Path, str]:
"""(path, prompt) of a gallery item Studio made, if its file has a readable recipe; else LookupError."""
gallery = _gallery(kind)
path = gallery.resolve(ref)
if path is not None:
meta = gallery.module._read_meta(
path if kind == "image" else gallery.module._sidecar_path(ref)
)
if path is None or meta is None:
raise LookupError(f"{kind}:{ref}")
return path, str(meta.get("prompt") or "")
def _open_owned(path: Path, item_id: str) -> BinaryIO:
try:
return open(path, "rb")
except (FileNotFoundError, IsADirectoryError, NotADirectoryError):
raise LookupError(item_id) from None
def open_item(item_id: str) -> ItemFile:
"""LookupError when gone; ValueError for items with no file of their own, such as chat attachments."""
kind, _, ref = item_id.partition(":")
if kind == "upload":
record = library_db.get_upload(ref)
path = upload_path(ref)
if record is None or path is None:
raise LookupError(item_id)
return ItemFile(
_open_owned(path, item_id),
safe_file_name(record["name"]),
"files",
safe_file_name(record["name"], item_id = item_id),
)
if kind in _GALLERIES:
path, prompt = _gallery_file(kind, ref)
name = safe_file_name(_prompt_name(prompt, ref, path.suffix.lstrip(".")), ref)
return ItemFile(_open_owned(path, item_id), name, _gallery(kind).folder, path.name)
if kind == "sandbox":
handle, path = _open_sandbox_file(ref)
name = os.path.basename(path)
return ItemFile(
handle, safe_file_name(name), "files", safe_file_name(name, item_id = item_id)
)
raise ValueError("This item cannot be added to a project.")
def project_item(item_id: str) -> ItemFile:
"""Like open_item, plus a chat attachment's image or clip, copied to a temp file; LookupError if
gone."""
kind, _, ref = item_id.partition(":")
if kind != "attachment":
return open_item(item_id)
from storage.studio_db import get_chat_attachment
mime_type, data = _attachment_media(ref)
message_id, attachment_id = _attachment_ref(ref)
record = get_chat_attachment(message_id, attachment_id) or {}
name = _named_for_type(str(record.get("name") or "Attachment"), mime_type)
handle = tempfile.TemporaryFile()
try:
handle.write(data)
handle.seek(0)
except BaseException:
handle.close()
raise
folder = "images" if mime_type.startswith("image/") else "videos"
return ItemFile(handle, safe_file_name(name), folder, safe_file_name(name, item_id = item_id))
def local_path(item_id: str) -> Path:
"""Same errors as open_item. Model paths come from the id, so they must sit inside the outputs or exports root."""
kind, _, ref = item_id.partition(":")
if kind == "upload":
path = upload_path(ref)
if library_db.get_upload(ref) is None or path is None or not path.is_file():
raise LookupError(item_id)
return path
if kind in _GALLERIES:
return _gallery_file(kind, ref)[0]
if kind == "sandbox":
path = Path(_sandbox_path(ref))
if not path.is_file():
raise LookupError(item_id)
return path
if kind != "model":
raise ValueError("This item cannot be added to a project.")
from utils.paths.storage_roots import exports_root, outputs_root
_origin, _, path = ref.partition(":")
resolved = os.path.realpath(path) if path else ""
roots = [os.path.realpath(root) for root in (outputs_root(), exports_root())]
if (
not resolved
or not any(is_path_within(resolved, root) for root in roots)
or not os.path.exists(resolved)
):
raise LookupError(item_id)
return Path(resolved)
_THUMBNAIL_WIDTH = 640
# The grid crops a picture past these heights (as a share of its width), so the rest is never sent.
_THUMBNAIL_MIN_RATIO = 2 / 3
_THUMBNAIL_MAX_RATIO = 3 / 2
_THUMBNAIL_MAX_PIXELS = 64_000_000
_THUMBNAIL_IMAGE_FORMATS = ("PNG", "JPEG", "WEBP", "GIF", "BMP", "TIFF", "AVIF")
_THUMBNAIL_VIDEO_CONTAINERS = {
"video/mp4": "mp4",
"video/quicktime": "mov",
"video/webm": "webm",
"video/x-matroska": "matroska",
"video/ogg": "ogg",
"video/x-msvideo": "avi",
}
_THUMBNAIL_DECODES = threading.BoundedSemaphore(2)
_THUMBNAILS = _Memo(size = 256)
def _attachment_media(ref: str) -> tuple[str, bytes]:
"""LookupError when it holds none. Images are stored as data URLs but clips as raw base64, so each needs its own decoding."""
import urllib.parse
from fastapi import HTTPException
from routes.chat_history import _decode_attachment_base64
from storage.studio_db import get_chat_attachment
message_id, attachment_id = _attachment_ref(ref)
attachment = get_chat_attachment(message_id, attachment_id) or {}
try:
for part in attachment.get("content") or []:
if not isinstance(part, dict):
continue
image = part.get("image")
if isinstance(image, str) and image[:5].lower() == "data:":
header, _, payload = image.partition(",")
mime_type = media_type(header[5:].split(";", 1)[0]) or ""
if not mime_type.startswith("image/"):
continue
if "base64" not in header.lower():
return mime_type, urllib.parse.unquote_to_bytes(payload)
return mime_type, _decode_attachment_base64(payload)
if part.get("type") != "file":
continue
data = part.get("data")
mime_type = media_type(part.get("mimeType") or attachment.get("contentType")) or ""
if isinstance(data, str) and data and mime_type.startswith(("image/", "video/")):
return mime_type, _decode_attachment_base64(data)
except HTTPException as exc:
raise RuntimeError("The attachment's data is corrupt.") from exc
raise LookupError(ref)
def _image_thumbnail(source: Union[Path, BinaryIO]) -> bytes:
"""The picture cropped as a card shows it and at most `_THUMBNAIL_WIDTH` wide, as WebP."""
try:
from PIL import Image, ImageOps
except Exception as exc: # noqa: BLE001 -- a missing dependency makes thumbnails unavailable
raise RuntimeError("Thumbnail generation needs the 'Pillow' package.") from exc
Image.init()
formats = [name for name in _THUMBNAIL_IMAGE_FORMATS if name in Image.OPEN]
try:
with Image.open(source, formats = formats) as opened:
if opened.width * opened.height > _THUMBNAIL_MAX_PIXELS:
raise RuntimeError(f"{opened.width}x{opened.height} is too large to thumbnail.")
opened.draft("RGB", (_THUMBNAIL_WIDTH, _THUMBNAIL_WIDTH))
image = ImageOps.exif_transpose(opened)
width, height = image.size
if height > width * _THUMBNAIL_MAX_RATIO:
image = image.crop((0, 0, width, round(width * _THUMBNAIL_MAX_RATIO)))
elif height < width * _THUMBNAIL_MIN_RATIO:
keep = round(height / _THUMBNAIL_MIN_RATIO)
left = (width - keep) // 2
image = image.crop((left, 0, left + keep, height))
image.thumbnail(
(_THUMBNAIL_WIDTH, round(_THUMBNAIL_WIDTH * _THUMBNAIL_MAX_RATIO)), Image.LANCZOS
)
if image.mode not in ("RGB", "RGBA"):
transparent = "A" in image.getbands() or "transparency" in image.info
image = image.convert("RGBA" if transparent else "RGB")
buf = io.BytesIO()
image.save(buf, format = "WEBP", quality = 85, method = 4)
return buf.getvalue()
except Exception as exc: # noqa: BLE001 -- surface decoder / WebP encoder failures
raise RuntimeError(f"Thumbnail generation failed to decode the image: {exc}") from exc
def _decode(mime_type: str, source: BinaryIO) -> bytes:
"""The thumbnail; the caller holds one of `_THUMBNAIL_DECODES`."""
from core.inference import video_gallery
if mime_type.startswith("video/") and mime_type in _THUMBNAIL_VIDEO_CONTAINERS:
return video_gallery.first_frame_webp(
source,
width = _THUMBNAIL_WIDTH,
container = _THUMBNAIL_VIDEO_CONTAINERS[mime_type],
max_pixels = _THUMBNAIL_MAX_PIXELS,
max_height = round(_THUMBNAIL_WIDTH * _THUMBNAIL_MAX_RATIO),
)
if mime_type.startswith("image/") and mime_type != "image/svg+xml":
return _image_thumbnail(source)
raise LookupError(mime_type)
def _decoded(mime_type: str, source: BinaryIO) -> bytes:
with _THUMBNAIL_DECODES:
return _decode(mime_type, source)
def thumbnail(item_id: str) -> bytes:
"""Remembered by file version, so an edit makes a new one. LookupError when gone or no picture; RuntimeError when it cannot be decoded."""
kind, _, ref = item_id.partition(":")
if kind == "attachment":
with _THUMBNAIL_DECODES:
mime_type, data = _attachment_media(ref)
version = (item_id, len(data), zlib.crc32(data))
return _THUMBNAILS.get(version, lambda: _decode(mime_type, io.BytesIO(data)))
mime_type = item_type(item_id)
if not mime_type.startswith(("image/", "video/")):
raise LookupError(item_id)
with open_item(item_id) as item:
version = (item_id, item.size, item.modified_ns)
return _THUMBNAILS.get(version, lambda: _decoded(mime_type, item.handle))
def item_exists(item_id: str, recorded: Optional[str] = None) -> bool:
"""Whether the source still has the item, without listing the source. Errors other than its
absence propagate, so a store that cannot be read is never taken for an empty one. With the
fingerprint an overlay row recorded, a new file at the item's path is not the item."""
kind, _, ref = item_id.partition(":")
try:
if kind == "attachment":
from storage.studio_db import get_chat_attachment
message_id, attachment_id = _attachment_ref(ref)
return get_chat_attachment(message_id, attachment_id) is not None
if kind in _GALLERIES:
return _gallery(kind).resolve(ref) is not None
path = local_path(item_id)
except (LookupError, ValueError):
return False
if recorded is None or kind not in ("sandbox", "model"):
return True
try:
return _fingerprint(os.stat(path)) == recorded
except OSError:
return False
_LOCATIONS = {
"uploads": "upload",
"images": "image",
"videos": "video",
"audio": "audio",
"fineTunes": "model:training",
"exports": "model:exported",
}
def _location_path(key: str) -> Path:
"""Where `key` lives, also while its chosen folder is unavailable."""
try:
return _SOURCE_ROOTS[_LOCATIONS[key]]()
except relocations.LocationUnavailable:
return relocations.chosen(key)
def locations() -> list[dict]:
"""available is false while a chosen drive is absent, and disk and device are then null; device tells folders on one disk apart."""
entries = []
for key in _LOCATIONS:
path = _location_path(key)
available = key not in relocations.MOVABLE or relocations.is_available(key)
device = _device(path) if available else None
entries.append(
{
"key": key,
"path": str(path),
"movable": key in relocations.MOVABLE,
"custom": relocations.chosen(key) is not None,
"available": available,
"disk": _disk(path) if available else None,
# A string: a device number can be wider than a JavaScript number is exact.
"device": None if device is None else str(device),
}
)
return entries
_OS_CLUTTER = frozenset({".DS_Store", "Thumbs.db", "desktop.ini"})
# Held for a whole move, so an upload lands in the folder the move leaves in use. Reentrant: the first read of the chosen folders can finish a move cut short.
_move_lock = threading.RLock()
def _in_progress(name: str) -> bool:
"""A save still being written (`.<id>.tmp`, renamed into place when whole), a delete setting its
file aside, or the write test. Never moved: the writer renames it by the path it started with."""
return name.startswith(".") and (
name.endswith((".tmp", ".deleting")) or name.startswith(".unsloth-write-test-")
)
def _identity(path) -> Optional[tuple[int, int]]:
try:
info = os.stat(path)
except (OSError, ValueError):
return None
# FAT and some shares give every entry file id 0; those compare by spelling instead.
return (info.st_dev, info.st_ino) if info.st_ino else None
def _loose(path: Path) -> str:
return os.path.normcase(str(path)).casefold()
def _inside(path: Path, folder: Path) -> bool:
"""Whether `path` (or its nearest existing parents) is `folder` or inside it, by what the disk
says: resolve() keeps the case it is given, so on a case-insensitive disk (macOS, exFAT, NTFS)
or through a bind mount two spellings name one folder."""
folder_id = _identity(folder)
if folder_id is None:
return any(_loose(candidate) == _loose(folder) for candidate in (path, *path.parents))
return any(_identity(candidate) == folder_id for candidate in (path, *path.parents))
def _same_folder(a: Path, b: Path) -> bool:
a_id, b_id = _identity(a), _identity(b)
if a_id is None and b_id is None:
return _loose(a) == _loose(b)
return a_id == b_id
def _location_default(key: str) -> Path:
return account_path("library") if key == "uploads" else storage_roots.studio_root() / key
def _scratch_and_system_folders() -> list[str]:
"""Folders Library files must not move to although the model download folder may be there:
temporary and cache folders the OS empties on its own, and system folders for programs."""
home = Path.home()
folders = [tempfile.gettempdir()]
system = platform.system()
if system == "Windows":
drive = os.environ.get("SystemDrive", "C:") + "\\"
folders += [
os.environ.get("TEMP", ""),
os.environ.get("TMP", ""),
os.path.join(os.environ.get("SystemRoot", drive + "Windows"), "Temp"),
os.path.join(os.environ.get("LOCALAPPDATA", str(home / "AppData" / "Local")), "Temp"),
os.environ.get("ProgramData", drive + "ProgramData"),
drive + "$Recycle.Bin",
]
else:
folders += ["/tmp", "/var/tmp", "/dev/shm", "/usr", "/bin", "/sbin", "/var", "/snap"]
folders += ["/lib", "/lib32", "/lib64", "/libx32", "/private/var", "/private/tmp"]
if system == "Darwin":
folders += ["/Applications", "/opt/homebrew"]
folders += [str(home / "Library" / "Caches"), str(home / ".Trash")]
elif str(home) != "/root":
folders.append("/root") # Another account's home, where this one cannot keep files.
return [folder for folder in folders if folder]
def _refuse_denied(resolved: Path) -> None:
from hub.storage.scan_folders import (
contains_sensitive_path_component,
is_denied_system_path,
is_within_any,
)
if is_denied_system_path(str(resolved)) or contains_sensitive_path_component(str(resolved)):
raise ValueError("System, credential and config folders cannot hold these files.")
if is_within_any(str(resolved), _scratch_and_system_folders()):
raise ValueError(
"Temporary, cache and program folders cannot hold these files: the system may empty them."
)
def _move_target(raw: str) -> Path:
"""An absolute, ordinary folder whose parent exists: the model download folder's rules, and no
temporary folders."""
value = raw.strip()
if not value:
raise ValueError("Choose a folder.")
candidate = Path(value).expanduser()
if not candidate.is_absolute():
raise ValueError("Choose an absolute folder path.")
try:
resolved = candidate.resolve(strict = False)
except (OSError, RuntimeError, ValueError) as exc:
raise ValueError("That folder path is invalid.") from exc
if resolved.parent == resolved:
raise ValueError("Choose a folder inside the drive, not the drive itself.")
_refuse_denied(resolved)
if not resolved.parent.is_dir():
raise ValueError("The parent folder does not exist.")
return resolved
_SUBFOLDERS = {
"uploads": "Unsloth Library",
"images": "Unsloth Images",
"videos": "Unsloth Videos",
"audio": "Unsloth Audio",
}
def _is_empty(folder: Path) -> bool:
return not any(entry.name not in _OS_CLUTTER for entry in folder.iterdir())
def _prepare_target(target: Path, key: str, current: Path) -> Path:
"""target itself when empty, else a named folder inside it, also at a drive's mount point so files stay together; reused if already current."""
try:
target.mkdir(exist_ok = True)
if not target.is_dir():
raise ValueError("That path is a file, not a folder.")
if os.path.ismount(target) or not _is_empty(target):
target = target / _SUBFOLDERS[key]
if _same_folder(target, current):
return target
# It can be a link out of the checked folder, so its destination is checked before use.
_refuse_denied(target.resolve(strict = False))
target.mkdir(exist_ok = True)
if not target.is_dir() or not _is_empty(target):
raise ValueError(f"{target} already holds files. Choose another folder.")
with tempfile.NamedTemporaryFile(prefix = ".unsloth-write-test-", dir = target):
pass
except PermissionError as exc:
raise ValueError("Unsloth cannot write to that folder.") from exc
except OSError as exc:
raise ValueError(f"Unsloth cannot use that folder: {exc.strerror or exc}") from exc
return target
def _refuse_overlap(target: Path, key: str, final: bool) -> None:
"""Refuses targets in another kind's folder, a chat sandbox (clearing deletes it), or the current folder; final must not hold them: moving into itself empties it."""
current = _location_path(key)
others = [_location_path(other) for other in _LOCATIONS if other != key]
others.append(_SOURCE_ROOTS["sandbox"]())
if any(_inside(target, root) for root in (*others, current)):
raise ValueError("That folder is inside another Unsloth folder.")
home = storage_roots.studio_root()
if _inside(target, home) and not _inside(target, _location_default(key)):
raise ValueError("That folder is inside Unsloth's own folder. Choose one outside it.")
if final and any(_inside(root, target) for root in (*others, current, home)):
raise ValueError("That folder holds Unsloth's own folders. Choose another folder.")
def _readable_size(size: int) -> str:
value = float(size)
for unit in ("B", "KB", "MB", "GB"):
if value < 1024:
return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}"
value /= 1024
return f"{value:.1f} TB"
def _refuse_short_space(current: Path, target: Path) -> None:
"""Cross-disk moves copy first, so refuse up front when the target cannot hold them all."""
if _device(current) == _device(target):
return
try:
need = _tree_stats(current)[0]
except OSError:
return
free = (_disk(target) or {}).get("freeBytes")
if free is not None and need + max(need // 50, 64 * 1024 * 1024) > free:
raise ValueError(
f"The files take {_readable_size(need)} and that drive has {_readable_size(free)} "
"free. Free up space there or choose another drive."
)
class _MoveLog:
"""What a move did, so a failure can undo it: each (source, dest, kept) that landed, where
`kept` means an identical file was already at `dest` and the original was dropped, and each
folder the move made."""
def __init__(self) -> None:
self.moved: list[tuple[Path, Path, bool]] = []
self.created: list[Path] = []
_rename = os.rename
_unlink = os.remove
def _discard(path: Path) -> None:
try:
path.unlink(missing_ok = True)
except OSError:
logger.error("library.move_cleanup_failed: %s", path, exc_info = True)
def _in_use_error(path: Path, exc: OSError) -> OSError:
# ERROR_SHARING_VIOLATION / ERROR_LOCK_VIOLATION; Windows reports most other holds as access denied.
if getattr(exc, "winerror", None) in (32, 33) or (
os.name == "nt" and isinstance(exc, PermissionError)
):
return OSError(exc.errno, f"{path} is in use by another program. Close it and try again.")
return OSError(exc.errno, f"{path} could not be moved: {exc.strerror or exc}")
def _free_name(dest: Path) -> Path:
"""`name (2).ext`, or the next number free, beside `dest`."""
for number in range(2, 10_000):
candidate = dest.with_name(f"{dest.stem} ({number}){dest.suffix}")
if not os.path.lexists(candidate):
return candidate
raise FileExistsError(errno.EEXIST, f"{dest} already exists")
def _renamed(entry: Path, dest: Path, log: _MoveLog) -> Optional[bool]:
"""True once `entry` is renamed to `dest`, None when it was deleted meanwhile, False when it
has to be copied instead (another drive)."""
try:
_rename(entry, dest)
except OSError as exc:
return None if isinstance(exc, FileNotFoundError) and not os.path.lexists(entry) else False
log.moved.append((entry, dest, False))
return True
def _move_file(entry: Path, dest: Path, log: _MoveLog) -> None:
"""Never overwrites or deletes a file it did not bring; clash keeps both, moved one renamed. Cross-drive: copy, remove original; if locked, copy goes instead."""
if entry.name == _FLAGS_STORE and os.path.lexists(dest):
# Copied over before the switch and written there since: the one left behind is older.
_unlink(entry)
return
kept = os.path.lexists(dest) and _same_bytes(entry, dest)
if os.path.lexists(dest) and not kept:
dest = _free_name(dest)
if not kept:
if _renamed(entry, dest, log) is not False:
return
# Stage under a hidden name and rename: a crash leaves no partial file under the real name.
staged = _staging_name(dest)
try:
shutil.copy2(entry, staged, follow_symlinks = False)
os.replace(staged, dest)
except BaseException as exc:
_discard(staged)
if isinstance(exc, FileNotFoundError) and not os.path.lexists(entry):
return
if isinstance(exc, PermissionError):
raise _in_use_error(entry, exc) from exc
raise
try:
_unlink(entry)
except FileNotFoundError:
pass
except OSError as exc:
if not kept:
_discard(dest)
raise _in_use_error(entry, exc) from exc
log.moved.append((entry, dest, kept))
_STAGING_SUFFIX = ".moving.tmp"
_FLAGS_STORE = ".flags.json"
def _staging_name(dest: Path) -> Path:
"""Where a copy across drives is written before it takes `dest`'s name. A dotted `.tmp`, so
a listing hides it and a move passes it by as a save in progress."""
return dest.with_name(f".{dest.name}{_STAGING_SUFFIX}")
def _clear_staging(folder: Path) -> None:
"""Drop copies a crash cut short, before the move they belonged to starts again."""
try:
leftovers = list(folder.rglob(f".*{_STAGING_SUFFIX}"))
except OSError:
return
for leftover in leftovers:
_discard(leftover)
def _same_bytes(a: Path, b: Path) -> bool:
try:
return a.is_file() and b.is_file() and filecmp.cmp(a, b, shallow = False)
except OSError:
return False
def _move_entry(entry: Path, dest: Path, log: _MoveLog) -> None:
"""Move `entry` to `dest`. A folder is renamed whole where it can be; across drives, or onto a
folder already there, it is merged in file by file, and the emptied original removed with
rmdir alone, which cannot take anything with it."""
if entry.name in _OS_CLUTTER or _in_progress(entry.name):
return
if not (entry.is_dir() and not entry.is_symlink()):
_move_file(entry, dest, log)
return
if not os.path.lexists(dest) and _renamed(entry, dest, log) is not False:
return
if os.path.lexists(dest) and not (dest.is_dir() and not dest.is_symlink()):
dest = _free_name(dest)
if not dest.is_dir():
dest.mkdir()
log.created.append(dest)
try:
children = list(entry.iterdir())
except FileNotFoundError:
return
for child in children:
_move_entry(child, dest / child.name, log)
try:
entry.rmdir()
except OSError:
pass # A save still writing in it, left for the next pass.
def _move_entries(
source: Path,
target: Path,
log: _MoveLog,
only = None,
) -> None:
"""Move everything in `source` into `target`, noting each move in `log` as it lands so a
failure part way can be undone. Never an entry that holds `target` itself."""
for entry in list(source.iterdir()):
if (only is None or only(entry)) and not _inside(target, entry):
_move_entry(entry, target / entry.name, log)
def _undo(log: _MoveLog) -> None:
"""Put back everything `log` moved, then the folders it made, as long as they are empty."""
for source, dest, kept in reversed(log.moved):
try:
source.parent.mkdir(parents = True, exist_ok = True)
if not kept:
_move_entry(dest, source, _MoveLog())
elif not os.path.lexists(source):
shutil.copy2(dest, source, follow_symlinks = False)
except OSError:
logger.error("library.move_rollback_failed: %s", dest, exc_info = True)
for folder in reversed(log.created):
try:
folder.rmdir()
except OSError:
pass # Something was saved into it meanwhile; the stray pass takes that back.
# Settle window for saves still writing to the old folder; stale leftovers do not count.
_SETTLE_SECONDS = 30.0
_WRITING_SECONDS = 10.0
def _writing_in(folder: Path) -> bool:
now = time.time()
try:
entries = list(folder.iterdir())
except OSError:
return False
for entry in entries:
if entry.name.endswith(".tmp") and _in_progress(entry.name):
try:
if now - entry.stat().st_mtime < _WRITING_SECONDS:
return True
except OSError:
continue
return False
def _settle(
source: Path,
target: Path,
wait: bool,
only = None,
) -> None:
"""Rescans source for late saves until a pass finds nothing new; with wait, also until nothing
is still being written; capped at _SETTLE_SECONDS."""
deadline = time.monotonic() + _SETTLE_SECONDS
while True:
log = _MoveLog()
try:
_move_entries(source, target, log, only)
except OSError:
logger.warning("library.move_straggler_failed: %s", source, exc_info = True)
return
writing = wait and _writing_in(source)
if not log.moved and not writing:
return
if time.monotonic() >= deadline:
if writing:
logger.warning("library.move_straggler_timeout: %s", source)
return
time.sleep(0.1)
def move_location(key: str, path: Optional[str]) -> Optional[str]:
"""Recorded before any file moves, so saves during the move land in the new folder. Owner only (caller checks); on failure the old folder stays in use."""
if key not in relocations.MOVABLE:
raise ValueError(
"These files stay where they are: training and chats remember them by path."
)
with _move_lock:
# Finish an interrupted move first, or a new one forgets files left in the old folder.
if relocations.moving_from(key) is not None:
_resume_move(key)
current = _location_path(key).resolve()
if not relocations.is_available(key):
if path is not None:
raise ValueError(
f"{current} is not available, so its files cannot move. Reconnect its drive, "
"or reset the folder."
)
relocations.set_chosen(key, None)
return str(current)
target = _move_target(path) if path is not None else _location_default(key).resolve()
if _same_folder(target, current):
return _settle_waiting_move(key, current)
_refuse_while_waiting(key)
_refuse_overlap(target, key, final = False)
_refuse_short_space(current, target)
# Re-resolve: the subfolder may be a link elsewhere or another kind's folder.
target = _prepare_target(target, key, current).resolve()
if _same_folder(target, current):
return _settle_waiting_move(key, current)
_refuse_overlap(target, key, final = True)
_refuse_short_space(current, target)
previous, previous_from = relocations.chosen(key), relocations.moving_from(key)
before = {entry.name for entry in target.iterdir()}
from core.inference import gallery_flags
# Copy the flag store first, under its lock, or Clear deletes the archive.
# Only where a store exists: the lock creates a lock file that would then move.
store = current / _FLAGS_STORE
with gallery_flags.exclusive(current) if store.is_file() else contextlib.nullcontext():
if store.is_file() and not os.path.lexists(target / _FLAGS_STORE):
shutil.copy2(store, target / _FLAGS_STORE)
# Recorded with the folder the files leave, so a crash part way is finished on restart.
relocations.set_chosen(key, target, moving_from = current)
log = _MoveLog()
try:
_move_entries(current, target, log)
except OSError as exc:
_undo(log)
relocations.set_chosen(key, previous, moving_from = previous_from)
_settle(target, current, wait = False, only = lambda entry: entry.name not in before)
raise RuntimeError(f"Could not move the files: {exc.strerror or exc}") from exc
_settle(current, target, wait = key != "uploads")
_finish_move(key, target)
return None
def _refuse_while_waiting(key: str) -> None:
"""ValueError while a move cut short still waits for a drive: files are on it, and a move from
here would leave them behind."""
if relocations.moving_from(key) is None:
return
missing = (
relocations.moving_from(key)
if relocations.chosen_available(key)
else relocations.chosen(key)
)
raise ValueError(
f"Some of these files are still in {missing}, which is not connected. Reconnect its "
"drive so their move can finish first."
)
def _settle_waiting_move(key: str, current: Path) -> Optional[str]:
"""Choosing the in-use folder while the chosen drive is missing means staying: the move is dropped."""
if relocations.moving_from(key) is None or relocations.chosen_available(key):
return None
waiting = relocations.chosen(key)
relocations.set_chosen(key, None if _same_folder(current, _location_default(key)) else current)
return str(waiting)
def _finish_move(key: str, target: Path) -> None:
relocations.set_chosen(key, None if _same_folder(target, _location_default(key)) else target)
def _resume_move(key: str) -> None:
"""Finishes a crash-interrupted move: leftovers merge into the chosen folder. Waits while either drive is absent; failures are logged and retried next start."""
with _move_lock:
source = relocations.moving_from(key)
if source is None:
return
target = relocations.chosen(key)
if not relocations.chosen_available(key):
# Move stays open until the drive returns; the old folder takes new files meanwhile.
logger.warning("library.move_resume_unavailable: %s", target)
relocations.wait_for_move(key)
return
if not relocations.moving_from_available(key):
# Move stays open until the source drive returns.
logger.warning("library.move_resume_source_unavailable: %s", source)
return
try:
_clear_staging(target)
if source.is_dir():
_move_entries(source, target, _MoveLog())
except OSError:
logger.warning("library.move_resume_failed: %s", source, exc_info = True)
return
_finish_move(key, target)
relocations.resume_move = _resume_move
def _delete_upload(upload_id: str, path: Path) -> bool:
"""Set the file aside before dropping its row, so a failure at either step leaves both."""
staged = path.with_name(f".{upload_id}.deleting")
with _upload_lock:
try:
os.replace(path, staged)
except FileNotFoundError:
return library_db.delete_upload(upload_id)
try:
deleted = library_db.delete_upload(upload_id)
except BaseException:
os.replace(staged, path)
raise
if deleted:
staged.unlink(missing_ok = True)
else:
os.replace(staged, path)
return deleted
class DeleteIncomplete(RuntimeError):
"""A delete that stopped part way; the item is still there to delete again."""
class ItemChanged(RuntimeError):
"""Another file now has the path the item was listed at; nothing was deleted."""
# Prevents an open from re-creating a row between a delete's item and row removal.
_overlay_lock = threading.Lock()
def mark_opened(item_id: str) -> bool:
"""Records an open against the file now at the item's path; False if the item is gone."""
with _overlay_lock, overlay_write():
found = fingerprint(item_id)
if found is None and (path_derived(item_id) or not item_exists(item_id)):
return False
library_db.mark_opened(item_id, fingerprint = found)
return True
def delete_item(item_id: str, fingerprint: Optional[str] = None) -> bool:
"""False if gone; a path-derived item needs its listed fingerprint to still match, else ItemChanged."""
with _overlay_lock:
deleted = _delete_item(item_id, fingerprint)
if deleted and item_id.startswith("attachment:"):
# May have held the last reference to an original. Swept outside the lock.
from core import chat_originals
chat_originals.sweep()
return deleted
def _delete_item(item_id: str, fingerprint: Optional[str]) -> bool:
kind, _, ref = item_id.partition(":")
deleted = False
if kind == "upload":
# Under the move lock, or a concurrent move leaves the file moved and its row dropped.
with _move_lock:
path = upload_path(ref)
if path is not None:
deleted = _delete_upload(ref, path)
elif kind == "attachment":
from storage.studio_db import delete_chat_attachment
message_id, attachment_id = _attachment_ref(ref)
deleted = delete_chat_attachment(message_id, attachment_id)
elif kind == "video":
from core.inference import video_gallery
from routes.video import _forget_openai_job, _forget_terminal_video
if video_gallery.get_record(ref) is not None:
# Forget the job first so /v1/videos keeps no ghost; a retry finishes a partial delete.
if not _forget_openai_job(ref):
raise DeleteIncomplete("Could not delete the video job; try again.")
deleted = video_gallery.delete(ref)
if deleted:
_forget_terminal_video(ref)
elif kind in _GALLERIES:
deleted = _gallery(kind).module.delete(ref)
elif kind == "sandbox":
deleted = _delete_sandbox_file(ref, fingerprint)
elif kind == "model":
_origin, _, path = ref.partition(":")
if not path or os.path.exists(path):
raise ValueError("Fine-tuned models are deleted from the model picker.")
deleted = True
else:
raise ValueError("Unknown library item")
invalidate_listing()
if not deleted and kind in _GALLERIES and item_exists(item_id):
raise DeleteIncomplete("Could not delete the file. Close any app using it and try again.")
library_db.delete_entry(item_id)
return deleted