Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Pin/archive/order flags in .flags.json beside media, not the recipe; corrupt store means no flags"""
from __future__ import annotations
import contextlib
import json
import math
import os
import threading
from pathlib import Path
from typing import Any, Optional
from loggers import get_logger
logger = get_logger(__name__)
_SCHEMA_VERSION = 1
_STORE_NAME = ".flags.json"
# Marks a store written over an illegible one: absence of a flag is not proof. See _carry_taint.
_TAINT_KEY = "unreadable"
_lock = threading.RLock()
def _store_path(directory: Path) -> Path:
return directory / _STORE_NAME
def _empty() -> dict[str, Any]:
return {"version": _SCHEMA_VERSION, "items": {}}
class FlagsUnavailable(RuntimeError):
"""The store exists but could not be trusted (unparseable, wrong shape, unreadable).
Distinct from "no store yet", which legitimately means no flags. Callers that only order or
display flags ignore this and fall back to no flags; callers that DELETE on the strength of a
flag must fail closed instead, or a corrupt store silently reads every archived item as active.
"""
def _valid_entry(entry: Any) -> bool:
"""Non-bool archived or bad pinned_at means damage; null reads as active, which clear deletes."""
if not isinstance(entry, dict):
return False
if "archived" in entry and not isinstance(entry["archived"], bool):
return False
if "pinned_at" in entry and _pinned_at(entry) is None:
return False
if "order_at" in entry and _finite(entry.get("order_at")) is None:
return False
return True
def _sanitize_entry(entry: Any) -> Optional[dict[str, Any]]:
"""Damaged archived resolves to True, never dropped: clear deletes anything it reads as active."""
if not isinstance(entry, dict):
return {"archived": True}
clean = dict(entry)
if "archived" in clean and not isinstance(clean["archived"], bool):
clean["archived"] = True
if "pinned_at" in clean and _pinned_at(clean) is None:
clean.pop("pinned_at")
if "order_at" in clean and _finite(clean.get("order_at")) is None:
clean.pop("order_at")
return clean or None
def _load(directory: Path) -> tuple[dict[str, Any], bool]:
"""``(data, trusted)``. ``trusted`` is False when a store is present but unusable, so a caller
can tell "nothing is flagged" apart from "we cannot say what is flagged"."""
try:
with open(_store_path(directory), encoding = "utf-8-sig") as f:
data = json.load(f)
# Validate items shape, not just version: a hand-edited non-dict items would crash every lookup.
if (
isinstance(data, dict)
and data.get("version") == _SCHEMA_VERSION
and isinstance(data.get("items"), dict)
):
# Written over an illegible store, so what it does NOT say is not evidence.
if data.get(_TAINT_KEY):
return data, False
# One bad entry costs the store its trust (clear() could delete an archived file), but
# readable entries are still returned; only destructive callers refuse.
if all(_valid_entry(v) for v in data["items"].values()):
return data, True
logger.warning(
"gallery_flags.unreadable: %s has a malformed entry", _store_path(directory)
)
return data, False
logger.warning(
"gallery_flags.unreadable: %s has an unrecognised shape", _store_path(directory)
)
return _empty(), False
except FileNotFoundError:
return _empty(), True
except Exception as exc:
logger.warning("gallery_flags.read_failed: %s", exc)
return _empty(), False
def _carry_taint(data: dict[str, Any], trusted: bool) -> dict[str, Any]:
"""Marks an unreadable store; destructive callers fail closed until clear(include_archived=True) runs."""
if not trusted and not data.get("items"):
data[_TAINT_KEY] = True
return data
def _save(directory: Path, data: dict[str, Any]) -> None:
"""Atomic tmp-then-replace write; raises on failure, since a silent miss reports an unstored pin."""
path = _store_path(directory)
tmp = directory / f".{_STORE_NAME}.tmp-{os.getpid()}"
try:
with open(tmp, "w", encoding = "utf-8") as f:
json.dump(data, f, indent = 2)
os.replace(tmp, path)
except Exception as exc:
logger.warning("gallery_flags.write_failed: %s", exc)
try:
tmp.unlink(missing_ok = True)
except OSError:
pass
raise
@contextlib.contextmanager
def _file_lock(directory: Path):
"""Best-effort cross-process exclusive lock over one directory's store. Generation runs in
subprocesses, so the in-process RLock alone would let two of them clobber each other on
``os.replace``. Yields whether the OS lock was taken, so destructive callers can fail closed."""
try:
fd = os.open(str(directory / f"{_STORE_NAME}.lock"), os.O_CREAT | os.O_RDWR, 0o600)
except Exception:
yield False
return
locked = False
try:
try:
if os.name == "nt":
import msvcrt
msvcrt.locking(fd, msvcrt.LK_LOCK, 1)
else:
import fcntl
fcntl.flock(fd, fcntl.LOCK_EX)
locked = True
except Exception:
pass # locking unavailable; the thread lock still applies
yield locked
finally:
# never let the release fail the call: a filesystem that cannot lock usually cannot unlock
try:
if locked:
with contextlib.suppress(Exception):
if os.name == "nt":
import msvcrt
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(fd, fcntl.LOCK_UN)
finally:
os.close(fd)
@contextlib.contextmanager
def exclusive(directory: Path, *, require_file_lock: bool = False):
"""Holds the write lock across read-then-act, so an archive cannot slip past clear's snapshot."""
with _lock, _file_lock(directory) as file_locked:
if require_file_lock and not file_locked:
raise FlagsUnavailable(f"{_store_path(directory)} could not be locked")
yield
def _entry(items: dict[str, Any], item_id: str) -> dict[str, Any]:
"""One id's entry, normalized. A non-dict entry (hand-edited) reads as no flags."""
entry = items.get(item_id)
return entry if isinstance(entry, dict) else {}
def read(directory: Path) -> dict[str, dict[str, Any]]:
"""Untrusted stores read as no flags: a lost pin beats an unlisted gallery; deletes need
read_trusted."""
with _lock:
items = _load(directory)[0].get("items", {})
return {k: v for k, v in items.items() if isinstance(v, dict)}
def is_trusted(directory: Path) -> bool:
"""Whether the store can be believed about what is NOT flagged."""
with _lock:
return _load(directory)[1]
def reset_locked(directory: Path) -> None:
"""Writes an empty trusted store; call inside exclusive() only after every owned item is removed."""
_save(directory, _empty())
def read_trusted(directory: Path) -> dict[str, dict[str, Any]]:
"""``read``, but raises FlagsUnavailable instead of pretending nothing is flagged. For callers
that delete based on a flag, where guessing "not archived" destroys the archive."""
with _lock:
data, trusted = _load(directory)
if not trusted:
raise FlagsUnavailable(f"{_store_path(directory)} could not be read")
items = data.get("items", {})
return {k: v for k, v in items.items() if isinstance(v, dict)}
def _finite(value: Any) -> Optional[float]:
"""Huge ints, NaN and infinity read as unset, so listing never raises or poisons the sort."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
try:
number = float(value)
except (OverflowError, ValueError):
return None
return number if math.isfinite(number) else None
def _pinned_at(entry: dict[str, Any]) -> Optional[float]:
"""The entry's pin time as a usable float, or None when it is absent or unusable."""
return _finite(entry.get("pinned_at"))
def flags_for(items: dict[str, dict[str, Any]], item_id: str) -> dict[str, Any]:
"""The public record fields for one id, from an already-read ``items`` map."""
entry = _entry(items, item_id)
return {
# Use the sort's conversion for pinned, so a value the sort cannot use never shows as pinned.
"pinned": _pinned_at(entry) is not None,
"archived": bool(entry.get("archived")),
}
def pin_rank(items: dict[str, dict[str, Any]], item_id: str) -> float:
"""Sort key for the pinned group: most recently pinned first. Unpinned sorts last."""
pinned_at = _pinned_at(_entry(items, item_id))
return pinned_at if pinned_at is not None else float("-inf")
def order_at(items: dict[str, dict[str, Any]], item_id: str) -> Optional[float]:
"""The id's manual sort key, or None if it was never dragged."""
return _finite(_entry(items, item_id).get("order_at"))
def order_rank(items: dict[str, dict[str, Any]], item_id: str, mtime: float) -> float:
"""Unpinned sort key (newest first): ``order_at`` if dragged, else mtime."""
manual = order_at(items, item_id)
return manual if manual is not None else mtime
def is_archived(items: dict[str, dict[str, Any]], item_id: str) -> bool:
return bool(_entry(items, item_id).get("archived"))
def set_flags(
directory: Path,
item_id: str,
*,
pinned: Optional[bool] = None,
archived: Optional[bool] = None,
) -> dict[str, Any]:
"""Unpinning drops the key rather than storing False, and an id left at defaults is removed entirely."""
with _lock, _file_lock(directory):
return set_flags_locked(directory, item_id, pinned = pinned, archived = archived)
def _load_repaired(directory: Path) -> dict[str, Any]:
"""The store ready for a rewrite, with unreadable entries dropped."""
data = _carry_taint(*_load(directory))
items: dict[str, Any] = {}
for key, value in data.get("items", {}).items():
clean = _sanitize_entry(value)
if clean is not None:
items[key] = clean
data["items"] = items
return data
def set_flags_locked(
directory: Path,
item_id: str,
*,
pinned: Optional[bool] = None,
archived: Optional[bool] = None,
) -> dict[str, Any]:
"""``set_flags`` for a caller already inside ``exclusive()``, so the ownership check and the
write land as one step. Separate for the same per-descriptor lock reason as ``forget_locked``."""
import time
# A write repairs the store: drop only unreadable entries so later clear() calls are not refused.
data = _load_repaired(directory)
items = data["items"]
entry = dict(_entry(items, item_id))
if pinned is not None:
if pinned:
# Strictly ahead of every stored stamp: Windows time.time() advances in ~16 ms steps.
latest = max(
(_pinned_at(v) for v in items.values() if _pinned_at(v) is not None),
default = float("-inf"),
)
now = time.time()
nudged = math.nextafter(latest, math.inf) if latest != float("-inf") else now
# Do not nudge past max float (json writes inf, _pinned_at refuses it); tie and fall back to mtime.
entry["pinned_at"] = (
now if now > latest else (nudged if math.isfinite(nudged) else latest)
)
else:
entry.pop("pinned_at", None)
if archived is not None:
if archived:
entry["archived"] = True
else:
entry.pop("archived", None)
if entry:
items[item_id] = entry
else:
items.pop(item_id, None)
_save(directory, data)
return {"pinned": entry.get("pinned_at") is not None, "archived": bool(entry.get("archived"))}
def forget(directory: Path, item_ids) -> None:
"""Drop flags for ids that no longer exist, so a deleted image cannot hand its pin to a
future id and the store cannot grow without bound. No-op when nothing is stored."""
with _lock, _file_lock(directory):
forget_locked(directory, item_ids)
def forget_locked(directory: Path, item_ids) -> None:
"""``forget`` for a caller already inside ``exclusive()``. Separate because the cross-process
lock is per file descriptor: re-taking it on a second descriptor in the same process blocks
against the one already held, so the nested call would deadlock rather than recurse."""
ids = {i for i in item_ids if i}
if not ids:
return
data = _load(directory)[0]
items = data.get("items", {})
if not any(i in items for i in ids):
return
for item_id in ids:
items.pop(item_id, None)
try:
_save(directory, data)
except Exception as exc: # noqa: BLE001 -- the media is already gone; a stale row is harmless
logger.warning("gallery_flags.prune_failed: %s", exc)
def _between(high: Optional[float], low: Optional[float], *, top: float) -> Optional[float]:
"""Key between high and low (descending), or None if both are None; top is used at the group
head so newer media sorts first."""
if high is not None and low is not None:
mid = (high + low) / 2
# Out of float precision: tie with the upper neighbour.
return mid if low < mid < high else high
if high is not None:
return high - 1.0
if low is not None:
return top if top > low else math.nextafter(low, math.inf)
return None
def place_locked(
directory: Path, item_id: str, ordered: list[tuple[str, float]], *, after_id: Optional[str]
) -> dict[str, Any]:
"""Rewrites only the moved item; call inside exclusive(); KeyError if after_id is not on the shelf."""
import time
data = _load_repaired(directory)
items = data["items"]
ids = [(i, m) for i, m in ordered if i != item_id]
position = 0
if after_id is not None:
position = next((n + 1 for n, (i, _) in enumerate(ids) if i == after_id), -1)
if position < 0:
raise KeyError(after_id)
above = ids[position - 1] if position > 0 else None
below = ids[position] if position < len(ids) else None
def is_pinned(pair):
return _pinned_at(_entry(items, pair[0])) is not None
entry = dict(_entry(items, item_id))
was_pinned = _pinned_at(entry) is not None
if above is not None and below is not None:
pinned = is_pinned(above) if is_pinned(above) == is_pinned(below) else was_pinned
elif above is not None or below is not None:
pinned = is_pinned(above or below)
else:
pinned = was_pinned
now = time.time()
if pinned:
high = _pinned_at(_entry(items, above[0])) if above and is_pinned(above) else None
low = _pinned_at(_entry(items, below[0])) if below and is_pinned(below) else None
entry["pinned_at"] = _between(high, low, top = now) if (high, low) != (None, None) else now
else:
entry.pop("pinned_at", None)
high = order_rank(items, above[0], above[1]) if above and not is_pinned(above) else None
low = order_rank(items, below[0], below[1]) if below and not is_pinned(below) else None
key = _between(high, low, top = now)
if key is None:
entry.pop("order_at", None)
else:
entry["order_at"] = key
if entry:
items[item_id] = entry
else:
items.pop(item_id, None)
_save(directory, data)
return {"pinned": entry.get("pinned_at") is not None, "archived": bool(entry.get("archived"))}