Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

19.4 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Opt-in extras: LoRA EMA (0.99 default decay, exported as a second adapter), conditioning cache
(bit-identical to a fresh encode), aspect bucketing; pure helpers avoid torch at import time"""

from __future__ import annotations

import hashlib
import json
import math
import random
import re
from pathlib import Path
from typing import Any, Iterable, Optional


def accumulate_loss(acc: Any, loss: Any, accumulation_steps: int) -> Any:
    """``acc + float(loss) / accumulation_steps`` without a per-micro-step host sync: an fp64 device sum is the same
    IEEE arithmetic as the Python float one. Devices without fp64 (MPS, Arc / integrated XPUs) keep the Python float.
    Read with loss_value."""
    detached = loss.detach()
    if detached.device.type not in ("cpu", "cuda"):
        from core.inference.video_minimax_h3_pdd import device_has_fp64
        if not device_has_fp64(detached.device):
            return (acc if acc is not None else 0.0) + float(detached) / accumulation_steps
    term = detached.double() / accumulation_steps
    return term if acc is None else acc.add_(term)


def loss_value(acc: Any) -> float:
    return 0.0 if acc is None else float(acc)


# Effective decay = min(decay, (1 + n) / (WARMUP_OFFSET + n)).

_EMA_WARMUP_OFFSET = 10.0


class LoRAEMA:
    """Exponential moving average over a model's TRAINABLE parameters only.

    For a LoRA run the trainable set is just the adapter matrices, so the
    shadow costs megabytes, not the gigabytes a full-model EMA would. Shadows
    are stored keyed by parameter name (stable across re-wraps) on the same
    device/dtype as the source params, and updated in-place:

        shadow = decay * shadow + (1 - decay) * param
    """

    def __init__(
        self,
        model: Any,
        decay: float = 0.99,
        warmup: bool = True,
    ):
        if not 0.0 <= float(decay) < 1.0:
            raise ValueError(f"ema decay must be in [0, 1), got {decay}")
        self.decay = float(decay)
        self.warmup = bool(warmup)
        self.updates = 0
        self._shadow: dict[str, Any] = {}
        for name, p in model.named_parameters():
            if not p.requires_grad:
                continue
            shadow = p.detach().clone()
            shadow.requires_grad = False
            self._shadow[name] = shadow

    def reseed_from(self, model: Any) -> None:
        """For a resume that first enables EMA: shadows would otherwise hold freshly initialised weights."""
        import torch

        with torch.no_grad():
            for name, p in model.named_parameters():
                shadow = self._shadow.get(name)
                if shadow is None or tuple(shadow.shape) != tuple(p.shape):
                    continue
                shadow.copy_(p.detach().to(device = shadow.device, dtype = shadow.dtype))
        self.updates = 0

    def effective_decay(self) -> float:
        """The decay used for the NEXT update (after ``updates`` prior ones)."""
        if not self.warmup:
            return self.decay
        step = self.updates + 1
        return min(self.decay, step / (_EMA_WARMUP_OFFSET + step))

    def _update_lists(self, model: Any) -> tuple[list, list]:
        """(shadows, params) for ``model``, cached per model."""
        import weakref

        cached = getattr(self, "_lists", None)
        if cached is not None and cached[0]() is model:
            return cached[1], cached[2]
        shadows, params = [], []
        for name, p in model.named_parameters():
            shadow = self._shadow.get(name)
            if shadow is None:
                continue
            shadows.append(shadow)
            params.append(p.detach())
        try:
            self._lists = (weakref.ref(model), shadows, params)
        except TypeError:  # not weak-referenceable: rebuild each call
            self._lists = None
        return shadows, params

    def update(self, model: Any) -> None:
        """Apply one EMA update from ``model``'s current trainable params."""
        import torch

        decay = self.effective_decay()
        shadows, params = self._update_lists(model)
        if shadows:
            with torch.no_grad():
                torch._foreach_mul_(shadows, decay)
                torch._foreach_add_(shadows, params, alpha = 1.0 - decay)
        self.updates += 1

    def state_dict(self) -> dict[str, Any]:
        return {name: t.detach().clone() for name, t in self._shadow.items()}

    def missing_from(self, state: dict[str, Any]) -> tuple[str, ...]:
        """load_state_dict skips missing shadows silently, which would blend restored and fresh weights."""
        saved = state or {}
        missing = []
        for name, shadow in self._shadow.items():
            entry = saved.get(name)
            if entry is None or tuple(entry.shape) != tuple(shadow.shape):
                missing.append(name)
        return tuple(missing)

    def load_state_dict(
        self,
        state: dict[str, Any],
        updates: int = 0,
    ) -> None:
        """Pass updates to keep the warmup ramp position; otherwise the ramp restarts on resume."""
        import torch

        with torch.no_grad():
            for name, shadow in self._shadow.items():
                saved = (state or {}).get(name)
                if saved is None or tuple(saved.shape) != tuple(shadow.shape):
                    continue
                shadow.copy_(saved.to(device = shadow.device, dtype = shadow.dtype))
        self.updates = max(0, int(updates or 0))

    def copy_to(self, model: Any) -> dict[str, Any]:
        """Write shadow values into params, returning displaced originals for restore to undo."""
        import torch

        backup: dict[str, Any] = {}
        with torch.no_grad():
            for name, p in model.named_parameters():
                shadow = self._shadow.get(name)
                if shadow is None:
                    continue
                backup[name] = p.detach().clone()
                p.copy_(shadow)
        return backup

    def restore(self, model: Any, backup: dict[str, Any]) -> None:
        import torch
        with torch.no_grad():
            for name, p in model.named_parameters():
                if name in backup:
                    p.copy_(backup[name])

    def __len__(self) -> int:
        return len(self._shadow)


def save_ema_adapter(ema: "LoRAEMA", transformer: Any, spec_save: Any, out_dir: str) -> str:
    """Temporarily swaps EMA shadows into the live LoRA params for the diffusers export path, then restores the trained weights."""
    from peft.utils import get_peft_model_state_dict

    ema_dir = Path(out_dir) / "ema"
    ema_dir.mkdir(parents = True, exist_ok = True)
    backup = ema.copy_to(transformer)
    try:
        layers = get_peft_model_state_dict(transformer)
        spec_save(None, str(ema_dir), layers)
    finally:
        ema.restore(transformer, backup)
    return str(ema_dir)


_CACHE_VERSION = "1"
# v2 composites transparent images onto white before encoding.
_LATENT_CACHE_VERSION = "2"


def _file_content_hash(path: str) -> str:
    """return a truncated sha256; renames preserve cache entries while edits invalidate them."""
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()[:20]


def _text_content_hash(text: str) -> str:
    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:20]


def _sanitize(token: str) -> str:
    return re.sub(r"[^a-z0-9.]+", "-", str(token).lower()).strip("-")


def _hub_cache_roots() -> list[str]:
    """HF_HUB_CACHE is an import-time snapshot, so the active Unsloth cache root is searched first."""
    import os  # noqa: PLC0415 - keep the module import list light for the subprocess

    roots: list[str] = []
    try:
        from utils.hf_cache_settings import active_hf_hub_cache  # noqa: PLC0415
        active = str(active_hf_hub_cache() or "").strip()
        if active:
            roots.append(active)
    except Exception:  # noqa: BLE001 -- settings unavailable in the subprocess: fall back
        pass
    for candidate in (os.environ.get("HF_HUB_CACHE"), os.environ.get("HUGGINGFACE_HUB_CACHE")):
        if candidate and candidate.strip() and candidate.strip() not in roots:
            roots.append(candidate.strip())
    try:
        from huggingface_hub import constants  # noqa: PLC0415
        if constants.HF_HUB_CACHE and str(constants.HF_HUB_CACHE) not in roots:
            roots.append(str(constants.HF_HUB_CACHE))
    except Exception:  # noqa: BLE001 -- no hub package: whatever we collected above stands
        pass
    return roots


# In-place edits here must change the fingerprint; "connectors" is LTX-2's cached connector.
_CACHE_SOURCE_SUBDIRS = ("text_encoder", "tokenizer", "vae", "connectors")


def source_revision(ref: Any) -> str:
    """A repo id or path is not a version; the marker tracks content so stale cache is not reused; it must never load the encoders."""
    import os  # noqa: PLC0415 - keep the module import list light for the subprocess
    try:
        name = str(ref or "").strip()
        if not name:
            return "none"
        if os.path.isdir(name):
            parts: list[str] = []
            roots = [name]
            with os.scandir(name) as it:
                roots += [
                    e.path for e in it if e.is_dir() and e.name.startswith(_CACHE_SOURCE_SUBDIRS)
                ]
            for root in roots:
                with os.scandir(root) as it:
                    for e in it:
                        if not e.is_file():
                            continue
                        st = e.stat()
                        parts.append(
                            f"{os.path.relpath(e.path, name)}:{st.st_size}:{st.st_mtime_ns}"
                        )
            return f"dir-{hashlib.sha256('|'.join(sorted(parts)).encode()).hexdigest()[:16]}"
        if "/" in name:
            org, _, repo = name.partition("/")
            for root in _hub_cache_roots():
                base = os.path.join(root, f"models--{org}--{repo}")
                ref_file = os.path.join(base, "refs", "main")
                if os.path.isfile(ref_file):
                    with open(ref_file, encoding = "utf-8") as fh:
                        sha = fh.read().strip()
                    if sha:
                        return f"rev-{sha[:16]}"
                snaps = os.path.join(base, "snapshots")
                if os.path.isdir(snaps):
                    names = sorted(os.listdir(snaps))
                    if len(names) == 1:
                        return f"rev-{names[0][:16]}"
        return "unresolved"
    except Exception:  # noqa: BLE001 - best-effort, never block a run
        return "unresolved"


class PersistentConditioningCache:
    """One safetensors file per cached item under ``cache_dir``.

    Keys carry everything that changes the encoded tensors: the cache format
    version, family, resolution (or explicit bucket shape), the source content
    hash, and -- for latents -- the crop/flip variant. Tensors are stored fp32
    exactly as the trainer's in-memory path holds them, so a reload is
    bit-identical to a fresh encode."""

    def __init__(self, cache_dir: str, family: str, resolution: int):
        self.root = Path(cache_dir).expanduser()
        self.family = _sanitize(family)
        self.resolution = int(resolution)
        self.root.mkdir(parents = True, exist_ok = True)

    def latent_key(
        self,
        image_path: str,
        variant: tuple[float, float, bool],
        shape: Optional[tuple[int, int]] = None,
    ) -> str:
        u_left, u_top, flip = variant
        geom = f"{shape[0]}x{shape[1]}" if shape else str(self.resolution)
        var = f"{u_left:.6f}_{u_top:.6f}_{int(bool(flip))}"
        return (
            f"lat_v{_LATENT_CACHE_VERSION}_{self.family}_{geom}_"
            f"{_file_content_hash(image_path)}_{var}"
        )

    def text_key(self, caption: str) -> str:
        return f"txt_v{_CACHE_VERSION}_{self.family}_{_text_content_hash(caption)}"

    def path_for(self, key: str) -> Path:
        return self.root / f"{key}.safetensors"

    def has(self, key: str) -> bool:
        return self.path_for(key).is_file()

    def put(self, key: str, tensors: Iterable[Any]) -> None:
        """store an ordered tensor tuple with None slots recorded for exact reconstruction."""
        from safetensors.torch import save_file

        named: dict[str, Any] = {}
        none_slots: list[int] = []
        for i, t in enumerate(tensors):
            if t is None:
                none_slots.append(i)
            else:
                named[f"t{i}"] = t.detach().cpu().contiguous()
        meta = {
            "version": _CACHE_VERSION,
            "family": self.family,
            "none_slots": json.dumps(none_slots),
            "count": str(len(none_slots) + len(named)),
        }
        tmp = self.path_for(key).with_suffix(".tmp")
        save_file(named, str(tmp), metadata = meta)
        tmp.replace(self.path_for(key))

    def get(self, key: str) -> Optional[tuple]:
        """Load a tuple previously stored with ``put``; None if absent/corrupt."""
        path = self.path_for(key)
        if not path.is_file():
            return None
        try:
            from safetensors import safe_open
            with safe_open(str(path), framework = "pt", device = "cpu") as f:
                meta = f.metadata() or {}
                count = int(meta.get("count", "0"))
                none_slots = set(json.loads(meta.get("none_slots", "[]")))
                out: list[Any] = []
                for i in range(count):
                    out.append(None if i in none_slots else f.get_tensor(f"t{i}"))
            return tuple(out)
        except Exception:  # noqa: BLE001 -- a corrupt entry is re-encoded, never fatal
            return None


# 64: a multiple of every family's latent grid (16 DiTs, 32 LTX-2, SDXL U-Net) and few compiled shapes.

BUCKET_DIVISOR = 64

# Widest bucket aspect ratio; more extreme images clamp to it, as is common for panoramas.
MAX_BUCKET_RATIO = 2.0


def bucket_resolutions(
    resolution: int,
    divisor: int = BUCKET_DIVISOR,
    max_ratio: float = MAX_BUCKET_RATIO,
) -> list[tuple[int, int]]:
    """The exact legacy square plus kohya BucketManager buckets (tallest grid height within resolution**2)."""
    base = int(resolution)
    area = base * base
    out = {(base, base)}
    w = divisor
    while w * w <= area * max_ratio:
        h = (area // w) // divisor * divisor
        if h >= divisor and 1.0 / max_ratio <= w / h <= max_ratio:
            out.add((w, h))
            out.add((h, w))
        w += divisor
    return sorted(out)


def nearest_bucket(width: int, height: int, buckets: list[tuple[int, int]]) -> tuple[int, int]:
    """Closest aspect in log space; ties: larger area, then sorted order (deterministic)."""
    if width <= 0 or height <= 0:
        raise ValueError("image dimensions must be positive")
    r = math.log(width / height)
    return min(buckets, key = lambda b: (abs(math.log(b[0] / b[1]) - r), -b[0] * b[1], b))


def compute_bucket(
    width: int,
    height: int,
    base_resolution: int,
    divisor: int = BUCKET_DIVISOR,
    max_ratio: float = MAX_BUCKET_RATIO,
) -> tuple[int, int]:
    """The (bucket_w, bucket_h) for an image of ``width`` x ``height``."""
    return nearest_bucket(width, height, bucket_resolutions(base_resolution, divisor, max_ratio))


def assign_buckets(
    sizes: list[tuple[int, int]],
    base_resolution: int,
    divisor: int = BUCKET_DIVISOR,
    max_ratio: float = MAX_BUCKET_RATIO,
) -> dict[tuple[int, int], list[int]]:
    choices = bucket_resolutions(base_resolution, divisor, max_ratio)
    buckets: dict[tuple[int, int], list[int]] = {}
    for i, (w, h) in enumerate(sizes):
        buckets.setdefault(nearest_bucket(w, h, choices), []).append(i)
    return buckets


def cover_resize_dims(width: int, height: int, canvas_w: int, canvas_h: int) -> tuple[int, int]:
    """Smallest uniform scale covering the canvas (square canvas = legacy short-side resize)."""
    scale = max(canvas_w / width, canvas_h / height)
    return max(canvas_w, round(width * scale)), max(canvas_h, round(height * scale))


def oriented_image_size(path: str) -> tuple[int, int]:
    """(width, height) after EXIF orientation, from the header only (no pixel decode)."""
    from PIL import Image

    with Image.open(path) as im:
        w, h = im.size
        try:
            orientation = im.getexif().get(0x0112, 1)
        except Exception:  # noqa: BLE001 -- unreadable EXIF is treated as upright, like exif_transpose
            orientation = 1
    return (h, w) if orientation in (5, 6, 7, 8) else (w, h)


class BucketBatchSampler:
    """Batch indices so every batch comes from ONE bucket (one latent shape).

    Epoch-style like PermutationBatchSampler: every image once per cycle; a bucket's last partial
    batch wraps within itself so shapes stay fixed."""

    def __init__(self, buckets: dict[tuple[int, int], list[int]], rng: random.Random):
        if not buckets or not any(buckets.values()):
            raise ValueError("BucketBatchSampler needs at least one bucketed item")
        self._rng = rng
        self._shapes = sorted(s for s in buckets if buckets[s])
        self._items = {s: list(buckets[s]) for s in self._shapes}
        self._batches: list[tuple[tuple[int, int], list[int]]] = []
        self._pos = 0

    def _refill(self, k: int) -> None:
        batches: list[tuple[tuple[int, int], list[int]]] = []
        for shape in self._shapes:
            order = list(self._items[shape])
            self._rng.shuffle(order)
            count = -(-len(order) // k)
            stream = (order * (-(-count * k // len(order))))[: count * k]
            batches.extend((shape, stream[b * k : (b + 1) * k]) for b in range(count))
        self._rng.shuffle(batches)
        self._batches, self._pos = batches, 0

    def next_batch(self, k: int) -> tuple[tuple[int, int], list[int]]:
        if self._pos >= len(self._batches) or len(self._batches[self._pos][1]) != k:
            self._refill(k)
        shape, idxs = self._batches[self._pos]
        self._pos += 1
        return shape, list(idxs)

    def _bucket_key(self) -> list[list[Any]]:
        return [[s[0], s[1], list(self._items[s])] for s in self._shapes]

    def state_dict(self) -> dict[str, Any]:
        return {
            "kind": "bucket",
            "buckets": self._bucket_key(),
            "batches": [[s[0], s[1], list(i)] for s, i in self._batches],
            "pos": int(self._pos),
        }

    def load_state_dict(self, state: Optional[dict[str, Any]]) -> bool:
        """False (never a silent fresh cycle) unless the state matches this exact bucket assignment."""
        if not isinstance(state, dict) or state.get("kind") != "bucket":
            return False
        try:
            if [[int(w), int(h), [int(i) for i in idx]] for w, h, idx in state["buckets"]] != (
                self._bucket_key()
            ):
                return False
            batches = [((int(w), int(h)), [int(i) for i in idx]) for w, h, idx in state["batches"]]
            pos = int(state.get("pos") or 0)
        except (KeyError, TypeError, ValueError):
            return False
        for shape, idxs in batches:
            if shape not in self._items or not set(idxs) <= set(self._items[shape]):
                return False
        if not 0 <= pos <= len(batches):
            return False
        self._batches, self._pos = batches, pos
        return True