Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

19.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""SAM 3 pre-quantized FP8/INT8/ConvRot detector weights sharing diffusion quant code; quantized
kernels CUDA only (CPU/MPS dequantize a lone quantized file); other exclusions refused at load"""

from __future__ import annotations

import os
import time
from contextlib import contextmanager
from typing import Any, Iterator, Optional

from loggers import get_logger

logger = get_logger(__name__)

# ── precision choice ───────────────────────────────────────────────────────────────────────────────────────────────
PRECISION_AUTO = "auto"
PRECISION_BF16 = "bf16"
PRECISION_FP8 = "fp8"
PRECISION_INT8 = "int8"
PRECISIONS = (PRECISION_AUTO, PRECISION_BF16, PRECISION_FP8, PRECISION_INT8)
QUANT_PRECISIONS = (PRECISION_FP8, PRECISION_INT8)
ENV_PRECISION = "UNSLOTH_SAM3_PRECISION"
# int8 loads the ConvRot artifact unless this is 0 (then the plain INT8 one).
ENV_INT8_CONVROT = "UNSLOTH_SAM3_INT8_CONVROT"

# What ``auto`` may pick on a CUDA card short of memory, best first (needs GATE_PASSED); empty: always bf16. With room
# auto runs bf16: at this size the 8-bit files save ~0.5 GB but are ~2x slower per Find (activation quant cost).
AUTO_ORDER: tuple[str, ...] = (PRECISION_INT8,)

# Minimum compute capability per scheme: fp8 scaled_mm needs Ada / Hopper (sm_89+), int8 _int_mm Ampere (sm_80+).
_MIN_CAPABILITY = {PRECISION_FP8: (8, 9), PRECISION_INT8: (8, 0)}

# ── what is quantized ──────────────────────────────────────────────────────────────────────────────────────────────
SEGMENTER_QUANT_FAMILY = "sam3"
# Keeps SAM 3's 256-wide DETR and mask Linears in bf16; only the ViT and CLIP text encoder quantize.
SEGMENTER_MIN_FEATURES = 512
SEGMENTER_CONVROT_GROUP = 256


def _first_vision_blocks(count: int) -> tuple[str, ...]:
    return tuple(f"backbone.layers.{index}." for index in range(count))


def artifact_tag(scheme: str, rotated: bool = False) -> str:
    """ "fp8", "int8" or "int8-convrot": one per published file."""
    return scheme + ("-convrot" if rotated else "")


# Early ViT blocks kept bf16 per file: smallest set from the sensitivity scan that brings it inside
# the gate tolerance (for fp8, closest to it).
SEGMENTER_EXCLUDE_TOKENS: dict[str, tuple[str, ...]] = {
    "fp8": _first_vision_blocks(8),
    "int8": _first_vision_blocks(4),
    "int8-convrot": _first_vision_blocks(8),
}
# Each file's verdict on the accuracy gate (unsloth/sam3.1 model card, Checks: the built file through the segment
# path, 53 prompts, every metric within 2x of what bf16 itself gives up against the fp32 weights, no kept-count,
# false-positive or absent-concept change). ``auto`` loads only a file that passed; an explicit precision loads any.
GATE_PASSED: dict[str, bool] = {"fp8": False, "int8": True, "int8-convrot": True}


def exclude_tokens(scheme: str, rotated: bool = False) -> tuple[str, ...]:
    return SEGMENTER_EXCLUDE_TOKENS.get(artifact_tag(scheme, rotated), ())


def auto_may_load(tag: str) -> bool:
    return bool(GATE_PASSED.get(tag))


def segmenter_filter_fn(
    scheme: str,
    *,
    exclude: Optional[tuple[str, ...]] = None,
    select: Any = None,
    min_features: int = SEGMENTER_MIN_FEATURES,
) -> Any:
    """The torchao ``quantize_`` filter for ``scheme``: the diffusion runtime's filter with SAM's exclusions.
    ``select(fqn)`` narrows it further (the sensitivity scan quantizes one group at a time)."""
    from .diffusion_transformer_quant import (
        _REQUIRE_BF16_SCHEMES,
        divisible_for_scheme,
        make_filter_fn,
    )

    base = make_filter_fn(
        min_features,
        exclude_name_tokens = exclude_tokens(scheme)
        if exclude is None
        else tuple(exclude),  # plain-file set
        require_bf16 = scheme in _REQUIRE_BF16_SCHEMES,
        require_divisible = divisible_for_scheme(scheme),
    )
    if select is None:
        return base
    return lambda module, fqn = "": bool(select(fqn)) and base(module, fqn)


def quantize_segmenter_(
    model: Any,
    scheme: str,
    *,
    convrot_group: int = 0,
    exclude: Optional[tuple[str, ...]] = None,
    select: Any = None,
    min_features: int = SEGMENTER_MIN_FEATURES,
) -> dict:
    """Quantizes in place; with convrot_group, rotates each selected Linear whose input width the
    group divides first; select and min_features are for the sensitivity scan only; artifacts use
    the defaults the loader checks."""
    from torchao.quantization import quantize_

    from .diffusion_transformer_quant import _REQUIRE_BF16_SCHEMES, _make_quant_config
    from .diffusion_transformer_quant import divisible_for_scheme

    if scheme not in QUANT_PRECISIONS:
        raise ValueError(f"unknown segmenter quant scheme {scheme!r}")
    if convrot_group and scheme != PRECISION_INT8:
        raise ValueError("ConvRot is an int8 rotation")
    if exclude is None:
        exclude = exclude_tokens(scheme, bool(convrot_group))
    filter_fn = segmenter_filter_fn(
        scheme, exclude = exclude, select = select, min_features = min_features
    )
    meta: dict = {
        "scheme": scheme,
        "min_features": int(min_features),
        "exclude_name_tokens": list(exclude),
        "require_bf16": scheme in _REQUIRE_BF16_SCHEMES,
        "require_divisible": divisible_for_scheme(scheme),
    }
    if convrot_group:
        from .diffusion_convrot import rotatable_fqns, rotate_linears_, rotation_metadata

        rotatable, plain = rotatable_fqns(model, filter_fn, int(convrot_group))
        if not rotatable:
            raise ValueError(f"ConvRot group {convrot_group} divides no quantized Linear")
        rotate_linears_(model, rotatable, int(convrot_group))
        meta.update(rotation_metadata(int(convrot_group), rotatable))
        meta["convrot_plain_linears"] = len(plain)
    quantize_(model, _make_quant_config(scheme), filter_fn = filter_fn)
    meta["quantized_linears"] = sum(
        1 for _n, m in model.named_modules() if _is_quantized_weight(getattr(m, "weight", None))
    )
    return meta


def _is_quantized_weight(weight: Any) -> bool:
    if weight is None:
        return False
    data = getattr(weight, "data", weight)
    import torch

    return type(data) not in (torch.Tensor, torch.nn.Parameter)


# ── artifacts ──────────────────────────────────────────────────────────────────────────────────────────────────────
def write_prequant_file(
    path: str,
    model: Any,
    quant_meta: dict,
    *,
    base_model_id: str,
    source_revision: Optional[str] = None,
) -> dict:
    """Saves a quantized Sam3Model as a prequant artifact; v2 when rotated, with canonical fingerprint."""
    import safetensors
    import torch
    import torchao
    import transformers

    from .diffusion_prequant import canonical_weight_fingerprint, prequant_format_for
    from .diffusion_transformer_quant import FP8_GRANULARITY, _resolve_fast_accum
    from .prequant_safetensors import save_prequant_safetensors

    state_dict = {
        k: (v.detach().to("cpu") if hasattr(v, "detach") else v)
        for k, v in model.state_dict().items()
    }
    scheme = quant_meta["scheme"]
    metadata = {
        "base_model_id": base_model_id,
        "source_revision": source_revision,
        "family": SEGMENTER_QUANT_FAMILY,
        "model_class": type(model).__name__,
        "component": "detector",
        "torch_dtype": "bfloat16",
        "quant_backend": "torchao",
        "fast_accum": _resolve_fast_accum(None) if scheme == PRECISION_FP8 else None,
        "torch_version": torch.__version__,
        "torchao_version": getattr(torchao, "__version__", "?"),
        "transformers_version": transformers.__version__,
        "safetensors_version": safetensors.__version__,
        **quant_meta,
    }
    if scheme == PRECISION_FP8:
        metadata["fp8_granularity"] = FP8_GRANULARITY
    metadata["fingerprint"] = canonical_weight_fingerprint(state_dict)
    fmt = prequant_format_for(metadata)
    save_prequant_safetensors(str(path), fmt = fmt, state_dict = state_dict, metadata = metadata)
    return {"format": fmt, **metadata}


def artifact_filename(
    prefix: str,
    scheme: str,
    *,
    rotated: bool = False,
) -> str:
    """``<prefix>-FP8.safetensors`` / ``-INT8.safetensors`` / ``-INT8-ConvRot.safetensors`` (the diffusion prequant
    naming)."""
    tag = scheme.upper() + ("-ConvRot" if rotated else "")
    return f"{prefix}-{tag}.safetensors"


def int8_convrot_enabled() -> bool:
    return (os.environ.get(ENV_INT8_CONVROT) or "").strip().lower() not in (
        "0",
        "off",
        "false",
        "no",
    )


def checkpoint_problem(
    fmt: Any,
    meta: Any,
    *,
    scheme: str,
    rotated: bool,
    base_model_id: Optional[str] = None,
    convrot_group: int = SEGMENTER_CONVROT_GROUP,
) -> Optional[str]:
    """Why a segmenter prequant header must not be loaded, or None. Torch free, header only. A rotated file must use
    ``convrot_group``: the accuracy gate and the exclusions were measured at that group, not at any power of four."""
    from .diffusion_convrot import ROTATION_GROUP_KEY, declares_rotation, rotation_metadata_error
    from .diffusion_prequant import PREQUANT_FORMAT, PREQUANT_FORMAT_ROTATED

    if not isinstance(meta, dict):
        return "no metadata"
    if meta.get("family") != SEGMENTER_QUANT_FAMILY:
        return f"family {meta.get('family')!r} is not {SEGMENTER_QUANT_FAMILY!r}"
    if meta.get("scheme") != scheme:
        return f"scheme {meta.get('scheme')!r}, expected {scheme!r}"
    if base_model_id and str(meta.get("base_model_id") or "").lower() != base_model_id.lower():
        return f"built from {meta.get('base_model_id')!r}, not {base_model_id!r}"
    if meta.get("min_features") != SEGMENTER_MIN_FEATURES:
        return f"min_features {meta.get('min_features')!r} != {SEGMENTER_MIN_FEATURES}"
    declared = declares_rotation(meta)
    if declared != (fmt == PREQUANT_FORMAT_ROTATED):
        return f"format {fmt!r} and the declared rotation ({declared}) disagree"
    if fmt not in (PREQUANT_FORMAT, PREQUANT_FORMAT_ROTATED):
        return f"unknown format {fmt!r}"
    if declared != rotated:
        return "rotation expected" if rotated else "unexpected rotation"
    if rotated and meta.get(ROTATION_GROUP_KEY) != convrot_group:
        return f"ConvRot group {meta.get(ROTATION_GROUP_KEY)!r}, expected {convrot_group}"
    if sorted(meta.get("exclude_name_tokens") or ()) != sorted(exclude_tokens(scheme, rotated)):
        return (
            f"excludes {meta.get('exclude_name_tokens')!r} differ from this build's "
            f"{list(exclude_tokens(scheme, rotated))!r}"
        )
    return rotation_metadata_error(meta)


@contextmanager
def _default_dtype(dtype: Any) -> Iterator[None]:
    import torch

    previous = torch.get_default_dtype()
    torch.set_default_dtype(dtype)
    try:
        yield
    finally:
        torch.set_default_dtype(previous)


def _build_skeleton(model_dir: str) -> Any:
    """Sam3Model with meta parameters (the file supplies every one) and real buffers. The buffers are built in
    float32, which is what ``from_pretrained(dtype = bfloat16)`` leaves them in: built under a bf16 default the
    global-attention RoPE tables of the ViT come out up to 0.04 off and every mask moves."""
    import torch
    from accelerate import init_empty_weights
    from transformers import Sam3Config, Sam3Model

    config = Sam3Config.from_pretrained(model_dir)
    with _default_dtype(torch.float32), init_empty_weights(include_buffers = False):
        return Sam3Model(config)


def _dequantize_state_dict(state_dict: dict, dtype: Any) -> dict:
    """Every quantized weight as a dense ``dtype`` tensor (rotated weights stay rotated)."""
    out = {}
    for key, value in state_dict.items():
        if _is_quantized_weight(value):
            value = value.dequantize().to(dtype)
        elif getattr(value, "is_floating_point", lambda: False)():
            value = value.to(dtype)
        out[key] = value
    return out


def load_prequantized_segmenter(
    model_dir: str,
    weights_path: str,
    *,
    scheme: str,
    rotated: bool,
    device: str,
    dtype: Any,
    base_model_id: Optional[str] = None,
    dequantize: bool = False,
    convrot_group: int = SEGMENTER_CONVROT_GROUP,
) -> tuple[Any, dict]:
    """Loads a prequant artifact; on CUDA the weights stay quantized unless dequantize, and a bad
    file raises so the caller uses bf16."""
    from .diffusion_convrot import apply_activation_rotation
    from .diffusion_prequant import _pin_kernel_preference, _verify_packed_fingerprint
    from .prequant_safetensors import load_prequant_safetensors

    started = time.perf_counter()
    quantized = device.startswith("cuda") and not dequantize
    ckpt = load_prequant_safetensors(weights_path, device = device if quantized else "cpu")
    fmt, meta, state_dict = ckpt["format"], ckpt["metadata"], ckpt["state_dict"]
    problem = checkpoint_problem(
        fmt,
        meta,
        scheme = scheme,
        rotated = rotated,
        base_model_id = base_model_id,
        convrot_group = convrot_group,
    )
    if problem:
        raise ValueError(f"segmenter prequant refused: {problem}")
    if not _verify_packed_fingerprint(state_dict, meta, logger = logger, path = weights_path):
        raise ValueError("segmenter prequant fingerprint mismatch")
    if quantized:
        _pin_kernel_preference(state_dict)
    else:
        state_dict = _dequantize_state_dict(state_dict, dtype)
    model = _build_skeleton(model_dir)
    model.load_state_dict(state_dict, strict = True, assign = True)
    rotated_fqns = apply_activation_rotation(model, meta, logger = logger)
    model.to(device)
    model.eval()
    stray = [n for n, p in model.named_parameters() if p.device.type == "meta"]
    stray += [n for n, b in model.named_buffers() if b.device.type == "meta"]
    if stray:
        raise ValueError(f"segmenter prequant left {len(stray)} tensors unset (e.g. {stray[0]})")
    info = {
        "precision": scheme if quantized else "bf16",
        "scheme": scheme,
        "rotated": bool(rotated_fqns),
        "rotated_linears": len(rotated_fqns),
        "quantized_linears": sum(
            1 for _n, m in model.named_modules() if _is_quantized_weight(getattr(m, "weight", None))
        ),
        "kernels": "quantized" if quantized else "dequantized",
        "file": os.path.basename(weights_path),
        "load_s": round(time.perf_counter() - started, 2),
    }
    logger.info(
        "images.segment: pre-quantized %s file engaged (%s weights, %d quantized linears, %d rotated, on %s): %s",
        scheme,
        info["kernels"],
        info["quantized_linears"],
        info["rotated_linears"],
        device,
        info["file"],
    )
    return model, info


# ── which precision runs ───────────────────────────────────────────────────────────────────────────────────────────
def requested_precision() -> str:
    """UNSLOTH_SAM3_PRECISION (auto / bf16 / fp8 / int8; "bfloat16" reads as bf16). Unknown values warn, read auto."""
    raw = (os.environ.get(ENV_PRECISION) or "").strip().lower()
    if raw in ("", PRECISION_AUTO):
        return PRECISION_AUTO
    if raw in ("bfloat16", "bf16", "dense", "none", "off"):
        return PRECISION_BF16
    if raw in QUANT_PRECISIONS:
        return raw
    logger.warning(
        "images.segment: unknown %s=%r (choices: %s), using auto",
        ENV_PRECISION,
        raw,
        ", ".join(PRECISIONS),
    )
    return PRECISION_AUTO


def _cuda_capability(device: str) -> Optional[tuple[int, int]]:
    """The card's compute capability; reads properties only, so no CUDA context is created."""
    try:
        import torch

        if not torch.cuda.is_available():
            return None
        ordinal = int(device.split(":", 1)[1]) if ":" in device else None
        major, minor = torch.cuda.get_device_capability(ordinal)
        return int(major), int(minor)
    except Exception:  # noqa: BLE001
        return None


def unsupported_reason(
    precision: str,
    device: str,
    dtype: Any = None,
) -> Optional[str]:
    """Why ``precision`` cannot run quantized on ``device``, or None."""
    if not device.startswith("cuda"):
        return "quantized kernels run on CUDA only"
    from core._torchao_stub import is_stubbed, torch_is_rocm

    # ROCm reports gfx versions through the CUDA capability API and these dynamic INT8 / FP8 paths are NVIDIA-only
    # (diffusion_transformer_quant.dense_transformer_supported refuses ROCm for the same reason).
    if torch_is_rocm():
        return "quantized kernels run on NVIDIA CUDA only (ROCm build)"
    if is_stubbed("torchao"):
        return "torchao is a stub on this install"
    if dtype is not None:
        import torch
        if dtype != torch.bfloat16:
            return "the card has no bf16"
    capability = _cuda_capability(device)
    need = _MIN_CAPABILITY[precision]
    if capability is None or capability < need:
        return f"{precision} needs compute capability {need[0]}.{need[1]}+"
    from .diffusion_transformer_quant import torchao_unavailable_reason
    from .prequant_safetensors import safetensors_prequant_supported

    why = torchao_unavailable_reason()
    if why:
        return why
    if not safetensors_prequant_supported():
        return "this torchao cannot read safetensors pre-quant files"
    return None


def resolve_precision(
    device: str,
    dtype: Any = None,
    *,
    available: tuple[str, ...] = QUANT_PRECISIONS,
    memory_tight: bool = False,
) -> tuple[str, str]:
    """``(precision, why)`` for a run on ``device``. ``available`` is the quantized arms the variant ships (and, at
    load, has on disk). bf16 when asked for, when nothing quantized can run here, when auto has no gated arm, or for
    auto on CUDA unless ``memory_tight``. An explicit fp8 / int8 is honoured either way."""
    requested = requested_precision()
    if requested == PRECISION_BF16:
        return PRECISION_BF16, f"{ENV_PRECISION}=bf16"
    if requested == PRECISION_AUTO and not memory_tight and device.startswith("cuda"):
        return PRECISION_BF16, "auto: bf16 fits and is faster"
    candidates = AUTO_ORDER if requested == PRECISION_AUTO else (requested,)
    reasons = []
    for precision in candidates:
        if precision not in available:
            reasons.append(f"{precision}: not available")
            continue
        why = unsupported_reason(precision, device, dtype)
        if why is None:
            return (
                precision,
                "auto" if requested == PRECISION_AUTO else f"{ENV_PRECISION}={precision}",
            )
        reasons.append(f"{precision}: {why}")
    if requested == PRECISION_AUTO and not AUTO_ORDER:
        return PRECISION_BF16, "auto: no quantized arm passed the accuracy gate"
    return PRECISION_BF16, "; ".join(reasons) or "bf16"


__all__ = [
    "AUTO_ORDER",
    "ENV_INT8_CONVROT",
    "ENV_PRECISION",
    "PRECISIONS",
    "PRECISION_AUTO",
    "PRECISION_BF16",
    "PRECISION_FP8",
    "PRECISION_INT8",
    "SEGMENTER_CONVROT_GROUP",
    "SEGMENTER_EXCLUDE_TOKENS",
    "SEGMENTER_MIN_FEATURES",
    "GATE_PASSED",
    "artifact_filename",
    "artifact_tag",
    "auto_may_load",
    "write_prequant_file",
    "checkpoint_problem",
    "load_prequantized_segmenter",
    "quantize_segmenter_",
    "requested_precision",
    "resolve_precision",
    "segmenter_filter_fn",
]