Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Native torchao-free denoiser cache, sharing diffusion_prequant_cache's root; key adds build spec."""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import tempfile
import time
from typing import Any, Optional
from . import diffusion_prequant_cache as _cache
NATIVE_LAYOUT_VERSION = 1
NATIVE_META_KEY = "unsloth_native_layers"
def native_spec(
scheme: str,
family: Optional[str],
*,
act_int8: bool,
scale_dtype: Any = None,
) -> dict:
"""Everything beside the dense weights that ``quantize_transformer``'s native path bakes into the bytes."""
from .diffusion_native_quant import NATIVE_INT8, int8_rotation_group
from .diffusion_transformer_quant import DEFAULT_MIN_LINEAR_FEATURES, exclude_tokens_for_scheme
act = bool(act_int8) and scheme == NATIVE_INT8
return {
"layout": NATIVE_LAYOUT_VERSION,
"scheme": scheme,
"act_int8": act,
"rot_group": int(int8_rotation_group()) if act else 0,
"scale_dtype": str(scale_dtype) if scale_dtype is not None and not act else None,
"min_features": DEFAULT_MIN_LINEAR_FEATURES,
"exclude_name_tokens": sorted(
set(exclude_tokens_for_scheme(scheme, family, baked = True)) | {"lora_"}
),
"require_bf16": True,
}
def _key_for(
fam: Any,
spec: dict,
base: Optional[str],
component: str,
*,
base_id: Optional[str] = None,
shards: Optional[list] = None,
) -> Optional[tuple[str, dict]]:
if spec.get("scheme") not in _cache.CACHEABLE_SCHEMES:
return None
if shards is None:
shards = _cache.source_shards(base, component)
if not shards:
return None
fp = _cache.source_fingerprint(shards)
if fp is None:
return None
try:
import torch
torch_minor = _cache._minor(torch.__version__)
except Exception: # noqa: BLE001
torch_minor = None
parts = {
"policy": _cache.POLICY_VERSION,
"family": str(getattr(fam, "name", fam if isinstance(fam, str) else None) or ""),
"transformer_class": str(getattr(fam, "transformer_class", None) or ""),
"scheme": spec["scheme"],
"component": component,
"source": fp,
"base": _cache.base_identity(base_id if base_id is not None else base),
"torch": torch_minor,
"native": dict(spec),
}
return _cache.cache_key(parts), parts
def _header_metadata(path: Any) -> Optional[dict]:
raw = _cache.read_header_bytes(path)
if raw is None:
return None
try:
meta = json.loads(raw[8:].decode("utf-8")).get("__metadata__") or {}
return json.loads(meta[NATIVE_META_KEY])
except Exception: # noqa: BLE001 - not a native entry
return None
def lookup(
fam: Any,
spec: dict,
*,
base_repo: Optional[str],
source_base: Optional[str] = None,
component: str = "transformer",
logger: Any = None,
) -> Optional[str]:
"""``source_base`` is where the dense shards are found (default ``base_repo``), as for ``save``.
Path of a cached native build of ``component`` for this exact dense source and ``spec``, or None. Local reads
(plus the Hub staleness probe a torchao hit also makes); never raises."""
if not _cache.enabled():
return None
try:
recorded_base = str(base_repo or getattr(fam, "base_repo", "") or "")
source = source_base or base_repo or getattr(fam, "base_repo", None)
key = shards = None
seen: set = set()
for candidate in (
_cache.source_shards(source, component),
*_cache._shard_sets(source, component),
):
if not candidate or tuple(candidate) in seen:
continue
seen.add(tuple(candidate))
keyed = _key_for(fam, spec, source, component, base_id = recorded_base, shards = candidate)
if (
keyed is not None
and (_cache.cache_root() / keyed[0] / _cache.MANIFEST_NAME).is_file()
):
key, shards = keyed[0], candidate
break
if key is None:
return None
if _cache._hub_source_stale(shards, logger):
return None
with _cache._cache_lock():
artifact = _cache._entry(key)
if artifact is None:
return None
with _cache._lock:
rejected = os.path.realpath(str(artifact)) in _cache._REJECTED
if rejected:
return None
_cache._touch(artifact.parent)
meta = _header_metadata(artifact)
if not isinstance(meta, dict) or meta.get("key") != key or meta.get("spec") != spec:
_cache.note_rejected(artifact, logger)
return None
return str(artifact)
except Exception as exc: # noqa: BLE001
_cache._log(logger, "warning", f"native lookup failed: {type(exc).__name__}: {exc}")
return None
def _layer_facts(transformer: Any, spec: dict) -> Optional[dict]:
"""``{fqn: facts}`` of every native Linear, or None when one disagrees with ``spec`` (a build the key does not
describe)."""
from .diffusion_native_quant import is_native_linear
layers: dict = {}
for name, mod in transformer.named_modules():
if not is_native_linear(mod):
continue
if mod.scheme != spec["scheme"] or bool(mod.act_int8) != bool(spec["act_int8"]):
return None
rot = int(getattr(mod, "rot_group", 0) or 0)
if rot not in (0, int(spec["rot_group"])):
return None
layers[name] = {
"scheme": mod.scheme,
"act_int8": bool(mod.act_int8),
"rot_group": rot,
"in_features": int(mod.in_features),
"out_features": int(mod.out_features),
"compute_dtype": str(mod.compute_dtype).replace("torch.", ""),
"fp8_format": str(getattr(mod, "fp8_format", "float8_e4m3fn")),
"bias": getattr(mod, "bias", None) is not None,
}
return layers or None
def _moves_with_offload(tensor: Any) -> bool:
"""Accelerator storage is what an offload release frees; a host tensor is a stable copy, kept as captured."""
return tensor.device.type != "cpu"
def _wait_for_transfers(value: Any) -> None:
"""Group offload onloads on its own stream with non_blocking copies: finish them before the writer reads."""
if value.device.type == "cuda":
from .diffusion_cuda_graph import synchronize_outside_capture
# A device sync inside another thread's CUDA graph capture would invalidate it.
synchronize_outside_capture(value.device)
def _live_refs(transformer: Any, state: dict) -> dict:
"""``state`` with module params / buffers swapped for live reads at write time, so a held state_dict does not
pin offloaded CUDA storage; hook-built tensors stay captured, a shape / dtype change fails the write."""
import torch
def resolve(name: str) -> Any:
prefix, _, attr = name.rpartition(".")
owner = transformer.get_submodule(prefix) if prefix else transformer
return getattr(owner, attr)
def getter(name: str, dtype: Any, shape: tuple) -> Any:
def get() -> Any:
value = resolve(name)
if value is None or value.dtype != dtype or tuple(value.shape) != shape:
raise RuntimeError(f"native cache write: {name} changed during the write")
# Detach before the sync: a later `param.data =` onload must not swap in a buffer still being copied.
value = value.detach()
_wait_for_transfers(value)
return value
return get
out: dict = {}
for name, tensor in state.items():
value = None
try:
if _moves_with_offload(tensor):
value = resolve(name)
# Same storage AND same view: a hook that hands back e.g. a transposed view keeps its captured tensor.
same = (
isinstance(value, torch.Tensor)
and value.dtype == tensor.dtype
and tuple(value.shape) == tuple(tensor.shape)
and value.data_ptr() == tensor.data_ptr()
and tuple(value.stride()) == tuple(tensor.stride())
)
except Exception: # noqa: BLE001 - not a module attribute (hook-built): keep the captured tensor
same = False
if same:
out[name] = _cache._FlatRef(getter(name, tensor.dtype, tuple(tensor.shape)), tensor)
else:
out[name] = tensor
return out
def _live_state(transformer: Any, state: dict) -> dict:
"""``_live_refs``, recapturing once if an offload moved weights after capture (else the write pins old storage)."""
import torch
layout = {k: (v.dtype, tuple(v.shape)) for k, v in state.items()}
refs = _live_refs(transformer, state)
if not any(isinstance(v, torch.Tensor) and _moves_with_offload(v) for v in refs.values()):
return refs
state = refs = None
fresh = transformer.state_dict()
if {k: (v.dtype, tuple(v.shape)) for k, v in fresh.items()} != layout or any(
type(v) not in (torch.Tensor, torch.nn.Parameter) for v in fresh.values()
):
raise RuntimeError("native cache write: the weights changed during the write")
return _live_refs(transformer, fresh)
def save(
transformer: Any,
fam: Any,
spec: dict,
*,
base_repo: Optional[str],
source_base: Optional[str] = None,
component: str = "transformer",
logger: Any = None,
cancel_check: Optional[Any] = None,
) -> Optional[str]:
"""Persist a just-built native denoiser; returns the artifact path or None (logged). Never raises."""
if not _cache.enabled():
return None
t0 = time.perf_counter()
root = _cache.cache_root()
try:
import torch
info = dict(getattr(transformer, "_unsloth_runtime_quant_info", None) or {})
if not info.get("native") or info.get("scheme") != spec.get("scheme"):
_cache._log(
logger,
"info",
f"native {component}: skipped (not a native {spec.get('scheme')} build)",
)
return None
if info.get("cache_path"):
return None
recorded_base = str(base_repo or getattr(fam, "base_repo", "") or "")
keyed = _key_for(fam, spec, source_base or base_repo, component, base_id = recorded_base)
if keyed is None:
_cache._log(
logger,
"info",
f"native {component}: skipped (dense source shards not found locally)",
)
return None
key, parts = keyed
layers = _layer_facts(transformer, spec)
if layers is None:
_cache._log(logger, "info", f"native {component}: skipped (layers differ from the key)")
return None
with _cache._writer_lock(root, key) as locked:
if not locked:
_cache._log(logger, "info", f"native {component}: skipped (no cross-process lock)")
return None
existing = _cache._entry(key)
replacing = None
if existing is not None:
real = os.path.realpath(str(existing))
with _cache._lock:
replacing = real if real in _cache._REJECTED else None
if replacing is None:
_cache._log(
logger, "info", f"native {component}: skipped (already cached as {key})"
)
return None
# an entry load() rejected (same size, bad bytes) would otherwise block every rebuild for good
_cache._log(
logger, "info", f"native {component}: replacing {key} (it failed to load)"
)
state = transformer.state_dict()
if any(".lora_" in k or k.startswith("lora_") for k in state):
_cache._log(logger, "info", f"native {component}: skipped (baked LoRA adapters)")
return None
for name, tensor in state.items():
if (
type(tensor) not in (torch.Tensor, torch.nn.Parameter)
or tensor.device.type == "meta"
):
_cache._log(
logger,
"info",
f"native {component}: skipped ({name} is not a plain tensor)",
)
return None
name = tensor = (
None # the loop's last tensor would otherwise stay referenced through the write
)
nbytes = sum(_cache._tensor_bytes(t) for t in state.values())
budget = _cache.max_cache_bytes()
if budget is not None and nbytes > budget:
_cache._log(
logger, "info", f"native {component}: skipped (larger than the cache budget)"
)
return None
root.mkdir(parents = True, exist_ok = True)
need = nbytes + max(_cache._DISK_MARGIN_BYTES, nbytes // 20)
free = shutil.disk_usage(str(root)).free
if free < need:
_cache.evict(root = root, extra_free_bytes = need - free, keep = key, logger = logger)
free = shutil.disk_usage(str(root)).free
if free < need:
_cache._log(
logger,
"info",
f"native {component}: skipped (disk short: {free / 2**30:.1f} GiB free, "
f"need {need / 2**30:.1f} GiB)",
)
return None
meta = {
"key": key,
"spec": spec,
"family": parts["family"],
"component": component,
"base_model_id": recorded_base,
"transformer_class": parts["transformer_class"] or type(transformer).__name__,
"torch_version": str(torch.__version__),
"layers": layers,
}
header = {NATIVE_META_KEY: json.dumps(meta, sort_keys = True)}
kdir = root / key
if replacing is not None:
_cache._reclaim(kdir)
with _cache._lock:
_cache._REJECTED.discard(replacing)
kdir.mkdir(parents = True, exist_ok = True)
fd, tmp = tempfile.mkstemp(
dir = str(kdir), prefix = f".{_cache.FILE_NAME}.", suffix = _cache._TEMP_SUFFIX
)
os.close(fd)
try:
state = _live_state(transformer, state)
hasher = hashlib.sha256()
size = _cache.write_streaming_safetensors(
tmp, state, header, cancel_check = cancel_check, hasher = hasher
)
del state
os.replace(tmp, kdir / _cache.FILE_NAME)
tmp = None
finally:
if tmp is not None:
try:
os.unlink(tmp)
except OSError:
pass
sha = hasher.hexdigest()
_cache._record_verified(kdir, sha)
manifest = {
"key": key,
"bytes": size,
"sha256": sha,
"created": time.time(),
"parts": parts,
}
_cache._atomic_bytes(
kdir / _cache.MANIFEST_NAME,
json.dumps(manifest, indent = 1, default = str).encode("utf-8"),
)
_cache._touch(kdir)
_cache._log(
logger,
"info",
f"native {component}: saved ({key}, {size / 2**30:.2f} GiB in {time.perf_counter() - t0:.1f} s)",
)
_cache.evict(logger = logger, keep = key)
return str(kdir / _cache.FILE_NAME)
except Exception as exc: # noqa: BLE001 - a cache write never fails a load (the caller re-checks a cancel)
_cache._log(
logger,
"warning",
f"native {component}: save failed: {type(exc).__name__}: {str(exc)[:200]}",
)
return None
def load(
path: str,
transformer_cls: Any,
config_source: str,
*,
component: str = "transformer",
spec: dict,
config_kwargs: Optional[dict] = None,
logger: Any = None,
cancel_check: Optional[Any] = None,
) -> Any:
"""Rebuild the cached native ``component`` (config from ``config_source``), or None. Never raises; a cancel
from ``cancel_check`` is a miss that leaves the entry usable (the caller re-checks it)."""
cancelled = [False]
def _check() -> None:
if cancel_check is None:
return
try:
cancel_check()
except BaseException:
cancelled[0] = True
raise
try:
import torch
from safetensors.torch import load_file
from .diffusion_native_quant import native_linear_class
if not _cache.verify_entry(path, logger, cancel_check = _check):
if not cancelled[0]:
_cache.note_rejected(path, logger)
return None
meta = _header_metadata(path)
if (
not isinstance(meta, dict)
or meta.get("spec") != spec
or meta.get("component") != component
):
_cache.note_rejected(path, logger)
return None
layers = meta.get("layers") or {}
config = transformer_cls.load_config(
config_source, subfolder = component, **(config_kwargs or {})
)
try:
from accelerate import init_empty_weights
except Exception: # noqa: BLE001
return None
# Buffers stay real: non-persistent ones (rope tables) are built by __init__ as a dense load builds them.
with init_empty_weights(include_buffers = False):
model = transformer_cls.from_config(config)
state = load_file(str(path))
cls = native_linear_class()
for fqn, facts in layers.items():
parent_name, _, leaf = fqn.rpartition(".")
parent = model.get_submodule(parent_name) if parent_name else model
stand_in = torch.nn.Linear(
int(facts["in_features"]),
int(facts["out_features"]),
bias = bool(facts["bias"]),
device = "meta",
dtype = getattr(torch, str(facts["compute_dtype"])),
)
codes = state[f"{fqn}.weight_q"]
if facts["scheme"] != "int8":
codes = codes.view(getattr(torch, str(facts.get("fp8_format") or "float8_e4m3fn")))
scale = state[f"{fqn}.weight_scale"].view(torch.float32)
setattr(
parent,
leaf,
cls(
stand_in,
facts["scheme"],
act_int8 = bool(facts["act_int8"]),
rot_group = int(facts["rot_group"] or 0),
codes = codes,
scale = scale,
),
)
from .prequant_safetensors import _unviewed
# No-copy aliases: a view's base survives offload's ``.data =`` re-point and keeps the whole file mapped.
state = {key: _unviewed(value) for key, value in state.items()}
model.load_state_dict(state, strict = True, assign = True)
del state
if any(t.device.type == "meta" for t in (*model.parameters(), *model.buffers())):
_cache._log(logger, "warning", f"native {component}: entry left meta tensors; not used")
_cache.note_rejected(path, logger)
return None
model.eval()
try:
model._unsloth_runtime_quant = spec["scheme"]
model._unsloth_runtime_quant_info = {
"scheme": spec["scheme"],
"native": True,
"cache_path": str(path),
}
except Exception: # noqa: BLE001
pass
_cache.note_hit(path, logger)
return model
except Exception as exc: # noqa: BLE001 - an unusable entry is a miss
if cancelled[0]:
return None
_cache._log(
logger,
"warning",
f"native {component}: read failed: {type(exc).__name__}: {str(exc)[:200]}",
)
try:
_cache.note_rejected(path, logger)
except Exception: # noqa: BLE001
pass
return None
STREAM_BUILD_ENV = "UNSLOTH_DIFFUSION_NATIVE_STREAM_BUILD"
# Dtypes a bf16 load never casts; any other floating dtype falls back to the dense load.
_STREAM_DTYPES = {
"BF16": "bfloat16",
"I64": "int64",
"I32": "int32",
"I16": "int16",
"I8": "int8",
"U8": "uint8",
"BOOL": "bool",
}
# ``cast_floats``: floating dtypes a bf16 load casts per tensor exactly as diffusers' loader does.
_CAST_DTYPES = {"F32": "float32", "F16": "float16", "F64": "float64"}
def stream_build_enabled() -> bool:
value = (os.environ.get(STREAM_BUILD_ENV) or "").strip().lower()
return value not in ("0", "off", "false", "no")
def _dense_shards(
source: Any,
component: str,
cache_dir: Optional[str],
weights_name: str = "diffusion_pytorch_model",
) -> Optional[list[str]]:
"""The shard files ``from_pretrained(source, subfolder = component)`` reads (index first, as diffusers), or None."""
stem = f"{component}/{weights_name}"
index = _cache._local_file(str(source), f"{stem}.safetensors.index.json", cache_dir)
if index is not None:
with open(index, "r", encoding = "utf-8") as fh:
weight_map = (json.load(fh) or {}).get("weight_map") or {}
names = sorted(set(str(v) for v in weight_map.values()))
paths = [_cache._local_file(str(source), f"{component}/{n}", cache_dir) for n in names]
return [str(p) for p in paths] if names and all(paths) else None
single = _cache._local_file(str(source), f"{stem}.safetensors", cache_dir)
return [str(single)] if single else None
def _shard_entries(path: str, dtypes: Optional[dict] = None) -> Optional[tuple[int, dict]]:
"""(data start, ``{key: header entry}``) of a shard holding only dtypes in ``dtypes`` (default
``_STREAM_DTYPES``), else None."""
dtypes = _STREAM_DTYPES if dtypes is None else dtypes
raw = _cache.read_header_bytes(path)
if raw is None:
return None
header = json.loads(raw[8:].decode("utf-8"))
header.pop("__metadata__", None)
if any(entry.get("dtype") not in dtypes for entry in header.values()):
return None
return len(raw), header
def _stored_dtype(tag: str) -> Any:
import torch
return getattr(torch, _STREAM_DTYPES.get(tag) or _CAST_DTYPES[tag])
def _loaded_dtype(key: str, tag: str, dtype: Any, keep_in_fp32: Any) -> Any:
"""The dtype ``from_pretrained(torch_dtype = dtype)`` gives the stored ``key``: diffusers'
``load_model_dict_into_meta`` rule (a floating param under a ``_keep_in_fp32_modules`` name goes fp32, any other
floating param goes ``dtype``; integers stay)."""
import torch
stored = _stored_dtype(tag)
if not stored.is_floating_point:
return stored
if keep_in_fp32 and any(m in key.split(".") for m in keep_in_fp32):
return torch.float32
return dtype
def _map_tensor(fd: int, start: int, entry: dict) -> Any:
"""Private mapping per tensor, unmapped when freed; a shard mapping stays while any tensor lives."""
import mmap
import torch
begin, end = (int(v) for v in entry["data_offsets"])
dtype = _stored_dtype(entry["dtype"])
shape = [int(d) for d in entry["shape"]]
count = 1
for d in shape:
count *= d
if end - begin != count * torch.empty((), dtype = dtype).element_size():
raise ValueError("shard entry size does not match its shape")
if end == begin:
return torch.empty(shape, dtype = dtype)
offset = start + begin
aligned = offset - offset % mmap.ALLOCATIONGRANULARITY
# Copy-on-write: writable, nothing reaches the file.
window = mmap.mmap(
fd, offset + (end - begin) - aligned, offset = aligned, access = mmap.ACCESS_COPY
)
flat = torch.frombuffer(window, dtype = torch.uint8, count = end - begin, offset = offset - aligned)
return flat.view(dtype).reshape(shape)
def build_from_shards(
transformer_cls: Any,
config_source: str,
spec: dict,
*,
family: Optional[str],
dtype: Any,
component: str = "transformer",
config_kwargs: Optional[dict] = None,
logger: Any = None,
cancel_check: Optional[Any] = None,
cast_floats: bool = False,
reraise_oom: bool = False,
) -> Any:
"""Build the native ``component`` per tensor, bit-identical to the in-place build, or None for the dense load
(also on cancel, which the caller re-checks). Never raises, except an out-of-memory under ``reraise_oom``.
``cast_floats``: fp32 / fp16 shards and ``_keep_in_fp32_modules`` classes are built too, each tensor cast as the
dense bf16 load casts it (one Linear's bf16 copy at a time). Inside ``quantise_host_linears_on(device)`` each
Linear is quantised on ``device`` and returned to the host, as the in-place build there does."""
cancelled = [False]
def _check() -> None:
if cancel_check is None:
return
try:
cancel_check()
except BaseException:
cancelled[0] = True
raise
t0 = time.perf_counter()
try:
import torch
from .diffusion_native_quant import (
_HOST_QUANT_DEVICE,
NATIVE_INT8,
_HostLinearMover,
native_linear_class,
)
from .diffusion_transformer_quant import exclude_tokens_for_scheme, make_filter_fn
def skip(why: str) -> None:
_cache._log(logger, "info", f"native {component}: per-tensor build skipped ({why})")
if not stream_build_enabled() or transformer_cls is None:
return None
if dtype is not torch.bfloat16:
return skip(f"loads at {str(dtype).replace('torch.', '')}, not bfloat16")
keep_in_fp32 = list(getattr(transformer_cls, "_keep_in_fp32_modules", None) or ())
if keep_in_fp32 and not cast_floats:
return skip("the class keeps fp32 modules")
scheme = spec["scheme"]
cache_dir = (config_kwargs or {}).get("cache_dir")
shards = _dense_shards(config_source, component, cache_dir)
if not shards:
return skip("dense shards not found locally")
if _cache._hub_source_stale(shards, logger):
return None
allowed = {**_STREAM_DTYPES, **_CAST_DTYPES} if cast_floats else _STREAM_DTYPES
entries = []
for shard in shards:
found = _shard_entries(shard, allowed)
if found is None:
return skip(
"a shard holds a dtype other than bf16"
if not cast_floats
else "a shard holds a dtype the per-tensor cast does not cover"
)
entries.append((shard, *found))
stored = {key: entry for _shard, _start, header in entries for key, entry in header.items()}
_check()
config = transformer_cls.load_config(
config_source, subfolder = component, **(config_kwargs or {})
)
try:
from accelerate import init_empty_weights
except Exception: # noqa: BLE001
return skip("accelerate is missing")
# Buffers stay real: non-persistent ones (rope tables) are built by __init__ as a dense load builds them.
with init_empty_weights(include_buffers = False):
model = transformer_cls.from_config(config)
# The in-place build filters the loaded module, whose weights carry the stored dtype.
for name, mod in model.named_modules():
entry = stored.get(f"{name}.weight") if name else None
if (
isinstance(mod, torch.nn.Linear)
and entry is not None
and mod.weight.device.type == "meta"
):
mod.weight = torch.nn.Parameter(
torch.empty(
mod.weight.shape,
dtype = _loaded_dtype(f"{name}.weight", entry["dtype"], dtype, keep_in_fp32)
if cast_floats
else _stored_dtype(entry["dtype"]),
device = "meta",
),
requires_grad = mod.weight.requires_grad,
)
filter_fn = make_filter_fn(
int(spec["min_features"]),
exclude_name_tokens = exclude_tokens_for_scheme(scheme, family, baked = True) + ("lora_",),
require_bf16 = True,
)
quant = {name for name, mod in model.named_modules() if name and filter_fn(mod, name)}
act_int8 = scheme == NATIVE_INT8 and bool(spec["act_int8"])
rot_group = int(spec["rot_group"]) if act_int8 else 0
cls = native_linear_class()
def make(linear: Any) -> Any:
return cls(linear, scheme, act_int8 = act_int8, rot_group = rot_group)
mover_device = _HOST_QUANT_DEVICE.get()
mover = _HostLinearMover(mover_device) if mover_device is not None else None
state: dict = {}
built = 0
for shard, start, header in entries:
fd = os.open(shard, os.O_RDONLY)
try:
for key, entry in sorted(
header.items(), key = lambda kv: int(kv[1]["data_offsets"][0])
):
_check()
tensor = _map_tensor(fd, start, entry)
if cast_floats:
# What load_model_dict_into_meta does: ``param.to(dtype)`` (a no-op at the stored dtype).
tensor = tensor.to(_loaded_dtype(key, entry["dtype"], dtype, keep_in_fp32))
fqn, _, leaf = key.rpartition(".")
if leaf != "weight" or fqn not in quant:
state[key] = tensor
continue
parent_name, _, attr = fqn.rpartition(".")
parent = model.get_submodule(parent_name) if parent_name else model
dense = getattr(parent, attr)
stand_in = torch.nn.Linear(
int(dense.in_features),
int(dense.out_features),
bias = dense.bias is not None,
device = "meta",
dtype = tensor.dtype,
)
stand_in.weight = torch.nn.Parameter(tensor, requires_grad = False)
del tensor
layer = mover.build(stand_in, make) if mover is not None else make(stand_in)
del stand_in
setattr(parent, attr, layer)
state[f"{fqn}.weight_q"] = layer.weight_q
state[f"{fqn}.weight_scale"] = layer.weight_scale
built += 1
finally:
# The mappings outlive the descriptor; each closes with its tensor.
os.close(fd)
if mover is not None:
mover.close(logger)
if built != len(quant):
return skip(f"{len(quant) - built} quantizable weights missing from the shards")
model.load_state_dict(state, strict = True, assign = True)
del state
if any(t.device.type == "meta" for t in (*model.parameters(), *model.buffers())):
return skip("tensors left on meta")
model.eval()
try:
model._unsloth_runtime_quant = scheme
model._unsloth_runtime_quant_info = {"scheme": scheme, "native": True}
except Exception: # noqa: BLE001
pass
_cache._log(
logger,
"info",
f"native {component}: {scheme} "
+ (
("W8A8 torch._int_mm" + (f" ConvRot g{rot_group}" if rot_group else ""))
if act_int8
else "weight-only"
)
+ f" built per tensor from {len(shards)} dense shard(s) on {built} linears "
f"in {time.perf_counter() - t0:.1f} s",
)
return model
except Exception as exc: # noqa: BLE001 - the caller loads the dense module whole instead
if cancelled[0]:
return None
if reraise_oom and _is_oom(exc):
raise
_cache._log(
logger,
"warning",
f"native {component}: per-tensor build failed: {type(exc).__name__}: {str(exc)[:200]}",
)
return None
def _is_oom(exc: BaseException) -> bool:
if isinstance(exc, MemoryError):
return True
try:
import torch
oom = getattr(torch, "OutOfMemoryError", None)
if oom is not None and isinstance(exc, oom):
return True
except Exception: # noqa: BLE001
pass
return "out of memory" in str(exc).lower()
def build_spec(transformer: Any, family: Optional[str]) -> Optional[dict]:
"""``native_spec`` of an in-memory native build, read from its layers, or None (no native layers, or a mix)."""
try:
from .diffusion_native_quant import is_native_linear
schemes, acts = set(), set()
for mod in transformer.modules():
if is_native_linear(mod):
schemes.add(mod.scheme)
acts.add(bool(mod.act_int8))
if len(schemes) != 1 or len(acts) != 1:
return None
return native_spec(next(iter(schemes)), family, act_int8 = next(iter(acts)))
except Exception: # noqa: BLE001
return None
class NativePendingWrite:
"""Background ``save``, awaited by ``finish_background_writes``; writes live host tensors (no snapshot copy)."""
def __init__(
self,
run: Any,
key: str,
logger: Any = None,
) -> None:
import threading
self._run_fn = run
self.key = key
self._logger = logger
self._cancel = threading.Event()
self._finished = threading.Event()
self._thread: Optional[Any] = None
self.result: Optional[str] = None
with _cache._lock:
_cache._BACKGROUND.add(self)
def _check(self) -> None:
if self._cancel.is_set():
raise RuntimeError("native cache write cancelled")
def start(self) -> bool:
import threading
with _cache._lock:
if self._thread is not None or self._run_fn is None or self._cancel.is_set():
return False
self._thread = threading.Thread(
target = self._run, name = f"native-cache-write-{self.key[:8]}", daemon = True
)
try:
self._thread.start()
except Exception: # noqa: BLE001 - no writer: the next load re-quantizes
with _cache._lock:
self._thread = None
self._close()
return False
return True
def _run(self) -> None:
try:
if self._run_fn is not None:
self.result = self._run_fn(self._check)
except Exception as exc: # noqa: BLE001 - a background cache write never surfaces
_cache._log(self._logger, "warning", f"native: background save failed: {exc}")
finally:
self._close()
def _close(self) -> None:
self._run_fn = None
with _cache._lock:
_cache._BACKGROUND.discard(self)
self._finished.set()
def discard(self) -> None:
with _cache._lock:
started = self._thread is not None
self._cancel.set()
if not started:
self._close()
def cancel(self) -> None:
self.discard()
def done(self) -> bool:
return self._finished.is_set()
def wait(self, timeout: Optional[float] = None) -> bool:
return self._finished.wait(timeout)
def save_deferred(
transformer: Any,
fam: Any,
*,
base_repo: Optional[str],
source_base: Optional[str] = None,
component: str = "transformer",
logger: Any = None,
cancel_check: Optional[Any] = None,
) -> Any:
"""A ``NativePendingWrite`` (start after publish), the sync ``save`` result, or None. Never raises."""
try:
if not _cache.enabled():
return None
spec = build_spec(transformer, getattr(fam, "name", None))
if spec is None:
_cache._log(logger, "info", f"native {component}: skipped (no single native recipe)")
return None
recorded_base = str(base_repo or getattr(fam, "base_repo", "") or "")
keyed = _key_for(fam, spec, source_base or base_repo, component, base_id = recorded_base)
if keyed is None:
_cache._log(
logger,
"info",
f"native {component}: skipped (dense source shards not found locally)",
)
return None
def run(check: Optional[Any] = None) -> Optional[str]:
def both() -> None:
if check is not None:
check()
if cancel_check is not None and check is None:
cancel_check()
return save(
transformer,
fam,
spec,
base_repo = base_repo,
source_base = source_base,
component = component,
logger = logger,
cancel_check = both,
)
if not _cache.async_enabled():
return run()
return NativePendingWrite(run, keyed[0], logger)
except Exception as exc: # noqa: BLE001
_cache._log(
logger, "warning", f"native {component}: save skipped ({type(exc).__name__}: {exc})"
)
return None
def read_for_load(
fam: Any,
spec: Optional[dict],
transformer_cls: Any,
*,
base_repo: Optional[str],
source_base: Optional[str],
component: str = "transformer",
config_kwargs: Optional[dict] = None,
logger: Any = None,
cancel_check: Optional[Any] = None,
) -> Any:
"""``lookup`` + ``load`` of the cached native ``component``, or None (a miss). Never raises."""
if spec is None or transformer_cls is None:
return None
path = lookup(
fam, spec, base_repo = base_repo, source_base = source_base, component = component, logger = logger
)
if path is None:
return None
return load(
path,
transformer_cls,
str(source_base or base_repo),
component = component,
spec = spec,
config_kwargs = config_kwargs,
logger = logger,
cancel_check = cancel_check,
)