Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read pre-quantized diffusion checkpoints from safetensors on torchao 0.17, 0.18 and main (#12645)

Daniel Han0f239ddca593
diff --git a/scripts/convert_prequant_to_safetensors.py b/scripts/convert_prequant_to_safetensors.py
new file mode 100644
index 00000000000..93c5309fcb6
--- /dev/null
+++ b/scripts/convert_prequant_to_safetensors.py
@@ -0,0 +1,364 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Convert a published ``.pt`` pre-quant checkpoint (denoiser or text encoder) to ``.safetensors``.
+
+ python scripts/convert_prequant_to_safetensors.py SRC.pt [SRC2.pt ...] --out-dir DIR \
+ [--repo owner/name --revision SHA]
+
+Writes ``DIR/<owner>/<name>/<same stem>.safetensors`` (``DIR/<stem>.safetensors`` without ``--repo``), the
+layout ``UNSLOTH_DIFFUSION_PREQUANT_MIRROR`` reads. Nothing is uploaded.
+
+Denoisers use the container Qwen-Image-2.1's published artifacts already use (``prequant_safetensors``:
+torchao's flattened tensors plus ``unsloth_format`` / ``unsloth_metadata``), so a released Studio that
+resolves ``<Model>-<SCHEME>.safetensors`` reads it too. Two header keys are added: ``unsloth_quant_layout``
+(how to rebuild every weight from its plain tensors without torchao, and for int8 that the weights came
+from torchao v1, so torchao <= 0.17 rebuilds the v1 class the ``.pt`` held) and ``unsloth_source`` (the
+file, its sha256 and the repo revision it was converted from). ``unsloth_metadata`` is copied unchanged.
+
+Text encoders are plain fp8 / bf16 tensors and are written in the plain layout
+``load_plain_prequant_safetensors`` reads, with no torchao involved.
+
+Every weight is checked after writing: the file is read back through Studio's loader and each tensor
+compared bit for bit with the ``.pt`` as the same install reads it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+BACKEND = Path(__file__).resolve().parent.parent / "studio" / "backend"
+
+
+def sha256_of(path: str, chunk: int = 64 << 20) -> str:
+ digest = hashlib.sha256()
+ with open(path, "rb") as fh:
+ while True:
+ block = fh.read(chunk)
+ if not block:
+ break
+ digest.update(block)
+ return digest.hexdigest()
+
+
+def _record_v1_facts(facts: list):
+ """Wrap the legacy decoder so each v1 weight's facts are recorded before it is rebuilt (torchao >= 0.18).
+
+ Patches a module global for the duration of one read, so conversions must not run concurrently in one
+ process (this is a single-threaded CLI)."""
+ from core.inference import prequant_legacy_int8 as legacy
+
+ original = legacy._rebuild_weight
+
+ def recording(name, w, standins, api):
+ facts.append(_v1_facts_of(w, standins))
+ return original(name, w, standins, api)
+
+ legacy._rebuild_weight = recording
+ return lambda: setattr(legacy, "_rebuild_weight", original)
+
+
+def _v1_facts_of(w, standins = None) -> dict:
+ """The INT8_V1_FACTS view of one v1 weight (a real torchao object or the decoder's stand-in)."""
+ from core.inference.prequant_legacy_int8 import _ACT_QUANT, _LAYOUT
+
+ state = getattr(w, "__dict__", {})
+ aqt = state.get("original_weight_tensor")
+ impl = getattr(aqt, "tensor_impl", None)
+ act = state.get("input_quant_func")
+ scale = getattr(impl, "scale", None)
+ zero_point = getattr(impl, "zero_point", None)
+ domain = getattr(aqt, "zero_point_domain", None)
+ layout = getattr(impl, "_layout", None)
+ if standins is not None:
+ # torchao >= 0.18 read the pickle into stand-ins registered under the v1 names: report those names.
+ if act is standins.get(_ACT_QUANT):
+ act = _ACT_QUANT
+ if isinstance(layout, standins.get(_LAYOUT, ())):
+ layout = _LAYOUT
+ return {
+ "act_quant": (act if isinstance(act, str) else getattr(act, "__name__", str(act))).rsplit(
+ ".", 1
+ )[-1],
+ "quant_kwargs": dict(state.get("quant_kwargs") or {}),
+ "zero_point": None if zero_point is None else "tensor",
+ "zero_point_domain": getattr(domain, "name", str(domain)),
+ "quant_min": getattr(aqt, "quant_min", "?"),
+ "quant_max": getattr(aqt, "quant_max", "?"),
+ "scale_shape": "out" if scale is not None and scale.dim() == 1 else "other",
+ "layout": layout.rsplit(".", 1)[-1] if isinstance(layout, str) else type(layout).__name__,
+ }
+
+
+def _to_int8_tensor_dict(state_dict: dict, facts: list) -> dict:
+ """torchao <= 0.17 hands back v1 objects: record their facts and rebuild them as Int8Tensor to flatten."""
+ from core.inference.prequant_legacy_int8 import (
+ LEGACY_INT8_CLASS_NAMES,
+ _ACT_QUANT,
+ _int8_tensor_api,
+ _rebuild_weight,
+ _resolve,
+ )
+
+ classes = {n: _resolve(n) for n in (*LEGACY_INT8_CLASS_NAMES, _ACT_QUANT)}
+ laqt = classes[LEGACY_INT8_CLASS_NAMES[0]]
+ if laqt is None:
+ return state_dict
+ api = _int8_tensor_api()
+ out = {}
+ for key, value in state_dict.items():
+ if isinstance(value, laqt):
+ facts.append(_v1_facts_of(value))
+ out[key] = _rebuild_weight(key, value, classes, api)
+ else:
+ out[key] = value
+ return out
+
+
+def _canonical(value):
+ """``(class name, {attr: tensor or value})`` of a weight, recursing into torchao subclasses."""
+ import torch
+
+ if type(value) is torch.Tensor:
+ return ("Tensor", value)
+ flatten = getattr(value, "__tensor_flatten__", None)
+ if flatten is None:
+ return (type(value).__name__, value)
+ names, ctx = flatten()
+ return (type(value).__name__, {n: _canonical(getattr(value, n)) for n in names}, repr(ctx))
+
+
+def _same(a, b) -> bool:
+ import torch
+
+ if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):
+ if a.dtype != b.dtype or tuple(a.shape) != tuple(b.shape):
+ return False
+ # Raw bytes, not torch.equal: -0.0 == 0.0 and NaN != NaN would otherwise decide.
+ return a.numel() == 0 or torch.equal(
+ a.detach().contiguous().cpu().view(torch.uint8),
+ b.detach().contiguous().cpu().view(torch.uint8),
+ )
+ if isinstance(a, tuple) and isinstance(b, tuple):
+ return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b, strict = True))
+ if isinstance(a, dict) and isinstance(b, dict):
+ return a.keys() == b.keys() and all(_same(a[k], b[k]) for k in a)
+ return a == b
+
+
+def compare_state_dicts(reference: dict, candidate: dict) -> list:
+ """Keys whose class, attributes or tensor bytes differ (empty = bit-identical)."""
+ bad = sorted(set(reference) ^ set(candidate))
+ for key in reference:
+ if key in candidate and not _same(_canonical(reference[key]), _canonical(candidate[key])):
+ bad.append(key)
+ return bad
+
+
+def _untie(flat: dict) -> dict:
+ """safetensors refuses tensors that share storage (tied embeddings): write each one as its own copy.
+
+ Studio's text-encoder loader re-ties after ``load_state_dict`` (``tie_weights``), exactly as it does for the
+ published safetensors encoders, so the copy costs disk only."""
+ seen: set = set()
+ out = {}
+ for key, value in flat.items():
+ value = value.contiguous()
+ ptr = value.untyped_storage().data_ptr() if value.numel() else None
+ if ptr is not None and ptr in seen:
+ value = value.clone()
+ elif ptr is not None:
+ seen.add(ptr)
+ out[key] = value
+ return out
+
+
+def write_plain_safetensors(
+ path: str, *, fmt: str, state_dict: dict, metadata: dict, extra: dict
+) -> None:
+ """The torchao-free plain layout ``load_plain_prequant_safetensors`` reads (text encoders)."""
+ import torch
+ from safetensors.torch import save_file
+
+ from core.inference.prequant_safetensors import (
+ UNSLOTH_FORMAT_KEY,
+ UNSLOTH_METADATA_KEY,
+ UNSLOTH_ROOT_KEYS_KEY,
+ UNSLOTH_ROOT_PREFIX,
+ )
+
+ names, roots, flat = [], [], {}
+ for key, value in state_dict.items():
+ if type(value) is not torch.Tensor:
+ raise ValueError(f"{key!r} is a {type(value).__name__}, not a plain tensor")
+ if "." in key:
+ names.append(key)
+ flat[key] = value.contiguous()
+ else:
+ roots.append(key)
+ flat[f"{UNSLOTH_ROOT_PREFIX}{key}"] = value.contiguous()
+ header = {name: json.dumps({"_type": "Tensor"}) for name in names}
+ header["tensor_names"] = json.dumps(names)
+ header[UNSLOTH_FORMAT_KEY] = fmt
+ header[UNSLOTH_METADATA_KEY] = json.dumps(metadata, default = str)
+ if roots:
+ header[UNSLOTH_ROOT_KEYS_KEY] = json.dumps(roots)
+ header.update(extra)
+ save_file(_untie(flat), path, metadata = header)
+
+
+def convert(
+ src: str, out_dir: str, *, repo: str | None, revision: str | None, sha: str | None
+) -> dict:
+ import torch
+
+ from core.inference import diffusion_prequant as dp
+ from core.inference import prequant_safetensors as ps
+ from core.inference.diffusion_te_prequant import TE_PREQUANT_FORMAT
+ from core.inference.prequant_native import (
+ INT8_SOURCE_V1,
+ INT8_V1_FACTS,
+ QUANT_LAYOUT_KEY,
+ SOURCE_KEY,
+ quant_layout_header,
+ )
+
+ t0 = time.time()
+ name = os.path.basename(src)
+ stem = os.path.splitext(name)[0] if name.endswith((".pt", ".pth")) else name
+ dest_dir = os.path.join(out_dir, *repo.split("/")) if repo else out_dir
+ os.makedirs(dest_dir, exist_ok = True)
+ dst = os.path.join(dest_dir, stem + ".safetensors")
+ source = {"file": name, "sha256": sha or sha256_of(src), "bytes": os.path.getsize(src)}
+ if repo:
+ source["repo"] = repo
+ if revision:
+ source["revision"] = revision
+ try:
+ import torchao
+ source["converter_torchao"] = torchao.__version__
+ except Exception: # noqa: BLE001
+ pass
+ source["converter_torch"] = torch.__version__
+
+ # Read exactly as Studio does (allowlisted weights_only); on torchao >= 0.18 v1 int8 comes back as Int8Tensor.
+ facts: list = []
+ restore = _record_v1_facts(facts)
+ try:
+ ckpt = dp._load_prequant_checkpoint(src, map_location = "cpu", mmap = True)
+ finally:
+ restore()
+ fmt = ckpt.get("format")
+ is_te = fmt == TE_PREQUANT_FORMAT
+ metadata = dict(ckpt.get("metadata") or {})
+ state_dict = ckpt["state_dict"]
+ extra = {SOURCE_KEY: json.dumps(source)}
+ if is_te:
+ write_plain_safetensors(dst, fmt = fmt, state_dict = state_dict, metadata = metadata, extra = extra)
+ back = ps.load_plain_prequant_safetensors(dst)
+ else:
+ if fmt not in dp.PREQUANT_FORMATS:
+ raise ValueError(f"{src}: unrecognised format {fmt!r}")
+ flat_sd = _to_int8_tensor_dict(state_dict, facts)
+ int8_source = None
+ if facts:
+ int8 = sum(type(v).__name__ == "Int8Tensor" for v in flat_sd.values())
+ if len(facts) != int8:
+ raise ValueError(
+ f"{src}: {int8} int8 weights but {len(facts)} came from torchao v1"
+ )
+ distinct = {json.dumps(f, sort_keys = True, default = str) for f in facts}
+ if distinct != {json.dumps(INT8_V1_FACTS, sort_keys = True)}:
+ raise ValueError(
+ f"{src}: v1 int8 weights do not match the supported v1 layout: {sorted(distinct)[:3]}"
+ )
+ int8_source = INT8_SOURCE_V1
+ layout = quant_layout_header(flat_sd, int8_source = int8_source)
+ # save_prequant_safetensors writes torchao's flattened layout plus our two keys; add the layout and source.
+ from safetensors.torch import save_file
+
+ helpers = ps._torchao_helpers()
+ if helpers is None:
+ raise RuntimeError("torchao >= 0.16 is required to write the flattened layout")
+ undotted = ps.unsupported_state_dict_keys(flat_sd)
+ if undotted:
+ raise ValueError(
+ f"{src}: root-level quantized weights cannot be flattened: {undotted[:8]}"
+ )
+ roots = ps._root_level_keys(flat_sd)
+ quantizable = {k: v for k, v in flat_sd.items() if k not in set(roots)}
+ flat, torchao_metadata = helpers[0](quantizable)
+ header = dict(torchao_metadata or {})
+ header[ps.UNSLOTH_FORMAT_KEY] = str(fmt)
+ header[ps.UNSLOTH_METADATA_KEY] = json.dumps(metadata, default = str)
+ header[QUANT_LAYOUT_KEY] = json.dumps(layout)
+ header.update(extra)
+ if roots:
+ flat = dict(flat)
+ for key in roots:
+ flat[f"{ps.UNSLOTH_ROOT_PREFIX}{key}"] = flat_sd[key].contiguous()
+ header[ps.UNSLOTH_ROOT_KEYS_KEY] = json.dumps([str(k) for k in roots])
+ save_file(_untie(dict(flat)), dst, metadata = header)
+ back = ps.load_prequant_safetensors(dst, mmap = True)
+ # Bit-identity against the .pt as THIS install reads it (v1 on <= 0.17 via the native v1 rebuild).
+ bad = compare_state_dicts(state_dict, back["state_dict"])
+ if bad:
+ raise AssertionError(f"{dst}: {len(bad)} tensors differ from {src}, e.g. {bad[:3]}")
+ if back["format"] != fmt or back["metadata"] != metadata:
+ raise AssertionError(f"{dst}: format or metadata changed in the round trip")
+ rec = {
+ "src": src,
+ "dst": dst,
+ "kind": "text_encoder" if is_te else "transformer",
+ "format": fmt,
+ "scheme": metadata.get("scheme"),
+ "tensors": len(state_dict),
+ "quantized": sum(1 for v in state_dict.values() if type(v) is not torch.Tensor),
+ "int8_source": None if is_te else layout.get("int8_source"),
+ "reader": back.get("reader"),
+ "src_bytes": source["bytes"],
+ "src_sha256": source["sha256"],
+ "dst_bytes": os.path.getsize(dst),
+ "dst_sha256": sha256_of(dst),
+ "seconds": round(time.time() - t0, 1),
+ "verified_bit_identical": True,
+ }
+ return rec
+
+
+def main(argv = None) -> int:
+ p = argparse.ArgumentParser(description = __doc__.split("\n\n")[0])
+ p.add_argument("src", nargs = "+")
+ p.add_argument("--out-dir", required = True)
+ p.add_argument(
+ "--repo", default = None, help = "owner/name the .pt came from (mirror layout + provenance)"
+ )
+ p.add_argument("--revision", default = None)
+ p.add_argument("--sha256", default = None, help = "known sha256 of a single SRC (skips hashing it)")
+ p.add_argument("--report", default = None, help = "append one JSON line per file here")
+ args = p.parse_args(argv)
+ sys.path.insert(0, str(BACKEND))
+ for src in args.src:
+ rec = convert(
+ src,
+ args.out_dir,
+ repo = args.repo,
+ revision = args.revision,
+ sha = args.sha256 if len(args.src) == 1 else None,
+ )
+ line = json.dumps(rec)
+ print(line, flush = True)
+ if args.report:
+ with open(args.report, "a") as fh:
+ fh.write(line + "\n")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.json
index b29ce92b528..40a6ae362e5 100644
--- a/scripts/risky_loaders_baseline.json
+++ b/scripts/risky_loaders_baseline.json
@@ -82,6 +82,14 @@
"count": 1,
"reason": "Module name comes from a fixed table or constant in this file, not from a model, request or download."
},
+ {
+ "file": "studio/backend/core/inference/diffusion_int8_fused.py",
+ "rule": "dynamic-import",
+ "sink": "import_module",
+ "digest": "8a43c55c018acccf",
+ "count": 1,
+ "reason": "Imports a name from the constant _TORCHAO_INTMM_MODULES tuple (torchao's safe_int_mm homes); never a model, request or download name."
+ },
{
"file": "studio/backend/core/inference/diffusion_prequant.py",
"rule": "dynamic-import",
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 5f0ca4759ba..640405d3b5d 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -1919,8 +1919,12 @@ def _dense_fast_path_reason(
kind: str,
path_override: Optional[str],
loras: Any = None,
+ failure: Optional[str] = None,
) -> str:
- """Name an unreadable hosted checkpoint only when it was in play (not override/GGUF/LoRA bake)."""
+ """Name an unreadable hosted checkpoint only when it was in play (not override/GGUF/LoRA bake).
+
+ ``failure`` is why a pre-quantized checkpoint that WAS tried did not load (``last_prequant_failure``):
+ planning thought it readable, so only the load can say what went wrong."""
note = (
prequant_unreadable_reason(fam, scheme, base_repo = base)
if kind == "pipeline" and not path_override and not _has_active_lora(loras)
@@ -1928,6 +1932,11 @@ def _dense_fast_path_reason(
)
if note:
return f"engaged on the dense fast path; {note}, so the dense bf16 transformer was quantized instead"
+ if failure:
+ return (
+ f"engaged on the dense fast path; the pre-quantized {scheme} checkpoint did not load "
+ f"({failure}), so the dense bf16 transformer was quantized instead"
+ )
return "engaged on the dense fast path"
@@ -5344,6 +5353,8 @@ class DiffusionBackend:
self._active_generate_cancel.set()
self._reserve_teardown_locked()
with self._model_transition_slot():
+ # Per load: a previous load's pre-quant failure must never reach this one's status.
+ self._prequant_fallback_note = None
with self._lock:
try:
self._raise_if_load_cancelled(_load_token)
@@ -6388,9 +6399,20 @@ class DiffusionBackend:
)
else:
# The plan was priced on the seed landing, so re-plan at bf16 without it.
+ from utils.native_path_leases import redact_native_paths
+
+ from .diffusion_prequant import last_prequant_failure
+
+ failure = last_prequant_failure()
+ self._prequant_fallback_note = (
+ redact_native_paths(failure)
+ if failure
+ else "the checkpoint was unavailable"
+ )
logger.warning(
"diffusion.denoiser_prequant: no pre-quantized denoiser was "
- "seeded; re-planning memory at the released bf16 size"
+ "seeded (%s); re-planning memory at the released bf16 size",
+ self._prequant_fallback_note,
)
pipeline_seed_scheme = None
plan = bf16_pipeline_plan
@@ -7243,6 +7265,7 @@ class DiffusionBackend:
kind,
transformer_prequant_path,
loras,
+ failure = getattr(self, "_prequant_fallback_note", None),
),
# Honored when the quant engaged AND when the ask was "off" (a request NOT to
# quantise, which the GGUF build satisfies)
@@ -7545,6 +7568,7 @@ class DiffusionBackend:
self._raise_if_load_cancelled(_load_token)
check_cancelled()
+ self._prequant_fallback_note = None
fetch_base = fetch_base or prefer_ungated_mirror(base, hf_token)
# 1. Pre-quantized checkpoint, when one is configured for the resolved scheme.
scheme = _planned_quant_scheme(
@@ -7593,6 +7617,25 @@ class DiffusionBackend:
placement_device = None if seed_device == device else seed_device,
)
check_cancelled()
+ if transformer is None:
+ # Never a silent swap: the status names why the checkpoint was not used.
+ from utils.native_path_leases import redact_native_paths
+
+ from .diffusion_prequant import last_prequant_failure
+
+ failure = last_prequant_failure()
+ self._prequant_fallback_note = (
+ redact_native_paths(failure)
+ if failure
+ else "the checkpoint was unavailable"
+ )
+ logger.warning(
+ "diffusion.prequant: %s checkpoint from %s not used (%s); quantizing the dense "
+ "transformer instead",
+ scheme,
+ getattr(source, "location", "?"),
+ self._prequant_fallback_note,
+ )
if transformer is not None:
if scheme == TQ_NVFP4:
from .diffusion_nvfp4_linear import nvfp4_prewarm
@@ -7619,8 +7662,11 @@ class DiffusionBackend:
if not allow_dense_fallback:
# The plan only budgeted the prequant-sized build, so the dense bf16 transformer would exceed it after
# eviction.
+ note = getattr(self, "_prequant_fallback_note", None)
raise RuntimeError(
- "prequant checkpoint unavailable and the dense transformer does not fit resident"
+ "prequant checkpoint unavailable"
+ + (f" ({note})" if note else "")
+ + " and the dense transformer does not fit resident"
)
# Deliberately the hub id, not base_local_dir: diffusers treats a local directory as terminal (_get_model_file
# raises rather than falling back to the hub) and a sharded load raises per missing shard, so a partial
diff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.py
index 28b4e7c60e2..588f8b1809e 100644
--- a/studio/backend/core/inference/diffusion_int8_fused.py
+++ b/studio/backend/core/inference/diffusion_int8_fused.py
@@ -36,6 +36,8 @@ _SWIGLU_ATTR = "_unsloth_i8_swiglu"
_LOCK = threading.Lock()
# Read by the traced forwards: dynamo must not trace into the lru_cache'd registration.
_OP_HANDLE: Any = None
+# torchao's safe_int_mm home, resolved at install (outside any trace) for the same reason.
+_INTMM_MODULE: Any = None
# Marker on each patched module (no global registry: it would pin an unloaded transformer).
_MARK = "_unsloth_i8_fused_prev"
_NO_PREV = object()
@@ -837,9 +839,31 @@ def _act_quant(x2d: Any, weight: Any) -> tuple:
return act.qdata, act.scale
+def _resolve_intmm() -> Any:
+ """The module holding torchao's ``safe_int_mm``, or None. torchao <= 0.18 ships it in ``torchao.kernel.intmm``;
+ main after pytorch/ao#4718 moved it to the int8 workflow and deleted ``torchao.kernel``. Read off the module at
+ call time, so Studio's capture-safe rebinding (diffusion_torchao_patches) is the one that runs."""
+ import importlib
+
+ from .diffusion_torchao_patches import _TORCHAO_INTMM_MODULES
+
+ for name in _TORCHAO_INTMM_MODULES:
+ try:
+ module = importlib.import_module(name)
+ except ImportError:
+ continue
+ if callable(getattr(module, "safe_int_mm", None)):
+ return module
+ return None
+
+
def _int_mm(a: Any, weight: Any) -> Any:
- from torchao.kernel.intmm import safe_int_mm
- return safe_int_mm(a, weight.qdata.contiguous().t())
+ module = _INTMM_MODULE
+ if module is None: # a direct call before any install (tests): resolve eagerly
+ module = _resolve_intmm()
+ if module is None:
+ raise ImportError("torchao ships no safe_int_mm in any known module")
+ return module.safe_int_mm(a, weight.qdata.contiguous().t())
def _linear_from_q(q: Any, xs: Any, weight: Any, bias: Any, out_dtype: Any) -> Any:
@@ -1233,13 +1257,19 @@ def _has_eligible(transformer: Any) -> bool:
def _finalize(transformer: Any, logger: Any = None) -> int:
- global _OP_HANDLE
+ global _OP_HANDLE, _INTMM_MODULE
dev = resident_cuda_device(transformer)
if dev is None:
return 0
# Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.
if not _has_eligible(transformer):
return 0
+ intmm = _resolve_intmm()
+ if (
+ intmm is None
+ ): # an int8 GEMM home this file does not know: keep the stock path rather than fail the render
+ return 0
+ _INTMM_MODULE = intmm
import torch
if not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index 23ce425949b..a4011a741f9 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -29,6 +29,7 @@ caller falls back to dense-quantise (then GGUF). Inert with nothing configured.
from __future__ import annotations
+import re as _re
import threading as _threading
from dataclasses import dataclass
from collections.abc import Sequence
@@ -338,12 +339,12 @@ def _load_prequant_checkpoint(path: str, **kwargs: Any) -> Any:
Dispatched on the file extension rather than on content sniffing: the extension is what the
resolver asked the Hub for, so a repo hosting both containers cannot serve one and be parsed as
- the other. ``kwargs`` are the pickle path's (``map_location``, ``mmap``); safetensors has no
- equivalent knobs and reads to CPU, which is where the pickle path maps too."""
+ the other. ``kwargs`` are the pickle path's (``map_location``, ``mmap``); safetensors reads to
+ CPU, which is where the pickle path maps too, and honours ``mmap`` the same way."""
from .prequant_safetensors import is_safetensors_checkpoint, load_prequant_safetensors
if is_safetensors_checkpoint(path):
- return load_prequant_safetensors(path)
+ return load_prequant_safetensors(path, mmap = bool(kwargs.get("mmap")))
return _torch_load_prequant(path, **kwargs)
@@ -432,6 +433,65 @@ def local_prequant_path_ready(path: str) -> bool:
return os.path.isfile(os.path.expanduser(path))
+# Operator-side mirror checked BEFORE the Hub: ``<root>/<owner>/<repo>/<filename>``, roots split by ``os.pathsep``.
+# Names come from the family tables, never a request; anything leaving the root is ignored.
+PREQUANT_MIRROR_ENV = "UNSLOTH_DIFFUSION_PREQUANT_MIRROR"
+
+
+def prequant_mirror_path(
+ repo_id: Optional[str],
+ name: Optional[str],
+ readable: Any = None,
+) -> Optional[str]:
+ """``<mirror>/<repo_id>/<name>`` when a configured mirror holds that file, else None. Never raises.
+
+ A converted safetensors copy of a requested pickle wins over the pickle itself (same weights, no
+ constructor allowlist), but only when ``readable(<sibling name>)`` says this install can open it:
+ otherwise a host without torchao's flatten helpers would be handed a file it then refuses, after
+ planning had already counted it as cached. ``readable`` defaults to "yes"."""
+ import os
+
+ raw = (os.environ.get(PREQUANT_MIRROR_ENV) or "").strip()
+ if not raw or not repo_id or not name:
+ return None
+ parts = [*str(repo_id).split("/"), *str(name).split("/")]
+ if any(p in ("", ".", "..") or "\\" in p for p in parts):
+ return None
+ wanted = [parts]
+ for suffix in (".pt", ".pth"):
+ if parts[-1].endswith(suffix):
+ sibling = parts[-1][: -len(suffix)] + ".safetensors"
+ try:
+ ok = readable is None or bool(readable(sibling))
+ except Exception: # noqa: BLE001 - an unanswerable question is a no
+ ok = False
+ if ok:
+ wanted.insert(0, [*parts[:-1], sibling])
+ break
+ for root in raw.split(os.pathsep):
+ root = root.strip()
+ if not root:
+ continue
+ for rel in wanted:
+ try:
+ base = os.path.realpath(os.path.expanduser(root))
+ candidate = os.path.realpath(os.path.join(base, *rel))
+ except Exception: # noqa: BLE001 - a bad root is simply not a mirror
+ break
+ if candidate.startswith(base.rstrip(os.sep) + os.sep) and os.path.isfile(candidate):
+ return candidate
+ return None
+
+
+def _first_mirrored(repo_id: Optional[str], names: Sequence[str], readable: Any) -> Optional[str]:
+ """The mirror's copy of the first of ``names`` it holds, checked for ALL names before any Hub call."""
+ for name in names:
+ hit = prequant_mirror_path(repo_id, name, readable)
+ if hit is not None:
+ return hit
+ return None
+
+
@dataclass(frozen = True)
class PrequantSource:
"""Where a pre-quantized checkpoint lives. ``kind`` is "path" (a local file) or "repo" (Hub repo
@@ -1049,15 +1109,23 @@ def cached_checkpoint_path(
``hf_hub_download`` falls back to huggingface_hub's import-time constant. Never raises."""
roots = (cache_dir, None) if cache_dir else (None,)
wanted = set(names) if names is not None else None
- for name in candidate_filenames_of(source):
- if wanted is not None and name not in wanted:
- continue
+
+ def _readable(name: str) -> bool:
try:
- readable = restricted_prequant_load_supported(None, name)
+ return bool(restricted_prequant_load_supported(None, name))
except Exception: # noqa: BLE001 - a pure lookup that never raises, as documented above
- readable = True
- if not readable:
- continue
+ return True
+
+ candidates = [
+ n
+ for n in candidate_filenames_of(source)
+ if (wanted is None or n in wanted) and _readable(n)
+ ]
+ if getattr(source, "kind", None) == "repo":
+ mirrored = _first_mirrored(getattr(source, "location", None), candidates, _readable)
+ if mirrored is not None:
+ return mirrored
+ for name in candidates:
for root in roots:
hit = _cached_in_root(source, root, name)
if hit is not None:
@@ -1437,6 +1505,12 @@ def _resolve_checkpoint_path(
names = readable or names
if not names:
return None
+ # Mirror first for EVERY name: else a Hub copy of an earlier name is downloaded and the mirror never used.
+ mirrored = _first_mirrored(
+ source.location, names, lambda n: restricted_prequant_load_supported(scheme, n)
+ )
+ if mirrored is not None:
+ return mirrored
for index, name in enumerate(names):
last = index == len(names) - 1
try:
@@ -2232,10 +2306,15 @@ def _has_meta_tensors(module: Any) -> bool:
_LAST_FAILURE = _threading.local()
+# An absolute POSIX or Windows path; the status keeps only its last component (the server's layout is not the
+# client's business). Not after a word, ':' or '/', so URLs and repo ids are left alone.
+_ABS_PATH = _re.compile(
+ r"(?<![\w:/.])(?:[A-Za-z]:[\\/]|/)(?:[^\s'\"<>|:;,()\[\]\\/]+[\\/])+(?=[^\s\\/])"
+)
def _warn(logger: Any, what: str, exc: Exception) -> None:
- _LAST_FAILURE.text = f"{type(exc).__name__}: {exc}"[:300]
+ _LAST_FAILURE.text = _ABS_PATH.sub("", f"{type(exc).__name__}: {exc}")[:300]
if logger is not None:
logger.warning("diffusion.prequant: %s failed: %s", what, exc)
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index 696c38907fe..2b585d49e6d 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -636,6 +636,12 @@ def _resolve_checkpoint_path(
)
names = [n for n in te_candidate_filenames(source) if te_candidate_is_readable(n)]
last: Optional[Exception] = None
+ from .diffusion_prequant import _first_mirrored
+
+ # The operator's mirror answers before the Hub is asked for any name (and so also offline).
+ mirrored = _first_mirrored(source.location, names, te_candidate_is_readable)
+ if mirrored is not None:
+ return mirrored
for name in names:
try:
return hf_hub_download(
diff --git a/studio/backend/core/inference/diffusion_torchao_patches.py b/studio/backend/core/inference/diffusion_torchao_patches.py
index 2916fd82f6b..5fdf3e24f9d 100644
--- a/studio/backend/core/inference/diffusion_torchao_patches.py
+++ b/studio/backend/core/inference/diffusion_torchao_patches.py
@@ -248,10 +248,11 @@ def install_torchao_int_mm_patch():
# inside ``dispatch_torchao`` for every LoRA target, torchao 0.18 deleted it, and Studio installs torchao
# 0.18 on torch >= 2.12, so every diffusion LoRA load raised. Both copies mark their wrapper
# ``__unsloth_patched__``, so whichever runs second leaves the first in place.
-# Matches both spellings a torchao removal produces: the class name, and the module that used to
-# define it. Only these two, so a torchao that is BROKEN rather than newer still raises.
+# The spellings a torchao removal produces (class, its old module, the whole ``torchao.dtypes`` package on main).
+# Only these, so a BROKEN torchao still raises.
_PEFT_TORCHAO_MISSING_TENSOR_SUBCLASS = re.compile(
- r"linear_?activation_?quantized_?tensor|affine_?quantized_?tensor",
+ r"linear_?activation_?quantized_?tensor|affine_?quantized_?tensor"
+ r"|no module named '?torchao\.dtypes'?(?![.\w])",
re.IGNORECASE | re.DOTALL,
)
diff --git a/studio/backend/core/inference/prequant_native.py b/studio/backend/core/inference/prequant_native.py
new file mode 100644
index 00000000000..15a33c24e6e
--- /dev/null
+++ b/studio/backend/core/inference/prequant_native.py
@@ -0,0 +1,436 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Rebuild a safetensors pre-quant checkpoint's int8 / fp8 weights without torchao's deserializer.
+
+The file layout is torchao's own (``flatten_tensor_state_dict``): ``<fqn>._weight_qdata`` and
+``<fqn>._weight_scale`` plus one JSON entry per weight. Reading it back through torchao's
+``unflatten_tensor_state_dict`` ties every load to that release's constructor kwargs, and that has
+already broken once (0.18 wrote ``reduce_range``, which 0.17 refuses). The two schemes Unsloth ships
+are simple enough to describe without torchao:
+
+ int8 w[n, k] = qdata[n, k] (int8) * scale[n] symmetric, per output row
+ activations: dynamic symmetric per-token int8
+ fp8 w[n, k] = qdata[n, k] (float8_e4m3fn) * scale[n] per output row (or one scale per tensor)
+ activations: dynamic fp8 with the recorded granularity
+
+so this module parses the JSON itself, validates the tensors against that description, and builds
+whatever class the INSTALLED torchao uses for the scheme. In particular an int8 artifact converted from
+a torchao v1 checkpoint (``unsloth_quant_layout.int8_source == "torchao_v1"``) is rebuilt as the v1
+``LinearActivationQuantizedTensor`` where torchao still ships it (<= 0.17), which makes it
+bit-identical, class for class, to reading the original ``.pt`` there; 0.18 and later get the
+``Int8Tensor`` the legacy-pickle decoder produces. Anything this module does not recognise
+returns None and the caller uses torchao's own reader, so nvfp4 / mxfp8 artifacts are unaffected.
+
+Kill switch: ``UNSLOTH_PREQUANT_NATIVE_REBUILD=0`` always uses torchao's reader.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+from typing import Any, Optional
+
+NATIVE_REBUILD_ENV = "UNSLOTH_PREQUANT_NATIVE_REBUILD"
+
+# Header keys the converter adds beside torchao's. Namespaced like the other unsloth_* keys, so torchao's
+# ``is_metadata_torchao`` (which walks ``tensor_names`` only) and every released Studio build ignore them.
+QUANT_LAYOUT_KEY = "unsloth_quant_layout"
+SOURCE_KEY = "unsloth_source"
+QUANT_LAYOUT_VERSION = 1
+INT8_SOURCE_V1 = "torchao_v1"
+
+# What a torchao v1 int8 pre-quant checkpoint holds, as validated by prequant_legacy_int8._rebuild_weight
+# and recorded by the converter. A file declaring anything else is not rebuilt as v1.
+INT8_V1_FACTS = {
+ "act_quant": "_int8_symm_per_token_reduced_range_quant",
+ "quant_kwargs": {},
+ "zero_point": None,
+ "zero_point_domain": "NONE",
+ "quant_min": None,
+ "quant_max": None,
+ "scale_shape": "out",
+ "layout": "PlainLayout",
+}
+
+# Unknown fields carrying one of these are behaviourally absent (same rule as prequant_safetensors).
+_INERT = (False, None, 0)
+
+_DTYPES = ("bfloat16", "float16", "float32", "float8_e4m3fn", "float8_e5m2", "int8")
+
+
+class _Unsupported(Exception):
+ """This entry needs torchao's own reader; not an error."""
+
+
+def native_rebuild_enabled() -> bool:
+ raw = (os.environ.get(NATIVE_REBUILD_ENV) or "").strip().lower()
+ return raw not in ("0", "false", "no", "off")
+
+
+def _torchao_api() -> Optional[dict]:
+ """The torchao names the rebuild needs, or None (absent / stubbed torchao)."""
+ try:
+ from core._torchao_stub import is_stubbed
+ if is_stubbed("torchao"):
+ return None
+ except Exception: # noqa: BLE001 - no stub module means nothing is stubbed
+ pass
+ try:
+ from torchao.float8.inference import Float8MMConfig
+ from torchao.quantization import Float8Tensor, PerRow, PerTensor
+ from torchao.quantization.quant_primitives import MappingType
+ from torchao.quantization.quantize_.common.kernel_preference import KernelPreference
+ from torchao.quantization.quantize_.workflows.float8.float8_tensor import (
+ QuantizeTensorToFloat8Kwargs,
+ )
+ except Exception: # noqa: BLE001
+ return None
+ api = {
+ "Float8Tensor": Float8Tensor,
+ "Float8MMConfig": Float8MMConfig,
+ "QuantizeTensorToFloat8Kwargs": QuantizeTensorToFloat8Kwargs,
+ "KernelPreference": KernelPreference,
+ "MappingType": MappingType,
+ "PerRow": PerRow,
+ "PerTensor": PerTensor,
+ }
+ try:
+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import (
+ Int8Tensor,
+ QuantizeTensorToInt8Kwargs,
+ )
+ api["Int8Tensor"] = Int8Tensor
+ api["QuantizeTensorToInt8Kwargs"] = QuantizeTensorToInt8Kwargs
+ except Exception: # noqa: BLE001 - no Int8Tensor: int8 entries go to torchao's reader
+ pass
+ return api
+
+
+def _v1_int8_api() -> Optional[dict]:
+ """torchao's v1 int8 classes when this release still ships ALL of them (<= 0.17), else None."""
+ try:
+ from torchao.dtypes.affine_quantized_tensor import AffineQuantizedTensor
+ from torchao.dtypes.uintx.plain_layout import PlainAQTTensorImpl
+ from torchao.dtypes.utils import PlainLayout
+ from torchao.quantization.linear_activation_quantized_tensor import (
+ LinearActivationQuantizedTensor,
+ )
+ from torchao.quantization.quant_api import _int8_symm_per_token_reduced_range_quant
+ from torchao.quantization.quant_primitives import ZeroPointDomain
+ except Exception: # noqa: BLE001
+ return None
+ return {
+ "AffineQuantizedTensor": AffineQuantizedTensor,
+ "PlainAQTTensorImpl": PlainAQTTensorImpl,
+ "PlainLayout": PlainLayout,
+ "LinearActivationQuantizedTensor": LinearActivationQuantizedTensor,
+ "act_quant": _int8_symm_per_token_reduced_range_quant,
+ "ZeroPointDomain": ZeroPointDomain,
+ }
+
+
+def _construct(cls: Any, data: dict, what: str) -> Any:
+ """``cls(**data)`` keeping only fields ``cls`` has; an unknown field must be inert to be dropped."""
+ if dataclasses.is_dataclass(cls):
+ known = {f.name for f in dataclasses.fields(cls)}
+ elif hasattr(cls, "_fields"):
+ known = set(cls._fields)
+ else:
+ raise _Unsupported(f"{what}: {cls!r} is neither a dataclass nor a NamedTuple")
+ extra = {k: v for k, v in data.items() if k not in known}
+ live = {k: v for k, v in extra.items() if v not in _INERT}
+ if live:
+ name = sorted(live)[0]
+ raise ValueError(
+ f"{what} records {name}={live[name]!r}, which this torchao cannot construct; "
+ "upgrade torchao to read this checkpoint"
+ )
+ return cls(**{k: v for k, v in data.items() if k in known})
+
+
+def _decode(value: Any, api: dict, what: str) -> Any:
+ """torchao's JSON encoding of one attribute, decoded against a FIXED table of types."""
+ if isinstance(value, list):
+ return [_decode(v, api, what) for v in value]
+ if not isinstance(value, dict) or "_type" not in value:
+ return value
+ kind, data = value.get("_type"), value.get("_data")
+ if kind == "torch.dtype":
+ import torch
+ if data not in _DTYPES:
+ raise _Unsupported(f"{what}: dtype {data!r}")
+ return getattr(torch, data)
+ if kind in ("MappingType", "KernelPreference"):
+ enum_cls = api[kind]
+ if not isinstance(data, str) or not hasattr(enum_cls, data):
+ raise _Unsupported(f"{what}: {kind} {data!r}")
+ return getattr(enum_cls, data)
+ if kind in ("PerRow", "PerTensor") and data is not None and not isinstance(data, dict):
+ raise _Unsupported(f"{what}: {kind} {data!r}")
+ if kind in ("PerRow", "PerTensor") and set(data or {}) - (
+ {"dim"} if kind == "PerRow" else set()
+ ):
+ raise _Unsupported(f"{what}: {kind} fields {sorted(set(data) - {'dim'})}")
+ if kind == "PerRow":
+ dim = (data or {}).get("dim", -1)
+ try:
+ return api["PerRow"](dim = dim)
+ except TypeError:
+ if dim != -1:
+ raise _Unsupported(f"{what}: PerRow(dim={dim}) on a torchao without dim") from None
+ return api["PerRow"]()
+ if kind == "PerTensor":
+ return api["PerTensor"]()
+ if kind in ("Float8MMConfig", "QuantizeTensorToFloat8Kwargs", "QuantizeTensorToInt8Kwargs"):
+ if kind not in api or not isinstance(data, dict):
+ raise _Unsupported(f"{what}: {kind}")
+ fields = {k: _decode(v, api, what) for k, v in data.items()}
+ return _construct(api[kind], fields, f"{what} {kind}")
+ raise _Unsupported(f"{what}: attribute type {kind!r}")
+
+
+def _flat_key(name: str, data_name: str) -> str:
+ module_fqn, weight_name = name.rsplit(".", 1)
+ return f"{module_fqn}._{weight_name}_{data_name}"
+
+
+def _granularity_name(g: Any) -> str:
+ return type(g).__name__
+
+
+def _rebuild_int8(
+ name: str, entry: dict, tensors: dict, used: set, api: dict, v1: Optional[dict]
+) -> Any:
+ import torch
+
+ data = entry.get("_data") or {}
+ data_names = list(entry.get("_tensor_data_names") or [])
+ if set(data_names) - {"qdata", "scale", "zero_point"} or not {"qdata", "scale"} <= set(
+ data_names
+ ):
+ raise _Unsupported(f"{name}: int8 tensor data {data_names}")
+ keys = {d: _flat_key(name, d) for d in data_names}
+ missing = [k for k in keys.values() if k not in tensors]
+ if missing:
+ raise ValueError(
+ f"int8 weight {name!r}: missing {missing[0]!r}; the checkpoint is incomplete"
+ )
+ qdata, scale = tensors[keys["qdata"]], tensors[keys["scale"]]
+ zero_point = tensors.get(keys["zero_point"]) if "zero_point" in keys else None
+ # Only the layout Unsloth ships is modelled here (2-D, symmetric, one scale per output row). Any other valid
+ # torchao int8 weight (per-tensor, 3-D experts) goes to torchao's own reader rather than being refused.
+ if qdata.dtype != torch.int8 or qdata.dim() != 2:
+ raise _Unsupported(f"{name}: int8 qdata {qdata.dtype} {tuple(qdata.shape)}")
+ n, k = qdata.shape
+ if not scale.is_floating_point() or scale.numel() != n:
+ raise _Unsupported(f"{name}: int8 scale {tuple(scale.shape)}")
+ if list(data.get("block_size") or []) != [1, k]:
+ raise _Unsupported(f"{name}: int8 block size {data.get('block_size')}")
+ known = {"block_size", "dtype", "act_quant_kwargs", "reduce_range"}
+ if any(v not in _INERT for k2, v in data.items() if k2 not in known):
+ raise _Unsupported(f"{name}: int8 attributes {sorted(set(data) - known)}")
+ if zero_point is not None and bool(torch.any(zero_point != 0)):
+ raise _Unsupported(f"{name}: asymmetric int8")
+ dtype = _decode(data.get("dtype"), api, name)
+ if dtype not in (torch.bfloat16, torch.float16, torch.float32):
+ raise _Unsupported(f"{name}: int8 output dtype {dtype}")
+ act = data.get("act_quant_kwargs")
+ act_fields = ((act or {}).get("_data") or {}) if act is not None else None
+ reduce_range = bool(data.get("reduce_range") or False)
+ if act_fields is not None:
+ gran = act_fields.get("granularity") or {}
+ mapping = act_fields.get("mapping_type") or {"_type": "MappingType", "_data": "SYMMETRIC"}
+ if gran.get("_type") != "PerRow" or (gran.get("_data") or {}).get("dim", -1) != -1:
+ raise _Unsupported(f"{name}: int8 activation granularity {gran}")
+ if mapping.get("_data") != "SYMMETRIC":
+ raise _Unsupported(f"{name}: int8 activation mapping {mapping}")
+ reduce_range = reduce_range or bool(act_fields.get("reduce_range") or False)
+ for d in keys.values():
+ used.add(d)
+
+ if v1 is not None and act_fields is not None and not reduce_range:
+ # Exactly the objects torchao <= 0.17 unpickles from the hosted .pt (see INT8_V1_FACTS).
+ impl = v1["PlainAQTTensorImpl"](qdata, scale.reshape(n), None, v1["PlainLayout"]())
+ aqt = v1["AffineQuantizedTensor"](
+ impl,
+ (1, k),
+ torch.Size((n, k)),
+ quant_min = None,
+ quant_max = None,
+ zero_point_domain = v1["ZeroPointDomain"].NONE,
+ dtype = dtype,
+ )
+ return v1["LinearActivationQuantizedTensor"](aqt, v1["act_quant"], {})
+
+ if "Int8Tensor" not in api:
+ raise _Unsupported(f"{name}: this torchao has no Int8Tensor")
+ import inspect
+
+ params = inspect.signature(api["Int8Tensor"].__new__).parameters
+ kwargs: dict = {}
+ if act_fields is not None:
+ kwargs["act_quant_kwargs"] = _decode(act, api, name)
+ if "reduce_range" in params:
+ kwargs["reduce_range"] = bool(data.get("reduce_range") or False)
+ elif data.get("reduce_range"):
+ raise ValueError(
+ f"int8 weight {name!r} records reduce_range=True, which this torchao cannot construct"
+ )
+ if zero_point is not None and "zero_point" in params:
+ kwargs["zero_point"] = zero_point
+ return api["Int8Tensor"](qdata, scale, [1, k], dtype, **kwargs)
+
+
+def _rebuild_fp8(name: str, entry: dict, tensors: dict, used: set, api: dict) -> Any:
+ import torch
+
+ data = entry.get("_data") or {}
+ data_names = list(entry.get("_tensor_data_names") or [])
+ if set(data_names) != {"qdata", "scale"}:
+ raise _Unsupported(f"{name}: fp8 tensor data {data_names}")
+ keys = {d: _flat_key(name, d) for d in data_names}
+ missing = [k for k in keys.values() if k not in tensors]
+ if missing:
+ raise ValueError(
+ f"fp8 weight {name!r}: missing {missing[0]!r}; the checkpoint is incomplete"
+ )
+ qdata, scale = tensors[keys["qdata"]], tensors[keys["scale"]]
+ # As for int8: anything but a 2-D per-row or per-tensor weight is torchao's to read.
+ if qdata.dtype not in (torch.float8_e4m3fn, torch.float8_e5m2) or qdata.dim() != 2:
+ raise _Unsupported(f"{name}: fp8 qdata {qdata.dtype} {tuple(qdata.shape)}")
+ n, k = qdata.shape
+ block = list(data.get("block_size") or [])
+ if not scale.is_floating_point() or not (
+ (block == [1, k] and scale.numel() == n) or (block == [n, k] and scale.numel() == 1)
+ ):
+ raise _Unsupported(f"{name}: fp8 block size {block} with scale {tuple(scale.shape)}")
+ known = {"block_size", "mm_config", "act_quant_kwargs", "kernel_preference", "dtype"}
+ extra = {k2: v for k2, v in data.items() if k2 not in known}
+ if any(v not in _INERT for v in extra.values()):
+ raise _Unsupported(f"{name}: fp8 attributes {sorted(extra)}")
+ dtype = _decode(data.get("dtype"), api, name)
+ mm_config = _decode(data.get("mm_config"), api, name)
+ act = _decode(data.get("act_quant_kwargs"), api, name)
+ if act is not None and _granularity_name(getattr(act, "granularity", None)) not in (
+ "PerRow",
+ "PerTensor",
+ ):
+ raise _Unsupported(f"{name}: fp8 activation granularity")
+ pref = _decode(data.get("kernel_preference"), api, name)
+ for d in keys.values():
+ used.add(d)
+ kwargs = {"block_size": block, "mm_config": mm_config, "act_quant_kwargs": act, "dtype": dtype}
+ if "kernel_preference" in data:
+ kwargs["kernel_preference"] = pref
+ return api["Float8Tensor"](qdata, scale, **kwargs)
+
+
+def read_quant_layout(raw: dict) -> dict:
+ try:
+ layout = json.loads(raw.get(QUANT_LAYOUT_KEY) or "{}")
+ except Exception: # noqa: BLE001 - a corrupt optional key is ignored, not fatal
+ return {}
+ return layout if isinstance(layout, dict) else {}
+
+
+def int8_rebuilds_as_v1(raw: dict) -> bool:
+ """Whether an int8 artifact converted from torchao v1 is rebuilt as v1 on THIS install."""
+ layout = read_quant_layout(raw)
+ if layout.get("int8_source") != INT8_SOURCE_V1:
+ return False
+ if (layout.get("int8_v1") or {}) != INT8_V1_FACTS:
+ return False
+ return _v1_int8_api() is not None
+
+
+def native_unflatten(
+ tensors: dict,
+ raw: dict,
+ *,
+ path: str = "",
+) -> Optional[dict]:
+ """``tensors`` + ``raw`` header -> state dict, or None when torchao's reader has to do it.
+
+ ``tensors`` is not modified when this returns None. Raises ValueError for a checkpoint that is
+ recognised but incomplete (a listed tensor or a weight part missing, a tensor the header does not
+ list, a live field this torchao lacks): torchao's reader would fail the same file, later and less
+ clearly. A weight of a layout this module does not model returns None instead.
+ """
+ api = _torchao_api()
+ if api is None:
+ return None
+ try:
+ names = json.loads(raw.get("tensor_names") or "null")
+ except Exception: # noqa: BLE001
+ return None
+ if (
+ not isinstance(names, list)
+ or not all(isinstance(n, str) and n for n in names)
+ or len(set(names)) != len(names)
+ ):
+ return None
+ v1 = _v1_int8_api() if int8_rebuilds_as_v1(raw) else None
+ out: dict = {}
+ used: set = set()
+ try:
+ for name in names:
+ try:
+ entry = json.loads(raw.get(name) or "null")
+ except Exception: # noqa: BLE001
+ raise _Unsupported(f"{name}: unreadable entry") from None
+ if not isinstance(entry, dict):
+ raise _Unsupported(f"{name}: no entry")
+ kind = entry.get("_type")
+ if kind == "Tensor":
+ if name not in tensors:
+ raise ValueError(f"{path} lists {name!r} but has no such tensor")
+ out[name] = tensors[name]
+ used.add(name)
+ elif "." not in name:
+ raise _Unsupported(f"{name}: root-level subclass")
+ elif kind == "Int8Tensor":
+ out[name] = _rebuild_int8(name, entry, tensors, used, api, v1)
+ elif kind == "Float8Tensor":
+ out[name] = _rebuild_fp8(name, entry, tensors, used, api)
+ else:
+ raise _Unsupported(f"{name}: {kind}")
+ except _Unsupported:
+ return None
+ leftover = sorted(set(tensors) - used)
+ if leftover:
+ raise ValueError(
+ f"{path} has {len(leftover)} tensor(s) its header does not account for "
+ f"(e.g. {leftover[0]!r}); the checkpoint is incomplete or was edited"
+ )
+ return out
+
+
+def quant_layout_header(state_dict: Any, *, int8_source: Optional[str] = None) -> dict:
+ """The ``unsloth_quant_layout`` description a converter writes for ``state_dict``."""
+ schemes = set()
+ for value in state_dict.values():
+ name = type(value).__name__
+ if name in ("Int8Tensor", "LinearActivationQuantizedTensor"):
+ schemes.add("int8")
+ elif name == "Float8Tensor":
+ schemes.add("fp8")
+ layout: dict = {"version": QUANT_LAYOUT_VERSION, "weights": {}}
+ if "int8" in schemes:
+ layout["weights"]["int8"] = {
+ "tensors": "<fqn>._weight_qdata int8 [out, in], <fqn>._weight_scale float [out, 1]",
+ "dequant": "weight = qdata * scale (symmetric, per output row, no zero point)",
+ "activation": "dynamic symmetric per-token int8",
+ }
+ if int8_source:
+ layout["int8_source"] = int8_source
+ if int8_source == INT8_SOURCE_V1:
+ layout["int8_v1"] = dict(INT8_V1_FACTS)
+ if "fp8" in schemes:
+ layout["weights"]["fp8"] = {
+ "tensors": "<fqn>._weight_qdata float8_e4m3fn [out, in], <fqn>._weight_scale float32 [out, 1]",
+ "dequant": "weight = qdata * scale (per output row)",
+ "activation": "dynamic float8 with the recorded granularity",
+ }
+ return layout
diff --git a/studio/backend/core/inference/prequant_safetensors.py b/studio/backend/core/inference/prequant_safetensors.py
index 630def0628a..febf12c9bca 100644
--- a/studio/backend/core/inference/prequant_safetensors.py
+++ b/studio/backend/core/inference/prequant_safetensors.py
@@ -47,6 +47,7 @@ a repo can host both and each build reads the one it understands.
from __future__ import annotations
import json
+import os
import re
from typing import Any, Iterable, Optional
@@ -61,6 +62,14 @@ UNSLOTH_METADATA_KEY = "unsloth_metadata"
# never sees a key it would try to rsplit on ".".
UNSLOTH_ROOT_PREFIX = "unsloth_root::"
UNSLOTH_ROOT_KEYS_KEY = "unsloth_root_tensors"
+# Our own header entries: never torchao tensor entries, so never pruned (layout + source: prequant_native).
+_UNSLOTH_HEADER_KEYS = (
+ UNSLOTH_FORMAT_KEY,
+ UNSLOTH_METADATA_KEY,
+ UNSLOTH_ROOT_KEYS_KEY,
+ "unsloth_quant_layout",
+ "unsloth_source",
+)
# The torchao release that first shipped the flatten/unflatten pair under this import path. Below it
# the helpers are absent and a safetensors artifact simply cannot be read, so the loader says so and
@@ -361,7 +370,7 @@ def _header_without_inert_tensor_field(
pruned = dict(header)
victims = []
for key, value in header.items():
- if key in (UNSLOTH_FORMAT_KEY, UNSLOTH_METADATA_KEY, UNSLOTH_ROOT_KEYS_KEY):
+ if key in _UNSLOTH_HEADER_KEYS:
continue
try:
parsed = json.loads(value)
@@ -433,7 +442,7 @@ def _header_without_unconstructible_fields(
removed: list = []
pruned = {}
for key, value in header.items():
- if key in (UNSLOTH_FORMAT_KEY, UNSLOTH_METADATA_KEY, UNSLOTH_ROOT_KEYS_KEY):
+ if key in _UNSLOTH_HEADER_KEYS:
pruned[key] = value
continue
try:
@@ -470,7 +479,106 @@ def _header_without_unconstructible_fields(
return header
-def load_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:
+_ST_DTYPES = {
+ "BF16": "bfloat16",
+ "F16": "float16",
+ "F32": "float32",
+ "F64": "float64",
+ "C64": "complex64",
+ "F8_E4M3": "float8_e4m3fn",
+ "F8_E5M2": "float8_e5m2",
+ "F8_E8M0": "float8_e8m0fnu",
+ "I8": "int8",
+ "U8": "uint8",
+ "I16": "int16",
+ "U16": "uint16",
+ "I32": "int32",
+ "U32": "uint32",
+ "I64": "int64",
+ "U64": "uint64",
+ "BOOL": "bool",
+}
+
+# safetensors refuses headers past 100 MB; a pre-quant header is a few MB at most.
+_MAX_HEADER_BYTES = 100_000_000 # the safetensors parser's own limit
+
+
+def _mapped_tensors(path: str) -> tuple:
+ """``(header metadata, {name: tensor})`` with every tensor a view of a private mapping of ``path``.
+
+ ``safe_open(...).get_tensor`` copies each tensor into anonymous memory, so a 34 GB checkpoint costs
+ 34 GB of host RAM before the first byte reaches the GPU; the pickle path maps the file instead
+ (``prequant_mmap_enabled``). This maps it with the same primitive ``torch.load(mmap = True)`` uses
+ (``UntypedStorage.from_file``, private), so the two containers cost the same on every OS: pages are
+ read when ``.to(device)`` touches them, and a write never reaches the file. The header is checked
+ the way safetensors checks it (known dtypes, non-negative integer shapes, byte ranges that tile the
+ data section exactly, a bounded header); anything else raises and the caller re-reads unmapped.
+ """
+ import struct
+
+ import torch
+
+ size = os.path.getsize(path)
+ with open(path, "rb") as fh:
+ head = fh.read(8)
+ if len(head) != 8:
+ raise ValueError(f"{path} is not a safetensors file")
+ (length,) = struct.unpack("<Q", head)
+ if length <= 0 or length > _MAX_HEADER_BYTES or 8 + length > size:
+ raise ValueError(f"{path} has a corrupt safetensors header")
+ header = json.loads(fh.read(length))
+ if not isinstance(header, dict):
+ raise ValueError(f"{path} has a corrupt safetensors header")
+ metadata = header.pop("__metadata__", None) or {}
+ base = 8 + length
+ entries = []
+ for name, info in header.items():
+ try:
+ dtype = getattr(torch, _ST_DTYPES[info["dtype"]])
+ start, end = (int(v) for v in info["data_offsets"])
+ shape = list(info["shape"])
+ except Exception: # noqa: BLE001 - unknown dtype (on this torch) or a malformed entry
+ raise ValueError(f"{path}: unreadable tensor entry {name!r}") from None
+ if not all(isinstance(d, int) and not isinstance(d, bool) and d >= 0 for d in shape):
+ raise ValueError(f"{path}: tensor {name!r} has an invalid shape")
+ count = 1
+ for d in shape:
+ count *= d
+ itemsize = torch.empty((), dtype = dtype).element_size()
+ if count * itemsize != end - start:
+ raise ValueError(f"{path}: tensor {name!r} size does not match its shape")
+ entries.append((start, end, name, dtype, shape, itemsize))
+ entries.sort(key = lambda e: (e[0], e[1]))
+ cursor = 0
+ for start, end, name, *_ in entries:
+ if start != cursor:
+ raise ValueError(f"{path}: tensor {name!r} overlaps another or leaves a gap")
+ cursor = end
+ if cursor != size - base:
+ raise ValueError(f"{path}: the data section does not match its header")
+ storage = (
+ torch.UntypedStorage.from_file(path, shared = False, nbytes = size) if size > base else None
+ )
+ tensors = {}
+ for start, end, name, dtype, shape, itemsize in entries:
+ offset = base + start
+ if start == end or storage is None:
+ tensors[name] = torch.empty(shape, dtype = dtype)
+ elif offset % itemsize == 0:
+ tensors[name] = torch.empty(0, dtype = dtype).set_(storage, offset // itemsize, shape)
+ else:
+ # safetensors packs tensors back to back, so one can start off its dtype's alignment: copy that one.
+ raw = torch.empty(0, dtype = torch.uint8).set_(storage, offset, (end - start,))
+ tensors[name] = raw.clone().view(dtype).reshape(shape)
+ return dict(metadata), tensors
+
+
+def load_prequant_safetensors(
+ path: str,
+ *,
+ device: str = "cpu",
+ mmap: bool = False,
+) -> dict:
"""Read ``path`` into the SAME dict shape the pickle path returns.
Returning ``{"format", "state_dict", "metadata"}`` rather than a new type is deliberate: every
@@ -478,6 +586,9 @@ def load_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:
activation-floor probe that reads the reconstructed tensors, the rotation biconditional, the
kernel-preference pin) then runs unchanged on both containers, so the two formats cannot drift
into having different acceptance rules.
+
+ ``mmap`` maps the file instead of copying it (CPU tensors only), as the pickle path does for an
+ accelerator destination.
"""
helpers = _torchao_helpers()
if helpers is None:
@@ -486,12 +597,14 @@ def load_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:
f"{'.'.join(str(p) for p in MIN_TORCHAO_VERSION)} is required for "
"torchao.prototype.safetensors.safetensors_support"
)
- from safetensors import safe_open
-
_, unflatten = helpers
- with safe_open(path, framework = "pt", device = device) as handle:
- raw = dict(handle.metadata() or {})
- tensors = {key: handle.get_tensor(key) for key in handle.keys()}
+ if mmap and str(device) == "cpu":
+ raw, tensors = _mapped_tensors(path)
+ else:
+ from safetensors import safe_open
+ with safe_open(path, framework = "pt", device = device) as handle:
+ raw = dict(handle.metadata() or {})
+ tensors = {key: handle.get_tensor(key) for key in handle.keys()}
fmt = raw.get(UNSLOTH_FORMAT_KEY)
if not fmt:
@@ -509,27 +622,31 @@ def load_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:
for key in [k for k in tensors if k.startswith(UNSLOTH_ROOT_PREFIX)]
}
- # A newer torchao can record a field an older one's constructor does not take, which is how a
- # published int8 checkpoint stopped loading. Dropped here when it is inert; see the helper.
- raw = _header_without_unconstructible_fields(unflatten, tensors, raw, path = path)
-
- # torchao reads its OWN keys out of the same header; ours are namespaced and simply ignored. The second element
- # is what it could NOT account for: a subclass missing one of its parts (a truncated or hand-edited file) is
- # skipped there rather than raised, which would reach load_state_dict as a bare missing-key error saying nothing
- # about the artifact. Name it here instead.
- rebuilt = unflatten(tensors, raw)
- state_dict = _first(rebuilt)
- leftover = rebuilt[1] if isinstance(rebuilt, tuple) and len(rebuilt) > 1 else None
- if leftover:
- raise ValueError(
- f"{path} has {len(leftover)} tensor(s) its header does not account for "
- f"(e.g. {sorted(leftover)[0]!r}); the checkpoint is incomplete or was edited"
- )
+ # Unsloth's own int8 / fp8 rebuild, independent of torchao's deserializer accepting another release's kwargs;
+ # None (nvfp4, mxfp8, ...) falls through to torchao below.
+ from .prequant_native import native_rebuild_enabled, native_unflatten
+
+ state_dict = native_unflatten(tensors, raw, path = path) if native_rebuild_enabled() else None
+ reader = "native" if state_dict is not None else "torchao"
+ if state_dict is None:
+ # A newer torchao's field an older constructor lacks (how a published int8 file broke): dropped if inert.
+ raw = _header_without_unconstructible_fields(unflatten, tensors, raw, path = path)
+
+ # The second element is what torchao could not account for (truncated / hand-edited file): named here, not left
+ # to surface as a bare missing-key error in load_state_dict.
+ rebuilt = unflatten(tensors, raw)
+ state_dict = _first(rebuilt)
+ leftover = rebuilt[1] if isinstance(rebuilt, tuple) and len(rebuilt) > 1 else None
+ if leftover:
+ raise ValueError(
+ f"{path} has {len(leftover)} tensor(s) its header does not account for "
+ f"(e.g. {sorted(leftover)[0]!r}); the checkpoint is incomplete or was edited"
+ )
if roots:
# Back under their bare names, so the caller's load_state_dict sees the shape the model
# declares. A dotted key can never collide with one of these.
state_dict.update(roots)
- return {"format": str(fmt), "state_dict": state_dict, "metadata": metadata}
+ return {"format": str(fmt), "state_dict": state_dict, "metadata": metadata, "reader": reader}
def scheme_is_flattenable(quant_config: Any, *, features: int = 512) -> Optional[bool]:
diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.py
index 0e72ca2c718..a3785c1ae99 100644
--- a/studio/backend/tests/test_diffusion_int8_fused.py
+++ b/studio/backend/tests/test_diffusion_int8_fused.py
@@ -76,6 +76,42 @@ def _clean_env(monkeypatch):
yield
+def test_int_mm_finds_safe_int_mm_after_torchao_kernel_was_removed(monkeypatch):
+ """torchao main moved ``safe_int_mm`` out of ``torchao.kernel.intmm`` (and deleted ``torchao.kernel``):
+ the fused forwards must find it in its new home, not fail every int8 render with ModuleNotFoundError."""
+ import sys
+ import types
+
+ calls = []
+ new_home = types.ModuleType("torchao.quantization.quantize_.workflows.int8.kernels")
+ new_home.safe_int_mm = lambda a, b: calls.append((a, b)) or "out"
+ monkeypatch.setitem(sys.modules, "torchao.kernel", None)
+ monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", None)
+ monkeypatch.setitem(sys.modules, new_home.__name__, new_home)
+ monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)
+ weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))
+ a = torch.ones(4, 2, dtype = torch.int8)
+ assert fused._int_mm(a, weight) == "out"
+ assert len(calls) == 1 and calls[0][0] is a and tuple(calls[0][1].shape) == (2, 3)
+
+
+def test_int_mm_prefers_the_released_home(monkeypatch):
+ """torchao <= 0.18 keeps ``torchao.kernel.intmm``: it wins over any other copy, so the capture-safe rebinding
+ Studio installs there is the one that runs."""
+ import sys
+ import types
+
+ old_home = types.ModuleType("torchao.kernel.intmm")
+ old_home.safe_int_mm = lambda a, b: "old"
+ new_home = types.ModuleType("torchao.quantization.quantize_.workflows.int8.kernels")
+ new_home.safe_int_mm = lambda a, b: "new"
+ monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", old_home)
+ monkeypatch.setitem(sys.modules, new_home.__name__, new_home)
+ monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)
+ weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))
+ assert fused._int_mm(torch.ones(4, 2, dtype = torch.int8), weight) == "old"
+
+
@pytest.mark.parametrize(
"version, ok",
[
diff --git a/studio/backend/tests/test_diffusion_peft_torchao_guard.py b/studio/backend/tests/test_diffusion_peft_torchao_guard.py
index b12d676d425..683949ff776 100644
--- a/studio/backend/tests/test_diffusion_peft_torchao_guard.py
+++ b/studio/backend/tests/test_diffusion_peft_torchao_guard.py
@@ -48,6 +48,21 @@ def test_studio_copy_matches_unsloth_import_fixes(name):
), f"{name} drifted from unsloth/import_fixes.py; port the change to both copies."
+def _assigned_value_dump(path: Path, name: str) -> str:
+ for node in ast.parse(path.read_text(encoding = "utf-8")).body:
+ if isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id == name for t in node.targets
+ ):
+ return ast.dump(node.value)
+ raise AssertionError(f"{path} has no top-level {name}")
+
+
+def test_studio_and_unsloth_match_the_same_torchao_removals():
+ """The matcher is shared state too: when unsloth patched first, its matcher is the one that runs."""
+ name = "_PEFT_TORCHAO_MISSING_TENSOR_SUBCLASS"
+ assert _assigned_value_dump(_PATCH_MODULE, name) == _assigned_value_dump(_IMPORT_FIXES, name)
+
+
@pytest.fixture(autouse = True)
def _restore_peft_dispatchers(monkeypatch):
"""The patcher wraps every loaded peft copy, real ones included. Put them back afterwards so a
@@ -139,6 +154,59 @@ def test_an_unrelated_import_error_still_surfaces(monkeypatch, fake_peft):
lora_model.dispatch_torchao(types.SimpleNamespace(weight = object()), "default", None)
+def test_a_torchao_without_the_dtypes_package_gets_a_plain_lora_layer(monkeypatch, fake_peft):
+ """torchao main (after 0.18) deleted the whole ``torchao.dtypes`` package, so peft <= 0.18's
+ first import inside ``dispatch_torchao`` is a ModuleNotFoundError naming the package, not the
+ class. That is the same removal, and every LoRA target must still fall through to peft's
+ ordinary layer instead of failing the load."""
+ lora_torchao, lora_model = fake_peft
+ monkeypatch.delitem(
+ sys.modules, "torchao.dtypes"
+ ) # the fake torchao has no __path__: the import fails
+ monkeypatch.delattr(sys.modules["torchao"], "dtypes")
+ monkeypatch.setattr(
+ sys.modules["torchao"], "__spec__", importlib.machinery.ModuleSpec("torchao", None)
+ )
+
+ # peft <= 0.18's first torchao import, defined in the fake peft module so the guard reads its globals.
+ namespace = dict(lora_torchao.__dict__)
+ exec(
+ "def dispatch_torchao(target, adapter_name, lora_config, **kwargs):\n"
+ " from torchao.dtypes import AffineQuantizedTensor\n",
+ namespace,
+ )
+ removed = namespace["dispatch_torchao"]
+ monkeypatch.setattr(lora_torchao, "dispatch_torchao", removed)
+ monkeypatch.setattr(lora_model, "dispatch_torchao", removed)
+ with pytest.raises(ModuleNotFoundError, match = "torchao.dtypes"):
+ removed(types.SimpleNamespace(weight = object()), "default", None)
+ patches._patch_peft_torchao_dispatchers()
+ assert (
+ lora_model.dispatch_torchao(types.SimpleNamespace(weight = object()), "default", None) is None
+ )
+
+
+ "message",
+ (
+ "No module named 'torchao'",
+ "No module named 'torchao.dtypes.uintx'",
+ "No module named 'torchao.dtypesx'",
+ ),
+)
+def test_other_missing_modules_still_surface(monkeypatch, fake_peft, message):
+ lora_torchao, lora_model = fake_peft
+
+ def broken(target, adapter_name, lora_config, **kwargs):
+ raise ModuleNotFoundError(message)
+
+ monkeypatch.setattr(lora_torchao, "dispatch_torchao", broken)
+ monkeypatch.setattr(lora_model, "dispatch_torchao", broken)
+ patches._patch_peft_torchao_dispatchers()
+ with pytest.raises(ModuleNotFoundError):
+ lora_model.dispatch_torchao(types.SimpleNamespace(weight = object()), "default", None)
+
+
def test_the_guard_waits_for_peft_instead_of_importing_it(monkeypatch):
monkeypatch.delitem(sys.modules, patches._PEFT_LORA_DISPATCH_MODULE, raising = False)
monkeypatch.setattr(
diff --git a/studio/backend/tests/test_prequant_native.py b/studio/backend/tests/test_prequant_native.py
new file mode 100644
index 00000000000..5a83b066593
--- /dev/null
+++ b/studio/backend/tests/test_prequant_native.py
@@ -0,0 +1,714 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Safetensors pre-quant checkpoints rebuilt without torchao's deserializer, the local mirror, and the converter."""
+
+from __future__ import annotations
+
+import importlib.util
+import inspect
+import json
+from pathlib import Path
+
+import pytest
+
+import core.inference.diffusion_prequant as pq
+import core.inference.prequant_native as pn
+import core.inference.prequant_safetensors as ps
+
+torch = pytest.importorskip("torch")
+torchao = pytest.importorskip("torchao")
+
+_HAVE_HELPERS = ps.safetensors_prequant_supported()
+needs_helpers = pytest.mark.skipif(
+ not _HAVE_HELPERS, reason = "needs torchao >= 0.16 flatten helpers"
+)
+_V1 = pn._v1_int8_api() is not None
+_API = pn._torchao_api() or {}
+needs_int8tensor = pytest.mark.skipif("Int8Tensor" not in _API, reason = "needs torchao Int8Tensor")
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+
+
+def _int8_tensor(
+ n = 8,
+ k = 32,
+ seed = 0,
+):
+ from core.inference.prequant_legacy_int8 import _int8_tensor_api, _to_int8_tensor
+
+ g = torch.Generator().manual_seed(seed)
+ qdata = torch.randint(-127, 128, (n, k), dtype = torch.int8, generator = g)
+ scale = (torch.rand(n, generator = g) / 50 + 1e-3).to(torch.bfloat16)
+ return _to_int8_tensor(qdata, scale, torch.bfloat16, _int8_tensor_api())
+
+
+def _fp8_tensor(
+ n = 8,
+ k = 32,
+ seed = 0,
+):
+ from torchao.float8.inference import Float8MMConfig
+ from torchao.quantization import Float8Tensor, PerRow
+ from torchao.quantization.quantize_.common.kernel_preference import KernelPreference
+ from torchao.quantization.quantize_.workflows.float8.float8_tensor import (
+ QuantizeTensorToFloat8Kwargs,
+ )
+
+ g = torch.Generator().manual_seed(seed)
+ qdata = (torch.randn(n, k, generator = g) * 50).to(torch.float8_e4m3fn)
+ scale = torch.rand(n, 1, generator = g) / 100 + 1e-4
+ act = QuantizeTensorToFloat8Kwargs(
+ float8_dtype = torch.float8_e4m3fn,
+ granularity = PerRow(),
+ mm_config = None,
+ hp_value_lb = 1e-12,
+ hp_value_ub = None,
+ kernel_preference = KernelPreference.AUTO,
+ )
+ return Float8Tensor(
+ qdata,
+ scale,
+ block_size = [1, k],
+ mm_config = Float8MMConfig(use_fast_accum = True),
+ act_quant_kwargs = act,
+ kernel_preference = KernelPreference.AUTO,
+ dtype = torch.bfloat16,
+ )
+
+
+def _write(
+ path,
+ state_dict,
+ *,
+ layout = None,
+ scheme = "int8",
+):
+ from safetensors.torch import save_file
+
+ flatten, _ = ps._torchao_helpers()
+ roots = ps._root_level_keys(state_dict)
+ flat, meta = flatten({k: v for k, v in state_dict.items() if k not in roots})
+ header = dict(meta)
+ header[ps.UNSLOTH_FORMAT_KEY] = pq.PREQUANT_FORMAT
+ header[ps.UNSLOTH_METADATA_KEY] = json.dumps({"scheme": scheme})
+ if layout is not None:
+ header[pn.QUANT_LAYOUT_KEY] = json.dumps(layout)
+ flat = dict(flat)
+ if roots:
+ for k in roots:
+ flat[ps.UNSLOTH_ROOT_PREFIX + k] = state_dict[k]
+ header[ps.UNSLOTH_ROOT_KEYS_KEY] = json.dumps(roots)
+ save_file({k: v.contiguous() for k, v in flat.items()}, str(path), metadata = header)
+ return path
+
+
+def _canon(value):
+ if type(value) is torch.Tensor:
+ return ("Tensor", value)
+ names, ctx = value.__tensor_flatten__()
+ return (type(value).__name__, {n: _canon(getattr(value, n)) for n in names}, repr(ctx))
+
+
+def _same(a, b):
+ if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):
+ return a.dtype == b.dtype and a.shape == b.shape and torch.equal(a, b)
+ if isinstance(a, (tuple, list)) and isinstance(b, (tuple, list)):
+ return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b, strict = True))
+ if isinstance(a, dict) and isinstance(b, dict):
+ return a.keys() == b.keys() and all(_same(a[k], b[k]) for k in a)
+ return a == b
+
+
+def _v1_layout():
+ return {"version": 1, "int8_source": pn.INT8_SOURCE_V1, "int8_v1": dict(pn.INT8_V1_FACTS)}
+
+
+@needs_helpers
+@needs_int8tensor
+def test_native_rebuild_matches_torchao_unflatten(tmp_path, monkeypatch):
+ sd = {
+ "blk.q.weight": _int8_tensor(),
+ "blk.f.weight": _fp8_tensor(),
+ "blk.norm.weight": torch.ones(32, dtype = torch.bfloat16),
+ "pad_token": torch.zeros(1, 32, dtype = torch.bfloat16),
+ }
+ path = _write(tmp_path / "M-INT8.safetensors", sd)
+ native = ps.load_prequant_safetensors(str(path))
+ monkeypatch.setenv(pn.NATIVE_REBUILD_ENV, "0")
+ stock = ps.load_prequant_safetensors(str(path))
+ assert native["reader"] == "native" and stock["reader"] == "torchao"
+ assert native["state_dict"].keys() == stock["state_dict"].keys() == sd.keys()
+ for key in sd:
+ assert _same(_canon(native["state_dict"][key]), _canon(stock["state_dict"][key])), key
+ assert _same(_canon(native["state_dict"][key]), _canon(sd[key])), key
+
+
+@needs_helpers
+@needs_int8tensor
+def test_mapped_read_is_the_same_tensors_as_a_copied_read(tmp_path):
+ sd = {"blk.q.weight": _int8_tensor(), "blk.f.weight": _fp8_tensor(), "x": torch.arange(4.0)}
+ path = _write(tmp_path / "M-INT8.safetensors", sd)
+ mapped = ps.load_prequant_safetensors(str(path), mmap = True)["state_dict"]
+ copied = ps.load_prequant_safetensors(str(path))["state_dict"]
+ for key in sd:
+ assert _same(_canon(mapped[key]), _canon(copied[key])), key
+
+
+def test_a_tensor_off_its_dtype_alignment_is_still_read(tmp_path):
+ a = torch.tensor([1, -2, 3], dtype = torch.int8)
+ b = torch.tensor([1.5, -0.25], dtype = torch.bfloat16)
+ header = {
+ "a": {"dtype": "I8", "shape": [3], "data_offsets": [0, 3]},
+ "b": {"dtype": "BF16", "shape": [2], "data_offsets": [3, 7]},
+ }
+ raw = json.dumps(header).encode()
+ raw += b" " * (-len(raw) % 8)
+ path = tmp_path / "odd.safetensors"
+ path.write_bytes(
+ len(raw).to_bytes(8, "little")
+ + raw
+ + a.numpy().tobytes()
+ + b.view(torch.int16).numpy().tobytes()
+ )
+ _, tensors = ps._mapped_tensors(str(path))
+ assert torch.equal(tensors["a"], a) and torch.equal(tensors["b"], b)
+
+
+def test_a_truncated_header_is_refused(tmp_path):
+ path = tmp_path / "bad.safetensors"
+ path.write_bytes((10_000).to_bytes(8, "little") + b"{}")
+ with pytest.raises(ValueError):
+ ps._mapped_tensors(str(path))
+
+
+@needs_helpers
+@needs_int8tensor
+def test_v1_converted_int8_rebuilds_as_v1_only_where_torchao_ships_it(tmp_path):
+ sd = {"blk.q.weight": _int8_tensor(), "blk.norm.weight": torch.ones(32, dtype = torch.bfloat16)}
+ path = _write(tmp_path / "M-INT8.safetensors", sd, layout = _v1_layout())
+ w = ps.load_prequant_safetensors(str(path))["state_dict"]["blk.q.weight"]
+ if _V1:
+ assert type(w).__name__ == "LinearActivationQuantizedTensor"
+ impl = w.original_weight_tensor.tensor_impl
+ assert torch.equal(impl.int_data, sd["blk.q.weight"].qdata)
+ assert torch.equal(impl.scale, sd["blk.q.weight"].scale.reshape(-1))
+ assert impl.zero_point is None
+ assert w.input_quant_func.__name__ == "_int8_symm_per_token_reduced_range_quant"
+ else:
+ assert type(w).__name__ == "Int8Tensor"
+ # Without the v1 declaration the artifact keeps the class it was written as, on every torchao.
+ plain = _write(tmp_path / "P-INT8.safetensors", sd)
+ assert (
+ type(ps.load_prequant_safetensors(str(plain))["state_dict"]["blk.q.weight"]).__name__
+ == "Int8Tensor"
+ )
+
+
[email protected](not _V1, reason = "needs torchao <= 0.17 (v1 int8 classes)")
+@needs_helpers
+def test_v1_rebuild_is_bit_identical_to_a_real_v1_quantize(tmp_path):
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ from core.inference.prequant_legacy_int8 import convert_legacy_int8_weights
+
+ model = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False)).to(torch.bfloat16)
+ quantize_(model, Int8DynamicActivationInt8WeightConfig())
+ v1 = model[0].weight
+ assert type(v1).__name__ == "LinearActivationQuantizedTensor"
+ reference = dict(model.state_dict())
+ converted = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False)).to(torch.bfloat16)
+ converted.load_state_dict(reference, assign = True)
+ assert convert_legacy_int8_weights(converted) == 1
+ path = _write(
+ tmp_path / "M-INT8.safetensors", dict(converted.state_dict()), layout = _v1_layout()
+ )
+ back = ps.load_prequant_safetensors(str(path))["state_dict"]
+ assert _same(_canon(back["0.weight"]), _canon(v1))
+ x = torch.randn(17, 64, dtype = torch.bfloat16)
+ assert torch.equal(
+ torch.nn.functional.linear(x, back["0.weight"]), torch.nn.functional.linear(x, v1)
+ )
+
+
+@needs_helpers
+@needs_int8tensor
+def test_unknown_entries_fall_back_to_torchao_and_leave_the_tensors_alone(tmp_path):
+ sd = {"blk.q.weight": _int8_tensor()}
+ flatten, _ = ps._torchao_helpers()
+ flat, meta = flatten(sd)
+ raw = dict(meta)
+ entry = json.loads(raw["blk.q.weight"])
+ entry["_type"] = "SomeFutureTensor"
+ raw["blk.q.weight"] = json.dumps(entry)
+ before = dict(flat)
+ assert pn.native_unflatten(flat, raw) is None
+ assert flat.keys() == before.keys()
+
+
+@needs_helpers
+@needs_int8tensor
+def test_a_missing_scale_or_a_stray_tensor_is_named(tmp_path):
+ flatten, _ = ps._torchao_helpers()
+ flat, meta = flatten({"blk.q.weight": _int8_tensor()})
+ no_scale = {k: v for k, v in flat.items() if not k.endswith("_scale")}
+ with pytest.raises(ValueError, match = "missing"):
+ pn.native_unflatten(no_scale, dict(meta), path = "x")
+ stray = dict(flat)
+ stray["blk.other._weight_qdata"] = torch.zeros(2, 2, dtype = torch.int8)
+ with pytest.raises(ValueError, match = "does not account for"):
+ pn.native_unflatten(stray, dict(meta), path = "x")
+
+
+@needs_helpers
+@needs_int8tensor
+def test_a_live_field_this_torchao_lacks_is_refused_and_an_inert_one_is_dropped():
+ flatten, _ = ps._torchao_helpers()
+ flat, meta = flatten({"blk.f.weight": _fp8_tensor()})
+ entry = json.loads(meta["blk.f.weight"])
+ entry["_data"]["mm_config"]["_data"]["some_new_knob"] = False
+ inert = dict(meta, **{"blk.f.weight": json.dumps(entry)})
+ out = pn.native_unflatten(dict(flat), inert)
+ assert type(out["blk.f.weight"]).__name__ == "Float8Tensor"
+ entry["_data"]["mm_config"]["_data"]["some_new_knob"] = 3
+ live = dict(meta, **{"blk.f.weight": json.dumps(entry)})
+ with pytest.raises(ValueError, match = "some_new_knob"):
+ pn.native_unflatten(dict(flat), live)
+
+
+@needs_helpers
+@needs_int8tensor
+def test_layouts_the_native_reader_does_not_model_go_to_torchao():
+ from torchao.quantization import Float8Tensor, PerRow, PerTensor
+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import (
+ Int8Tensor,
+ QuantizeTensorToInt8Kwargs,
+ )
+
+ flatten, _ = ps._torchao_helpers()
+ per_tensor = Int8Tensor.from_hp(
+ torch.randn(16, 32, dtype = torch.bfloat16),
+ PerTensor(),
+ act_quant_kwargs = QuantizeTensorToInt8Kwargs(granularity = PerRow()),
+ )
+ experts = Float8Tensor.from_hp(
+ torch.randn(2, 16, 32, dtype = torch.bfloat16), granularity = PerRow()
+ )
+ for weight in (per_tensor, experts):
+ flat, meta = flatten({"blk.w.weight": weight})
+ assert pn.native_unflatten(dict(flat), dict(meta)) is None
+ # an int8 field this reader does not know is torchao's call too, never silently dropped
+ flat, meta = flatten({"blk.w.weight": _int8_tensor()})
+ entry = json.loads(meta["blk.w.weight"])
+ entry["_data"]["future_live_field"] = "per_block_128"
+ assert (
+ pn.native_unflatten(dict(flat), dict(meta, **{"blk.w.weight": json.dumps(entry)})) is None
+ )
+
+
+@needs_helpers
+@needs_int8tensor
+def test_duplicate_names_and_unknown_granularity_fields_go_to_torchao():
+ flatten, _ = ps._torchao_helpers()
+ flat, meta = flatten({"blk.f.weight": _fp8_tensor()})
+ dup = dict(meta, tensor_names = json.dumps(["blk.f.weight", "blk.f.weight"]))
+ assert pn.native_unflatten(dict(flat), dup) is None
+ entry = json.loads(meta["blk.f.weight"])
+ text = json.dumps(entry)
+ assert '"_type": "PerRow", "_data": {"dim": -1}' in text
+ odd = dict(
+ meta,
+ **{
+ "blk.f.weight": text.replace(
+ '"_type": "PerRow", "_data": {"dim": -1}',
+ '"_type": "PerRow", "_data": {"dim": -1, "block": 128}',
+ )
+ },
+ )
+ assert pn.native_unflatten(dict(flat), odd) is None
+
+
+def test_overlapping_tensor_bytes_are_refused(tmp_path):
+ header = {
+ "x": {"dtype": "F32", "shape": [4], "data_offsets": [0, 16]},
+ "y": {"dtype": "F32", "shape": [4], "data_offsets": [0, 16]},
+ }
+ raw = json.dumps(header).encode()
+ raw += b" " * (-len(raw) % 8)
+ path = tmp_path / "overlap.safetensors"
+ path.write_bytes(len(raw).to_bytes(8, "little") + raw + bytes(16))
+ with pytest.raises(ValueError):
+ ps._mapped_tensors(str(path))
+
+
[email protected](not hasattr(torch, "float8_e8m0fnu"), reason = "needs float8_e8m0fnu")
+def test_the_mapped_read_takes_mxfp8_scale_dtypes(tmp_path):
+ from safetensors.torch import save_file
+
+ t = torch.ones(8, 4).to(torch.float8_e8m0fnu)
+ save_file({"a.b": t}, str(tmp_path / "e.safetensors"))
+ _, tensors = ps._mapped_tensors(str(tmp_path / "e.safetensors"))
+ assert tensors["a.b"].dtype == torch.float8_e8m0fnu and torch.equal(
+ tensors["a.b"].view(torch.uint8), t.view(torch.uint8)
+ )
+
+
+def _mirror(tmp_path, monkeypatch, *files):
+ root = tmp_path / "mirror"
+ out = []
+ for rel in files:
+ path = root / rel
+ path.parent.mkdir(parents = True, exist_ok = True)
+ path.write_bytes(b"x")
+ out.append(path)
+ monkeypatch.setenv(pq.PREQUANT_MIRROR_ENV, str(root))
+ return out
+
+
+def test_a_mirrored_sibling_this_host_cannot_read_is_not_served(tmp_path, monkeypatch):
+ _mirror(tmp_path, monkeypatch, "unsloth/Model-INT8/Model-INT8.safetensors")
+ monkeypatch.setattr(ps, "safetensors_prequant_supported", lambda: False)
+ monkeypatch.setattr(pq, "_register_prequant_safe_globals", lambda: True)
+ monkeypatch.setattr(pq, "_download_checkpoint_name", lambda *a, **k: "/hub/Model-INT8.pt")
+ src = pq.PrequantSource(
+ kind = "repo",
+ location = "unsloth/Model-INT8",
+ filename = "Model-INT8.pt",
+ declared_filenames = ("Model-INT8.pt",),
+ )
+ assert pq._resolve_checkpoint_path(src, None, None, scheme = "int8") == "/hub/Model-INT8.pt"
+ cached = pq.cached_checkpoint_path(src)
+ assert cached is None or not ps.is_safetensors_checkpoint(cached)
+
+
+def test_any_mirrored_name_wins_before_the_hub_is_asked(tmp_path, monkeypatch):
+ (pt,) = _mirror(tmp_path, monkeypatch, "unsloth/Model-INT8/Model-INT8.pt")
+ monkeypatch.setattr(
+ pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True
+ )
+ calls = []
+ monkeypatch.setattr(
+ pq,
+ "_download_checkpoint_name",
+ lambda source, name, *a, **k: calls.append(name) or f"/hub/{name}",
+ )
+ src = pq.PrequantSource(
+ kind = "repo",
+ location = "unsloth/Model-INT8",
+ filename = "Model-INT8.safetensors",
+ fallback_filenames = ("Model-INT8.pt",),
+ )
+ assert (
+ pq._resolve_checkpoint_path(src, None, None, scheme = "int8") == str(pt.resolve())
+ and calls == []
+ )
+
+
+def test_the_text_encoder_mirror_is_used_when_the_hub_is_unreachable(tmp_path, monkeypatch):
+ import huggingface_hub
+ from huggingface_hub.errors import LocalEntryNotFoundError
+
+ import core.inference.diffusion_te_prequant as te
+
+ (pt,) = _mirror(tmp_path, monkeypatch, "unsloth/Model-FP8/Model-text_encoder-FP8.pt")
+
+ def offline(*a, **k):
+ raise LocalEntryNotFoundError("connection error")
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", offline)
+ src = te.TePrequantSource(
+ kind = "repo",
+ location = "unsloth/Model-FP8",
+ filename = "Model-text_encoder-FP8.safetensors",
+ fallback_filenames = ("Model-text_encoder-FP8.pt",),
+ )
+ assert te._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path / "hub")) == str(
+ pt.resolve()
+ )
+
+
+def test_the_mirror_is_read_before_the_hub_and_cannot_escape_its_root(tmp_path, monkeypatch):
+ root = tmp_path / "mirror"
+ target = root / "unsloth" / "Model-FP8" / "Model-INT8.safetensors"
+ target.parent.mkdir(parents = True)
+ target.write_bytes(b"x")
+ outside = tmp_path / "secret.safetensors"
+ outside.write_bytes(b"x")
+ monkeypatch.setenv(pq.PREQUANT_MIRROR_ENV, str(root))
+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "Model-INT8.safetensors") == str(
+ target.resolve()
+ )
+ # a request for the pickle gets its converted sibling; a pickle with no sibling is not mirrored
+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "Model-INT8.pt") == str(target.resolve())
+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "Model-FP8.pt") is None
+ assert pq.prequant_mirror_path("unsloth/..", "secret.safetensors") is None
+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "../../../secret.safetensors") is None
+ monkeypatch.delenv(pq.PREQUANT_MIRROR_ENV)
+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "Model-INT8.safetensors") is None
+
+
+def test_the_resolver_takes_the_mirrored_safetensors_without_asking_the_hub(tmp_path, monkeypatch):
+ root = tmp_path / "mirror"
+ target = root / "unsloth" / "Model-FP8" / "Model-INT8.safetensors"
+ target.parent.mkdir(parents = True)
+ target.write_bytes(b"x")
+ monkeypatch.setenv(pq.PREQUANT_MIRROR_ENV, str(root))
+ monkeypatch.setattr(
+ pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True
+ )
+
+ def no_hub(*args, **kwargs):
+ raise AssertionError("the Hub was asked for a mirrored file")
+
+ monkeypatch.setattr(pq, "_download_checkpoint_name", no_hub)
+ src = pq.PrequantSource(
+ kind = "repo",
+ location = "unsloth/Model-FP8",
+ filename = "Model-INT8.safetensors",
+ fallback_filenames = ("Model-INT8.pt",),
+ )
+ assert pq._resolve_checkpoint_path(src, None, None, scheme = "int8") == str(target.resolve())
+ assert pq.cached_checkpoint_path(src) == str(target.resolve())
+
+
+def test_the_text_encoder_resolver_reads_the_mirror_too(tmp_path, monkeypatch):
+ import core.inference.diffusion_te_prequant as te
+
+ root = tmp_path / "mirror"
+ target = root / "unsloth" / "Model-FP8" / "Model-text_encoder-FP8.safetensors"
+ target.parent.mkdir(parents = True)
+ target.write_bytes(b"x")
+ monkeypatch.setenv(pq.PREQUANT_MIRROR_ENV, str(root))
+ import huggingface_hub
+
+ def no_hub(*args, **kwargs):
+ raise AssertionError("the Hub was asked for a mirrored file")
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", no_hub)
+ src = te.TePrequantSource(
+ kind = "repo",
+ location = "unsloth/Model-FP8",
+ filename = "Model-text_encoder-FP8.safetensors",
+ fallback_filenames = ("Model-text_encoder-FP8.pt",),
+ )
+ assert te._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path / "hub")) == str(
+ target.resolve()
+ )
+
+
+def test_the_dense_fast_path_reason_names_a_load_time_failure(monkeypatch):
+ import core.inference.diffusion as diffusion
+
+ monkeypatch.setattr(diffusion, "prequant_unreadable_reason", lambda *a, **k: None)
+ reason = diffusion._dense_fast_path_reason(
+ object(), "int8", "org/base", "pipeline", None, None, failure = "ValueError: bad header"
+ )
+ assert "did not load (ValueError: bad header)" in reason and "quantized instead" in reason
+ assert (
+ diffusion._dense_fast_path_reason(object(), "int8", "org/base", "pipeline", None, None)
+ == "engaged on the dense fast path"
+ )
+
+
+ "message, expected",
+ [
+ (
+ "/srv/hf/hub/models--unsloth--X/snapshots/abc/X-INT8.safetensors: the data section does not match its header",
+ "ValueError: X-INT8.safetensors: the data section does not match its header",
+ ),
+ (
+ r"C:\Users\me\mirror\unsloth\X\X-FP8.safetensors is truncated",
+ "ValueError: X-FP8.safetensors is truncated",
+ ),
+ (
+ "unsloth/X-FP8 has no X-FP8.pt (https://huggingface.co/unsloth/X-FP8)",
+ "ValueError: unsloth/X-FP8 has no X-FP8.pt (https://huggingface.co/unsloth/X-FP8)",
+ ),
+ ],
+)
+def test_the_status_failure_note_carries_no_server_directory(message, expected):
+ import core.inference.diffusion_prequant as dp
+ dp._warn(None, "load", ValueError(message))
+ assert dp.last_prequant_failure() == expected
+
+
+def test_every_load_starts_without_the_previous_loads_failure_note():
+ import core.inference.diffusion as diffusion
+
+ src = inspect.getsource(diffusion.DiffusionBackend.load_pipeline)
+ reset = src.find("self._prequant_fallback_note = None")
+ read = src.find('getattr(self, "_prequant_fallback_note", None)')
+ assert 0 <= reset < read
+
+
+def _converter():
+ spec = importlib.util.spec_from_file_location(
+ "convert_prequant_to_safetensors",
+ REPO_ROOT / "scripts" / "convert_prequant_to_safetensors.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _legacy_writer():
+ spec = importlib.util.spec_from_file_location(
+ "_legacy_int8_fixture", Path(__file__).with_name("test_prequant_legacy_int8.py")
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def no_foreign_legacy_registrations():
+ """Other test files register placeholders under the v1 int8 names process-wide (see test_prequant_legacy_int8)."""
+ import core.inference.prequant_legacy_int8 as li
+
+ saved = torch.serialization.get_safe_globals()
+ legacy = {*li.LEGACY_INT8_CLASS_NAMES, li._ACT_QUANT}
+ torch.serialization.clear_safe_globals()
+ torch.serialization.add_safe_globals(
+ [g for g in saved if not (isinstance(g, tuple) and g[1] in legacy)]
+ )
+ yield
+ torch.serialization.clear_safe_globals()
+ torch.serialization.add_safe_globals(saved)
+
+
+@needs_helpers
+@needs_int8tensor
+def test_the_converter_writes_a_bit_identical_v1_int8_artifact(
+ tmp_path, monkeypatch, no_foreign_legacy_registrations
+):
+ src = tmp_path / "Model-INT8.pt"
+ if _V1:
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ model = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False)).to(torch.bfloat16)
+ quantize_(model, Int8DynamicActivationInt8WeightConfig())
+ sd = {
+ "blk.proj.weight": model[0].weight,
+ "blk.norm.weight": torch.ones(64, dtype = torch.bfloat16),
+ }
+ torch.save(
+ {"format": pq.PREQUANT_FORMAT, "metadata": {"scheme": "int8"}, "state_dict": sd},
+ str(src),
+ )
+ else:
+ import core.inference.prequant_legacy_int8 as li
+ if not li.legacy_int8_decode_supported():
+ pytest.skip("no v1 classes and no legacy decoder")
+ _legacy_writer()._write_legacy_checkpoint(monkeypatch, src)
+ conv = _converter()
+ rec = conv.convert(
+ str(src), str(tmp_path / "out"), repo = "unsloth/Model-FP8", revision = "abc", sha = None
+ )
+ assert rec["verified_bit_identical"] and rec["int8_source"] == pn.INT8_SOURCE_V1
+ dst = tmp_path / "out" / "unsloth" / "Model-FP8" / "Model-INT8.safetensors"
+ assert rec["dst"] == str(dst) and dst.is_file()
+ header = ps.read_prequant_header(str(dst))
+ assert header["format"] == pq.PREQUANT_FORMAT and header["metadata"] == {"scheme": "int8"}
+ from safetensors import safe_open
+
+ with safe_open(str(dst), framework = "pt") as handle:
+ raw = handle.metadata()
+ assert json.loads(raw[pn.SOURCE_KEY])["sha256"] == conv.sha256_of(str(src))
+ assert json.loads(raw[pn.QUANT_LAYOUT_KEY])["int8_v1"] == pn.INT8_V1_FACTS
+ # the same pickle, read by Studio, and the new file, read by Studio: same objects, same bytes
+ a = pq._load_prequant_checkpoint(str(src), map_location = "cpu")["state_dict"]
+ b = pq._load_prequant_checkpoint(str(dst), map_location = "cpu", mmap = True)["state_dict"]
+ assert conv.compare_state_dicts(a, b) == []
+
+
+def test_the_converter_writes_a_text_encoder_with_tied_weights(tmp_path):
+ from core.inference.diffusion_te_prequant import TE_PREQUANT_FORMAT
+
+ embed = torch.randn(16, 8).to(torch.float8_e4m3fn)
+ sd = {
+ "model.embed_tokens.weight": embed,
+ "lm_head.weight": embed,
+ "model.norm.weight": torch.ones(8),
+ }
+ src = tmp_path / "Model-text_encoder-FP8.pt"
+ torch.save(
+ {
+ "format": TE_PREQUANT_FORMAT,
+ "metadata": {"scheme": "fp8", "component": "text_encoder"},
+ "state_dict": sd,
+ },
+ str(src),
+ )
+ rec = _converter().convert(str(src), str(tmp_path / "out"), repo = None, revision = None, sha = None)
+ assert rec["kind"] == "text_encoder" and rec["verified_bit_identical"]
+ assert rec["dst"].endswith("Model-text_encoder-FP8.safetensors")
+ pth = tmp_path / "Other-text_encoder-FP8.pth"
+ pth.write_bytes(src.read_bytes())
+ assert (
+ _converter()
+ .convert(str(pth), str(tmp_path / "out"), repo = None, revision = None, sha = None)["dst"]
+ .endswith("Other-text_encoder-FP8.safetensors")
+ )
+ back = ps.load_plain_prequant_safetensors(rec["dst"])
+ assert back["format"] == TE_PREQUANT_FORMAT
+ for key in sd:
+ assert back["state_dict"][key].dtype == sd[key].dtype and torch.equal(
+ back["state_dict"][key].float(), sd[key].float()
+ )
+
+
+def test_the_converters_comparison_sees_every_kind_of_difference():
+ conv = _converter()
+ w = _int8_tensor()
+ assert (
+ conv.compare_state_dicts({"a.w": w, "b": torch.zeros(2)}, {"a.w": w, "b": torch.zeros(2)})
+ == []
+ )
+ assert conv.compare_state_dicts({"a.w": w}, {"a.w": _int8_tensor(seed = 1)}) == ["a.w"]
+ assert conv.compare_state_dicts({"b": torch.zeros(2)}, {"b": -torch.zeros(2)}) == ["b"]
+ assert conv.compare_state_dicts(
+ {"b": torch.zeros(2)}, {"b": torch.zeros(2, dtype = torch.float64)}
+ ) == ["b"]
+ assert conv.compare_state_dicts({"b": torch.zeros(2)}, {"c": torch.zeros(2)}) == ["b", "c"]
+ nan = torch.tensor([float("nan")])
+ assert conv.compare_state_dicts({"b": nan}, {"b": nan.clone()}) == []
+
+
+def test_the_converter_refuses_v1_weights_off_the_supported_layout_and_restores_the_decoder(
+ tmp_path, monkeypatch, no_foreign_legacy_registrations
+):
+ import core.inference.prequant_legacy_int8 as li
+
+ conv = _converter()
+ original = li._rebuild_weight
+ restore = conv._record_v1_facts([])
+ assert li._rebuild_weight is not original
+ restore()
+ assert li._rebuild_weight is original
+
+ def odd_facts(w, standins = None):
+ return dict(pn.INT8_V1_FACTS, quant_kwargs = {"reduce_range": False})
+
+ monkeypatch.setattr(conv, "_v1_facts_of", odd_facts)
+ src = tmp_path / "Model-INT8.pt"
+ if _V1:
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ model = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False)).to(torch.bfloat16)
+ quantize_(model, Int8DynamicActivationInt8WeightConfig())
+ torch.save(
+ {
+ "format": pq.PREQUANT_FORMAT,
+ "metadata": {"scheme": "int8"},
+ "state_dict": {"blk.p.weight": model[0].weight},
+ },
+ str(src),
+ )
+ elif li.legacy_int8_decode_supported():
+ _legacy_writer()._write_legacy_checkpoint(monkeypatch, src)
+ else:
+ pytest.skip("no v1 classes and no legacy decoder")
+ with pytest.raises(ValueError, match = "supported v1 layout"):
+ conv.convert(str(src), str(tmp_path / "out"), repo = None, revision = None, sha = None)
+ assert li._rebuild_weight is original
diff --git a/tests/test_peft_torchao_missing_tensor_subclass.py b/tests/test_peft_torchao_missing_tensor_subclass.py
index 9124d4d1984..4048e50721d 100644
--- a/tests/test_peft_torchao_missing_tensor_subclass.py
+++ b/tests/test_peft_torchao_missing_tensor_subclass.py
@@ -269,6 +269,8 @@ def test_the_warning_does_not_claim_the_other_class_is_gone_too(peft_env, fake_t
"cannot import name 'LinearActivationQuantizedTensor' from 'torchao.quantization'",
# Code reaching past the package for the same class.
"No module named 'torchao.quantization.linear_activation_quantized_tensor'",
+ # torchao main deleted the whole package that defined AffineQuantizedTensor.
+ "No module named 'torchao.dtypes'",
],
)
def test_every_spelling_of_the_missing_class_is_handled(peft_env, fake_torchao, message):
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 11c6700095f..1fac14719df 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -10327,10 +10327,11 @@ def fix_peft_stale_torchao_import_error():
return patched
-# Matches both spellings a torchao removal produces: the class name, and the module that used to
-# define it. Only these two, so a torchao that is BROKEN rather than newer still raises.
+# The spellings a torchao removal produces (class, its old module, the whole ``torchao.dtypes`` package on main).
+# Only these, so a BROKEN torchao still raises.
_PEFT_TORCHAO_MISSING_TENSOR_SUBCLASS = re.compile(
- r"linear_?activation_?quantized_?tensor|affine_?quantized_?tensor",
+ r"linear_?activation_?quantized_?tensor|affine_?quantized_?tensor"
+ r"|no module named '?torchao\.dtypes'?(?![.\w])",
re.IGNORECASE | re.DOTALL,
)