Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read pre-quantized diffusion checkpoints from safetensors on torchao 0.17, 0.18 and main (#12645)
Daniel Han0f239ddca593
diff --git a/scripts/convert_prequant_to_safetensors.py b/scripts/convert_prequant_to_safetensors.pynew file mode 100644index 00000000000..93c5309fcb6--- /dev/null+++ b/scripts/convert_prequant_to_safetensors.py@@ -0,0 +1,364 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Convert a published ``.pt`` pre-quant checkpoint (denoiser or text encoder) to ``.safetensors``.++ python scripts/convert_prequant_to_safetensors.py SRC.pt [SRC2.pt ...] --out-dir DIR \+ [--repo owner/name --revision SHA]++Writes ``DIR/<owner>/<name>/<same stem>.safetensors`` (``DIR/<stem>.safetensors`` without ``--repo``), the+layout ``UNSLOTH_DIFFUSION_PREQUANT_MIRROR`` reads. Nothing is uploaded.++Denoisers use the container Qwen-Image-2.1's published artifacts already use (``prequant_safetensors``:+torchao's flattened tensors plus ``unsloth_format`` / ``unsloth_metadata``), so a released Studio that+resolves ``<Model>-<SCHEME>.safetensors`` reads it too. Two header keys are added: ``unsloth_quant_layout``+(how to rebuild every weight from its plain tensors without torchao, and for int8 that the weights came+from torchao v1, so torchao <= 0.17 rebuilds the v1 class the ``.pt`` held) and ``unsloth_source`` (the+file, its sha256 and the repo revision it was converted from). ``unsloth_metadata`` is copied unchanged.++Text encoders are plain fp8 / bf16 tensors and are written in the plain layout+``load_plain_prequant_safetensors`` reads, with no torchao involved.++Every weight is checked after writing: the file is read back through Studio's loader and each tensor+compared bit for bit with the ``.pt`` as the same install reads it.+"""++from __future__ import annotations++import argparse+import hashlib+import json+import os+import sys+import time+from pathlib import Path++BACKEND = Path(__file__).resolve().parent.parent / "studio" / "backend"+++def sha256_of(path: str, chunk: int = 64 << 20) -> str:+ digest = hashlib.sha256()+ with open(path, "rb") as fh:+ while True:+ block = fh.read(chunk)+ if not block:+ break+ digest.update(block)+ return digest.hexdigest()+++def _record_v1_facts(facts: list):+ """Wrap the legacy decoder so each v1 weight's facts are recorded before it is rebuilt (torchao >= 0.18).++ Patches a module global for the duration of one read, so conversions must not run concurrently in one+ process (this is a single-threaded CLI)."""+ from core.inference import prequant_legacy_int8 as legacy++ original = legacy._rebuild_weight++ def recording(name, w, standins, api):+ facts.append(_v1_facts_of(w, standins))+ return original(name, w, standins, api)++ legacy._rebuild_weight = recording+ return lambda: setattr(legacy, "_rebuild_weight", original)+++def _v1_facts_of(w, standins = None) -> dict:+ """The INT8_V1_FACTS view of one v1 weight (a real torchao object or the decoder's stand-in)."""+ from core.inference.prequant_legacy_int8 import _ACT_QUANT, _LAYOUT++ state = getattr(w, "__dict__", {})+ aqt = state.get("original_weight_tensor")+ impl = getattr(aqt, "tensor_impl", None)+ act = state.get("input_quant_func")+ scale = getattr(impl, "scale", None)+ zero_point = getattr(impl, "zero_point", None)+ domain = getattr(aqt, "zero_point_domain", None)+ layout = getattr(impl, "_layout", None)+ if standins is not None:+ # torchao >= 0.18 read the pickle into stand-ins registered under the v1 names: report those names.+ if act is standins.get(_ACT_QUANT):+ act = _ACT_QUANT+ if isinstance(layout, standins.get(_LAYOUT, ())):+ layout = _LAYOUT+ return {+ "act_quant": (act if isinstance(act, str) else getattr(act, "__name__", str(act))).rsplit(+ ".", 1+ )[-1],+ "quant_kwargs": dict(state.get("quant_kwargs") or {}),+ "zero_point": None if zero_point is None else "tensor",+ "zero_point_domain": getattr(domain, "name", str(domain)),+ "quant_min": getattr(aqt, "quant_min", "?"),+ "quant_max": getattr(aqt, "quant_max", "?"),+ "scale_shape": "out" if scale is not None and scale.dim() == 1 else "other",+ "layout": layout.rsplit(".", 1)[-1] if isinstance(layout, str) else type(layout).__name__,+ }+++def _to_int8_tensor_dict(state_dict: dict, facts: list) -> dict:+ """torchao <= 0.17 hands back v1 objects: record their facts and rebuild them as Int8Tensor to flatten."""+ from core.inference.prequant_legacy_int8 import (+ LEGACY_INT8_CLASS_NAMES,+ _ACT_QUANT,+ _int8_tensor_api,+ _rebuild_weight,+ _resolve,+ )++ classes = {n: _resolve(n) for n in (*LEGACY_INT8_CLASS_NAMES, _ACT_QUANT)}+ laqt = classes[LEGACY_INT8_CLASS_NAMES[0]]+ if laqt is None:+ return state_dict+ api = _int8_tensor_api()+ out = {}+ for key, value in state_dict.items():+ if isinstance(value, laqt):+ facts.append(_v1_facts_of(value))+ out[key] = _rebuild_weight(key, value, classes, api)+ else:+ out[key] = value+ return out+++def _canonical(value):+ """``(class name, {attr: tensor or value})`` of a weight, recursing into torchao subclasses."""+ import torch++ if type(value) is torch.Tensor:+ return ("Tensor", value)+ flatten = getattr(value, "__tensor_flatten__", None)+ if flatten is None:+ return (type(value).__name__, value)+ names, ctx = flatten()+ return (type(value).__name__, {n: _canonical(getattr(value, n)) for n in names}, repr(ctx))+++def _same(a, b) -> bool:+ import torch++ if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):+ if a.dtype != b.dtype or tuple(a.shape) != tuple(b.shape):+ return False+ # Raw bytes, not torch.equal: -0.0 == 0.0 and NaN != NaN would otherwise decide.+ return a.numel() == 0 or torch.equal(+ a.detach().contiguous().cpu().view(torch.uint8),+ b.detach().contiguous().cpu().view(torch.uint8),+ )+ if isinstance(a, tuple) and isinstance(b, tuple):+ return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b, strict = True))+ if isinstance(a, dict) and isinstance(b, dict):+ return a.keys() == b.keys() and all(_same(a[k], b[k]) for k in a)+ return a == b+++def compare_state_dicts(reference: dict, candidate: dict) -> list:+ """Keys whose class, attributes or tensor bytes differ (empty = bit-identical)."""+ bad = sorted(set(reference) ^ set(candidate))+ for key in reference:+ if key in candidate and not _same(_canonical(reference[key]), _canonical(candidate[key])):+ bad.append(key)+ return bad+++def _untie(flat: dict) -> dict:+ """safetensors refuses tensors that share storage (tied embeddings): write each one as its own copy.++ Studio's text-encoder loader re-ties after ``load_state_dict`` (``tie_weights``), exactly as it does for the+ published safetensors encoders, so the copy costs disk only."""+ seen: set = set()+ out = {}+ for key, value in flat.items():+ value = value.contiguous()+ ptr = value.untyped_storage().data_ptr() if value.numel() else None+ if ptr is not None and ptr in seen:+ value = value.clone()+ elif ptr is not None:+ seen.add(ptr)+ out[key] = value+ return out+++def write_plain_safetensors(+ path: str, *, fmt: str, state_dict: dict, metadata: dict, extra: dict+) -> None:+ """The torchao-free plain layout ``load_plain_prequant_safetensors`` reads (text encoders)."""+ import torch+ from safetensors.torch import save_file++ from core.inference.prequant_safetensors import (+ UNSLOTH_FORMAT_KEY,+ UNSLOTH_METADATA_KEY,+ UNSLOTH_ROOT_KEYS_KEY,+ UNSLOTH_ROOT_PREFIX,+ )++ names, roots, flat = [], [], {}+ for key, value in state_dict.items():+ if type(value) is not torch.Tensor:+ raise ValueError(f"{key!r} is a {type(value).__name__}, not a plain tensor")+ if "." in key:+ names.append(key)+ flat[key] = value.contiguous()+ else:+ roots.append(key)+ flat[f"{UNSLOTH_ROOT_PREFIX}{key}"] = value.contiguous()+ header = {name: json.dumps({"_type": "Tensor"}) for name in names}+ header["tensor_names"] = json.dumps(names)+ header[UNSLOTH_FORMAT_KEY] = fmt+ header[UNSLOTH_METADATA_KEY] = json.dumps(metadata, default = str)+ if roots:+ header[UNSLOTH_ROOT_KEYS_KEY] = json.dumps(roots)+ header.update(extra)+ save_file(_untie(flat), path, metadata = header)+++def convert(+ src: str, out_dir: str, *, repo: str | None, revision: str | None, sha: str | None+) -> dict:+ import torch++ from core.inference import diffusion_prequant as dp+ from core.inference import prequant_safetensors as ps+ from core.inference.diffusion_te_prequant import TE_PREQUANT_FORMAT+ from core.inference.prequant_native import (+ INT8_SOURCE_V1,+ INT8_V1_FACTS,+ QUANT_LAYOUT_KEY,+ SOURCE_KEY,+ quant_layout_header,+ )++ t0 = time.time()+ name = os.path.basename(src)+ stem = os.path.splitext(name)[0] if name.endswith((".pt", ".pth")) else name+ dest_dir = os.path.join(out_dir, *repo.split("/")) if repo else out_dir+ os.makedirs(dest_dir, exist_ok = True)+ dst = os.path.join(dest_dir, stem + ".safetensors")+ source = {"file": name, "sha256": sha or sha256_of(src), "bytes": os.path.getsize(src)}+ if repo:+ source["repo"] = repo+ if revision:+ source["revision"] = revision+ try:+ import torchao+ source["converter_torchao"] = torchao.__version__+ except Exception: # noqa: BLE001+ pass+ source["converter_torch"] = torch.__version__++ # Read exactly as Studio does (allowlisted weights_only); on torchao >= 0.18 v1 int8 comes back as Int8Tensor.+ facts: list = []+ restore = _record_v1_facts(facts)+ try:+ ckpt = dp._load_prequant_checkpoint(src, map_location = "cpu", mmap = True)+ finally:+ restore()+ fmt = ckpt.get("format")+ is_te = fmt == TE_PREQUANT_FORMAT+ metadata = dict(ckpt.get("metadata") or {})+ state_dict = ckpt["state_dict"]+ extra = {SOURCE_KEY: json.dumps(source)}+ if is_te:+ write_plain_safetensors(dst, fmt = fmt, state_dict = state_dict, metadata = metadata, extra = extra)+ back = ps.load_plain_prequant_safetensors(dst)+ else:+ if fmt not in dp.PREQUANT_FORMATS:+ raise ValueError(f"{src}: unrecognised format {fmt!r}")+ flat_sd = _to_int8_tensor_dict(state_dict, facts)+ int8_source = None+ if facts:+ int8 = sum(type(v).__name__ == "Int8Tensor" for v in flat_sd.values())+ if len(facts) != int8:+ raise ValueError(+ f"{src}: {int8} int8 weights but {len(facts)} came from torchao v1"+ )+ distinct = {json.dumps(f, sort_keys = True, default = str) for f in facts}+ if distinct != {json.dumps(INT8_V1_FACTS, sort_keys = True)}:+ raise ValueError(+ f"{src}: v1 int8 weights do not match the supported v1 layout: {sorted(distinct)[:3]}"+ )+ int8_source = INT8_SOURCE_V1+ layout = quant_layout_header(flat_sd, int8_source = int8_source)+ # save_prequant_safetensors writes torchao's flattened layout plus our two keys; add the layout and source.+ from safetensors.torch import save_file++ helpers = ps._torchao_helpers()+ if helpers is None:+ raise RuntimeError("torchao >= 0.16 is required to write the flattened layout")+ undotted = ps.unsupported_state_dict_keys(flat_sd)+ if undotted:+ raise ValueError(+ f"{src}: root-level quantized weights cannot be flattened: {undotted[:8]}"+ )+ roots = ps._root_level_keys(flat_sd)+ quantizable = {k: v for k, v in flat_sd.items() if k not in set(roots)}+ flat, torchao_metadata = helpers[0](quantizable)+ header = dict(torchao_metadata or {})+ header[ps.UNSLOTH_FORMAT_KEY] = str(fmt)+ header[ps.UNSLOTH_METADATA_KEY] = json.dumps(metadata, default = str)+ header[QUANT_LAYOUT_KEY] = json.dumps(layout)+ header.update(extra)+ if roots:+ flat = dict(flat)+ for key in roots:+ flat[f"{ps.UNSLOTH_ROOT_PREFIX}{key}"] = flat_sd[key].contiguous()+ header[ps.UNSLOTH_ROOT_KEYS_KEY] = json.dumps([str(k) for k in roots])+ save_file(_untie(dict(flat)), dst, metadata = header)+ back = ps.load_prequant_safetensors(dst, mmap = True)+ # Bit-identity against the .pt as THIS install reads it (v1 on <= 0.17 via the native v1 rebuild).+ bad = compare_state_dicts(state_dict, back["state_dict"])+ if bad:+ raise AssertionError(f"{dst}: {len(bad)} tensors differ from {src}, e.g. {bad[:3]}")+ if back["format"] != fmt or back["metadata"] != metadata:+ raise AssertionError(f"{dst}: format or metadata changed in the round trip")+ rec = {+ "src": src,+ "dst": dst,+ "kind": "text_encoder" if is_te else "transformer",+ "format": fmt,+ "scheme": metadata.get("scheme"),+ "tensors": len(state_dict),+ "quantized": sum(1 for v in state_dict.values() if type(v) is not torch.Tensor),+ "int8_source": None if is_te else layout.get("int8_source"),+ "reader": back.get("reader"),+ "src_bytes": source["bytes"],+ "src_sha256": source["sha256"],+ "dst_bytes": os.path.getsize(dst),+ "dst_sha256": sha256_of(dst),+ "seconds": round(time.time() - t0, 1),+ "verified_bit_identical": True,+ }+ return rec+++def main(argv = None) -> int:+ p = argparse.ArgumentParser(description = __doc__.split("\n\n")[0])+ p.add_argument("src", nargs = "+")+ p.add_argument("--out-dir", required = True)+ p.add_argument(+ "--repo", default = None, help = "owner/name the .pt came from (mirror layout + provenance)"+ )+ p.add_argument("--revision", default = None)+ p.add_argument("--sha256", default = None, help = "known sha256 of a single SRC (skips hashing it)")+ p.add_argument("--report", default = None, help = "append one JSON line per file here")+ args = p.parse_args(argv)+ sys.path.insert(0, str(BACKEND))+ for src in args.src:+ rec = convert(+ src,+ args.out_dir,+ repo = args.repo,+ revision = args.revision,+ sha = args.sha256 if len(args.src) == 1 else None,+ )+ line = json.dumps(rec)+ print(line, flush = True)+ if args.report:+ with open(args.report, "a") as fh:+ fh.write(line + "\n")+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.jsonindex b29ce92b528..40a6ae362e5 100644--- a/scripts/risky_loaders_baseline.json+++ b/scripts/risky_loaders_baseline.json@@ -82,6 +82,14 @@"count": 1,"reason": "Module name comes from a fixed table or constant in this file, not from a model, request or download."},+ {+ "file": "studio/backend/core/inference/diffusion_int8_fused.py",+ "rule": "dynamic-import",+ "sink": "import_module",+ "digest": "8a43c55c018acccf",+ "count": 1,+ "reason": "Imports a name from the constant _TORCHAO_INTMM_MODULES tuple (torchao's safe_int_mm homes); never a model, request or download name."+ },{"file": "studio/backend/core/inference/diffusion_prequant.py","rule": "dynamic-import",diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 5f0ca4759ba..640405d3b5d 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -1919,8 +1919,12 @@ def _dense_fast_path_reason(kind: str,path_override: Optional[str],loras: Any = None,+ failure: Optional[str] = None,) -> str:- """Name an unreadable hosted checkpoint only when it was in play (not override/GGUF/LoRA bake)."""+ """Name an unreadable hosted checkpoint only when it was in play (not override/GGUF/LoRA bake).++ ``failure`` is why a pre-quantized checkpoint that WAS tried did not load (``last_prequant_failure``):+ planning thought it readable, so only the load can say what went wrong."""note = (prequant_unreadable_reason(fam, scheme, base_repo = base)if kind == "pipeline" and not path_override and not _has_active_lora(loras)@@ -1928,6 +1932,11 @@ def _dense_fast_path_reason()if note:return f"engaged on the dense fast path; {note}, so the dense bf16 transformer was quantized instead"+ if failure:+ return (+ f"engaged on the dense fast path; the pre-quantized {scheme} checkpoint did not load "+ f"({failure}), so the dense bf16 transformer was quantized instead"+ )return "engaged on the dense fast path"@@ -5344,6 +5353,8 @@ class DiffusionBackend:self._active_generate_cancel.set()self._reserve_teardown_locked()with self._model_transition_slot():+ # Per load: a previous load's pre-quant failure must never reach this one's status.+ self._prequant_fallback_note = Nonewith self._lock:try:self._raise_if_load_cancelled(_load_token)@@ -6388,9 +6399,20 @@ class DiffusionBackend:)else:# The plan was priced on the seed landing, so re-plan at bf16 without it.+ from utils.native_path_leases import redact_native_paths++ from .diffusion_prequant import last_prequant_failure++ failure = last_prequant_failure()+ self._prequant_fallback_note = (+ redact_native_paths(failure)+ if failure+ else "the checkpoint was unavailable"+ )logger.warning("diffusion.denoiser_prequant: no pre-quantized denoiser was "- "seeded; re-planning memory at the released bf16 size"+ "seeded (%s); re-planning memory at the released bf16 size",+ self._prequant_fallback_note,)pipeline_seed_scheme = Noneplan = bf16_pipeline_plan@@ -7243,6 +7265,7 @@ class DiffusionBackend:kind,transformer_prequant_path,loras,+ failure = getattr(self, "_prequant_fallback_note", None),),# Honored when the quant engaged AND when the ask was "off" (a request NOT to# quantise, which the GGUF build satisfies)@@ -7545,6 +7568,7 @@ class DiffusionBackend:self._raise_if_load_cancelled(_load_token)check_cancelled()+ self._prequant_fallback_note = Nonefetch_base = fetch_base or prefer_ungated_mirror(base, hf_token)# 1. Pre-quantized checkpoint, when one is configured for the resolved scheme.scheme = _planned_quant_scheme(@@ -7593,6 +7617,25 @@ class DiffusionBackend:placement_device = None if seed_device == device else seed_device,)check_cancelled()+ if transformer is None:+ # Never a silent swap: the status names why the checkpoint was not used.+ from utils.native_path_leases import redact_native_paths++ from .diffusion_prequant import last_prequant_failure++ failure = last_prequant_failure()+ self._prequant_fallback_note = (+ redact_native_paths(failure)+ if failure+ else "the checkpoint was unavailable"+ )+ logger.warning(+ "diffusion.prequant: %s checkpoint from %s not used (%s); quantizing the dense "+ "transformer instead",+ scheme,+ getattr(source, "location", "?"),+ self._prequant_fallback_note,+ )if transformer is not None:if scheme == TQ_NVFP4:from .diffusion_nvfp4_linear import nvfp4_prewarm@@ -7619,8 +7662,11 @@ class DiffusionBackend:if not allow_dense_fallback:# The plan only budgeted the prequant-sized build, so the dense bf16 transformer would exceed it after# eviction.+ note = getattr(self, "_prequant_fallback_note", None)raise RuntimeError(- "prequant checkpoint unavailable and the dense transformer does not fit resident"+ "prequant checkpoint unavailable"+ + (f" ({note})" if note else "")+ + " and the dense transformer does not fit resident")# Deliberately the hub id, not base_local_dir: diffusers treats a local directory as terminal (_get_model_file# raises rather than falling back to the hub) and a sharded load raises per missing shard, so a partialdiff --git a/studio/backend/core/inference/diffusion_int8_fused.py b/studio/backend/core/inference/diffusion_int8_fused.pyindex 28b4e7c60e2..588f8b1809e 100644--- a/studio/backend/core/inference/diffusion_int8_fused.py+++ b/studio/backend/core/inference/diffusion_int8_fused.py@@ -36,6 +36,8 @@ _SWIGLU_ATTR = "_unsloth_i8_swiglu"_LOCK = threading.Lock()# Read by the traced forwards: dynamo must not trace into the lru_cache'd registration._OP_HANDLE: Any = None+# torchao's safe_int_mm home, resolved at install (outside any trace) for the same reason.+_INTMM_MODULE: Any = None# Marker on each patched module (no global registry: it would pin an unloaded transformer)._MARK = "_unsloth_i8_fused_prev"_NO_PREV = object()@@ -837,9 +839,31 @@ def _act_quant(x2d: Any, weight: Any) -> tuple:return act.qdata, act.scale+def _resolve_intmm() -> Any:+ """The module holding torchao's ``safe_int_mm``, or None. torchao <= 0.18 ships it in ``torchao.kernel.intmm``;+ main after pytorch/ao#4718 moved it to the int8 workflow and deleted ``torchao.kernel``. Read off the module at+ call time, so Studio's capture-safe rebinding (diffusion_torchao_patches) is the one that runs."""+ import importlib++ from .diffusion_torchao_patches import _TORCHAO_INTMM_MODULES++ for name in _TORCHAO_INTMM_MODULES:+ try:+ module = importlib.import_module(name)+ except ImportError:+ continue+ if callable(getattr(module, "safe_int_mm", None)):+ return module+ return None++def _int_mm(a: Any, weight: Any) -> Any:- from torchao.kernel.intmm import safe_int_mm- return safe_int_mm(a, weight.qdata.contiguous().t())+ module = _INTMM_MODULE+ if module is None: # a direct call before any install (tests): resolve eagerly+ module = _resolve_intmm()+ if module is None:+ raise ImportError("torchao ships no safe_int_mm in any known module")+ return module.safe_int_mm(a, weight.qdata.contiguous().t())def _linear_from_q(q: Any, xs: Any, weight: Any, bias: Any, out_dtype: Any) -> Any:@@ -1233,13 +1257,19 @@ def _has_eligible(transformer: Any) -> bool:def _finalize(transformer: Any, logger: Any = None) -> int:- global _OP_HANDLE+ global _OP_HANDLE, _INTMM_MODULEdev = resident_cuda_device(transformer)if dev is None:return 0# Before the probe: it compiles and launches both Triton kernels, pure latency for a bf16 / fp16 model.if not _has_eligible(transformer):return 0+ intmm = _resolve_intmm()+ if (+ intmm is None+ ): # an int8 GEMM home this file does not know: keep the stock path rather than fail the render+ return 0+ _INTMM_MODULE = intmmimport torchif not _device_ok(dev.index if dev.index is not None else torch.cuda.current_device()):diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex 23ce425949b..a4011a741f9 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -29,6 +29,7 @@ caller falls back to dense-quantise (then GGUF). Inert with nothing configured.from __future__ import annotations+import re as _reimport threading as _threadingfrom dataclasses import dataclassfrom collections.abc import Sequence@@ -338,12 +339,12 @@ def _load_prequant_checkpoint(path: str, **kwargs: Any) -> Any:Dispatched on the file extension rather than on content sniffing: the extension is what theresolver asked the Hub for, so a repo hosting both containers cannot serve one and be parsed as- the other. ``kwargs`` are the pickle path's (``map_location``, ``mmap``); safetensors has no- equivalent knobs and reads to CPU, which is where the pickle path maps too."""+ the other. ``kwargs`` are the pickle path's (``map_location``, ``mmap``); safetensors reads to+ CPU, which is where the pickle path maps too, and honours ``mmap`` the same way."""from .prequant_safetensors import is_safetensors_checkpoint, load_prequant_safetensorsif is_safetensors_checkpoint(path):- return load_prequant_safetensors(path)+ return load_prequant_safetensors(path, mmap = bool(kwargs.get("mmap")))return _torch_load_prequant(path, **kwargs)@@ -432,6 +433,65 @@ def local_prequant_path_ready(path: str) -> bool:return os.path.isfile(os.path.expanduser(path))+# Operator-side mirror checked BEFORE the Hub: ``<root>/<owner>/<repo>/<filename>``, roots split by ``os.pathsep``.+# Names come from the family tables, never a request; anything leaving the root is ignored.+PREQUANT_MIRROR_ENV = "UNSLOTH_DIFFUSION_PREQUANT_MIRROR"+++def prequant_mirror_path(+ repo_id: Optional[str],+ name: Optional[str],+ readable: Any = None,+) -> Optional[str]:+ """``<mirror>/<repo_id>/<name>`` when a configured mirror holds that file, else None. Never raises.++ A converted safetensors copy of a requested pickle wins over the pickle itself (same weights, no+ constructor allowlist), but only when ``readable(<sibling name>)`` says this install can open it:+ otherwise a host without torchao's flatten helpers would be handed a file it then refuses, after+ planning had already counted it as cached. ``readable`` defaults to "yes"."""+ import os++ raw = (os.environ.get(PREQUANT_MIRROR_ENV) or "").strip()+ if not raw or not repo_id or not name:+ return None+ parts = [*str(repo_id).split("/"), *str(name).split("/")]+ if any(p in ("", ".", "..") or "\\" in p for p in parts):+ return None+ wanted = [parts]+ for suffix in (".pt", ".pth"):+ if parts[-1].endswith(suffix):+ sibling = parts[-1][: -len(suffix)] + ".safetensors"+ try:+ ok = readable is None or bool(readable(sibling))+ except Exception: # noqa: BLE001 - an unanswerable question is a no+ ok = False+ if ok:+ wanted.insert(0, [*parts[:-1], sibling])+ break+ for root in raw.split(os.pathsep):+ root = root.strip()+ if not root:+ continue+ for rel in wanted:+ try:+ base = os.path.realpath(os.path.expanduser(root))+ candidate = os.path.realpath(os.path.join(base, *rel))+ except Exception: # noqa: BLE001 - a bad root is simply not a mirror+ break+ if candidate.startswith(base.rstrip(os.sep) + os.sep) and os.path.isfile(candidate):+ return candidate+ return None+++def _first_mirrored(repo_id: Optional[str], names: Sequence[str], readable: Any) -> Optional[str]:+ """The mirror's copy of the first of ``names`` it holds, checked for ALL names before any Hub call."""+ for name in names:+ hit = prequant_mirror_path(repo_id, name, readable)+ if hit is not None:+ return hit+ return None++@dataclass(frozen = True)class PrequantSource:"""Where a pre-quantized checkpoint lives. ``kind`` is "path" (a local file) or "repo" (Hub repo@@ -1049,15 +1109,23 @@ def cached_checkpoint_path(``hf_hub_download`` falls back to huggingface_hub's import-time constant. Never raises."""roots = (cache_dir, None) if cache_dir else (None,)wanted = set(names) if names is not None else None- for name in candidate_filenames_of(source):- if wanted is not None and name not in wanted:- continue++ def _readable(name: str) -> bool:try:- readable = restricted_prequant_load_supported(None, name)+ return bool(restricted_prequant_load_supported(None, name))except Exception: # noqa: BLE001 - a pure lookup that never raises, as documented above- readable = True- if not readable:- continue+ return True++ candidates = [+ n+ for n in candidate_filenames_of(source)+ if (wanted is None or n in wanted) and _readable(n)+ ]+ if getattr(source, "kind", None) == "repo":+ mirrored = _first_mirrored(getattr(source, "location", None), candidates, _readable)+ if mirrored is not None:+ return mirrored+ for name in candidates:for root in roots:hit = _cached_in_root(source, root, name)if hit is not None:@@ -1437,6 +1505,12 @@ def _resolve_checkpoint_path(names = readable or namesif not names:return None+ # Mirror first for EVERY name: else a Hub copy of an earlier name is downloaded and the mirror never used.+ mirrored = _first_mirrored(+ source.location, names, lambda n: restricted_prequant_load_supported(scheme, n)+ )+ if mirrored is not None:+ return mirroredfor index, name in enumerate(names):last = index == len(names) - 1try:@@ -2232,10 +2306,15 @@ def _has_meta_tensors(module: Any) -> bool:_LAST_FAILURE = _threading.local()+# An absolute POSIX or Windows path; the status keeps only its last component (the server's layout is not the+# client's business). Not after a word, ':' or '/', so URLs and repo ids are left alone.+_ABS_PATH = _re.compile(+ r"(?<![\w:/.])(?:[A-Za-z]:[\\/]|/)(?:[^\s'\"<>|:;,()\[\]\\/]+[\\/])+(?=[^\s\\/])"+)def _warn(logger: Any, what: str, exc: Exception) -> None:- _LAST_FAILURE.text = f"{type(exc).__name__}: {exc}"[:300]+ _LAST_FAILURE.text = _ABS_PATH.sub("", f"{type(exc).__name__}: {exc}")[:300]if logger is not None:logger.warning("diffusion.prequant: %s failed: %s", what, exc)diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex 696c38907fe..2b585d49e6d 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -636,6 +636,12 @@ def _resolve_checkpoint_path()names = [n for n in te_candidate_filenames(source) if te_candidate_is_readable(n)]last: Optional[Exception] = None+ from .diffusion_prequant import _first_mirrored++ # The operator's mirror answers before the Hub is asked for any name (and so also offline).+ mirrored = _first_mirrored(source.location, names, te_candidate_is_readable)+ if mirrored is not None:+ return mirroredfor name in names:try:return hf_hub_download(diff --git a/studio/backend/core/inference/diffusion_torchao_patches.py b/studio/backend/core/inference/diffusion_torchao_patches.pyindex 2916fd82f6b..5fdf3e24f9d 100644--- a/studio/backend/core/inference/diffusion_torchao_patches.py+++ b/studio/backend/core/inference/diffusion_torchao_patches.py@@ -248,10 +248,11 @@ def install_torchao_int_mm_patch():# inside ``dispatch_torchao`` for every LoRA target, torchao 0.18 deleted it, and Studio installs torchao# 0.18 on torch >= 2.12, so every diffusion LoRA load raised. Both copies mark their wrapper# ``__unsloth_patched__``, so whichever runs second leaves the first in place.-# Matches both spellings a torchao removal produces: the class name, and the module that used to-# define it. Only these two, so a torchao that is BROKEN rather than newer still raises.+# The spellings a torchao removal produces (class, its old module, the whole ``torchao.dtypes`` package on main).+# Only these, so a BROKEN torchao still raises._PEFT_TORCHAO_MISSING_TENSOR_SUBCLASS = re.compile(- r"linear_?activation_?quantized_?tensor|affine_?quantized_?tensor",+ r"linear_?activation_?quantized_?tensor|affine_?quantized_?tensor"+ r"|no module named '?torchao\.dtypes'?(?![.\w])",re.IGNORECASE | re.DOTALL,)diff --git a/studio/backend/core/inference/prequant_native.py b/studio/backend/core/inference/prequant_native.pynew file mode 100644index 00000000000..15a33c24e6e--- /dev/null+++ b/studio/backend/core/inference/prequant_native.py@@ -0,0 +1,436 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Rebuild a safetensors pre-quant checkpoint's int8 / fp8 weights without torchao's deserializer.++The file layout is torchao's own (``flatten_tensor_state_dict``): ``<fqn>._weight_qdata`` and+``<fqn>._weight_scale`` plus one JSON entry per weight. Reading it back through torchao's+``unflatten_tensor_state_dict`` ties every load to that release's constructor kwargs, and that has+already broken once (0.18 wrote ``reduce_range``, which 0.17 refuses). The two schemes Unsloth ships+are simple enough to describe without torchao:++ int8 w[n, k] = qdata[n, k] (int8) * scale[n] symmetric, per output row+ activations: dynamic symmetric per-token int8+ fp8 w[n, k] = qdata[n, k] (float8_e4m3fn) * scale[n] per output row (or one scale per tensor)+ activations: dynamic fp8 with the recorded granularity++so this module parses the JSON itself, validates the tensors against that description, and builds+whatever class the INSTALLED torchao uses for the scheme. In particular an int8 artifact converted from+a torchao v1 checkpoint (``unsloth_quant_layout.int8_source == "torchao_v1"``) is rebuilt as the v1+``LinearActivationQuantizedTensor`` where torchao still ships it (<= 0.17), which makes it+bit-identical, class for class, to reading the original ``.pt`` there; 0.18 and later get the+``Int8Tensor`` the legacy-pickle decoder produces. Anything this module does not recognise+returns None and the caller uses torchao's own reader, so nvfp4 / mxfp8 artifacts are unaffected.++Kill switch: ``UNSLOTH_PREQUANT_NATIVE_REBUILD=0`` always uses torchao's reader.+"""++from __future__ import annotations++import dataclasses+import json+import os+from typing import Any, Optional++NATIVE_REBUILD_ENV = "UNSLOTH_PREQUANT_NATIVE_REBUILD"++# Header keys the converter adds beside torchao's. Namespaced like the other unsloth_* keys, so torchao's+# ``is_metadata_torchao`` (which walks ``tensor_names`` only) and every released Studio build ignore them.+QUANT_LAYOUT_KEY = "unsloth_quant_layout"+SOURCE_KEY = "unsloth_source"+QUANT_LAYOUT_VERSION = 1+INT8_SOURCE_V1 = "torchao_v1"++# What a torchao v1 int8 pre-quant checkpoint holds, as validated by prequant_legacy_int8._rebuild_weight+# and recorded by the converter. A file declaring anything else is not rebuilt as v1.+INT8_V1_FACTS = {+ "act_quant": "_int8_symm_per_token_reduced_range_quant",+ "quant_kwargs": {},+ "zero_point": None,+ "zero_point_domain": "NONE",+ "quant_min": None,+ "quant_max": None,+ "scale_shape": "out",+ "layout": "PlainLayout",+}++# Unknown fields carrying one of these are behaviourally absent (same rule as prequant_safetensors).+_INERT = (False, None, 0)++_DTYPES = ("bfloat16", "float16", "float32", "float8_e4m3fn", "float8_e5m2", "int8")+++class _Unsupported(Exception):+ """This entry needs torchao's own reader; not an error."""+++def native_rebuild_enabled() -> bool:+ raw = (os.environ.get(NATIVE_REBUILD_ENV) or "").strip().lower()+ return raw not in ("0", "false", "no", "off")+++def _torchao_api() -> Optional[dict]:+ """The torchao names the rebuild needs, or None (absent / stubbed torchao)."""+ try:+ from core._torchao_stub import is_stubbed+ if is_stubbed("torchao"):+ return None+ except Exception: # noqa: BLE001 - no stub module means nothing is stubbed+ pass+ try:+ from torchao.float8.inference import Float8MMConfig+ from torchao.quantization import Float8Tensor, PerRow, PerTensor+ from torchao.quantization.quant_primitives import MappingType+ from torchao.quantization.quantize_.common.kernel_preference import KernelPreference+ from torchao.quantization.quantize_.workflows.float8.float8_tensor import (+ QuantizeTensorToFloat8Kwargs,+ )+ except Exception: # noqa: BLE001+ return None+ api = {+ "Float8Tensor": Float8Tensor,+ "Float8MMConfig": Float8MMConfig,+ "QuantizeTensorToFloat8Kwargs": QuantizeTensorToFloat8Kwargs,+ "KernelPreference": KernelPreference,+ "MappingType": MappingType,+ "PerRow": PerRow,+ "PerTensor": PerTensor,+ }+ try:+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import (+ Int8Tensor,+ QuantizeTensorToInt8Kwargs,+ )+ api["Int8Tensor"] = Int8Tensor+ api["QuantizeTensorToInt8Kwargs"] = QuantizeTensorToInt8Kwargs+ except Exception: # noqa: BLE001 - no Int8Tensor: int8 entries go to torchao's reader+ pass+ return api+++def _v1_int8_api() -> Optional[dict]:+ """torchao's v1 int8 classes when this release still ships ALL of them (<= 0.17), else None."""+ try:+ from torchao.dtypes.affine_quantized_tensor import AffineQuantizedTensor+ from torchao.dtypes.uintx.plain_layout import PlainAQTTensorImpl+ from torchao.dtypes.utils import PlainLayout+ from torchao.quantization.linear_activation_quantized_tensor import (+ LinearActivationQuantizedTensor,+ )+ from torchao.quantization.quant_api import _int8_symm_per_token_reduced_range_quant+ from torchao.quantization.quant_primitives import ZeroPointDomain+ except Exception: # noqa: BLE001+ return None+ return {+ "AffineQuantizedTensor": AffineQuantizedTensor,+ "PlainAQTTensorImpl": PlainAQTTensorImpl,+ "PlainLayout": PlainLayout,+ "LinearActivationQuantizedTensor": LinearActivationQuantizedTensor,+ "act_quant": _int8_symm_per_token_reduced_range_quant,+ "ZeroPointDomain": ZeroPointDomain,+ }+++def _construct(cls: Any, data: dict, what: str) -> Any:+ """``cls(**data)`` keeping only fields ``cls`` has; an unknown field must be inert to be dropped."""+ if dataclasses.is_dataclass(cls):+ known = {f.name for f in dataclasses.fields(cls)}+ elif hasattr(cls, "_fields"):+ known = set(cls._fields)+ else:+ raise _Unsupported(f"{what}: {cls!r} is neither a dataclass nor a NamedTuple")+ extra = {k: v for k, v in data.items() if k not in known}+ live = {k: v for k, v in extra.items() if v not in _INERT}+ if live:+ name = sorted(live)[0]+ raise ValueError(+ f"{what} records {name}={live[name]!r}, which this torchao cannot construct; "+ "upgrade torchao to read this checkpoint"+ )+ return cls(**{k: v for k, v in data.items() if k in known})+++def _decode(value: Any, api: dict, what: str) -> Any:+ """torchao's JSON encoding of one attribute, decoded against a FIXED table of types."""+ if isinstance(value, list):+ return [_decode(v, api, what) for v in value]+ if not isinstance(value, dict) or "_type" not in value:+ return value+ kind, data = value.get("_type"), value.get("_data")+ if kind == "torch.dtype":+ import torch+ if data not in _DTYPES:+ raise _Unsupported(f"{what}: dtype {data!r}")+ return getattr(torch, data)+ if kind in ("MappingType", "KernelPreference"):+ enum_cls = api[kind]+ if not isinstance(data, str) or not hasattr(enum_cls, data):+ raise _Unsupported(f"{what}: {kind} {data!r}")+ return getattr(enum_cls, data)+ if kind in ("PerRow", "PerTensor") and data is not None and not isinstance(data, dict):+ raise _Unsupported(f"{what}: {kind} {data!r}")+ if kind in ("PerRow", "PerTensor") and set(data or {}) - (+ {"dim"} if kind == "PerRow" else set()+ ):+ raise _Unsupported(f"{what}: {kind} fields {sorted(set(data) - {'dim'})}")+ if kind == "PerRow":+ dim = (data or {}).get("dim", -1)+ try:+ return api["PerRow"](dim = dim)+ except TypeError:+ if dim != -1:+ raise _Unsupported(f"{what}: PerRow(dim={dim}) on a torchao without dim") from None+ return api["PerRow"]()+ if kind == "PerTensor":+ return api["PerTensor"]()+ if kind in ("Float8MMConfig", "QuantizeTensorToFloat8Kwargs", "QuantizeTensorToInt8Kwargs"):+ if kind not in api or not isinstance(data, dict):+ raise _Unsupported(f"{what}: {kind}")+ fields = {k: _decode(v, api, what) for k, v in data.items()}+ return _construct(api[kind], fields, f"{what} {kind}")+ raise _Unsupported(f"{what}: attribute type {kind!r}")+++def _flat_key(name: str, data_name: str) -> str:+ module_fqn, weight_name = name.rsplit(".", 1)+ return f"{module_fqn}._{weight_name}_{data_name}"+++def _granularity_name(g: Any) -> str:+ return type(g).__name__+++def _rebuild_int8(+ name: str, entry: dict, tensors: dict, used: set, api: dict, v1: Optional[dict]+) -> Any:+ import torch++ data = entry.get("_data") or {}+ data_names = list(entry.get("_tensor_data_names") or [])+ if set(data_names) - {"qdata", "scale", "zero_point"} or not {"qdata", "scale"} <= set(+ data_names+ ):+ raise _Unsupported(f"{name}: int8 tensor data {data_names}")+ keys = {d: _flat_key(name, d) for d in data_names}+ missing = [k for k in keys.values() if k not in tensors]+ if missing:+ raise ValueError(+ f"int8 weight {name!r}: missing {missing[0]!r}; the checkpoint is incomplete"+ )+ qdata, scale = tensors[keys["qdata"]], tensors[keys["scale"]]+ zero_point = tensors.get(keys["zero_point"]) if "zero_point" in keys else None+ # Only the layout Unsloth ships is modelled here (2-D, symmetric, one scale per output row). Any other valid+ # torchao int8 weight (per-tensor, 3-D experts) goes to torchao's own reader rather than being refused.+ if qdata.dtype != torch.int8 or qdata.dim() != 2:+ raise _Unsupported(f"{name}: int8 qdata {qdata.dtype} {tuple(qdata.shape)}")+ n, k = qdata.shape+ if not scale.is_floating_point() or scale.numel() != n:+ raise _Unsupported(f"{name}: int8 scale {tuple(scale.shape)}")+ if list(data.get("block_size") or []) != [1, k]:+ raise _Unsupported(f"{name}: int8 block size {data.get('block_size')}")+ known = {"block_size", "dtype", "act_quant_kwargs", "reduce_range"}+ if any(v not in _INERT for k2, v in data.items() if k2 not in known):+ raise _Unsupported(f"{name}: int8 attributes {sorted(set(data) - known)}")+ if zero_point is not None and bool(torch.any(zero_point != 0)):+ raise _Unsupported(f"{name}: asymmetric int8")+ dtype = _decode(data.get("dtype"), api, name)+ if dtype not in (torch.bfloat16, torch.float16, torch.float32):+ raise _Unsupported(f"{name}: int8 output dtype {dtype}")+ act = data.get("act_quant_kwargs")+ act_fields = ((act or {}).get("_data") or {}) if act is not None else None+ reduce_range = bool(data.get("reduce_range") or False)+ if act_fields is not None:+ gran = act_fields.get("granularity") or {}+ mapping = act_fields.get("mapping_type") or {"_type": "MappingType", "_data": "SYMMETRIC"}+ if gran.get("_type") != "PerRow" or (gran.get("_data") or {}).get("dim", -1) != -1:+ raise _Unsupported(f"{name}: int8 activation granularity {gran}")+ if mapping.get("_data") != "SYMMETRIC":+ raise _Unsupported(f"{name}: int8 activation mapping {mapping}")+ reduce_range = reduce_range or bool(act_fields.get("reduce_range") or False)+ for d in keys.values():+ used.add(d)++ if v1 is not None and act_fields is not None and not reduce_range:+ # Exactly the objects torchao <= 0.17 unpickles from the hosted .pt (see INT8_V1_FACTS).+ impl = v1["PlainAQTTensorImpl"](qdata, scale.reshape(n), None, v1["PlainLayout"]())+ aqt = v1["AffineQuantizedTensor"](+ impl,+ (1, k),+ torch.Size((n, k)),+ quant_min = None,+ quant_max = None,+ zero_point_domain = v1["ZeroPointDomain"].NONE,+ dtype = dtype,+ )+ return v1["LinearActivationQuantizedTensor"](aqt, v1["act_quant"], {})++ if "Int8Tensor" not in api:+ raise _Unsupported(f"{name}: this torchao has no Int8Tensor")+ import inspect++ params = inspect.signature(api["Int8Tensor"].__new__).parameters+ kwargs: dict = {}+ if act_fields is not None:+ kwargs["act_quant_kwargs"] = _decode(act, api, name)+ if "reduce_range" in params:+ kwargs["reduce_range"] = bool(data.get("reduce_range") or False)+ elif data.get("reduce_range"):+ raise ValueError(+ f"int8 weight {name!r} records reduce_range=True, which this torchao cannot construct"+ )+ if zero_point is not None and "zero_point" in params:+ kwargs["zero_point"] = zero_point+ return api["Int8Tensor"](qdata, scale, [1, k], dtype, **kwargs)+++def _rebuild_fp8(name: str, entry: dict, tensors: dict, used: set, api: dict) -> Any:+ import torch++ data = entry.get("_data") or {}+ data_names = list(entry.get("_tensor_data_names") or [])+ if set(data_names) != {"qdata", "scale"}:+ raise _Unsupported(f"{name}: fp8 tensor data {data_names}")+ keys = {d: _flat_key(name, d) for d in data_names}+ missing = [k for k in keys.values() if k not in tensors]+ if missing:+ raise ValueError(+ f"fp8 weight {name!r}: missing {missing[0]!r}; the checkpoint is incomplete"+ )+ qdata, scale = tensors[keys["qdata"]], tensors[keys["scale"]]+ # As for int8: anything but a 2-D per-row or per-tensor weight is torchao's to read.+ if qdata.dtype not in (torch.float8_e4m3fn, torch.float8_e5m2) or qdata.dim() != 2:+ raise _Unsupported(f"{name}: fp8 qdata {qdata.dtype} {tuple(qdata.shape)}")+ n, k = qdata.shape+ block = list(data.get("block_size") or [])+ if not scale.is_floating_point() or not (+ (block == [1, k] and scale.numel() == n) or (block == [n, k] and scale.numel() == 1)+ ):+ raise _Unsupported(f"{name}: fp8 block size {block} with scale {tuple(scale.shape)}")+ known = {"block_size", "mm_config", "act_quant_kwargs", "kernel_preference", "dtype"}+ extra = {k2: v for k2, v in data.items() if k2 not in known}+ if any(v not in _INERT for v in extra.values()):+ raise _Unsupported(f"{name}: fp8 attributes {sorted(extra)}")+ dtype = _decode(data.get("dtype"), api, name)+ mm_config = _decode(data.get("mm_config"), api, name)+ act = _decode(data.get("act_quant_kwargs"), api, name)+ if act is not None and _granularity_name(getattr(act, "granularity", None)) not in (+ "PerRow",+ "PerTensor",+ ):+ raise _Unsupported(f"{name}: fp8 activation granularity")+ pref = _decode(data.get("kernel_preference"), api, name)+ for d in keys.values():+ used.add(d)+ kwargs = {"block_size": block, "mm_config": mm_config, "act_quant_kwargs": act, "dtype": dtype}+ if "kernel_preference" in data:+ kwargs["kernel_preference"] = pref+ return api["Float8Tensor"](qdata, scale, **kwargs)+++def read_quant_layout(raw: dict) -> dict:+ try:+ layout = json.loads(raw.get(QUANT_LAYOUT_KEY) or "{}")+ except Exception: # noqa: BLE001 - a corrupt optional key is ignored, not fatal+ return {}+ return layout if isinstance(layout, dict) else {}+++def int8_rebuilds_as_v1(raw: dict) -> bool:+ """Whether an int8 artifact converted from torchao v1 is rebuilt as v1 on THIS install."""+ layout = read_quant_layout(raw)+ if layout.get("int8_source") != INT8_SOURCE_V1:+ return False+ if (layout.get("int8_v1") or {}) != INT8_V1_FACTS:+ return False+ return _v1_int8_api() is not None+++def native_unflatten(+ tensors: dict,+ raw: dict,+ *,+ path: str = "",+) -> Optional[dict]:+ """``tensors`` + ``raw`` header -> state dict, or None when torchao's reader has to do it.++ ``tensors`` is not modified when this returns None. Raises ValueError for a checkpoint that is+ recognised but incomplete (a listed tensor or a weight part missing, a tensor the header does not+ list, a live field this torchao lacks): torchao's reader would fail the same file, later and less+ clearly. A weight of a layout this module does not model returns None instead.+ """+ api = _torchao_api()+ if api is None:+ return None+ try:+ names = json.loads(raw.get("tensor_names") or "null")+ except Exception: # noqa: BLE001+ return None+ if (+ not isinstance(names, list)+ or not all(isinstance(n, str) and n for n in names)+ or len(set(names)) != len(names)+ ):+ return None+ v1 = _v1_int8_api() if int8_rebuilds_as_v1(raw) else None+ out: dict = {}+ used: set = set()+ try:+ for name in names:+ try:+ entry = json.loads(raw.get(name) or "null")+ except Exception: # noqa: BLE001+ raise _Unsupported(f"{name}: unreadable entry") from None+ if not isinstance(entry, dict):+ raise _Unsupported(f"{name}: no entry")+ kind = entry.get("_type")+ if kind == "Tensor":+ if name not in tensors:+ raise ValueError(f"{path} lists {name!r} but has no such tensor")+ out[name] = tensors[name]+ used.add(name)+ elif "." not in name:+ raise _Unsupported(f"{name}: root-level subclass")+ elif kind == "Int8Tensor":+ out[name] = _rebuild_int8(name, entry, tensors, used, api, v1)+ elif kind == "Float8Tensor":+ out[name] = _rebuild_fp8(name, entry, tensors, used, api)+ else:+ raise _Unsupported(f"{name}: {kind}")+ except _Unsupported:+ return None+ leftover = sorted(set(tensors) - used)+ if leftover:+ raise ValueError(+ f"{path} has {len(leftover)} tensor(s) its header does not account for "+ f"(e.g. {leftover[0]!r}); the checkpoint is incomplete or was edited"+ )+ return out+++def quant_layout_header(state_dict: Any, *, int8_source: Optional[str] = None) -> dict:+ """The ``unsloth_quant_layout`` description a converter writes for ``state_dict``."""+ schemes = set()+ for value in state_dict.values():+ name = type(value).__name__+ if name in ("Int8Tensor", "LinearActivationQuantizedTensor"):+ schemes.add("int8")+ elif name == "Float8Tensor":+ schemes.add("fp8")+ layout: dict = {"version": QUANT_LAYOUT_VERSION, "weights": {}}+ if "int8" in schemes:+ layout["weights"]["int8"] = {+ "tensors": "<fqn>._weight_qdata int8 [out, in], <fqn>._weight_scale float [out, 1]",+ "dequant": "weight = qdata * scale (symmetric, per output row, no zero point)",+ "activation": "dynamic symmetric per-token int8",+ }+ if int8_source:+ layout["int8_source"] = int8_source+ if int8_source == INT8_SOURCE_V1:+ layout["int8_v1"] = dict(INT8_V1_FACTS)+ if "fp8" in schemes:+ layout["weights"]["fp8"] = {+ "tensors": "<fqn>._weight_qdata float8_e4m3fn [out, in], <fqn>._weight_scale float32 [out, 1]",+ "dequant": "weight = qdata * scale (per output row)",+ "activation": "dynamic float8 with the recorded granularity",+ }+ return layoutdiff --git a/studio/backend/core/inference/prequant_safetensors.py b/studio/backend/core/inference/prequant_safetensors.pyindex 630def0628a..febf12c9bca 100644--- a/studio/backend/core/inference/prequant_safetensors.py+++ b/studio/backend/core/inference/prequant_safetensors.py@@ -47,6 +47,7 @@ a repo can host both and each build reads the one it understands.from __future__ import annotationsimport json+import osimport refrom typing import Any, Iterable, Optional@@ -61,6 +62,14 @@ UNSLOTH_METADATA_KEY = "unsloth_metadata"# never sees a key it would try to rsplit on ".".UNSLOTH_ROOT_PREFIX = "unsloth_root::"UNSLOTH_ROOT_KEYS_KEY = "unsloth_root_tensors"+# Our own header entries: never torchao tensor entries, so never pruned (layout + source: prequant_native).+_UNSLOTH_HEADER_KEYS = (+ UNSLOTH_FORMAT_KEY,+ UNSLOTH_METADATA_KEY,+ UNSLOTH_ROOT_KEYS_KEY,+ "unsloth_quant_layout",+ "unsloth_source",+)# The torchao release that first shipped the flatten/unflatten pair under this import path. Below it# the helpers are absent and a safetensors artifact simply cannot be read, so the loader says so and@@ -361,7 +370,7 @@ def _header_without_inert_tensor_field(pruned = dict(header)victims = []for key, value in header.items():- if key in (UNSLOTH_FORMAT_KEY, UNSLOTH_METADATA_KEY, UNSLOTH_ROOT_KEYS_KEY):+ if key in _UNSLOTH_HEADER_KEYS:continuetry:parsed = json.loads(value)@@ -433,7 +442,7 @@ def _header_without_unconstructible_fields(removed: list = []pruned = {}for key, value in header.items():- if key in (UNSLOTH_FORMAT_KEY, UNSLOTH_METADATA_KEY, UNSLOTH_ROOT_KEYS_KEY):+ if key in _UNSLOTH_HEADER_KEYS:pruned[key] = valuecontinuetry:@@ -470,7 +479,106 @@ def _header_without_unconstructible_fields(return header-def load_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:+_ST_DTYPES = {+ "BF16": "bfloat16",+ "F16": "float16",+ "F32": "float32",+ "F64": "float64",+ "C64": "complex64",+ "F8_E4M3": "float8_e4m3fn",+ "F8_E5M2": "float8_e5m2",+ "F8_E8M0": "float8_e8m0fnu",+ "I8": "int8",+ "U8": "uint8",+ "I16": "int16",+ "U16": "uint16",+ "I32": "int32",+ "U32": "uint32",+ "I64": "int64",+ "U64": "uint64",+ "BOOL": "bool",+}++# safetensors refuses headers past 100 MB; a pre-quant header is a few MB at most.+_MAX_HEADER_BYTES = 100_000_000 # the safetensors parser's own limit+++def _mapped_tensors(path: str) -> tuple:+ """``(header metadata, {name: tensor})`` with every tensor a view of a private mapping of ``path``.++ ``safe_open(...).get_tensor`` copies each tensor into anonymous memory, so a 34 GB checkpoint costs+ 34 GB of host RAM before the first byte reaches the GPU; the pickle path maps the file instead+ (``prequant_mmap_enabled``). This maps it with the same primitive ``torch.load(mmap = True)`` uses+ (``UntypedStorage.from_file``, private), so the two containers cost the same on every OS: pages are+ read when ``.to(device)`` touches them, and a write never reaches the file. The header is checked+ the way safetensors checks it (known dtypes, non-negative integer shapes, byte ranges that tile the+ data section exactly, a bounded header); anything else raises and the caller re-reads unmapped.+ """+ import struct++ import torch++ size = os.path.getsize(path)+ with open(path, "rb") as fh:+ head = fh.read(8)+ if len(head) != 8:+ raise ValueError(f"{path} is not a safetensors file")+ (length,) = struct.unpack("<Q", head)+ if length <= 0 or length > _MAX_HEADER_BYTES or 8 + length > size:+ raise ValueError(f"{path} has a corrupt safetensors header")+ header = json.loads(fh.read(length))+ if not isinstance(header, dict):+ raise ValueError(f"{path} has a corrupt safetensors header")+ metadata = header.pop("__metadata__", None) or {}+ base = 8 + length+ entries = []+ for name, info in header.items():+ try:+ dtype = getattr(torch, _ST_DTYPES[info["dtype"]])+ start, end = (int(v) for v in info["data_offsets"])+ shape = list(info["shape"])+ except Exception: # noqa: BLE001 - unknown dtype (on this torch) or a malformed entry+ raise ValueError(f"{path}: unreadable tensor entry {name!r}") from None+ if not all(isinstance(d, int) and not isinstance(d, bool) and d >= 0 for d in shape):+ raise ValueError(f"{path}: tensor {name!r} has an invalid shape")+ count = 1+ for d in shape:+ count *= d+ itemsize = torch.empty((), dtype = dtype).element_size()+ if count * itemsize != end - start:+ raise ValueError(f"{path}: tensor {name!r} size does not match its shape")+ entries.append((start, end, name, dtype, shape, itemsize))+ entries.sort(key = lambda e: (e[0], e[1]))+ cursor = 0+ for start, end, name, *_ in entries:+ if start != cursor:+ raise ValueError(f"{path}: tensor {name!r} overlaps another or leaves a gap")+ cursor = end+ if cursor != size - base:+ raise ValueError(f"{path}: the data section does not match its header")+ storage = (+ torch.UntypedStorage.from_file(path, shared = False, nbytes = size) if size > base else None+ )+ tensors = {}+ for start, end, name, dtype, shape, itemsize in entries:+ offset = base + start+ if start == end or storage is None:+ tensors[name] = torch.empty(shape, dtype = dtype)+ elif offset % itemsize == 0:+ tensors[name] = torch.empty(0, dtype = dtype).set_(storage, offset // itemsize, shape)+ else:+ # safetensors packs tensors back to back, so one can start off its dtype's alignment: copy that one.+ raw = torch.empty(0, dtype = torch.uint8).set_(storage, offset, (end - start,))+ tensors[name] = raw.clone().view(dtype).reshape(shape)+ return dict(metadata), tensors+++def load_prequant_safetensors(+ path: str,+ *,+ device: str = "cpu",+ mmap: bool = False,+) -> dict:"""Read ``path`` into the SAME dict shape the pickle path returns.Returning ``{"format", "state_dict", "metadata"}`` rather than a new type is deliberate: every@@ -478,6 +586,9 @@ def load_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:activation-floor probe that reads the reconstructed tensors, the rotation biconditional, thekernel-preference pin) then runs unchanged on both containers, so the two formats cannot driftinto having different acceptance rules.++ ``mmap`` maps the file instead of copying it (CPU tensors only), as the pickle path does for an+ accelerator destination."""helpers = _torchao_helpers()if helpers is None:@@ -486,12 +597,14 @@ def load_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:f"{'.'.join(str(p) for p in MIN_TORCHAO_VERSION)} is required for ""torchao.prototype.safetensors.safetensors_support")- from safetensors import safe_open-_, unflatten = helpers- with safe_open(path, framework = "pt", device = device) as handle:- raw = dict(handle.metadata() or {})- tensors = {key: handle.get_tensor(key) for key in handle.keys()}+ if mmap and str(device) == "cpu":+ raw, tensors = _mapped_tensors(path)+ else:+ from safetensors import safe_open+ with safe_open(path, framework = "pt", device = device) as handle:+ raw = dict(handle.metadata() or {})+ tensors = {key: handle.get_tensor(key) for key in handle.keys()}fmt = raw.get(UNSLOTH_FORMAT_KEY)if not fmt:@@ -509,27 +622,31 @@ def load_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:for key in [k for k in tensors if k.startswith(UNSLOTH_ROOT_PREFIX)]}- # A newer torchao can record a field an older one's constructor does not take, which is how a- # published int8 checkpoint stopped loading. Dropped here when it is inert; see the helper.- raw = _header_without_unconstructible_fields(unflatten, tensors, raw, path = path)-- # torchao reads its OWN keys out of the same header; ours are namespaced and simply ignored. The second element- # is what it could NOT account for: a subclass missing one of its parts (a truncated or hand-edited file) is- # skipped there rather than raised, which would reach load_state_dict as a bare missing-key error saying nothing- # about the artifact. Name it here instead.- rebuilt = unflatten(tensors, raw)- state_dict = _first(rebuilt)- leftover = rebuilt[1] if isinstance(rebuilt, tuple) and len(rebuilt) > 1 else None- if leftover:- raise ValueError(- f"{path} has {len(leftover)} tensor(s) its header does not account for "- f"(e.g. {sorted(leftover)[0]!r}); the checkpoint is incomplete or was edited"- )+ # Unsloth's own int8 / fp8 rebuild, independent of torchao's deserializer accepting another release's kwargs;+ # None (nvfp4, mxfp8, ...) falls through to torchao below.+ from .prequant_native import native_rebuild_enabled, native_unflatten++ state_dict = native_unflatten(tensors, raw, path = path) if native_rebuild_enabled() else None+ reader = "native" if state_dict is not None else "torchao"+ if state_dict is None:+ # A newer torchao's field an older constructor lacks (how a published int8 file broke): dropped if inert.+ raw = _header_without_unconstructible_fields(unflatten, tensors, raw, path = path)++ # The second element is what torchao could not account for (truncated / hand-edited file): named here, not left+ # to surface as a bare missing-key error in load_state_dict.+ rebuilt = unflatten(tensors, raw)+ state_dict = _first(rebuilt)+ leftover = rebuilt[1] if isinstance(rebuilt, tuple) and len(rebuilt) > 1 else None+ if leftover:+ raise ValueError(+ f"{path} has {len(leftover)} tensor(s) its header does not account for "+ f"(e.g. {sorted(leftover)[0]!r}); the checkpoint is incomplete or was edited"+ )if roots:# Back under their bare names, so the caller's load_state_dict sees the shape the model# declares. A dotted key can never collide with one of these.state_dict.update(roots)- return {"format": str(fmt), "state_dict": state_dict, "metadata": metadata}+ return {"format": str(fmt), "state_dict": state_dict, "metadata": metadata, "reader": reader}def scheme_is_flattenable(quant_config: Any, *, features: int = 512) -> Optional[bool]:diff --git a/studio/backend/tests/test_diffusion_int8_fused.py b/studio/backend/tests/test_diffusion_int8_fused.pyindex 0e72ca2c718..a3785c1ae99 100644--- a/studio/backend/tests/test_diffusion_int8_fused.py+++ b/studio/backend/tests/test_diffusion_int8_fused.py@@ -76,6 +76,42 @@ def _clean_env(monkeypatch):yield+def test_int_mm_finds_safe_int_mm_after_torchao_kernel_was_removed(monkeypatch):+ """torchao main moved ``safe_int_mm`` out of ``torchao.kernel.intmm`` (and deleted ``torchao.kernel``):+ the fused forwards must find it in its new home, not fail every int8 render with ModuleNotFoundError."""+ import sys+ import types++ calls = []+ new_home = types.ModuleType("torchao.quantization.quantize_.workflows.int8.kernels")+ new_home.safe_int_mm = lambda a, b: calls.append((a, b)) or "out"+ monkeypatch.setitem(sys.modules, "torchao.kernel", None)+ monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", None)+ monkeypatch.setitem(sys.modules, new_home.__name__, new_home)+ monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)+ weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))+ a = torch.ones(4, 2, dtype = torch.int8)+ assert fused._int_mm(a, weight) == "out"+ assert len(calls) == 1 and calls[0][0] is a and tuple(calls[0][1].shape) == (2, 3)+++def test_int_mm_prefers_the_released_home(monkeypatch):+ """torchao <= 0.18 keeps ``torchao.kernel.intmm``: it wins over any other copy, so the capture-safe rebinding+ Studio installs there is the one that runs."""+ import sys+ import types++ old_home = types.ModuleType("torchao.kernel.intmm")+ old_home.safe_int_mm = lambda a, b: "old"+ new_home = types.ModuleType("torchao.quantization.quantize_.workflows.int8.kernels")+ new_home.safe_int_mm = lambda a, b: "new"+ monkeypatch.setitem(sys.modules, "torchao.kernel.intmm", old_home)+ monkeypatch.setitem(sys.modules, new_home.__name__, new_home)+ monkeypatch.setattr(fused, "_INTMM_MODULE", None, raising = False)+ weight = types.SimpleNamespace(qdata = torch.ones(3, 2, dtype = torch.int8))+ assert fused._int_mm(torch.ones(4, 2, dtype = torch.int8), weight) == "old"++@pytest.mark.parametrize("version, ok",[diff --git a/studio/backend/tests/test_diffusion_peft_torchao_guard.py b/studio/backend/tests/test_diffusion_peft_torchao_guard.pyindex b12d676d425..683949ff776 100644--- a/studio/backend/tests/test_diffusion_peft_torchao_guard.py+++ b/studio/backend/tests/test_diffusion_peft_torchao_guard.py@@ -48,6 +48,21 @@ def test_studio_copy_matches_unsloth_import_fixes(name):), f"{name} drifted from unsloth/import_fixes.py; port the change to both copies."+def _assigned_value_dump(path: Path, name: str) -> str:+ for node in ast.parse(path.read_text(encoding = "utf-8")).body:+ if isinstance(node, ast.Assign) and any(+ isinstance(t, ast.Name) and t.id == name for t in node.targets+ ):+ return ast.dump(node.value)+ raise AssertionError(f"{path} has no top-level {name}")+++def test_studio_and_unsloth_match_the_same_torchao_removals():+ """The matcher is shared state too: when unsloth patched first, its matcher is the one that runs."""+ name = "_PEFT_TORCHAO_MISSING_TENSOR_SUBCLASS"+ assert _assigned_value_dump(_PATCH_MODULE, name) == _assigned_value_dump(_IMPORT_FIXES, name)++@pytest.fixture(autouse = True)def _restore_peft_dispatchers(monkeypatch):"""The patcher wraps every loaded peft copy, real ones included. Put them back afterwards so a@@ -139,6 +154,59 @@ def test_an_unrelated_import_error_still_surfaces(monkeypatch, fake_peft):lora_model.dispatch_torchao(types.SimpleNamespace(weight = object()), "default", None)+def test_a_torchao_without_the_dtypes_package_gets_a_plain_lora_layer(monkeypatch, fake_peft):+ """torchao main (after 0.18) deleted the whole ``torchao.dtypes`` package, so peft <= 0.18's+ first import inside ``dispatch_torchao`` is a ModuleNotFoundError naming the package, not the+ class. That is the same removal, and every LoRA target must still fall through to peft's+ ordinary layer instead of failing the load."""+ lora_torchao, lora_model = fake_peft+ monkeypatch.delitem(+ sys.modules, "torchao.dtypes"+ ) # the fake torchao has no __path__: the import fails+ monkeypatch.delattr(sys.modules["torchao"], "dtypes")+ monkeypatch.setattr(+ sys.modules["torchao"], "__spec__", importlib.machinery.ModuleSpec("torchao", None)+ )++ # peft <= 0.18's first torchao import, defined in the fake peft module so the guard reads its globals.+ namespace = dict(lora_torchao.__dict__)+ exec(+ "def dispatch_torchao(target, adapter_name, lora_config, **kwargs):\n"+ " from torchao.dtypes import AffineQuantizedTensor\n",+ namespace,+ )+ removed = namespace["dispatch_torchao"]+ monkeypatch.setattr(lora_torchao, "dispatch_torchao", removed)+ monkeypatch.setattr(lora_model, "dispatch_torchao", removed)+ with pytest.raises(ModuleNotFoundError, match = "torchao.dtypes"):+ removed(types.SimpleNamespace(weight = object()), "default", None)+ patches._patch_peft_torchao_dispatchers()+ assert (+ lora_model.dispatch_torchao(types.SimpleNamespace(weight = object()), "default", None) is None+ )+++ "message",+ (+ "No module named 'torchao'",+ "No module named 'torchao.dtypes.uintx'",+ "No module named 'torchao.dtypesx'",+ ),+)+def test_other_missing_modules_still_surface(monkeypatch, fake_peft, message):+ lora_torchao, lora_model = fake_peft++ def broken(target, adapter_name, lora_config, **kwargs):+ raise ModuleNotFoundError(message)++ monkeypatch.setattr(lora_torchao, "dispatch_torchao", broken)+ monkeypatch.setattr(lora_model, "dispatch_torchao", broken)+ patches._patch_peft_torchao_dispatchers()+ with pytest.raises(ModuleNotFoundError):+ lora_model.dispatch_torchao(types.SimpleNamespace(weight = object()), "default", None)++def test_the_guard_waits_for_peft_instead_of_importing_it(monkeypatch):monkeypatch.delitem(sys.modules, patches._PEFT_LORA_DISPATCH_MODULE, raising = False)monkeypatch.setattr(diff --git a/studio/backend/tests/test_prequant_native.py b/studio/backend/tests/test_prequant_native.pynew file mode 100644index 00000000000..5a83b066593--- /dev/null+++ b/studio/backend/tests/test_prequant_native.py@@ -0,0 +1,714 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Safetensors pre-quant checkpoints rebuilt without torchao's deserializer, the local mirror, and the converter."""++from __future__ import annotations++import importlib.util+import inspect+import json+from pathlib import Path++import pytest++import core.inference.diffusion_prequant as pq+import core.inference.prequant_native as pn+import core.inference.prequant_safetensors as ps++torch = pytest.importorskip("torch")+torchao = pytest.importorskip("torchao")++_HAVE_HELPERS = ps.safetensors_prequant_supported()+needs_helpers = pytest.mark.skipif(+ not _HAVE_HELPERS, reason = "needs torchao >= 0.16 flatten helpers"+)+_V1 = pn._v1_int8_api() is not None+_API = pn._torchao_api() or {}+needs_int8tensor = pytest.mark.skipif("Int8Tensor" not in _API, reason = "needs torchao Int8Tensor")++REPO_ROOT = Path(__file__).resolve().parents[3]+++def _int8_tensor(+ n = 8,+ k = 32,+ seed = 0,+):+ from core.inference.prequant_legacy_int8 import _int8_tensor_api, _to_int8_tensor++ g = torch.Generator().manual_seed(seed)+ qdata = torch.randint(-127, 128, (n, k), dtype = torch.int8, generator = g)+ scale = (torch.rand(n, generator = g) / 50 + 1e-3).to(torch.bfloat16)+ return _to_int8_tensor(qdata, scale, torch.bfloat16, _int8_tensor_api())+++def _fp8_tensor(+ n = 8,+ k = 32,+ seed = 0,+):+ from torchao.float8.inference import Float8MMConfig+ from torchao.quantization import Float8Tensor, PerRow+ from torchao.quantization.quantize_.common.kernel_preference import KernelPreference+ from torchao.quantization.quantize_.workflows.float8.float8_tensor import (+ QuantizeTensorToFloat8Kwargs,+ )++ g = torch.Generator().manual_seed(seed)+ qdata = (torch.randn(n, k, generator = g) * 50).to(torch.float8_e4m3fn)+ scale = torch.rand(n, 1, generator = g) / 100 + 1e-4+ act = QuantizeTensorToFloat8Kwargs(+ float8_dtype = torch.float8_e4m3fn,+ granularity = PerRow(),+ mm_config = None,+ hp_value_lb = 1e-12,+ hp_value_ub = None,+ kernel_preference = KernelPreference.AUTO,+ )+ return Float8Tensor(+ qdata,+ scale,+ block_size = [1, k],+ mm_config = Float8MMConfig(use_fast_accum = True),+ act_quant_kwargs = act,+ kernel_preference = KernelPreference.AUTO,+ dtype = torch.bfloat16,+ )+++def _write(+ path,+ state_dict,+ *,+ layout = None,+ scheme = "int8",+):+ from safetensors.torch import save_file++ flatten, _ = ps._torchao_helpers()+ roots = ps._root_level_keys(state_dict)+ flat, meta = flatten({k: v for k, v in state_dict.items() if k not in roots})+ header = dict(meta)+ header[ps.UNSLOTH_FORMAT_KEY] = pq.PREQUANT_FORMAT+ header[ps.UNSLOTH_METADATA_KEY] = json.dumps({"scheme": scheme})+ if layout is not None:+ header[pn.QUANT_LAYOUT_KEY] = json.dumps(layout)+ flat = dict(flat)+ if roots:+ for k in roots:+ flat[ps.UNSLOTH_ROOT_PREFIX + k] = state_dict[k]+ header[ps.UNSLOTH_ROOT_KEYS_KEY] = json.dumps(roots)+ save_file({k: v.contiguous() for k, v in flat.items()}, str(path), metadata = header)+ return path+++def _canon(value):+ if type(value) is torch.Tensor:+ return ("Tensor", value)+ names, ctx = value.__tensor_flatten__()+ return (type(value).__name__, {n: _canon(getattr(value, n)) for n in names}, repr(ctx))+++def _same(a, b):+ if isinstance(a, torch.Tensor) and isinstance(b, torch.Tensor):+ return a.dtype == b.dtype and a.shape == b.shape and torch.equal(a, b)+ if isinstance(a, (tuple, list)) and isinstance(b, (tuple, list)):+ return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b, strict = True))+ if isinstance(a, dict) and isinstance(b, dict):+ return a.keys() == b.keys() and all(_same(a[k], b[k]) for k in a)+ return a == b+++def _v1_layout():+ return {"version": 1, "int8_source": pn.INT8_SOURCE_V1, "int8_v1": dict(pn.INT8_V1_FACTS)}+++@needs_helpers+@needs_int8tensor+def test_native_rebuild_matches_torchao_unflatten(tmp_path, monkeypatch):+ sd = {+ "blk.q.weight": _int8_tensor(),+ "blk.f.weight": _fp8_tensor(),+ "blk.norm.weight": torch.ones(32, dtype = torch.bfloat16),+ "pad_token": torch.zeros(1, 32, dtype = torch.bfloat16),+ }+ path = _write(tmp_path / "M-INT8.safetensors", sd)+ native = ps.load_prequant_safetensors(str(path))+ monkeypatch.setenv(pn.NATIVE_REBUILD_ENV, "0")+ stock = ps.load_prequant_safetensors(str(path))+ assert native["reader"] == "native" and stock["reader"] == "torchao"+ assert native["state_dict"].keys() == stock["state_dict"].keys() == sd.keys()+ for key in sd:+ assert _same(_canon(native["state_dict"][key]), _canon(stock["state_dict"][key])), key+ assert _same(_canon(native["state_dict"][key]), _canon(sd[key])), key+++@needs_helpers+@needs_int8tensor+def test_mapped_read_is_the_same_tensors_as_a_copied_read(tmp_path):+ sd = {"blk.q.weight": _int8_tensor(), "blk.f.weight": _fp8_tensor(), "x": torch.arange(4.0)}+ path = _write(tmp_path / "M-INT8.safetensors", sd)+ mapped = ps.load_prequant_safetensors(str(path), mmap = True)["state_dict"]+ copied = ps.load_prequant_safetensors(str(path))["state_dict"]+ for key in sd:+ assert _same(_canon(mapped[key]), _canon(copied[key])), key+++def test_a_tensor_off_its_dtype_alignment_is_still_read(tmp_path):+ a = torch.tensor([1, -2, 3], dtype = torch.int8)+ b = torch.tensor([1.5, -0.25], dtype = torch.bfloat16)+ header = {+ "a": {"dtype": "I8", "shape": [3], "data_offsets": [0, 3]},+ "b": {"dtype": "BF16", "shape": [2], "data_offsets": [3, 7]},+ }+ raw = json.dumps(header).encode()+ raw += b" " * (-len(raw) % 8)+ path = tmp_path / "odd.safetensors"+ path.write_bytes(+ len(raw).to_bytes(8, "little")+ + raw+ + a.numpy().tobytes()+ + b.view(torch.int16).numpy().tobytes()+ )+ _, tensors = ps._mapped_tensors(str(path))+ assert torch.equal(tensors["a"], a) and torch.equal(tensors["b"], b)+++def test_a_truncated_header_is_refused(tmp_path):+ path = tmp_path / "bad.safetensors"+ path.write_bytes((10_000).to_bytes(8, "little") + b"{}")+ with pytest.raises(ValueError):+ ps._mapped_tensors(str(path))+++@needs_helpers+@needs_int8tensor+def test_v1_converted_int8_rebuilds_as_v1_only_where_torchao_ships_it(tmp_path):+ sd = {"blk.q.weight": _int8_tensor(), "blk.norm.weight": torch.ones(32, dtype = torch.bfloat16)}+ path = _write(tmp_path / "M-INT8.safetensors", sd, layout = _v1_layout())+ w = ps.load_prequant_safetensors(str(path))["state_dict"]["blk.q.weight"]+ if _V1:+ assert type(w).__name__ == "LinearActivationQuantizedTensor"+ impl = w.original_weight_tensor.tensor_impl+ assert torch.equal(impl.int_data, sd["blk.q.weight"].qdata)+ assert torch.equal(impl.scale, sd["blk.q.weight"].scale.reshape(-1))+ assert impl.zero_point is None+ assert w.input_quant_func.__name__ == "_int8_symm_per_token_reduced_range_quant"+ else:+ assert type(w).__name__ == "Int8Tensor"+ # Without the v1 declaration the artifact keeps the class it was written as, on every torchao.+ plain = _write(tmp_path / "P-INT8.safetensors", sd)+ assert (+ type(ps.load_prequant_safetensors(str(plain))["state_dict"]["blk.q.weight"]).__name__+ == "Int8Tensor"+ )++[email protected](not _V1, reason = "needs torchao <= 0.17 (v1 int8 classes)")+@needs_helpers+def test_v1_rebuild_is_bit_identical_to_a_real_v1_quantize(tmp_path):+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ from core.inference.prequant_legacy_int8 import convert_legacy_int8_weights++ model = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False)).to(torch.bfloat16)+ quantize_(model, Int8DynamicActivationInt8WeightConfig())+ v1 = model[0].weight+ assert type(v1).__name__ == "LinearActivationQuantizedTensor"+ reference = dict(model.state_dict())+ converted = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False)).to(torch.bfloat16)+ converted.load_state_dict(reference, assign = True)+ assert convert_legacy_int8_weights(converted) == 1+ path = _write(+ tmp_path / "M-INT8.safetensors", dict(converted.state_dict()), layout = _v1_layout()+ )+ back = ps.load_prequant_safetensors(str(path))["state_dict"]+ assert _same(_canon(back["0.weight"]), _canon(v1))+ x = torch.randn(17, 64, dtype = torch.bfloat16)+ assert torch.equal(+ torch.nn.functional.linear(x, back["0.weight"]), torch.nn.functional.linear(x, v1)+ )+++@needs_helpers+@needs_int8tensor+def test_unknown_entries_fall_back_to_torchao_and_leave_the_tensors_alone(tmp_path):+ sd = {"blk.q.weight": _int8_tensor()}+ flatten, _ = ps._torchao_helpers()+ flat, meta = flatten(sd)+ raw = dict(meta)+ entry = json.loads(raw["blk.q.weight"])+ entry["_type"] = "SomeFutureTensor"+ raw["blk.q.weight"] = json.dumps(entry)+ before = dict(flat)+ assert pn.native_unflatten(flat, raw) is None+ assert flat.keys() == before.keys()+++@needs_helpers+@needs_int8tensor+def test_a_missing_scale_or_a_stray_tensor_is_named(tmp_path):+ flatten, _ = ps._torchao_helpers()+ flat, meta = flatten({"blk.q.weight": _int8_tensor()})+ no_scale = {k: v for k, v in flat.items() if not k.endswith("_scale")}+ with pytest.raises(ValueError, match = "missing"):+ pn.native_unflatten(no_scale, dict(meta), path = "x")+ stray = dict(flat)+ stray["blk.other._weight_qdata"] = torch.zeros(2, 2, dtype = torch.int8)+ with pytest.raises(ValueError, match = "does not account for"):+ pn.native_unflatten(stray, dict(meta), path = "x")+++@needs_helpers+@needs_int8tensor+def test_a_live_field_this_torchao_lacks_is_refused_and_an_inert_one_is_dropped():+ flatten, _ = ps._torchao_helpers()+ flat, meta = flatten({"blk.f.weight": _fp8_tensor()})+ entry = json.loads(meta["blk.f.weight"])+ entry["_data"]["mm_config"]["_data"]["some_new_knob"] = False+ inert = dict(meta, **{"blk.f.weight": json.dumps(entry)})+ out = pn.native_unflatten(dict(flat), inert)+ assert type(out["blk.f.weight"]).__name__ == "Float8Tensor"+ entry["_data"]["mm_config"]["_data"]["some_new_knob"] = 3+ live = dict(meta, **{"blk.f.weight": json.dumps(entry)})+ with pytest.raises(ValueError, match = "some_new_knob"):+ pn.native_unflatten(dict(flat), live)+++@needs_helpers+@needs_int8tensor+def test_layouts_the_native_reader_does_not_model_go_to_torchao():+ from torchao.quantization import Float8Tensor, PerRow, PerTensor+ from torchao.quantization.quantize_.workflows.int8.int8_tensor import (+ Int8Tensor,+ QuantizeTensorToInt8Kwargs,+ )++ flatten, _ = ps._torchao_helpers()+ per_tensor = Int8Tensor.from_hp(+ torch.randn(16, 32, dtype = torch.bfloat16),+ PerTensor(),+ act_quant_kwargs = QuantizeTensorToInt8Kwargs(granularity = PerRow()),+ )+ experts = Float8Tensor.from_hp(+ torch.randn(2, 16, 32, dtype = torch.bfloat16), granularity = PerRow()+ )+ for weight in (per_tensor, experts):+ flat, meta = flatten({"blk.w.weight": weight})+ assert pn.native_unflatten(dict(flat), dict(meta)) is None+ # an int8 field this reader does not know is torchao's call too, never silently dropped+ flat, meta = flatten({"blk.w.weight": _int8_tensor()})+ entry = json.loads(meta["blk.w.weight"])+ entry["_data"]["future_live_field"] = "per_block_128"+ assert (+ pn.native_unflatten(dict(flat), dict(meta, **{"blk.w.weight": json.dumps(entry)})) is None+ )+++@needs_helpers+@needs_int8tensor+def test_duplicate_names_and_unknown_granularity_fields_go_to_torchao():+ flatten, _ = ps._torchao_helpers()+ flat, meta = flatten({"blk.f.weight": _fp8_tensor()})+ dup = dict(meta, tensor_names = json.dumps(["blk.f.weight", "blk.f.weight"]))+ assert pn.native_unflatten(dict(flat), dup) is None+ entry = json.loads(meta["blk.f.weight"])+ text = json.dumps(entry)+ assert '"_type": "PerRow", "_data": {"dim": -1}' in text+ odd = dict(+ meta,+ **{+ "blk.f.weight": text.replace(+ '"_type": "PerRow", "_data": {"dim": -1}',+ '"_type": "PerRow", "_data": {"dim": -1, "block": 128}',+ )+ },+ )+ assert pn.native_unflatten(dict(flat), odd) is None+++def test_overlapping_tensor_bytes_are_refused(tmp_path):+ header = {+ "x": {"dtype": "F32", "shape": [4], "data_offsets": [0, 16]},+ "y": {"dtype": "F32", "shape": [4], "data_offsets": [0, 16]},+ }+ raw = json.dumps(header).encode()+ raw += b" " * (-len(raw) % 8)+ path = tmp_path / "overlap.safetensors"+ path.write_bytes(len(raw).to_bytes(8, "little") + raw + bytes(16))+ with pytest.raises(ValueError):+ ps._mapped_tensors(str(path))++[email protected](not hasattr(torch, "float8_e8m0fnu"), reason = "needs float8_e8m0fnu")+def test_the_mapped_read_takes_mxfp8_scale_dtypes(tmp_path):+ from safetensors.torch import save_file++ t = torch.ones(8, 4).to(torch.float8_e8m0fnu)+ save_file({"a.b": t}, str(tmp_path / "e.safetensors"))+ _, tensors = ps._mapped_tensors(str(tmp_path / "e.safetensors"))+ assert tensors["a.b"].dtype == torch.float8_e8m0fnu and torch.equal(+ tensors["a.b"].view(torch.uint8), t.view(torch.uint8)+ )+++def _mirror(tmp_path, monkeypatch, *files):+ root = tmp_path / "mirror"+ out = []+ for rel in files:+ path = root / rel+ path.parent.mkdir(parents = True, exist_ok = True)+ path.write_bytes(b"x")+ out.append(path)+ monkeypatch.setenv(pq.PREQUANT_MIRROR_ENV, str(root))+ return out+++def test_a_mirrored_sibling_this_host_cannot_read_is_not_served(tmp_path, monkeypatch):+ _mirror(tmp_path, monkeypatch, "unsloth/Model-INT8/Model-INT8.safetensors")+ monkeypatch.setattr(ps, "safetensors_prequant_supported", lambda: False)+ monkeypatch.setattr(pq, "_register_prequant_safe_globals", lambda: True)+ monkeypatch.setattr(pq, "_download_checkpoint_name", lambda *a, **k: "/hub/Model-INT8.pt")+ src = pq.PrequantSource(+ kind = "repo",+ location = "unsloth/Model-INT8",+ filename = "Model-INT8.pt",+ declared_filenames = ("Model-INT8.pt",),+ )+ assert pq._resolve_checkpoint_path(src, None, None, scheme = "int8") == "/hub/Model-INT8.pt"+ cached = pq.cached_checkpoint_path(src)+ assert cached is None or not ps.is_safetensors_checkpoint(cached)+++def test_any_mirrored_name_wins_before_the_hub_is_asked(tmp_path, monkeypatch):+ (pt,) = _mirror(tmp_path, monkeypatch, "unsloth/Model-INT8/Model-INT8.pt")+ monkeypatch.setattr(+ pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True+ )+ calls = []+ monkeypatch.setattr(+ pq,+ "_download_checkpoint_name",+ lambda source, name, *a, **k: calls.append(name) or f"/hub/{name}",+ )+ src = pq.PrequantSource(+ kind = "repo",+ location = "unsloth/Model-INT8",+ filename = "Model-INT8.safetensors",+ fallback_filenames = ("Model-INT8.pt",),+ )+ assert (+ pq._resolve_checkpoint_path(src, None, None, scheme = "int8") == str(pt.resolve())+ and calls == []+ )+++def test_the_text_encoder_mirror_is_used_when_the_hub_is_unreachable(tmp_path, monkeypatch):+ import huggingface_hub+ from huggingface_hub.errors import LocalEntryNotFoundError++ import core.inference.diffusion_te_prequant as te++ (pt,) = _mirror(tmp_path, monkeypatch, "unsloth/Model-FP8/Model-text_encoder-FP8.pt")++ def offline(*a, **k):+ raise LocalEntryNotFoundError("connection error")++ monkeypatch.setattr(huggingface_hub, "hf_hub_download", offline)+ src = te.TePrequantSource(+ kind = "repo",+ location = "unsloth/Model-FP8",+ filename = "Model-text_encoder-FP8.safetensors",+ fallback_filenames = ("Model-text_encoder-FP8.pt",),+ )+ assert te._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path / "hub")) == str(+ pt.resolve()+ )+++def test_the_mirror_is_read_before_the_hub_and_cannot_escape_its_root(tmp_path, monkeypatch):+ root = tmp_path / "mirror"+ target = root / "unsloth" / "Model-FP8" / "Model-INT8.safetensors"+ target.parent.mkdir(parents = True)+ target.write_bytes(b"x")+ outside = tmp_path / "secret.safetensors"+ outside.write_bytes(b"x")+ monkeypatch.setenv(pq.PREQUANT_MIRROR_ENV, str(root))+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "Model-INT8.safetensors") == str(+ target.resolve()+ )+ # a request for the pickle gets its converted sibling; a pickle with no sibling is not mirrored+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "Model-INT8.pt") == str(target.resolve())+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "Model-FP8.pt") is None+ assert pq.prequant_mirror_path("unsloth/..", "secret.safetensors") is None+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "../../../secret.safetensors") is None+ monkeypatch.delenv(pq.PREQUANT_MIRROR_ENV)+ assert pq.prequant_mirror_path("unsloth/Model-FP8", "Model-INT8.safetensors") is None+++def test_the_resolver_takes_the_mirrored_safetensors_without_asking_the_hub(tmp_path, monkeypatch):+ root = tmp_path / "mirror"+ target = root / "unsloth" / "Model-FP8" / "Model-INT8.safetensors"+ target.parent.mkdir(parents = True)+ target.write_bytes(b"x")+ monkeypatch.setenv(pq.PREQUANT_MIRROR_ENV, str(root))+ monkeypatch.setattr(+ pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True+ )++ def no_hub(*args, **kwargs):+ raise AssertionError("the Hub was asked for a mirrored file")++ monkeypatch.setattr(pq, "_download_checkpoint_name", no_hub)+ src = pq.PrequantSource(+ kind = "repo",+ location = "unsloth/Model-FP8",+ filename = "Model-INT8.safetensors",+ fallback_filenames = ("Model-INT8.pt",),+ )+ assert pq._resolve_checkpoint_path(src, None, None, scheme = "int8") == str(target.resolve())+ assert pq.cached_checkpoint_path(src) == str(target.resolve())+++def test_the_text_encoder_resolver_reads_the_mirror_too(tmp_path, monkeypatch):+ import core.inference.diffusion_te_prequant as te++ root = tmp_path / "mirror"+ target = root / "unsloth" / "Model-FP8" / "Model-text_encoder-FP8.safetensors"+ target.parent.mkdir(parents = True)+ target.write_bytes(b"x")+ monkeypatch.setenv(pq.PREQUANT_MIRROR_ENV, str(root))+ import huggingface_hub++ def no_hub(*args, **kwargs):+ raise AssertionError("the Hub was asked for a mirrored file")++ monkeypatch.setattr(huggingface_hub, "hf_hub_download", no_hub)+ src = te.TePrequantSource(+ kind = "repo",+ location = "unsloth/Model-FP8",+ filename = "Model-text_encoder-FP8.safetensors",+ fallback_filenames = ("Model-text_encoder-FP8.pt",),+ )+ assert te._resolve_checkpoint_path(src, None, cache_dir = str(tmp_path / "hub")) == str(+ target.resolve()+ )+++def test_the_dense_fast_path_reason_names_a_load_time_failure(monkeypatch):+ import core.inference.diffusion as diffusion++ monkeypatch.setattr(diffusion, "prequant_unreadable_reason", lambda *a, **k: None)+ reason = diffusion._dense_fast_path_reason(+ object(), "int8", "org/base", "pipeline", None, None, failure = "ValueError: bad header"+ )+ assert "did not load (ValueError: bad header)" in reason and "quantized instead" in reason+ assert (+ diffusion._dense_fast_path_reason(object(), "int8", "org/base", "pipeline", None, None)+ == "engaged on the dense fast path"+ )+++ "message, expected",+ [+ (+ "/srv/hf/hub/models--unsloth--X/snapshots/abc/X-INT8.safetensors: the data section does not match its header",+ "ValueError: X-INT8.safetensors: the data section does not match its header",+ ),+ (+ r"C:\Users\me\mirror\unsloth\X\X-FP8.safetensors is truncated",+ "ValueError: X-FP8.safetensors is truncated",+ ),+ (+ "unsloth/X-FP8 has no X-FP8.pt (https://huggingface.co/unsloth/X-FP8)",+ "ValueError: unsloth/X-FP8 has no X-FP8.pt (https://huggingface.co/unsloth/X-FP8)",+ ),+ ],+)+def test_the_status_failure_note_carries_no_server_directory(message, expected):+ import core.inference.diffusion_prequant as dp+ dp._warn(None, "load", ValueError(message))+ assert dp.last_prequant_failure() == expected+++def test_every_load_starts_without_the_previous_loads_failure_note():+ import core.inference.diffusion as diffusion++ src = inspect.getsource(diffusion.DiffusionBackend.load_pipeline)+ reset = src.find("self._prequant_fallback_note = None")+ read = src.find('getattr(self, "_prequant_fallback_note", None)')+ assert 0 <= reset < read+++def _converter():+ spec = importlib.util.spec_from_file_location(+ "convert_prequant_to_safetensors",+ REPO_ROOT / "scripts" / "convert_prequant_to_safetensors.py",+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _legacy_writer():+ spec = importlib.util.spec_from_file_location(+ "_legacy_int8_fixture", Path(__file__).with_name("test_prequant_legacy_int8.py")+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def no_foreign_legacy_registrations():+ """Other test files register placeholders under the v1 int8 names process-wide (see test_prequant_legacy_int8)."""+ import core.inference.prequant_legacy_int8 as li++ saved = torch.serialization.get_safe_globals()+ legacy = {*li.LEGACY_INT8_CLASS_NAMES, li._ACT_QUANT}+ torch.serialization.clear_safe_globals()+ torch.serialization.add_safe_globals(+ [g for g in saved if not (isinstance(g, tuple) and g[1] in legacy)]+ )+ yield+ torch.serialization.clear_safe_globals()+ torch.serialization.add_safe_globals(saved)+++@needs_helpers+@needs_int8tensor+def test_the_converter_writes_a_bit_identical_v1_int8_artifact(+ tmp_path, monkeypatch, no_foreign_legacy_registrations+):+ src = tmp_path / "Model-INT8.pt"+ if _V1:+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ model = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False)).to(torch.bfloat16)+ quantize_(model, Int8DynamicActivationInt8WeightConfig())+ sd = {+ "blk.proj.weight": model[0].weight,+ "blk.norm.weight": torch.ones(64, dtype = torch.bfloat16),+ }+ torch.save(+ {"format": pq.PREQUANT_FORMAT, "metadata": {"scheme": "int8"}, "state_dict": sd},+ str(src),+ )+ else:+ import core.inference.prequant_legacy_int8 as li+ if not li.legacy_int8_decode_supported():+ pytest.skip("no v1 classes and no legacy decoder")+ _legacy_writer()._write_legacy_checkpoint(monkeypatch, src)+ conv = _converter()+ rec = conv.convert(+ str(src), str(tmp_path / "out"), repo = "unsloth/Model-FP8", revision = "abc", sha = None+ )+ assert rec["verified_bit_identical"] and rec["int8_source"] == pn.INT8_SOURCE_V1+ dst = tmp_path / "out" / "unsloth" / "Model-FP8" / "Model-INT8.safetensors"+ assert rec["dst"] == str(dst) and dst.is_file()+ header = ps.read_prequant_header(str(dst))+ assert header["format"] == pq.PREQUANT_FORMAT and header["metadata"] == {"scheme": "int8"}+ from safetensors import safe_open++ with safe_open(str(dst), framework = "pt") as handle:+ raw = handle.metadata()+ assert json.loads(raw[pn.SOURCE_KEY])["sha256"] == conv.sha256_of(str(src))+ assert json.loads(raw[pn.QUANT_LAYOUT_KEY])["int8_v1"] == pn.INT8_V1_FACTS+ # the same pickle, read by Studio, and the new file, read by Studio: same objects, same bytes+ a = pq._load_prequant_checkpoint(str(src), map_location = "cpu")["state_dict"]+ b = pq._load_prequant_checkpoint(str(dst), map_location = "cpu", mmap = True)["state_dict"]+ assert conv.compare_state_dicts(a, b) == []+++def test_the_converter_writes_a_text_encoder_with_tied_weights(tmp_path):+ from core.inference.diffusion_te_prequant import TE_PREQUANT_FORMAT++ embed = torch.randn(16, 8).to(torch.float8_e4m3fn)+ sd = {+ "model.embed_tokens.weight": embed,+ "lm_head.weight": embed,+ "model.norm.weight": torch.ones(8),+ }+ src = tmp_path / "Model-text_encoder-FP8.pt"+ torch.save(+ {+ "format": TE_PREQUANT_FORMAT,+ "metadata": {"scheme": "fp8", "component": "text_encoder"},+ "state_dict": sd,+ },+ str(src),+ )+ rec = _converter().convert(str(src), str(tmp_path / "out"), repo = None, revision = None, sha = None)+ assert rec["kind"] == "text_encoder" and rec["verified_bit_identical"]+ assert rec["dst"].endswith("Model-text_encoder-FP8.safetensors")+ pth = tmp_path / "Other-text_encoder-FP8.pth"+ pth.write_bytes(src.read_bytes())+ assert (+ _converter()+ .convert(str(pth), str(tmp_path / "out"), repo = None, revision = None, sha = None)["dst"]+ .endswith("Other-text_encoder-FP8.safetensors")+ )+ back = ps.load_plain_prequant_safetensors(rec["dst"])+ assert back["format"] == TE_PREQUANT_FORMAT+ for key in sd:+ assert back["state_dict"][key].dtype == sd[key].dtype and torch.equal(+ back["state_dict"][key].float(), sd[key].float()+ )+++def test_the_converters_comparison_sees_every_kind_of_difference():+ conv = _converter()+ w = _int8_tensor()+ assert (+ conv.compare_state_dicts({"a.w": w, "b": torch.zeros(2)}, {"a.w": w, "b": torch.zeros(2)})+ == []+ )+ assert conv.compare_state_dicts({"a.w": w}, {"a.w": _int8_tensor(seed = 1)}) == ["a.w"]+ assert conv.compare_state_dicts({"b": torch.zeros(2)}, {"b": -torch.zeros(2)}) == ["b"]+ assert conv.compare_state_dicts(+ {"b": torch.zeros(2)}, {"b": torch.zeros(2, dtype = torch.float64)}+ ) == ["b"]+ assert conv.compare_state_dicts({"b": torch.zeros(2)}, {"c": torch.zeros(2)}) == ["b", "c"]+ nan = torch.tensor([float("nan")])+ assert conv.compare_state_dicts({"b": nan}, {"b": nan.clone()}) == []+++def test_the_converter_refuses_v1_weights_off_the_supported_layout_and_restores_the_decoder(+ tmp_path, monkeypatch, no_foreign_legacy_registrations+):+ import core.inference.prequant_legacy_int8 as li++ conv = _converter()+ original = li._rebuild_weight+ restore = conv._record_v1_facts([])+ assert li._rebuild_weight is not original+ restore()+ assert li._rebuild_weight is original++ def odd_facts(w, standins = None):+ return dict(pn.INT8_V1_FACTS, quant_kwargs = {"reduce_range": False})++ monkeypatch.setattr(conv, "_v1_facts_of", odd_facts)+ src = tmp_path / "Model-INT8.pt"+ if _V1:+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ model = torch.nn.Sequential(torch.nn.Linear(64, 32, bias = False)).to(torch.bfloat16)+ quantize_(model, Int8DynamicActivationInt8WeightConfig())+ torch.save(+ {+ "format": pq.PREQUANT_FORMAT,+ "metadata": {"scheme": "int8"},+ "state_dict": {"blk.p.weight": model[0].weight},+ },+ str(src),+ )+ elif li.legacy_int8_decode_supported():+ _legacy_writer()._write_legacy_checkpoint(monkeypatch, src)+ else:+ pytest.skip("no v1 classes and no legacy decoder")+ with pytest.raises(ValueError, match = "supported v1 layout"):+ conv.convert(str(src), str(tmp_path / "out"), repo = None, revision = None, sha = None)+ assert li._rebuild_weight is originaldiff --git a/tests/test_peft_torchao_missing_tensor_subclass.py b/tests/test_peft_torchao_missing_tensor_subclass.pyindex 9124d4d1984..4048e50721d 100644--- a/tests/test_peft_torchao_missing_tensor_subclass.py+++ b/tests/test_peft_torchao_missing_tensor_subclass.py@@ -269,6 +269,8 @@ def test_the_warning_does_not_claim_the_other_class_is_gone_too(peft_env, fake_t"cannot import name 'LinearActivationQuantizedTensor' from 'torchao.quantization'",# Code reaching past the package for the same class."No module named 'torchao.quantization.linear_activation_quantized_tensor'",+ # torchao main deleted the whole package that defined AffineQuantizedTensor.+ "No module named 'torchao.dtypes'",],)def test_every_spelling_of_the_missing_class_is_handled(peft_env, fake_torchao, message):diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 11c6700095f..1fac14719df 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -10327,10 +10327,11 @@ def fix_peft_stale_torchao_import_error():return patched-# Matches both spellings a torchao removal produces: the class name, and the module that used to-# define it. Only these two, so a torchao that is BROKEN rather than newer still raises.+# The spellings a torchao removal produces (class, its old module, the whole ``torchao.dtypes`` package on main).+# Only these, so a BROKEN torchao still raises._PEFT_TORCHAO_MISSING_TENSOR_SUBCLASS = re.compile(- r"linear_?activation_?quantized_?tensor|affine_?quantized_?tensor",+ r"linear_?activation_?quantized_?tensor|affine_?quantized_?tensor"+ r"|no module named '?torchao\.dtypes'?(?![.\w])",re.IGNORECASE | re.DOTALL,)