Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: load plain FP8 encoder safetensors without torchao (#12333)
oobaboogafa595c6fad56
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex e9943bc719a..b60a40b3db1 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -212,15 +212,13 @@ def te_candidate_is_readable(name: Optional[str]) -> bool:NOT the transformer's ``restricted_prequant_load_supported``: this state dict is plaintensors, read under a bare ``weights_only`` load with no constructor allowlist, so a ``.pt``is always readable and asking the DiT's question would refuse one on every install whose- torchao lacks some DiT scheme's constructors. Only the safetensors container has a- requirement, and a plan that drops the dense encoder for an artifact this install cannot open- leaves the load with neither.+ torchao lacks some DiT scheme's constructors. The safetensors container needs safetensors, not torchao."""if not name:return False- from .prequant_safetensors import is_safetensors_checkpoint, safetensors_prequant_supported+ from .prequant_safetensors import is_safetensors_checkpoint, plain_safetensors_supported- return safetensors_prequant_supported() if is_safetensors_checkpoint(name) else True+ return plain_safetensors_supported() if is_safetensors_checkpoint(name) else Truedef family_te_prequant_repo(fam: Any, scheme: str, component: str) -> Optional[str]:@@ -419,15 +417,15 @@ def load_prequant_text_encoder(import torch- from .prequant_safetensors import is_safetensors_checkpoint, load_prequant_safetensors+ from .prequant_safetensors import is_safetensors_checkpoint, load_plain_prequant_safetensors# The layerwise-fp8 state dict is plain tensors, so weights_only=True suffices and no pickle code runs even for# a local path. A future torchao-subclass scheme needs a format bump AND the DiT module's allowlist.- # A ``.safetensors`` artifact is read through the shared reader instead, which returns the same dict shape, so+ # A ``.safetensors`` artifact is read through the plain-tensor reader instead, which returns the same dict shape, so# ``_validate_checkpoint`` and everything after it are unchanged. Dispatch is on the extension the resolver# asked the Hub for, never on sniffing the bytes.if is_safetensors_checkpoint(path):- ckpt = load_prequant_safetensors(path)+ ckpt = load_plain_prequant_safetensors(path)else:ckpt = torch.load(path, weights_only = True, map_location = "cpu")if not _validate_checkpoint(ckpt, scheme, component, base, logger):diff --git a/studio/backend/core/inference/prequant_safetensors.py b/studio/backend/core/inference/prequant_safetensors.pyindex 2307fca8d0d..825341329c8 100644--- a/studio/backend/core/inference/prequant_safetensors.py+++ b/studio/backend/core/inference/prequant_safetensors.py@@ -135,6 +135,54 @@ def safetensors_prequant_supported() -> bool:return True+def plain_safetensors_supported() -> bool:+ """The layerwise-cast encoder format stores plain tensors, without torchao subclasses."""+ try:+ from safetensors import safe_open # noqa: F401+ except ImportError:+ return False+ return True+++def load_plain_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:+ """Read a plain-tensor prequant checkpoint without torchao; refuses subclass metadata and unlisted tensors."""+ from safetensors import safe_open++ with safe_open(path, framework = "pt", device = device) as handle:+ raw = handle.metadata() or {}+ fmt = raw.get(UNSLOTH_FORMAT_KEY)+ if not fmt:+ raise ValueError(f"{path} is not an Unsloth pre-quant checkpoint")+ metadata = json.loads(raw.get(UNSLOTH_METADATA_KEY) or "{}")+ names = json.loads(raw.get("tensor_names") or "null")+ if not isinstance(metadata, dict):+ raise ValueError(f"{path} has invalid pre-quant metadata")+ if (+ not isinstance(names, list)+ or not all(isinstance(name, str) and name for name in names)+ or len(names) != len(set(names))+ ):+ raise ValueError(f"{path} has invalid tensor_names")+ keys = set(handle.keys())+ roots = {key for key in keys if key.startswith(UNSLOTH_ROOT_PREFIX)}+ root_names = {key[len(UNSLOTH_ROOT_PREFIX) :] for key in roots}+ if (+ any(not name or "." in name for name in root_names)+ or root_names.intersection(names)+ or set(names).intersection(roots)+ or set(names) != keys - roots+ ):+ raise ValueError(f"{path} has tensors its header does not account for")+ for name in names:+ if json.loads(raw.get(name) or "null") != {"_type": "Tensor"}:+ raise ValueError(f"{path} requires a tensor-subclass reader for {name!r}")+ state_dict = {name: handle.get_tensor(name) for name in names}+ state_dict.update(+ (key[len(UNSLOTH_ROOT_PREFIX) :], handle.get_tensor(key)) for key in roots+ )+ return {"format": str(fmt), "state_dict": state_dict, "metadata": metadata}++def _first(value: Any) -> Any:"""Both torchao helpers return a 2-tuple whose first element is the dict we want.diff --git a/studio/backend/tests/test_prequant_safetensors.py b/studio/backend/tests/test_prequant_safetensors.pyindex 74c507d35c9..8e0737ce2d6 100644--- a/studio/backend/tests/test_prequant_safetensors.py+++ b/studio/backend/tests/test_prequant_safetensors.py@@ -222,6 +222,34 @@ def test_plain_tensor_round_trip(tmp_path):assert torch.equal(ckpt["state_dict"][key], value)+def test_plain_reader_matches_real_writer_without_torchao(tmp_path, monkeypatch):+ _real_libs()+ import torch+ from core.inference.diffusion_te_prequant import TE_PREQUANT_FORMAT++ state = {+ "encoder.weight": torch.arange(8).reshape(2, 4).to(torch.float8_e4m3fn),+ "encoder.bias": torch.zeros(2),+ "buffer": torch.ones(1),+ }+ metadata = {"scheme": "fp8", "component": "text_encoder", "base_model_id": "org/base"}+ path = str(tmp_path / "encoder.safetensors")+ ps.save_prequant_safetensors(path, fmt = TE_PREQUANT_FORMAT, state_dict = state, metadata = metadata)+ shared = ps.load_prequant_safetensors(path)++ # Exercise the Windows reader after the real writer has produced the header.+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)+ monkeypatch.setitem(sys.modules, "torchao", None)+ plain = ps.load_plain_prequant_safetensors(path)+ assert plain["format"] == shared["format"] == TE_PREQUANT_FORMAT+ assert plain["metadata"] == shared["metadata"] == metadata+ assert set(plain["state_dict"]) == set(shared["state_dict"]) == set(state)+ for name, expected in state.items():+ for actual in (plain["state_dict"][name], shared["state_dict"][name]):+ assert actual.dtype == expected.dtype+ assert torch.equal(actual.float(), expected.float())++def test_header_is_readable_as_a_torchao_checkpoint(tmp_path):"""Interop, and the trap that makes it fail silently.@@ -594,3 +622,126 @@ def test_torchao_older_than_the_floor_is_not_safetensors_support(monkeypatch):monkeypatch.setattr(ps, "_torchao_version", lambda: "0.14.0")assert ps._torchao_helpers() is Noneassert ps.safetensors_prequant_supported() is False+++def plain_checkpoint(tmp_path, monkeypatch):+ torch = pytest.importorskip("torch")+ save_file = pytest.importorskip("safetensors.torch").save_file+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)++ def write(*, tensors = None, header = None):+ tensors = {"encoder.weight": torch.ones(2, 3)} if tensors is None else tensors+ metadata = {+ ps.UNSLOTH_FORMAT_KEY: "unsloth_prequant_text_encoder_state_dict_v1",+ ps.UNSLOTH_METADATA_KEY: json.dumps({"scheme": "fp8", "component": "text_encoder"}),+ "tensor_names": json.dumps(["encoder.weight"]),+ "encoder.weight": json.dumps({"_type": "Tensor"}),+ }+ metadata.update(header or {})+ path = str(tmp_path / "encoder.safetensors")+ save_file(tensors, path, metadata = metadata)+ return path++ return write+++def test_plain_encoder_needs_no_torchao(plain_checkpoint):+ import core.inference.diffusion_te_prequant as te+ import torch++ path = plain_checkpoint(tensors = {"encoder.weight": torch.ones(2, 3).to(torch.float8_e4m3fn)})+ assert te.te_candidate_is_readable(path)+ assert not ps.safetensors_prequant_supported()+ loaded = ps.load_plain_prequant_safetensors(path)+ assert loaded["state_dict"]["encoder.weight"].dtype == torch.float8_e4m3fn+ assert torch.equal(loaded["state_dict"]["encoder.weight"].float(), torch.ones(2, 3))+ assert loaded["metadata"]["scheme"] == "fp8"+++ "header",+ [+ {"tensor_names": '["encoder.weight", "encoder.weight"]'},+ {"tensor_names": '["missing.weight"]'},+ {"tensor_names": '"encoder.weight"'},+ {"tensor_names": "[1]"},+ {"tensor_names": "invalid"},+ {"encoder.weight": '{"_type":"Float8Tensor"}'},+ {"encoder.weight": '{"_type":"Tensor","unexpected":true}'},+ {ps.UNSLOTH_METADATA_KEY: "[]"},+ {ps.UNSLOTH_FORMAT_KEY: ""},+ ],+)+def test_plain_reader_rejects_invalid_headers(plain_checkpoint, header):+ with pytest.raises(ValueError):+ ps.load_plain_prequant_safetensors(plain_checkpoint(header = header))+++def test_plain_reader_rejects_unaccounted_tensor(plain_checkpoint):+ import torch+ path = plain_checkpoint(+ tensors = {"encoder.weight": torch.ones(2), "extra.weight": torch.ones(2)}+ )+ with pytest.raises(ValueError, match = "does not account"):+ ps.load_plain_prequant_safetensors(path)+++def test_plain_reader_restores_root_buffers(plain_checkpoint):+ import torch++ path = plain_checkpoint(+ tensors = {+ "encoder.weight": torch.ones(2),+ ps.UNSLOTH_ROOT_PREFIX + "buffer": torch.zeros(1),+ }+ )+ state = ps.load_plain_prequant_safetensors(path)["state_dict"]+ assert set(state) == {"encoder.weight", "buffer"}+ assert torch.equal(state["buffer"], torch.zeros(1))+++def test_plain_reader_rejects_root_collision(plain_checkpoint):+ import torch+ path = plain_checkpoint(+ tensors = {"buffer": torch.ones(2), ps.UNSLOTH_ROOT_PREFIX + "buffer": torch.zeros(1)},+ header = {"tensor_names": '["buffer"]', "buffer": '{"_type":"Tensor"}'},+ )+ with pytest.raises(ValueError, match = "does not account"):+ ps.load_plain_prequant_safetensors(path)+++def test_plain_encoder_gate_requires_safetensors(monkeypatch):+ from core.inference.diffusion_te_prequant import te_candidate_is_readable++ monkeypatch.setitem(sys.modules, "safetensors", None)+ assert not te_candidate_is_readable("encoder.safetensors")+ assert te_candidate_is_readable("encoder.pt")+++def test_plain_encoder_resolver_uses_safetensors_without_torchao(monkeypatch):+ from core.inference.diffusion_te_prequant import (+ TePrequantSource,+ _resolve_checkpoint_path,+ )++ pytest.importorskip("safetensors")+ hub = pytest.importorskip("huggingface_hub")+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)+ requested = []++ def download(**kwargs):+ requested.append(kwargs["filename"])+ return "/cache/encoder.safetensors"++ monkeypatch.setattr(hub, "hf_hub_download", download)+ source = TePrequantSource(+ kind = "repo",+ location = "org/encoder",+ filename = "encoder.safetensors",+ fallback_filenames = ("encoder.pt",),+ )+ assert (+ _resolve_checkpoint_path(source, None, cache_dir = "/cache") == "/cache/encoder.safetensors"+ )+ assert requested == ["encoder.safetensors"]