Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: load plain FP8 encoder safetensors without torchao (#12333)

oobaboogafa595c6fad56
diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.py
index e9943bc719a..b60a40b3db1 100644
--- a/studio/backend/core/inference/diffusion_te_prequant.py
+++ b/studio/backend/core/inference/diffusion_te_prequant.py
@@ -212,15 +212,13 @@ def te_candidate_is_readable(name: Optional[str]) -> bool:
NOT the transformer's ``restricted_prequant_load_supported``: this state dict is plain
tensors, read under a bare ``weights_only`` load with no constructor allowlist, so a ``.pt``
is always readable and asking the DiT's question would refuse one on every install whose
- torchao lacks some DiT scheme's constructors. Only the safetensors container has a
- requirement, and a plan that drops the dense encoder for an artifact this install cannot open
- leaves the load with neither.
+ torchao lacks some DiT scheme's constructors. The safetensors container needs safetensors, not torchao.
"""
if not name:
return False
- from .prequant_safetensors import is_safetensors_checkpoint, safetensors_prequant_supported
+ from .prequant_safetensors import is_safetensors_checkpoint, plain_safetensors_supported
- return safetensors_prequant_supported() if is_safetensors_checkpoint(name) else True
+ return plain_safetensors_supported() if is_safetensors_checkpoint(name) else True
def family_te_prequant_repo(fam: Any, scheme: str, component: str) -> Optional[str]:
@@ -419,15 +417,15 @@ def load_prequant_text_encoder(
import torch
- from .prequant_safetensors import is_safetensors_checkpoint, load_prequant_safetensors
+ from .prequant_safetensors import is_safetensors_checkpoint, load_plain_prequant_safetensors
# The layerwise-fp8 state dict is plain tensors, so weights_only=True suffices and no pickle code runs even for
# a local path. A future torchao-subclass scheme needs a format bump AND the DiT module's allowlist.
- # A ``.safetensors`` artifact is read through the shared reader instead, which returns the same dict shape, so
+ # A ``.safetensors`` artifact is read through the plain-tensor reader instead, which returns the same dict shape, so
# ``_validate_checkpoint`` and everything after it are unchanged. Dispatch is on the extension the resolver
# asked the Hub for, never on sniffing the bytes.
if is_safetensors_checkpoint(path):
- ckpt = load_prequant_safetensors(path)
+ ckpt = load_plain_prequant_safetensors(path)
else:
ckpt = torch.load(path, weights_only = True, map_location = "cpu")
if not _validate_checkpoint(ckpt, scheme, component, base, logger):
diff --git a/studio/backend/core/inference/prequant_safetensors.py b/studio/backend/core/inference/prequant_safetensors.py
index 2307fca8d0d..825341329c8 100644
--- a/studio/backend/core/inference/prequant_safetensors.py
+++ b/studio/backend/core/inference/prequant_safetensors.py
@@ -135,6 +135,54 @@ def safetensors_prequant_supported() -> bool:
return True
+def plain_safetensors_supported() -> bool:
+ """The layerwise-cast encoder format stores plain tensors, without torchao subclasses."""
+ try:
+ from safetensors import safe_open # noqa: F401
+ except ImportError:
+ return False
+ return True
+
+
+def load_plain_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:
+ """Read a plain-tensor prequant checkpoint without torchao; refuses subclass metadata and unlisted tensors."""
+ from safetensors import safe_open
+
+ with safe_open(path, framework = "pt", device = device) as handle:
+ raw = handle.metadata() or {}
+ fmt = raw.get(UNSLOTH_FORMAT_KEY)
+ if not fmt:
+ raise ValueError(f"{path} is not an Unsloth pre-quant checkpoint")
+ metadata = json.loads(raw.get(UNSLOTH_METADATA_KEY) or "{}")
+ names = json.loads(raw.get("tensor_names") or "null")
+ if not isinstance(metadata, dict):
+ raise ValueError(f"{path} has invalid pre-quant metadata")
+ if (
+ not isinstance(names, list)
+ or not all(isinstance(name, str) and name for name in names)
+ or len(names) != len(set(names))
+ ):
+ raise ValueError(f"{path} has invalid tensor_names")
+ keys = set(handle.keys())
+ roots = {key for key in keys if key.startswith(UNSLOTH_ROOT_PREFIX)}
+ root_names = {key[len(UNSLOTH_ROOT_PREFIX) :] for key in roots}
+ if (
+ any(not name or "." in name for name in root_names)
+ or root_names.intersection(names)
+ or set(names).intersection(roots)
+ or set(names) != keys - roots
+ ):
+ raise ValueError(f"{path} has tensors its header does not account for")
+ for name in names:
+ if json.loads(raw.get(name) or "null") != {"_type": "Tensor"}:
+ raise ValueError(f"{path} requires a tensor-subclass reader for {name!r}")
+ state_dict = {name: handle.get_tensor(name) for name in names}
+ state_dict.update(
+ (key[len(UNSLOTH_ROOT_PREFIX) :], handle.get_tensor(key)) for key in roots
+ )
+ return {"format": str(fmt), "state_dict": state_dict, "metadata": metadata}
+
+
def _first(value: Any) -> Any:
"""Both torchao helpers return a 2-tuple whose first element is the dict we want.
diff --git a/studio/backend/tests/test_prequant_safetensors.py b/studio/backend/tests/test_prequant_safetensors.py
index 74c507d35c9..8e0737ce2d6 100644
--- a/studio/backend/tests/test_prequant_safetensors.py
+++ b/studio/backend/tests/test_prequant_safetensors.py
@@ -222,6 +222,34 @@ def test_plain_tensor_round_trip(tmp_path):
assert torch.equal(ckpt["state_dict"][key], value)
+def test_plain_reader_matches_real_writer_without_torchao(tmp_path, monkeypatch):
+ _real_libs()
+ import torch
+ from core.inference.diffusion_te_prequant import TE_PREQUANT_FORMAT
+
+ state = {
+ "encoder.weight": torch.arange(8).reshape(2, 4).to(torch.float8_e4m3fn),
+ "encoder.bias": torch.zeros(2),
+ "buffer": torch.ones(1),
+ }
+ metadata = {"scheme": "fp8", "component": "text_encoder", "base_model_id": "org/base"}
+ path = str(tmp_path / "encoder.safetensors")
+ ps.save_prequant_safetensors(path, fmt = TE_PREQUANT_FORMAT, state_dict = state, metadata = metadata)
+ shared = ps.load_prequant_safetensors(path)
+
+ # Exercise the Windows reader after the real writer has produced the header.
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ monkeypatch.setitem(sys.modules, "torchao", None)
+ plain = ps.load_plain_prequant_safetensors(path)
+ assert plain["format"] == shared["format"] == TE_PREQUANT_FORMAT
+ assert plain["metadata"] == shared["metadata"] == metadata
+ assert set(plain["state_dict"]) == set(shared["state_dict"]) == set(state)
+ for name, expected in state.items():
+ for actual in (plain["state_dict"][name], shared["state_dict"][name]):
+ assert actual.dtype == expected.dtype
+ assert torch.equal(actual.float(), expected.float())
+
+
def test_header_is_readable_as_a_torchao_checkpoint(tmp_path):
"""Interop, and the trap that makes it fail silently.
@@ -594,3 +622,126 @@ def test_torchao_older_than_the_floor_is_not_safetensors_support(monkeypatch):
monkeypatch.setattr(ps, "_torchao_version", lambda: "0.14.0")
assert ps._torchao_helpers() is None
assert ps.safetensors_prequant_supported() is False
+
+
+def plain_checkpoint(tmp_path, monkeypatch):
+ torch = pytest.importorskip("torch")
+ save_file = pytest.importorskip("safetensors.torch").save_file
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+
+ def write(*, tensors = None, header = None):
+ tensors = {"encoder.weight": torch.ones(2, 3)} if tensors is None else tensors
+ metadata = {
+ ps.UNSLOTH_FORMAT_KEY: "unsloth_prequant_text_encoder_state_dict_v1",
+ ps.UNSLOTH_METADATA_KEY: json.dumps({"scheme": "fp8", "component": "text_encoder"}),
+ "tensor_names": json.dumps(["encoder.weight"]),
+ "encoder.weight": json.dumps({"_type": "Tensor"}),
+ }
+ metadata.update(header or {})
+ path = str(tmp_path / "encoder.safetensors")
+ save_file(tensors, path, metadata = metadata)
+ return path
+
+ return write
+
+
+def test_plain_encoder_needs_no_torchao(plain_checkpoint):
+ import core.inference.diffusion_te_prequant as te
+ import torch
+
+ path = plain_checkpoint(tensors = {"encoder.weight": torch.ones(2, 3).to(torch.float8_e4m3fn)})
+ assert te.te_candidate_is_readable(path)
+ assert not ps.safetensors_prequant_supported()
+ loaded = ps.load_plain_prequant_safetensors(path)
+ assert loaded["state_dict"]["encoder.weight"].dtype == torch.float8_e4m3fn
+ assert torch.equal(loaded["state_dict"]["encoder.weight"].float(), torch.ones(2, 3))
+ assert loaded["metadata"]["scheme"] == "fp8"
+
+
+ "header",
+ [
+ {"tensor_names": '["encoder.weight", "encoder.weight"]'},
+ {"tensor_names": '["missing.weight"]'},
+ {"tensor_names": '"encoder.weight"'},
+ {"tensor_names": "[1]"},
+ {"tensor_names": "invalid"},
+ {"encoder.weight": '{"_type":"Float8Tensor"}'},
+ {"encoder.weight": '{"_type":"Tensor","unexpected":true}'},
+ {ps.UNSLOTH_METADATA_KEY: "[]"},
+ {ps.UNSLOTH_FORMAT_KEY: ""},
+ ],
+)
+def test_plain_reader_rejects_invalid_headers(plain_checkpoint, header):
+ with pytest.raises(ValueError):
+ ps.load_plain_prequant_safetensors(plain_checkpoint(header = header))
+
+
+def test_plain_reader_rejects_unaccounted_tensor(plain_checkpoint):
+ import torch
+ path = plain_checkpoint(
+ tensors = {"encoder.weight": torch.ones(2), "extra.weight": torch.ones(2)}
+ )
+ with pytest.raises(ValueError, match = "does not account"):
+ ps.load_plain_prequant_safetensors(path)
+
+
+def test_plain_reader_restores_root_buffers(plain_checkpoint):
+ import torch
+
+ path = plain_checkpoint(
+ tensors = {
+ "encoder.weight": torch.ones(2),
+ ps.UNSLOTH_ROOT_PREFIX + "buffer": torch.zeros(1),
+ }
+ )
+ state = ps.load_plain_prequant_safetensors(path)["state_dict"]
+ assert set(state) == {"encoder.weight", "buffer"}
+ assert torch.equal(state["buffer"], torch.zeros(1))
+
+
+def test_plain_reader_rejects_root_collision(plain_checkpoint):
+ import torch
+ path = plain_checkpoint(
+ tensors = {"buffer": torch.ones(2), ps.UNSLOTH_ROOT_PREFIX + "buffer": torch.zeros(1)},
+ header = {"tensor_names": '["buffer"]', "buffer": '{"_type":"Tensor"}'},
+ )
+ with pytest.raises(ValueError, match = "does not account"):
+ ps.load_plain_prequant_safetensors(path)
+
+
+def test_plain_encoder_gate_requires_safetensors(monkeypatch):
+ from core.inference.diffusion_te_prequant import te_candidate_is_readable
+
+ monkeypatch.setitem(sys.modules, "safetensors", None)
+ assert not te_candidate_is_readable("encoder.safetensors")
+ assert te_candidate_is_readable("encoder.pt")
+
+
+def test_plain_encoder_resolver_uses_safetensors_without_torchao(monkeypatch):
+ from core.inference.diffusion_te_prequant import (
+ TePrequantSource,
+ _resolve_checkpoint_path,
+ )
+
+ pytest.importorskip("safetensors")
+ hub = pytest.importorskip("huggingface_hub")
+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)
+ requested = []
+
+ def download(**kwargs):
+ requested.append(kwargs["filename"])
+ return "/cache/encoder.safetensors"
+
+ monkeypatch.setattr(hub, "hf_hub_download", download)
+ source = TePrequantSource(
+ kind = "repo",
+ location = "org/encoder",
+ filename = "encoder.safetensors",
+ fallback_filenames = ("encoder.pt",),
+ )
+ assert (
+ _resolve_checkpoint_path(source, None, cache_dir = "/cache") == "/cache/encoder.safetensors"
+ )
+ assert requested == ["encoder.safetensors"]