Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: skip the unused Qwen-Image-2.1 text-encoder lm_head (0.6 to 1.3 GiB less VRAM, bit-identical images) (#12669)
Daniel Hanbd58e1f9d2b2
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 476f1678b27..8b9803cc542 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -7060,6 +7060,17 @@ class DiffusionBackend:"compiled; eager torchao quant is ~30x slower than GGUF here",transformer_quant_engaged,)+ # Before the cast and placement, so the unused head is never cast or moved.+ from .diffusion_text_encoder_trim import trim_text_encoder++ te_trim = trim_text_encoder(+ getattr(pipe, "text_encoder", None), family = fam.name+ )+ if te_trim.get("lm_head") == "dropped" and te_trim.get("params"):+ logger.info(+ "diffusion.text_encoder: dropped unused lm_head (%.2fM params); hidden states unchanged",+ te_trim["params"] / 1e6,+ )# Quantise the dense companion text encoder(s) before placement so offload moves the smaller# weights.te_outcome = quantize_text_encoders(diff --git a/studio/backend/core/inference/diffusion_precision.py b/studio/backend/core/inference/diffusion_precision.pyindex 876cdfb2609..89611e9d4a6 100644--- a/studio/backend/core/inference/diffusion_precision.py+++ b/studio/backend/core/inference/diffusion_precision.py@@ -457,7 +457,12 @@ def _cast_fp8(encoder: Any, target: Any) -> None:)out_emb = get_out() if callable(get_out) else Nonein_emb = get_in() if callable(get_in) else None- if out_emb is not None and in_emb is not None and out_emb.weight is in_emb.weight:+ # getattr: a dropped head (diffusion_text_encoder_trim.NoLogitsHead) has no weight.+ if (+ out_emb is not None+ and in_emb is not None+ and getattr(out_emb, "weight", None) is getattr(in_emb, "weight", object())+ ):tied_name = next((n for n, m in encoder.named_modules() if m is out_emb), None)if tied_name:skip += (rf"^{re.escape(tied_name)}$",)diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex b60a40b3db1..696c38907fe 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -377,6 +377,7 @@ def load_prequant_text_encoder(config_subfolder: Optional[str] = None,config_overrides: Optional[dict] = None,local_files_only: bool = False,+ trim_lm_head: bool = False,) -> Optional[Any]:"""Load the pre-cast text encoder described by ``source`` (on CPU, for pipelineassembly to place), with the layerwise upcast hooks already installed.@@ -389,7 +390,9 @@ def load_prequant_text_encoder(the component name; "" means the repo root, for encoders assembled from a separatestandalone repo like HiDream's Llama TE4). ``config_overrides`` sets config fieldsthe pipeline's assembly normally passes to ``from_pretrained`` (forward-behaviour- flags only; the state dict is unaffected by them)."""+ flags only; the state dict is unaffected by them).+ ``trim_lm_head`` builds the encoder without its untied ``lm_head`` and never reads that tensor+ (``diffusion_text_encoder_trim``)."""try:if source.kind == "path" and not _local_prequant_path_allowed(source.location):_warn(@@ -424,14 +427,27 @@ def load_prequant_text_encoder(# A ``.safetensors`` artifact is read through the plain-tensor reader instead, which returns the same dict shape, so# ``_validate_checkpoint`` and everything after it are unchanged. Dispatch is on the extension the resolver# asked the Hub for, never on sniffing the bytes.+ from .diffusion_text_encoder_trim import (+ LM_HEAD_KEY,+ class_trims_lm_head,+ config_ties_lm_head,+ trim_text_encoder,+ )++ te_class = None+ skip = ()if is_safetensors_checkpoint(path):- ckpt = load_plain_prequant_safetensors(path)+ if trim_lm_head:+ te_class = _safetensors_te_class(path)+ skip = (LM_HEAD_KEY,) if trim_lm_head and class_trims_lm_head(te_class) else ()+ ckpt = load_plain_prequant_safetensors(path, skip_names = skip)else:ckpt = torch.load(path, weights_only = True, map_location = "cpu")if not _validate_checkpoint(ckpt, scheme, component, base, logger):return Nonestate_dict = ckpt["state_dict"]te_class = (ckpt.get("metadata") or {}).get("te_class")+ trim = trim_lm_head and class_trims_lm_head(te_class)import transformers@@ -460,15 +476,26 @@ def load_prequant_text_encoder(remap_rope_parameters(getattr(config, "text_config", config))for key, value in (config_overrides or {}).items():setattr(config, key, value)+ if trim and config_ties_lm_head(config):+ # Tied: nothing to drop.+ trim = False+ if LM_HEAD_KEY in skip:+ state_dict[LM_HEAD_KEY] = _read_safetensors_tensor(path, LM_HEAD_KEY)+ if trim:+ state_dict.pop(LM_HEAD_KEY, None)from accelerate import init_empty_weightswith init_empty_weights():encoder = encoder_cls(config)+ if trim:+ trim_text_encoder(encoder)encoder.load_state_dict(state_dict, strict = True, assign = True)if _has_meta_tensors(encoder):# Non-persistent buffers (built in __init__, absent from the state dict) stay on meta. Rebuild on CPU so# they hold real values, then re-assign the cast weights.encoder = encoder_cls(config)+ if trim:+ trim_text_encoder(encoder)encoder.load_state_dict(state_dict, strict = True, assign = True)# assign=True swaps in SEPARATE tensors for tied weights (the saved dict carries a copy per key), untying e.g.# Qwen3's lm_head from embed_tokens and defeating _cast_fp8's tied-projection skip. Re-tie to the@@ -524,6 +551,7 @@ def te_prequant_pipe_kwargs(status reporting truthful."""try:from .diffusion_precision import TE_QUANT_FP8+ from .diffusion_text_encoder_trim import family_trims_lm_headsources = te_prequant_sources_for_base(fam,@@ -544,6 +572,8 @@ def te_prequant_pipe_kwargs(scheme = mode,logger = logger,local_files_only = local_files_only,+ trim_lm_head = component == "text_encoder"+ and family_trims_lm_head(getattr(fam, "name", None)),)if encoder is not None:injected[component] = encoder@@ -553,6 +583,29 @@ def te_prequant_pipe_kwargs(return {}+def _safetensors_te_class(path: str) -> Optional[str]:+ try:+ import json++ from safetensors import safe_open++ from .prequant_safetensors import UNSLOTH_METADATA_KEY++ with safe_open(path, framework = "pt", device = "cpu") as handle:+ raw = handle.metadata() or {}+ metadata = json.loads(raw.get(UNSLOTH_METADATA_KEY) or "{}")+ value = metadata.get("te_class") if isinstance(metadata, dict) else None+ return str(value) if value else None+ except Exception: # noqa: BLE001 - unreadable header: read every tensor, trim after+ return None+++def _read_safetensors_tensor(path: str, name: str) -> Any:+ from safetensors import safe_open+ with safe_open(path, framework = "pt", device = "cpu") as handle:+ return handle.get_tensor(name)++def _resolve_checkpoint_path(source: TePrequantSource,hf_token: Optional[str],diff --git a/studio/backend/core/inference/diffusion_text_encoder_trim.py b/studio/backend/core/inference/diffusion_text_encoder_trim.pynew file mode 100644index 00000000000..61bb8cff794--- /dev/null+++ b/studio/backend/core/inference/diffusion_text_encoder_trim.py@@ -0,0 +1,96 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Drop the unused ``lm_head`` of a Qwen3-VL text encoder whose pipeline reads only hidden states.++Qwen-Image-2.1 reads ``hidden_states[-1]`` (diffusers ``QwenImage21Pipeline._get_qwen_prompt_embeds``); the untied+151936 x 4096 head runs after every hidden state, so replacing it with :class:`NoLogitsHead` is bit-identical. The+vision tower stays: edit requests on the same pipeline run ``model.visual``. Kill switch: ``UNSLOTH_TE_KEEP_LM_HEAD=1``.+"""++from __future__ import annotations++import os+from functools import lru_cache+from typing import Any, Optional++KEEP_LM_HEAD_ENV = "UNSLOTH_TE_KEEP_LM_HEAD"++# By class name so this module never imports transformers.+_HIDDEN_STATE_ONLY_CLASSES = frozenset({"Qwen3VLForConditionalGeneration"})++_HIDDEN_STATE_ONLY_FAMILIES = frozenset({"qwen-image-2.1"})++LM_HEAD_KEY = "lm_head.weight"+++def trim_enabled() -> bool:+ return os.environ.get(KEEP_LM_HEAD_ENV, "").strip().lower() not in ("1", "true", "yes", "on")+++def family_trims_lm_head(family: Optional[str]) -> bool:+ return trim_enabled() and (family or "").strip().lower() in _HIDDEN_STATE_ONLY_FAMILIES+++def class_trims_lm_head(class_name: Optional[str]) -> bool:+ return (class_name or "") in _HIDDEN_STATE_ONLY_CLASSES+++def config_ties_lm_head(config: Any) -> bool:+ if config is None:+ return False+ text_config = getattr(config, "text_config", None) or config+ return bool(getattr(text_config, "tie_word_embeddings", False)) or bool(+ getattr(config, "tie_word_embeddings", False)+ )+++@lru_cache(maxsize = None)+def _no_logits_head_class() -> Any:+ """Built once: a fresh class per load would be a fresh dynamo type guard."""+ from torch import nn++ class NoLogitsHead(nn.Module):+ """Stand-in for an unused ``lm_head``: no weight, returns an empty logits view."""++ def __init__(self, in_features: int, out_features: int) -> None:+ super().__init__()+ self.in_features = in_features+ self.out_features = out_features++ def forward(self, hidden_states: Any) -> Any: # noqa: D102+ return hidden_states[..., :0]++ def extra_repr(self) -> str: # noqa: D102+ return f"in_features={self.in_features}, out_features={self.out_features}, dropped"++ return NoLogitsHead+++def is_trimmed(text_encoder: Any) -> bool:+ head = getattr(text_encoder, "lm_head", None)+ return head is not None and type(head).__name__ == "NoLogitsHead"+++def trim_text_encoder(text_encoder: Any, *, family: Optional[str] = None) -> dict:+ """Replace ``text_encoder.lm_head`` in place; returns ``{"lm_head": "dropped", "params": N}`` or why it was kept."""+ if text_encoder is None:+ return {"lm_head": "kept", "reason": "no encoder"}+ if not trim_enabled():+ return {"lm_head": "kept", "reason": KEEP_LM_HEAD_ENV}+ if family is not None and not family_trims_lm_head(family):+ return {"lm_head": "kept", "reason": f"family {family}"}+ if not class_trims_lm_head(type(text_encoder).__name__):+ return {"lm_head": "kept", "reason": f"class {type(text_encoder).__name__}"}+ if is_trimmed(text_encoder):+ return {"lm_head": "dropped", "params": 0, "already": True}+ head = getattr(text_encoder, "lm_head", None)+ weight = getattr(head, "weight", None)+ if head is None or weight is None or getattr(weight, "ndim", 0) != 2:+ return {"lm_head": "kept", "reason": "no 2D lm_head weight"}+ if config_ties_lm_head(getattr(text_encoder, "config", None)):+ return {"lm_head": "kept", "reason": "tied to embed_tokens"}+ out_features, in_features = int(weight.shape[0]), int(weight.shape[1])+ params = int(weight.numel())+ text_encoder.lm_head = _no_logits_head_class()(in_features, out_features)+ return {"lm_head": "dropped", "params": params}diff --git a/studio/backend/core/inference/prequant_safetensors.py b/studio/backend/core/inference/prequant_safetensors.pyindex 825341329c8..630def0628a 100644--- a/studio/backend/core/inference/prequant_safetensors.py+++ b/studio/backend/core/inference/prequant_safetensors.py@@ -48,7 +48,7 @@ from __future__ import annotationsimport jsonimport re-from typing import Any, Optional+from typing import Any, Iterable, OptionalSAFETENSORS_SUFFIX = ".safetensors"@@ -144,8 +144,15 @@ def plain_safetensors_supported() -> bool:return True-def load_plain_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:- """Read a plain-tensor prequant checkpoint without torchao; refuses subclass metadata and unlisted tensors."""+def load_plain_prequant_safetensors(+ path: str,+ *,+ device: str = "cpu",+ skip_names: Iterable[str] = (),+) -> dict:+ """Read a plain-tensor prequant checkpoint without torchao; refuses subclass metadata and unlisted tensors.++ ``skip_names`` are validated but never read."""from safetensors import safe_openwith safe_open(path, framework = "pt", device = device) as handle:@@ -176,7 +183,8 @@ def load_plain_prequant_safetensors(path: str, *, device: str = "cpu") -> dict:for name in names:if json.loads(raw.get(name) or "null") != {"_type": "Tensor"}:raise ValueError(f"{path} requires a tensor-subclass reader for {name!r}")- state_dict = {name: handle.get_tensor(name) for name in names}+ skip = frozenset(skip_names)+ state_dict = {name: handle.get_tensor(name) for name in names if name not in skip}state_dict.update((key[len(UNSLOTH_ROOT_PREFIX) :], handle.get_tensor(key)) for key in roots)diff --git a/studio/backend/tests/test_diffusion_text_encoder_trim.py b/studio/backend/tests/test_diffusion_text_encoder_trim.pynew file mode 100644index 00000000000..5e85d820bfe--- /dev/null+++ b/studio/backend/tests/test_diffusion_text_encoder_trim.py@@ -0,0 +1,269 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Qwen-Image-2.1's Qwen3-VL text encoder drops its unused lm_head without changing a hidden state."""++from __future__ import annotations++import json+import types++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("transformers.models.qwen3_vl")++from core.inference import diffusion_text_encoder_trim as trim_mod # noqa: E402+from core.inference.diffusion_text_encoder_trim import ( # noqa: E402+ KEEP_LM_HEAD_ENV,+ LM_HEAD_KEY,+ is_trimmed,+ trim_text_encoder,+)++VOCAB, HIDDEN = 96, 32+++def _config(tie: bool = False):+ from transformers.models.qwen3_vl.configuration_qwen3_vl import (+ Qwen3VLConfig,+ Qwen3VLTextConfig,+ Qwen3VLVisionConfig,+ )++ config = Qwen3VLConfig(+ vision_config = Qwen3VLVisionConfig(+ depth = 1,+ hidden_size = 32,+ intermediate_size = 64,+ num_heads = 4,+ out_hidden_size = HIDDEN,+ deepstack_visual_indexes = [],+ ).to_dict(),+ text_config = Qwen3VLTextConfig(+ hidden_size = HIDDEN,+ intermediate_size = 64,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ vocab_size = VOCAB,+ tie_word_embeddings = tie,+ # transformers 4.57 reads mrope_section from rope_scaling; 5.x maps it onto rope_parameters.+ rope_scaling = {+ "rope_type": "default",+ "mrope_section": [2, 1, 1],+ "mrope_interleaved": True,+ },+ ).to_dict(),+ tie_word_embeddings = tie,+ )+ config._attn_implementation = "sdpa"+ return config+++def _encoder(tie: bool = False, seed: int = 0):+ from transformers import Qwen3VLForConditionalGeneration+ torch.manual_seed(seed)+ return Qwen3VLForConditionalGeneration(_config(tie)).eval()+++def _pipeline_hidden_states(encoder, input_ids):+ """What QwenImage21Pipeline._get_qwen_prompt_embeds reads: every hidden state with the final norm bypassed."""+ text_model = getattr(encoder.model, "language_model", encoder.model)+ handle = text_model.norm.register_forward_hook(lambda module, args, output: args[0])+ try:+ with torch.no_grad():+ out = encoder(+ input_ids = input_ids,+ attention_mask = torch.ones_like(input_ids),+ output_hidden_states = True,+ )+ finally:+ handle.remove()+ return out+++def test_trim_keeps_every_hidden_state_bit_identical_and_drops_the_head():+ encoder = _encoder()+ ids = torch.randint(0, VOCAB, (2, 11))+ before = _pipeline_hidden_states(encoder, ids)+ params_before = sum(p.numel() for p in encoder.parameters())+ record = trim_text_encoder(encoder, family = "qwen-image-2.1")+ after = _pipeline_hidden_states(encoder, ids)++ assert record == {"lm_head": "dropped", "params": VOCAB * HIDDEN}+ assert is_trimmed(encoder)+ assert sum(p.numel() for p in encoder.parameters()) == params_before - VOCAB * HIDDEN+ assert not any("lm_head" in name for name, _ in encoder.named_parameters())+ assert len(before.hidden_states) == len(after.hidden_states)+ for a, b in zip(before.hidden_states, after.hidden_states):+ assert torch.equal(a, b)+ # No logits are computed: an empty view, not a [batch, seq, vocab] projection.+ assert after.logits.shape == (2, 11, 0)+ # The vision tower stays: the same pipeline serves image-conditioned requests.+ assert sum(p.numel() for p in encoder.model.visual.parameters()) > 0+++def test_trim_is_idempotent_and_gated():+ encoder = _encoder()+ assert trim_text_encoder(encoder, family = "qwen-image")["lm_head"] == "kept"+ assert not is_trimmed(encoder)+ assert trim_text_encoder(encoder, family = "qwen-image-2.1")["lm_head"] == "dropped"+ assert trim_text_encoder(encoder, family = "qwen-image-2.1") == {+ "lm_head": "dropped",+ "params": 0,+ "already": True,+ }+ other = torch.nn.Module()+ other.lm_head = torch.nn.Linear(4, 8, bias = False)+ assert trim_text_encoder(other, family = "qwen-image-2.1")["lm_head"] == "kept"+ assert isinstance(other.lm_head, torch.nn.Linear)+ assert trim_text_encoder(None)["lm_head"] == "kept"+++def test_kill_switch_keeps_the_head(monkeypatch):+ monkeypatch.setenv(KEEP_LM_HEAD_ENV, "1")+ encoder = _encoder()+ assert trim_text_encoder(encoder, family = "qwen-image-2.1")["lm_head"] == "kept"+ assert isinstance(encoder.lm_head, torch.nn.Linear)+++def test_a_tied_head_is_kept():+ encoder = _encoder(tie = True)+ assert trim_text_encoder(encoder, family = "qwen-image-2.1") == {+ "lm_head": "kept",+ "reason": "tied to embed_tokens",+ }+++def test_layerwise_fp8_cast_accepts_a_trimmed_encoder():+ pytest.importorskip("diffusers.hooks")+ from core.inference.diffusion_precision import _cast_fp8++ encoder = _encoder()+ trim_text_encoder(encoder, family = "qwen-image-2.1")+ _cast_fp8(encoder, types.SimpleNamespace(dtype = torch.float32))+ assert getattr(encoder, "_unsloth_te_cast_complete", False)+ assert is_trimmed(encoder)+++def _write_precast(tmp_path, encoder, base):+ from safetensors.torch import save_file++ import core.inference.prequant_safetensors as ps+ from core.inference.diffusion_te_prequant import TE_PREQUANT_FORMAT++ state = {k: v.detach().clone().contiguous() for k, v in encoder.state_dict().items()}+ metadata = {+ ps.UNSLOTH_FORMAT_KEY: TE_PREQUANT_FORMAT,+ ps.UNSLOTH_METADATA_KEY: json.dumps(+ {+ "scheme": "fp8",+ "component": "text_encoder",+ "base_model_id": base,+ "te_class": "Qwen3VLForConditionalGeneration",+ }+ ),+ "tensor_names": json.dumps(sorted(state)),+ }+ metadata.update({name: json.dumps({"_type": "Tensor"}) for name in state})+ path = tmp_path / "Qwen-Image-2.1-text_encoder-FP8.safetensors"+ save_file(state, str(path), metadata = metadata)+ return path++[email protected]("trim", [False, True])+def test_precast_loader_skips_the_head_before_reading_it(monkeypatch, tmp_path, trim):+ pytest.importorskip("diffusers.hooks")+ pytest.importorskip("accelerate")+ import transformers++ import core.inference.diffusion_te_prequant as tpq+ import core.inference.prequant_safetensors as ps+ from core.inference.diffusion_prequant import ALLOW_LOCAL_PREQUANT_PATH_ENV++ base = "Qwen/Qwen-Image-2.1"+ reference = _encoder(seed = 3)+ path = _write_precast(tmp_path, reference, base)+ monkeypatch.setenv(ALLOW_LOCAL_PREQUANT_PATH_ENV, str(tmp_path))+ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)+ monkeypatch.setattr(transformers.AutoConfig, "from_pretrained", lambda *_a, **_k: _config())+ read: list = []+ real_reader = ps.load_plain_prequant_safetensors++ def spy(p, **kwargs):+ ckpt = real_reader(p, **kwargs)+ read.extend(ckpt["state_dict"])+ return ckpt++ monkeypatch.setattr(ps, "load_plain_prequant_safetensors", spy)+ encoder = tpq.load_prequant_text_encoder(+ base,+ "text_encoder",+ tpq.TePrequantSource(kind = "path", location = str(path)),+ dtype = torch.float32,+ local_files_only = True,+ trim_lm_head = trim,+ )+ assert encoder is not None+ assert is_trimmed(encoder) is trim+ assert (LM_HEAD_KEY in read) is (not trim)+ ids = torch.randint(0, VOCAB, (1, 9))+ expected = _pipeline_hidden_states(_reference_through_same_cast(reference), ids).hidden_states+ got = _pipeline_hidden_states(encoder, ids).hidden_states+ for a, b in zip(expected, got):+ assert torch.equal(a, b)+++def _reference_through_same_cast(encoder):+ from core.inference.diffusion_precision import _cast_fp8+ _cast_fp8(encoder, types.SimpleNamespace(dtype = torch.float32))+ return encoder+++def test_pipe_kwargs_asks_the_loader_to_trim_only_for_hidden_state_families(monkeypatch):+ import core.inference.diffusion_te_prequant as tpq++ seen: list = []+ monkeypatch.setattr(+ tpq,+ "te_prequant_sources_for_base",+ lambda *a, **k: {"text_encoder": tpq.TePrequantSource(kind = "repo", location = "x/y")},+ )+ monkeypatch.setattr(+ tpq,+ "load_prequant_text_encoder",+ lambda *a, **k: seen.append(k.get("trim_lm_head")) or object(),+ )+ for name in ("qwen-image-2.1", "qwen-image"):+ tpq.te_prequant_pipe_kwargs(+ types.SimpleNamespace(name = name),+ "base",+ te_quant_mode = "fp8",+ target = types.SimpleNamespace(device = "cuda", dtype = None),+ dtype = None,+ )+ assert seen == [True, False]+++def test_plain_reader_skip_names_never_reads_the_tensor(tmp_path, monkeypatch):+ from safetensors.torch import save_file++ import core.inference.prequant_safetensors as ps++ monkeypatch.setattr(ps, "_torchao_helpers", lambda: None)+ names = ["a.weight", "lm_head.weight"]+ metadata = {+ ps.UNSLOTH_FORMAT_KEY: "unsloth_prequant_text_encoder_state_dict_v1",+ ps.UNSLOTH_METADATA_KEY: json.dumps({"scheme": "fp8"}),+ "tensor_names": json.dumps(names),+ **{n: json.dumps({"_type": "Tensor"}) for n in names},+ }+ path = str(tmp_path / "e.safetensors")+ save_file({"a.weight": torch.ones(2), "lm_head.weight": torch.ones(3)}, path, metadata = metadata)+ assert set(ps.load_plain_prequant_safetensors(path)["state_dict"]) == set(names)+ assert set(+ ps.load_plain_prequant_safetensors(path, skip_names = ["lm_head.weight"])["state_dict"]+ ) == {"a.weight"}+ assert trim_mod.LM_HEAD_KEY == "lm_head.weight"