Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): load merged.pt for the v2 checkpoint instead of the unmerged PEFT safetensors (#2716)

Raúl46da91b2f137
diff --git a/headroom/onnx_runtime.py b/headroom/onnx_runtime.py
index 0ee6353a4..aa3e7f7b2 100644
--- a/headroom/onnx_runtime.py
+++ b/headroom/onnx_runtime.py
@@ -117,6 +117,25 @@ def hf_hub_download_local_first(
return str(hf_hub_download(repo_id, filename, revision=revision))
+def hf_entry_known_absent(repo_id: str, filename: str, *, revision: str | None = None) -> bool:
+ """True only if a prior network lookup already confirmed ``filename`` does
+ not exist in ``repo_id`` at the resolved revision.
+
+ Backed by ``huggingface_hub``'s own cache of negative lookups (the
+ ``.no_exist`` marker written after a real 404), so this never makes a
+ network call itself. Returns ``False`` both when the file is cached and
+ when nothing is known yet about it, on purpose: callers in a cache-only
+ (``allow_network=False``) code path can use this to tell "confirmed
+ missing upstream, safe to use a fallback file" apart from "just never
+ checked yet, do not guess."
+ """
+ from huggingface_hub import _CACHED_NO_EXIST, try_to_load_from_cache
+
+ revision = _resolve_revision(repo_id, revision)
+ result = try_to_load_from_cache(repo_id, filename, revision=revision)
+ return result is _CACHED_NO_EXIST
+
+
def create_cpu_session_options(
ort: Any,
*,
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 0f5f0c91d..c2a39870b 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -29,6 +29,7 @@ from ..config import TransformResult
from ..onnx_runtime import (
ONNX_CPU_ARENA_ENV,
create_cpu_session_options,
+ hf_entry_known_absent,
hf_hub_download_local_first,
trim_process_heap,
)
@@ -721,6 +722,104 @@ def _load_modernbert_tokenizer(auto_tokenizer: Any, *, allow_download: bool) ->
raise
+# Sub-state-dict keys inside a merged v2-style checkpoint (see
+# scripts/export_kompress_v2_onnx.py, which this mirrors).
+_MERGED_CHECKPOINT_KEYS = ("encoder_state_dict", "token_head_state_dict", "span_conv_state_dict")
+
+
+def _load_merged_state_dict(model: Any, ckpt_path: str, model_id: str) -> None:
+ """Load a merged v2-style checkpoint (LoRA already folded into the encoder).
+
+ The checkpoint is a dict of per-submodule state-dicts
+ (``encoder_state_dict`` / ``token_head_state_dict`` / ``span_conv_state_dict``)
+ rather than a single flat state-dict, so each piece is loaded into its
+ matching submodule directly instead of via a single ``load_state_dict``
+ call on the whole model.
+ """
+ import torch
+
+ ckpt = torch.load(ckpt_path, map_location="cpu")
+ missing_sections = [k for k in _MERGED_CHECKPOINT_KEYS if k not in ckpt]
+ if missing_sections:
+ raise RuntimeError(
+ f"merged.pt for {model_id} is missing {missing_sections}; found keys: "
+ f"{sorted(ckpt)}. This checkpoint format is not what the loader expects."
+ )
+
+ for section, submodule in (
+ ("encoder_state_dict", model.encoder),
+ ("token_head_state_dict", model.token_head),
+ ("span_conv_state_dict", model.span_conv),
+ ):
+ missing, unexpected = submodule.load_state_dict(ckpt[section], strict=False)
+ if missing or unexpected:
+ raise RuntimeError(
+ f"{model_id} {section}: state_dict mismatch against {type(submodule).__name__} "
+ f"(missing={list(missing)[:5]}, unexpected={list(unexpected)[:5]}). "
+ "The checkpoint no longer matches HeadroomCompressorModel's architecture."
+ )
+
+
+def _load_plain_state_dict(model: Any, weights_path: str, model_id: str) -> None:
+ """Load a plain, already-merged full state-dict (the pre-v2 / non-PEFT format)."""
+ from safetensors.torch import load_file
+
+ state_dict = load_file(weights_path)
+ missing, unexpected = model.load_state_dict(state_dict, strict=False)
+ if missing or unexpected:
+ raise RuntimeError(
+ f"{model_id} model.safetensors: state_dict mismatch against "
+ f"HeadroomCompressorModel (missing={list(missing)[:5]}, "
+ f"unexpected={list(unexpected)[:5]}). Refusing to run with unloaded weights."
+ )
+
+
+def _load_pytorch_weights(model: Any, model_id: str, *, allow_download: bool) -> None:
+ """Load PyTorch weights into ``model``, preferring the merged v2 checkpoint.
+
+ ``merged.pt`` (when the repo ships one) holds LoRA-merged sub-state-dicts
+ keyed by submodule name. In a PEFT-trained repo, ``model.safetensors`` is
+ the *unmerged* adapter checkpoint (encoder keys prefixed
+ ``encoder.base_model.model...``) and does not map onto this module tree at
+ all, so it is only used as a fallback for repos that never shipped a
+ merged checkpoint (e.g. the original non-LoRA kompress-base).
+
+ In cache-only mode (``allow_download=False``) a ``merged.pt`` cache miss is
+ ambiguous: it could mean the repo has no merged checkpoint (safe to use the
+ plain fallback), or it could mean the repo has one but it just is not
+ downloaded yet (in which case a *stale* cached ``model.safetensors`` from a
+ prior run must not be used, since for a PEFT repo it is the wrong format).
+ ``hf_entry_known_absent`` disambiguates without a network call, using
+ HuggingFace Hub's own cache of confirmed-404 lookups.
+ """
+ try:
+ ckpt_path = hf_hub_download_local_first(model_id, "merged.pt", allow_network=allow_download)
+ except _NOT_CACHED_ERRORS as exc:
+ if not allow_download:
+ if not hf_entry_known_absent(model_id, "merged.pt"):
+ raise KompressModelNotCached(model_id) from exc
+ try:
+ weights_path = hf_hub_download_local_first(
+ model_id, "model.safetensors", allow_network=False
+ )
+ except _NOT_CACHED_ERRORS:
+ raise KompressModelNotCached(model_id) from exc
+ _load_plain_state_dict(model, weights_path, model_id)
+ return
+ if isinstance(exc, EntryNotFoundError):
+ # merged.pt genuinely does not exist in this repo (confirmed by a
+ # real network lookup, not just a cache miss) - fall back to the
+ # plain format instead of treating it as a download failure.
+ weights_path = hf_hub_download_local_first(
+ model_id, "model.safetensors", allow_network=allow_download
+ )
+ _load_plain_state_dict(model, weights_path, model_id)
+ return
+ raise
+ else:
+ _load_merged_state_dict(model, ckpt_path, model_id)
+
+
def _load_kompress_pytorch(
model_id: str, device: str = "auto", *, allow_download: bool = True
) -> tuple[Any, Any, str]:
@@ -738,22 +837,10 @@ def _load_kompress_pytorch(
logger.info("Downloading Kompress PyTorch model from %s ...", model_id)
- try:
- weights_path = hf_hub_download_local_first(
- model_id, "model.safetensors", allow_network=allow_download
- )
- except _NOT_CACHED_ERRORS as exc:
- if not allow_download:
- raise KompressModelNotCached(model_id) from exc
- raise
-
HeadroomCompressorModel = _get_model_class()
model = HeadroomCompressorModel()
- from safetensors.torch import load_file
-
- state_dict = load_file(weights_path)
- model.load_state_dict(state_dict, strict=False)
+ _load_pytorch_weights(model, model_id, allow_download=allow_download)
if device == "auto":
if torch.cuda.is_available():
diff --git a/tests/test_transforms/test_kompress_compressor.py b/tests/test_transforms/test_kompress_compressor.py
index f782b8f2a..37276791d 100644
--- a/tests/test_transforms/test_kompress_compressor.py
+++ b/tests/test_transforms/test_kompress_compressor.py
@@ -539,3 +539,217 @@ class TestOnnxBackendPrefixGating:
with pytest.raises(AttributeError):
compressor._timed_canary(model, pt_tokenizer, "pytorch")
+
+
+class TestPytorchWeightLoading:
+ """_load_pytorch_weights must load the merged v2 checkpoint format correctly,
+ fall back to the plain format only when the repo genuinely has no merged.pt,
+ and refuse to run on a state-dict mismatch instead of silently ignoring it.
+ """
+
+ @staticmethod
+ def _make_model(torch):
+ import torch.nn as nn
+
+ model = nn.Module()
+ model.encoder = nn.Linear(4, 4)
+ model.token_head = nn.Linear(4, 2)
+ model.span_conv = nn.Sequential(nn.Conv1d(4, 4, 1), nn.GELU())
+ return model
+
+ def test_merged_checkpoint_loads_into_matching_submodules(self, tmp_path, monkeypatch) -> None:
+ import pytest
+
+ torch = pytest.importorskip("torch")
+ import headroom.transforms.kompress_compressor as kmod
+
+ model = self._make_model(torch)
+ ckpt_path = tmp_path / "merged.pt"
+ torch.save(
+ {
+ "encoder_state_dict": model.encoder.state_dict(),
+ "token_head_state_dict": model.token_head.state_dict(),
+ "span_conv_state_dict": model.span_conv.state_dict(),
+ },
+ ckpt_path,
+ )
+
+ fresh_model = self._make_model(torch)
+ monkeypatch.setattr(kmod, "hf_hub_download_local_first", lambda *a, **k: str(ckpt_path))
+
+ kmod._load_pytorch_weights(fresh_model, "some/repo", allow_download=True)
+
+ for name, param in model.encoder.state_dict().items():
+ assert torch.equal(param, fresh_model.encoder.state_dict()[name])
+
+ def test_merged_checkpoint_missing_section_raises(self, tmp_path, monkeypatch) -> None:
+ import pytest
+
+ torch = pytest.importorskip("torch")
+ import headroom.transforms.kompress_compressor as kmod
+
+ ckpt_path = tmp_path / "merged.pt"
+ torch.save({"encoder_state_dict": {}}, ckpt_path)
+
+ fresh_model = self._make_model(torch)
+ monkeypatch.setattr(kmod, "hf_hub_download_local_first", lambda *a, **k: str(ckpt_path))
+
+ with pytest.raises(RuntimeError, match="missing"):
+ kmod._load_pytorch_weights(fresh_model, "some/repo", allow_download=True)
+
+ def test_merged_checkpoint_key_mismatch_raises_instead_of_silently_dropping(
+ self, tmp_path, monkeypatch
+ ) -> None:
+ """Regression test for the bug this loader used to have: loading a
+ state-dict that does not match the module tree (e.g. an unmerged PEFT
+ checkpoint) must fail loudly, not silently skip the mismatched keys.
+ """
+ import pytest
+
+ torch = pytest.importorskip("torch")
+ import headroom.transforms.kompress_compressor as kmod
+
+ ckpt_path = tmp_path / "merged.pt"
+ torch.save(
+ {
+ # Wrong prefix, mimics the unmerged PEFT structure documented
+ # in scripts/export_kompress_v2_onnx.py.
+ "encoder_state_dict": {"base_model.model.weight": torch.zeros(4, 4)},
+ "token_head_state_dict": {},
+ "span_conv_state_dict": {},
+ },
+ ckpt_path,
+ )
+
+ fresh_model = self._make_model(torch)
+ monkeypatch.setattr(kmod, "hf_hub_download_local_first", lambda *a, **k: str(ckpt_path))
+
+ with pytest.raises(RuntimeError, match="state_dict mismatch"):
+ kmod._load_pytorch_weights(fresh_model, "some/repo", allow_download=True)
+
+ def test_missing_merged_pt_falls_back_to_plain_safetensors(self, tmp_path, monkeypatch) -> None:
+ import pytest
+
+ torch = pytest.importorskip("torch")
+ safetensors_torch = pytest.importorskip("safetensors.torch")
+ import headroom.transforms.kompress_compressor as kmod
+
+ model = self._make_model(torch)
+ weights_path = tmp_path / "model.safetensors"
+ safetensors_torch.save_file(dict(model.state_dict()), str(weights_path))
+
+ fresh_model = self._make_model(torch)
+
+ def fake_download(model_id, filename, *, allow_network=True, **kwargs): # noqa: ANN001
+ if filename == "merged.pt":
+ raise kmod.EntryNotFoundError("no merged.pt in this repo")
+ assert filename == "model.safetensors"
+ return str(weights_path)
+
+ monkeypatch.setattr(kmod, "hf_hub_download_local_first", fake_download)
+
+ kmod._load_pytorch_weights(fresh_model, "some/v1/repo", allow_download=True)
+
+ for name, param in model.state_dict().items():
+ assert torch.equal(param, fresh_model.state_dict()[name])
+
+ def test_plain_safetensors_key_mismatch_raises(self, tmp_path, monkeypatch) -> None:
+ import pytest
+
+ torch = pytest.importorskip("torch")
+ safetensors_torch = pytest.importorskip("safetensors.torch")
+ import headroom.transforms.kompress_compressor as kmod
+
+ weights_path = tmp_path / "model.safetensors"
+ safetensors_torch.save_file({"totally.unrelated.key": torch.zeros(2)}, str(weights_path))
+
+ fresh_model = self._make_model(torch)
+
+ def fake_download(model_id, filename, *, allow_network=True, **kwargs): # noqa: ANN001
+ if filename == "merged.pt":
+ raise kmod.EntryNotFoundError("no merged.pt in this repo")
+ return str(weights_path)
+
+ monkeypatch.setattr(kmod, "hf_hub_download_local_first", fake_download)
+
+ with pytest.raises(RuntimeError, match="state_dict mismatch"):
+ kmod._load_pytorch_weights(fresh_model, "some/v1/repo", allow_download=True)
+
+ def test_cache_only_miss_raises_kompress_model_not_cached(self, monkeypatch) -> None:
+ import pytest
+
+ pytest.importorskip("torch")
+ import headroom.transforms.kompress_compressor as kmod
+
+ def fake_download(model_id, filename, *, allow_network=True, **kwargs): # noqa: ANN001
+ raise kmod.LocalEntryNotFoundError("not cached")
+
+ monkeypatch.setattr(kmod, "hf_hub_download_local_first", fake_download)
+ monkeypatch.setattr(kmod, "hf_entry_known_absent", lambda *a, **k: False)
+
+ with pytest.raises(kmod.KompressModelNotCached):
+ kmod._load_pytorch_weights(SimpleNamespace(), "some/repo", allow_download=False)
+
+ def test_cache_only_defers_instead_of_using_stale_plain_checkpoint(self, monkeypatch) -> None:
+ """Regression test: if merged.pt is not cached yet and we have no
+ confirmation it is genuinely absent upstream, a stale model.safetensors
+ left over from a previous (pre-fix) run must NOT be used as a silent
+ fallback - that would reintroduce the original bug for exactly the
+ upgrade scenario that motivated this fix. It must defer instead.
+ """
+ import pytest
+
+ pytest.importorskip("torch")
+ import headroom.transforms.kompress_compressor as kmod
+
+ plain_download_calls: list[str] = []
+
+ def fake_download(model_id, filename, *, allow_network=True, **kwargs): # noqa: ANN001
+ if filename == "merged.pt":
+ raise kmod.LocalEntryNotFoundError("merged.pt not cached yet")
+ plain_download_calls.append(filename)
+ return "/fake/cached/model.safetensors"
+
+ monkeypatch.setattr(kmod, "hf_hub_download_local_first", fake_download)
+ # Nothing has ever confirmed merged.pt is absent from this repo -
+ # simulates a v2-style repo mid-upgrade, not a v1-style repo.
+ monkeypatch.setattr(kmod, "hf_entry_known_absent", lambda *a, **k: False)
+
+ with pytest.raises(kmod.KompressModelNotCached):
+ kmod._load_pytorch_weights(
+ SimpleNamespace(), "chopratejas/kompress-v2-base", allow_download=False
+ )
+
+ assert plain_download_calls == [], (
+ "must not fall back to model.safetensors without confirming merged.pt is absent"
+ )
+
+ def test_cache_only_uses_plain_checkpoint_when_merged_pt_confirmed_absent(
+ self, tmp_path, monkeypatch
+ ) -> None:
+ import pytest
+
+ torch = pytest.importorskip("torch")
+ safetensors_torch = pytest.importorskip("safetensors.torch")
+ import headroom.transforms.kompress_compressor as kmod
+
+ model = self._make_model(torch)
+ weights_path = tmp_path / "model.safetensors"
+ safetensors_torch.save_file(dict(model.state_dict()), str(weights_path))
+
+ fresh_model = self._make_model(torch)
+
+ def fake_download(model_id, filename, *, allow_network=True, **kwargs): # noqa: ANN001
+ if filename == "merged.pt":
+ raise kmod.LocalEntryNotFoundError("merged.pt not cached")
+ return str(weights_path)
+
+ monkeypatch.setattr(kmod, "hf_hub_download_local_first", fake_download)
+ # A prior real network lookup already confirmed this repo has no
+ # merged.pt at all (the v1-style case), so the plain fallback is safe.
+ monkeypatch.setattr(kmod, "hf_entry_known_absent", lambda *a, **k: True)
+
+ kmod._load_pytorch_weights(fresh_model, "chopratejas/kompress-base", allow_download=False)
+
+ for name, param in model.state_dict().items():
+ assert torch.equal(param, fresh_model.state_dict()[name])