Skip to content

AirLLM 2.8T LLM inference with single 4GB GPU

feat: support Kimi K3 (2.8T MXFP4 multimodal MoE) and fix streaming gaps it exposed

Yu Li2decbfd2af3e
diff --git a/air_llm/airllm/__init__.py b/air_llm/airllm/__init__.py
index c86ba93..7fe9536 100644
--- a/air_llm/airllm/__init__.py
+++ b/air_llm/airllm/__init__.py
@@ -31,6 +31,7 @@ else:
("AirLLMInternLM", ".airllm_internlm"),
("AirLLMMistral", ".airllm_mistral"),
("AirLLMMixtral", ".airllm_mixtral"),
+ ("AirLLMKimiK3", ".airllm_kimi_k3"),
):
try:
_mod = __import__(__name__ + _module, fromlist=[_name])
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.py
index e1fe4fc..a3a1f73 100644
--- a/air_llm/airllm/airllm_base.py
+++ b/air_llm/airllm/airllm_base.py
@@ -7,6 +7,7 @@ from concurrent.futures import ThreadPoolExecutor
import torch
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig
+from transformers.generation import GenerationMixin
from accelerate import init_empty_weights
from accelerate.utils.modeling import set_module_tensor_to_device
from transformers.quantizers import AutoHfQuantizer
@@ -40,6 +41,10 @@ class AirLLMBaseModel:
them.
"""
+ # Upper bound on how much pinned (page-locked) host memory a single prefetched layer may use.
+ # Layers larger than this are loaded into ordinary pageable memory instead.
+ max_pinned_layer_bytes = 2 * 1024 ** 3
+
# Subclasses override this to point at non-standard module names.
def set_layer_names_dict(self):
self.layer_names_dict = {'embed': 'model.embed_tokens',
@@ -157,6 +162,7 @@ class AirLLMBaseModel:
self.max_seq_len = max_seq_len
self.set_layers_from_layer_names()
+ self._load_resident_modules()
self._install_streaming_hooks()
# ---- customization hooks for subclasses -------------------------------------------------
@@ -220,7 +226,16 @@ class AirLLMBaseModel:
running_dtype = self.running_dtype
base_cls = type(self.model)
- class _AirLLMRuntimeModel(base_cls):
+ # transformers >= 4.50 removed GenerationMixin from PreTrainedModel, so model classes that
+ # predate that change (or ship as remote code, like Kimi K3's multimodal wrapper) no longer
+ # have .generate(). They still define prepare_inputs_for_generation, so mixing the class
+ # back in restores generation. It must come after the model class, per transformers.
+ extra_bases = () if isinstance(self.model, GenerationMixin) else (GenerationMixin,)
+ if extra_bases:
+ print(f"{base_cls.__name__} does not inherit GenerationMixin; mixing it in so "
+ f"generate() works.")
+
+ class _AirLLMRuntimeModel(base_cls, *extra_bases):
@property
def device(self):
return running_device
@@ -270,8 +285,19 @@ class AirLLMBaseModel:
state_dict = load_layer_output
if self.prefetching and torch.cuda.is_available():
- for k in state_dict.keys():
- state_dict[k].pin_memory()
+ # pin_memory() returns a pinned copy rather than pinning in place, so the result has to
+ # be kept for the faster host->device copy to actually happen. Pinned memory can't be
+ # paged out, so we only spend it on layers small enough to be safe: a frontier MoE
+ # checkpoint has ~17GB layers, and with prefetching two are in flight at once, which
+ # would lock up ~34GB of RAM for a copy that is dwarfed by the disk read anyway.
+ total_bytes = sum(v.numel() * v.element_size() for v in state_dict.values())
+ if total_bytes <= self.max_pinned_layer_bytes:
+ try:
+ for k in state_dict.keys():
+ state_dict[k] = state_dict[k].pin_memory()
+ except RuntimeError:
+ # Out of pinned memory: fall back to pageable, which is slower but always works.
+ pass
return state_dict
@@ -284,12 +310,10 @@ class AirLLMBaseModel:
self.hf_quantizer.create_quantized_param(self.model, state_dict[param_name], param_name,
self.running_device, state_dict)
else:
- # Normal load. Pre-quantized weights (fp8) and their block scales must be placed
- # verbatim: casting an fp8 weight to fp16 silently drops the quantization and the
- # accompanying weight_scale_inv, producing garbage. Only ordinary high-precision
- # tensors get cast to the runtime dtype.
+ # Normal load. Only ordinary high-precision tensors get cast to the runtime dtype;
+ # pre-quantized payloads must be placed verbatim (see _should_load_verbatim).
value = state_dict[param_name]
- if value.dtype in (torch.float8_e4m3fn, torch.float8_e5m2) or param_name.endswith("_scale_inv"):
+ if self._should_load_verbatim(param_name, value):
set_module_tensor_to_device(self.model, param_name, self.running_device, value=value)
else:
set_module_tensor_to_device(self.model, param_name, self.running_device,
@@ -297,6 +321,27 @@ class AirLLMBaseModel:
moved.append(param_name)
return moved
+ # Suffixes of the companion tensors that pre-quantized checkpoints ship alongside a weight
+ # (fp8 block scales, compressed-tensors/MXFP4 packed payloads and their scales, GPTQ indices).
+ _QUANT_COMPANION_SUFFIXES = ("_scale", "_scale_inv", "_packed", "_zero_point", "_g_idx", "_shape")
+
+ def _should_load_verbatim(self, param_name, value):
+ """Whether a checkpoint tensor must be placed on the device without a dtype cast.
+
+ Casting a pre-quantized payload to the runtime dtype destroys it: an fp8 weight loses its
+ quantization, and a 4-bit MXFP4 ``weight_packed`` tensor (stored as packed integers) becomes
+ meaningless floats. Equally important for very large models, decompressing on load would
+ multiply a layer's footprint by ~4x, which is what keeps Kimi-K3-class checkpoints from
+ fitting on a single GPU. So we keep anything that isn't a plain high-precision float as-is.
+ """
+ if not value.is_floating_point():
+ # Packed 4-bit payloads, zero points, g_idx, shape metadata.
+ return True
+ if value.element_size() == 1:
+ # Any 8-bit float: fp8 e4m3/e5m2 weights, and the e8m0 scales MXFP4 uses.
+ return True
+ return param_name.endswith(self._QUANT_COMPANION_SUFFIXES)
+
def _needs_quantization(self, param_name):
q = self.hf_quantizer
# transformers renamed check_quantized_param -> param_needs_quantization.
@@ -319,6 +364,22 @@ class AirLLMBaseModel:
names.append(param_name)
return names
+ def _load_resident_modules(self):
+ """Load modules that sit outside the streamed embed -> layers -> norm -> lm_head sequence.
+
+ Multimodal checkpoints carry a vision tower and projector, and some architectures add
+ extra top-level norms. They never get a streaming hook, so without this they would stay on
+ the meta device and fail the moment they run. They are small (well under a GB), so we load
+ them once and leave them resident.
+ """
+ for name in self.layer_names_dict.get('resident', []):
+ try:
+ state_dict = self.load_layer_to_cpu(name)
+ except FileNotFoundError:
+ # Not every checkpoint of a given architecture ships every optional module.
+ continue
+ self.move_layer_to_device(state_dict)
+
def _install_streaming_hooks(self):
# Modules execute in this order during a forward: embed -> layers -> norm -> lm_head.
n = len(self.layer_names)
diff --git a/air_llm/airllm/airllm_kimi_k3.py b/air_llm/airllm/airllm_kimi_k3.py
new file mode 100644
index 0000000..3732f6f
--- /dev/null
+++ b/air_llm/airllm/airllm_kimi_k3.py
@@ -0,0 +1,27 @@
+from .airllm_base import AirLLMBaseModel
+
+
+class AirLLMKimiK3(AirLLMBaseModel):
+ """Kimi K3 (``KimiK3ForConditionalGeneration``).
+
+ K3 is a multimodal MoE checkpoint, so the decoder lives one level deeper than usual, under
+ ``language_model``, and the checkpoint carries a vision tower and projector alongside it.
+ It also adds a pair of top-level Attention Residual modules that sit outside the normal
+ embed -> layers -> norm -> lm_head sequence. Everything else (MXFP4 weights, per-layer
+ streaming) is handled by the generic base class.
+ """
+
+ def set_layer_names_dict(self):
+ self.layer_names_dict = {
+ 'embed': 'language_model.model.embed_tokens',
+ 'layer_prefix': 'language_model.model.layers',
+ 'norm': 'language_model.model.norm',
+ 'lm_head': 'language_model.lm_head',
+ # Not streamed: loaded once and kept resident. Together these are well under 1GB.
+ 'resident': [
+ 'language_model.model.output_attn_res_norm',
+ 'language_model.model.output_attn_res_proj',
+ 'mm_projector',
+ 'vision_tower',
+ ],
+ }
diff --git a/air_llm/airllm/auto_model.py b/air_llm/airllm/auto_model.py
index c6d9b1e..f8607b7 100644
--- a/air_llm/airllm/auto_model.py
+++ b/air_llm/airllm/auto_model.py
@@ -21,6 +21,7 @@ ARCH_OVERRIDES = {
"BaichuanForCausalLM": "AirLLMBaichuan",
"BaiChuanForCausalLM": "AirLLMBaichuan",
"InternLMForCausalLM": "AirLLMInternLM",
+ "KimiK3ForConditionalGeneration": "AirLLMKimiK3",
}
diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.py
index 556a14f..af7153d 100644
--- a/air_llm/airllm/utils.py
+++ b/air_llm/airllm/utils.py
@@ -185,6 +185,33 @@ def remove_real_and_linked_file(to_delete):
+def link_or_copy_file(src, dst):
+ """Point dst at src's data without duplicating it, falling back to a real copy.
+
+ A hard link is preferred over a symlink because it keeps the data alive even if the original
+ checkpoint file is later deleted (``delete_original``), and because it costs no extra disk.
+ Hard links need both paths on one filesystem, so we degrade to a symlink and finally to a copy.
+ Hugging Face caches store files as symlinks into a blob dir, so we always link the real file.
+ """
+ src = Path(os.path.realpath(str(src)))
+ dst = Path(dst)
+ if dst.exists() or dst.is_symlink():
+ dst.unlink()
+
+ try:
+ os.link(src, dst)
+ return 'hardlink'
+ except OSError:
+ pass
+ try:
+ os.symlink(src, dst)
+ return 'symlink'
+ except OSError:
+ pass
+ shutil.copyfile(src, dst)
+ return 'copy'
+
+
def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitted_model_dir_name='splitted_model',
compression=None, layer_names=None, delete_original=False, repo_id=None, hf_token=None):
"""
@@ -243,6 +270,10 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
if 'rotary_pos_emb' in layer_names:
layers = [layer_names['rotary_pos_emb']] + layers
+ # Modules that are not part of the streamed sequence but still need their weights on disk,
+ # e.g. a multimodal model's vision tower / projector, or extra top-level norms. They get
+ # their own shard and are loaded once and kept resident.
+ layers = layers + list(layer_names.get('resident', []))
layers = [l + "." for l in layers]
# Drop layers that have no weights in the checkpoint. This happens for tied embeddings,
@@ -250,6 +281,18 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
# would try to save an empty shard (which fails) and never detect the split as complete.
layers = [l for l in layers if any(k.startswith(l) for k in index.keys())]
+ # Split in ascending shard order. The loop below only ever walks the shard counter forward, so
+ # a module whose weights sit in an earlier shard than its predecessor's would silently be saved
+ # incomplete. That ordering isn't guaranteed once non-sequential modules (a vision tower, extra
+ # norms) are in the list, so sort by the last shard each module touches. This is a stable sort,
+ # so plain embed -> layers -> norm -> lm_head checkpoints keep their existing order.
+ def _last_shard_of(layer):
+ nums = [int(v.split('-')[1]) for k, v in index.items()
+ if k.startswith(layer) and '-' in v and len(v.split('-')) > 1]
+ return max(nums) if nums else -1
+
+ layers.sort(key=_last_shard_of)
+
# check if splitting exists and all files are there
found_layers = None
@@ -269,7 +312,36 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
else:
print(f"some layer splits found, some are not, re-save all layers in case there's some corruptions.")
- if not delete_original:
+ # Some checkpoints are already sharded exactly one module per file (Kimi K3, for instance, ships
+ # one ~17GB shard per decoder layer). Re-writing those into per-layer files would duplicate the
+ # entire checkpoint on disk -- 1.5TB+ for a 2.8T-parameter model -- and take hours, to produce
+ # byte-identical content. When a shard holds nothing but one module's tensors we link to it
+ # instead of copying.
+ passthrough = {}
+ # Linking only produces a file the loader can read when shards are stored in the same format
+ # the persister writes; the MLX persister, for instance, writes .mlx.npz.
+ persister_is_safetensors = type(ModelPersister.get_model_persister()).__name__ == 'SafetensorModelPersister'
+ if compression is None and safetensors_format and persister_is_safetensors:
+ shard_contents = defaultdict(list)
+ for k, v in index.items():
+ shard_contents[v].append(k)
+ for layer in layers:
+ files = {v for k, v in index.items() if k.startswith(layer)}
+ if len(files) != 1:
+ continue
+ only_file = next(iter(files))
+ if all(k.startswith(layer) for k in shard_contents[only_file]):
+ passthrough[layer] = only_file
+
+ if passthrough:
+ print(f"{len(passthrough)}/{len(layers)} modules are already one-per-shard; "
+ f"linking to the original files instead of copying them.")
+
+ # Must exist before check_space, which stats the filesystem it lives on.
+ saving_path.mkdir(parents=True, exist_ok=True)
+
+ # A copy is only made for the layers we cannot link, so only those need free space.
+ if not delete_original and len(passthrough) < len(layers):
check_space(checkpoint_path, layer_shards_saving_path, compression, splitted_model_dir_name=splitted_model_dir_name)
@@ -289,14 +361,29 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
except ValueError:
pass
- if not os.path.exists(saving_path):
- #os.makedirs(saving_path)
- saving_path.mkdir(parents=True, exist_ok=True)
-
single_modelfile = None
for layer in tqdm(layers):
+ if layer in passthrough:
+ src = checkpoint_path / passthrough[layer]
+ if not os.path.exists(src):
+ assert repo_id is not None
+ huggingface_hub.snapshot_download(repo_id, allow_patterns=os.path.basename(src),
+ token=hf_token)
+ if not ModelPersister.get_model_persister().model_persist_exist(layer, saving_path):
+ link_or_copy_file(src, saving_path / (layer + 'safetensors'))
+ (saving_path / (layer + 'safetensors.done')).touch()
+ # Keep the shard cursor in step with what we skipped, so a later layer that does need
+ # loading doesn't walk back through (and read) every shard we just linked past.
+ src_parts = passthrough[layer].split('-')
+ if len(src_parts) > 1:
+ try:
+ shard = max(shard, int(src_parts[1]))
+ except ValueError:
+ pass
+ continue
+
# Optionnally load next shard
# checking whether after spliting from '-', if second element exists. otherwise it throws errors for single 'model.safetensor' files
shards = [int(v.split('-')[1]) for k, v in index.items() if k.startswith(layer) and '-' in v and len(v.split('-')) > 1]
@@ -395,9 +482,13 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards
# try local model path, if the model exist split and save there
if os.path.exists(model_local_path_or_repo_id):
- if os.path.exists(Path(model_local_path_or_repo_id) / 'pytorch_model.bin.index.json') or \
- os.path.exists(Path(model_local_path_or_repo_id) / 'model.safetensors.index.json'):
- print(f"found index file...")
+ # Accept single-file checkpoints too, not just sharded ones with an index: the splitter
+ # handles both, so requiring an index needlessly sent local single-file models down the
+ # "treat it as a repo id" path, where they fail as an invalid repo name.
+ local_weight_files = ('pytorch_model.bin.index.json', 'model.safetensors.index.json',
+ 'model.safetensors', 'pytorch_model.bin')
+ if any(os.path.exists(Path(model_local_path_or_repo_id) / f) for f in local_weight_files):
+ print(f"found local checkpoint...")
return Path(model_local_path_or_repo_id), split_and_save_layers(model_local_path_or_repo_id, layer_shards_saving_path,
compression=compression, layer_names=layer_names, delete_original=delete_original)
else:
diff --git a/air_llm/tests/test_kimi_k3_split.py b/air_llm/tests/test_kimi_k3_split.py
new file mode 100644
index 0000000..c9c6543
--- /dev/null
+++ b/air_llm/tests/test_kimi_k3_split.py
@@ -0,0 +1,224 @@
+"""Structural tests for Kimi-K3-style checkpoints.
+
+K3 is multimodal (decoder nested under ``language_model``), ships one shard per decoder layer, and
+carries modules outside the streamed sequence (vision tower, projector, extra residual norms).
+These tests build a miniature checkpoint with the same shape and check that the splitter links
+rather than copies the one-module-per-shard files, still materialises the shared shard correctly,
+and never loses a module because of shard ordering.
+"""
+
+import json
+import os
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+
+import torch
+from safetensors.torch import save_file, load_file
+
+_AIRLLM_DIR = Path(__file__).resolve().parents[1] / 'airllm'
+
+# Bind the package without running airllm/__init__.py: that module pulls in the MLX backend on
+# macOS and the full transformers stack elsewhere, none of which these splitter tests need.
+if 'airllm' not in sys.modules:
+ _pkg = types.ModuleType('airllm')
+ _pkg.__path__ = [str(_AIRLLM_DIR)]
+ sys.modules['airllm'] = _pkg
+
+from airllm.persist import model_persister as _persister_mod
+from airllm.persist.safetensor_model_persister import SafetensorModelPersister
+
+# The persister is otherwise chosen by platform, so pin the safetensors one to exercise the
+# Linux/CUDA path regardless of where the test runs.
+_persister_mod.model_persister = SafetensorModelPersister()
+
+from airllm.utils import split_and_save_layers
+
+N_LAYERS = 3
+LAYER_PREFIX = "language_model.model.layers"
+
+KIMI_LAYER_NAMES = {
+ 'embed': 'language_model.model.embed_tokens',
+ 'layer_prefix': LAYER_PREFIX,
+ 'norm': 'language_model.model.norm',
+ 'lm_head': 'language_model.lm_head',
+ 'resident': [
+ 'language_model.model.output_attn_res_norm',
+ 'language_model.model.output_attn_res_proj',
+ 'mm_projector',
+ 'vision_tower',
+ ],
+}
+
+
+def build_fake_checkpoint(root):
+ """One shard per layer, then a shared shard, then projector and vision tower."""
+ n_shards = N_LAYERS + 3
+ shards = {}
+
+ for i in range(N_LAYERS):
+ name = f"model-{i + 1:05d}-of-{n_shards:06d}.safetensors"
+ shards[name] = {
+ # A packed 4-bit payload plus its scale, like MXFP4 stores.
+ f"{LAYER_PREFIX}.{i}.block_sparse_moe.experts.0.w1.weight_packed":
+ torch.randint(0, 255, (8, 4), dtype=torch.uint8),
+ f"{LAYER_PREFIX}.{i}.block_sparse_moe.experts.0.w1.weight_scale":
+ torch.randn(8, 1, dtype=torch.float32),
+ f"{LAYER_PREFIX}.{i}.input_layernorm.weight": torch.randn(8),
+ }
+
+ shared = f"model-{N_LAYERS + 1:05d}-of-{n_shards:06d}.safetensors"
+ shards[shared] = {
+ "language_model.model.embed_tokens.weight": torch.randn(16, 8),
+ "language_model.model.norm.weight": torch.randn(8),
+ "language_model.lm_head.weight": torch.randn(16, 8),
+ "language_model.model.output_attn_res_norm.weight": torch.randn(8),
+ "language_model.model.output_attn_res_proj.weight": torch.randn(8, 8),
+ }
+ # Deliberately out of order relative to the resident list: the projector lives in an *earlier*
+ # shard than the vision tower, which the splitter must handle without losing tensors.
+ proj = f"model-{N_LAYERS + 2:05d}-of-{n_shards:06d}.safetensors"
+ shards[proj] = {"mm_projector.proj.0.weight": torch.randn(8, 8)}
+ vis = f"model-{N_LAYERS + 3:05d}-of-{n_shards:06d}.safetensors"
+ shards[vis] = {"vision_tower.encoder.blocks.0.mlp.fc0.weight": torch.randn(8, 8)}
+
+ weight_map = {}
+ for fname, sd in shards.items():
+ save_file(sd, str(Path(root) / fname))
+ for k in sd:
+ weight_map[k] = fname
+ with open(Path(root) / "model.safetensors.index.json", "w") as f:
+ json.dump({"metadata": {"total_size": 1}, "weight_map": weight_map}, f)
+ return shards, weight_map
+
+
+class TestKimiK3Split(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self.tmp.name)
+ self.shards, self.weight_map = build_fake_checkpoint(self.root)
+ self.out = Path(split_and_save_layers(self.root, layer_names=KIMI_LAYER_NAMES))
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def _split_file(self, module):
+ return self.out / f"{module}.safetensors"
+
+ def test_every_module_is_split_out(self):
+ expected = ([KIMI_LAYER_NAMES['embed']]
+ + [f"{LAYER_PREFIX}.{i}" for i in range(N_LAYERS)]
+ + [KIMI_LAYER_NAMES['norm'], KIMI_LAYER_NAMES['lm_head']]
+ + KIMI_LAYER_NAMES['resident'])
+ for module in expected:
+ self.assertTrue(self._split_file(module).exists(), f"missing split for {module}")
+
+ def test_one_shard_per_layer_is_linked_not_copied(self):
+ """The whole point: a 1.5TB checkpoint must not be duplicated."""
+ for i in range(N_LAYERS):
+ src = self.root / f"model-{i + 1:05d}-of-{N_LAYERS + 3:06d}.safetensors"
+ dst = self._split_file(f"{LAYER_PREFIX}.{i}")
+ self.assertEqual(os.stat(src).st_ino, os.stat(dst).st_ino,
+ f"layer {i} was copied instead of linked")
+
+ def test_projector_and_vision_tower_are_linked(self):
+ for module, shard_no in (('mm_projector', N_LAYERS + 2), ('vision_tower', N_LAYERS + 3)):
+ src = self.root / f"model-{shard_no:05d}-of-{N_LAYERS + 3:06d}.safetensors"
+ self.assertEqual(os.stat(src).st_ino, os.stat(self._split_file(module)).st_ino,
+ f"{module} was copied instead of linked")
+
+ def test_shared_shard_modules_are_materialised_separately(self):
+ """embed / norm / lm_head / residual modules share one shard, so they must be real files."""
+ shared_src = self.root / f"model-{N_LAYERS + 1:05d}-of-{N_LAYERS + 3:06d}.safetensors"
+ for module in ('language_model.model.embed_tokens', 'language_model.model.norm',
+ 'language_model.lm_head', 'language_model.model.output_attn_res_norm',
+ 'language_model.model.output_attn_res_proj'):
+ dst = self._split_file(module)
+ self.assertNotEqual(os.stat(shared_src).st_ino, os.stat(dst).st_ino)
+ keys = set(load_file(str(dst)).keys())
+ self.assertTrue(keys, f"{module} split is empty")
+ self.assertTrue(all(k.startswith(module + '.') for k in keys),
+ f"{module} split leaked other tensors: {keys}")
+
+ def test_contents_match_the_original_checkpoint(self):
+ """Linked or copied, every tensor must survive the split bit-for-bit."""
+ original = {}
+ for fname in self.shards:
+ original.update(load_file(str(self.root / fname)))
+
+ recovered = {}
+ for f in self.out.glob('*.safetensors'):
+ recovered.update(load_file(str(f)))
+
+ self.assertEqual(set(original.keys()), set(recovered.keys()))
+ for k, v in original.items():
+ self.assertEqual(v.dtype, recovered[k].dtype, f"{k} changed dtype")
+ self.assertTrue(torch.equal(v, recovered[k]), f"{k} changed value")
+
+ def test_packed_4bit_payload_keeps_its_integer_dtype(self):
+ sd = load_file(str(self._split_file(f"{LAYER_PREFIX}.0")))
+ packed = [v for k, v in sd.items() if k.endswith('weight_packed')][0]
+ self.assertEqual(packed.dtype, torch.uint8)
+
+
+class TestStandardCheckpointStillSplits(unittest.TestCase):
+ """Regression guard: the ordinary layout (several layers per shard) must be unaffected.
+
+ Passthrough must not kick in here, because no shard holds exactly one module.
+ """
+
+ N = 4
+
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self.tmp.name)
+
+ shard_a = {"model.embed_tokens.weight": torch.randn(16, 8)}
+ for i in range(2):
+ shard_a[f"model.layers.{i}.self_attn.q_proj.weight"] = torch.randn(8, 8)
+ shard_b = {}
+ for i in range(2, self.N):
+ shard_b[f"model.layers.{i}.self_attn.q_proj.weight"] = torch.randn(8, 8)
+ shard_b["model.norm.weight"] = torch.randn(8)
+ shard_b["lm_head.weight"] = torch.randn(16, 8)
+
+ weight_map = {}
+ for fname, sd in (("model-00001-of-00002.safetensors", shard_a),
+ ("model-00002-of-00002.safetensors", shard_b)):
+ save_file(sd, str(self.root / fname))
+ for k in sd:
+ weight_map[k] = fname
+ with open(self.root / "model.safetensors.index.json", "w") as f:
+ json.dump({"metadata": {"total_size": 1}, "weight_map": weight_map}, f)
+
+ self.original = dict(shard_a)
+ self.original.update(shard_b)
+ self.out = Path(split_and_save_layers(self.root))
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def test_all_modules_written_as_real_files(self):
+ expected = (['model.embed_tokens'] + [f'model.layers.{i}' for i in range(self.N)]
+ + ['model.norm', 'lm_head'])
+ src_inodes = {os.stat(self.root / f).st_ino
+ for f in os.listdir(self.root) if f.endswith('.safetensors')}
+ for module in expected:
+ dst = self.out / f"{module}.safetensors"
+ self.assertTrue(dst.exists(), f"missing split for {module}")
+ self.assertNotIn(os.stat(dst).st_ino, src_inodes,
+ f"{module} was linked, but this layout should be copied")
+
+ def test_contents_round_trip(self):
+ recovered = {}
+ for f in self.out.glob('*.safetensors'):
+ recovered.update(load_file(str(f)))
+ self.assertEqual(set(self.original.keys()), set(recovered.keys()))
+ for k, v in self.original.items():
+ self.assertTrue(torch.equal(v, recovered[k]), f"{k} changed value")
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)