AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
feat: support Kimi K3 (2.8T MXFP4 multimodal MoE) and fix streaming gaps it exposed
Yu Li2decbfd2af3e
diff --git a/air_llm/airllm/__init__.py b/air_llm/airllm/__init__.pyindex c86ba93..7fe9536 100644--- a/air_llm/airllm/__init__.py+++ b/air_llm/airllm/__init__.py@@ -31,6 +31,7 @@ else:("AirLLMInternLM", ".airllm_internlm"),("AirLLMMistral", ".airllm_mistral"),("AirLLMMixtral", ".airllm_mixtral"),+ ("AirLLMKimiK3", ".airllm_kimi_k3"),):try:_mod = __import__(__name__ + _module, fromlist=[_name])diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.pyindex e1fe4fc..a3a1f73 100644--- a/air_llm/airllm/airllm_base.py+++ b/air_llm/airllm/airllm_base.py@@ -7,6 +7,7 @@ from concurrent.futures import ThreadPoolExecutorimport torchfrom transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, GenerationConfig+from transformers.generation import GenerationMixinfrom accelerate import init_empty_weightsfrom accelerate.utils.modeling import set_module_tensor_to_devicefrom transformers.quantizers import AutoHfQuantizer@@ -40,6 +41,10 @@ class AirLLMBaseModel:them."""+ # Upper bound on how much pinned (page-locked) host memory a single prefetched layer may use.+ # Layers larger than this are loaded into ordinary pageable memory instead.+ max_pinned_layer_bytes = 2 * 1024 ** 3+# Subclasses override this to point at non-standard module names.def set_layer_names_dict(self):self.layer_names_dict = {'embed': 'model.embed_tokens',@@ -157,6 +162,7 @@ class AirLLMBaseModel:self.max_seq_len = max_seq_lenself.set_layers_from_layer_names()+ self._load_resident_modules()self._install_streaming_hooks()# ---- customization hooks for subclasses -------------------------------------------------@@ -220,7 +226,16 @@ class AirLLMBaseModel:running_dtype = self.running_dtypebase_cls = type(self.model)- class _AirLLMRuntimeModel(base_cls):+ # transformers >= 4.50 removed GenerationMixin from PreTrainedModel, so model classes that+ # predate that change (or ship as remote code, like Kimi K3's multimodal wrapper) no longer+ # have .generate(). They still define prepare_inputs_for_generation, so mixing the class+ # back in restores generation. It must come after the model class, per transformers.+ extra_bases = () if isinstance(self.model, GenerationMixin) else (GenerationMixin,)+ if extra_bases:+ print(f"{base_cls.__name__} does not inherit GenerationMixin; mixing it in so "+ f"generate() works.")++ class _AirLLMRuntimeModel(base_cls, *extra_bases):@propertydef device(self):return running_device@@ -270,8 +285,19 @@ class AirLLMBaseModel:state_dict = load_layer_outputif self.prefetching and torch.cuda.is_available():- for k in state_dict.keys():- state_dict[k].pin_memory()+ # pin_memory() returns a pinned copy rather than pinning in place, so the result has to+ # be kept for the faster host->device copy to actually happen. Pinned memory can't be+ # paged out, so we only spend it on layers small enough to be safe: a frontier MoE+ # checkpoint has ~17GB layers, and with prefetching two are in flight at once, which+ # would lock up ~34GB of RAM for a copy that is dwarfed by the disk read anyway.+ total_bytes = sum(v.numel() * v.element_size() for v in state_dict.values())+ if total_bytes <= self.max_pinned_layer_bytes:+ try:+ for k in state_dict.keys():+ state_dict[k] = state_dict[k].pin_memory()+ except RuntimeError:+ # Out of pinned memory: fall back to pageable, which is slower but always works.+ passreturn state_dict@@ -284,12 +310,10 @@ class AirLLMBaseModel:self.hf_quantizer.create_quantized_param(self.model, state_dict[param_name], param_name,self.running_device, state_dict)else:- # Normal load. Pre-quantized weights (fp8) and their block scales must be placed- # verbatim: casting an fp8 weight to fp16 silently drops the quantization and the- # accompanying weight_scale_inv, producing garbage. Only ordinary high-precision- # tensors get cast to the runtime dtype.+ # Normal load. Only ordinary high-precision tensors get cast to the runtime dtype;+ # pre-quantized payloads must be placed verbatim (see _should_load_verbatim).value = state_dict[param_name]- if value.dtype in (torch.float8_e4m3fn, torch.float8_e5m2) or param_name.endswith("_scale_inv"):+ if self._should_load_verbatim(param_name, value):set_module_tensor_to_device(self.model, param_name, self.running_device, value=value)else:set_module_tensor_to_device(self.model, param_name, self.running_device,@@ -297,6 +321,27 @@ class AirLLMBaseModel:moved.append(param_name)return moved+ # Suffixes of the companion tensors that pre-quantized checkpoints ship alongside a weight+ # (fp8 block scales, compressed-tensors/MXFP4 packed payloads and their scales, GPTQ indices).+ _QUANT_COMPANION_SUFFIXES = ("_scale", "_scale_inv", "_packed", "_zero_point", "_g_idx", "_shape")++ def _should_load_verbatim(self, param_name, value):+ """Whether a checkpoint tensor must be placed on the device without a dtype cast.++ Casting a pre-quantized payload to the runtime dtype destroys it: an fp8 weight loses its+ quantization, and a 4-bit MXFP4 ``weight_packed`` tensor (stored as packed integers) becomes+ meaningless floats. Equally important for very large models, decompressing on load would+ multiply a layer's footprint by ~4x, which is what keeps Kimi-K3-class checkpoints from+ fitting on a single GPU. So we keep anything that isn't a plain high-precision float as-is.+ """+ if not value.is_floating_point():+ # Packed 4-bit payloads, zero points, g_idx, shape metadata.+ return True+ if value.element_size() == 1:+ # Any 8-bit float: fp8 e4m3/e5m2 weights, and the e8m0 scales MXFP4 uses.+ return True+ return param_name.endswith(self._QUANT_COMPANION_SUFFIXES)+def _needs_quantization(self, param_name):q = self.hf_quantizer# transformers renamed check_quantized_param -> param_needs_quantization.@@ -319,6 +364,22 @@ class AirLLMBaseModel:names.append(param_name)return names+ def _load_resident_modules(self):+ """Load modules that sit outside the streamed embed -> layers -> norm -> lm_head sequence.++ Multimodal checkpoints carry a vision tower and projector, and some architectures add+ extra top-level norms. They never get a streaming hook, so without this they would stay on+ the meta device and fail the moment they run. They are small (well under a GB), so we load+ them once and leave them resident.+ """+ for name in self.layer_names_dict.get('resident', []):+ try:+ state_dict = self.load_layer_to_cpu(name)+ except FileNotFoundError:+ # Not every checkpoint of a given architecture ships every optional module.+ continue+ self.move_layer_to_device(state_dict)+def _install_streaming_hooks(self):# Modules execute in this order during a forward: embed -> layers -> norm -> lm_head.n = len(self.layer_names)diff --git a/air_llm/airllm/airllm_kimi_k3.py b/air_llm/airllm/airllm_kimi_k3.pynew file mode 100644index 0000000..3732f6f--- /dev/null+++ b/air_llm/airllm/airllm_kimi_k3.py@@ -0,0 +1,27 @@+from .airllm_base import AirLLMBaseModel+++class AirLLMKimiK3(AirLLMBaseModel):+ """Kimi K3 (``KimiK3ForConditionalGeneration``).++ K3 is a multimodal MoE checkpoint, so the decoder lives one level deeper than usual, under+ ``language_model``, and the checkpoint carries a vision tower and projector alongside it.+ It also adds a pair of top-level Attention Residual modules that sit outside the normal+ embed -> layers -> norm -> lm_head sequence. Everything else (MXFP4 weights, per-layer+ streaming) is handled by the generic base class.+ """++ def set_layer_names_dict(self):+ self.layer_names_dict = {+ 'embed': 'language_model.model.embed_tokens',+ 'layer_prefix': 'language_model.model.layers',+ 'norm': 'language_model.model.norm',+ 'lm_head': 'language_model.lm_head',+ # Not streamed: loaded once and kept resident. Together these are well under 1GB.+ 'resident': [+ 'language_model.model.output_attn_res_norm',+ 'language_model.model.output_attn_res_proj',+ 'mm_projector',+ 'vision_tower',+ ],+ }diff --git a/air_llm/airllm/auto_model.py b/air_llm/airllm/auto_model.pyindex c6d9b1e..f8607b7 100644--- a/air_llm/airllm/auto_model.py+++ b/air_llm/airllm/auto_model.py@@ -21,6 +21,7 @@ ARCH_OVERRIDES = {"BaichuanForCausalLM": "AirLLMBaichuan","BaiChuanForCausalLM": "AirLLMBaichuan","InternLMForCausalLM": "AirLLMInternLM",+ "KimiK3ForConditionalGeneration": "AirLLMKimiK3",}diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.pyindex 556a14f..af7153d 100644--- a/air_llm/airllm/utils.py+++ b/air_llm/airllm/utils.py@@ -185,6 +185,33 @@ def remove_real_and_linked_file(to_delete):+def link_or_copy_file(src, dst):+ """Point dst at src's data without duplicating it, falling back to a real copy.++ A hard link is preferred over a symlink because it keeps the data alive even if the original+ checkpoint file is later deleted (``delete_original``), and because it costs no extra disk.+ Hard links need both paths on one filesystem, so we degrade to a symlink and finally to a copy.+ Hugging Face caches store files as symlinks into a blob dir, so we always link the real file.+ """+ src = Path(os.path.realpath(str(src)))+ dst = Path(dst)+ if dst.exists() or dst.is_symlink():+ dst.unlink()++ try:+ os.link(src, dst)+ return 'hardlink'+ except OSError:+ pass+ try:+ os.symlink(src, dst)+ return 'symlink'+ except OSError:+ pass+ shutil.copyfile(src, dst)+ return 'copy'++def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitted_model_dir_name='splitted_model',compression=None, layer_names=None, delete_original=False, repo_id=None, hf_token=None):"""@@ -243,6 +270,10 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splittif 'rotary_pos_emb' in layer_names:layers = [layer_names['rotary_pos_emb']] + layers+ # Modules that are not part of the streamed sequence but still need their weights on disk,+ # e.g. a multimodal model's vision tower / projector, or extra top-level norms. They get+ # their own shard and are loaded once and kept resident.+ layers = layers + list(layer_names.get('resident', []))layers = [l + "." for l in layers]# Drop layers that have no weights in the checkpoint. This happens for tied embeddings,@@ -250,6 +281,18 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt# would try to save an empty shard (which fails) and never detect the split as complete.layers = [l for l in layers if any(k.startswith(l) for k in index.keys())]+ # Split in ascending shard order. The loop below only ever walks the shard counter forward, so+ # a module whose weights sit in an earlier shard than its predecessor's would silently be saved+ # incomplete. That ordering isn't guaranteed once non-sequential modules (a vision tower, extra+ # norms) are in the list, so sort by the last shard each module touches. This is a stable sort,+ # so plain embed -> layers -> norm -> lm_head checkpoints keep their existing order.+ def _last_shard_of(layer):+ nums = [int(v.split('-')[1]) for k, v in index.items()+ if k.startswith(layer) and '-' in v and len(v.split('-')) > 1]+ return max(nums) if nums else -1++ layers.sort(key=_last_shard_of)+# check if splitting exists and all files are therefound_layers = None@@ -269,7 +312,36 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splittelse:print(f"some layer splits found, some are not, re-save all layers in case there's some corruptions.")- if not delete_original:+ # Some checkpoints are already sharded exactly one module per file (Kimi K3, for instance, ships+ # one ~17GB shard per decoder layer). Re-writing those into per-layer files would duplicate the+ # entire checkpoint on disk -- 1.5TB+ for a 2.8T-parameter model -- and take hours, to produce+ # byte-identical content. When a shard holds nothing but one module's tensors we link to it+ # instead of copying.+ passthrough = {}+ # Linking only produces a file the loader can read when shards are stored in the same format+ # the persister writes; the MLX persister, for instance, writes .mlx.npz.+ persister_is_safetensors = type(ModelPersister.get_model_persister()).__name__ == 'SafetensorModelPersister'+ if compression is None and safetensors_format and persister_is_safetensors:+ shard_contents = defaultdict(list)+ for k, v in index.items():+ shard_contents[v].append(k)+ for layer in layers:+ files = {v for k, v in index.items() if k.startswith(layer)}+ if len(files) != 1:+ continue+ only_file = next(iter(files))+ if all(k.startswith(layer) for k in shard_contents[only_file]):+ passthrough[layer] = only_file++ if passthrough:+ print(f"{len(passthrough)}/{len(layers)} modules are already one-per-shard; "+ f"linking to the original files instead of copying them.")++ # Must exist before check_space, which stats the filesystem it lives on.+ saving_path.mkdir(parents=True, exist_ok=True)++ # A copy is only made for the layers we cannot link, so only those need free space.+ if not delete_original and len(passthrough) < len(layers):check_space(checkpoint_path, layer_shards_saving_path, compression, splitted_model_dir_name=splitted_model_dir_name)@@ -289,14 +361,29 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splittexcept ValueError:pass- if not os.path.exists(saving_path):- #os.makedirs(saving_path)- saving_path.mkdir(parents=True, exist_ok=True)-single_modelfile = Nonefor layer in tqdm(layers):+ if layer in passthrough:+ src = checkpoint_path / passthrough[layer]+ if not os.path.exists(src):+ assert repo_id is not None+ huggingface_hub.snapshot_download(repo_id, allow_patterns=os.path.basename(src),+ token=hf_token)+ if not ModelPersister.get_model_persister().model_persist_exist(layer, saving_path):+ link_or_copy_file(src, saving_path / (layer + 'safetensors'))+ (saving_path / (layer + 'safetensors.done')).touch()+ # Keep the shard cursor in step with what we skipped, so a later layer that does need+ # loading doesn't walk back through (and read) every shard we just linked past.+ src_parts = passthrough[layer].split('-')+ if len(src_parts) > 1:+ try:+ shard = max(shard, int(src_parts[1]))+ except ValueError:+ pass+ continue+# Optionnally load next shard# checking whether after spliting from '-', if second element exists. otherwise it throws errors for single 'model.safetensor' filesshards = [int(v.split('-')[1]) for k, v in index.items() if k.startswith(layer) and '-' in v and len(v.split('-')) > 1]@@ -395,9 +482,13 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards# try local model path, if the model exist split and save thereif os.path.exists(model_local_path_or_repo_id):- if os.path.exists(Path(model_local_path_or_repo_id) / 'pytorch_model.bin.index.json') or \- os.path.exists(Path(model_local_path_or_repo_id) / 'model.safetensors.index.json'):- print(f"found index file...")+ # Accept single-file checkpoints too, not just sharded ones with an index: the splitter+ # handles both, so requiring an index needlessly sent local single-file models down the+ # "treat it as a repo id" path, where they fail as an invalid repo name.+ local_weight_files = ('pytorch_model.bin.index.json', 'model.safetensors.index.json',+ 'model.safetensors', 'pytorch_model.bin')+ if any(os.path.exists(Path(model_local_path_or_repo_id) / f) for f in local_weight_files):+ print(f"found local checkpoint...")return Path(model_local_path_or_repo_id), split_and_save_layers(model_local_path_or_repo_id, layer_shards_saving_path,compression=compression, layer_names=layer_names, delete_original=delete_original)else:diff --git a/air_llm/tests/test_kimi_k3_split.py b/air_llm/tests/test_kimi_k3_split.pynew file mode 100644index 0000000..c9c6543--- /dev/null+++ b/air_llm/tests/test_kimi_k3_split.py@@ -0,0 +1,224 @@+"""Structural tests for Kimi-K3-style checkpoints.++K3 is multimodal (decoder nested under ``language_model``), ships one shard per decoder layer, and+carries modules outside the streamed sequence (vision tower, projector, extra residual norms).+These tests build a miniature checkpoint with the same shape and check that the splitter links+rather than copies the one-module-per-shard files, still materialises the shared shard correctly,+and never loses a module because of shard ordering.+"""++import json+import os+import sys+import tempfile+import types+import unittest+from pathlib import Path++import torch+from safetensors.torch import save_file, load_file++_AIRLLM_DIR = Path(__file__).resolve().parents[1] / 'airllm'++# Bind the package without running airllm/__init__.py: that module pulls in the MLX backend on+# macOS and the full transformers stack elsewhere, none of which these splitter tests need.+if 'airllm' not in sys.modules:+ _pkg = types.ModuleType('airllm')+ _pkg.__path__ = [str(_AIRLLM_DIR)]+ sys.modules['airllm'] = _pkg++from airllm.persist import model_persister as _persister_mod+from airllm.persist.safetensor_model_persister import SafetensorModelPersister++# The persister is otherwise chosen by platform, so pin the safetensors one to exercise the+# Linux/CUDA path regardless of where the test runs.+_persister_mod.model_persister = SafetensorModelPersister()++from airllm.utils import split_and_save_layers++N_LAYERS = 3+LAYER_PREFIX = "language_model.model.layers"++KIMI_LAYER_NAMES = {+ 'embed': 'language_model.model.embed_tokens',+ 'layer_prefix': LAYER_PREFIX,+ 'norm': 'language_model.model.norm',+ 'lm_head': 'language_model.lm_head',+ 'resident': [+ 'language_model.model.output_attn_res_norm',+ 'language_model.model.output_attn_res_proj',+ 'mm_projector',+ 'vision_tower',+ ],+}+++def build_fake_checkpoint(root):+ """One shard per layer, then a shared shard, then projector and vision tower."""+ n_shards = N_LAYERS + 3+ shards = {}++ for i in range(N_LAYERS):+ name = f"model-{i + 1:05d}-of-{n_shards:06d}.safetensors"+ shards[name] = {+ # A packed 4-bit payload plus its scale, like MXFP4 stores.+ f"{LAYER_PREFIX}.{i}.block_sparse_moe.experts.0.w1.weight_packed":+ torch.randint(0, 255, (8, 4), dtype=torch.uint8),+ f"{LAYER_PREFIX}.{i}.block_sparse_moe.experts.0.w1.weight_scale":+ torch.randn(8, 1, dtype=torch.float32),+ f"{LAYER_PREFIX}.{i}.input_layernorm.weight": torch.randn(8),+ }++ shared = f"model-{N_LAYERS + 1:05d}-of-{n_shards:06d}.safetensors"+ shards[shared] = {+ "language_model.model.embed_tokens.weight": torch.randn(16, 8),+ "language_model.model.norm.weight": torch.randn(8),+ "language_model.lm_head.weight": torch.randn(16, 8),+ "language_model.model.output_attn_res_norm.weight": torch.randn(8),+ "language_model.model.output_attn_res_proj.weight": torch.randn(8, 8),+ }+ # Deliberately out of order relative to the resident list: the projector lives in an *earlier*+ # shard than the vision tower, which the splitter must handle without losing tensors.+ proj = f"model-{N_LAYERS + 2:05d}-of-{n_shards:06d}.safetensors"+ shards[proj] = {"mm_projector.proj.0.weight": torch.randn(8, 8)}+ vis = f"model-{N_LAYERS + 3:05d}-of-{n_shards:06d}.safetensors"+ shards[vis] = {"vision_tower.encoder.blocks.0.mlp.fc0.weight": torch.randn(8, 8)}++ weight_map = {}+ for fname, sd in shards.items():+ save_file(sd, str(Path(root) / fname))+ for k in sd:+ weight_map[k] = fname+ with open(Path(root) / "model.safetensors.index.json", "w") as f:+ json.dump({"metadata": {"total_size": 1}, "weight_map": weight_map}, f)+ return shards, weight_map+++class TestKimiK3Split(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.root = Path(self.tmp.name)+ self.shards, self.weight_map = build_fake_checkpoint(self.root)+ self.out = Path(split_and_save_layers(self.root, layer_names=KIMI_LAYER_NAMES))++ def tearDown(self):+ self.tmp.cleanup()++ def _split_file(self, module):+ return self.out / f"{module}.safetensors"++ def test_every_module_is_split_out(self):+ expected = ([KIMI_LAYER_NAMES['embed']]+ + [f"{LAYER_PREFIX}.{i}" for i in range(N_LAYERS)]+ + [KIMI_LAYER_NAMES['norm'], KIMI_LAYER_NAMES['lm_head']]+ + KIMI_LAYER_NAMES['resident'])+ for module in expected:+ self.assertTrue(self._split_file(module).exists(), f"missing split for {module}")++ def test_one_shard_per_layer_is_linked_not_copied(self):+ """The whole point: a 1.5TB checkpoint must not be duplicated."""+ for i in range(N_LAYERS):+ src = self.root / f"model-{i + 1:05d}-of-{N_LAYERS + 3:06d}.safetensors"+ dst = self._split_file(f"{LAYER_PREFIX}.{i}")+ self.assertEqual(os.stat(src).st_ino, os.stat(dst).st_ino,+ f"layer {i} was copied instead of linked")++ def test_projector_and_vision_tower_are_linked(self):+ for module, shard_no in (('mm_projector', N_LAYERS + 2), ('vision_tower', N_LAYERS + 3)):+ src = self.root / f"model-{shard_no:05d}-of-{N_LAYERS + 3:06d}.safetensors"+ self.assertEqual(os.stat(src).st_ino, os.stat(self._split_file(module)).st_ino,+ f"{module} was copied instead of linked")++ def test_shared_shard_modules_are_materialised_separately(self):+ """embed / norm / lm_head / residual modules share one shard, so they must be real files."""+ shared_src = self.root / f"model-{N_LAYERS + 1:05d}-of-{N_LAYERS + 3:06d}.safetensors"+ for module in ('language_model.model.embed_tokens', 'language_model.model.norm',+ 'language_model.lm_head', 'language_model.model.output_attn_res_norm',+ 'language_model.model.output_attn_res_proj'):+ dst = self._split_file(module)+ self.assertNotEqual(os.stat(shared_src).st_ino, os.stat(dst).st_ino)+ keys = set(load_file(str(dst)).keys())+ self.assertTrue(keys, f"{module} split is empty")+ self.assertTrue(all(k.startswith(module + '.') for k in keys),+ f"{module} split leaked other tensors: {keys}")++ def test_contents_match_the_original_checkpoint(self):+ """Linked or copied, every tensor must survive the split bit-for-bit."""+ original = {}+ for fname in self.shards:+ original.update(load_file(str(self.root / fname)))++ recovered = {}+ for f in self.out.glob('*.safetensors'):+ recovered.update(load_file(str(f)))++ self.assertEqual(set(original.keys()), set(recovered.keys()))+ for k, v in original.items():+ self.assertEqual(v.dtype, recovered[k].dtype, f"{k} changed dtype")+ self.assertTrue(torch.equal(v, recovered[k]), f"{k} changed value")++ def test_packed_4bit_payload_keeps_its_integer_dtype(self):+ sd = load_file(str(self._split_file(f"{LAYER_PREFIX}.0")))+ packed = [v for k, v in sd.items() if k.endswith('weight_packed')][0]+ self.assertEqual(packed.dtype, torch.uint8)+++class TestStandardCheckpointStillSplits(unittest.TestCase):+ """Regression guard: the ordinary layout (several layers per shard) must be unaffected.++ Passthrough must not kick in here, because no shard holds exactly one module.+ """++ N = 4++ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.root = Path(self.tmp.name)++ shard_a = {"model.embed_tokens.weight": torch.randn(16, 8)}+ for i in range(2):+ shard_a[f"model.layers.{i}.self_attn.q_proj.weight"] = torch.randn(8, 8)+ shard_b = {}+ for i in range(2, self.N):+ shard_b[f"model.layers.{i}.self_attn.q_proj.weight"] = torch.randn(8, 8)+ shard_b["model.norm.weight"] = torch.randn(8)+ shard_b["lm_head.weight"] = torch.randn(16, 8)++ weight_map = {}+ for fname, sd in (("model-00001-of-00002.safetensors", shard_a),+ ("model-00002-of-00002.safetensors", shard_b)):+ save_file(sd, str(self.root / fname))+ for k in sd:+ weight_map[k] = fname+ with open(self.root / "model.safetensors.index.json", "w") as f:+ json.dump({"metadata": {"total_size": 1}, "weight_map": weight_map}, f)++ self.original = dict(shard_a)+ self.original.update(shard_b)+ self.out = Path(split_and_save_layers(self.root))++ def tearDown(self):+ self.tmp.cleanup()++ def test_all_modules_written_as_real_files(self):+ expected = (['model.embed_tokens'] + [f'model.layers.{i}' for i in range(self.N)]+ + ['model.norm', 'lm_head'])+ src_inodes = {os.stat(self.root / f).st_ino+ for f in os.listdir(self.root) if f.endswith('.safetensors')}+ for module in expected:+ dst = self.out / f"{module}.safetensors"+ self.assertTrue(dst.exists(), f"missing split for {module}")+ self.assertNotIn(os.stat(dst).st_ino, src_inodes,+ f"{module} was linked, but this layout should be copied")++ def test_contents_round_trip(self):+ recovered = {}+ for f in self.out.glob('*.safetensors'):+ recovered.update(load_file(str(f)))+ self.assertEqual(set(self.original.keys()), set(recovered.keys()))+ for k, v in self.original.items():+ self.assertTrue(torch.equal(v, recovered[k]), f"{k} changed value")+++if __name__ == '__main__':+ unittest.main(verbosity=2)