Skip to content

AirLLM 2.8T LLM inference with single 4GB GPU

feat: support Qwen3.8-Flash-Next on 5.95GB (3.3.0)

Yu Liee09fafc170d
diff --git a/README.md b/README.md
index d1341b6..bc22c68 100644
--- a/README.md
+++ b/README.md
@@ -31,6 +31,8 @@
* [Bloome — build & run AI agent teams in the cloud, zero setup](https://bloome.im/app?ref=G6BYnov0&utm_medium=github&utm_source=lyogavin-airllm-ivor-202606)
## Updates
+[2026/08] **Qwen3.8-Flash-Next** support: Qwen's 125B MoE flagship (`Qwen4ExpForConditionalGeneration`) with a ~51B n-gram embedding runs in **5.95GB** of VRAM, measured end to end on one RTX 4090. The n-gram table is file-mapped on the host (a 64GB machine is enough); decoder layers stream. Needs a `transformers` build with in-tree `qwen4_exp` (`pip install git+https://github.com/huggingface/transformers.git` today) and ~360GB of checkpoint disk (`delete_original=True` reclaims the originals after the split).
+
[2026/08] **Qwen3.8-27B** support: Qwen's new dense VL (Gated DeltaNet + Gated Attention, native vision) runs in **3.33GB** of VRAM, measured end to end on one RTX 3090. Needs `transformers` 5.8+.
[2026/07] **Kimi K3 (2.8T)** support: the largest open-source model runs on a single card in **3.72GB** of VRAM, measured end to end on one RTX 6000 Ada. Per-expert streaming loads only the experts a token actually routes to. K3 brings three requirements of its own: `pip install compressed-tensors flash-attn` (its model code mandates flash attention regardless of what you request), a CUDA 12 build of torch, since no prebuilt flash-attn wheel exists for CUDA 13 yet, and `transformers` 4.56.x, as its remote code does not load on 5.x.
@@ -106,6 +108,7 @@ model = AutoModel.from_pretrained("Qwen/Qwen3-32B")
# go bigger with the exact same one line:
#model = AutoModel.from_pretrained("Qwen/Qwen3.8-27B") # 27B dense VL, 3.33GB
+#model = AutoModel.from_pretrained("Qwen/Qwen3.8-Flash-Next") # 125B MoE + 51B PLE, 5.95GB
#model = AutoModel.from_pretrained("Qwen/Qwen3-235B-A22B") # 235B, runs in ~3GB
#model = AutoModel.from_pretrained("deepseek-ai/DeepSeek-V3") # 671B, runs in ~12GB
@@ -275,7 +278,7 @@ model.tokenizer.decode(generation_output.sequences[0])
AirLLM works out of the box with **virtually every popular open LLM** — just pass its Hugging Face ID to `AutoModel.from_pretrained(...)`. That covers all the major families:
-**Llama** (2 / 3 / 3.1 / 3.3 / 4) · **Qwen** (1 / 2 / 2.5 / 3 / 3.5 / 3.8, including MoE, FP8, and native VL) · **DeepSeek** (V2 / V3 / R1) · **Mistral & Mixtral** · **Phi** · **Gemma** · **ChatGLM** · **Baichuan** · **InternLM** · **Yi** · **Kimi K3** — and most new models the day they're released.
+**Llama** (2 / 3 / 3.1 / 3.3 / 4) · **Qwen** (1 / 2 / 2.5 / 3 / 3.5 / 3.8, including MoE, Flash-Next, FP8, and native VL) · **DeepSeek** (V2 / V3 / R1) · **Mistral & Mixtral** · **Phi** · **Gemma** · **ChatGLM** · **Baichuan** · **InternLM** · **Yi** · **Kimi K3** — and most new models the day they're released.
### Tiny GPU, huge models
@@ -286,6 +289,7 @@ The trick: AirLLM only ever keeps **one layer on the GPU at a time**, so the VRA
| Qwen3 / Mistral / Phi (≈8B) | 8B | **~1–2 GB** |
| Qwen3-30B / Mixtral (MoE) | 30–47B | **~1–3 GB** |
| Qwen3.8-27B (dense VL) | 27B | **3.33 GB** |
+| Qwen3.8-Flash-Next (MoE + PLE) | ~180B | **5.95 GB** |
| Qwen3-235B (MoE) | 235B | **~3 GB** |
| Llama 3.x 70B (full precision) | 70B | **~4 GB** |
| Llama 3.1 405B | 405B | **~8 GB** |
diff --git a/air_llm/airllm/__init__.py b/air_llm/airllm/__init__.py
index bfff72f..5ce56df 100644
--- a/air_llm/airllm/__init__.py
+++ b/air_llm/airllm/__init__.py
@@ -33,6 +33,7 @@ else:
("AirLLMMixtral", ".airllm_mixtral"),
("AirLLMKimiK3", ".airllm_kimi_k3"),
("AirLLMQwen3_5", ".airllm_qwen3_5"),
+ ("AirLLMQwen4Exp", ".airllm_qwen4_exp"),
):
try:
_mod = __import__(__name__ + _module, fromlist=[_name])
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.py
index efd9959..8da7ab9 100644
--- a/air_llm/airllm/airllm_base.py
+++ b/air_llm/airllm/airllm_base.py
@@ -16,7 +16,8 @@ from transformers.quantizers import AutoHfQuantizer
from .profiler import LayeredProfiler
from .utils import clean_memory, load_layer, layer_tensor_names, load_layer_subset, \
- find_or_create_local_splitted_path
+ find_or_create_local_splitted_path, load_merged_ngram_embedding, \
+ open_ngram_mmap_table, MmapEmbedding, _force_meta_embeddings
from .persist import ModelPersister
try:
@@ -198,6 +199,7 @@ class AirLLMBaseModel:
self.set_layers_from_layer_names()
self._load_resident_modules()
+ self._load_cpu_resident_modules()
self._install_streaming_hooks()
# ---- customization hooks for subclasses -------------------------------------------------
@@ -289,29 +291,30 @@ class AirLLMBaseModel:
"trust_remote_code": self.trust_remote_code,
}
errors = []
- for name, cls in self._auto_model_classes():
- try:
- with init_empty_weights(include_buffers=False):
- model = cls.from_config(self.config, **kwargs)
- except TypeError:
- # Older Auto factories don't take attn_implementation.
+ with _force_meta_embeddings():
+ for name, cls in self._auto_model_classes():
try:
with init_empty_weights(include_buffers=False):
- model = cls.from_config(self.config, trust_remote_code=self.trust_remote_code)
+ model = cls.from_config(self.config, **kwargs)
+ except TypeError:
+ # Older Auto factories don't take attn_implementation.
+ try:
+ with init_empty_weights(include_buffers=False):
+ model = cls.from_config(self.config, trust_remote_code=self.trust_remote_code)
+ except Exception as e: # noqa: BLE001 - try the next factory
+ errors.append(f"{name}: {type(e).__name__}: {e}")
+ continue
except Exception as e: # noqa: BLE001 - try the next factory
errors.append(f"{name}: {type(e).__name__}: {e}")
continue
- except Exception as e: # noqa: BLE001 - try the next factory
- errors.append(f"{name}: {type(e).__name__}: {e}")
- continue
- if not self._model_matches_layer_names(model):
- errors.append(
- f"{name}: built {type(model).__name__} but it is missing "
- f"{self.layer_names_dict['layer_prefix']}"
- )
- continue
- print(f"built empty {type(model).__name__} via {name} (attn={attn_implementation})")
- return model
+ if not self._model_matches_layer_names(model):
+ errors.append(
+ f"{name}: built {type(model).__name__} but it is missing "
+ f"{self.layer_names_dict['layer_prefix']}"
+ )
+ continue
+ print(f"built empty {type(model).__name__} via {name} (attn={attn_implementation})")
+ return model
raise RuntimeError(
"Could not instantiate the model on meta from config. "
f"architecture={getattr(self.config, 'architectures', None)} "
@@ -586,8 +589,15 @@ class AirLLMBaseModel:
def move_layer_to_device(self, state_dict):
self._restore_plain_weight_modules(state_dict)
state_dict = self._decompress_state_dict(state_dict)
+ marker = self.layer_names_dict.get('cpu_resident_marker')
moved = []
for param_name in self._param_names_from_state_dict(state_dict):
+ # A nested CPU table (Flash-Next PLE) must never be placed on the GPU, even if a
+ # decoder-layer shard still contains it.
+ if param_name in getattr(self, '_cpu_resident_params', ()):
+ continue
+ if marker and marker in param_name:
+ continue
if self.hf_quantizer is not None and self._needs_quantization(param_name):
# On-the-fly-quantizing schemes (e.g. bitsandbytes) reconstruct the param from the
# weight plus companion quant-state tensors carried in state_dict.
@@ -665,6 +675,66 @@ class AirLLMBaseModel:
continue
self.move_layer_to_device(state_dict)
+ def _cpu_resident_names(self):
+ """Module prefixes whose weights stay on CPU for the lifetime of the process.
+
+ Flash-Next's n-gram table is the motivating case: transformers already gathers rows onto
+ the embedding's own device, so keeping ~102GB of bf16 on the host is the designed path.
+ Names come from ``cpu_resident`` and from split files matching ``cpu_resident_marker``.
+ """
+ names = list(self.layer_names_dict.get('cpu_resident', []))
+ marker = self.layer_names_dict.get('cpu_resident_marker')
+ if marker:
+ for path in Path(self.checkpoint_path).glob('*.mmap.json'):
+ stem = path.name[:-len('.mmap.json')]
+ if stem.endswith(marker) and stem not in names:
+ names.append(stem)
+ for path in Path(self.checkpoint_path).glob('*.safetensors'):
+ stem = path.name[:-len('.safetensors')]
+ if stem.endswith(marker) and stem not in names:
+ names.append(stem)
+ return names
+
+ def _install_mmap_embedding(self, module_name, table):
+ parent_name, _, attr = module_name.rpartition('.')
+ parent = self.model.get_submodule(parent_name)
+ setattr(parent, attr, MmapEmbedding(table))
+
+ def _load_cpu_resident_modules(self):
+ """Load oversized lookup tables onto CPU and never stream them to the GPU.
+
+ Flash-Next's n-gram table is ~102GB bf16. We mmap it from disk so a 64GB host can still
+ run; transformers already gathers rows on ``ngram_embedding.weight.device``.
+ ``module.to('meta')`` after a decoder layer would otherwise evict a child embedding that
+ lives under that layer, so we also record the parameter names and evict selectively.
+ """
+ self._cpu_resident_params = set()
+ for name in self._cpu_resident_names():
+ mmap_meta = Path(self.checkpoint_path) / f"{name}.mmap.json"
+ if mmap_meta.exists():
+ table = open_ngram_mmap_table(self.checkpoint_path, name)
+ self._install_mmap_embedding(name, table)
+ self._cpu_resident_params.add(f"{name}.weight")
+ print(f"cpu-resident mmap embedding: {name} "
+ f"{tuple(table.shape)} {table.dtype} (gathered from disk, not copied to RAM)")
+ continue
+ try:
+ state_dict = load_merged_ngram_embedding(self.checkpoint_path, name)
+ except FileNotFoundError:
+ continue
+ for param_name, value in state_dict.items():
+ self._adopt_checkpoint_shape(param_name, value)
+ if self._should_load_verbatim(param_name, value):
+ set_module_tensor_to_device(self.model, param_name, 'cpu', value=value)
+ else:
+ set_module_tensor_to_device(
+ self.model, param_name, 'cpu', value=value, dtype=self.running_dtype)
+ self._cpu_resident_params.add(param_name)
+ if self._cpu_resident_params:
+ n = len(self._cpu_resident_params)
+ print(f"cpu-resident modules: left {n} tensors on host RAM "
+ f"(n-gram / PLE table is gathered on CPU, not streamed to the GPU).")
+
def _install_streaming_hooks(self):
# Modules execute in this order during a forward: embed -> layers -> norm -> lm_head.
n = len(self.layer_names)
@@ -814,9 +884,13 @@ class AirLLMBaseModel:
self._prefetched_idx = nxt
def _post_hook(self, module, args, output):
- # module.to('meta') would also evict the experts, which manage their own lifetime, so with
- # expert streaming we only release exactly what this hook placed.
- if self.hf_quantizer is not None or getattr(self, '_expert_streaming', False):
+ # module.to('meta') would also evict experts (which manage their own lifetime) and any
+ # cpu-resident child such as Flash-Next's n-gram table, which lives under a decoder layer
+ # but must stay materialised on the host. When any of those are in play, only release the
+ # tensors this hook actually placed.
+ if (self.hf_quantizer is not None
+ or getattr(self, '_expert_streaming', False)
+ or getattr(self, '_cpu_resident_params', None)):
for param_name in getattr(module, '_airllm_moved', []):
set_module_tensor_to_device(self.model, param_name, 'meta')
else:
diff --git a/air_llm/airllm/airllm_qwen4_exp.py b/air_llm/airllm/airllm_qwen4_exp.py
new file mode 100644
index 0000000..df81c22
--- /dev/null
+++ b/air_llm/airllm/airllm_qwen4_exp.py
@@ -0,0 +1,62 @@
+from .airllm_base import AirLLMBaseModel
+from .utils import _wrap_forward_int64_scatter
+
+
+def _ensure_qwen4_exp_indexer_scatter_int64():
+ """transformers qwen4_exp writes int32 indexer indices; torch.scatter requires int64."""
+ try:
+ from transformers.models.qwen4_exp.modeling_qwen4_exp import Qwen4ExpTextQSAIndexer
+ except ImportError:
+ return
+ if getattr(Qwen4ExpTextQSAIndexer.forward, '_airllm_int64_scatter', False):
+ return
+ Qwen4ExpTextQSAIndexer.forward = _wrap_forward_int64_scatter(Qwen4ExpTextQSAIndexer.forward)
+ Qwen4ExpTextQSAIndexer.forward._airllm_int64_scatter = True
+
+
+class AirLLMQwen4Exp(AirLLMBaseModel):
+ """Qwen3.8-Flash-Next / Qwen4-Exp (``Qwen4ExpForConditionalGeneration``).
+
+ Flash-Next is not the 27B dense VL. It is a 48-layer hybrid MoE (512 experts, 10 routed + 1
+ shared) with Gated DeltaNet, Qwen Sparse Attention, hyper-connections, and a ~51B n-gram
+ embedding (PLE) injected at one decoder layer. Transformers stores that table as
+ ``ngram_embedding.shard_N.weight`` and concatenates it at load time; the modeling code already
+ gathers rows on whatever device the table lives on (``_no_placement_params``).
+
+ AirLLM therefore:
+
+ * streams the decoder under ``model.language_model`` (there is no final RMSNorm; the output
+ mix is ``hyper_connection_mixer``)
+ * keeps ``model.visual`` resident, same as Qwen3.8-27B
+ * constructs that ``nn.Embedding`` on the meta device (accelerate would otherwise allocate
+ ~191GB of empty fp32 on CPU before moving it) then peels the table out of its decoder
+ layer and file-mmaps it on the host, so ~102GB of bf16 never lands in GPU VRAM or
+ anonymous RAM
+ * drops ``mtp.*`` (transformers ignores the Multi-Token Prediction head)
+ * works around a transformers indexer bug that scatters ``int32`` indices (torch wants
+ ``int64``) so sparse-attention layers can run
+
+ Packed MoE experts are 3D tensors on a single module, not a ``ModuleList``, so Kimi-style
+ per-expert module hooks do not apply. A whole MoE layer is a few GB of bf16; that is what
+ streams today.
+
+ Needs a transformers build that includes in-tree ``qwen4_exp`` (the class is not remote code).
+ Optional ``fla`` / ``causal-conv1d`` speed up DeltaNet; transformers falls back to PyTorch
+ when they are missing.
+ """
+
+ def set_layer_names_dict(self):
+ _ensure_qwen4_exp_indexer_scatter_int64()
+ self.layer_names_dict = {
+ 'embed': 'model.language_model.embed_tokens',
+ 'layer_prefix': 'model.language_model.layers',
+ 'norm': 'model.language_model.hyper_connection_mixer',
+ 'lm_head': 'lm_head',
+ 'resident': [
+ 'model.visual',
+ ],
+ # Matches ``model.language_model.layers.{i}.ple.ple_embedding.ngram_embedding`` for
+ # whatever layer ``ple_layer_ids`` selected. The splitter peels those tensors out of
+ # the parent decoder layer so streaming that layer cannot drag ~102GB onto the GPU.
+ 'cpu_resident_marker': 'ple.ple_embedding.ngram_embedding',
+ }
diff --git a/air_llm/airllm/auto_model.py b/air_llm/airllm/auto_model.py
index 2a48c01..f6608df 100644
--- a/air_llm/airllm/auto_model.py
+++ b/air_llm/airllm/auto_model.py
@@ -23,6 +23,7 @@ ARCH_OVERRIDES = {
"InternLMForCausalLM": "AirLLMInternLM",
"KimiK3ForConditionalGeneration": "AirLLMKimiK3",
"Qwen3_5ForConditionalGeneration": "AirLLMQwen3_5",
+ "Qwen4ExpForConditionalGeneration": "AirLLMQwen4Exp",
}
diff --git a/air_llm/airllm/persist/safetensor_model_persister.py b/air_llm/airllm/persist/safetensor_model_persister.py
index 23f5cc7..8c6caca 100644
--- a/air_llm/airllm/persist/safetensor_model_persister.py
+++ b/air_llm/airllm/persist/safetensor_model_persister.py
@@ -19,6 +19,11 @@ class SafetensorModelPersister(ModelPersister):
def model_persist_exist(self, layer_name, saving_path):
+ mmap_exists = os.path.exists(str(saving_path / (layer_name + 'mmap')))
+ mmap_done = os.path.exists(str(saving_path / (layer_name + 'mmap.done')))
+ if mmap_exists and mmap_done:
+ return True
+
safetensor_exists = os.path.exists(str(saving_path / (layer_name + 'safetensors')))
done_marker_exists = os.path.exists(str(saving_path / (layer_name + 'safetensors.done')))
diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.py
index ad45f99..75740db 100644
--- a/air_llm/airllm/utils.py
+++ b/air_llm/airllm/utils.py
@@ -1,6 +1,7 @@
import gc
import json
import os
+import re
import ctypes
import shutil
from tqdm import tqdm
@@ -9,6 +10,7 @@ from glob import glob
import time
from collections import OrderedDict, defaultdict
+from contextlib import contextmanager
from typing import Dict, List, Optional, Tuple, Union
from sys import platform
@@ -113,6 +115,297 @@ def uncompress_layer_state_dict(layer_state_dict):
return layer_state_dict if uncompressed_layer_state_dict is None else uncompressed_layer_state_dict
+
+# Qwen4-Exp / Flash-Next stores the ~51B PLE n-gram table as ``*.ngram_embedding.shard_N.weight``.
+# Transformers concatenates those shards into a single ``nn.Embedding.weight`` at load time.
+# On machines that cannot hold ~102GB of host RAM we instead stream the shards into a file-backed
+# mmap and gather rows from disk (the modeling code already looks up on ``weight.device``).
+_NGRAM_SHARD_RE = re.compile(r'^(?P<prefix>.*\.ngram_embedding)\.shard_(?P<idx>\d+)\.weight$')
+_DTYPE_TAGS = {
+ torch.bfloat16: 'bf16',
+ torch.float16: 'fp16',
+ torch.float32: 'fp32',
+}
+_TAG_DTYPES = {tag: dtype for dtype, tag in _DTYPE_TAGS.items()}
+
+
+def merge_ngram_embedding_shards(state_dict):
+ """Concatenate ``ngram_embedding.shard_N.weight`` tensors into ``ngram_embedding.weight``.
+
+ Keys that are not n-gram shards pass through unchanged. Shard indices must be a contiguous
+ 0..N-1 range; a gap would silently drop rows from the hash table.
+ """
+ groups = {}
+ out = {}
+ for key, value in state_dict.items():
+ match = _NGRAM_SHARD_RE.match(key)
+ if match:
+ groups.setdefault(match.group('prefix'), []).append((int(match.group('idx')), value))
+ else:
+ out[key] = value
+ for prefix, shards in groups.items():
+ shards.sort(key=lambda item: item[0])
+ got = [idx for idx, _ in shards]
+ expected = list(range(len(shards)))
+ if got != expected:
+ raise ValueError(
+ f"ngram embedding shards for {prefix} are not contiguous 0..{len(shards) - 1}: {got}"
+ )
+ out[f'{prefix}.weight'] = torch.cat([tensor for _, tensor in shards], dim=0)
+ return out
+
+
+def load_merged_ngram_embedding(local_path, layer_name):
+ """Load a split n-gram file and concatenate shards without holding two full copies.
+
+ ``load_file`` plus ``torch.cat`` would peak at ~2x the table (~200GB for Flash-Next). This
+ reads one shard at a time into a preallocated host tensor.
+ """
+ filepath = Path(local_path) / (layer_name + ".safetensors")
+ with safe_open(str(filepath), framework="pt") as handle:
+ groups = {}
+ rest = {}
+ for key in handle.keys():
+ match = _NGRAM_SHARD_RE.match(key)
+ if match:
+ groups.setdefault(match.group('prefix'), []).append((int(match.group('idx')), key))
+ else:
+ rest[key] = handle.get_tensor(key)
+ for prefix, shards in groups.items():
+ shards.sort(key=lambda item: item[0])
+ got = [idx for idx, _ in shards]
+ expected = list(range(len(shards)))
+ if got != expected:
+ raise ValueError(
+ f"ngram embedding shards for {prefix} are not contiguous "
+ f"0..{len(shards) - 1}: {got}"
+ )
+ shapes = [tuple(handle.get_slice(key).get_shape()) for _, key in shards]
+ first = handle.get_tensor(shards[0][1])
+ dest = first.new_empty((sum(shape[0] for shape in shapes),) + shapes[0][1:])
+ dest[0:shapes[0][0]].copy_(first)
+ del first
+ offset = shapes[0][0]
+ for (_, key), shape in zip(shards[1:], shapes[1:]):
+ piece = handle.get_tensor(key)
+ dest[offset:offset + shape[0]].copy_(piece)
+ offset += shape[0]
+ del piece
+ rest[f'{prefix}.weight'] = dest
+ return rest
+
+
+def ngram_mmap_exists(saving_path, layer_name):
+ return (os.path.exists(str(saving_path / (layer_name + 'mmap')))
+ and os.path.exists(str(saving_path / (layer_name + 'mmap.json')))
+ and os.path.exists(str(saving_path / (layer_name + 'mmap.done'))))
+
+
+def _ngram_shard_items(layer_prefix, index):
+ items = []
+ for key, filename in index.items():
+ if not key.startswith(layer_prefix):
+ continue
+ match = _NGRAM_SHARD_RE.match(key)
+ if not match:
+ continue
+ items.append((int(match.group('idx')), key, filename))
+ items.sort(key=lambda item: item[0])
+ got = [idx for idx, _, _ in items]
+ if got != list(range(len(items))):
+ raise ValueError(
+ f"ngram embedding shards for {layer_prefix} are not contiguous "
+ f"0..{len(items) - 1}: {got}"
+ )
+ return items
+
+
+def persist_ngram_mmap(layer_prefix, index, checkpoint_path, saving_path,
+ repo_id=None, hf_token=None):
+ """Stream n-gram shards into a file-backed mmap without holding the full table in RAM."""
+ if ngram_mmap_exists(saving_path, layer_prefix):
+ return
+ items = _ngram_shard_items(layer_prefix, index)
+ if not items:
+ return
+
+ first_file = Path(checkpoint_path) / items[0][2]
+ if not first_file.exists():
+ assert repo_id is not None
+ huggingface_hub.snapshot_download(repo_id, allow_patterns=os.path.basename(first_file),
+ token=hf_token)
+
+ open_handles = {}
+ try:
+ def _handle(filename):
+ path = str(Path(checkpoint_path) / filename)
+ if path not in open_handles:
+ src = Path(checkpoint_path) / filename
+ if not src.exists():
+ assert repo_id is not None
+ huggingface_hub.snapshot_download(
+ repo_id, allow_patterns=os.path.basename(src), token=hf_token)
+ cm = safe_open(path, framework='pt')
+ open_handles[path] = (cm, cm.__enter__())
+ return open_handles[path][1]
+
+ def _tensor(filename, key):
+ return _handle(filename).get_tensor(key)
+
+ first = _tensor(items[0][2], items[0][1])
+ cols = first.shape[1]
+ dtype = first.dtype
+ if dtype not in _DTYPE_TAGS:
+ raise ValueError(f"unsupported n-gram mmap dtype {dtype}")
+ rows = first.shape[0]
+ shapes_rest = []
+ for _, key, filename in items[1:]:
+ shapes_rest.append(tuple(_handle(filename).get_slice(key).get_shape()))
+ rows += shapes_rest[-1][0]
+
+ raw_path = Path(saving_path) / (layer_prefix + 'mmap')
+ tmp_path = Path(saving_path) / (layer_prefix + 'mmap.tmp')
+ nbytes = rows * cols * first.element_size()
+ print(f"writing n-gram mmap ({rows} x {cols} {_DTYPE_TAGS[dtype]}, "
+ f"{nbytes / (1024 ** 3):.1f}GB) to {raw_path}")
+ with open(tmp_path, 'wb') as out:
+ out.write(first.contiguous().view(torch.uint8).numpy().tobytes())
+ del first
+ for (_, key, filename), shape in zip(items[1:], shapes_rest):
+ piece = _tensor(filename, key)
+ if tuple(piece.shape) != shape:
+ raise ValueError(f"{key} shape {tuple(piece.shape)} != {shape}")
+ out.write(piece.contiguous().view(torch.uint8).numpy().tobytes())
+ del piece
+ tmp_path.replace(raw_path)
+ meta = {
+ 'rows': rows,
+ 'cols': cols,
+ 'dtype': _DTYPE_TAGS[dtype],
+ 'nbytes': nbytes,
+ }
+ with open(Path(saving_path) / (layer_prefix + 'mmap.json'), 'w') as f:
+ json.dump(meta, f)
+ (Path(saving_path) / (layer_prefix + 'mmap.done')).touch()
+ print(f"saved n-gram mmap as: {raw_path}")
+ finally:
+ for cm, _handle_obj in open_handles.values():
+ cm.__exit__(None, None, None)
+
+
+def open_ngram_mmap_table(local_path, layer_name):
+ """Map a persisted n-gram table without copying it into anonymous RAM."""
+ meta_path = Path(local_path) / (layer_name + '.mmap.json')
+ raw_path = Path(local_path) / (layer_name + '.mmap')
+ meta = json.loads(meta_path.read_text())
+ dtype = _TAG_DTYPES[meta['dtype']]
+ rows, cols, nbytes = meta['rows'], meta['cols'], meta['nbytes']
+ storage = torch.UntypedStorage.from_file(str(raw_path), shared=True, nbytes=nbytes)
+ table = torch.tensor([], dtype=dtype)
+ table.set_(storage, 0, (rows, cols))
+ return table
+
+
+class MmapEmbedding(nn.Module):
+ """``nn.Embedding`` stand-in over a file-backed CPU table.
+
+ The table is a plain attribute, not a parameter or buffer, so a parent decoder layer's
+ ``module.to('meta')`` cannot evict it. A zero-size ``weight`` lives on CPU so modeling code
+ that gathers on ``self.ngram_embedding.weight.device`` still runs the lookup on the host.
+ """
+
+ def __init__(self, table):
+ super().__init__()
+ if table.dim() != 2:
+ raise ValueError(f"mmap embedding table must be 2D, got {tuple(table.shape)}")
+ self.num_embeddings, self.embedding_dim = table.shape
+ self._table = table
+ self.weight = nn.Parameter(
+ torch.empty(0, table.shape[1], dtype=table.dtype), requires_grad=False)
+
+ def forward(self, input):
+ return nn.functional.embedding(input, self._table)
+
+
+@contextmanager
+def _force_meta_embeddings():
+ """Construct ``nn.Embedding`` on meta so huge tables never materialize on CPU.
+
+ ``accelerate.init_empty_weights`` allocates each Parameter on CPU and then ``.to('meta')``.
+ That is fine for ordinary weights, but Flash-Next's PLE table is
+ ``nn.Embedding(~320M, 160)`` -- ~191GB of empty fp32 -- which OOMs a 64GB host before the
+ move. Creating the module with ``device=meta`` keeps construction memory-free; AirLLM later
+ replaces this module with a file-backed ``MmapEmbedding``.
+ """
+ orig = torch.nn.Embedding.__init__
+
+ def _init(self, num_embeddings, embedding_dim, *args, **kwargs):
+ kwargs = dict(kwargs)
+ kwargs.setdefault('device', torch.device('meta'))
+ orig(self, num_embeddings, embedding_dim, *args, **kwargs)
+
+ torch.nn.Embedding.__init__ = _init
+ try:
+ yield
+ finally:
+ torch.nn.Embedding.__init__ = orig
+
+
+def _wrap_forward_int64_scatter(orig_forward):
+ """Make ``tensor.scatter(..., index)`` accept int32 indices for one forward.
+
+ Transformers' Qwen4-Exp sparse-attention indexer fills ``selected_token_indices`` as
+ ``int32`` and scatters them into a bool mask. ``torch.Tensor.scatter`` requires ``int64``,
+ so generate crashes on the first sparse-attention layer. The wrap is scoped to that
+ forward so the rest of PyTorch is unchanged.
+ """
+ def forward(self, *args, **kwargs):
+ orig_scatter = torch.Tensor.scatter
+
+ def scatter(tensor, dim, index, *a, **kw):
+ if torch.is_tensor(index) and index.dtype != torch.int64:
+ index = index.to(torch.int64)
+ return orig_scatter(tensor, dim, index, *a, **kw)
+
+ torch.Tensor.scatter = scatter
+ try:
+ return orig_forward(self, *args, **kwargs)
+ finally:
+ torch.Tensor.scatter = orig_scatter
+
+ return forward
+
+
+def cpu_resident_module_names(layer_names, index_keys):
+ """Module prefixes whose weights should stay on CPU after the split.
+
+ ``cpu_resident`` is an explicit list. ``cpu_resident_marker`` (e.g. the Flash-Next PLE table)
+ is scanned out of the checkpoint so we do not have to hard-code which decoder layer owns it.
+ """
+ names = list(layer_names.get('cpu_resident', []))
+ marker = layer_names.get('cpu_resident_marker')
+ if marker:
+ found = []
+ for key in index_keys:
+ pos = key.find(marker)
+ if pos == -1:
+ continue
+ prefix = key[:pos + len(marker)]
+ if prefix not in names and prefix not in found:
+ found.append(prefix)
+ names.extend(found)
+ return names
+
+
+def layer_owner(key, layer_prefixes):
+ """Longest matching prefix wins, so a nested cpu-resident module is not swallowed by its parent."""
+ owner = None
+ for prefix in layer_prefixes:
+ if key.startswith(prefix) and (owner is None or len(prefix) > len(owner)):
+ owner = prefix
+ return owner
+
+
def layer_tensor_names(local_path, layer_name):
"""List the tensors in a layer shard without reading any tensor data."""
with safe_open(str(Path(local_path) / (layer_name + ".safetensors")), framework="pt") as f:
@@ -294,24 +587,29 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
# e.g. a multimodal model's vision tower / projector, or extra top-level norms. They get
# their own shard and are loaded once and kept resident.
layers = layers + list(layer_names.get('resident', []))
+ layers = layers + cpu_resident_module_names(layer_names, index.keys())
layers = [l + "." for l in layers]
# Drop layers that have no weights in the checkpoint. This happens for tied embeddings,
# where lm_head shares storage with embed_tokens and has no entry of its own. Without this we
# would try to save an empty shard (which fails) and never detect the split as complete.
layers = [l for l in layers if any(k.startswith(l) for k in index.keys())]
+ owned = {layer_owner(k, layers) for k in index}
+ layers = [l for l in layers if l in owned]
# Split in ascending shard order. The loop below only ever walks the shard counter forward, so
# a module whose weights sit in an earlier shard than its predecessor's would silently be saved
# incomplete. That ordering isn't guaranteed once non-sequential modules (a vision tower, extra
# norms) are in the list, so sort by the last shard each module touches. This is a stable sort,
# so plain embed -> layers -> norm -> lm_head checkpoints keep their existing order.
+ # When two modules share a last shard, the longer prefix goes first so a nested cpu-resident
+ # table is extracted before its parent decoder layer can swallow it.
def _last_shard_of(layer):
nums = [int(v.split('-')[1]) for k, v in index.items()
if k.startswith(layer) and '-' in v and len(v.split('-')) > 1]
return max(nums) if nums else -1
- layers.sort(key=_last_shard_of)
+ layers.sort(key=lambda layer: (_last_shard_of(layer), -len(layer)))
# check if splitting exists and all files are there
@@ -385,6 +683,12 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
for layer in tqdm(layers):
+ marker = (layer_names or {}).get('cpu_resident_marker')
+ if marker and marker in layer:
+ persist_ngram_mmap(layer, index, checkpoint_path, saving_path,
+ repo_id=repo_id, hf_token=hf_token)
+ continue
+
if layer in passthrough:
src = checkpoint_path / passthrough[layer]
if not os.path.exists(src):
@@ -431,9 +735,13 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
token=hf_token)
if not safetensors_format:
- state_dict.update(torch.load(to_load, map_location='cpu'))
+ loaded = torch.load(to_load, map_location='cpu')
else:
- state_dict.update(load_file(to_load, device='cpu'))
+ loaded = load_file(to_load, device='cpu')
+ marker = (layer_names or {}).get('cpu_resident_marker')
+ if marker:
+ loaded = {k: v for k, v in loaded.items() if marker not in k}
+ state_dict.update(loaded)
else:
shards = [v for k, v in index.items() if k.startswith(layer)]
@@ -445,12 +753,18 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
huggingface_hub.snapshot_download(repo_id, allow_patterns=os.path.basename(to_load),
token=hf_token)
if not safetensors_format:
- state_dict.update(torch.load(to_load, map_location='cpu'))
+ loaded = torch.load(to_load, map_location='cpu')
else:
- state_dict.update(load_file(to_load, device='cpu'))
-
- # Get layer state dict
- layer_state_dict = dict([(k, v) for k, v in state_dict.items() if k.startswith(layer)])
+ loaded = load_file(to_load, device='cpu')
+ marker = (layer_names or {}).get('cpu_resident_marker')
+ if marker:
+ loaded = {k: v for k, v in loaded.items() if marker not in k}
+ state_dict.update(loaded)
+
+ # Get layer state dict. Longest prefix wins, so e.g. a PLE n-gram table nested under a
+ # decoder layer is not written into that layer's shard (and later streamed onto the GPU).
+ layer_state_dict = dict([(k, v) for k, v in state_dict.items()
+ if layer_owner(k, layers) == layer])
layer_state_dict = compress_layer_state_dict(layer_state_dict, compression)
diff --git a/air_llm/setup.py b/air_llm/setup.py
index 3470c76..b349ed7 100644
--- a/air_llm/setup.py
+++ b/air_llm/setup.py
@@ -15,12 +15,12 @@ for _readme in (os.path.join(here, "README.md"), os.path.join(here, os.pardir, "
setuptools.setup(
name="airllm",
- version="3.2.0",
+ version="3.3.0",
author="Gavin Li",
author_email="[email protected]",
description="AirLLM runs 70B large language models on a single 4GB GPU without quantization, "
"distillation or pruning. 405B Llama 3.1 on 8GB, DeepSeek-V3 671B on ~12GB, "
- "Kimi K3 2.8T on under 4GB, Qwen3.8-27B on 3.3GB.",
+ "Kimi K3 2.8T on under 4GB, Qwen3.8-27B on 3.3GB, Qwen3.8-Flash-Next on 6GB.",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/lyogavin/airllm",
diff --git a/air_llm/tests/test_qwen38_flash_next_split.py b/air_llm/tests/test_qwen38_flash_next_split.py
new file mode 100644
index 0000000..0b46d3e
--- /dev/null
+++ b/air_llm/tests/test_qwen38_flash_next_split.py
@@ -0,0 +1,297 @@
+"""Structural tests for Qwen3.8-Flash-Next / Qwen4-Exp checkpoints.
+
+Flash-Next is ``Qwen4ExpForConditionalGeneration``: decoder nested under ``model.language_model``,
+vision at ``model.visual``, no final RMSNorm (output mix is ``hyper_connection_mixer``), packed MoE
+expert tensors, a sharded n-gram embedding under one decoder layer, and an ``mtp.*`` head that
+transformers ignores. These tests check that the splitter peels the n-gram table out of its parent
+layer, keeps the vision tower, drops MTP, and concatenates n-gram shards into a single weight.
+"""
+
+import json
+import os
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+
+import torch
+from safetensors.torch import save_file, load_file
+
+_AIRLLM_DIR = Path(__file__).resolve().parents[1] / 'airllm'
+
+if 'airllm' not in sys.modules:
+ _pkg = types.ModuleType('airllm')
+ _pkg.__path__ = [str(_AIRLLM_DIR)]
+ sys.modules['airllm'] = _pkg
+
+from airllm.persist import model_persister as _persister_mod
+from airllm.persist.safetensor_model_persister import SafetensorModelPersister
+
+_persister_mod.model_persister = SafetensorModelPersister()
+
+from airllm.utils import (
+ split_and_save_layers,
+ merge_ngram_embedding_shards,
+ load_merged_ngram_embedding,
+ open_ngram_mmap_table,
+ cpu_resident_module_names,
+ layer_owner,
+ _force_meta_embeddings,
+ _wrap_forward_int64_scatter,
+)
+
+N_LAYERS = 4
+LAYER_PREFIX = "model.language_model.layers"
+NGRAM_MODULE = f"{LAYER_PREFIX}.1.ple.ple_embedding.ngram_embedding"
+
+FLASH_LAYER_NAMES = {
+ 'embed': 'model.language_model.embed_tokens',
+ 'layer_prefix': LAYER_PREFIX,
+ 'norm': 'model.language_model.hyper_connection_mixer',
+ 'lm_head': 'lm_head',
+ 'resident': [
+ 'model.visual',
+ ],
+ 'cpu_resident_marker': 'ple.ple_embedding.ngram_embedding',
+}
+
+
+def build_fake_checkpoint(root):
+ """Layer 1 packs MoE experts plus n-gram shards in the same file as the rest of that layer."""
+ shards = {}
+
+ shard0 = {
+ "model.visual.patch_embed.proj.weight": torch.randn(8, 3, 2, 2),
+ "model.visual.blocks.0.attn.qkv.weight": torch.randn(8, 8),
+ "model.visual.merger.linear_fc1.weight": torch.randn(8, 8),
+ f"{LAYER_PREFIX}.0.linear_attn.in_proj_a.weight": torch.randn(4, 8),
+ f"{LAYER_PREFIX}.0.mlp.experts.gate_up_proj": torch.randn(4, 8, 8),
+ f"{LAYER_PREFIX}.0.mlp.experts.down_proj": torch.randn(4, 8, 4),
+ f"{LAYER_PREFIX}.0.mlp.shared_expert.gate_proj.weight": torch.randn(8, 8),
+ }
+ shards["model-00001-of-00003.safetensors"] = shard0
+
+ shard1 = {
+ "model.language_model.embed_tokens.weight": torch.randn(16, 8),
+ f"{LAYER_PREFIX}.1.linear_attn.out_proj.weight": torch.randn(8, 8),
+ f"{LAYER_PREFIX}.1.mlp.experts.gate_up_proj": torch.randn(4, 8, 8),
+ f"{LAYER_PREFIX}.1.mlp.experts.down_proj": torch.randn(4, 8, 4),
+ f"{LAYER_PREFIX}.1.ple.key_proj.weight": torch.randn(8, 8),
+ f"{NGRAM_MODULE}.shard_0.weight": torch.arange(12, dtype=torch.float32).reshape(3, 4),
+ f"{NGRAM_MODULE}.shard_1.weight": torch.arange(12, 24, dtype=torch.float32).reshape(3, 4),
+ f"{LAYER_PREFIX}.2.self_attn.q_proj.weight": torch.randn(8, 8),
+ f"{LAYER_PREFIX}.3.mlp.experts.gate_up_proj": torch.randn(4, 8, 8),
+ "model.language_model.hyper_connection_mixer.hc_norm.weight": torch.randn(8),
+ "model.language_model.hyper_connection_mixer.input_mix_weight_down.weight": torch.randn(4, 8),
+ }
+ shards["model-00002-of-00003.safetensors"] = shard1
+
+ shard2 = {
+ "lm_head.weight": torch.randn(16, 8),
+ "mtp.fc.weight": torch.randn(8, 8),
+ "mtp.layers.0.mlp.gate_proj.weight": torch.randn(8, 8),
+ "mtp.hyper_connection_mixer.hc_norm.weight": torch.randn(8),
+ }
+ shards["model-00003-of-00003.safetensors"] = shard2
+
+ weight_map = {}
+ for fname, sd in shards.items():
+ save_file(sd, str(Path(root) / fname))
+ for k in sd:
+ weight_map[k] = fname
+ with open(Path(root) / "model.safetensors.index.json", "w") as f:
+ json.dump({"metadata": {"total_size": 1}, "weight_map": weight_map}, f)
+ return shards, weight_map
+
+
+class TestNgramShardMerge(unittest.TestCase):
+ def test_concatenates_contiguous_shards_along_dim0(self):
+ prefix = NGRAM_MODULE
+ shard0 = torch.arange(6, dtype=torch.float32).reshape(2, 3)
+ shard1 = torch.arange(6, 12, dtype=torch.float32).reshape(2, 3)
+ merged = merge_ngram_embedding_shards({
+ f"{prefix}.shard_1.weight": shard1,
+ f"{prefix}.shard_0.weight": shard0,
+ f"{LAYER_PREFIX}.1.ple.key_proj.weight": torch.ones(3),
+ })
+ self.assertIn(f"{prefix}.weight", merged)
+ self.assertNotIn(f"{prefix}.shard_0.weight", merged)
+ self.assertTrue(torch.equal(merged[f"{prefix}.weight"], torch.cat([shard0, shard1], dim=0)))
+ self.assertTrue(torch.equal(merged[f"{LAYER_PREFIX}.1.ple.key_proj.weight"], torch.ones(3)))
+
+ def test_file_loader_concatenates_without_keeping_shard_keys(self):
+ tmp = tempfile.TemporaryDirectory()
+ try:
+ prefix = NGRAM_MODULE
+ shard0 = torch.arange(6, dtype=torch.float32).reshape(2, 3)
+ shard1 = torch.arange(6, 12, dtype=torch.float32).reshape(2, 3)
+ path = Path(tmp.name) / f"{prefix}.safetensors"
+ save_file({
+ f"{prefix}.shard_0.weight": shard0,
+ f"{prefix}.shard_1.weight": shard1,
+ f"{LAYER_PREFIX}.1.ple.key_proj.weight": torch.ones(3),
+ }, str(path))
+ loaded = load_merged_ngram_embedding(tmp.name, prefix)
+ self.assertIn(f"{prefix}.weight", loaded)
+ self.assertNotIn(f"{prefix}.shard_0.weight", loaded)
+ self.assertTrue(torch.equal(loaded[f"{prefix}.weight"], torch.cat([shard0, shard1], dim=0)))
+ self.assertTrue(torch.equal(loaded[f"{LAYER_PREFIX}.1.ple.key_proj.weight"], torch.ones(3)))
+ finally:
+ tmp.cleanup()
+
+ def test_rejects_gapped_shard_indices(self):
+ with self.assertRaises(ValueError):
+ merge_ngram_embedding_shards({
+ f"{NGRAM_MODULE}.shard_0.weight": torch.zeros(2, 2),
+ f"{NGRAM_MODULE}.shard_2.weight": torch.ones(2, 2),
+ })
+
+ def test_marker_discovers_the_parent_layer(self):
+ keys = [
+ f"{NGRAM_MODULE}.shard_0.weight",
+ f"{LAYER_PREFIX}.1.mlp.experts.down_proj",
+ ]
+ names = cpu_resident_module_names(FLASH_LAYER_NAMES, keys)
+ self.assertEqual(names, [NGRAM_MODULE])
+
+ def test_longest_prefix_wins(self):
+ prefixes = [f"{LAYER_PREFIX}.1.", f"{NGRAM_MODULE}."]
+ self.assertEqual(layer_owner(f"{NGRAM_MODULE}.shard_0.weight", prefixes), f"{NGRAM_MODULE}.")
+ self.assertEqual(
+ layer_owner(f"{LAYER_PREFIX}.1.mlp.experts.down_proj", prefixes),
+ f"{LAYER_PREFIX}.1.",
+ )
+
+
+class TestQwen38FlashNextSplit(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self.tmp.name)
+ self.shards, self.weight_map = build_fake_checkpoint(self.root)
+ self.out = Path(split_and_save_layers(self.root, layer_names=FLASH_LAYER_NAMES))
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def _split_file(self, module):
+ return self.out / f"{module}.safetensors"
+
+ def test_architecture_maps_to_the_qwen4_exp_subclass(self):
+ src = (_AIRLLM_DIR / 'auto_model.py').read_text()
+ self.assertIn('"Qwen4ExpForConditionalGeneration": "AirLLMQwen4Exp"', src)
+ init_src = (_AIRLLM_DIR / '__init__.py').read_text()
+ self.assertIn('AirLLMQwen4Exp', init_src)
+ self.assertTrue((_AIRLLM_DIR / 'airllm_qwen4_exp.py').exists())
+
+ def test_every_streamed_and_resident_module_is_split_out(self):
+ expected = ([FLASH_LAYER_NAMES['embed']]
+ + [f"{LAYER_PREFIX}.{i}" for i in range(N_LAYERS)]
+ + [FLASH_LAYER_NAMES['norm'], FLASH_LAYER_NAMES['lm_head']]
+ + FLASH_LAYER_NAMES['resident'])
+ for module in expected:
+ self.assertTrue(self._split_file(module).exists(), f"missing split for {module}")
+ self.assertTrue((self.out / f"{NGRAM_MODULE}.mmap").exists(), "missing n-gram mmap")
+ self.assertTrue((self.out / f"{NGRAM_MODULE}.mmap.json").exists())
+
+ def test_ngram_table_is_peeled_out_of_its_decoder_layer(self):
+ layer1 = load_file(str(self._split_file(f"{LAYER_PREFIX}.1")))
+ table = open_ngram_mmap_table(self.out, NGRAM_MODULE)
+ shard0 = load_file(str(self.root / "model-00002-of-00003.safetensors"))[
+ f"{NGRAM_MODULE}.shard_0.weight"]
+ shard1 = load_file(str(self.root / "model-00002-of-00003.safetensors"))[
+ f"{NGRAM_MODULE}.shard_1.weight"]
+ self.assertTrue(torch.equal(table, torch.cat([shard0, shard1], dim=0)))
+ self.assertFalse(any('ngram_embedding' in k for k in layer1),
+ f"n-gram table leaked into decoder layer 1: {set(layer1)}")
+ self.assertFalse(self._split_file(NGRAM_MODULE).exists(),
+ "n-gram table should be an mmap, not a safetensors shard")
+ self.assertIn(f"{LAYER_PREFIX}.1.mlp.experts.gate_up_proj", layer1)
+ self.assertIn(f"{LAYER_PREFIX}.1.ple.key_proj.weight", layer1)
+
+ def test_packed_experts_stay_on_the_decoder_layer(self):
+ layer0 = load_file(str(self._split_file(f"{LAYER_PREFIX}.0")))
+ self.assertIn(f"{LAYER_PREFIX}.0.mlp.experts.gate_up_proj", layer0)
+ self.assertIn(f"{LAYER_PREFIX}.0.mlp.experts.down_proj", layer0)
+ self.assertFalse(any(f"{LAYER_PREFIX}.0.mlp.experts.0." in k for k in layer0))
+
+ def test_output_mix_is_the_norm_slot(self):
+ sd = load_file(str(self._split_file("model.language_model.hyper_connection_mixer")))
+ self.assertIn("model.language_model.hyper_connection_mixer.hc_norm.weight", sd)
+ self.assertFalse(self._split_file("model.language_model.norm").exists())
+
+ def test_vision_tower_is_one_shard_and_excludes_decoder_weights(self):
+ sd = load_file(str(self._split_file("model.visual")))
+ self.assertTrue(sd, "vision split is empty")
+ self.assertTrue(all(k.startswith("model.visual.") for k in sd),
+ f"vision split leaked decoder tensors: {set(sd)}")
+
+ def test_mtp_is_dropped(self):
+ recovered = {}
+ for f in self.out.glob('*.safetensors'):
+ recovered.update(load_file(str(f)))
+ self.assertFalse(any(k.startswith("mtp.") for k in recovered),
+ f"MTP leaked into the split: {[k for k in recovered if k.startswith('mtp.')]}")
+ self.assertFalse(any(p.name.startswith("mtp") for p in self.out.glob('*.safetensors')))
+
+ def test_non_mtp_round_trip(self):
+ original = {}
+ for fname in self.shards:
+ original.update(load_file(str(self.root / fname)))
+ ngram_keys = [k for k in original if 'ngram_embedding.shard_' in k]
+ ngram_cat = torch.cat(
+ [original[k] for k in sorted(ngram_keys, key=lambda k: int(k.rsplit('_', 1)[-1].split('.')[0]))],
+ dim=0)
+ keep = {k: v for k, v in original.items()
+ if not k.startswith("mtp.") and k not in ngram_keys}
+
+ recovered = {}
+ for f in self.out.glob('*.safetensors'):
+ recovered.update(load_file(str(f)))
+
+ self.assertEqual(set(keep.keys()), set(recovered.keys()))
+ for k, v in keep.items():
+ self.assertEqual(v.dtype, recovered[k].dtype, f"{k} changed dtype")
+ self.assertTrue(torch.equal(v, recovered[k]), f"{k} changed value")
+ self.assertTrue(torch.equal(open_ngram_mmap_table(self.out, NGRAM_MODULE), ngram_cat))
+
+ def test_shared_shards_are_copied_not_linked(self):
+ src_inodes = {os.stat(self.root / f).st_ino
+ for f in os.listdir(self.root) if f.endswith('.safetensors')}
+ for module in ([f"{LAYER_PREFIX}.{i}" for i in range(N_LAYERS)]
+ + ['model.language_model.embed_tokens', 'model.visual', 'lm_head']):
+ dst = self._split_file(module)
+ self.assertNotIn(os.stat(dst).st_ino, src_inodes,
+ f"{module} was linked, but this layout should be copied")
+
+
+class TestForceMetaEmbeddings(unittest.TestCase):
+ """Flash-Next's PLE table is ``nn.Embedding(~320M, 160)``. Born on CPU that is ~191GB fp32."""
+
+ def test_huge_embedding_is_born_on_meta(self):
+ with _force_meta_embeddings():
+ emb = torch.nn.Embedding(320_001_536, 160)
+ self.assertEqual(emb.weight.device.type, 'meta')
+ self.assertEqual(tuple(emb.weight.shape), (320_001_536, 160))
+ # Patch must not leak: a later Embedding should use the default (CPU) device.
+ small = torch.nn.Embedding(4, 8)
+ self.assertEqual(small.weight.device.type, 'cpu')
+ self.assertEqual(tuple(small.weight.shape), (4, 8))
+
+
+class TestIndexerScatterInt64(unittest.TestCase):
+ def test_int32_scatter_indices_are_promoted(self):
+ class Dummy(torch.nn.Module):
+ def forward(self, _):
+ mask = torch.zeros(1, 5, dtype=torch.bool)
+ index = torch.tensor([[1, 2]], dtype=torch.int32)
+ return mask.scatter(-1, index, True)
+
+ Dummy.forward = _wrap_forward_int64_scatter(Dummy.forward)
+ out = Dummy()(None)
+ self.assertEqual(out.dtype, torch.bool)
+ self.assertEqual(out.tolist(), [[False, True, True, False, False]])
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=2)