AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
feat: support Qwen3.8-Flash-Next on 5.95GB (3.3.0)
Yu Liee09fafc170d
diff --git a/README.md b/README.mdindex d1341b6..bc22c68 100644--- a/README.md+++ b/README.md@@ -31,6 +31,8 @@* [Bloome — build & run AI agent teams in the cloud, zero setup](https://bloome.im/app?ref=G6BYnov0&utm_medium=github&utm_source=lyogavin-airllm-ivor-202606)## Updates+[2026/08] **Qwen3.8-Flash-Next** support: Qwen's 125B MoE flagship (`Qwen4ExpForConditionalGeneration`) with a ~51B n-gram embedding runs in **5.95GB** of VRAM, measured end to end on one RTX 4090. The n-gram table is file-mapped on the host (a 64GB machine is enough); decoder layers stream. Needs a `transformers` build with in-tree `qwen4_exp` (`pip install git+https://github.com/huggingface/transformers.git` today) and ~360GB of checkpoint disk (`delete_original=True` reclaims the originals after the split).+[2026/08] **Qwen3.8-27B** support: Qwen's new dense VL (Gated DeltaNet + Gated Attention, native vision) runs in **3.33GB** of VRAM, measured end to end on one RTX 3090. Needs `transformers` 5.8+.[2026/07] **Kimi K3 (2.8T)** support: the largest open-source model runs on a single card in **3.72GB** of VRAM, measured end to end on one RTX 6000 Ada. Per-expert streaming loads only the experts a token actually routes to. K3 brings three requirements of its own: `pip install compressed-tensors flash-attn` (its model code mandates flash attention regardless of what you request), a CUDA 12 build of torch, since no prebuilt flash-attn wheel exists for CUDA 13 yet, and `transformers` 4.56.x, as its remote code does not load on 5.x.@@ -106,6 +108,7 @@ model = AutoModel.from_pretrained("Qwen/Qwen3-32B")# go bigger with the exact same one line:#model = AutoModel.from_pretrained("Qwen/Qwen3.8-27B") # 27B dense VL, 3.33GB+#model = AutoModel.from_pretrained("Qwen/Qwen3.8-Flash-Next") # 125B MoE + 51B PLE, 5.95GB#model = AutoModel.from_pretrained("Qwen/Qwen3-235B-A22B") # 235B, runs in ~3GB#model = AutoModel.from_pretrained("deepseek-ai/DeepSeek-V3") # 671B, runs in ~12GB@@ -275,7 +278,7 @@ model.tokenizer.decode(generation_output.sequences[0])AirLLM works out of the box with **virtually every popular open LLM** — just pass its Hugging Face ID to `AutoModel.from_pretrained(...)`. That covers all the major families:-**Llama** (2 / 3 / 3.1 / 3.3 / 4) · **Qwen** (1 / 2 / 2.5 / 3 / 3.5 / 3.8, including MoE, FP8, and native VL) · **DeepSeek** (V2 / V3 / R1) · **Mistral & Mixtral** · **Phi** · **Gemma** · **ChatGLM** · **Baichuan** · **InternLM** · **Yi** · **Kimi K3** — and most new models the day they're released.+**Llama** (2 / 3 / 3.1 / 3.3 / 4) · **Qwen** (1 / 2 / 2.5 / 3 / 3.5 / 3.8, including MoE, Flash-Next, FP8, and native VL) · **DeepSeek** (V2 / V3 / R1) · **Mistral & Mixtral** · **Phi** · **Gemma** · **ChatGLM** · **Baichuan** · **InternLM** · **Yi** · **Kimi K3** — and most new models the day they're released.### Tiny GPU, huge models@@ -286,6 +289,7 @@ The trick: AirLLM only ever keeps **one layer on the GPU at a time**, so the VRA| Qwen3 / Mistral / Phi (≈8B) | 8B | **~1–2 GB** || Qwen3-30B / Mixtral (MoE) | 30–47B | **~1–3 GB** || Qwen3.8-27B (dense VL) | 27B | **3.33 GB** |+| Qwen3.8-Flash-Next (MoE + PLE) | ~180B | **5.95 GB** || Qwen3-235B (MoE) | 235B | **~3 GB** || Llama 3.x 70B (full precision) | 70B | **~4 GB** || Llama 3.1 405B | 405B | **~8 GB** |diff --git a/air_llm/airllm/__init__.py b/air_llm/airllm/__init__.pyindex bfff72f..5ce56df 100644--- a/air_llm/airllm/__init__.py+++ b/air_llm/airllm/__init__.py@@ -33,6 +33,7 @@ else:("AirLLMMixtral", ".airllm_mixtral"),("AirLLMKimiK3", ".airllm_kimi_k3"),("AirLLMQwen3_5", ".airllm_qwen3_5"),+ ("AirLLMQwen4Exp", ".airllm_qwen4_exp"),):try:_mod = __import__(__name__ + _module, fromlist=[_name])diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.pyindex efd9959..8da7ab9 100644--- a/air_llm/airllm/airllm_base.py+++ b/air_llm/airllm/airllm_base.py@@ -16,7 +16,8 @@ from transformers.quantizers import AutoHfQuantizerfrom .profiler import LayeredProfilerfrom .utils import clean_memory, load_layer, layer_tensor_names, load_layer_subset, \- find_or_create_local_splitted_path+ find_or_create_local_splitted_path, load_merged_ngram_embedding, \+ open_ngram_mmap_table, MmapEmbedding, _force_meta_embeddingsfrom .persist import ModelPersistertry:@@ -198,6 +199,7 @@ class AirLLMBaseModel:self.set_layers_from_layer_names()self._load_resident_modules()+ self._load_cpu_resident_modules()self._install_streaming_hooks()# ---- customization hooks for subclasses -------------------------------------------------@@ -289,29 +291,30 @@ class AirLLMBaseModel:"trust_remote_code": self.trust_remote_code,}errors = []- for name, cls in self._auto_model_classes():- try:- with init_empty_weights(include_buffers=False):- model = cls.from_config(self.config, **kwargs)- except TypeError:- # Older Auto factories don't take attn_implementation.+ with _force_meta_embeddings():+ for name, cls in self._auto_model_classes():try:with init_empty_weights(include_buffers=False):- model = cls.from_config(self.config, trust_remote_code=self.trust_remote_code)+ model = cls.from_config(self.config, **kwargs)+ except TypeError:+ # Older Auto factories don't take attn_implementation.+ try:+ with init_empty_weights(include_buffers=False):+ model = cls.from_config(self.config, trust_remote_code=self.trust_remote_code)+ except Exception as e: # noqa: BLE001 - try the next factory+ errors.append(f"{name}: {type(e).__name__}: {e}")+ continueexcept Exception as e: # noqa: BLE001 - try the next factoryerrors.append(f"{name}: {type(e).__name__}: {e}")continue- except Exception as e: # noqa: BLE001 - try the next factory- errors.append(f"{name}: {type(e).__name__}: {e}")- continue- if not self._model_matches_layer_names(model):- errors.append(- f"{name}: built {type(model).__name__} but it is missing "- f"{self.layer_names_dict['layer_prefix']}"- )- continue- print(f"built empty {type(model).__name__} via {name} (attn={attn_implementation})")- return model+ if not self._model_matches_layer_names(model):+ errors.append(+ f"{name}: built {type(model).__name__} but it is missing "+ f"{self.layer_names_dict['layer_prefix']}"+ )+ continue+ print(f"built empty {type(model).__name__} via {name} (attn={attn_implementation})")+ return modelraise RuntimeError("Could not instantiate the model on meta from config. "f"architecture={getattr(self.config, 'architectures', None)} "@@ -586,8 +589,15 @@ class AirLLMBaseModel:def move_layer_to_device(self, state_dict):self._restore_plain_weight_modules(state_dict)state_dict = self._decompress_state_dict(state_dict)+ marker = self.layer_names_dict.get('cpu_resident_marker')moved = []for param_name in self._param_names_from_state_dict(state_dict):+ # A nested CPU table (Flash-Next PLE) must never be placed on the GPU, even if a+ # decoder-layer shard still contains it.+ if param_name in getattr(self, '_cpu_resident_params', ()):+ continue+ if marker and marker in param_name:+ continueif self.hf_quantizer is not None and self._needs_quantization(param_name):# On-the-fly-quantizing schemes (e.g. bitsandbytes) reconstruct the param from the# weight plus companion quant-state tensors carried in state_dict.@@ -665,6 +675,66 @@ class AirLLMBaseModel:continueself.move_layer_to_device(state_dict)+ def _cpu_resident_names(self):+ """Module prefixes whose weights stay on CPU for the lifetime of the process.++ Flash-Next's n-gram table is the motivating case: transformers already gathers rows onto+ the embedding's own device, so keeping ~102GB of bf16 on the host is the designed path.+ Names come from ``cpu_resident`` and from split files matching ``cpu_resident_marker``.+ """+ names = list(self.layer_names_dict.get('cpu_resident', []))+ marker = self.layer_names_dict.get('cpu_resident_marker')+ if marker:+ for path in Path(self.checkpoint_path).glob('*.mmap.json'):+ stem = path.name[:-len('.mmap.json')]+ if stem.endswith(marker) and stem not in names:+ names.append(stem)+ for path in Path(self.checkpoint_path).glob('*.safetensors'):+ stem = path.name[:-len('.safetensors')]+ if stem.endswith(marker) and stem not in names:+ names.append(stem)+ return names++ def _install_mmap_embedding(self, module_name, table):+ parent_name, _, attr = module_name.rpartition('.')+ parent = self.model.get_submodule(parent_name)+ setattr(parent, attr, MmapEmbedding(table))++ def _load_cpu_resident_modules(self):+ """Load oversized lookup tables onto CPU and never stream them to the GPU.++ Flash-Next's n-gram table is ~102GB bf16. We mmap it from disk so a 64GB host can still+ run; transformers already gathers rows on ``ngram_embedding.weight.device``.+ ``module.to('meta')`` after a decoder layer would otherwise evict a child embedding that+ lives under that layer, so we also record the parameter names and evict selectively.+ """+ self._cpu_resident_params = set()+ for name in self._cpu_resident_names():+ mmap_meta = Path(self.checkpoint_path) / f"{name}.mmap.json"+ if mmap_meta.exists():+ table = open_ngram_mmap_table(self.checkpoint_path, name)+ self._install_mmap_embedding(name, table)+ self._cpu_resident_params.add(f"{name}.weight")+ print(f"cpu-resident mmap embedding: {name} "+ f"{tuple(table.shape)} {table.dtype} (gathered from disk, not copied to RAM)")+ continue+ try:+ state_dict = load_merged_ngram_embedding(self.checkpoint_path, name)+ except FileNotFoundError:+ continue+ for param_name, value in state_dict.items():+ self._adopt_checkpoint_shape(param_name, value)+ if self._should_load_verbatim(param_name, value):+ set_module_tensor_to_device(self.model, param_name, 'cpu', value=value)+ else:+ set_module_tensor_to_device(+ self.model, param_name, 'cpu', value=value, dtype=self.running_dtype)+ self._cpu_resident_params.add(param_name)+ if self._cpu_resident_params:+ n = len(self._cpu_resident_params)+ print(f"cpu-resident modules: left {n} tensors on host RAM "+ f"(n-gram / PLE table is gathered on CPU, not streamed to the GPU).")+def _install_streaming_hooks(self):# Modules execute in this order during a forward: embed -> layers -> norm -> lm_head.n = len(self.layer_names)@@ -814,9 +884,13 @@ class AirLLMBaseModel:self._prefetched_idx = nxtdef _post_hook(self, module, args, output):- # module.to('meta') would also evict the experts, which manage their own lifetime, so with- # expert streaming we only release exactly what this hook placed.- if self.hf_quantizer is not None or getattr(self, '_expert_streaming', False):+ # module.to('meta') would also evict experts (which manage their own lifetime) and any+ # cpu-resident child such as Flash-Next's n-gram table, which lives under a decoder layer+ # but must stay materialised on the host. When any of those are in play, only release the+ # tensors this hook actually placed.+ if (self.hf_quantizer is not None+ or getattr(self, '_expert_streaming', False)+ or getattr(self, '_cpu_resident_params', None)):for param_name in getattr(module, '_airllm_moved', []):set_module_tensor_to_device(self.model, param_name, 'meta')else:diff --git a/air_llm/airllm/airllm_qwen4_exp.py b/air_llm/airllm/airllm_qwen4_exp.pynew file mode 100644index 0000000..df81c22--- /dev/null+++ b/air_llm/airllm/airllm_qwen4_exp.py@@ -0,0 +1,62 @@+from .airllm_base import AirLLMBaseModel+from .utils import _wrap_forward_int64_scatter+++def _ensure_qwen4_exp_indexer_scatter_int64():+ """transformers qwen4_exp writes int32 indexer indices; torch.scatter requires int64."""+ try:+ from transformers.models.qwen4_exp.modeling_qwen4_exp import Qwen4ExpTextQSAIndexer+ except ImportError:+ return+ if getattr(Qwen4ExpTextQSAIndexer.forward, '_airllm_int64_scatter', False):+ return+ Qwen4ExpTextQSAIndexer.forward = _wrap_forward_int64_scatter(Qwen4ExpTextQSAIndexer.forward)+ Qwen4ExpTextQSAIndexer.forward._airllm_int64_scatter = True+++class AirLLMQwen4Exp(AirLLMBaseModel):+ """Qwen3.8-Flash-Next / Qwen4-Exp (``Qwen4ExpForConditionalGeneration``).++ Flash-Next is not the 27B dense VL. It is a 48-layer hybrid MoE (512 experts, 10 routed + 1+ shared) with Gated DeltaNet, Qwen Sparse Attention, hyper-connections, and a ~51B n-gram+ embedding (PLE) injected at one decoder layer. Transformers stores that table as+ ``ngram_embedding.shard_N.weight`` and concatenates it at load time; the modeling code already+ gathers rows on whatever device the table lives on (``_no_placement_params``).++ AirLLM therefore:++ * streams the decoder under ``model.language_model`` (there is no final RMSNorm; the output+ mix is ``hyper_connection_mixer``)+ * keeps ``model.visual`` resident, same as Qwen3.8-27B+ * constructs that ``nn.Embedding`` on the meta device (accelerate would otherwise allocate+ ~191GB of empty fp32 on CPU before moving it) then peels the table out of its decoder+ layer and file-mmaps it on the host, so ~102GB of bf16 never lands in GPU VRAM or+ anonymous RAM+ * drops ``mtp.*`` (transformers ignores the Multi-Token Prediction head)+ * works around a transformers indexer bug that scatters ``int32`` indices (torch wants+ ``int64``) so sparse-attention layers can run++ Packed MoE experts are 3D tensors on a single module, not a ``ModuleList``, so Kimi-style+ per-expert module hooks do not apply. A whole MoE layer is a few GB of bf16; that is what+ streams today.++ Needs a transformers build that includes in-tree ``qwen4_exp`` (the class is not remote code).+ Optional ``fla`` / ``causal-conv1d`` speed up DeltaNet; transformers falls back to PyTorch+ when they are missing.+ """++ def set_layer_names_dict(self):+ _ensure_qwen4_exp_indexer_scatter_int64()+ self.layer_names_dict = {+ 'embed': 'model.language_model.embed_tokens',+ 'layer_prefix': 'model.language_model.layers',+ 'norm': 'model.language_model.hyper_connection_mixer',+ 'lm_head': 'lm_head',+ 'resident': [+ 'model.visual',+ ],+ # Matches ``model.language_model.layers.{i}.ple.ple_embedding.ngram_embedding`` for+ # whatever layer ``ple_layer_ids`` selected. The splitter peels those tensors out of+ # the parent decoder layer so streaming that layer cannot drag ~102GB onto the GPU.+ 'cpu_resident_marker': 'ple.ple_embedding.ngram_embedding',+ }diff --git a/air_llm/airllm/auto_model.py b/air_llm/airllm/auto_model.pyindex 2a48c01..f6608df 100644--- a/air_llm/airllm/auto_model.py+++ b/air_llm/airllm/auto_model.py@@ -23,6 +23,7 @@ ARCH_OVERRIDES = {"InternLMForCausalLM": "AirLLMInternLM","KimiK3ForConditionalGeneration": "AirLLMKimiK3","Qwen3_5ForConditionalGeneration": "AirLLMQwen3_5",+ "Qwen4ExpForConditionalGeneration": "AirLLMQwen4Exp",}diff --git a/air_llm/airllm/persist/safetensor_model_persister.py b/air_llm/airllm/persist/safetensor_model_persister.pyindex 23f5cc7..8c6caca 100644--- a/air_llm/airllm/persist/safetensor_model_persister.py+++ b/air_llm/airllm/persist/safetensor_model_persister.py@@ -19,6 +19,11 @@ class SafetensorModelPersister(ModelPersister):def model_persist_exist(self, layer_name, saving_path):+ mmap_exists = os.path.exists(str(saving_path / (layer_name + 'mmap')))+ mmap_done = os.path.exists(str(saving_path / (layer_name + 'mmap.done')))+ if mmap_exists and mmap_done:+ return True+safetensor_exists = os.path.exists(str(saving_path / (layer_name + 'safetensors')))done_marker_exists = os.path.exists(str(saving_path / (layer_name + 'safetensors.done')))diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.pyindex ad45f99..75740db 100644--- a/air_llm/airllm/utils.py+++ b/air_llm/airllm/utils.py@@ -1,6 +1,7 @@import gcimport jsonimport os+import reimport ctypesimport shutilfrom tqdm import tqdm@@ -9,6 +10,7 @@ from glob import globimport timefrom collections import OrderedDict, defaultdict+from contextlib import contextmanagerfrom typing import Dict, List, Optional, Tuple, Unionfrom sys import platform@@ -113,6 +115,297 @@ def uncompress_layer_state_dict(layer_state_dict):return layer_state_dict if uncompressed_layer_state_dict is None else uncompressed_layer_state_dict++# Qwen4-Exp / Flash-Next stores the ~51B PLE n-gram table as ``*.ngram_embedding.shard_N.weight``.+# Transformers concatenates those shards into a single ``nn.Embedding.weight`` at load time.+# On machines that cannot hold ~102GB of host RAM we instead stream the shards into a file-backed+# mmap and gather rows from disk (the modeling code already looks up on ``weight.device``).+_NGRAM_SHARD_RE = re.compile(r'^(?P<prefix>.*\.ngram_embedding)\.shard_(?P<idx>\d+)\.weight$')+_DTYPE_TAGS = {+ torch.bfloat16: 'bf16',+ torch.float16: 'fp16',+ torch.float32: 'fp32',+}+_TAG_DTYPES = {tag: dtype for dtype, tag in _DTYPE_TAGS.items()}+++def merge_ngram_embedding_shards(state_dict):+ """Concatenate ``ngram_embedding.shard_N.weight`` tensors into ``ngram_embedding.weight``.++ Keys that are not n-gram shards pass through unchanged. Shard indices must be a contiguous+ 0..N-1 range; a gap would silently drop rows from the hash table.+ """+ groups = {}+ out = {}+ for key, value in state_dict.items():+ match = _NGRAM_SHARD_RE.match(key)+ if match:+ groups.setdefault(match.group('prefix'), []).append((int(match.group('idx')), value))+ else:+ out[key] = value+ for prefix, shards in groups.items():+ shards.sort(key=lambda item: item[0])+ got = [idx for idx, _ in shards]+ expected = list(range(len(shards)))+ if got != expected:+ raise ValueError(+ f"ngram embedding shards for {prefix} are not contiguous 0..{len(shards) - 1}: {got}"+ )+ out[f'{prefix}.weight'] = torch.cat([tensor for _, tensor in shards], dim=0)+ return out+++def load_merged_ngram_embedding(local_path, layer_name):+ """Load a split n-gram file and concatenate shards without holding two full copies.++ ``load_file`` plus ``torch.cat`` would peak at ~2x the table (~200GB for Flash-Next). This+ reads one shard at a time into a preallocated host tensor.+ """+ filepath = Path(local_path) / (layer_name + ".safetensors")+ with safe_open(str(filepath), framework="pt") as handle:+ groups = {}+ rest = {}+ for key in handle.keys():+ match = _NGRAM_SHARD_RE.match(key)+ if match:+ groups.setdefault(match.group('prefix'), []).append((int(match.group('idx')), key))+ else:+ rest[key] = handle.get_tensor(key)+ for prefix, shards in groups.items():+ shards.sort(key=lambda item: item[0])+ got = [idx for idx, _ in shards]+ expected = list(range(len(shards)))+ if got != expected:+ raise ValueError(+ f"ngram embedding shards for {prefix} are not contiguous "+ f"0..{len(shards) - 1}: {got}"+ )+ shapes = [tuple(handle.get_slice(key).get_shape()) for _, key in shards]+ first = handle.get_tensor(shards[0][1])+ dest = first.new_empty((sum(shape[0] for shape in shapes),) + shapes[0][1:])+ dest[0:shapes[0][0]].copy_(first)+ del first+ offset = shapes[0][0]+ for (_, key), shape in zip(shards[1:], shapes[1:]):+ piece = handle.get_tensor(key)+ dest[offset:offset + shape[0]].copy_(piece)+ offset += shape[0]+ del piece+ rest[f'{prefix}.weight'] = dest+ return rest+++def ngram_mmap_exists(saving_path, layer_name):+ return (os.path.exists(str(saving_path / (layer_name + 'mmap')))+ and os.path.exists(str(saving_path / (layer_name + 'mmap.json')))+ and os.path.exists(str(saving_path / (layer_name + 'mmap.done'))))+++def _ngram_shard_items(layer_prefix, index):+ items = []+ for key, filename in index.items():+ if not key.startswith(layer_prefix):+ continue+ match = _NGRAM_SHARD_RE.match(key)+ if not match:+ continue+ items.append((int(match.group('idx')), key, filename))+ items.sort(key=lambda item: item[0])+ got = [idx for idx, _, _ in items]+ if got != list(range(len(items))):+ raise ValueError(+ f"ngram embedding shards for {layer_prefix} are not contiguous "+ f"0..{len(items) - 1}: {got}"+ )+ return items+++def persist_ngram_mmap(layer_prefix, index, checkpoint_path, saving_path,+ repo_id=None, hf_token=None):+ """Stream n-gram shards into a file-backed mmap without holding the full table in RAM."""+ if ngram_mmap_exists(saving_path, layer_prefix):+ return+ items = _ngram_shard_items(layer_prefix, index)+ if not items:+ return++ first_file = Path(checkpoint_path) / items[0][2]+ if not first_file.exists():+ assert repo_id is not None+ huggingface_hub.snapshot_download(repo_id, allow_patterns=os.path.basename(first_file),+ token=hf_token)++ open_handles = {}+ try:+ def _handle(filename):+ path = str(Path(checkpoint_path) / filename)+ if path not in open_handles:+ src = Path(checkpoint_path) / filename+ if not src.exists():+ assert repo_id is not None+ huggingface_hub.snapshot_download(+ repo_id, allow_patterns=os.path.basename(src), token=hf_token)+ cm = safe_open(path, framework='pt')+ open_handles[path] = (cm, cm.__enter__())+ return open_handles[path][1]++ def _tensor(filename, key):+ return _handle(filename).get_tensor(key)++ first = _tensor(items[0][2], items[0][1])+ cols = first.shape[1]+ dtype = first.dtype+ if dtype not in _DTYPE_TAGS:+ raise ValueError(f"unsupported n-gram mmap dtype {dtype}")+ rows = first.shape[0]+ shapes_rest = []+ for _, key, filename in items[1:]:+ shapes_rest.append(tuple(_handle(filename).get_slice(key).get_shape()))+ rows += shapes_rest[-1][0]++ raw_path = Path(saving_path) / (layer_prefix + 'mmap')+ tmp_path = Path(saving_path) / (layer_prefix + 'mmap.tmp')+ nbytes = rows * cols * first.element_size()+ print(f"writing n-gram mmap ({rows} x {cols} {_DTYPE_TAGS[dtype]}, "+ f"{nbytes / (1024 ** 3):.1f}GB) to {raw_path}")+ with open(tmp_path, 'wb') as out:+ out.write(first.contiguous().view(torch.uint8).numpy().tobytes())+ del first+ for (_, key, filename), shape in zip(items[1:], shapes_rest):+ piece = _tensor(filename, key)+ if tuple(piece.shape) != shape:+ raise ValueError(f"{key} shape {tuple(piece.shape)} != {shape}")+ out.write(piece.contiguous().view(torch.uint8).numpy().tobytes())+ del piece+ tmp_path.replace(raw_path)+ meta = {+ 'rows': rows,+ 'cols': cols,+ 'dtype': _DTYPE_TAGS[dtype],+ 'nbytes': nbytes,+ }+ with open(Path(saving_path) / (layer_prefix + 'mmap.json'), 'w') as f:+ json.dump(meta, f)+ (Path(saving_path) / (layer_prefix + 'mmap.done')).touch()+ print(f"saved n-gram mmap as: {raw_path}")+ finally:+ for cm, _handle_obj in open_handles.values():+ cm.__exit__(None, None, None)+++def open_ngram_mmap_table(local_path, layer_name):+ """Map a persisted n-gram table without copying it into anonymous RAM."""+ meta_path = Path(local_path) / (layer_name + '.mmap.json')+ raw_path = Path(local_path) / (layer_name + '.mmap')+ meta = json.loads(meta_path.read_text())+ dtype = _TAG_DTYPES[meta['dtype']]+ rows, cols, nbytes = meta['rows'], meta['cols'], meta['nbytes']+ storage = torch.UntypedStorage.from_file(str(raw_path), shared=True, nbytes=nbytes)+ table = torch.tensor([], dtype=dtype)+ table.set_(storage, 0, (rows, cols))+ return table+++class MmapEmbedding(nn.Module):+ """``nn.Embedding`` stand-in over a file-backed CPU table.++ The table is a plain attribute, not a parameter or buffer, so a parent decoder layer's+ ``module.to('meta')`` cannot evict it. A zero-size ``weight`` lives on CPU so modeling code+ that gathers on ``self.ngram_embedding.weight.device`` still runs the lookup on the host.+ """++ def __init__(self, table):+ super().__init__()+ if table.dim() != 2:+ raise ValueError(f"mmap embedding table must be 2D, got {tuple(table.shape)}")+ self.num_embeddings, self.embedding_dim = table.shape+ self._table = table+ self.weight = nn.Parameter(+ torch.empty(0, table.shape[1], dtype=table.dtype), requires_grad=False)++ def forward(self, input):+ return nn.functional.embedding(input, self._table)+++@contextmanager+def _force_meta_embeddings():+ """Construct ``nn.Embedding`` on meta so huge tables never materialize on CPU.++ ``accelerate.init_empty_weights`` allocates each Parameter on CPU and then ``.to('meta')``.+ That is fine for ordinary weights, but Flash-Next's PLE table is+ ``nn.Embedding(~320M, 160)`` -- ~191GB of empty fp32 -- which OOMs a 64GB host before the+ move. Creating the module with ``device=meta`` keeps construction memory-free; AirLLM later+ replaces this module with a file-backed ``MmapEmbedding``.+ """+ orig = torch.nn.Embedding.__init__++ def _init(self, num_embeddings, embedding_dim, *args, **kwargs):+ kwargs = dict(kwargs)+ kwargs.setdefault('device', torch.device('meta'))+ orig(self, num_embeddings, embedding_dim, *args, **kwargs)++ torch.nn.Embedding.__init__ = _init+ try:+ yield+ finally:+ torch.nn.Embedding.__init__ = orig+++def _wrap_forward_int64_scatter(orig_forward):+ """Make ``tensor.scatter(..., index)`` accept int32 indices for one forward.++ Transformers' Qwen4-Exp sparse-attention indexer fills ``selected_token_indices`` as+ ``int32`` and scatters them into a bool mask. ``torch.Tensor.scatter`` requires ``int64``,+ so generate crashes on the first sparse-attention layer. The wrap is scoped to that+ forward so the rest of PyTorch is unchanged.+ """+ def forward(self, *args, **kwargs):+ orig_scatter = torch.Tensor.scatter++ def scatter(tensor, dim, index, *a, **kw):+ if torch.is_tensor(index) and index.dtype != torch.int64:+ index = index.to(torch.int64)+ return orig_scatter(tensor, dim, index, *a, **kw)++ torch.Tensor.scatter = scatter+ try:+ return orig_forward(self, *args, **kwargs)+ finally:+ torch.Tensor.scatter = orig_scatter++ return forward+++def cpu_resident_module_names(layer_names, index_keys):+ """Module prefixes whose weights should stay on CPU after the split.++ ``cpu_resident`` is an explicit list. ``cpu_resident_marker`` (e.g. the Flash-Next PLE table)+ is scanned out of the checkpoint so we do not have to hard-code which decoder layer owns it.+ """+ names = list(layer_names.get('cpu_resident', []))+ marker = layer_names.get('cpu_resident_marker')+ if marker:+ found = []+ for key in index_keys:+ pos = key.find(marker)+ if pos == -1:+ continue+ prefix = key[:pos + len(marker)]+ if prefix not in names and prefix not in found:+ found.append(prefix)+ names.extend(found)+ return names+++def layer_owner(key, layer_prefixes):+ """Longest matching prefix wins, so a nested cpu-resident module is not swallowed by its parent."""+ owner = None+ for prefix in layer_prefixes:+ if key.startswith(prefix) and (owner is None or len(prefix) > len(owner)):+ owner = prefix+ return owner++def layer_tensor_names(local_path, layer_name):"""List the tensors in a layer shard without reading any tensor data."""with safe_open(str(Path(local_path) / (layer_name + ".safetensors")), framework="pt") as f:@@ -294,24 +587,29 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt# e.g. a multimodal model's vision tower / projector, or extra top-level norms. They get# their own shard and are loaded once and kept resident.layers = layers + list(layer_names.get('resident', []))+ layers = layers + cpu_resident_module_names(layer_names, index.keys())layers = [l + "." for l in layers]# Drop layers that have no weights in the checkpoint. This happens for tied embeddings,# where lm_head shares storage with embed_tokens and has no entry of its own. Without this we# would try to save an empty shard (which fails) and never detect the split as complete.layers = [l for l in layers if any(k.startswith(l) for k in index.keys())]+ owned = {layer_owner(k, layers) for k in index}+ layers = [l for l in layers if l in owned]# Split in ascending shard order. The loop below only ever walks the shard counter forward, so# a module whose weights sit in an earlier shard than its predecessor's would silently be saved# incomplete. That ordering isn't guaranteed once non-sequential modules (a vision tower, extra# norms) are in the list, so sort by the last shard each module touches. This is a stable sort,# so plain embed -> layers -> norm -> lm_head checkpoints keep their existing order.+ # When two modules share a last shard, the longer prefix goes first so a nested cpu-resident+ # table is extracted before its parent decoder layer can swallow it.def _last_shard_of(layer):nums = [int(v.split('-')[1]) for k, v in index.items()if k.startswith(layer) and '-' in v and len(v.split('-')) > 1]return max(nums) if nums else -1- layers.sort(key=_last_shard_of)+ layers.sort(key=lambda layer: (_last_shard_of(layer), -len(layer)))# check if splitting exists and all files are there@@ -385,6 +683,12 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splittfor layer in tqdm(layers):+ marker = (layer_names or {}).get('cpu_resident_marker')+ if marker and marker in layer:+ persist_ngram_mmap(layer, index, checkpoint_path, saving_path,+ repo_id=repo_id, hf_token=hf_token)+ continue+if layer in passthrough:src = checkpoint_path / passthrough[layer]if not os.path.exists(src):@@ -431,9 +735,13 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitttoken=hf_token)if not safetensors_format:- state_dict.update(torch.load(to_load, map_location='cpu'))+ loaded = torch.load(to_load, map_location='cpu')else:- state_dict.update(load_file(to_load, device='cpu'))+ loaded = load_file(to_load, device='cpu')+ marker = (layer_names or {}).get('cpu_resident_marker')+ if marker:+ loaded = {k: v for k, v in loaded.items() if marker not in k}+ state_dict.update(loaded)else:shards = [v for k, v in index.items() if k.startswith(layer)]@@ -445,12 +753,18 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitthuggingface_hub.snapshot_download(repo_id, allow_patterns=os.path.basename(to_load),token=hf_token)if not safetensors_format:- state_dict.update(torch.load(to_load, map_location='cpu'))+ loaded = torch.load(to_load, map_location='cpu')else:- state_dict.update(load_file(to_load, device='cpu'))-- # Get layer state dict- layer_state_dict = dict([(k, v) for k, v in state_dict.items() if k.startswith(layer)])+ loaded = load_file(to_load, device='cpu')+ marker = (layer_names or {}).get('cpu_resident_marker')+ if marker:+ loaded = {k: v for k, v in loaded.items() if marker not in k}+ state_dict.update(loaded)++ # Get layer state dict. Longest prefix wins, so e.g. a PLE n-gram table nested under a+ # decoder layer is not written into that layer's shard (and later streamed onto the GPU).+ layer_state_dict = dict([(k, v) for k, v in state_dict.items()+ if layer_owner(k, layers) == layer])layer_state_dict = compress_layer_state_dict(layer_state_dict, compression)diff --git a/air_llm/setup.py b/air_llm/setup.pyindex 3470c76..b349ed7 100644--- a/air_llm/setup.py+++ b/air_llm/setup.py@@ -15,12 +15,12 @@ for _readme in (os.path.join(here, "README.md"), os.path.join(here, os.pardir, "setuptools.setup(name="airllm",- version="3.2.0",+ version="3.3.0",author="Gavin Li",author_email="[email protected]",description="AirLLM runs 70B large language models on a single 4GB GPU without quantization, ""distillation or pruning. 405B Llama 3.1 on 8GB, DeepSeek-V3 671B on ~12GB, "- "Kimi K3 2.8T on under 4GB, Qwen3.8-27B on 3.3GB.",+ "Kimi K3 2.8T on under 4GB, Qwen3.8-27B on 3.3GB, Qwen3.8-Flash-Next on 6GB.",long_description=long_description,long_description_content_type="text/markdown",url="https://github.com/lyogavin/airllm",diff --git a/air_llm/tests/test_qwen38_flash_next_split.py b/air_llm/tests/test_qwen38_flash_next_split.pynew file mode 100644index 0000000..0b46d3e--- /dev/null+++ b/air_llm/tests/test_qwen38_flash_next_split.py@@ -0,0 +1,297 @@+"""Structural tests for Qwen3.8-Flash-Next / Qwen4-Exp checkpoints.++Flash-Next is ``Qwen4ExpForConditionalGeneration``: decoder nested under ``model.language_model``,+vision at ``model.visual``, no final RMSNorm (output mix is ``hyper_connection_mixer``), packed MoE+expert tensors, a sharded n-gram embedding under one decoder layer, and an ``mtp.*`` head that+transformers ignores. These tests check that the splitter peels the n-gram table out of its parent+layer, keeps the vision tower, drops MTP, and concatenates n-gram shards into a single weight.+"""++import json+import os+import sys+import tempfile+import types+import unittest+from pathlib import Path++import torch+from safetensors.torch import save_file, load_file++_AIRLLM_DIR = Path(__file__).resolve().parents[1] / 'airllm'++if 'airllm' not in sys.modules:+ _pkg = types.ModuleType('airllm')+ _pkg.__path__ = [str(_AIRLLM_DIR)]+ sys.modules['airllm'] = _pkg++from airllm.persist import model_persister as _persister_mod+from airllm.persist.safetensor_model_persister import SafetensorModelPersister++_persister_mod.model_persister = SafetensorModelPersister()++from airllm.utils import (+ split_and_save_layers,+ merge_ngram_embedding_shards,+ load_merged_ngram_embedding,+ open_ngram_mmap_table,+ cpu_resident_module_names,+ layer_owner,+ _force_meta_embeddings,+ _wrap_forward_int64_scatter,+)++N_LAYERS = 4+LAYER_PREFIX = "model.language_model.layers"+NGRAM_MODULE = f"{LAYER_PREFIX}.1.ple.ple_embedding.ngram_embedding"++FLASH_LAYER_NAMES = {+ 'embed': 'model.language_model.embed_tokens',+ 'layer_prefix': LAYER_PREFIX,+ 'norm': 'model.language_model.hyper_connection_mixer',+ 'lm_head': 'lm_head',+ 'resident': [+ 'model.visual',+ ],+ 'cpu_resident_marker': 'ple.ple_embedding.ngram_embedding',+}+++def build_fake_checkpoint(root):+ """Layer 1 packs MoE experts plus n-gram shards in the same file as the rest of that layer."""+ shards = {}++ shard0 = {+ "model.visual.patch_embed.proj.weight": torch.randn(8, 3, 2, 2),+ "model.visual.blocks.0.attn.qkv.weight": torch.randn(8, 8),+ "model.visual.merger.linear_fc1.weight": torch.randn(8, 8),+ f"{LAYER_PREFIX}.0.linear_attn.in_proj_a.weight": torch.randn(4, 8),+ f"{LAYER_PREFIX}.0.mlp.experts.gate_up_proj": torch.randn(4, 8, 8),+ f"{LAYER_PREFIX}.0.mlp.experts.down_proj": torch.randn(4, 8, 4),+ f"{LAYER_PREFIX}.0.mlp.shared_expert.gate_proj.weight": torch.randn(8, 8),+ }+ shards["model-00001-of-00003.safetensors"] = shard0++ shard1 = {+ "model.language_model.embed_tokens.weight": torch.randn(16, 8),+ f"{LAYER_PREFIX}.1.linear_attn.out_proj.weight": torch.randn(8, 8),+ f"{LAYER_PREFIX}.1.mlp.experts.gate_up_proj": torch.randn(4, 8, 8),+ f"{LAYER_PREFIX}.1.mlp.experts.down_proj": torch.randn(4, 8, 4),+ f"{LAYER_PREFIX}.1.ple.key_proj.weight": torch.randn(8, 8),+ f"{NGRAM_MODULE}.shard_0.weight": torch.arange(12, dtype=torch.float32).reshape(3, 4),+ f"{NGRAM_MODULE}.shard_1.weight": torch.arange(12, 24, dtype=torch.float32).reshape(3, 4),+ f"{LAYER_PREFIX}.2.self_attn.q_proj.weight": torch.randn(8, 8),+ f"{LAYER_PREFIX}.3.mlp.experts.gate_up_proj": torch.randn(4, 8, 8),+ "model.language_model.hyper_connection_mixer.hc_norm.weight": torch.randn(8),+ "model.language_model.hyper_connection_mixer.input_mix_weight_down.weight": torch.randn(4, 8),+ }+ shards["model-00002-of-00003.safetensors"] = shard1++ shard2 = {+ "lm_head.weight": torch.randn(16, 8),+ "mtp.fc.weight": torch.randn(8, 8),+ "mtp.layers.0.mlp.gate_proj.weight": torch.randn(8, 8),+ "mtp.hyper_connection_mixer.hc_norm.weight": torch.randn(8),+ }+ shards["model-00003-of-00003.safetensors"] = shard2++ weight_map = {}+ for fname, sd in shards.items():+ save_file(sd, str(Path(root) / fname))+ for k in sd:+ weight_map[k] = fname+ with open(Path(root) / "model.safetensors.index.json", "w") as f:+ json.dump({"metadata": {"total_size": 1}, "weight_map": weight_map}, f)+ return shards, weight_map+++class TestNgramShardMerge(unittest.TestCase):+ def test_concatenates_contiguous_shards_along_dim0(self):+ prefix = NGRAM_MODULE+ shard0 = torch.arange(6, dtype=torch.float32).reshape(2, 3)+ shard1 = torch.arange(6, 12, dtype=torch.float32).reshape(2, 3)+ merged = merge_ngram_embedding_shards({+ f"{prefix}.shard_1.weight": shard1,+ f"{prefix}.shard_0.weight": shard0,+ f"{LAYER_PREFIX}.1.ple.key_proj.weight": torch.ones(3),+ })+ self.assertIn(f"{prefix}.weight", merged)+ self.assertNotIn(f"{prefix}.shard_0.weight", merged)+ self.assertTrue(torch.equal(merged[f"{prefix}.weight"], torch.cat([shard0, shard1], dim=0)))+ self.assertTrue(torch.equal(merged[f"{LAYER_PREFIX}.1.ple.key_proj.weight"], torch.ones(3)))++ def test_file_loader_concatenates_without_keeping_shard_keys(self):+ tmp = tempfile.TemporaryDirectory()+ try:+ prefix = NGRAM_MODULE+ shard0 = torch.arange(6, dtype=torch.float32).reshape(2, 3)+ shard1 = torch.arange(6, 12, dtype=torch.float32).reshape(2, 3)+ path = Path(tmp.name) / f"{prefix}.safetensors"+ save_file({+ f"{prefix}.shard_0.weight": shard0,+ f"{prefix}.shard_1.weight": shard1,+ f"{LAYER_PREFIX}.1.ple.key_proj.weight": torch.ones(3),+ }, str(path))+ loaded = load_merged_ngram_embedding(tmp.name, prefix)+ self.assertIn(f"{prefix}.weight", loaded)+ self.assertNotIn(f"{prefix}.shard_0.weight", loaded)+ self.assertTrue(torch.equal(loaded[f"{prefix}.weight"], torch.cat([shard0, shard1], dim=0)))+ self.assertTrue(torch.equal(loaded[f"{LAYER_PREFIX}.1.ple.key_proj.weight"], torch.ones(3)))+ finally:+ tmp.cleanup()++ def test_rejects_gapped_shard_indices(self):+ with self.assertRaises(ValueError):+ merge_ngram_embedding_shards({+ f"{NGRAM_MODULE}.shard_0.weight": torch.zeros(2, 2),+ f"{NGRAM_MODULE}.shard_2.weight": torch.ones(2, 2),+ })++ def test_marker_discovers_the_parent_layer(self):+ keys = [+ f"{NGRAM_MODULE}.shard_0.weight",+ f"{LAYER_PREFIX}.1.mlp.experts.down_proj",+ ]+ names = cpu_resident_module_names(FLASH_LAYER_NAMES, keys)+ self.assertEqual(names, [NGRAM_MODULE])++ def test_longest_prefix_wins(self):+ prefixes = [f"{LAYER_PREFIX}.1.", f"{NGRAM_MODULE}."]+ self.assertEqual(layer_owner(f"{NGRAM_MODULE}.shard_0.weight", prefixes), f"{NGRAM_MODULE}.")+ self.assertEqual(+ layer_owner(f"{LAYER_PREFIX}.1.mlp.experts.down_proj", prefixes),+ f"{LAYER_PREFIX}.1.",+ )+++class TestQwen38FlashNextSplit(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.root = Path(self.tmp.name)+ self.shards, self.weight_map = build_fake_checkpoint(self.root)+ self.out = Path(split_and_save_layers(self.root, layer_names=FLASH_LAYER_NAMES))++ def tearDown(self):+ self.tmp.cleanup()++ def _split_file(self, module):+ return self.out / f"{module}.safetensors"++ def test_architecture_maps_to_the_qwen4_exp_subclass(self):+ src = (_AIRLLM_DIR / 'auto_model.py').read_text()+ self.assertIn('"Qwen4ExpForConditionalGeneration": "AirLLMQwen4Exp"', src)+ init_src = (_AIRLLM_DIR / '__init__.py').read_text()+ self.assertIn('AirLLMQwen4Exp', init_src)+ self.assertTrue((_AIRLLM_DIR / 'airllm_qwen4_exp.py').exists())++ def test_every_streamed_and_resident_module_is_split_out(self):+ expected = ([FLASH_LAYER_NAMES['embed']]+ + [f"{LAYER_PREFIX}.{i}" for i in range(N_LAYERS)]+ + [FLASH_LAYER_NAMES['norm'], FLASH_LAYER_NAMES['lm_head']]+ + FLASH_LAYER_NAMES['resident'])+ for module in expected:+ self.assertTrue(self._split_file(module).exists(), f"missing split for {module}")+ self.assertTrue((self.out / f"{NGRAM_MODULE}.mmap").exists(), "missing n-gram mmap")+ self.assertTrue((self.out / f"{NGRAM_MODULE}.mmap.json").exists())++ def test_ngram_table_is_peeled_out_of_its_decoder_layer(self):+ layer1 = load_file(str(self._split_file(f"{LAYER_PREFIX}.1")))+ table = open_ngram_mmap_table(self.out, NGRAM_MODULE)+ shard0 = load_file(str(self.root / "model-00002-of-00003.safetensors"))[+ f"{NGRAM_MODULE}.shard_0.weight"]+ shard1 = load_file(str(self.root / "model-00002-of-00003.safetensors"))[+ f"{NGRAM_MODULE}.shard_1.weight"]+ self.assertTrue(torch.equal(table, torch.cat([shard0, shard1], dim=0)))+ self.assertFalse(any('ngram_embedding' in k for k in layer1),+ f"n-gram table leaked into decoder layer 1: {set(layer1)}")+ self.assertFalse(self._split_file(NGRAM_MODULE).exists(),+ "n-gram table should be an mmap, not a safetensors shard")+ self.assertIn(f"{LAYER_PREFIX}.1.mlp.experts.gate_up_proj", layer1)+ self.assertIn(f"{LAYER_PREFIX}.1.ple.key_proj.weight", layer1)++ def test_packed_experts_stay_on_the_decoder_layer(self):+ layer0 = load_file(str(self._split_file(f"{LAYER_PREFIX}.0")))+ self.assertIn(f"{LAYER_PREFIX}.0.mlp.experts.gate_up_proj", layer0)+ self.assertIn(f"{LAYER_PREFIX}.0.mlp.experts.down_proj", layer0)+ self.assertFalse(any(f"{LAYER_PREFIX}.0.mlp.experts.0." in k for k in layer0))++ def test_output_mix_is_the_norm_slot(self):+ sd = load_file(str(self._split_file("model.language_model.hyper_connection_mixer")))+ self.assertIn("model.language_model.hyper_connection_mixer.hc_norm.weight", sd)+ self.assertFalse(self._split_file("model.language_model.norm").exists())++ def test_vision_tower_is_one_shard_and_excludes_decoder_weights(self):+ sd = load_file(str(self._split_file("model.visual")))+ self.assertTrue(sd, "vision split is empty")+ self.assertTrue(all(k.startswith("model.visual.") for k in sd),+ f"vision split leaked decoder tensors: {set(sd)}")++ def test_mtp_is_dropped(self):+ recovered = {}+ for f in self.out.glob('*.safetensors'):+ recovered.update(load_file(str(f)))+ self.assertFalse(any(k.startswith("mtp.") for k in recovered),+ f"MTP leaked into the split: {[k for k in recovered if k.startswith('mtp.')]}")+ self.assertFalse(any(p.name.startswith("mtp") for p in self.out.glob('*.safetensors')))++ def test_non_mtp_round_trip(self):+ original = {}+ for fname in self.shards:+ original.update(load_file(str(self.root / fname)))+ ngram_keys = [k for k in original if 'ngram_embedding.shard_' in k]+ ngram_cat = torch.cat(+ [original[k] for k in sorted(ngram_keys, key=lambda k: int(k.rsplit('_', 1)[-1].split('.')[0]))],+ dim=0)+ keep = {k: v for k, v in original.items()+ if not k.startswith("mtp.") and k not in ngram_keys}++ recovered = {}+ for f in self.out.glob('*.safetensors'):+ recovered.update(load_file(str(f)))++ self.assertEqual(set(keep.keys()), set(recovered.keys()))+ for k, v in keep.items():+ self.assertEqual(v.dtype, recovered[k].dtype, f"{k} changed dtype")+ self.assertTrue(torch.equal(v, recovered[k]), f"{k} changed value")+ self.assertTrue(torch.equal(open_ngram_mmap_table(self.out, NGRAM_MODULE), ngram_cat))++ def test_shared_shards_are_copied_not_linked(self):+ src_inodes = {os.stat(self.root / f).st_ino+ for f in os.listdir(self.root) if f.endswith('.safetensors')}+ for module in ([f"{LAYER_PREFIX}.{i}" for i in range(N_LAYERS)]+ + ['model.language_model.embed_tokens', 'model.visual', 'lm_head']):+ dst = self._split_file(module)+ self.assertNotIn(os.stat(dst).st_ino, src_inodes,+ f"{module} was linked, but this layout should be copied")+++class TestForceMetaEmbeddings(unittest.TestCase):+ """Flash-Next's PLE table is ``nn.Embedding(~320M, 160)``. Born on CPU that is ~191GB fp32."""++ def test_huge_embedding_is_born_on_meta(self):+ with _force_meta_embeddings():+ emb = torch.nn.Embedding(320_001_536, 160)+ self.assertEqual(emb.weight.device.type, 'meta')+ self.assertEqual(tuple(emb.weight.shape), (320_001_536, 160))+ # Patch must not leak: a later Embedding should use the default (CPU) device.+ small = torch.nn.Embedding(4, 8)+ self.assertEqual(small.weight.device.type, 'cpu')+ self.assertEqual(tuple(small.weight.shape), (4, 8))+++class TestIndexerScatterInt64(unittest.TestCase):+ def test_int32_scatter_indices_are_promoted(self):+ class Dummy(torch.nn.Module):+ def forward(self, _):+ mask = torch.zeros(1, 5, dtype=torch.bool)+ index = torch.tensor([[1, 2]], dtype=torch.int32)+ return mask.scatter(-1, index, True)++ Dummy.forward = _wrap_forward_int64_scatter(Dummy.forward)+ out = Dummy()(None)+ self.assertEqual(out.dtype, torch.bool)+ self.assertEqual(out.tolist(), [[False, True, True, False, False]])+++if __name__ == '__main__':+ unittest.main(verbosity=2)