Skip to content

AirLLM 2.8T LLM inference with single 4GB GPU

Support tied embeddings and route all standard archs through generic model

Yu Li8f62eeca24da
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.py
index 411373b..5ccd7ea 100644
--- a/air_llm/airllm/airllm_base.py
+++ b/air_llm/airllm/airllm_base.py
@@ -280,12 +280,35 @@ class AirLLMBaseModel:
return names
def _install_streaming_hooks(self):
- # All modules execute in this order during a forward pass: embed -> layers -> norm -> lm_head.
- for idx, (name, module) in enumerate(zip(self.layer_names, self.layers)):
+ # Modules execute in this order during a forward: embed -> layers -> norm -> lm_head.
+ n = len(self.layer_names)
+
+ # Detect tied input/output embeddings. When tied, lm_head shares the embedding weight, so
+ # there is no separate lm_head shard. We keep the embedding resident on the GPU (it is the
+ # only copy and such models are small) and re-tie lm_head to it, then stream only the
+ # decoder layers and the final norm.
+ self.tie_word_embeddings = bool(getattr(self.config, "tie_word_embeddings", False))
+
+ if self.tie_word_embeddings:
+ embed_state = self.load_layer_to_cpu(self.layer_names[0])
+ self.move_layer_to_device(embed_state)
+ self.model.tie_weights()
+ self._streamed_indices = list(range(1, n - 1)) # decoder layers + final norm
+ else:
+ self._streamed_indices = list(range(n))
+
+ self._streamed_set = set(self._streamed_indices)
+
+ for idx in self._streamed_indices:
+ module = self.layers[idx]
module._airllm_idx = idx
module.register_forward_pre_hook(self._pre_hook)
module.register_forward_hook(self._post_hook)
+ def _next_streamed_idx(self, idx):
+ nxt = idx + 1
+ return nxt if nxt in self._streamed_set else None
+
def _pre_hook(self, module, args):
idx = module._airllm_idx
@@ -297,9 +320,11 @@ class AirLLMBaseModel:
module._airllm_moved = self.move_layer_to_device(state_dict)
- if self.prefetching and (idx + 1) < len(self.layer_names):
- self._prefetch_future = self._executor.submit(self.load_layer_to_cpu, self.layer_names[idx + 1])
- self._prefetched_idx = idx + 1
+ if self.prefetching:
+ nxt = self._next_streamed_idx(idx)
+ if nxt is not None:
+ self._prefetch_future = self._executor.submit(self.load_layer_to_cpu, self.layer_names[nxt])
+ self._prefetched_idx = nxt
def _post_hook(self, module, args, output):
if self.hf_quantizer is not None:
diff --git a/air_llm/airllm/auto_model.py b/air_llm/airllm/auto_model.py
index c740980..c6d9b1e 100644
--- a/air_llm/airllm/auto_model.py
+++ b/air_llm/airllm/auto_model.py
@@ -10,47 +10,51 @@ if platform == "darwin":
if is_on_mac_os:
from airllm import AirLLMLlamaMlx
+# Architectures that need a dedicated AirLLM subclass because of a non-standard module layout
+# (custom remote-code models). Everything else uses the generic AirLLMBaseModel, which streams any
+# standard *ForCausalLM (model.model.layers + lm_head / norm) and lets transformers own the
+# forward pass, so newly released architectures work without code changes.
+ARCH_OVERRIDES = {
+ "ChatGLMModel": "AirLLMChatGLM",
+ "ChatGLMForConditionalGeneration": "AirLLMChatGLM",
+ "QWenLMHeadModel": "AirLLMQWen",
+ "BaichuanForCausalLM": "AirLLMBaichuan",
+ "BaiChuanForCausalLM": "AirLLMBaichuan",
+ "InternLMForCausalLM": "AirLLMInternLM",
+}
+
+
class AutoModel:
def __init__(self):
raise EnvironmentError(
"AutoModel is designed to be instantiated "
"using the `AutoModel.from_pretrained(pretrained_model_name_or_path)` method."
)
+
@classmethod
def get_module_class(cls, pretrained_model_name_or_path, *inputs, **kwargs):
if 'hf_token' in kwargs:
- print(f"using hf_token")
- config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True, token=kwargs['hf_token'])
+ config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True,
+ token=kwargs['hf_token'])
else:
config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True)
- if "Qwen2ForCausalLM" in config.architectures[0]:
- return "airllm", "AirLLMQWen2"
- elif "QWen" in config.architectures[0]:
- return "airllm", "AirLLMQWen"
- elif "Baichuan" in config.architectures[0]:
- return "airllm", "AirLLMBaichuan"
- elif "ChatGLM" in config.architectures[0]:
- return "airllm", "AirLLMChatGLM"
- elif "InternLM" in config.architectures[0]:
- return "airllm", "AirLLMInternLM"
- elif "Mistral" in config.architectures[0]:
- return "airllm", "AirLLMMistral"
- elif "Mixtral" in config.architectures[0]:
- return "airllm", "AirLLMMixtral"
- elif "Llama" in config.architectures[0]:
- return "airllm", "AirLLMLlama2"
- else:
- print(f"unknown artichitecture: {config.architectures[0]}, try to use Llama2...")
- return "airllm", "AirLLMLlama2"
+ architectures = getattr(config, "architectures", None) or []
+ arch = architectures[0] if architectures else ""
+
+ cls_name = ARCH_OVERRIDES.get(arch)
+ if cls_name is None:
+ print(f"using generic AirLLM streaming model for architecture: {arch or 'unknown'}")
+ cls_name = "AirLLMBaseModel"
+ return "airllm", cls_name
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
if is_on_mac_os:
- return AirLLMLlamaMlx(pretrained_model_name_or_path, *inputs, ** kwargs)
+ return AirLLMLlamaMlx(pretrained_model_name_or_path, *inputs, **kwargs)
- module, cls = AutoModel.get_module_class(pretrained_model_name_or_path, *inputs, **kwargs)
+ module, class_name = AutoModel.get_module_class(pretrained_model_name_or_path, *inputs, **kwargs)
module = importlib.import_module(module)
- class_ = getattr(module, cls)
- return class_(pretrained_model_name_or_path, *inputs, ** kwargs)
\ No newline at end of file
+ class_ = getattr(module, class_name)
+ return class_(pretrained_model_name_or_path, *inputs, **kwargs)
diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.py
index a2e02b5..2d3518e 100644
--- a/air_llm/airllm/utils.py
+++ b/air_llm/airllm/utils.py
@@ -245,6 +245,11 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
layers = [layer_names['rotary_pos_emb']] + layers
layers = [l + "." for l in layers]
+ # Drop layers that have no weights in the checkpoint. This happens for tied embeddings,
+ # where lm_head shares storage with embed_tokens and has no entry of its own. Without this we
+ # would try to save an empty shard (which fails) and never detect the split as complete.
+ layers = [l for l in layers if any(k.startswith(l) for k in index.keys())]
+
# check if splitting exists and all files are there
found_layers = None