AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
Support tied embeddings and route all standard archs through generic model
Yu Li8f62eeca24da
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.pyindex 411373b..5ccd7ea 100644--- a/air_llm/airllm/airllm_base.py+++ b/air_llm/airllm/airllm_base.py@@ -280,12 +280,35 @@ class AirLLMBaseModel:return namesdef _install_streaming_hooks(self):- # All modules execute in this order during a forward pass: embed -> layers -> norm -> lm_head.- for idx, (name, module) in enumerate(zip(self.layer_names, self.layers)):+ # Modules execute in this order during a forward: embed -> layers -> norm -> lm_head.+ n = len(self.layer_names)++ # Detect tied input/output embeddings. When tied, lm_head shares the embedding weight, so+ # there is no separate lm_head shard. We keep the embedding resident on the GPU (it is the+ # only copy and such models are small) and re-tie lm_head to it, then stream only the+ # decoder layers and the final norm.+ self.tie_word_embeddings = bool(getattr(self.config, "tie_word_embeddings", False))++ if self.tie_word_embeddings:+ embed_state = self.load_layer_to_cpu(self.layer_names[0])+ self.move_layer_to_device(embed_state)+ self.model.tie_weights()+ self._streamed_indices = list(range(1, n - 1)) # decoder layers + final norm+ else:+ self._streamed_indices = list(range(n))++ self._streamed_set = set(self._streamed_indices)++ for idx in self._streamed_indices:+ module = self.layers[idx]module._airllm_idx = idxmodule.register_forward_pre_hook(self._pre_hook)module.register_forward_hook(self._post_hook)+ def _next_streamed_idx(self, idx):+ nxt = idx + 1+ return nxt if nxt in self._streamed_set else None+def _pre_hook(self, module, args):idx = module._airllm_idx@@ -297,9 +320,11 @@ class AirLLMBaseModel:module._airllm_moved = self.move_layer_to_device(state_dict)- if self.prefetching and (idx + 1) < len(self.layer_names):- self._prefetch_future = self._executor.submit(self.load_layer_to_cpu, self.layer_names[idx + 1])- self._prefetched_idx = idx + 1+ if self.prefetching:+ nxt = self._next_streamed_idx(idx)+ if nxt is not None:+ self._prefetch_future = self._executor.submit(self.load_layer_to_cpu, self.layer_names[nxt])+ self._prefetched_idx = nxtdef _post_hook(self, module, args, output):if self.hf_quantizer is not None:diff --git a/air_llm/airllm/auto_model.py b/air_llm/airllm/auto_model.pyindex c740980..c6d9b1e 100644--- a/air_llm/airllm/auto_model.py+++ b/air_llm/airllm/auto_model.py@@ -10,47 +10,51 @@ if platform == "darwin":if is_on_mac_os:from airllm import AirLLMLlamaMlx+# Architectures that need a dedicated AirLLM subclass because of a non-standard module layout+# (custom remote-code models). Everything else uses the generic AirLLMBaseModel, which streams any+# standard *ForCausalLM (model.model.layers + lm_head / norm) and lets transformers own the+# forward pass, so newly released architectures work without code changes.+ARCH_OVERRIDES = {+ "ChatGLMModel": "AirLLMChatGLM",+ "ChatGLMForConditionalGeneration": "AirLLMChatGLM",+ "QWenLMHeadModel": "AirLLMQWen",+ "BaichuanForCausalLM": "AirLLMBaichuan",+ "BaiChuanForCausalLM": "AirLLMBaichuan",+ "InternLMForCausalLM": "AirLLMInternLM",+}++class AutoModel:def __init__(self):raise EnvironmentError("AutoModel is designed to be instantiated ""using the `AutoModel.from_pretrained(pretrained_model_name_or_path)` method.")+@classmethoddef get_module_class(cls, pretrained_model_name_or_path, *inputs, **kwargs):if 'hf_token' in kwargs:- print(f"using hf_token")- config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True, token=kwargs['hf_token'])+ config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True,+ token=kwargs['hf_token'])else:config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True)- if "Qwen2ForCausalLM" in config.architectures[0]:- return "airllm", "AirLLMQWen2"- elif "QWen" in config.architectures[0]:- return "airllm", "AirLLMQWen"- elif "Baichuan" in config.architectures[0]:- return "airllm", "AirLLMBaichuan"- elif "ChatGLM" in config.architectures[0]:- return "airllm", "AirLLMChatGLM"- elif "InternLM" in config.architectures[0]:- return "airllm", "AirLLMInternLM"- elif "Mistral" in config.architectures[0]:- return "airllm", "AirLLMMistral"- elif "Mixtral" in config.architectures[0]:- return "airllm", "AirLLMMixtral"- elif "Llama" in config.architectures[0]:- return "airllm", "AirLLMLlama2"- else:- print(f"unknown artichitecture: {config.architectures[0]}, try to use Llama2...")- return "airllm", "AirLLMLlama2"+ architectures = getattr(config, "architectures", None) or []+ arch = architectures[0] if architectures else ""++ cls_name = ARCH_OVERRIDES.get(arch)+ if cls_name is None:+ print(f"using generic AirLLM streaming model for architecture: {arch or 'unknown'}")+ cls_name = "AirLLMBaseModel"+ return "airllm", cls_name@classmethoddef from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):if is_on_mac_os:- return AirLLMLlamaMlx(pretrained_model_name_or_path, *inputs, ** kwargs)+ return AirLLMLlamaMlx(pretrained_model_name_or_path, *inputs, **kwargs)- module, cls = AutoModel.get_module_class(pretrained_model_name_or_path, *inputs, **kwargs)+ module, class_name = AutoModel.get_module_class(pretrained_model_name_or_path, *inputs, **kwargs)module = importlib.import_module(module)- class_ = getattr(module, cls)- return class_(pretrained_model_name_or_path, *inputs, ** kwargs)\ No newline at end of file+ class_ = getattr(module, class_name)+ return class_(pretrained_model_name_or_path, *inputs, **kwargs)diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.pyindex a2e02b5..2d3518e 100644--- a/air_llm/airllm/utils.py+++ b/air_llm/airllm/utils.py@@ -245,6 +245,11 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splittlayers = [layer_names['rotary_pos_emb']] + layerslayers = [l + "." for l in layers]+ # Drop layers that have no weights in the checkpoint. This happens for tied embeddings,+ # where lm_head shares storage with embed_tokens and has no entry of its own. Without this we+ # would try to save an empty shard (which fails) and never detect the split as complete.+ layers = [l for l in layers if any(k.startswith(l) for k in index.keys())]+# check if splitting exists and all files are therefound_layers = None