Skip to content

AirLLM 2.8T LLM inference with single 4GB GPU

add support of chatglm

Yu Li64f3c1cd1e36
diff --git a/air_llm/README.md b/air_llm/README.md
index 21b8889..ee2baa4 100644
--- a/air_llm/README.md
+++ b/air_llm/README.md
@@ -7,6 +7,8 @@ AirLLM优化inference内存,4GB单卡GPU可以运行70B大语言模型推理
## Updates
+[2023/12/03] added support of **ChatGLM**!
+
[2023/12/02] added support for safetensors. Now support all top 10 models in open llm leaderboard.
[2023/12/01] airllm 2.0. Support compressions: **3x run time speed up!**
@@ -117,18 +119,18 @@ When initialize the model, we support the following configurations:
@12/01/23
-| Rank | Model | Supported |
-| ------------- | ------------- | ------------- |
-| 1 | TigerResearch/tigerbot-70b-chat-v2 | ✅ |
-| 2 | upstage/SOLAR-0-70b-16bit | ✅ |
-| 3 | ICBU-NPU/FashionGPT-70B-V1.1 | ✅ |
-| 4 | sequelbox/StellarBright | ✅ |
-| 5 | bhenrym14/platypus-yi-34b | ✅ |
-| 6 | MayaPH/GodziLLa2-70B | ✅ |
-| 7 | 01-ai/Yi-34B | ✅ |
-| 8 | garage-bAInd/Platypus2-70B-instruct | ✅ |
-| 9 | jondurbin/airoboros-l2-70b-2.2.1 | ✅ |
-| 10 | chargoddard/Yi-34B-Llama | ✅ |
+| Rank | Model | Supported | Model Class |
+| ------------- | ------------- | ------------- | ------------- |
+| 1 | TigerResearch/tigerbot-70b-chat-v2 | ✅ | AirLLMLlama2 |
+| 2 | upstage/SOLAR-0-70b-16bit | ✅ | AirLLMLlama2 |
+| 3 | ICBU-NPU/FashionGPT-70B-V1.1 | ✅ | AirLLMLlama2 |
+| 4 | sequelbox/StellarBright | ✅ | AirLLMLlama2 |
+| 5 | bhenrym14/platypus-yi-34b | ✅ | AirLLMLlama2 |
+| 6 | MayaPH/GodziLLa2-70B | ✅ | AirLLMLlama2 |
+| 7 | 01-ai/Yi-34B | ✅ | AirLLMLlama2 |
+| 8 | garage-bAInd/Platypus2-70B-instruct | ✅ | AirLLMLlama2 |
+| 9 | jondurbin/airoboros-l2-70b-2.2.1 | ✅ | AirLLMLlama2 |
+| 10 | chargoddard/Yi-34B-Llama | ✅ | AirLLMLlama2 |
#### [opencompass leaderboard](https://opencompass.org.cn/leaderboard-llm) top models
@@ -137,18 +139,39 @@ When initialize the model, we support the following configurations:
@12/01/23
-| Rank | Model | Supported |
-| ------------- | ------------- | ------------- |
-| 1 | GPT-4 | closed.ai😓 |
-| 2 | TigerResearch/tigerbot-70b-chat-v2 | ✅ |
-| 3 | THUDM/chatglm3-6b-base | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) |
-| 4 | Qwen/Qwen-14B | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) |
-| 5 | 01-ai/Yi-34B | ✅ |
-| 6 | ChatGPT | closed.ai😓 |
-| 7 | OrionStarAI/OrionStar-Yi-34B-Chat | ✅ |
-| 8 | Qwen/Qwen-14B-Chat | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) |
-| 9 | Duxiaoman-DI/XuanYuan-70B | ✅ |
-| 10 | internlm/internlm-20b | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) |
+| Rank | Model | Supported | Model Class |
+| ------------- | ------------- | ------------- | ------------- |
+| 1 | GPT-4 | closed.ai😓 | N/A |
+| 2 | TigerResearch/tigerbot-70b-chat-v2 | ✅ | AirLLMLlama2 |
+| 3 | THUDM/chatglm3-6b-base | ✅ | AirLLMChatGLM |
+| 4 | Qwen/Qwen-14B | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) | AirLLMLlama2 |
+| 5 | 01-ai/Yi-34B | ✅ | AirLLMLlama2 |
+| 6 | ChatGPT | closed.ai😓 | N/A |
+| 7 | OrionStarAI/OrionStar-Yi-34B-Chat | ✅ | AirLLMLlama2 |
+| 8 | Qwen/Qwen-14B-Chat | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) | AirLLMLlama2 |
+| 9 | Duxiaoman-DI/XuanYuan-70B | ✅ | AirLLMLlama2 |
+| 10 | internlm/internlm-20b | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) | AirLLMLlama2 |
+
+#### example of other models:
+
+```python
+from airllm import AirLLMChatGLM
+MAX_LENGTH = 128
+model = AirLLMChatGLM("THUDM/chatglm3-6b-base")
+input_text = ['What is the capital of China?',]
+input_tokens = model.tokenizer(input_text,
+ return_tensors="pt",
+ return_attention_mask=False,
+ truncation=True,
+ max_length=MAX_LENGTH,
+ padding=True)
+generation_output = model.generate(
+ input_tokens['input_ids'].cuda(),
+ max_new_tokens=5,
+ use_cache=False,
+ return_dict_in_generate=True)
+model.tokenizer.decode(generation_output.sequences[0])
+```
## Acknowledgement
diff --git a/air_llm/airllm/__init__.py b/air_llm/airllm/__init__.py
index e92a4b6..ab19ca9 100644
--- a/air_llm/airllm/__init__.py
+++ b/air_llm/airllm/__init__.py
@@ -1,3 +1,4 @@
from .airllm import AirLLMLlama2
+from .airllm_chatglm import AirLLMChatGLM
from .utils import split_and_save_layers
from .utils import NotEnoughSpaceException
\ No newline at end of file
diff --git a/air_llm/airllm/airllm_chatglm.py b/air_llm/airllm/airllm_chatglm.py
new file mode 100644
index 0000000..d44c11d
--- /dev/null
+++ b/air_llm/airllm/airllm_chatglm.py
@@ -0,0 +1,403 @@
+import gc
+import json
+import os
+from typing import List, Optional, Tuple, Union
+import ctypes
+import shutil
+from tqdm import tqdm
+from pathlib import Path
+from glob import glob
+import time
+
+import torch
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, AutoModel, GenerationMixin, LlamaForCausalLM, GenerationConfig
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from accelerate import init_empty_weights
+from accelerate.utils.modeling import set_module_tensor_to_device
+from safetensors.torch import load_file, save_file
+from optimum.bettertransformer import BetterTransformer
+import huggingface_hub
+
+from .utils import save_quant_state_to_dict, NotEnoughSpaceException, clean_memory, uncompress_layer_state_dict, load_layer, \
+ check_space, compress_layer_state_dict, split_and_save_layers, find_or_create_local_splitted_path
+
+try:
+ import bitsandbytes as bnb
+
+ bitsandbytes_installed = True
+ print('>>>> bitsandbytes installed')
+except ImportError:
+ bitsandbytes_installed = False
+
+total_disk_loading_time = None
+total_gpu_loading_time = None
+total_compression_overhead_time = None
+
+
+
+class AirLLMChatGLM(GenerationMixin):
+ def __init__(self, model_local_path_or_repo_id, device="cuda:0", dtype=torch.float16, max_seq_len=512,
+ layer_shards_saving_path=None, profiling_mode=False, compression=None):
+ """
+ Sharded version of LlamaForCausalLM : the model is splitted into layer shards to reduce GPU memory usage.
+ During the forward pass, the inputs are processed layer by layer, and the GPU memory is freed after each layer.
+ To avoid loading the layers multiple times, we could save all the intermediate activations in RAM.
+
+ Parameters
+ ----------
+ model_local_path_or_repo_id : str or Path
+ path to the local model checkpoint or huggingface repo id
+ device : str, optional
+ device, by default "cuda:0"
+ dtype : torch.dtype, optional
+ dtype, by default torch.float16
+ max_seq_len : int, optional
+ max seq lenght, by default 512
+ layer_shards_saving_path : str, optional
+ optional path to save layered shards model file, by default just save to the local cache of model, subdir named splitted_model will be saved
+ profiling_mode : book, optional
+ if to profile the model loading time, default to False
+ compression: str, optinal
+ setting to '4bit' or '8bit' to enable compression from 16 bits to 4 bits/8 bits which speeed up 4x or 2x inference time with a tiny accuracy loss.
+ """
+
+
+ self.profiling_mode = profiling_mode
+
+ if compression is not None:
+ if not bitsandbytes_installed:
+ raise ImportError('WARNING: bitsandbytes not found. Compression needs bitsandbytes. To use compression, please install bitsandbytes: `pip install bitsandbytes`')
+
+
+ self.compression = compression
+
+ # Save parameters
+ self.layer_names_dict = {'embed': 'transformer.embedding.word_embeddings',
+ 'layer_prefix': 'transformer.encoder.layers',
+ 'norm': 'transformer.encoder.final_layernorm',
+ 'lm_head': 'transformer.output_layer',
+ 'rotary_pos_emb': 'transformer.rotary_pos_emb'}
+ self.model_local_path, self.checkpoint_path = find_or_create_local_splitted_path(model_local_path_or_repo_id,
+ layer_shards_saving_path,
+ compression=compression,
+ layer_names=self.layer_names_dict)
+ self.running_device = device
+ self.device = torch.device(self.running_device)
+ self.running_dtype = dtype
+ self.dtype = self.running_dtype
+
+ # Create model
+ self.config = AutoConfig.from_pretrained(self.model_local_path, trust_remote_code=True)
+ self.generation_config = GenerationConfig()#GenerationConfig.from_pretrained(self.model_local_path)
+ #print(f"using generation_config: {self.generation_config}")
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_local_path, trust_remote_code=True)
+ #self.tokenizer.pad_token = self.tokenizer.eos_token
+ #self.tokenizer.padding_side = "right"
+ self.init_model()
+ self.layer_names = [self.layer_names_dict['embed']] + [f'{self.layer_names_dict["layer_prefix"]}.{i}' for i in range(len(self.model.transformer.encoder.layers))] + \
+ [self.layer_names_dict['norm'], self.layer_names_dict['lm_head']]
+
+ self.max_seq_len = max_seq_len
+
+ self.main_input_name = "input_ids"
+
+ def init_model(self):
+
+ # Load meta model (no memory used)
+ with init_empty_weights():
+ self.model = AutoModelForCausalLM.from_config(self.config, trust_remote_code=True)
+ self.model.eval()
+ #self.model = BetterTransformer.transform(self.model) # enable flash attention
+ self.model.tie_weights()
+
+ self.layers = [self.model.transformer.embedding] + list(self.model.transformer.encoder.layers) + \
+ [self.model.transformer.encoder.final_layernorm, self.model.transformer.output_layer]
+
+ # Move buffers to device (not that much GPU memory used)
+ for buffer_name, buffer in self.model.named_buffers():
+ set_module_tensor_to_device(self.model, buffer_name, self.running_device, value=buffer,
+ dtype=self.running_dtype)
+
+ # for glm keep rotary_pos_emb in gpu
+ self.load_rotary_pos_emb_to_device()
+
+ def load_rotary_pos_emb_to_device(self):
+ state_dict = load_layer(self.checkpoint_path, 'transformer.rotary_pos_emb')
+ self.move_layer_to_device(state_dict)
+
+ def load_layer_to_cpu(self, layer_name, profiling=False):
+
+ t = time.process_time()
+ load_layer_output = load_layer(self.checkpoint_path, layer_name, profiling)
+ elapsed_time = time.process_time() - t
+
+ if profiling:
+ state_dict, compression_time = load_layer_output
+ disk_loading_time = elapsed_time - compression_time
+ return state_dict, disk_loading_time, compression_time
+ else:
+ state_dict = load_layer_output
+
+ return state_dict
+
+ def move_layer_to_device(self, state_dict):
+ for param_name, param in state_dict.items():
+ assert param.dtype != torch.int8, "int8 not supported (need to add fp16_statistics)"
+ set_module_tensor_to_device(self.model, param_name, self.running_device, value=param,
+ dtype=self.running_dtype)
+
+ # make GenerationMixin happy
+ def can_generate(self):
+ return True
+
+ def prepare_inputs_for_generation(
+ self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+ ):
+ if past_key_values is not None:
+ past_length = past_key_values[0][0].shape[2]
+
+ # Some generation methods already pass only the last input ID
+ if input_ids.shape[1] > past_length:
+ remove_prefix_length = past_length
+ else:
+ # Default to old behavior: keep only final ID
+ remove_prefix_length = input_ids.shape[1] - 1
+
+ input_ids = input_ids[:, remove_prefix_length:]
+
+ position_ids = kwargs.get("position_ids", None)
+ if attention_mask is not None and position_ids is None:
+ # create position_ids on the fly for batch generation
+ position_ids = attention_mask.long().cumsum(-1) - 1
+ position_ids.masked_fill_(attention_mask == 0, 1)
+ if past_key_values:
+ position_ids = position_ids[:, -input_ids.shape[1]:]
+
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+ if inputs_embeds is not None and past_key_values is None:
+ model_inputs = {"inputs_embeds": inputs_embeds}
+ else:
+ model_inputs = {"input_ids": input_ids}
+
+ model_inputs.update(
+ {
+ "position_ids": position_ids,
+ "past_key_values": past_key_values,
+ "use_cache": kwargs.get("use_cache"),
+ "attention_mask": attention_mask,
+ }
+ )
+ return model_inputs
+
+ def __call__(self, *args, **kwargs):
+ return self.forward(*args, **kwargs)
+
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
+ print(f"input_ids shape: {input_ids.shape}")
+
+ global total_disk_loading_time, total_gpu_loading_time, total_compression_overhead_time
+
+ if self.profiling_mode:
+ total_disk_loading_time = []
+ total_gpu_loading_time = []
+ total_compression_overhead_time = []
+ forward_start = time.process_time()
+
+ # Reboot the model to make sure buffers are loaded and memory is clean
+ del self.model
+ clean_memory()
+ self.init_model()
+
+ batch = [input_ids_unit.to(self.running_device).unsqueeze(0) for input_ids_unit in input_ids]
+ n_seq = len(batch[0])
+ print(f"batch[0] shape:{batch[0].shape}")
+ batch_eos = [(input_ids_unit != self.tokenizer.pad_token_id).sum(0) - 1 for input_ids_unit in input_ids]
+
+ # Create attention mask for the largest input, and position ids to use KV cache
+ attention_mask = torch.ones(self.max_seq_len, self.max_seq_len)
+ attention_mask = attention_mask.triu(diagonal=1)[None, None, ...] == 0
+ attention_mask = attention_mask.to(self.running_device)
+ position_ids = torch.arange(self.max_seq_len, dtype=torch.long, device=self.running_device)[None, :]
+
+ kv_cache_list = [] if use_cache else None
+ if use_cache:
+ for x in self.layers:
+ kv_cache_list.append(([], []))
+ all_hidden_states = [] * len(self.layers) if output_hidden_states else None
+ all_self_attns = None
+
+ with torch.inference_mode():
+
+ for i, (layer_name, layer) in tqdm(enumerate(zip(self.layer_names, self.layers)), desc=self.running_device,
+ total=len(self.layers)):
+ load_layer_to_cpu_output = self.load_layer_to_cpu(layer_name, self.profiling_mode)
+ # profile
+ if self.profiling_mode:
+ state_dict, disk_loading_time, compression_time = load_layer_to_cpu_output
+ total_disk_loading_time.append(disk_loading_time)
+ total_compression_overhead_time.append(compression_time)
+ else:
+ state_dict = load_layer_to_cpu_output
+
+ t = time.process_time()
+ self.move_layer_to_device(state_dict)
+ elapsed_time = time.process_time() - t
+ # profile
+ if self.profiling_mode:
+ total_gpu_loading_time.append(elapsed_time)
+
+ # Run layer
+
+ for j, seq in enumerate(batch):
+ #print(f"{j}th in batch shape: {seq.shape}")
+
+ if layer_name == self.layer_names_dict['embed']:
+ batch[j] = layer(seq)
+ elif layer_name == self.layer_names_dict['norm']:
+ #batch[j] = layer(seq[torch.arange(n_seq), batch_eos[j]][:, None])
+ batch[j] = layer(seq)
+
+ #if output_attentions:
+ # all_hidden_states[i].append(batch[j])
+ elif layer_name == self.layer_names_dict['lm_head']:
+ layer_res = layer(seq)
+ batch[j] = layer_res.transpose(0, 1).contiguous().float()
+ else:
+
+ #if output_attentions:
+ # all_hidden_states[i].append(new_seq)
+
+ if past_key_values is not None:
+ #print(f"len past_key_values: {len(past_key_values)}, past_key_values[0][0] shape:{past_key_values[0][0].shape}")
+ # join past kv
+ k_cache, v_cache = past_key_values[i - 1]
+ len_p = past_key_values[0][0].shape[0]
+ len_s = seq.shape[0]
+
+ pos = position_ids[:, len_p:len_p + len_s]
+
+ # Rotary positional embeddings
+ rotary_pos_emb = self.model.transformer.rotary_pos_emb(self.config.seq_length)
+ rotary_pos_emb = rotary_pos_emb[None, : len_s]
+ rotary_pos_emb = rotary_pos_emb.transpose(0, 1).contiguous()
+
+
+ attn = attention_mask[:, :, -len_s:, -len_p - len_s:]
+ kv_cache = (k_cache,
+ v_cache,
+ )
+
+ layer_outputs = layer(seq,
+ use_cache=True,
+ #output_attentions=output_attentions,
+ kv_cache=kv_cache,
+ #position_ids=pos,
+ rotary_pos_emb=rotary_pos_emb,
+ attention_mask=None #attn
+ )
+ new_seq = layer_outputs[0]
+
+ #if output_attentions:
+ # all_self_attns[i].append(layer_outputs[1])
+
+ if use_cache:
+ (k_cache, v_cache) = layer_outputs[1]
+ kv_cache_list[i][0].append(k_cache)
+ kv_cache_list[i][1].append(v_cache)
+
+
+ else:
+ # Note: glm shape is [s b h]
+ len_seq = seq.shape[0]
+
+ # Rotary positional embeddings
+ rotary_pos_emb = self.model.transformer.rotary_pos_emb(self.config.seq_length)
+ rotary_pos_emb = rotary_pos_emb[position_ids[:, :len_seq]]
+ rotary_pos_emb = rotary_pos_emb.transpose(0, 1).contiguous()
+
+ if not use_cache:
+ new_seq = layer(seq,
+ rotary_pos_emb=rotary_pos_emb,
+ attention_mask=None#attention_mask[:, :, -len_seq:, -len_seq:]
+ )[0]
+ else:
+ new_seq, (k_cache, v_cache) = layer(seq,
+ use_cache=True,
+ rotary_pos_emb=rotary_pos_emb,
+ attention_mask=None #attention_mask[:, :, -len_seq:,-len_seq:]
+ )
+ kv_cache_list[i][0].append(k_cache)
+ kv_cache_list[i][1].append(v_cache)
+
+ # print(f"k_cache size: {k_cache.shape}")
+ # print(f"k_cache sizes: {[len(x[1]) for x in kv_cache_list]}")
+
+ batch[j] = new_seq
+
+ if output_hidden_states:
+ all_hidden_states += (torch.cat(batch, 0),)
+
+ # Remove previous layer from memory (including buffers)
+ layer.to("meta")
+ clean_memory() # proposed by CPMP
+
+ logits = torch.cat(batch, 0)
+ if use_cache:
+ kv_cache_list = kv_cache_list[1:-2]
+ for i in range(len(kv_cache_list)):
+ # print(f"{i} - {kv_cache_list[i][0].shape}")
+ kv_cache_list[i] = (torch.cat(kv_cache_list[i][0], 0), torch.cat(kv_cache_list[i][1], 0))
+ #print(f"returning kvcache size: {kv_cache_list[0][0].shape}")
+
+ #if output_attentions:
+ # all_self_attns = all_self_attns[0:-2]
+ # for i in range(len(all_self_attns)):
+ # all_self_attns[i] = torch.cat(all_self_attns[i], 0)
+
+ if output_hidden_states:
+ all_hidden_states = all_hidden_states[0:-2]
+ for i in range(len(all_hidden_states)):
+ all_hidden_states[i] = torch.cat(all_hidden_states[i], 0)
+
+ if not return_dict:
+ return tuple(v for v in [logits,
+ tuple(kv_cache_list) if kv_cache_list is not None else None,
+ tuple(all_hidden_states) if all_hidden_states is not None else None,
+ tuple(all_self_attns) if all_self_attns is not None else None] if v is not None)
+ if self.profiling_mode:
+
+ forward_elapsed_time = time.process_time() - forward_start
+
+ if self.compression:
+ print(f"total disk loading time: {sum(total_disk_loading_time):.04f}")
+ print(f"total gpu loading time: {sum(total_gpu_loading_time):.04f}")
+ print(f"total compression overhead time: {sum(total_compression_overhead_time):.04f}")
+ else:
+ # loading is async/lazy, so can't really distinguish them...
+ print(f"total disk+gpu loading time: {sum(total_disk_loading_time) + sum(total_gpu_loading_time):.04f}")
+ print(f"total infer time(including all above plus gpu compute): {forward_elapsed_time:.04f}")
+
+ total_disk_loading_time = []
+ total_gpu_loading_time = []
+ total_compression_overhead_time = []
+
+
+ return CausalLMOutputWithPast(
+ loss=None,
+ logits=logits,
+ past_key_values=tuple(kv_cache_list) if kv_cache_list is not None else None,
+ hidden_states=tuple(all_hidden_states) if all_hidden_states is not None else None,
+ attentions=tuple(all_self_attns) if all_hidden_states is not None else None,
+ )
\ No newline at end of file
diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.py
index 705b7e6..38592a3 100644
--- a/air_llm/airllm/utils.py
+++ b/air_llm/airllm/utils.py
@@ -155,7 +155,8 @@ def compress_layer_state_dict(layer_state_dict, compression=None):
return compressed_layer_state_dict if compressed_layer_state_dict is not None else layer_state_dict
-def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitted_model_dir_name='splitted_model', compression=None):
+def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitted_model_dir_name='splitted_model',
+ compression=None, layer_names=None):
"""
Save the all layers of a model sharded checkpoint using safetensors.
"""
@@ -190,8 +191,19 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
with open(checkpoint_path / 'model.safetensors.index.json', 'rb') as f:
index = json.load(f)['weight_map']
- n_layers = len(set([int(k.split('.')[2]) for k in index.keys() if 'model.layers' in k]))
- layers = ['model.embed_tokens.'] + [f'model.layers.{i}.' for i in range(n_layers)] + ['model.norm.', 'lm_head.']
+ if layer_names is None:
+ n_layers = len(set([int(k.split('.')[2]) for k in index.keys() if 'model.layers' in k]))
+ else:
+ n_layers = len(set([int(k[len(layer_names['layer_prefix']):].split('.')[1]) for k in index.keys() if layer_names['layer_prefix'] in k]))
+
+ if layer_names is None:
+ layers = ['model.embed_tokens.'] + [f'model.layers.{i}.' for i in range(n_layers)] + ['model.norm.', 'lm_head.']
+ else:
+ layers = [layer_names['embed']] + [f'{layer_names["layer_prefix"]}.{i}' for i in range(n_layers)] + [layer_names['norm'], layer_names['lm_head']]
+
+ if 'rotary_pos_emb' in layer_names:
+ layers = [layer_names['rotary_pos_emb']] + layers
+ layers = [l + "." for l in layers]
shard = 0
n_shards = len(set(index.values()))
state_dict = {}
@@ -235,7 +247,8 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
return str(saving_path)
-def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards_saving_path=None, compression=None):
+def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards_saving_path=None, compression=None,
+ layer_names=None):
"""
find the model's local cache path, download the cache if not exists, then split and save the model.
@@ -260,7 +273,8 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards
if os.path.exists(model_local_path_or_repo_id):
if os.path.exists(Path(model_local_path_or_repo_id) / 'pytorch_model.bin.index.json') or \
os.path.exists(Path(model_local_path_or_repo_id) / 'model.safetensors.index.json'):
- return Path(model_local_path_or_repo_id), split_and_save_layers(model_local_path_or_repo_id, layer_shards_saving_path, compression=compression)
+ return Path(model_local_path_or_repo_id), split_and_save_layers(model_local_path_or_repo_id, layer_shards_saving_path,
+ compression=compression, layer_names=layer_names)
else:
print(
f"Found local directory in {model_local_path_or_repo_id}, but didn't find downloaded model. Try using {model_local_path_or_repo_id} as a HF repo...")
@@ -272,5 +286,6 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards
f"{hf_cache_path}/pytorch_model.bin.index.json or {hf_cache_path}/model.safetensors.index.json should exists."
# if splitted_model subdir exists under cache use it, otherwise split and save
- return Path(hf_cache_path), split_and_save_layers(hf_cache_path, layer_shards_saving_path, compression=compression)
+ return Path(hf_cache_path), split_and_save_layers(hf_cache_path, layer_shards_saving_path,
+ compression=compression, layer_names=layer_names)
diff --git a/air_llm/setup.py b/air_llm/setup.py
index f89cc94..b7250df 100644
--- a/air_llm/setup.py
+++ b/air_llm/setup.py
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
setuptools.setup(
name="airllm",
- version="2.1.1",
+ version="2.2.0",
author="Gavin Li",
author_email="[email protected]",
description="AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning.",