AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
add support of chatglm
Yu Li64f3c1cd1e36
diff --git a/air_llm/README.md b/air_llm/README.mdindex 21b8889..ee2baa4 100644--- a/air_llm/README.md+++ b/air_llm/README.md@@ -7,6 +7,8 @@ AirLLM优化inference内存,4GB单卡GPU可以运行70B大语言模型推理## Updates+[2023/12/03] added support of **ChatGLM**!+[2023/12/02] added support for safetensors. Now support all top 10 models in open llm leaderboard.[2023/12/01] airllm 2.0. Support compressions: **3x run time speed up!**@@ -117,18 +119,18 @@ When initialize the model, we support the following configurations:@12/01/23-| Rank | Model | Supported |-| ------------- | ------------- | ------------- |-| 1 | TigerResearch/tigerbot-70b-chat-v2 | ✅ |-| 2 | upstage/SOLAR-0-70b-16bit | ✅ |-| 3 | ICBU-NPU/FashionGPT-70B-V1.1 | ✅ |-| 4 | sequelbox/StellarBright | ✅ |-| 5 | bhenrym14/platypus-yi-34b | ✅ |-| 6 | MayaPH/GodziLLa2-70B | ✅ |-| 7 | 01-ai/Yi-34B | ✅ |-| 8 | garage-bAInd/Platypus2-70B-instruct | ✅ |-| 9 | jondurbin/airoboros-l2-70b-2.2.1 | ✅ |-| 10 | chargoddard/Yi-34B-Llama | ✅ |+| Rank | Model | Supported | Model Class |+| ------------- | ------------- | ------------- | ------------- |+| 1 | TigerResearch/tigerbot-70b-chat-v2 | ✅ | AirLLMLlama2 |+| 2 | upstage/SOLAR-0-70b-16bit | ✅ | AirLLMLlama2 |+| 3 | ICBU-NPU/FashionGPT-70B-V1.1 | ✅ | AirLLMLlama2 |+| 4 | sequelbox/StellarBright | ✅ | AirLLMLlama2 |+| 5 | bhenrym14/platypus-yi-34b | ✅ | AirLLMLlama2 |+| 6 | MayaPH/GodziLLa2-70B | ✅ | AirLLMLlama2 |+| 7 | 01-ai/Yi-34B | ✅ | AirLLMLlama2 |+| 8 | garage-bAInd/Platypus2-70B-instruct | ✅ | AirLLMLlama2 |+| 9 | jondurbin/airoboros-l2-70b-2.2.1 | ✅ | AirLLMLlama2 |+| 10 | chargoddard/Yi-34B-Llama | ✅ | AirLLMLlama2 |#### [opencompass leaderboard](https://opencompass.org.cn/leaderboard-llm) top models@@ -137,18 +139,39 @@ When initialize the model, we support the following configurations:@12/01/23-| Rank | Model | Supported |-| ------------- | ------------- | ------------- |-| 1 | GPT-4 | closed.ai😓 |-| 2 | TigerResearch/tigerbot-70b-chat-v2 | ✅ |-| 3 | THUDM/chatglm3-6b-base | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) |-| 4 | Qwen/Qwen-14B | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) |-| 5 | 01-ai/Yi-34B | ✅ |-| 6 | ChatGPT | closed.ai😓 |-| 7 | OrionStarAI/OrionStar-Yi-34B-Chat | ✅ |-| 8 | Qwen/Qwen-14B-Chat | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) |-| 9 | Duxiaoman-DI/XuanYuan-70B | ✅ |-| 10 | internlm/internlm-20b | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) |+| Rank | Model | Supported | Model Class |+| ------------- | ------------- | ------------- | ------------- |+| 1 | GPT-4 | closed.ai😓 | N/A |+| 2 | TigerResearch/tigerbot-70b-chat-v2 | ✅ | AirLLMLlama2 |+| 3 | THUDM/chatglm3-6b-base | ✅ | AirLLMChatGLM |+| 4 | Qwen/Qwen-14B | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) | AirLLMLlama2 |+| 5 | 01-ai/Yi-34B | ✅ | AirLLMLlama2 |+| 6 | ChatGPT | closed.ai😓 | N/A |+| 7 | OrionStarAI/OrionStar-Yi-34B-Chat | ✅ | AirLLMLlama2 |+| 8 | Qwen/Qwen-14B-Chat | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) | AirLLMLlama2 |+| 9 | Duxiaoman-DI/XuanYuan-70B | ✅ | AirLLMLlama2 |+| 10 | internlm/internlm-20b | ⏰(adding, [to accelerate😀](https://bmc.link/lyogavinQ)) | AirLLMLlama2 |++#### example of other models:++```python+from airllm import AirLLMChatGLM+MAX_LENGTH = 128+model = AirLLMChatGLM("THUDM/chatglm3-6b-base")+input_text = ['What is the capital of China?',]+input_tokens = model.tokenizer(input_text,+ return_tensors="pt",+ return_attention_mask=False,+ truncation=True,+ max_length=MAX_LENGTH,+ padding=True)+generation_output = model.generate(+ input_tokens['input_ids'].cuda(),+ max_new_tokens=5,+ use_cache=False,+ return_dict_in_generate=True)+model.tokenizer.decode(generation_output.sequences[0])+```## Acknowledgementdiff --git a/air_llm/airllm/__init__.py b/air_llm/airllm/__init__.pyindex e92a4b6..ab19ca9 100644--- a/air_llm/airllm/__init__.py+++ b/air_llm/airllm/__init__.py@@ -1,3 +1,4 @@from .airllm import AirLLMLlama2+from .airllm_chatglm import AirLLMChatGLMfrom .utils import split_and_save_layersfrom .utils import NotEnoughSpaceException\ No newline at end of filediff --git a/air_llm/airllm/airllm_chatglm.py b/air_llm/airllm/airllm_chatglm.pynew file mode 100644index 0000000..d44c11d--- /dev/null+++ b/air_llm/airllm/airllm_chatglm.py@@ -0,0 +1,403 @@+import gc+import json+import os+from typing import List, Optional, Tuple, Union+import ctypes+import shutil+from tqdm import tqdm+from pathlib import Path+from glob import glob+import time++import torch+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, AutoModel, GenerationMixin, LlamaForCausalLM, GenerationConfig+from transformers.modeling_outputs import CausalLMOutputWithPast+from accelerate import init_empty_weights+from accelerate.utils.modeling import set_module_tensor_to_device+from safetensors.torch import load_file, save_file+from optimum.bettertransformer import BetterTransformer+import huggingface_hub++from .utils import save_quant_state_to_dict, NotEnoughSpaceException, clean_memory, uncompress_layer_state_dict, load_layer, \+ check_space, compress_layer_state_dict, split_and_save_layers, find_or_create_local_splitted_path++try:+ import bitsandbytes as bnb++ bitsandbytes_installed = True+ print('>>>> bitsandbytes installed')+except ImportError:+ bitsandbytes_installed = False++total_disk_loading_time = None+total_gpu_loading_time = None+total_compression_overhead_time = None++++class AirLLMChatGLM(GenerationMixin):+ def __init__(self, model_local_path_or_repo_id, device="cuda:0", dtype=torch.float16, max_seq_len=512,+ layer_shards_saving_path=None, profiling_mode=False, compression=None):+ """+ Sharded version of LlamaForCausalLM : the model is splitted into layer shards to reduce GPU memory usage.+ During the forward pass, the inputs are processed layer by layer, and the GPU memory is freed after each layer.+ To avoid loading the layers multiple times, we could save all the intermediate activations in RAM.++ Parameters+ ----------+ model_local_path_or_repo_id : str or Path+ path to the local model checkpoint or huggingface repo id+ device : str, optional+ device, by default "cuda:0"+ dtype : torch.dtype, optional+ dtype, by default torch.float16+ max_seq_len : int, optional+ max seq lenght, by default 512+ layer_shards_saving_path : str, optional+ optional path to save layered shards model file, by default just save to the local cache of model, subdir named splitted_model will be saved+ profiling_mode : book, optional+ if to profile the model loading time, default to False+ compression: str, optinal+ setting to '4bit' or '8bit' to enable compression from 16 bits to 4 bits/8 bits which speeed up 4x or 2x inference time with a tiny accuracy loss.+ """+++ self.profiling_mode = profiling_mode++ if compression is not None:+ if not bitsandbytes_installed:+ raise ImportError('WARNING: bitsandbytes not found. Compression needs bitsandbytes. To use compression, please install bitsandbytes: `pip install bitsandbytes`')+++ self.compression = compression++ # Save parameters+ self.layer_names_dict = {'embed': 'transformer.embedding.word_embeddings',+ 'layer_prefix': 'transformer.encoder.layers',+ 'norm': 'transformer.encoder.final_layernorm',+ 'lm_head': 'transformer.output_layer',+ 'rotary_pos_emb': 'transformer.rotary_pos_emb'}+ self.model_local_path, self.checkpoint_path = find_or_create_local_splitted_path(model_local_path_or_repo_id,+ layer_shards_saving_path,+ compression=compression,+ layer_names=self.layer_names_dict)+ self.running_device = device+ self.device = torch.device(self.running_device)+ self.running_dtype = dtype+ self.dtype = self.running_dtype++ # Create model+ self.config = AutoConfig.from_pretrained(self.model_local_path, trust_remote_code=True)+ self.generation_config = GenerationConfig()#GenerationConfig.from_pretrained(self.model_local_path)+ #print(f"using generation_config: {self.generation_config}")+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_local_path, trust_remote_code=True)+ #self.tokenizer.pad_token = self.tokenizer.eos_token+ #self.tokenizer.padding_side = "right"+ self.init_model()+ self.layer_names = [self.layer_names_dict['embed']] + [f'{self.layer_names_dict["layer_prefix"]}.{i}' for i in range(len(self.model.transformer.encoder.layers))] + \+ [self.layer_names_dict['norm'], self.layer_names_dict['lm_head']]++ self.max_seq_len = max_seq_len++ self.main_input_name = "input_ids"++ def init_model(self):++ # Load meta model (no memory used)+ with init_empty_weights():+ self.model = AutoModelForCausalLM.from_config(self.config, trust_remote_code=True)+ self.model.eval()+ #self.model = BetterTransformer.transform(self.model) # enable flash attention+ self.model.tie_weights()++ self.layers = [self.model.transformer.embedding] + list(self.model.transformer.encoder.layers) + \+ [self.model.transformer.encoder.final_layernorm, self.model.transformer.output_layer]++ # Move buffers to device (not that much GPU memory used)+ for buffer_name, buffer in self.model.named_buffers():+ set_module_tensor_to_device(self.model, buffer_name, self.running_device, value=buffer,+ dtype=self.running_dtype)++ # for glm keep rotary_pos_emb in gpu+ self.load_rotary_pos_emb_to_device()++ def load_rotary_pos_emb_to_device(self):+ state_dict = load_layer(self.checkpoint_path, 'transformer.rotary_pos_emb')+ self.move_layer_to_device(state_dict)++ def load_layer_to_cpu(self, layer_name, profiling=False):++ t = time.process_time()+ load_layer_output = load_layer(self.checkpoint_path, layer_name, profiling)+ elapsed_time = time.process_time() - t++ if profiling:+ state_dict, compression_time = load_layer_output+ disk_loading_time = elapsed_time - compression_time+ return state_dict, disk_loading_time, compression_time+ else:+ state_dict = load_layer_output++ return state_dict++ def move_layer_to_device(self, state_dict):+ for param_name, param in state_dict.items():+ assert param.dtype != torch.int8, "int8 not supported (need to add fp16_statistics)"+ set_module_tensor_to_device(self.model, param_name, self.running_device, value=param,+ dtype=self.running_dtype)++ # make GenerationMixin happy+ def can_generate(self):+ return True++ def prepare_inputs_for_generation(+ self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs+ ):+ if past_key_values is not None:+ past_length = past_key_values[0][0].shape[2]++ # Some generation methods already pass only the last input ID+ if input_ids.shape[1] > past_length:+ remove_prefix_length = past_length+ else:+ # Default to old behavior: keep only final ID+ remove_prefix_length = input_ids.shape[1] - 1++ input_ids = input_ids[:, remove_prefix_length:]++ position_ids = kwargs.get("position_ids", None)+ if attention_mask is not None and position_ids is None:+ # create position_ids on the fly for batch generation+ position_ids = attention_mask.long().cumsum(-1) - 1+ position_ids.masked_fill_(attention_mask == 0, 1)+ if past_key_values:+ position_ids = position_ids[:, -input_ids.shape[1]:]++ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step+ if inputs_embeds is not None and past_key_values is None:+ model_inputs = {"inputs_embeds": inputs_embeds}+ else:+ model_inputs = {"input_ids": input_ids}++ model_inputs.update(+ {+ "position_ids": position_ids,+ "past_key_values": past_key_values,+ "use_cache": kwargs.get("use_cache"),+ "attention_mask": attention_mask,+ }+ )+ return model_inputs++ def __call__(self, *args, **kwargs):+ return self.forward(*args, **kwargs)++ def forward(+ self,+ input_ids: torch.LongTensor = None,+ attention_mask: Optional[torch.Tensor] = None,+ position_ids: Optional[torch.LongTensor] = None,+ past_key_values: Optional[List[torch.FloatTensor]] = None,+ inputs_embeds: Optional[torch.FloatTensor] = None,+ labels: Optional[torch.LongTensor] = None,+ use_cache: Optional[bool] = None,+ output_attentions: Optional[bool] = None,+ output_hidden_states: Optional[bool] = None,+ return_dict: Optional[bool] = None,+ ) -> Union[Tuple, CausalLMOutputWithPast]:+ print(f"input_ids shape: {input_ids.shape}")++ global total_disk_loading_time, total_gpu_loading_time, total_compression_overhead_time++ if self.profiling_mode:+ total_disk_loading_time = []+ total_gpu_loading_time = []+ total_compression_overhead_time = []+ forward_start = time.process_time()++ # Reboot the model to make sure buffers are loaded and memory is clean+ del self.model+ clean_memory()+ self.init_model()++ batch = [input_ids_unit.to(self.running_device).unsqueeze(0) for input_ids_unit in input_ids]+ n_seq = len(batch[0])+ print(f"batch[0] shape:{batch[0].shape}")+ batch_eos = [(input_ids_unit != self.tokenizer.pad_token_id).sum(0) - 1 for input_ids_unit in input_ids]++ # Create attention mask for the largest input, and position ids to use KV cache+ attention_mask = torch.ones(self.max_seq_len, self.max_seq_len)+ attention_mask = attention_mask.triu(diagonal=1)[None, None, ...] == 0+ attention_mask = attention_mask.to(self.running_device)+ position_ids = torch.arange(self.max_seq_len, dtype=torch.long, device=self.running_device)[None, :]++ kv_cache_list = [] if use_cache else None+ if use_cache:+ for x in self.layers:+ kv_cache_list.append(([], []))+ all_hidden_states = [] * len(self.layers) if output_hidden_states else None+ all_self_attns = None++ with torch.inference_mode():++ for i, (layer_name, layer) in tqdm(enumerate(zip(self.layer_names, self.layers)), desc=self.running_device,+ total=len(self.layers)):+ load_layer_to_cpu_output = self.load_layer_to_cpu(layer_name, self.profiling_mode)+ # profile+ if self.profiling_mode:+ state_dict, disk_loading_time, compression_time = load_layer_to_cpu_output+ total_disk_loading_time.append(disk_loading_time)+ total_compression_overhead_time.append(compression_time)+ else:+ state_dict = load_layer_to_cpu_output++ t = time.process_time()+ self.move_layer_to_device(state_dict)+ elapsed_time = time.process_time() - t+ # profile+ if self.profiling_mode:+ total_gpu_loading_time.append(elapsed_time)++ # Run layer++ for j, seq in enumerate(batch):+ #print(f"{j}th in batch shape: {seq.shape}")++ if layer_name == self.layer_names_dict['embed']:+ batch[j] = layer(seq)+ elif layer_name == self.layer_names_dict['norm']:+ #batch[j] = layer(seq[torch.arange(n_seq), batch_eos[j]][:, None])+ batch[j] = layer(seq)++ #if output_attentions:+ # all_hidden_states[i].append(batch[j])+ elif layer_name == self.layer_names_dict['lm_head']:+ layer_res = layer(seq)+ batch[j] = layer_res.transpose(0, 1).contiguous().float()+ else:++ #if output_attentions:+ # all_hidden_states[i].append(new_seq)++ if past_key_values is not None:+ #print(f"len past_key_values: {len(past_key_values)}, past_key_values[0][0] shape:{past_key_values[0][0].shape}")+ # join past kv+ k_cache, v_cache = past_key_values[i - 1]+ len_p = past_key_values[0][0].shape[0]+ len_s = seq.shape[0]++ pos = position_ids[:, len_p:len_p + len_s]++ # Rotary positional embeddings+ rotary_pos_emb = self.model.transformer.rotary_pos_emb(self.config.seq_length)+ rotary_pos_emb = rotary_pos_emb[None, : len_s]+ rotary_pos_emb = rotary_pos_emb.transpose(0, 1).contiguous()+++ attn = attention_mask[:, :, -len_s:, -len_p - len_s:]+ kv_cache = (k_cache,+ v_cache,+ )++ layer_outputs = layer(seq,+ use_cache=True,+ #output_attentions=output_attentions,+ kv_cache=kv_cache,+ #position_ids=pos,+ rotary_pos_emb=rotary_pos_emb,+ attention_mask=None #attn+ )+ new_seq = layer_outputs[0]++ #if output_attentions:+ # all_self_attns[i].append(layer_outputs[1])++ if use_cache:+ (k_cache, v_cache) = layer_outputs[1]+ kv_cache_list[i][0].append(k_cache)+ kv_cache_list[i][1].append(v_cache)+++ else:+ # Note: glm shape is [s b h]+ len_seq = seq.shape[0]++ # Rotary positional embeddings+ rotary_pos_emb = self.model.transformer.rotary_pos_emb(self.config.seq_length)+ rotary_pos_emb = rotary_pos_emb[position_ids[:, :len_seq]]+ rotary_pos_emb = rotary_pos_emb.transpose(0, 1).contiguous()++ if not use_cache:+ new_seq = layer(seq,+ rotary_pos_emb=rotary_pos_emb,+ attention_mask=None#attention_mask[:, :, -len_seq:, -len_seq:]+ )[0]+ else:+ new_seq, (k_cache, v_cache) = layer(seq,+ use_cache=True,+ rotary_pos_emb=rotary_pos_emb,+ attention_mask=None #attention_mask[:, :, -len_seq:,-len_seq:]+ )+ kv_cache_list[i][0].append(k_cache)+ kv_cache_list[i][1].append(v_cache)++ # print(f"k_cache size: {k_cache.shape}")+ # print(f"k_cache sizes: {[len(x[1]) for x in kv_cache_list]}")++ batch[j] = new_seq++ if output_hidden_states:+ all_hidden_states += (torch.cat(batch, 0),)++ # Remove previous layer from memory (including buffers)+ layer.to("meta")+ clean_memory() # proposed by CPMP++ logits = torch.cat(batch, 0)+ if use_cache:+ kv_cache_list = kv_cache_list[1:-2]+ for i in range(len(kv_cache_list)):+ # print(f"{i} - {kv_cache_list[i][0].shape}")+ kv_cache_list[i] = (torch.cat(kv_cache_list[i][0], 0), torch.cat(kv_cache_list[i][1], 0))+ #print(f"returning kvcache size: {kv_cache_list[0][0].shape}")++ #if output_attentions:+ # all_self_attns = all_self_attns[0:-2]+ # for i in range(len(all_self_attns)):+ # all_self_attns[i] = torch.cat(all_self_attns[i], 0)++ if output_hidden_states:+ all_hidden_states = all_hidden_states[0:-2]+ for i in range(len(all_hidden_states)):+ all_hidden_states[i] = torch.cat(all_hidden_states[i], 0)++ if not return_dict:+ return tuple(v for v in [logits,+ tuple(kv_cache_list) if kv_cache_list is not None else None,+ tuple(all_hidden_states) if all_hidden_states is not None else None,+ tuple(all_self_attns) if all_self_attns is not None else None] if v is not None)+ if self.profiling_mode:++ forward_elapsed_time = time.process_time() - forward_start++ if self.compression:+ print(f"total disk loading time: {sum(total_disk_loading_time):.04f}")+ print(f"total gpu loading time: {sum(total_gpu_loading_time):.04f}")+ print(f"total compression overhead time: {sum(total_compression_overhead_time):.04f}")+ else:+ # loading is async/lazy, so can't really distinguish them...+ print(f"total disk+gpu loading time: {sum(total_disk_loading_time) + sum(total_gpu_loading_time):.04f}")+ print(f"total infer time(including all above plus gpu compute): {forward_elapsed_time:.04f}")++ total_disk_loading_time = []+ total_gpu_loading_time = []+ total_compression_overhead_time = []+++ return CausalLMOutputWithPast(+ loss=None,+ logits=logits,+ past_key_values=tuple(kv_cache_list) if kv_cache_list is not None else None,+ hidden_states=tuple(all_hidden_states) if all_hidden_states is not None else None,+ attentions=tuple(all_self_attns) if all_hidden_states is not None else None,+ )\ No newline at end of filediff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.pyindex 705b7e6..38592a3 100644--- a/air_llm/airllm/utils.py+++ b/air_llm/airllm/utils.py@@ -155,7 +155,8 @@ def compress_layer_state_dict(layer_state_dict, compression=None):return compressed_layer_state_dict if compressed_layer_state_dict is not None else layer_state_dict-def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitted_model_dir_name='splitted_model', compression=None):+def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitted_model_dir_name='splitted_model',+ compression=None, layer_names=None):"""Save the all layers of a model sharded checkpoint using safetensors."""@@ -190,8 +191,19 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splittwith open(checkpoint_path / 'model.safetensors.index.json', 'rb') as f:index = json.load(f)['weight_map']- n_layers = len(set([int(k.split('.')[2]) for k in index.keys() if 'model.layers' in k]))- layers = ['model.embed_tokens.'] + [f'model.layers.{i}.' for i in range(n_layers)] + ['model.norm.', 'lm_head.']+ if layer_names is None:+ n_layers = len(set([int(k.split('.')[2]) for k in index.keys() if 'model.layers' in k]))+ else:+ n_layers = len(set([int(k[len(layer_names['layer_prefix']):].split('.')[1]) for k in index.keys() if layer_names['layer_prefix'] in k]))++ if layer_names is None:+ layers = ['model.embed_tokens.'] + [f'model.layers.{i}.' for i in range(n_layers)] + ['model.norm.', 'lm_head.']+ else:+ layers = [layer_names['embed']] + [f'{layer_names["layer_prefix"]}.{i}' for i in range(n_layers)] + [layer_names['norm'], layer_names['lm_head']]++ if 'rotary_pos_emb' in layer_names:+ layers = [layer_names['rotary_pos_emb']] + layers+ layers = [l + "." for l in layers]shard = 0n_shards = len(set(index.values()))state_dict = {}@@ -235,7 +247,8 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splittreturn str(saving_path)-def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards_saving_path=None, compression=None):+def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards_saving_path=None, compression=None,+ layer_names=None):"""find the model's local cache path, download the cache if not exists, then split and save the model.@@ -260,7 +273,8 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shardsif os.path.exists(model_local_path_or_repo_id):if os.path.exists(Path(model_local_path_or_repo_id) / 'pytorch_model.bin.index.json') or \os.path.exists(Path(model_local_path_or_repo_id) / 'model.safetensors.index.json'):- return Path(model_local_path_or_repo_id), split_and_save_layers(model_local_path_or_repo_id, layer_shards_saving_path, compression=compression)+ return Path(model_local_path_or_repo_id), split_and_save_layers(model_local_path_or_repo_id, layer_shards_saving_path,+ compression=compression, layer_names=layer_names)else:print(f"Found local directory in {model_local_path_or_repo_id}, but didn't find downloaded model. Try using {model_local_path_or_repo_id} as a HF repo...")@@ -272,5 +286,6 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shardsf"{hf_cache_path}/pytorch_model.bin.index.json or {hf_cache_path}/model.safetensors.index.json should exists."# if splitted_model subdir exists under cache use it, otherwise split and save- return Path(hf_cache_path), split_and_save_layers(hf_cache_path, layer_shards_saving_path, compression=compression)+ return Path(hf_cache_path), split_and_save_layers(hf_cache_path, layer_shards_saving_path,+ compression=compression, layer_names=layer_names)diff --git a/air_llm/setup.py b/air_llm/setup.pyindex f89cc94..b7250df 100644--- a/air_llm/setup.py+++ b/air_llm/setup.py@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:setuptools.setup(name="airllm",- version="2.1.1",+ version="2.2.0",author="Gavin Li",author_email="[email protected]",description="AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning.",