Skip to content

AirLLM 2.8T LLM inference with single 4GB GPU

support hf token

Yu Li693f01c682e1
diff --git a/air_llm/README.md b/air_llm/README.md
index ddc1c43..c3e4ba0 100644
--- a/air_llm/README.md
+++ b/air_llm/README.md
@@ -118,7 +118,7 @@ Quantization normally needs to quantize both weights and activations to really s
While in our case the bottleneck is mainly at the disk loading, we only need to make the model loading size smaller. So we get to only quantize the weights part, which is easier to ensure the accuracy.
-### 4. All supported configurations
+### 4. Configurations
When initialize the model, we support the following configurations:
@@ -127,6 +127,7 @@ When initialize the model, we support the following configurations:
* **compression**: supported options: 4bit, 8bit for 4-bit or 8-bit block-wise quantization, or by default None for no compression
* **profiling_mode**: supported options: True to output time consumptions or by default False
* **layer_shards_saving_path**: optionally another path to save the splitted model
+* **hf_token**: huggingface token can be provided here if downloading gated models like: *meta-llama/Llama-2-7b-hf*
### 5. Supported Models
diff --git a/air_llm/airllm/airllm.py b/air_llm/airllm/airllm.py
index cae4b37..2db897d 100644
--- a/air_llm/airllm/airllm.py
+++ b/air_llm/airllm/airllm.py
@@ -37,7 +37,8 @@ total_compression_overhead_time = None
class AirLLMLlama2(GenerationMixin):
def __init__(self, model_local_path_or_repo_id, device="cuda:0", dtype=torch.float16, max_seq_len=512,
- layer_shards_saving_path=None, profiling_mode=False, compression=None):
+ layer_shards_saving_path=None, profiling_mode=False, compression=None,
+ hf_token=None):
"""
Sharded version of LlamaForCausalLM : the model is splitted into layer shards to reduce GPU memory usage.
During the forward pass, the inputs are processed layer by layer, and the GPU memory is freed after each layer.
@@ -59,6 +60,8 @@ class AirLLMLlama2(GenerationMixin):
if to profile the model loading time, default to False
compression: str, optinal
setting to '4bit' or '8bit' to enable compression from 16 bits to 4 bits/8 bits which speeed up 4x or 2x inference time with a tiny accuracy loss.
+ hf_token: str, optional
+ huggingface api token could be provided, by default None
"""
@@ -70,21 +73,32 @@ class AirLLMLlama2(GenerationMixin):
self.compression = compression
+ self.hf_token = hf_token
# Save parameters
self.model_local_path, self.checkpoint_path = find_or_create_local_splitted_path(model_local_path_or_repo_id,
layer_shards_saving_path,
- compression=compression)
+ compression=compression,
+ hf_token=hf_token)
self.running_device = device
self.device = torch.device(self.running_device)
self.running_dtype = dtype
self.dtype = self.running_dtype
# Create model
- self.config = AutoConfig.from_pretrained(self.model_local_path)
+ if hf_token is not None:
+ self.config = AutoConfig.from_pretrained(self.model_local_path, token=hf_token)
+ else:
+ self.config = AutoConfig.from_pretrained(self.model_local_path)
+
self.generation_config = GenerationConfig.from_pretrained(self.model_local_path)
#print(f"using generation_config: {self.generation_config}")
- self.tokenizer = AutoTokenizer.from_pretrained(self.model_local_path)
+
+ if hf_token is not None:
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_local_path, token=hf_token)
+ else:
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_local_path)
+
self.tokenizer.pad_token = self.tokenizer.eos_token
self.tokenizer.padding_side = "right"
self.init_model()
diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.py
index 2ce49bb..bdc6d28 100644
--- a/air_llm/airllm/utils.py
+++ b/air_llm/airllm/utils.py
@@ -293,7 +293,7 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
return str(saving_path)
def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards_saving_path=None, compression=None,
- layer_names=None):
+ layer_names=None, hf_token=None):
"""
find the model's local cache path, download the cache if not exists, then split and save the model.
@@ -312,6 +312,8 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards
the path saved layer shards
compression: str, optinal
setting to '4bit' or '8bit' to enable compression from 16 bits to 4 bits/8 bits which speeed up 4x or 2x inference time with a tiny accuracy loss.
+ hf_token: str, optional
+ huggingface api token could be provided, by default None
"""
# try local model path, if the model exist split and save there
@@ -325,7 +327,10 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards
f"Found local directory in {model_local_path_or_repo_id}, but didn't find downloaded model. Try using {model_local_path_or_repo_id} as a HF repo...")
# it should be a repo id at this point...
- hf_cache_path = huggingface_hub.snapshot_download(model_local_path_or_repo_id)
+ if hf_token is not None:
+ hf_cache_path = huggingface_hub.snapshot_download(model_local_path_or_repo_id, token=hf_token)
+ else:
+ hf_cache_path = huggingface_hub.snapshot_download(model_local_path_or_repo_id)
assert os.path.exists(Path(hf_cache_path) / 'pytorch_model.bin.index.json') or \
os.path.exists(Path(hf_cache_path) / 'model.safetensors.index.json'), \
f"{hf_cache_path}/pytorch_model.bin.index.json or {hf_cache_path}/model.safetensors.index.json should exists."
diff --git a/air_llm/setup.py b/air_llm/setup.py
index dda18a9..a7f2b14 100644
--- a/air_llm/setup.py
+++ b/air_llm/setup.py
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
setuptools.setup(
name="airllm",
- version="2.4.1",
+ version="2.4.2",
author="Gavin Li",
author_email="[email protected]",
description="AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning.",