Skip to content

AirLLM 2.8T LLM inference with single 4GB GPU

Fix airllm install/import and single-file model splitting

Yu Li9478b3f2fee4
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.py
index d8e2835..f7449b2 100644
--- a/air_llm/airllm/airllm_base.py
+++ b/air_llm/airllm/airllm_base.py
@@ -15,8 +15,6 @@ from transformers.quantizers import AutoHfQuantizer, HfQuantizer
from .profiler import LayeredProfiler
-from optimum.bettertransformer import BetterTransformer
-
from .utils import clean_memory, load_layer, \
find_or_create_local_splitted_path
@@ -180,39 +178,23 @@ class AirLLMBaseModel(GenerationMixin):
def init_model(self):
- # try way 1 better transformers...
- # Load meta model (no memory used)
+ # Load meta model (no real memory used). Prefer PyTorch's SDPA (flash / memory-efficient
+ # attention), which is built into transformers now. BetterTransformer is deprecated and
+ # has been removed from optimum, so we no longer depend on it.
self.model = None
- if self.get_use_better_transformer():
- try:
- with init_empty_weights():
- self.model = AutoModelForCausalLM.from_config(self.config, trust_remote_code=True)
- self.model = BetterTransformer.transform(self.model) # enable flash attention
- except ValueError as ve:
- del self.model
- clean_memory()
- self.model = None
-
- if self.model is None:
- # try way 2.
- try:
-
- print(f"new version of transfomer, no need to use BetterTransformer, try setting attn impl to sdpa...")
- self.config.attn_implementation = "sdpa"
-
- with init_empty_weights():
- self.model = AutoModelForCausalLM.from_config(self.config, attn_implementation="sdpa", trust_remote_code=True)
- print(f"attn imp: {type(self.model.model.layers[3].self_attn)}")
-
- except TypeError as ve:
- del self.model
- clean_memory()
- self.model = None
-
- # fallback to original way
+ try:
+ with init_empty_weights():
+ self.model = AutoModelForCausalLM.from_config(
+ self.config, attn_implementation="sdpa", trust_remote_code=True)
+ except (ValueError, TypeError) as e:
+ # This architecture may not implement an SDPA path; fall back to the default attention.
+ print(f"attn_implementation='sdpa' not available ({e}), creating model with default attn implementation")
+ del self.model
+ clean_memory()
+ self.model = None
+
if self.model is None:
- print(f"either BetterTransformer or attn_implementation='sdpa' is available, creating model directly")
with init_empty_weights():
self.model = AutoModelForCausalLM.from_config(self.config, trust_remote_code=True)
diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.py
index e2536e9..a2e02b5 100644
--- a/air_llm/airllm/utils.py
+++ b/air_llm/airllm/utils.py
@@ -204,15 +204,32 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splitt
saving_path = Path(layer_shards_saving_path) / splitted_model_dir_name
+ # Build a weight_map (param name -> file that stores it). Multi-shard checkpoints ship an
+ # index.json; small/modern models often ship a single file with no index, so synthesize one.
safetensors_format = False
if os.path.exists(checkpoint_path / 'pytorch_model.bin.index.json'):
with open(checkpoint_path / 'pytorch_model.bin.index.json', 'rb') as f:
index = json.load(f)['weight_map']
- else:
+ elif os.path.exists(checkpoint_path / 'model.safetensors.index.json'):
safetensors_format = True
- assert os.path.exists(checkpoint_path / 'model.safetensors.index.json'), f'model.safetensors.index.json should exist.'
with open(checkpoint_path / 'model.safetensors.index.json', 'rb') as f:
index = json.load(f)['weight_map']
+ elif os.path.exists(checkpoint_path / 'model.safetensors'):
+ # single-file safetensors checkpoint: map every tensor to that one file
+ safetensors_format = True
+ from safetensors import safe_open
+ with safe_open(str(checkpoint_path / 'model.safetensors'), framework='pt') as f:
+ index = {k: 'model.safetensors' for k in f.keys()}
+ elif os.path.exists(checkpoint_path / 'pytorch_model.bin'):
+ # single-file torch checkpoint: map every tensor to that one file
+ safetensors_format = False
+ single_sd = torch.load(checkpoint_path / 'pytorch_model.bin', map_location='cpu')
+ index = {k: 'pytorch_model.bin' for k in single_sd.keys()}
+ del single_sd
+ else:
+ raise FileNotFoundError(
+ f"No model weights found under {checkpoint_path}. Expected one of: "
+ f"model.safetensors(.index.json) or pytorch_model.bin(.index.json).")
if layer_names is None:
n_layers = len(set([int(k.split('.')[2]) for k in index.keys() if 'model.layers' in k]))
@@ -374,10 +391,21 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shards
f"Found local directory in {model_local_path_or_repo_id}, but didn't find downloaded model. Try using {model_local_path_or_repo_id} as a HF repo...")
# it should be a repo id at this point...
+ # First grab everything except the (potentially huge) weight files. For multi-shard models the
+ # index.json tells us the structure and we stream each shard on demand during splitting.
hf_cache_path = huggingface_hub.snapshot_download(model_local_path_or_repo_id, token=hf_token,
#allow_patterns= ["model.safetensors.index.json", 'pytorch_model.bin.index.json'],
ignore_patterns=['*.safetensors', '*.bin'])
+ # Single-file checkpoints have no index.json, so there's nothing to stream on demand and we
+ # can't infer the structure without the file itself. Download the single weight file now.
+ has_index = os.path.exists(Path(hf_cache_path) / 'model.safetensors.index.json') or \
+ os.path.exists(Path(hf_cache_path) / 'pytorch_model.bin.index.json')
+ if not has_index:
+ hf_cache_path = huggingface_hub.snapshot_download(
+ model_local_path_or_repo_id, token=hf_token,
+ allow_patterns=['model.safetensors', 'pytorch_model.bin'])
+
# check if there's safetensors saved, if so, exclude torch saves
# delay download now...
diff --git a/air_llm/setup.py b/air_llm/setup.py
index fcfead4..1b8cc30 100644
--- a/air_llm/setup.py
+++ b/air_llm/setup.py
@@ -1,16 +1,4 @@
-import sys
import setuptools
-from setuptools.command.install import install
-import subprocess
-
-# upgrade transformers to latest version to avoid "`rope_scaling` must be a dictionary with two fields" error
-class PostInstallCommand(install):
- def run(self):
- install.run(self)
- try:
- subprocess.check_call([sys.executable, "-m", "pip", "install", "--upgrade", "transformers"])
- except subprocess.CalledProcessError:
- print("Warning: Unable to upgrade transformers package. Please upgrade manually.")
# Windows uses a different default encoding (use a consistent encoding)
with open("README.md", "r", encoding="utf-8") as fh:
@@ -18,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="airllm",
- version="2.11.0",
+ version="3.0.0",
author="Gavin Li",
author_email="[email protected]",
description="AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning. 8GB vmem to run 405B Llama3.1.",
@@ -26,23 +14,21 @@ setuptools.setup(
long_description_content_type="text/markdown",
url="https://github.com/lyogavin/airllm",
packages=setuptools.find_packages(),
+ # Keep the dependency surface small and pinned to a tested range so a plain
+ # `pip install airllm` gives users a known-good stack with no manual upgrades.
install_requires=[
'tqdm',
- 'torch',
- 'transformers',
- 'accelerate',
+ 'torch>=2.4',
+ 'transformers>=4.49,<5.13',
+ 'accelerate>=1.0',
'safetensors',
- 'optimum',
'huggingface-hub',
'scipy',
- #'bitsandbytes' set it to optional to support fallback when not installable
+ # 'bitsandbytes' is optional (used only for --compression); we fall back gracefully when absent.
],
- cmdclass={
- 'install': PostInstallCommand,
- },
classifiers=[
"Programming Language :: Python :: 3",
- "License :: OSI Approved :: MIT License",
+ "License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
],
)