AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
Fix airllm install/import and single-file model splitting
Yu Li9478b3f2fee4
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.pyindex d8e2835..f7449b2 100644--- a/air_llm/airllm/airllm_base.py+++ b/air_llm/airllm/airllm_base.py@@ -15,8 +15,6 @@ from transformers.quantizers import AutoHfQuantizer, HfQuantizerfrom .profiler import LayeredProfiler-from optimum.bettertransformer import BetterTransformer-from .utils import clean_memory, load_layer, \find_or_create_local_splitted_path@@ -180,39 +178,23 @@ class AirLLMBaseModel(GenerationMixin):def init_model(self):- # try way 1 better transformers...- # Load meta model (no memory used)+ # Load meta model (no real memory used). Prefer PyTorch's SDPA (flash / memory-efficient+ # attention), which is built into transformers now. BetterTransformer is deprecated and+ # has been removed from optimum, so we no longer depend on it.self.model = None- if self.get_use_better_transformer():- try:- with init_empty_weights():- self.model = AutoModelForCausalLM.from_config(self.config, trust_remote_code=True)- self.model = BetterTransformer.transform(self.model) # enable flash attention- except ValueError as ve:- del self.model- clean_memory()- self.model = None-- if self.model is None:- # try way 2.- try:-- print(f"new version of transfomer, no need to use BetterTransformer, try setting attn impl to sdpa...")- self.config.attn_implementation = "sdpa"-- with init_empty_weights():- self.model = AutoModelForCausalLM.from_config(self.config, attn_implementation="sdpa", trust_remote_code=True)- print(f"attn imp: {type(self.model.model.layers[3].self_attn)}")-- except TypeError as ve:- del self.model- clean_memory()- self.model = None-- # fallback to original way+ try:+ with init_empty_weights():+ self.model = AutoModelForCausalLM.from_config(+ self.config, attn_implementation="sdpa", trust_remote_code=True)+ except (ValueError, TypeError) as e:+ # This architecture may not implement an SDPA path; fall back to the default attention.+ print(f"attn_implementation='sdpa' not available ({e}), creating model with default attn implementation")+ del self.model+ clean_memory()+ self.model = None+if self.model is None:- print(f"either BetterTransformer or attn_implementation='sdpa' is available, creating model directly")with init_empty_weights():self.model = AutoModelForCausalLM.from_config(self.config, trust_remote_code=True)diff --git a/air_llm/airllm/utils.py b/air_llm/airllm/utils.pyindex e2536e9..a2e02b5 100644--- a/air_llm/airllm/utils.py+++ b/air_llm/airllm/utils.py@@ -204,15 +204,32 @@ def split_and_save_layers(checkpoint_path, layer_shards_saving_path=None, splittsaving_path = Path(layer_shards_saving_path) / splitted_model_dir_name+ # Build a weight_map (param name -> file that stores it). Multi-shard checkpoints ship an+ # index.json; small/modern models often ship a single file with no index, so synthesize one.safetensors_format = Falseif os.path.exists(checkpoint_path / 'pytorch_model.bin.index.json'):with open(checkpoint_path / 'pytorch_model.bin.index.json', 'rb') as f:index = json.load(f)['weight_map']- else:+ elif os.path.exists(checkpoint_path / 'model.safetensors.index.json'):safetensors_format = True- assert os.path.exists(checkpoint_path / 'model.safetensors.index.json'), f'model.safetensors.index.json should exist.'with open(checkpoint_path / 'model.safetensors.index.json', 'rb') as f:index = json.load(f)['weight_map']+ elif os.path.exists(checkpoint_path / 'model.safetensors'):+ # single-file safetensors checkpoint: map every tensor to that one file+ safetensors_format = True+ from safetensors import safe_open+ with safe_open(str(checkpoint_path / 'model.safetensors'), framework='pt') as f:+ index = {k: 'model.safetensors' for k in f.keys()}+ elif os.path.exists(checkpoint_path / 'pytorch_model.bin'):+ # single-file torch checkpoint: map every tensor to that one file+ safetensors_format = False+ single_sd = torch.load(checkpoint_path / 'pytorch_model.bin', map_location='cpu')+ index = {k: 'pytorch_model.bin' for k in single_sd.keys()}+ del single_sd+ else:+ raise FileNotFoundError(+ f"No model weights found under {checkpoint_path}. Expected one of: "+ f"model.safetensors(.index.json) or pytorch_model.bin(.index.json).")if layer_names is None:n_layers = len(set([int(k.split('.')[2]) for k in index.keys() if 'model.layers' in k]))@@ -374,10 +391,21 @@ def find_or_create_local_splitted_path(model_local_path_or_repo_id, layer_shardsf"Found local directory in {model_local_path_or_repo_id}, but didn't find downloaded model. Try using {model_local_path_or_repo_id} as a HF repo...")# it should be a repo id at this point...+ # First grab everything except the (potentially huge) weight files. For multi-shard models the+ # index.json tells us the structure and we stream each shard on demand during splitting.hf_cache_path = huggingface_hub.snapshot_download(model_local_path_or_repo_id, token=hf_token,#allow_patterns= ["model.safetensors.index.json", 'pytorch_model.bin.index.json'],ignore_patterns=['*.safetensors', '*.bin'])+ # Single-file checkpoints have no index.json, so there's nothing to stream on demand and we+ # can't infer the structure without the file itself. Download the single weight file now.+ has_index = os.path.exists(Path(hf_cache_path) / 'model.safetensors.index.json') or \+ os.path.exists(Path(hf_cache_path) / 'pytorch_model.bin.index.json')+ if not has_index:+ hf_cache_path = huggingface_hub.snapshot_download(+ model_local_path_or_repo_id, token=hf_token,+ allow_patterns=['model.safetensors', 'pytorch_model.bin'])+# check if there's safetensors saved, if so, exclude torch saves# delay download now...diff --git a/air_llm/setup.py b/air_llm/setup.pyindex fcfead4..1b8cc30 100644--- a/air_llm/setup.py+++ b/air_llm/setup.py@@ -1,16 +1,4 @@-import sysimport setuptools-from setuptools.command.install import install-import subprocess--# upgrade transformers to latest version to avoid "`rope_scaling` must be a dictionary with two fields" error-class PostInstallCommand(install):- def run(self):- install.run(self)- try:- subprocess.check_call([sys.executable, "-m", "pip", "install", "--upgrade", "transformers"])- except subprocess.CalledProcessError:- print("Warning: Unable to upgrade transformers package. Please upgrade manually.")# Windows uses a different default encoding (use a consistent encoding)with open("README.md", "r", encoding="utf-8") as fh:@@ -18,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:setuptools.setup(name="airllm",- version="2.11.0",+ version="3.0.0",author="Gavin Li",author_email="[email protected]",description="AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning. 8GB vmem to run 405B Llama3.1.",@@ -26,23 +14,21 @@ setuptools.setup(long_description_content_type="text/markdown",url="https://github.com/lyogavin/airllm",packages=setuptools.find_packages(),+ # Keep the dependency surface small and pinned to a tested range so a plain+ # `pip install airllm` gives users a known-good stack with no manual upgrades.install_requires=['tqdm',- 'torch',- 'transformers',- 'accelerate',+ 'torch>=2.4',+ 'transformers>=4.49,<5.13',+ 'accelerate>=1.0','safetensors',- 'optimum','huggingface-hub','scipy',- #'bitsandbytes' set it to optional to support fallback when not installable+ # 'bitsandbytes' is optional (used only for --compression); we fall back gracefully when absent.],- cmdclass={- 'install': PostInstallCommand,- },classifiers=["Programming Language :: Python :: 3",- "License :: OSI Approved :: MIT License",+ "License :: OSI Approved :: Apache Software License","Operating System :: OS Independent",],)