AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
fix layer_name
Yu Lib1e6311cc2ce
diff --git a/air_llm/airllm/airllm_base.py b/air_llm/airllm/airllm_base.pyindex 6dcbbb0..b214cb1 100644--- a/air_llm/airllm/airllm_base.py+++ b/air_llm/airllm/airllm_base.py@@ -298,7 +298,7 @@ class AirLLMBaseModel(GenerationMixin):layers = []for param_name, param in state_dict.items():if self.hf_quantizer is None:- layers.append(layer_name)+ layers.append(param_name)else:if '.weight' in param_name:layer_name = param_name[:param_name.index(".weight") + len(".weight")]diff --git a/air_llm/setup.py b/air_llm/setup.pyindex 4c79cde..697caff 100644--- a/air_llm/setup.py+++ b/air_llm/setup.py@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:setuptools.setup(name="airllm",- version="2.9",+ version="2.9.1",author="Gavin Li",author_email="[email protected]",description="AirLLM allows single 4GB GPU card to run 70B large language models without quantization, distillation or pruning. 8GB vmem to run 405B Llama3.1.",