AirLLM 2.8T LLM inference with single 4GB GPU
Python chinese-llmchinese-nlpfinetunegenerative-aiinstruct-gptinstruction-setllamallmloraopen-modelsopen-sourceopen-source-modelsqlora
2048 block size for 8bit
Yu Lif3d9067c5cde
diff --git a/air_llm/airllm/airllm.py b/air_llm/airllm/airllm.pyindex 3413944..4257c8c 100644--- a/air_llm/airllm/airllm.py+++ b/air_llm/airllm/airllm.py@@ -97,7 +97,7 @@ def uncompress_layer_state_dict(layer_state_dict):absmax = layer_state_dict[k + ".8bit.absmax"]code = layer_state_dict[k + ".8bit.code"]- dqv = bnb.functional.dequantize(v, (absmax, code))+ dqv = bnb.functional.dequantize_blockwise(v, (absmax, code), blocksize=2048)uncompressed_layer_state_dict[k] = dqvdel layer_state_dict@@ -132,7 +132,7 @@ def compress_layer_state_dict(layer_state_dict, compression=None):elif compression == '8bit':compressed_layer_state_dict = {}for k, v in layer_state_dict.items():- v_quant, (absmax, code) = bnb.functional.quantize(v)+ v_quant, (absmax, code) = bnb.functional.quantize_blockwise(v, blocksize=2048)compressed_layer_state_dict[k] = v_quantcompressed_layer_state_dict[k + ".8bit.absmax"] = absmaxcompressed_layer_state_dict[k + ".8bit.code"] = code