PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
low_cpu_mem refinement (#186)
n1ck-guo7b9611ed36fa
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 383ee5695..674b758bc 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -45,7 +45,7 @@ from .utils import (mv_module_from_gpu,)-from .layer_wise.utils import get_layers_before_block+from .low_cpu_mem.utils import get_layers_before_blockclass AutoRound(object):"""This is Signround+ which is an advanced version of Signround. For more information,@@ -86,6 +86,7 @@ class AutoRound(object):lr (float): The learning rate (default is None, will be set to 1.0/iters).minmax_lr (float): The learning rate for min-max tuning (default is None, it will be set to lr automatically).low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).iters (int): Number of iterations (default is 200).seqlen (int): Data length of the sequence for tuning (default is 2048).nsamples (int): Number of samples (default is 128).@@ -127,6 +128,7 @@ class AutoRound(object):lr: float = None,minmax_lr: float = None,low_gpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = False,iters: int = 200,seqlen: int = 2048,nsamples: int = 128,@@ -143,7 +145,6 @@ class AutoRound(object):act_group_size: int = None,act_sym: bool = None,act_dynamic: bool = True,- low_cpu_mem_usage: bool = False,**kwargs,):self.quantized = False@@ -1253,6 +1254,7 @@ class AutoOPTRound(AutoRound):lr (float): The learning rate (default is 0.005).minmax_lr (float): The learning rate for min-max tuning (default is None).low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).iters (int): Number of iterations (default is 200).seqlen (int): Length of the sequence.nsamples (int): Number of samples (default is 128).@@ -1295,6 +1297,7 @@ class AutoOPTRound(AutoRound):lr: float = None,minmax_lr: float = None,low_gpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = False,iters: int = 200,seqlen: int = 2048,nsamples: int = 128,@@ -1331,6 +1334,7 @@ class AutoOPTRound(AutoRound):lr,minmax_lr,low_gpu_mem_usage,+ low_cpu_mem_usage,iters,seqlen,nsamples,@@ -1414,6 +1418,7 @@ class AutoAdamRound(AutoOPTRound):lr (float): The learning rate (default is 0.005).minmax_lr (float): The learning rate for min-max tuning (default is None).low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).iters (int): Number of iterations (default is 200).seqlen (int): Length of the sequence.nsamples (int): Number of samples (default is 128).@@ -1455,6 +1460,7 @@ class AutoAdamRound(AutoOPTRound):lr: float = None,minmax_lr: float = None,low_gpu_mem_usage: bool = False,+ low_cpu_mem_usage: bool = False,iters: int = 200,seqlen: int = 2048,nsamples: int = 128,@@ -1491,6 +1497,7 @@ class AutoAdamRound(AutoOPTRound):lr,minmax_lr,low_gpu_mem_usage,+ low_cpu_mem_usage,iters,seqlen,nsamples,diff --git a/auto_round/layer_wise/__init__.py b/auto_round/low_cpu_mem/__init__.pysimilarity index 100%rename from auto_round/layer_wise/__init__.pyrename to auto_round/low_cpu_mem/__init__.pydiff --git a/auto_round/layer_wise/load.py b/auto_round/low_cpu_mem/load.pysimilarity index 99%rename from auto_round/layer_wise/load.pyrename to auto_round/low_cpu_mem/load.pyindex d50fdf734..1ff291b16 100644--- a/auto_round/layer_wise/load.py+++ b/auto_round/low_cpu_mem/load.py@@ -32,7 +32,7 @@ from torch.serialization import (_open_zipfile_reader,)-from ..layer_wise import modified_pickle as pickle+from ..low_cpu_mem import modified_pickle as picklefrom .utils import torchdiff --git a/auto_round/layer_wise/modified_pickle.py b/auto_round/low_cpu_mem/modified_pickle.pysimilarity index 100%rename from auto_round/layer_wise/modified_pickle.pyrename to auto_round/low_cpu_mem/modified_pickle.pydiff --git a/auto_round/layer_wise/utils.py b/auto_round/low_cpu_mem/utils.pysimilarity index 99%rename from auto_round/layer_wise/utils.pyrename to auto_round/low_cpu_mem/utils.pyindex 92c14bbac..26fdcb855 100644--- a/auto_round/layer_wise/utils.py+++ b/auto_round/low_cpu_mem/utils.py@@ -33,9 +33,9 @@ from transformers.models.auto.auto_factory import _BaseAutoModelClassfrom .load import loadlogging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s")-logger = logging.getLogger("layer_wise_tools")+logger = logging.getLogger("low_cpu_mem_tools")-LWQ_WORKSPACE = os.path.join("layer_wise_tmp")+LWQ_WORKSPACE = os.path.join("low_cpu_mem_tmp")def get_module(model, key):diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 195c2586c..f7bdd1f27 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -167,6 +167,15 @@ def to_device(input, device=torch.device("cpu")):return inputdef mv_module_from_gpu(module, low_cpu_mem_usage=False):+ """Moves module from gpu to cpu or meta if low_cpu_mem_usage is true.++ Args:+ module: The module to be moved.+ low_cpu_mem_usage: Whether to use low CPU memory. If true, move module to meta.++ Returns:+ The module on the specified device.+ """if hasattr(module, "device"):target_device ="meta" if low_cpu_mem_usage else "cpu"if module.device.type == target_device:diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex bb28a0677..bc3fec102 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -52,7 +52,6 @@ pip install -r requirements.txtCUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128```- **Reduced GPU Memory Usage:**--enable low_gpu_mem_usage(more tuning cost)set "--train_bs 1 --gradient_accumulate_steps 8" (more tuning cost)@@ -63,6 +62,12 @@ reduce the seqlen to 512 (potential accuracy drop)or combine them+- **Reduced CPU Memory Usage:**+set "--low_cpu_mem_mode 1" to use block-wise mode, load the weights from disk of each block when tuning and release the memory of the block after tuning. (more tuning cost)++set "--low_cpu_mem_mode 2" to use layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed.++- **Speedup the tuning:**reduce the train bs to 4(little accuracy drop)diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 942a51063..3cd5e660d 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -121,7 +121,13 @@ if __name__ == '__main__':help="quant_lm_head")parser.add_argument("--low_cpu_mem_mode", default=0, type=int,- help="choose low cpu memory mode, 1 for block-wise, 2 for layer-wise, others means not use low cpu memory.")+ help="Choose which low cpu memory mode to use. Can significantly reduce cpu memory footprint but cost more time."+ "1 means choose block-wise mode, load the weights of each block from disk when tuning and release the memory of the block after tuning."+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed."+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")++ parser.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,+ help="temp work space to store the temporary files when using low cpu memory mode. Will remove after tuning.")parser.add_argument("--model_dtype", default=None, type=str,help="force to convert the dtype, some backends supports fp16 dtype better")@@ -180,25 +186,27 @@ if __name__ == '__main__':is_glm = bool(re.search("chatglm", model_name.lower()))low_cpu_mem_usage = Falsemodel_cls = AutoModel if is_glm else AutoModelForCausalLM+ if args.low_cpu_mem_tmp_dir is None:+ args.low_cpu_mem_tmp_dir = os.path.join(args.output_dir, "low_cpu_mem_tmp")if args.low_cpu_mem_mode == 2:- from auto_round.layer_wise.utils import load_model_with_hooks+ from auto_round.low_cpu_mem.utils import load_model_with_hooksmodel = load_model_with_hooks(model_name,model_cls,device=device_str,clean_weight=True,- saved_path=os.path.join(args.output_dir, "layer_wise_tmp"),+ saved_path=args.low_cpu_mem_tmp_dir,torch_dtype=torch_dtype,trust_remote_code=not args.disable_trust_remote_code)elif args.low_cpu_mem_mode == 1:- from auto_round.layer_wise.utils import load_empty_model+ from auto_round.low_cpu_mem.utils import load_empty_modellow_cpu_mem_usage = Truemodel = load_empty_model(model_name,model_cls,device=device_str,- saved_path=os.path.join(args.output_dir, "layer_wise_tmp"),+ saved_path=args.low_cpu_mem_tmp_dir,torch_dtype=torch_dtype,trust_remote_code=not args.disable_trust_remote_code)@@ -300,7 +308,7 @@ if __name__ == '__main__':model_name = args.model_name.rstrip("/")if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:import shutil- shutil.rmtree(os.path.join(args.output_dir, "layer_wise_tmp"), ignore_errors=True)+ shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)model.eval()if "cpu" not in device_str:diff --git a/test/test_low_cpu_mem.py b/test/test_low_cpu_mem.pyindex 3f8dd0211..6f368dc70 100644--- a/test/test_low_cpu_mem.py+++ b/test/test_low_cpu_mem.py@@ -6,7 +6,7 @@ sys.path.insert(0, "..")import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round.layer_wise.utils import (+from auto_round.low_cpu_mem.utils import (load_model_with_hooks,load_empty_model,get_layers_before_block,