Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

low_cpu_mem refinement (#186)

n1ck-guo7b9611ed36fa
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 383ee5695..674b758bc 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -45,7 +45,7 @@ from .utils import (
mv_module_from_gpu,
)
-from .layer_wise.utils import get_layers_before_block
+from .low_cpu_mem.utils import get_layers_before_block
class AutoRound(object):
"""This is Signround+ which is an advanced version of Signround. For more information,
@@ -86,6 +86,7 @@ class AutoRound(object):
lr (float): The learning rate (default is None, will be set to 1.0/iters).
minmax_lr (float): The learning rate for min-max tuning (default is None, it will be set to lr automatically).
low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).
iters (int): Number of iterations (default is 200).
seqlen (int): Data length of the sequence for tuning (default is 2048).
nsamples (int): Number of samples (default is 128).
@@ -127,6 +128,7 @@ class AutoRound(object):
lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = False,
+ low_cpu_mem_usage: bool = False,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
@@ -143,7 +145,6 @@ class AutoRound(object):
act_group_size: int = None,
act_sym: bool = None,
act_dynamic: bool = True,
- low_cpu_mem_usage: bool = False,
**kwargs,
):
self.quantized = False
@@ -1253,6 +1254,7 @@ class AutoOPTRound(AutoRound):
lr (float): The learning rate (default is 0.005).
minmax_lr (float): The learning rate for min-max tuning (default is None).
low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).
iters (int): Number of iterations (default is 200).
seqlen (int): Length of the sequence.
nsamples (int): Number of samples (default is 128).
@@ -1295,6 +1297,7 @@ class AutoOPTRound(AutoRound):
lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = False,
+ low_cpu_mem_usage: bool = False,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
@@ -1331,6 +1334,7 @@ class AutoOPTRound(AutoRound):
lr,
minmax_lr,
low_gpu_mem_usage,
+ low_cpu_mem_usage,
iters,
seqlen,
nsamples,
@@ -1414,6 +1418,7 @@ class AutoAdamRound(AutoOPTRound):
lr (float): The learning rate (default is 0.005).
minmax_lr (float): The learning rate for min-max tuning (default is None).
low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
+ low_cpu_mem_usage (bool): Whether to use low CPU memory (default is False).
iters (int): Number of iterations (default is 200).
seqlen (int): Length of the sequence.
nsamples (int): Number of samples (default is 128).
@@ -1455,6 +1460,7 @@ class AutoAdamRound(AutoOPTRound):
lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = False,
+ low_cpu_mem_usage: bool = False,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
@@ -1491,6 +1497,7 @@ class AutoAdamRound(AutoOPTRound):
lr,
minmax_lr,
low_gpu_mem_usage,
+ low_cpu_mem_usage,
iters,
seqlen,
nsamples,
diff --git a/auto_round/layer_wise/__init__.py b/auto_round/low_cpu_mem/__init__.py
similarity index 100%
rename from auto_round/layer_wise/__init__.py
rename to auto_round/low_cpu_mem/__init__.py
diff --git a/auto_round/layer_wise/load.py b/auto_round/low_cpu_mem/load.py
similarity index 99%
rename from auto_round/layer_wise/load.py
rename to auto_round/low_cpu_mem/load.py
index d50fdf734..1ff291b16 100644
--- a/auto_round/layer_wise/load.py
+++ b/auto_round/low_cpu_mem/load.py
@@ -32,7 +32,7 @@ from torch.serialization import (
_open_zipfile_reader,
)
-from ..layer_wise import modified_pickle as pickle
+from ..low_cpu_mem import modified_pickle as pickle
from .utils import torch
diff --git a/auto_round/layer_wise/modified_pickle.py b/auto_round/low_cpu_mem/modified_pickle.py
similarity index 100%
rename from auto_round/layer_wise/modified_pickle.py
rename to auto_round/low_cpu_mem/modified_pickle.py
diff --git a/auto_round/layer_wise/utils.py b/auto_round/low_cpu_mem/utils.py
similarity index 99%
rename from auto_round/layer_wise/utils.py
rename to auto_round/low_cpu_mem/utils.py
index 92c14bbac..26fdcb855 100644
--- a/auto_round/layer_wise/utils.py
+++ b/auto_round/low_cpu_mem/utils.py
@@ -33,9 +33,9 @@ from transformers.models.auto.auto_factory import _BaseAutoModelClass
from .load import load
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s")
-logger = logging.getLogger("layer_wise_tools")
+logger = logging.getLogger("low_cpu_mem_tools")
-LWQ_WORKSPACE = os.path.join("layer_wise_tmp")
+LWQ_WORKSPACE = os.path.join("low_cpu_mem_tmp")
def get_module(model, key):
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 195c2586c..f7bdd1f27 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -167,6 +167,15 @@ def to_device(input, device=torch.device("cpu")):
return input
def mv_module_from_gpu(module, low_cpu_mem_usage=False):
+ """Moves module from gpu to cpu or meta if low_cpu_mem_usage is true.
+
+ Args:
+ module: The module to be moved.
+ low_cpu_mem_usage: Whether to use low CPU memory. If true, move module to meta.
+
+ Returns:
+ The module on the specified device.
+ """
if hasattr(module, "device"):
target_device ="meta" if low_cpu_mem_usage else "cpu"
if module.device.type == target_device:
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index bb28a0677..bc3fec102 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -52,7 +52,6 @@ pip install -r requirements.txt
CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128
```
- **Reduced GPU Memory Usage:**
--
enable low_gpu_mem_usage(more tuning cost)
set "--train_bs 1 --gradient_accumulate_steps 8" (more tuning cost)
@@ -63,6 +62,12 @@ reduce the seqlen to 512 (potential accuracy drop)
or combine them
+- **Reduced CPU Memory Usage:**
+set "--low_cpu_mem_mode 1" to use block-wise mode, load the weights from disk of each block when tuning and release the memory of the block after tuning. (more tuning cost)
+
+set "--low_cpu_mem_mode 2" to use layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed.
+
+
- **Speedup the tuning:**
reduce the train bs to 4(little accuracy drop)
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 942a51063..3cd5e660d 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -121,7 +121,13 @@ if __name__ == '__main__':
help="quant_lm_head")
parser.add_argument("--low_cpu_mem_mode", default=0, type=int,
- help="choose low cpu memory mode, 1 for block-wise, 2 for layer-wise, others means not use low cpu memory.")
+ help="Choose which low cpu memory mode to use. Can significantly reduce cpu memory footprint but cost more time."
+ "1 means choose block-wise mode, load the weights of each block from disk when tuning and release the memory of the block after tuning."
+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed."
+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")
+
+ parser.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,
+ help="temp work space to store the temporary files when using low cpu memory mode. Will remove after tuning.")
parser.add_argument("--model_dtype", default=None, type=str,
help="force to convert the dtype, some backends supports fp16 dtype better")
@@ -180,25 +186,27 @@ if __name__ == '__main__':
is_glm = bool(re.search("chatglm", model_name.lower()))
low_cpu_mem_usage = False
model_cls = AutoModel if is_glm else AutoModelForCausalLM
+ if args.low_cpu_mem_tmp_dir is None:
+ args.low_cpu_mem_tmp_dir = os.path.join(args.output_dir, "low_cpu_mem_tmp")
if args.low_cpu_mem_mode == 2:
- from auto_round.layer_wise.utils import load_model_with_hooks
+ from auto_round.low_cpu_mem.utils import load_model_with_hooks
model = load_model_with_hooks(
model_name,
model_cls,
device=device_str,
clean_weight=True,
- saved_path=os.path.join(args.output_dir, "layer_wise_tmp"),
+ saved_path=args.low_cpu_mem_tmp_dir,
torch_dtype=torch_dtype,
trust_remote_code=not args.disable_trust_remote_code
)
elif args.low_cpu_mem_mode == 1:
- from auto_round.layer_wise.utils import load_empty_model
+ from auto_round.low_cpu_mem.utils import load_empty_model
low_cpu_mem_usage = True
model = load_empty_model(
model_name,
model_cls,
device=device_str,
- saved_path=os.path.join(args.output_dir, "layer_wise_tmp"),
+ saved_path=args.low_cpu_mem_tmp_dir,
torch_dtype=torch_dtype,
trust_remote_code=not args.disable_trust_remote_code
)
@@ -300,7 +308,7 @@ if __name__ == '__main__':
model_name = args.model_name.rstrip("/")
if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:
import shutil
- shutil.rmtree(os.path.join(args.output_dir, "layer_wise_tmp"), ignore_errors=True)
+ shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)
model.eval()
if "cpu" not in device_str:
diff --git a/test/test_low_cpu_mem.py b/test/test_low_cpu_mem.py
index 3f8dd0211..6f368dc70 100644
--- a/test/test_low_cpu_mem.py
+++ b/test/test_low_cpu_mem.py
@@ -6,7 +6,7 @@ sys.path.insert(0, "..")
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round.layer_wise.utils import (
+from auto_round.low_cpu_mem.utils import (
load_model_with_hooks,
load_empty_model,
get_layers_before_block,