Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

open source delta loss (#1300)

Wenhua Cheng587eadb86775
diff --git a/auto_round/auto_scheme/__init__.py b/auto_round/auto_scheme/__init__.py
index 583086c6b..0ebf7d258 100644
--- a/auto_round/auto_scheme/__init__.py
+++ b/auto_round/auto_scheme/__init__.py
@@ -13,20 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.logger import logger
-
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
-
-
-def __getattr__(name):
- if name == "AUTO_SCHEME_METHODS":
- try:
- import auto_round.auto_scheme.default_alg
- except ImportError:
- logger.warning("AutoScheme is currently supported only on Linux.")
-
- from auto_round.auto_scheme.register import AUTO_SCHEME_METHODS
-
- return AUTO_SCHEME_METHODS
-
- raise AttributeError(f"auto-scheme has no attribute '{name}'")
+import auto_round.auto_scheme.delta_loss
+from auto_round.auto_scheme.register import AUTO_SCHEME_METHODS
diff --git a/auto_round/auto_scheme/default_alg.abi3.so b/auto_round/auto_scheme/default_alg.abi3.so
deleted file mode 100755
index e06c2094e..000000000
Binary files a/auto_round/auto_scheme/default_alg.abi3.so and /dev/null differ
diff --git a/auto_round/auto_scheme/default_alg.pyd b/auto_round/auto_scheme/default_alg.pyd
deleted file mode 100644
index 28a00b252..000000000
Binary files a/auto_round/auto_scheme/default_alg.pyd and /dev/null differ
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
new file mode 100644
index 000000000..701b9ac53
--- /dev/null
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -0,0 +1,1010 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+from dataclasses import asdict
+from functools import wraps
+from typing import Iterable, Union
+
+import torch
+from accelerate import dispatch_model
+from tqdm import tqdm
+
+from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+from auto_round.auto_scheme.register import register_scheme_methods
+from auto_round.auto_scheme.utils import (
+ apply_quant_scheme,
+ compute_avg_bits_for_scheme,
+ compute_layer_bits,
+ dispatch_model_by_all_available_devices,
+ parse_shared_layers,
+ remove_quant_scheme,
+)
+from auto_round.calib_dataset import get_dataloader
+from auto_round.data_type.gguf import (
+ quant_tensor_gguf_asym_dq,
+ quant_tensor_gguf_sym_dq,
+ search_gguf_scale_min_asym,
+ search_gguf_scale_min_sym,
+)
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.logger import logger
+from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
+from auto_round.utils import (
+ SUPPORTED_LAYER_TYPES,
+ check_to_quantized,
+ clear_memory,
+ get_block_names,
+ get_major_device,
+ get_module,
+ llm_load_model,
+ parse_available_devices,
+ set_avg_auto_device_map,
+ set_module,
+ set_non_auto_device_map,
+ to_device,
+)
+from auto_round.wrapper import WrapperLinear
+
+__all__ = ["gen_layer_config"]
+
+
+class AutoSchemeWrapperLinear(WrapperLinear):
+ def __init__(
+ self,
+ orig_layer,
+ enable_minmax_tuning=True,
+ enable_norm_bias_tuning=False,
+ device="cpu",
+ enable_round_tuning=True,
+ need_weight_grad=False,
+ enable_torch_compile=False,
+ **kwargs,
+ ):
+ super().__init__(
+ orig_layer,
+ enable_minmax_tuning,
+ enable_norm_bias_tuning,
+ device,
+ enable_round_tuning,
+ enable_torch_compile=enable_torch_compile,
+ **kwargs,
+ )
+ self.total_act_score = 0.0
+ self.act_score = 0.0
+ self.avg_act_score = 0.0
+ self.act_cnt = 0.0
+ self.weight_score = 0.0
+ self.mix_score = 0.0
+ self.super_qdq_func = super()._qdq_weight
+ self.act_qdq_func = super()._qdq_act
+ # self.device = device
+ self.max_act_value = 0
+ self.need_weight_grad = need_weight_grad
+ self.grad_mode = False
+ if self.need_weight_grad:
+ self.orig_layer.weight.requires_grad = True
+
+ def _qdq_act(self, x, act_max_scale, act_max=None):
+ if hasattr(self.orig_layer, "act_bits") and self.orig_layer.act_bits > 8:
+ return x, 1.0, None
+
+ qdq_x, scale, zp = self.act_qdq_func(x, act_max_scale, act_max)
+ if self.grad_mode:
+ with torch.no_grad():
+ self.max_act_value = torch.abs(x).max()
+ if torch.abs(x).max() != 0:
+ self.act_cnt += 1
+ x_diff = x - qdq_x
+ self.x_diff = x_diff.to("cpu")
+
+ def save_grad(grad):
+ if self.max_act_value == 0:
+ if torch.abs(grad).max() != 0:
+ raise ValueError
+ """
+ this ut will cause NAN issue sometimes, need to investigate
+ @multi_card
+ def test_multi_card(self):
+ model_name = "/models/Qwen3-8B"
+ """
+ if torch.isnan(grad).any() or torch.isnan(self.x_diff).any():
+ self.act_cnt -= 1
+ return None
+
+ self.total_act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()
+ self.act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
+ self.mix_score = self.weight_score + self.act_score
+ self.x_diff = None
+ return None
+
+ qdq_x.register_hook(save_grad)
+ return qdq_x, scale, zp
+
+ def _qdq_weight(self, value, min_scale, max_scale):
+ device = self.device
+ if self.orig_layer.bits > 8 or not self.need_weight_grad:
+ qdq_w, scale, zp = super()._qdq_weight(
+ torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
+ )
+
+ return qdq_w, 1.0, None
+
+ qdq_w, scale, zp = super()._qdq_weight(
+ torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
+ )
+
+ if self.grad_mode:
+
+ def save_grad(grad):
+ qdq_w, scale, zp = self.super_qdq_func(
+ torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device)
+ )
+ w_diff = self.orig_layer.weight - qdq_w.to(self.orig_layer.weight.device)
+ self.weight_score += torch.abs((grad.to(w_diff.device) * w_diff)).sum().item()
+ act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
+ self.mix_score = self.weight_score + act_score
+ return None
+
+ qdq_w.register_hook(save_grad)
+ return qdq_w, 1.0, None
+
+
+class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):
+ def __init__(
+ self,
+ orig_layer,
+ enable_minmax_tuning=True,
+ enable_norm_bias_tuning=False,
+ device="cpu",
+ enable_round_tuning=True,
+ need_weight_grad=False,
+ **kwargs,
+ ):
+ super().__init__(
+ orig_layer,
+ enable_minmax_tuning,
+ enable_norm_bias_tuning,
+ device,
+ enable_round_tuning,
+ need_weight_grad,
+ **kwargs,
+ )
+ with torch.no_grad():
+ qdq_w, scale, zp = self.super_qdq_func(
+ torch.tensor(0).to(device), torch.tensor(1.0).to(device), torch.tensor(1.0).to(device)
+ )
+ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
+
+ if self.need_weight_grad:
+
+ def save_grad(grad):
+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
+ # TODO strange, grad could be in CPU
+ self.weight_score += torch.abs((grad.to(w_diff.device) * w_diff)).sum().item() # TODO add 2nd order
+ act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
+ self.mix_score = self.weight_score + act_score
+ return None
+
+ self.qdq_w.requires_grad_(True)
+ self.orig_layer.weight.requires_grad_(False)
+
+ self.qdq_w.register_hook(save_grad)
+
+ def _qdq_weight(self, value, min_scale, max_scale):
+ return self.qdq_w, 1.0, None
+
+
+class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
+ def __init__(
+ self,
+ orig_layer,
+ enable_minmax_tuning=True,
+ enable_norm_bias_tuning=False,
+ device="cpu",
+ enable_round_tuning=True,
+ need_weight_grad=False,
+ enable_torch_compile=False,
+ **kwargs,
+ ):
+ super().__init__(
+ orig_layer,
+ enable_minmax_tuning,
+ enable_norm_bias_tuning,
+ device,
+ enable_round_tuning,
+ need_weight_grad,
+ enable_torch_compile=enable_torch_compile,
+ **kwargs,
+ )
+ with torch.no_grad():
+ qdq_w = self._init_scale().detach()
+ self.register_buffer("qdq_w", qdq_w.detach().clone().to(self.orig_layer.weight.device))
+ if self.need_weight_grad:
+
+ def save_grad(grad):
+ w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)
+ self.weight_score += torch.abs((grad * w_diff.to(grad.device))).sum().item()
+ act_score = 0.0 if self.act_cnt <= 0 else self.total_act_score / self.act_cnt
+ self.mix_score = self.weight_score + act_score
+ return None
+
+ self.qdq_w.requires_grad_(True)
+ self.orig_layer.weight.requires_grad_(False)
+
+ self.qdq_w.register_hook(save_grad)
+
+ @torch.no_grad()
+ def _init_scale(self):
+ tensor = self.orig_layer.weight.data
+ bits = self.orig_layer.bits
+ scale_dtype = self.orig_layer.scale_dtype
+ imatrix = self.orig_layer.imatrix
+ orig_dtype = tensor.dtype
+ if self.orig_layer.bits in [2, 4, 5]:
+ group_size = 16 if bits == 2 else 32
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(tensor, bits, scale_dtype, imatrix)
+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
+
+ qdq_w, _, _ = quant_tensor_gguf_asym_dq(
+ tensor=tensor,
+ bits=bits,
+ scale_dtype=scale_dtype,
+ imatrix=imatrix,
+ scale=scale,
+ wmin=wmin,
+ d_scale=d_scale,
+ d_wmin=d_wmin,
+ )
+ elif bits in [3, 6]:
+ group_size = 16
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ scale, d_scale = search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num=1)
+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
+ qdq_w, _, _ = quant_tensor_gguf_sym_dq(
+ tensor=tensor, bits=bits, scale_dtype=scale_dtype, imatrix=imatrix, scale=scale, d_scale=d_scale
+ )
+ else:
+ raise ValueError("bits must be in [2,3,4,5,6]")
+ return qdq_w.to(orig_dtype)
+
+ def _qdq_weight(self, value, min_scale, max_scale):
+ return self.qdq_w, 1.0, None
+
+
+def cal_imatrix(model, dataloader):
+ def register_act_hook(model):
+ """Registers hooks to accumulate activation squared norms into `imatrix`."""
+
+ def get_imatrix_hook(module, input, output):
+ input = input[0] if isinstance(input, (tuple, list)) else input
+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
+ squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)
+
+ if not hasattr(module, "imatrix"):
+ module.imatrix = squared.to("cpu")
+ else:
+ module.imatrix += squared.to(module.imatrix.device).to("cpu")
+
+ hook_handles = []
+ for name, module in model.named_modules():
+ if isinstance(module, SUPPORTED_LAYER_TYPES):
+ hook = module.register_forward_hook(get_imatrix_hook)
+ hook_handles.append(hook)
+ return hook_handles
+
+ hooks = register_act_hook(model)
+ for data in dataloader:
+ model.forward(**to_device(data, model.device))
+ for hook in hooks:
+ hook.remove()
+
+
+class MyCustomError(Exception):
+
+ def __init__(self, message):
+ super().__init__(message)
+
+
+last_grad_input = None
+
+
+def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_device="cpu"):
+ block_inputs.clear()
+ for n, m in model.named_modules():
+ if hasattr(m, "grad_mode"):
+ m.grad_mode = False
+
+ block_names = get_block_names(model)[0]
+
+ def wrap_forward(module, module_name):
+ original_forward = module.forward
+
+ @wraps(original_forward)
+ def new_forward(*args, **kwargs):
+ move_module_to_tuning_device(module, major_device=major_device)
+
+ # Call the original forward
+ with torch.no_grad():
+ result = original_forward(*args, **kwargs)
+
+ # Save input information and ensure tensors are on CPU
+ input_info = {
+ "args": [arg.detach().clone().to("cpu") if isinstance(arg, torch.Tensor) else arg for arg in args],
+ "kwargs": {
+ k: v.detach().clone().to("cpu") if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()
+ },
+ }
+ block_inputs[module_name] = input_info
+
+ module.to("cpu")
+ # torch.cuda.empty_cache()
+
+ # Enable gradients for the output of the last block
+ if module.tmp_name == block_names[-1]:
+ if isinstance(result, torch.Tensor):
+ result = result.requires_grad_(True)
+ elif isinstance(result, tuple):
+ result = tuple(r.requires_grad_(True) if isinstance(r, torch.Tensor) else r for r in result)
+
+ pbar.update(1)
+ return result
+
+ return new_forward
+
+ # Assign a temporary name to each module
+ for n, m in model.named_modules():
+ m.tmp_name = n
+
+ # Wrap the forward method of each block
+ for block_name in block_names:
+ module = get_module(model, block_name)
+ module.forward = wrap_forward(module, block_name)
+
+
+def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
+ block_inputs = {}
+
+ block_names = get_block_names(model)[0]
+ for name in block_names:
+ module = get_module(model, name)
+ module.orig_forward = module.forward
+
+ def backward_pre_hook(module, grad_input):
+ """Hook executed before backward propagation."""
+ global last_grad_input
+ last_grad_input = grad_input
+ if torch.cuda.is_available():
+ torch.cuda.synchronize()
+ raise MyCustomError("Interrupt backward pass")
+
+ for data in dataloader:
+ prepare_model_low_gpu(model, block_inputs, major_device=major_device, pbar=pbar)
+
+ # Register backward hook on the last block
+ last_block = get_module(model, block_names[-1])
+ last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)
+
+ data = to_device(data, model.device)
+ output = model.forward(**data, labels=data["input_ids"], use_cache=False)
+
+ try:
+ # Backward pass (will be interrupted by the hook)
+ output.loss.backward()
+ except MyCustomError:
+ pass
+
+ current_grad = last_grad_input
+
+ # Manually compute gradients block by block
+ last_block_backward_hook.remove()
+
+ for name in block_names:
+ module = get_module(model, name)
+ module.forward = module.orig_forward
+
+ for block_name in reversed(block_names):
+
+ # Retrieve stored inputs for the block
+ block_input_info = block_inputs.get(block_name, {})
+
+ block_input_args = to_device(block_input_info.get("args", []), major_device)
+ block_input_kwargs = to_device(block_input_info.get("kwargs", {}), major_device)
+ block_input_args[0].requires_grad_(True)
+
+ # Move the block module to GPU
+ block_module = get_module(model, block_name)
+ for n, m in block_module.named_modules():
+ if hasattr(m, "grad_mode"):
+ m.grad_mode = True
+ if hasattr(m, "orig_layer"):
+ m.to(m.orig_layer.tuning_device)
+ elif hasattr(m, "tuning_device"):
+ m.to(m.tuning_device)
+ elif len(list(m.children())) == 0:
+ m.to(major_device)
+
+ # Set the block to eval mode while enabling gradient computation
+ block_module.eval()
+
+ # Recompute the block output
+ block_output = block_module(*block_input_args, **block_input_kwargs)
+
+ # Ensure the output requires gradients
+ if isinstance(block_output, tuple):
+ # For tuple outputs, we usually care about the first element (hidden states)
+ main_output = block_output[0]
+ main_output = main_output.requires_grad_(True)
+ else:
+ main_output = block_output.requires_grad_(True)
+
+ # Backward pass for the current block
+ torch.autograd.backward(
+ tensors=main_output,
+ # inputs=block_input_args,
+ grad_tensors=current_grad,
+ retain_graph=True, # False may lead to zero gradients for some cases (e.g., MXFP4)
+ )
+
+ # Extract gradients w.r.t. the block input
+ if block_input_args and isinstance(block_input_args[0], torch.Tensor):
+ current_grad = block_input_args[0].grad.detach().clone()
+ else:
+ print(f"Warning: No suitable input gradient found for {block_name}")
+ break
+
+ block_module.to("cpu")
+ # clear_memory()
+ pbar.update(1)
+
+
+def get_score_for_scheme(
+ model,
+ tokenizer,
+ quant_layer_names,
+ fixed_layer_scheme,
+ dataset,
+ ignore_scale_zp_bits=False,
+ nsamples=16,
+ seqlen=256,
+ pbar=None,
+ shared_layers=None,
+ need_weight_grad=False,
+ enable_torch_compile=False,
+ low_gpu_mem_usage=True,
+ major_device="cpu",
+ batch_size=1,
+ disable_opt_rtn=True,
+):
+ scores_dict = {} # Key=name,Val=[quant_total_bits, loss]
+ for n, m in model.named_modules():
+ if type(m) in SUPPORTED_LAYER_TYPES:
+ m.weight.requires_grad = False
+ if hasattr(m, "bias") and m.bias is not None:
+ m.bias.requires_grad = False
+
+ has_imatrix = False
+ for name in quant_layer_names:
+ if name in fixed_layer_scheme.keys():
+ continue
+ m = get_module(model, name)
+ if hasattr(m, "imatrix") and m.imatrix is not None:
+ has_imatrix = True
+ break
+
+ for name in quant_layer_names:
+ if name in fixed_layer_scheme.keys():
+ continue
+ m = get_module(model, name)
+ if not check_to_quantized(m):
+ layer_bits, _ = compute_layer_bits(m, ignore_scale_zp_bits)
+ scores_dict[name] = [layer_bits, 0.0]
+ continue
+ if m.act_bits > 8 and m.super_bits is not None:
+ m.scale_dtype = torch.float32 # TODO set this via API
+ elif m.act_bits > 8:
+ m.scale_dtype = torch.float16
+ else:
+ m.scale_dtype = torch.bfloat16
+
+ WrapperLayer = AutoSchemeWrapperLinear
+ if hasattr(m, "super_group_size") and m.super_group_size is not None:
+ if has_imatrix:
+ WrapperLayer = AutoSchemeWrapperLinearForGGUFKImatrix
+ else:
+ WrapperLayer = AutoSchemeWrapperLinearForGGUFK
+
+ with torch.no_grad():
+ if low_gpu_mem_usage:
+ device = m.tuning_device if hasattr(m, "tuning_device") else major_device
+ if "cuda" in device or "xpu" in device:
+ device = major_device
+ else:
+ device = m.weight.device
+ m.tuning_device = m.weight.device
+
+ new_m = WrapperLayer(
+ m,
+ device=device,
+ enable_minmax_tuning=False, # TODO this should be change
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ need_weight_grad=need_weight_grad,
+ enable_torch_compile=enable_torch_compile,
+ disable_opt_rtn=disable_opt_rtn,
+ )
+ set_module(model, name, new_m)
+ if low_gpu_mem_usage:
+ dataloader = get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)
+
+ model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar)
+ else:
+ dataloader = get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)
+ for data in dataloader:
+ data = to_device(data, model.device)
+ output = model.forward(**data, labels=data["input_ids"], use_cache=False)
+ output.loss.backward()
+ for n, m in model.named_parameters(): # This should be kept to reduce VRAM footprint
+ m.grad = None
+ if pbar is not None:
+ pbar.update(1)
+
+ for n, m in model.named_parameters():
+ m.grad = None
+
+ scores_dict = {}
+ for n, m in model.named_modules():
+ if hasattr(m, "mix_score"):
+ if m.orig_layer.act_bits <= 8:
+ if m.act_cnt == 0:
+ logger.warning_once(
+ "layer{n} max abs activation is 0, please use more data to improve the accuracy"
+ )
+ layer_bits, _ = compute_layer_bits(m.orig_layer, ignore_scale_zp_bits=ignore_scale_zp_bits)
+ scores_dict[n] = [layer_bits, m.mix_score]
+ for n, m in model.named_modules():
+ if hasattr(m, "orig_layer"):
+ set_module(model, n, m.orig_layer)
+ return scores_dict
+
+
+def choose_bits_per_layer_with_path(layers: dict, P: int):
+ """
+ Args:
+ layers: A dict mapping each layer name to a list of candidate options.
+ Each option is a tuple of (scheme, bits_cost, loss_cost, layer_names).
+ P: Upper bound on the total parameter (bit) budget.
+
+ Returns:
+ (min_loss, best_path), where best_path is a list of
+ (layer_names, scheme) for each layer, or (None, None) if no feasible
+ solution exists.
+ """
+ # dp: total_params -> (accumulated_loss, chosen_path)
+ # The path explicitly stores the selected options.
+ dp: dict[int, tuple[float, list]] = {0: (0.0, [])}
+
+ for layer_name, opts in layers.items():
+ new_dp: dict[int, tuple[float, list]] = {}
+ for cur_params, (cur_loss, cur_path) in dp.items():
+ for opt in opts:
+ scheme, bits_cost, loss_cost, layer_names = opt
+ np_total = cur_params + bits_cost
+ if np_total > P:
+ continue
+
+ new_loss = cur_loss + loss_cost
+ new_path = cur_path + [(layer_names, scheme)]
+
+ # Keep the path with smaller loss for the same parameter budget
+ if np_total not in new_dp or new_loss < new_dp[np_total][0]:
+ new_dp[np_total] = (new_loss, new_path)
+
+ if not new_dp:
+ return None, None
+
+ # Pareto pruning: remove dominated (params, loss) states
+ items = sorted(new_dp.items(), key=lambda x: x[0]) # (params, (loss, path))
+ pruned: dict[int, tuple[float, list]] = {}
+ best_loss_so_far = float("inf")
+ for params_val, (loss_val, path_val) in items:
+ if loss_val < best_loss_so_far:
+ pruned[params_val] = (loss_val, path_val)
+ best_loss_so_far = loss_val
+
+ dp = pruned
+
+ # Select the solution with the minimum loss
+ best_params = min(dp.keys(), key=lambda k: dp[k][0])
+ best_loss, best_path = dp[best_params]
+ return best_loss, best_path
+
+
+def move_module_to_tuning_device(module, major_device="cpu"):
+ for n, m in module.named_modules():
+ if hasattr(m, "orig_layer"):
+ m.to(m.orig_layer.tuning_device)
+ elif hasattr(m, "tuning_device"):
+ m.to(m.tuning_device)
+ elif len(list(m.children())) == 0:
+ m.to(major_device)
+
+
+def _gen_layer_config(
+ auto_scheme: AutoScheme,
+ model: Union[str, torch.nn.Module],
+ quant_layer_names: Iterable[str],
+ fixed_layer_scheme: dict[str, dict],
+ min_avg_bit_scheme,
+ dataset: str = "pile-10k",
+ tokenizer=None,
+ device_map=None,
+ enable_torch_compile=False,
+ disable_opt_rtn=True,
+ model_name=None,
+ major_device="cpu",
+ device_list=None,
+):
+ target_bits = auto_scheme.avg_bits
+ model.eval()
+
+ block_name = get_block_names(model)[0] # TODO need change to support vlm
+ for name in block_name:
+ module = get_module(model, name)
+ module.in_block = True
+ for n, m in module.named_modules():
+ m.in_block = True
+
+ for n, m in model.named_modules():
+ if len(list(m.children())) == 0:
+ if not hasattr(m, "in_block"):
+ m.in_block = False
+ if not m.in_block and auto_scheme.low_gpu_mem_usage:
+ m.to(major_device)
+
+ total_scores = {}
+ schemes = auto_scheme.options
+
+ def check_bf16_scheme(scheme):
+ if isinstance(scheme, str) and scheme.upper() == "BF16":
+ return True
+ if isinstance(scheme, QuantizationScheme):
+ scheme = asdict(scheme)
+ if scheme["bits"] >= 16 and scheme["act_bits"] >= 16:
+ return True
+ return False
+
+ if auto_scheme.nsamples is not None:
+ nsamples = auto_scheme.nsamples
+ else:
+ is_moe_model = False
+ if hasattr(model, "config"):
+ for key in model.config.to_dict().keys():
+ if "moe" in key or "expert" in key:
+ is_moe_model = True
+ break
+ if is_moe_model:
+ logger.info(
+ "The model appears to be an MoE model. "
+ "Using more samples to help generate a better auto-scheme recipe."
+ )
+ nsamples = 64
+ else:
+ nsamples = 16
+
+ seqlen = auto_scheme.seqlen if auto_scheme.seqlen is not None else 256
+
+ if auto_scheme.batch_size is not None:
+ batch_size = auto_scheme.batch_size
+ else:
+ if auto_scheme.low_gpu_mem_usage:
+ batch_size = 8
+ else:
+ batch_size = 1
+
+ pbar_cnt = 0
+ need_weight_grad = False
+ for index, scheme in enumerate(schemes):
+ if check_bf16_scheme(scheme):
+ continue
+ if isinstance(scheme, str):
+ scheme = asdict(preset_name_to_scheme(scheme))
+ elif isinstance(scheme, QuantizationScheme):
+ scheme = asdict(scheme)
+ bits = scheme.get("bits", 16)
+ act_bits = scheme.get("act_bits", 16)
+ if bits <= 8 < act_bits:
+ need_weight_grad = True
+ if not auto_scheme.low_gpu_mem_usage:
+ pbar_cnt += nsamples
+ if auto_scheme.low_gpu_mem_usage:
+ pbar_cnt += len(block_name) * 2 * ((nsamples + batch_size - 1) // batch_size) # forward backward
+ shared_layers = parse_shared_layers(model, auto_scheme.shared_layers)
+
+ embedding_layers_names = []
+ for name in quant_layer_names:
+ module = get_module(model, name)
+ if isinstance(module, torch.nn.Embedding):
+ embedding_layers_names.append(name)
+ quant_layer_names = list(set(quant_layer_names) - set(embedding_layers_names))
+
+ options_scores = []
+ if auto_scheme.low_gpu_mem_usage and not disable_opt_rtn:
+ logger.warning("low_gpu_mem_usage is enabled, disable_opt_rtn will be set to True automatically")
+ disable_opt_rtn = True
+ if not disable_opt_rtn:
+ need_imatrix = False
+ for scheme in schemes:
+ if isinstance(scheme, str):
+ scheme = asdict(preset_name_to_scheme(scheme))
+ elif isinstance(scheme, QuantizationScheme):
+ scheme = asdict(scheme)
+
+ need_imatrix = scheme["super_bits"] is not None
+ if need_imatrix:
+ break
+ if need_imatrix: # TODO change to block way in low_gpu_mem_usage
+ dataloader = get_dataloader(tokenizer, seqlen=256, dataset_name=dataset, seed=42, bs=8, nsamples=16)
+ logger.info("start to compute imatrix for GGUF-K quantization in AutoScheme")
+ cal_imatrix(model, dataloader)
+ logger.info("finish calculating imatrix")
+
+ pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")
+ for index, scheme in enumerate(schemes):
+ apply_quant_scheme(
+ model, quant_layer_names=quant_layer_names, fixed_layer_scheme=fixed_layer_scheme, scheme=scheme
+ )
+ scores = {} # name: bits, loss
+ if check_bf16_scheme(scheme):
+ for n in quant_layer_names:
+ if n in fixed_layer_scheme.keys():
+ continue
+ m = get_module(model, n)
+ bits, _ = compute_layer_bits(m, auto_scheme.ignore_scale_zp_bits)
+ scores[n] = [bits, 0.0]
+ else:
+ scores = get_score_for_scheme(
+ model,
+ tokenizer,
+ quant_layer_names,
+ fixed_layer_scheme,
+ dataset,
+ ignore_scale_zp_bits=auto_scheme.ignore_scale_zp_bits,
+ pbar=pbar,
+ nsamples=nsamples,
+ seqlen=seqlen,
+ need_weight_grad=need_weight_grad,
+ enable_torch_compile=enable_torch_compile,
+ low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,
+ major_device=major_device,
+ batch_size=batch_size,
+ disable_opt_rtn=auto_scheme.disable_opt_rtn,
+ )
+ new_scores = {}
+ for share_layer in shared_layers:
+ param_bits = 0
+ tmp_loss = 0
+ name_list = []
+ for name in share_layer:
+ if name in scores.keys():
+ param_bits += scores[name][0]
+ tmp_loss += scores[name][1]
+ name_list.append(name)
+ scores.pop(name)
+ new_scores[name_list[0]] = [index, param_bits, tmp_loss, name_list]
+ for name, item in scores.items():
+ new_scores[name] = [index, item[0], item[1], [name]]
+ options_total_loss = 0.0
+ for key, item in new_scores.items():
+ options_total_loss += item[2]
+ if key in total_scores:
+ total_scores[key].append(item)
+ else:
+ total_scores[key] = [item]
+ options_scores.append(options_total_loss)
+ clear_memory(device_list=device_list)
+
+ total_params = 0
+ for n, m in model.named_modules():
+ if n in quant_layer_names + embedding_layers_names:
+ total_params += m.weight.numel()
+
+ target_params_cnt = int(total_params * target_bits)
+ sorted_indices = sorted(range(len(options_scores)), key=lambda i: options_scores[i])
+ # Layers that are not fixed in fixed_layer_scheme
+ not_fixed_embedding_layers_names = [name for name in embedding_layers_names if name not in fixed_layer_scheme]
+ for sorted_index in sorted_indices:
+ tmp_scheme = schemes[sorted_index]
+ if isinstance(tmp_scheme, str):
+ tmp_scheme = asdict(preset_name_to_scheme(tmp_scheme))
+
+ for embedding_layer_name in not_fixed_embedding_layers_names:
+
+ fixed_layer_scheme[embedding_layer_name] = tmp_scheme
+ embedding_layer = get_module(model, embedding_layer_name)
+ for key, item in tmp_scheme.items():
+ setattr(embedding_layer, key, item)
+ current_avg_bits, _ = compute_avg_bits_for_scheme(
+ model,
+ quant_layer_names + embedding_layers_names,
+ fixed_layer_scheme,
+ min_avg_bit_scheme,
+ ignore_scale_zp_bits=auto_scheme.ignore_scale_zp_bits,
+ clean_scheme=False,
+ )
+
+ if current_avg_bits <= target_bits: # compute_avg_bit remove setting
+ for embedding_layer_name in not_fixed_embedding_layers_names:
+
+ fixed_layer_scheme[embedding_layer_name] = tmp_scheme
+ embedding_layer = get_module(model, embedding_layer_name)
+ for key, item in tmp_scheme.items():
+ setattr(embedding_layer, key, item)
+ break
+
+ # Minus fixed_layer
+ for name in fixed_layer_scheme.keys(): # The Scheme should have been applied
+ m = get_module(model, name)
+ layer_bits, _ = compute_layer_bits(m, auto_scheme.ignore_scale_zp_bits)
+ target_params_cnt -= layer_bits
+ if target_params_cnt <= 0:
+ raise ValueError("Avg bits is too small")
+
+ remove_quant_scheme(model) # Must place after minus fixed_layer
+
+ best_loss, best_path = choose_bits_per_layer_with_path(total_scores, target_params_cnt)
+ # print(best_loss, best_path) # TODO better log
+ layer_config = copy.deepcopy(fixed_layer_scheme)
+ options = copy.deepcopy(auto_scheme.options)
+ # Replace scheme preset names with actual QuantizationScheme objects
+ for index in range(len(options)):
+ if isinstance(options[index], str):
+ options[index] = preset_name_to_scheme(options[index])
+ for item in best_path:
+ layer_names = item[0]
+ layer_scheme = options[item[1]]
+ for layer_name in layer_names:
+ layer_config[layer_name] = asdict(layer_scheme)
+
+ if model_name is not None:
+ model = None
+ del model
+ else:
+ model = model.to("cpu") # TODO this requires large ram
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+
+ accelerate.hooks.remove_hook_from_submodules(model)
+ delattr(model, "hf_device_map")
+ for n, m in model.named_modules():
+ if hasattr(m, "scale_dtype"): # TODO refine code
+ delattr(m, "scale_dtype")
+ if hasattr(m, "imatrix"):
+ delattr(m, "imatrix")
+ if hasattr(m, "tuning_device"):
+ delattr(m, "tuning_device")
+ for n, m in model.named_parameters():
+ if hasattr(m, "grad"):
+ m.grad = None
+ global last_grad_input
+ last_grad_input = None
+ clear_memory(device_list=device_list)
+ pbar.close()
+ return layer_config
+
+
+# Supports model with gradient clearing between iterations
+@register_scheme_methods(("default", "DeltaLoss"))
+def gen_layer_config(
+ auto_scheme: AutoScheme,
+ model: Union[str, torch.nn.Module],
+ quant_layer_names: Iterable[str],
+ fixed_layer_scheme: dict[str, dict],
+ dataset: str = "pile-10k",
+ tokenizer=None,
+ device_map=None,
+ enable_torch_compile=False,
+ disable_opt_rtn=True,
+ low_gpu_mem_usage=True,
+ min_avg_bit_scheme=None,
+ **kwargs,
+):
+ model_name = None
+ if isinstance(model, str):
+ model_name = model
+ # Load model on CPU only; do not apply automatic device map or tuning-aware placement at load time.
+ model, tokenizer, _ = llm_load_model(model_name, device_map="cpu")
+ # Get major device
+ major_device = get_major_device(device_map)
+ if not low_gpu_mem_usage:
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ model = dispatch_model(model, device_map=model.hf_device_map)
+ else:
+ model = dispatch_model_by_all_available_devices(model, device_map)
+ else:
+ model.to("cpu")
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+
+ accelerate.hooks.remove_hook_from_submodules(model)
+ if (isinstance(device_map, str) and "," in device_map) or device_map == "auto":
+ set_avg_auto_device_map(model, device_map)
+ else:
+ set_non_auto_device_map(model, device_map)
+
+ for n in quant_layer_names:
+ m = get_module(model, n)
+ if not hasattr(m, "tuning_device"):
+ m.tuning_device = major_device
+
+ device_list = parse_available_devices(device_map)
+
+ # Enable gradient checkpointing if supported
+ if model.supports_gradient_checkpointing:
+ model.gradient_checkpointing_enable()
+
+ for name in quant_layer_names:
+ m = get_module(model, name)
+ m.tmp_name = name
+
+ try:
+ res = _gen_layer_config(
+ auto_scheme,
+ model,
+ quant_layer_names,
+ fixed_layer_scheme,
+ dataset=dataset,
+ tokenizer=tokenizer,
+ model_name=model_name,
+ enable_torch_compile=enable_torch_compile,
+ disable_opt_rtn=disable_opt_rtn,
+ device_map=device_map,
+ major_device=major_device,
+ device_list=device_list,
+ min_avg_bit_scheme=min_avg_bit_scheme,
+ )
+ except torch.OutOfMemoryError:
+ logger.warning(
+ "Fallback to CPU for automatic scheme generation."
+ " Using multiple devices is strongly recommended (e.g., --device_map 0,1,2,3)."
+ )
+ model.to("cpu")
+ for n, m in model.named_modules():
+ if hasattr(m, "orig_layer"):
+ set_module(model, n, m.orig_layer)
+ clear_memory(device_list=device_list)
+ if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
+ import accelerate
+
+ accelerate.hooks.remove_hook_from_submodules(model)
+ delattr(model, "hf_device_map")
+ res = _gen_layer_config(
+ auto_scheme,
+ model,
+ quant_layer_names,
+ fixed_layer_scheme,
+ dataset=dataset,
+ tokenizer=tokenizer,
+ model_name=model_name,
+ enable_torch_compile=enable_torch_compile,
+ disable_opt_rtn=disable_opt_rtn,
+ device_map=device_map,
+ major_device=major_device,
+ device_list=device_list,
+ min_avg_bit_scheme=min_avg_bit_scheme,
+ )
+
+ return res