Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Refine code (#291)

wenhuach21f41094a9dd32
diff --git a/README.md b/README.md
index 8ccbe5da3..e0d76b868 100644
--- a/README.md
+++ b/README.md
@@ -60,7 +60,8 @@ A user guide detailing the full list of supported arguments is provided by calli
Alternatively, you can use ```auto_round``` instead of ```auto-round```.
```bash
-auto-round --model facebook/opt-125m \
+CUDA_VISIBLE_DEVICES=0 auto-round \
+ --model facebook/opt-125m \
--bits 4 \
--group_size 128 \
--format auto_round \
@@ -74,7 +75,8 @@ We provide two recipes for best accuracy and fast running speed with low memory.
```bash
## best accuracy, 3X slower, low_gpu_mem_usage could save ~20G but ~30% slower
- auto-round --model facebook/opt-125m \
+CUDA_VISIBLE_DEVICES=0 auto-round \
+ --model facebook/opt-125m \
--bits 4 \
--group_size 128 \
--nsamples 512 \
@@ -85,7 +87,8 @@ We provide two recipes for best accuracy and fast running speed with low memory.
```bash
## fast and low memory, 2-3X speedup, slight accuracy drop at W4G128
- auto-round --model facebook/opt-125m \
+CUDA_VISIBLE_DEVICES=0 auto-round \
+ --model facebook/opt-125m \
--bits 4 \
--group_size 128 \
--nsamples 128 \
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 7ed64d1b3..4d0a8e05b 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -47,7 +47,7 @@ from .utils import (
to_dtype,
get_layer_names_in_block,
mv_module_from_gpu,
- unsupport_meta_device, detect_device_count,
+ unsupport_meta_device, detect_device_count, clear_memory,
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -153,12 +153,14 @@ class AutoRound(object):
):
self.quantized = False
self.model_orig_dtype = model.dtype
- self.low_cpu_mem_usage = low_cpu_mem_usage
+ self.seed = seed
+ set_seed(self.seed)
assert not unsupport_meta_device(model), (
"autoround does not support for params on meta device by transformers` interfaces,"
"please do not using device_map='auto' in model loading, "
"or follow examples/language-modeling/main.py to enable low_cpu_mem_usage")
- self.model = model.eval()
+
+ ## important tuning hype-parameters
self.amp = amp
self.enable_quanted_input = enable_quanted_input
self.enable_minmax_tuning = enable_minmax_tuning
@@ -166,51 +168,48 @@ class AutoRound(object):
self.nblocks = nblocks
self.bits = bits
self.enable_norm_bias_tuning = enable_norm_bias_tuning
- if self.enable_norm_bias_tuning is None:
- if self.bits == 2:
- self.enable_norm_bias_tuning = True
- else:
- self.enable_norm_bias_tuning = False
self.group_size = group_size
self.sym = sym
self.low_gpu_mem_usage = low_gpu_mem_usage
- self.data_type = data_type
- self.supported_types = [torch.nn.Linear, transformers.modeling_utils.Conv1D]
+ self.low_cpu_mem_usage = low_cpu_mem_usage
self.layer_config = {} if layer_config is None else layer_config
- self.seed = seed
- set_seed(self.seed)
- self.tokenizer = tokenizer
self.seqlen = seqlen
self.train_bs = batch_size
self.nblocks = nblocks
- self.device = detect_device(device)
- self.scale_dtype = convert_dtype_str2torch(scale_dtype)
- self.set_amp_dtype()
- self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
self.dataset = dataset
-
self.iters = iters
- self.quant_block_list = quant_block_list
if self.iters <= 0:
logger.warning("iters must be positive, reset it to 200")
self.iters = 200
- self.lr = lr or (1.0 / self.iters)
+ self.lr = lr or (1.0 / self.iters) ##must after iter setting
self.minmax_lr = minmax_lr or self.lr
+ ##activation
+ self.act_group_size = act_group_size if not (act_group_size is None) else self.group_size
+ self.act_bits = act_bits if not (act_bits is None) else self.bits
+ self.act_sym = act_sym if not (act_sym is None) else self.sym
+ self.act_dynamic = act_dynamic
+
+ self.data_type = data_type
+ self.supported_types = [torch.nn.Linear, transformers.modeling_utils.Conv1D]
+ self.model = model.eval()
+ self.tokenizer = tokenizer
+ self.device = detect_device(device)
+ self.scale_dtype = convert_dtype_str2torch(scale_dtype)
+ self.set_amp_dtype()
+ self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
+ self.quant_block_list = quant_block_list
+
self.sampler = sampler
self.gradient_accumulate_steps = gradient_accumulate_steps
self.not_use_best_mse = not_use_best_mse
self.dynamic_max_gap = dynamic_max_gap
self.lr_scheduler = lr_scheduler
self.optimizer = self.get_optimizer(None)
- self.share_attention_mask_flag = None
- self.hidden_dim_flag = None
+ self.share_attention_mask_flag = None ##TODO remove it later
self.infer_bs_coeff = 1
- self.act_group_size = act_group_size if not (act_group_size is None) else self.group_size
- self.act_bits = act_bits if not (act_bits is None) else self.bits
- self.act_sym = act_sym if not (act_sym is None) else self.sym
- self.act_dynamic = act_dynamic
- self.set_layerwise_config(self.layer_config)
+
+ self.set_layerwise_config(self.layer_config) ##better place in the end
torch.set_printoptions(precision=3, sci_mode=True)
self.check_configs()
logger.info(f"using {self.model.dtype} for quantization tuning")
@@ -281,8 +280,9 @@ class AutoRound(object):
for block_names in all_blocks:
inputs = all_inputs[block_names[0]]
all_inputs.pop(block_names[0])
- self.inputs = None
- del self.inputs
+
+ clear_memory(self.inputs)
+
if "input_ids" in inputs.keys():
total_samples = len(inputs["input_ids"])
self.n_samples = total_samples
@@ -290,7 +290,7 @@ class AutoRound(object):
self.train_bs = total_samples
logger.warning(f"force the train batch size to {total_samples}")
- torch.cuda.empty_cache()
+
self.quant_blocks(
self.model,
inputs,
@@ -381,18 +381,15 @@ class AutoRound(object):
self.model) ##self.model.hf_device_map has not been changed
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- torch.cuda.empty_cache()
+ clear_memory()
for layer_name in layer_names:
layer_input = layer_inputs[layer_name]
layer_input = to_device(layer_input, self.cache_device)
q_layer_input = q_layer_inputs[layer_name] if enable_quanted_input else None
q_layer_input = to_device(q_layer_input, self.cache_device)
self.quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
- for i in range(len(layer_input)):
- layer_input[i] = None
- if q_layer_input is not None:
- q_layer_input[i] = None
- torch.cuda.empty_cache()
+ del layer_input
+ clear_memory(q_layer_input)
def set_layerwise_config(self, layer_config):
"""Sets the layer-wise configuration based on the provided layer_config.
@@ -468,7 +465,8 @@ class AutoRound(object):
output.append(tmp_output)
else:
output.extend(list(torch.split(tmp_output, 1, dim=self.input_dim)))
- torch.cuda.empty_cache()
+ if self.low_gpu_mem_usage:
+ clear_memory()
return output
@@ -566,10 +564,10 @@ class AutoRound(object):
if self.low_cpu_mem_usage:
for n, m in embed_layers:
m = m.to("meta")
- # torch.cuda.empty_cache()
+
@torch.no_grad()
- def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=[], last_cache_name=None):
+ def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):
"""Attempts to cache intermediate data on GPU, if failed, then using CPU.
Args:
@@ -584,6 +582,8 @@ class AutoRound(object):
Raises:
Exception: If caching on GPU fails, switches to CPU and caches there.
"""
+ if layer_names is None:
+ layer_names = []
try:
if not self.model.device.type == "meta":
if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
@@ -594,7 +594,7 @@ class AutoRound(object):
block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
)
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- torch.cuda.empty_cache()
+ clear_memory()
except:
logger.info("switch to cpu to cache inputs")
if "lm_head" in self.layer_config and self.layer_config["lm_head"]["bits"] < 8:
@@ -603,14 +603,14 @@ class AutoRound(object):
f" for optimal performance during calibration when enabling lm-head quantization. "
f"Otherwise, the process may be significantly slower.")
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
- torch.cuda.empty_cache()
+ clear_memory()
all_inputs = self.cache_inter_data(
block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
)
return all_inputs
@torch.no_grad()
- def cache_inter_data(self, block_names, nsamples, layer_names=[], last_cache_name=None):
+ def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):
"""Save the inputs of block_name for calibration.
This method temporarily replaces the forward method of the model to capture
@@ -627,6 +627,8 @@ class AutoRound(object):
Returns:
dict: A dictionary containing the inputs for the specified block.
"""
+ if layer_names is None:
+ layer_names = []
self.inputs = {}
self.to_cached_layers = block_names + layer_names
tmp_dtype = None
@@ -727,8 +729,8 @@ class AutoRound(object):
self.inputs[name][key].append(to_device(kwargs[key], device=torch.device("cpu")))
elif key not in self.inputs[name].keys():
self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \
- if self.not_share_position_ids_flag \
- else to_device(kwargs[key], device=torch.device("cpu"))
+ if self.not_share_position_ids_flag \
+ else to_device(kwargs[key], device=torch.device("cpu"))
elif kwargs[key] is not None and self.not_share_position_ids_flag:
self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))
elif 'rotary_pos_emb' in key or 'cu_seqlens' in key:
@@ -745,7 +747,7 @@ class AutoRound(object):
self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))
elif key not in self.inputs[name].keys():
self.inputs[name][key] = to_device(kwargs[key], device=torch.device("cpu"))
-
+
if name == self.last_cache_name:
raise NotImplementedError
else:
@@ -849,6 +851,7 @@ class AutoRound(object):
pick_samples = train_bs * gradient_accumulate_steps
if self.sampler != "rand":
whole_indices = torch.randperm(nsamples)[:pick_samples]
+ total_loss = 0
for i in range(self.iters):
total_loss = 0
if self.sampler == "rand":
@@ -891,7 +894,7 @@ class AutoRound(object):
best_params = collect_best_params(wrapper_linear)
if not self.not_use_best_mse:
- if self.dynamic_max_gap > 0 and i - last_best_iter >= self.dynamic_max_gap:
+ if 0 < self.dynamic_max_gap <= i - last_best_iter:
break
self.step(scaler, optimizer, lr_schedule)
@@ -902,7 +905,7 @@ class AutoRound(object):
best_iter = last_best_iter
with torch.no_grad():
unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)
- layer = mv_module_from_gpu(layer, self.low_cpu_mem_usage)
+ mv_module_from_gpu(layer, self.low_cpu_mem_usage)
dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
logger.info(dump_info)
@@ -973,6 +976,7 @@ class AutoRound(object):
scaler = self.get_scaler() # pylint: disable=assignment-from-none
init_loss = None
best_params = {}
+ total_loss = 0
for i in range(self.iters):
total_loss = 0
if self.sampler == "rand":
@@ -1022,7 +1026,7 @@ class AutoRound(object):
best_params = collect_best_params(block)
if not self.not_use_best_mse:
- if self.dynamic_max_gap > 0 and i - last_best_iter >= self.dynamic_max_gap:
+ if 0 < self.dynamic_max_gap <= i - last_best_iter:
break
self.step(scaler, optimizer, lr_schedule)
@@ -1047,18 +1051,14 @@ class AutoRound(object):
block, input_ids, input_others, self.train_bs * self.infer_bs_coeff, device,
cache_device=self.cache_device
)
- block = mv_module_from_gpu(block, self.low_cpu_mem_usage)
- for i in range(len(input_ids)):
- input_ids[i] = None
- torch.cuda.empty_cache()
+ mv_module_from_gpu(block, self.low_cpu_mem_usage)
+ clear_memory(input_ids)
return q_outputs, output
else:
- block = mv_module_from_gpu(block, self.low_cpu_mem_usage)
- for i in range(len(input_ids)):
- input_ids[i] = None
- torch.cuda.empty_cache()
+ mv_module_from_gpu(block, self.low_cpu_mem_usage)
+ clear_memory(input_ids)
return None, output
def quant_blocks(
@@ -1082,13 +1082,13 @@ class AutoRound(object):
None
"""
q_input = None
- torch.cuda.empty_cache()
+ clear_memory()
for n, m in model.named_parameters():
m.requires_grad_(False)
input_ids = inputs["input_ids"]
inputs.pop("input_ids", None)
input_others = inputs
- torch.cuda.empty_cache()
+ clear_memory()
input_ids = to_device(input_ids, self.cache_device)
input_others = to_device(input_others, self.cache_device)
## as in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
@@ -1127,7 +1127,6 @@ class AutoRound(object):
device=device,
)
- torch.cuda.empty_cache()
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
del q_input
@@ -1135,7 +1134,7 @@ class AutoRound(object):
del input_others
del inputs
- torch.cuda.empty_cache()
+ clear_memory()
def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
"""Save the quantized model to the specified output directory in the specified format.
@@ -1175,7 +1174,7 @@ class AutoRound(object):
" particularly for 2-bit quantization and smaller models."
" We recommend exporting to either the AutoAWQ format (4 bits) or "
"the AutoRound format (2 bits) to enhance performance."
- )
+ )
if "awq" in format and not self.bits == 4:
raise ValueError("The AWQ format only supports W4 quantization ")
@@ -1452,8 +1451,6 @@ class AutoOPTRound(AutoRound):
**kwargs,
)
- if layer_config is None:
- layer_config = {}
self.optimizer = self.get_optimizer(optimizer)
def get_optimizer(self, optimizer):
@@ -1618,6 +1615,3 @@ class AutoAdamRound(AutoOPTRound):
optimizer=optimizer,
**kwargs,
)
-
-
-
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index 2ebd5ced3..6dc00c158 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -13,14 +13,14 @@
# limitations under the License.
import torch
-from .utils import floor_ste, round_ste
-from auto_round.data_type.register import register_dtype, QUANT_FUNC_WITH_DTYPE
+from .utils import round_ste
+from auto_round.data_type.register import register_dtype
@register_dtype("int_sym")
def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,
weight_max=None, q_scale_thresh=0.0, **kwargs):
- """Quantizes and dequantizes weight asymmetrically. full range, credict goes to llamacpp community
+ """Quantize and de-quantize weight asymmetrically. full range, credict goes to llamacpp community
Args:
weight: Tensor containing the weight to be quantized
@@ -30,9 +30,11 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dt
max_scale: Maximum scale coefficient for weight
weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
Returns:
- Quantized and dequantized weight, scale, zero-point
+ Quantized and de-quantized weight, scale, zero-point
"""
maxq = torch.tensor(2 ** (bits - 1))
if weight_min is None or weight_max is None:
@@ -41,13 +43,11 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dt
else:
wmin_tmp = weight_min
wmax_tmp = weight_max
- if isinstance(min_scale, torch.Tensor):
- wmin = wmin_tmp * min_scale
- wmax = wmax_tmp * max_scale
- else:
- wmin = wmin_tmp
- wmax = wmax_tmp
- max_v = (2 * (torch.abs(wmax) < torch.abs(wmin)).int() - 1) * torch.max(torch.abs(wmax), torch.abs(wmin))
+
+ wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130
+ wmax_abs = wmax_tmp * max_scale
+
+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)
scale = (max_v / maxq).to(scale_dtype)
scale = torch.where(scale < 0, torch.clamp(scale, max=-q_scale_thresh), torch.clamp(scale, min=q_scale_thresh))
@@ -63,7 +63,7 @@ def quant_tensor_sym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dt
@register_dtype("int_asym")
def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
weight_min=None, weight_max=None, q_scale_thresh=0.0, **kwargs):
- """Quantizes and dequantizes weight asymmetrically.
+ """Quantize and de-quantize weight asymmetrically.
Args:
weight: Tensor containing the weight to be quantized
@@ -73,9 +73,11 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_d
max_scale: Maximum scale coefficient for weight
weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
Returns:
- Quantized and dequantized weight, scale, zero-point
+ Quantized and de-quantized weight, scale, zero-point
"""
maxq = torch.tensor(2 ** bits - 1)
if weight_min is None or weight_max is None:
@@ -103,8 +105,8 @@ def quant_tensor_asym(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_d
@register_dtype("int_sym_gptq")
def quant_tensor_sym_gptq(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16, weight_min=None,
- weight_max=None, q_scale_thresh=0.0, **kwargs):
- """Quantizes and dequantizes weight symmetrically.
+ weight_max=None, q_scale_thresh=0.0, **kwargs):
+ """Quantize and de-quantize weight asymmetrically.
Args:
weight: Tensor containing the weight to be quantized
@@ -114,9 +116,11 @@ def quant_tensor_sym_gptq(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, sca
max_scale: Maximum scale coefficient for weight
weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
Returns:
- Quantized and dequantized weight, scale, zero-point
+ Quantized and de-quantized weight, scale, zero-point
"""
maxq = torch.tensor(2 ** bits - 1)
if weight_min is None or weight_max is None:
@@ -149,10 +153,9 @@ def quant_tensor_sym_gptq(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, sca
return qdq_result, scale, zp
-
def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,
weight_min=None, weight_max=None, q_scale_thresh=0.0, **kwargs):
- """Quantizes and dequantizes weight asymmetrically without rounding, this is mainly for tuning bias, norm.
+ """Quantize and de-quantize weight asymmetrically without rounding, this is mainly for tuning bias, norm.
Args:
weight: Tensor containing the weight to be quantized
@@ -162,9 +165,11 @@ def quant_tensor_asym_wo_round(weight, bits=4, v=0, min_scale=1.0, max_scale=1.0
max_scale: Maximum scale coefficient for weight
weight_min (Tensor, optional): Minimum weight value for quantization. Defaults to None.
weight_max (Tensor, optional): Maximum weight value for quantization. Defaults to None.
+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import
+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability
Returns:
- Quantized and dequantized weight, scale, zero-point
+ Quantized and de-quantize weight, scale, zero-point
"""
maxq = torch.tensor(2 ** bits - 1)
if weight_min is None or weight_max is None:
diff --git a/auto_round/quantizer.py b/auto_round/quantizer.py
index 411e1ab1d..97a714461 100644
--- a/auto_round/quantizer.py
+++ b/auto_round/quantizer.py
@@ -132,8 +132,7 @@ class WrapperLayerNorm(torch.nn.Module):
torch.zeros(self.orig_layer.weight.shape, device=self.device, dtype=weight_dtype),
self.group_size),
requires_grad=True)
- self.params = {}
- self.params["v"] = self.v
+ self.params = {"v": self.v}
from auto_round.data_type.int import quant_tensor_asym_wo_round
self.quant_func = quant_tensor_asym_wo_round
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 4d0da4a4b..ad8cdd52b 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -28,7 +28,7 @@ from torch.amp import autocast
from functools import lru_cache
from packaging import version
-
+import gc
@lru_cache(None)
def warning_once(self, msg: str):
@@ -368,7 +368,7 @@ def sampling_inputs(input_ids, input_others, indices, seqlen,
for key in input_others.keys():
if not share_attention_mask_flag and ("attention_mask" in key or "alibi" in key) \
or (not_share_position_ids_flag and ("position_ids" in key or \
- "cache_position" in key or "position_embeddings" in key)) \
+ "cache_position" in key or "position_embeddings" in key)) \
or (not_share_rotary_pos_emb_flag and ("rotary_pos_emb" in key or 'cu_seqlens' in key)) \
or "cross_attention_states" in key:
current_input_others[key] = None
@@ -403,23 +403,14 @@ def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.flo
input_ids = to_device(input_ids, device)
input_others = to_device(input_others, device)
input_tuple = input_others.pop("positional_inputs", None)
- if "alibi" in input_others.keys():
- alibi = input_others.pop("alibi")
- if alibi is not None:
- alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
- if amp:
- with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no cover
- output = block(
- input_ids, alibi=alibi, *input_tuple, **input_others
- ) ##TODO is this correct for all models with alibi?
- else:
- output = block(input_ids, alibi=alibi, *input_tuple, **input_others)
- else:
- if amp:
- with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no cover
- output = block(input_ids, *input_tuple, **input_others)
- else:
+ if "alibi" in input_others.keys() and input_others["alibi"] is not None:
+ alibi = input_others["alibi"]
+ input_others["alibi"] = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
+ if amp:
+ with autocast(device_type=device.split(":")[0], dtype=amp_dtype): # pragma: no cover
output = block(input_ids, *input_tuple, **input_others)
+ else:
+ output = block(input_ids, *input_tuple, **input_others)
if isinstance(output, list) or isinstance(output, tuple):
output = output[0]
return output
@@ -761,8 +752,6 @@ def is_autoround_exllamav2_available():
return res
-
-
def is_hpu_supported(): # pragma: no cover
try:
import subprocess
@@ -868,3 +857,10 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
use_marlin=not disable_marlin,
)
return QuantLinear
+
+
+def clear_memory(tensor=None):
+ if tensor is not None:
+ del tensor
+ gc.collect()
+ torch.cuda.empty_cache()