Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

increase precision of loss logging in quantization tuning for debugging (#2229)

Xin He385a48c94afa
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index 4e4aeed57..1df932c83 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -507,6 +507,8 @@ class SignRoundQuantizer(BaseQuantizer):
if i == 0:
init_loss = total_loss
+ current_lr = optimizer.param_groups[0]["lr"]
+ logger.debug("iter %d loss: %.3e lr: %s", i, total_loss, current_lr)
if total_loss < best_loss:
best_loss = total_loss
@@ -530,7 +532,7 @@ class SignRoundQuantizer(BaseQuantizer):
if self.iters > 0:
dump_info = (
f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
- f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
+ f"layers in the block, loss iter 0: {init_loss:.3e} -> iter {best_iter}: {last_loss:.3e}"
)
else:
dump_info = (
@@ -733,6 +735,8 @@ class SignRoundQuantizer(BaseQuantizer):
self._scale_loss_and_backward(scaler, loss)
if i == 0:
init_loss = total_loss
+ current_lr = optimizer.param_groups[0]["lr"]
+ logger.debug("iter %d loss: %.3e lr: %s", i, total_loss, current_lr)
if total_loss < best_loss:
best_loss = total_loss
@@ -755,7 +759,7 @@ class SignRoundQuantizer(BaseQuantizer):
with torch.no_grad():
unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)
mv_module_from_gpu(layer)
- dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
+ dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.3e} -> iter {best_iter}: {last_loss:.3e}"
logger.info(dump_info)
def finalize_run(self) -> None:
diff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.py
index c816c3d98..b67b84073 100644
--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py
+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py
@@ -587,7 +587,7 @@ class SpinQuantPreprocessor:
logger.info(
f"[SpinQuant] Training complete: {result.steps} steps, "
- f"best_loss={result.best_loss:.6f}, ortho_dev={result.final_ortho_deviation:.2e}"
+ f"best_loss={result.best_loss:.3e}, ortho_dev={result.final_ortho_deviation:.2e}"
)
del original_model
diff --git a/auto_round/algorithms/transforms/spinquant/training.py b/auto_round/algorithms/transforms/spinquant/training.py
index 618bfa7b1..e61793315 100644
--- a/auto_round/algorithms/transforms/spinquant/training.py
+++ b/auto_round/algorithms/transforms/spinquant/training.py
@@ -286,7 +286,7 @@ def run_training_loop(
on_step_end(step, loss_val, avg)
elif step % log_interval == 0:
avg = sum(loss_history[-50:]) / len(loss_history[-50:])
- logger.info(f"[SpinQuant] Step {step}/{max_iters}, loss={loss_val:.6f} (avg={avg:.6f})")
+ logger.info(f"[SpinQuant] Step {step}/{max_iters}, loss={loss_val:.3e} (avg={avg:.3e})")
step += 1
@@ -569,7 +569,7 @@ class LossLogger(RotationTrainerCallback):
if step % self.log_interval == 0:
loss = state.get("loss", 0.0)
avg_loss = state.get("avg_loss", 0.0)
- print(f" [LossLogger] step={step}/{args.iters} loss={loss:.6f} avg={avg_loss:.6f}")
+ print(f" [LossLogger] step={step}/{args.iters} loss={loss:.3e} avg={avg_loss:.3e}")
class RotationTrainer:
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index c7207bea6..f407127ae 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -3205,7 +3205,7 @@ def _gen_layer_config(
)
total_loss = _record_scheme_scores(index, per_op_scores)
logger.info(
- "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",
+ "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.3e)",
index + 1,
len(schemes),
total_loss,
@@ -3314,7 +3314,7 @@ def _gen_layer_config(
total_loss = _record_scheme_scores(index, per_op_scores)
logger.info(
- "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",
+ "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.3e)",
index + 1,
len(schemes),
total_loss,
diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.py
index b987a263d..c8e4f5e2a 100644
--- a/auto_round/auto_scheme/utils.py
+++ b/auto_round/auto_scheme/utils.py
@@ -561,7 +561,7 @@ def _log_score_summary_by_block_and_nonblock(
for name, avg_loss in rows_in_order:
rank = rank_map.get(name, -1)
ratio_pct = 0.0 if abs(total_avg_loss) < _ZERO_EPS else (avg_loss / total_avg_loss) * 100.0
- logger.debug("AutoScheme | %d | %s | %.6f | %.2f%% |", rank, name, avg_loss, ratio_pct)
+ logger.debug("AutoScheme | %d | %s | %.3e | %.2f%% |", rank, name, avg_loss, ratio_pct)
if head_name is not None and head_name not in scores_dict:
logger.debug("AutoScheme | - | %s | N/A | N/A |", head_name)
@@ -569,7 +569,7 @@ def _log_score_summary_by_block_and_nonblock(
if non_block_items:
non_block_items.sort(key=lambda x: x[0])
for layer_name, layer_loss in non_block_items:
- logger.info("AutoScheme non_block=%s loss=%.6f", layer_name, layer_loss)
+ logger.info("AutoScheme non_block=%s loss=%.3e", layer_name, layer_loss)
def _collect_current_scores(model):
@@ -611,7 +611,7 @@ def _log_batch_avg_loss(model, batch_idx: int, pbar=None, block_names=None, tota
avg_loss = 0.0 if layer_cnt == 0 else total_loss / layer_cnt
tag = f"[{scheme_tag}] " if scheme_tag else ""
batch_str = f"{batch_idx}/{total_batches}" if total_batches is not None else str(batch_idx)
- msg = f"AutoScheme {tag}cumulative batch {batch_str} avg_loss={avg_loss:.6f} layers={layer_cnt}"
+ msg = f"AutoScheme {tag}cumulative batch {batch_str} avg_loss={avg_loss:.3e} layers={layer_cnt}"
if pbar is not None:
pbar.write(msg)
logger.debug(msg)