PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
increase precision of loss logging in quantization tuning for debugging (#2229)
Xin He385a48c94afa
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex 4e4aeed57..1df932c83 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -507,6 +507,8 @@ class SignRoundQuantizer(BaseQuantizer):if i == 0:init_loss = total_loss+ current_lr = optimizer.param_groups[0]["lr"]+ logger.debug("iter %d loss: %.3e lr: %s", i, total_loss, current_lr)if total_loss < best_loss:best_loss = total_loss@@ -530,7 +532,7 @@ class SignRoundQuantizer(BaseQuantizer):if self.iters > 0:dump_info = (f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "- f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"+ f"layers in the block, loss iter 0: {init_loss:.3e} -> iter {best_iter}: {last_loss:.3e}")else:dump_info = (@@ -733,6 +735,8 @@ class SignRoundQuantizer(BaseQuantizer):self._scale_loss_and_backward(scaler, loss)if i == 0:init_loss = total_loss+ current_lr = optimizer.param_groups[0]["lr"]+ logger.debug("iter %d loss: %.3e lr: %s", i, total_loss, current_lr)if total_loss < best_loss:best_loss = total_loss@@ -755,7 +759,7 @@ class SignRoundQuantizer(BaseQuantizer):with torch.no_grad():unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)mv_module_from_gpu(layer)- dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"+ dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.3e} -> iter {best_iter}: {last_loss:.3e}"logger.info(dump_info)def finalize_run(self) -> None:diff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.pyindex c816c3d98..b67b84073 100644--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py@@ -587,7 +587,7 @@ class SpinQuantPreprocessor:logger.info(f"[SpinQuant] Training complete: {result.steps} steps, "- f"best_loss={result.best_loss:.6f}, ortho_dev={result.final_ortho_deviation:.2e}"+ f"best_loss={result.best_loss:.3e}, ortho_dev={result.final_ortho_deviation:.2e}")del original_modeldiff --git a/auto_round/algorithms/transforms/spinquant/training.py b/auto_round/algorithms/transforms/spinquant/training.pyindex 618bfa7b1..e61793315 100644--- a/auto_round/algorithms/transforms/spinquant/training.py+++ b/auto_round/algorithms/transforms/spinquant/training.py@@ -286,7 +286,7 @@ def run_training_loop(on_step_end(step, loss_val, avg)elif step % log_interval == 0:avg = sum(loss_history[-50:]) / len(loss_history[-50:])- logger.info(f"[SpinQuant] Step {step}/{max_iters}, loss={loss_val:.6f} (avg={avg:.6f})")+ logger.info(f"[SpinQuant] Step {step}/{max_iters}, loss={loss_val:.3e} (avg={avg:.3e})")step += 1@@ -569,7 +569,7 @@ class LossLogger(RotationTrainerCallback):if step % self.log_interval == 0:loss = state.get("loss", 0.0)avg_loss = state.get("avg_loss", 0.0)- print(f" [LossLogger] step={step}/{args.iters} loss={loss:.6f} avg={avg_loss:.6f}")+ print(f" [LossLogger] step={step}/{args.iters} loss={loss:.3e} avg={avg_loss:.3e}")class RotationTrainer:diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex c7207bea6..f407127ae 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -3205,7 +3205,7 @@ def _gen_layer_config()total_loss = _record_scheme_scores(index, per_op_scores)logger.info(- "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",+ "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.3e)",index + 1,len(schemes),total_loss,@@ -3314,7 +3314,7 @@ def _gen_layer_config(total_loss = _record_scheme_scores(index, per_op_scores)logger.info(- "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",+ "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.3e)",index + 1,len(schemes),total_loss,diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pyindex b987a263d..c8e4f5e2a 100644--- a/auto_round/auto_scheme/utils.py+++ b/auto_round/auto_scheme/utils.py@@ -561,7 +561,7 @@ def _log_score_summary_by_block_and_nonblock(for name, avg_loss in rows_in_order:rank = rank_map.get(name, -1)ratio_pct = 0.0 if abs(total_avg_loss) < _ZERO_EPS else (avg_loss / total_avg_loss) * 100.0- logger.debug("AutoScheme | %d | %s | %.6f | %.2f%% |", rank, name, avg_loss, ratio_pct)+ logger.debug("AutoScheme | %d | %s | %.3e | %.2f%% |", rank, name, avg_loss, ratio_pct)if head_name is not None and head_name not in scores_dict:logger.debug("AutoScheme | - | %s | N/A | N/A |", head_name)@@ -569,7 +569,7 @@ def _log_score_summary_by_block_and_nonblock(if non_block_items:non_block_items.sort(key=lambda x: x[0])for layer_name, layer_loss in non_block_items:- logger.info("AutoScheme non_block=%s loss=%.6f", layer_name, layer_loss)+ logger.info("AutoScheme non_block=%s loss=%.3e", layer_name, layer_loss)def _collect_current_scores(model):@@ -611,7 +611,7 @@ def _log_batch_avg_loss(model, batch_idx: int, pbar=None, block_names=None, totaavg_loss = 0.0 if layer_cnt == 0 else total_loss / layer_cnttag = f"[{scheme_tag}] " if scheme_tag else ""batch_str = f"{batch_idx}/{total_batches}" if total_batches is not None else str(batch_idx)- msg = f"AutoScheme {tag}cumulative batch {batch_str} avg_loss={avg_loss:.6f} layers={layer_cnt}"+ msg = f"AutoScheme {tag}cumulative batch {batch_str} avg_loss={avg_loss:.3e} layers={layer_cnt}"if pbar is not None:pbar.write(msg)logger.debug(msg)