Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix nemoron gate inference issue (#2325)

Wenhua Cheng0ea7c93c3450
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index 2a9bc7f1c..cf0954b1b 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -276,15 +276,22 @@ def linear_loop_experts_forward(
# Get this expert's container with its projection layers
expert = getattr(self, str(expert_idx))
- gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)
up_out = expert.up_proj(expert_input) # (num_samples, intermediate_dim)
- # Apply gating
- if hasattr(self, "_apply_gate"):
- gate_up_out = torch.cat([gate_out, up_out], dim=-1)
- gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)
+ # Non-gated MLP experts (e.g. NemotronH's @use_experts_implementation(has_gate=False))
+ # expose only up_proj/down_proj: out = down_proj(act_fn(up_proj(x))). Gated experts
+ # (Mixtral/Qwen3-MoE/...) additionally own gate_proj. Mirror the grouped path
+ # (grouped_linear_experts_forward) which keys the same branch off gate_proj presence.
+ if hasattr(expert, "gate_proj"):
+ gate_out = expert.gate_proj(expert_input) # (num_samples, intermediate_dim)
+ # Apply gating
+ if hasattr(self, "_apply_gate"):
+ gate_up_out = torch.cat([gate_out, up_out], dim=-1)
+ gated_out = self._apply_gate(gate_up_out) # (num_samples, intermediate_dim)
+ else:
+ gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)
else:
- gated_out = self.act_fn(gate_out) * up_out # (num_samples, intermediate_dim)
+ gated_out = self.act_fn(up_out) # (num_samples, intermediate_dim)
# Down projection
expert_out = expert.down_proj(gated_out) # (num_samples, hidden_dim)
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index c8f09f2a9..010b3669e 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -2877,11 +2877,6 @@ def place_ngram_embeddings_for_tuning_(module: torch.nn.Module, gpu_devices: lis
len(fallback_ngram_names),
", ".join(fallback_ngram_names[:3]),
)
- else:
- logger.info_once(
- "AR_NGRAM_DEVICE=%s is set, but no ngram modules were found under this block.",
- setting or "auto",
- )
return []
total_ngram_nbytes = sum(_module_storage_nbytes(sub) for _, sub in ngram_modules)
logger.info_once(