Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Reduce mem usage of GPT-OSS (#1013)

Yi Liu6d3d87d07230
diff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modelling/gpt_oss.py
index 78f73075c..2acae3bb1 100644
--- a/auto_round/modelling/gpt_oss.py
+++ b/auto_round/modelling/gpt_oss.py
@@ -103,20 +103,27 @@ class SequentialGPTOSSMoE(nn.Module):
# Use the original router (it returns scores and indices already softmaxed over top-k)
router_scores, router_indices = self.router(x) # scores: [tokens, E], indices: [tokens, k]
- out = self.shared_expert(x) if self.shared_expert is not None else torch.zeros_like(x)
-
- # Accumulate expert outputs for chosen experts only
- for j in range(self.top_k):
- idx = router_indices[:, j]
- w = router_scores[torch.arange(idx.size(0), device=idx.device), idx].unsqueeze(-1)
- unique_experts = torch.unique(idx)
- for e in unique_experts:
- mask = idx == e
- out[mask] += self.experts[e](x[mask]) * w[mask]
-
- out = out.view(B, T, H)
- router_scores = router_scores.view(B * T, -1) # shape doesn't matter much; it’s ignored by the decoder
- return out, router_scores
+ final_hidden_states = self.shared_expert(x) if self.shared_expert is not None else torch.zeros_like(x)
+ num_all_tokens, total_num_experts = x.size(0), self.num_experts
+ mask_weights = torch.zeros((num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device)
+ topk_ids, experts_mask = router_indices, router_scores
+ topk_ids = topk_ids.to(torch.int64)
+
+ mask_weights.scatter_(-1, topk_ids, 1)
+
+ mask_weights = mask_weights[:num_all_tokens, :total_num_experts]
+ mask_weights = mask_weights.transpose(0, 1)
+ experts_mask = experts_mask[:num_all_tokens, :total_num_experts]
+ experts_mask = experts_mask.transpose(0, 1)
+ num_experts = total_num_experts
+ for expert_index in range(num_experts):
+ mask_weight = mask_weights[expert_index].unsqueeze(1)
+ current_state_static = x * mask_weight
+ expert = self.experts[expert_index]
+ expert_output = expert(current_state_static)
+ expert_output = expert_output * experts_mask[expert_index].unsqueeze(1)
+ final_hidden_states += expert_output
+ return final_hidden_states.view(B, T, H), router_scores.view(B * T, -1)
def get_replacement_info(config):