Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gemma4 12b vllm issue (#2491)

Wenhua Cheng7ef18f8c38e9
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index dc9596854..07ae7b67b 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -369,7 +369,20 @@ def _experts_supports_decorator(module: nn.Module) -> bool:
return hasattr(forward_method, "__wrapped__")
-def _detect_expert_projections(module: nn.Module) -> dict[str, dict]:
+def _module_name_is_experts_container(module_name: str) -> bool:
+ """True when ``module_name`` names a fused-MoE experts container.
+
+ Fused expert weights always live on a module whose path carries an experts/MoE
+ token (e.g. ``model.layers.0.mlp.experts``, ``...block_sparse_moe.experts``,
+ ``...ffn.experts``). A vision/positional embedder such as gemma-4 unified's
+ ``model.embed_vision`` (which owns a 3D ``pos_embedding`` table) does not, so it
+ must never be mistaken for an experts module.
+ """
+ tokens = module_name.lower().split(".")
+ return any(("expert" in token) or token == "moe" for token in tokens)
+
+
+def _detect_expert_projections(module: nn.Module, module_name: str = "") -> dict[str, dict]:
"""Detect which expert projections exist in the module.
This function scans the module for any 3D nn.Parameter attributes.
@@ -381,6 +394,13 @@ def _detect_expert_projections(module: nn.Module) -> dict[str, dict]:
``nn.Module.__getattr__``, which searches three dicts before raising. A parameter is
always registered in ``_parameters``, so the cheap membership test is equivalent.
+ Args:
+ module: The candidate experts module.
+ module_name: The module's qualified path in the model tree (e.g.
+ ``model.layers.0.mlp.experts``). Used to gate the *unknown* 3D-parameter
+ fallback so only genuine experts/MoE containers are unfused. When empty
+ (name unavailable), the fallback stays permissive for backward compatibility.
+
Returns:
Dict mapping projection names to their config, only for projections that exist
as 3D nn.Parameter in the module.
@@ -406,17 +426,29 @@ def _detect_expert_projections(module: nn.Module) -> dict[str, dict]:
if proj_name in three_dim:
detected[proj_name] = config
- # If no known patterns found, take any 3D parameter (future-proofing), but exclude
- # known non-expert 3D parameters that exist on some transformer architectures
- # (e.g. scale_shift_table on WanTransformer3DModel).
+ # Known 3D parameters that are *never* fused expert GEMMs and must never be split,
+ # regardless of the module path (e.g. ``scale_shift_table`` on WanTransformer3DModel).
_NON_EXPERT_3D_PARAMS = frozenset(["scale_shift_table"])
- if not detected:
+
+ # If no known pattern matched, fall back to treating an unknown 3D parameter as a
+ # future expert projection -- but ONLY when this module is actually a fused-MoE
+ # experts container (by path). This is the stable signal: an expert projection
+ # always lives on a ``*.experts``/``*.moe`` module. Without it, legitimate 3D
+ # parameters on non-expert modules get misclassified and split into bogus experts,
+ # e.g. gemma-4 unified's ``model.embed_vision.pos_embedding``
+ # ``(mm_posemb_size, 2, mm_embed_dim)`` -> ``embed_vision.{i}.pos_embedding.weight``.
+ # When ``module_name`` is unavailable we cannot verify the container, so stay
+ # permissive to preserve the previous behaviour -- but still honour the explicit
+ # non-expert denylist in both cases.
+ if not detected and (not module_name or _module_name_is_experts_container(module_name)):
for name in three_dim:
if name in _NON_EXPERT_3D_PARAMS:
+ logger.debug(f"Skipping known non-expert 3D parameter: {name}")
continue
- # Use default config for unknown projections
logger.debug(f"Discovered unknown 3D projection: {name}")
detected[name] = {"is_input_proj": True, "output_multiplier": 1}
+ elif not detected:
+ logger.debug(f"Skipping 3D parameter(s) on non-experts module '{module_name}': {list(three_dim)}")
return detected
@@ -584,13 +616,12 @@ def _warn_if_unfusing_allocates(module: nn.Module, detected_projections: dict[st
total_bytes = sum(param.numel() * param.element_size() for param in params)
logger.warning(
- "[MoE Prep] fused expert weights are on %s (%.2f GB for '%s' alone), not meta. "
- "Splitting them in memory needs the fused tensor and its per-expert copies at the "
- "same time. AutoRound normally avoids this by building a meta skeleton and reading "
- "each expert from disk; that requires a local checkpoint directory.",
- params[0].device.type,
- total_bytes / 1024**3,
+ "[MoE Prep] unfusing '%s' (%.2f GB on %s) in memory needs ~2x that peak. "
+ "For large MoE, enable AutoRound's disk-stream path (AR_DISK_STREAM_MODEL=1, "
+ "needs a local checkpoint + transformers>=5) to unfuse on meta instead.",
module.__class__.__name__,
+ total_bytes / 1024**3,
+ params[0].device.type,
)
@@ -598,6 +629,7 @@ def _unfuse_experts_weights_inplace(
module: nn.Module,
check_decorator: bool = True,
projection_names: list[str] | None = None,
+ module_name: str = "",
) -> bool:
"""Convert fused 3D expert weights to per-expert containers of nn.Linear layers.
@@ -610,6 +642,9 @@ def _unfuse_experts_weights_inplace(
@use_experts_implementation decorator. Default is True.
projection_names: Optional list of projection names to unfuse.
If None, auto-detects from KNOWN_PROJECTION_PATTERNS.
+ module_name: The module's qualified path in the model tree. Forwarded to
+ :func:`_detect_expert_projections` so the unknown-3D-parameter fallback
+ only fires on genuine experts/MoE containers.
Returns:
True if unfusing was successful, False if module doesn't match pattern
@@ -620,7 +655,7 @@ def _unfuse_experts_weights_inplace(
name: config for name, config in KNOWN_PROJECTION_PATTERNS.items() if name in projection_names
}
else:
- detected_projections = _detect_expert_projections(module)
+ detected_projections = _detect_expert_projections(module, module_name=module_name)
if not detected_projections:
return False
@@ -787,7 +822,7 @@ def prepare_model_for_moe_quantization(
for name, module in model.named_modules():
if skip_prefixes and any(name.startswith(p) for p in skip_prefixes):
continue
- if _unfuse_experts_weights_inplace(module):
+ if _unfuse_experts_weights_inplace(module, module_name=name):
unfused_modules.append(name)
logger.debug(f"[MoE Prep] Unfused '{name}'")
diff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.py
index c8f9bb53a..c9e687c1d 100644
--- a/auto_round/utils/disk_stream_util.py
+++ b/auto_round/utils/disk_stream_util.py
@@ -541,6 +541,7 @@ def _wildcard_concat_converters_for(model_type):
target = entry.target_patterns[0].rstrip("$")
if "*" not in source or "*" in target:
continue
+
converters.append((target, source, getattr(entry.operations[0], "dim", 0), getattr(entry, "force_cpu", False)))
return tuple(converters)
@@ -583,6 +584,7 @@ def _wildcard_split_converters_for(model_type):
target = entry.target_patterns[0].rstrip("$")
if "*" not in source or "*" in target:
continue
+
op = entry.operations[0]
converters.append((target, source, getattr(op, "dim", 0), getattr(op, "num_shards_attribute", None)))
return tuple(converters)