Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
qwen3_5: gather packed gate_up experts in one launch
Jesse Gross08edcb8f2c23
diff --git a/x/models/qwen3_5/qwen3_5.go b/x/models/qwen3_5/qwen3_5.goindex 16324053f..158396c6d 100644--- a/x/models/qwen3_5/qwen3_5.go+++ b/x/models/qwen3_5/qwen3_5.go@@ -153,29 +153,22 @@ type SparseMoE struct {SharedExpertGate nn.LinearLayer}-// SwitchMLP executes selected expert MLPs.+// SwitchMLP executes selected expert MLPs. Gate and up are always held+// packed as one gate_up tensor; checkpoints that ship them separately are+// fused at load.type SwitchMLP struct {- GateWeight *mlx.Array- UpWeight *mlx.Array- DownWeight *mlx.Array-- GateWeightQ, GateScales, GateBiases *mlx.Array- UpWeightQ, UpScales, UpBiases *mlx.Array- DownWeightQ, DownScales, DownBiases *mlx.Array-- GateBits int- UpBits int- DownBits int-- GateGroupSize int- UpGroupSize int- DownGroupSize int-- GateMode string- UpMode string- DownMode string-- UseQuantized bool+ GateUpWeight *mlx.Array+ DownWeight *mlx.Array++ GateUpWeightQ, GateUpScales, GateUpBias *mlx.Array+ DownWeightQ, DownScales, DownBiases *mlx.Array++ GateUpBits int+ DownBits int+ GateUpGroupSize int+ DownGroupSize int+ GateUpMode string+ DownMode string}type stackedExpertWeights struct {@@ -468,57 +461,58 @@ func transposeExpertWeightForGatherMM(w *mlx.Array) *mlx.Array {return cloned}-func describeMoEProjection(prefix string, w *stackedExpertWeights) string {- if w == nil {- return prefix + "=missing"- }- if w.Scales != nil {- return fmt.Sprintf("%s=qmm(mode=%s,bits=%d,gs=%d)", prefix, w.Mode, w.Bits, w.GroupSize)- }- if w.Bits > 0 || w.Mode != "" {- reason := "dequantized"- if !supportsGatherQMM(w.Mode, w.Bits) {- reason = "unsupported_gather_qmm"- }- return fmt.Sprintf("%s=%s(mode=%s,bits=%d,gs=%d)", prefix, reason, w.Mode, w.Bits, w.GroupSize)+func fuseExpertStacks(a, b *mlx.Array, axis int) *mlx.Array {+ if a == nil || !a.Valid() || b == nil || !b.Valid() {+ return nil}- return prefix + "=fp"+ out := mlx.Concatenate([]*mlx.Array{a, b}, axis).Clone()+ mlx.Eval(out)+ return out}-func summarizeMoEFallbackReason(gateW, upW, downW *stackedExpertWeights) string {- for _, w := range []*stackedExpertWeights{gateW, upW, downW} {- if w == nil {- return "missing_projection"- }- if w.Scales != nil {- continue+// fuseGateUpProjections joins gate and up stacks along the output dimension,+// which is exact for quantized stacks: groups run along the input dimension.+func fuseGateUpProjections(gate, up *stackedExpertWeights) *stackedExpertWeights {+ if gate == nil || up == nil {+ return nil+ }+ if gate.Weight == nil || gate.Weight.NumDims() != 3 || up.Weight == nil || up.Weight.NumDims() != 3 {+ return nil+ }+ if gate.Scales != nil && up.Scales != nil &&+ gate.Bits == up.Bits && gate.GroupSize == up.GroupSize && gate.Mode == up.Mode &&+ (gate.Biases == nil) == (up.Biases == nil) {+ fused := &stackedExpertWeights{+ Weight: fuseExpertStacks(gate.Weight, up.Weight, 1),+ Scales: fuseExpertStacks(gate.Scales, up.Scales, 1),+ Bits: gate.Bits,+ GroupSize: gate.GroupSize,+ Mode: gate.Mode,}- if w.Bits > 0 || w.Mode != "" {- if !supportsGatherQMM(w.Mode, w.Bits) {- return fmt.Sprintf("unsupported_gather_qmm(mode=%s,bits=%d)", w.Mode, w.Bits)- }- return "dequantized_quant_weights"+ if gate.Biases != nil {+ fused.Biases = fuseExpertStacks(gate.Biases, up.Biases, 1)}+ return fused}- return "unquantized_weights"-}--func sliceStackedExpertAxis1(a *mlx.Array, start, stop int32) *mlx.Array {- if a == nil || !a.Valid() {- return nil+ if gate.Scales != nil || up.Scales != nil {+ slog.Warn("dequantizing gate/up experts: quantization parameters differ",+ "gate_mode", gate.Mode, "gate_bits", gate.Bits, "gate_group_size", gate.GroupSize,+ "up_mode", up.Mode, "up_bits", up.Bits, "up_group_size", up.GroupSize)}- dims := a.Dims()- if len(dims) < 2 {- return nil+ gateWeight := gate.Weight+ if gate.Scales != nil {+ gateWeight = mlx.Dequantize(gate.Weight, gate.Scales, gate.Biases, gate.GroupSize, gate.Bits, gate.Mode)}- beg := make([]int32, len(dims))- end := make([]int32, len(dims))- for i, d := range dims {- end[i] = int32(d)+ upWeight := up.Weight+ if up.Scales != nil {+ upWeight = mlx.Dequantize(up.Weight, up.Scales, up.Biases, up.GroupSize, up.Bits, up.Mode)+ }+ return &stackedExpertWeights{+ Weight: mlx.Concatenate([]*mlx.Array{gateWeight, upWeight}, 1),+ Bits: gate.Bits,+ GroupSize: gate.GroupSize,+ Mode: gate.Mode,}- beg[1] = start- end[1] = stop- return mlx.SliceStartStop(a, beg, end)}func loadStackedProjection(tensors map[string]*mlx.Array, cfg *Config, useQuantized bool, bases ...string) *stackedExpertWeights {@@ -554,6 +548,9 @@ func loadStackedProjection(tensors map[string]*mlx.Array, cfg *Config, useQuanti}}+ if useQuantized {+ slog.Warn("dequantizing expert weights: no gather kernel for format", "tensor", key, "mode", mode, "bits", bits)+ }return &stackedExpertWeights{Weight: mlx.Dequantize(w, scales, qbiases, groupSize, bits, mode),Bits: bits,@@ -570,6 +567,7 @@ func collectPerExpertProjection(tensors map[string]*mlx.Array, cfg *Config, useQscales := make([]*mlx.Array, 0, numExperts)biases := make([]*mlx.Array, 0, numExperts)consumedKeys := make([]string, 0, numExperts*3)+ numDequantized := 0bits := 0groupSize := 0mode := cfg.QuantMode@@ -614,8 +612,13 @@ func collectPerExpertProjection(tensors map[string]*mlx.Array, cfg *Config, useQ}} else {weights = append(weights, mlx.Dequantize(w, s, qb, gs, b, m))+ numDequantized++}}+ if useQuantized && numDequantized > 0 {+ slog.Warn("dequantizing expert weights: no gather kernel for format",+ "tensor", fmt.Sprintf("%s.mlp.experts.*.%s", layerPrefix, proj), "mode", mode, "bits", bits)+ }if len(weights) == 0 {return nil@@ -632,122 +635,125 @@ func collectPerExpertProjection(tensors map[string]*mlx.Array, cfg *Config, useQreturn out}-func splitGateUpProjection(tensors map[string]*mlx.Array, cfg *Config, useQuantized bool, layerPrefix string) (gate, up, down *stackedExpertWeights) {+// combinedGateUpProjection loads the packed mlp.experts.gate_up_proj stack;+// nil when the checkpoint ships gate/up separately.+func combinedGateUpProjection(tensors map[string]*mlx.Array, cfg *Config, useQuantized bool, layerPrefix string) *stackedExpertWeights {gateUp, key := tensorAny(tensors,layerPrefix+".mlp.experts.gate_up_proj.weight",layerPrefix+".mlp.experts.gate_up_proj",)- if gateUp == nil {- return nil, nil, nil- }-- if scales := tensors[key+"_scale"]; scales != nil {- qbiases := tensors[key+"_qbias"]- groupSize, bits, mode := model.ResolveLinearQuantParams(- cfg.QuantGroupSize,- cfg.QuantBits,- cfg.QuantMode,- cfg.TensorQuant,- key,- gateUp,- scales,- )- if useQuantized && supportsGatherQMM(mode, bits) {- gate = &stackedExpertWeights{- Bits: bits,- GroupSize: groupSize,- Mode: mode,- }- up = &stackedExpertWeights{- Bits: bits,- GroupSize: groupSize,- Mode: mode,- }- // Keep quantized packed tensor and split along the out-dim (axis=1).- // This assumes MLX quantization preserves the leading [experts, out, ...] layout.- if gateUp.NumDims() != 3 {- return nil, nil, nil- }- shape := gateUp.Dims()- nExperts, twoHidden, inHidden := int32(shape[0]), int32(shape[1]), int32(shape[2])- _ = nExperts- _ = inHidden- mid := twoHidden / 2-- gate.Weight = sliceStackedExpertAxis1(gateUp, 0, mid)- up.Weight = sliceStackedExpertAxis1(gateUp, mid, twoHidden)- gate.Scales = sliceStackedExpertAxis1(scales, 0, mid)- up.Scales = sliceStackedExpertAxis1(scales, mid, twoHidden)- if qbiases != nil {- gate.Biases = sliceStackedExpertAxis1(qbiases, 0, mid)- up.Biases = sliceStackedExpertAxis1(qbiases, mid, twoHidden)- }- } else {- gateUp = mlx.Dequantize(gateUp, scales, qbiases, groupSize, bits, mode)- gate = &stackedExpertWeights{Bits: bits, GroupSize: groupSize, Mode: mode}- up = &stackedExpertWeights{Bits: bits, GroupSize: groupSize, Mode: mode}- }+ if gateUp == nil || gateUp.NumDims() != 3 {+ return nil}- if gateUp.NumDims() != 3 {- return nil, nil, nil+ scales := tensors[key+"_scale"]+ if scales == nil {+ return &stackedExpertWeights{Weight: gateUp}}- shape := gateUp.Dims()- nExperts, twoHidden, inHidden := int32(shape[0]), int32(shape[1]), int32(shape[2])- mid := twoHidden / 2- if gate == nil {- gate = &stackedExpertWeights{}+ qbiases := tensors[key+"_qbias"]+ groupSize, bits, mode := model.ResolveLinearQuantParams(+ cfg.QuantGroupSize,+ cfg.QuantBits,+ cfg.QuantMode,+ cfg.TensorQuant,+ key,+ gateUp,+ scales,+ )+ if useQuantized && supportsGatherQMM(mode, bits) {+ return &stackedExpertWeights{+ Weight: gateUp,+ Scales: scales,+ Biases: qbiases,+ Bits: bits,+ GroupSize: groupSize,+ Mode: mode,+ }}- if up == nil {- up = &stackedExpertWeights{}+ if useQuantized {+ slog.Warn("dequantizing expert weights: no gather kernel for format", "tensor", key, "mode", mode, "bits", bits)}- if gate.Weight == nil {- gate.Weight = mlx.SliceStartStop(gateUp, []int32{0, 0, 0}, []int32{nExperts, mid, inHidden})+ return &stackedExpertWeights{+ Weight: mlx.Dequantize(gateUp, scales, qbiases, groupSize, bits, mode),+ Bits: bits,+ GroupSize: groupSize,+ Mode: mode,}- if up.Weight == nil {- up.Weight = mlx.SliceStartStop(gateUp, []int32{0, mid, 0}, []int32{nExperts, twoHidden, inHidden})+}++// splitLastAxisHalves views the two halves of a's last axis.+func splitLastAxisHalves(a *mlx.Array) (lo, hi *mlx.Array) {+ dims := a.Dims()+ nd := len(dims)+ beg := make([]int32, nd)+ end := make([]int32, nd)+ for i, d := range dims {+ end[i] = int32(d)}+ mid := int32(dims[nd-1]) / 2+ endLo := append([]int32(nil), end...)+ endLo[nd-1] = mid+ begHi := append([]int32(nil), beg...)+ begHi[nd-1] = mid+ return mlx.SliceStartStop(a, beg, endLo), mlx.SliceStartStop(a, begHi, end)+}- downW, downKey := tensorAny(- tensors,- layerPrefix+".mlp.experts.down_proj.weight",+// loadSwitchMLP assembles a layer's routed experts from any supported+// checkpoint layout.+func loadSwitchMLP(tensors map[string]*mlx.Array, cfg *Config, useQuantized bool, layerPrefix string) (*SwitchMLP, error) {+ gateUpW := combinedGateUpProjection(tensors, cfg, useQuantized, layerPrefix)+ downW := loadStackedProjection(tensors, cfg, useQuantized,+ layerPrefix+".mlp.switch_mlp.down_proj",layerPrefix+".mlp.experts.down_proj",)- if downW == nil {- return gate, up, nil- }- if scales := tensors[downKey+"_scale"]; scales != nil {- qbiases := tensors[downKey+"_qbias"]- groupSize, bits, mode := model.ResolveLinearQuantParams(- cfg.QuantGroupSize,- cfg.QuantBits,- cfg.QuantMode,- cfg.TensorQuant,- downKey,- downW,- scales,+ if gateUpW == nil {+ gateW := loadStackedProjection(tensors, cfg, useQuantized,+ layerPrefix+".mlp.switch_mlp.gate_proj",+ layerPrefix+".mlp.experts.gate_proj",)- if useQuantized && supportsGatherQMM(mode, bits) {- down = &stackedExpertWeights{- Weight: downW,- Scales: scales,- Biases: qbiases,- Bits: bits,- GroupSize: groupSize,- Mode: mode,- }- return gate, up, down+ upW := loadStackedProjection(tensors, cfg, useQuantized,+ layerPrefix+".mlp.switch_mlp.up_proj",+ layerPrefix+".mlp.experts.up_proj",+ )+ if gateW == nil {+ gateW = collectPerExpertProjection(tensors, cfg, useQuantized, layerPrefix, "gate_proj", cfg.NumExperts)}- downW = mlx.Dequantize(downW, scales, qbiases, groupSize, bits, mode)- down = &stackedExpertWeights{Bits: bits, GroupSize: groupSize, Mode: mode}+ if upW == nil {+ upW = collectPerExpertProjection(tensors, cfg, useQuantized, layerPrefix, "up_proj", cfg.NumExperts)+ }+ gateUpW = fuseGateUpProjections(gateW, upW)+ }+ if downW == nil {+ downW = collectPerExpertProjection(tensors, cfg, useQuantized, layerPrefix, "down_proj", cfg.NumExperts)}- if down == nil {- down = &stackedExpertWeights{}+ if gateUpW == nil || downW == nil {+ return nil, fmt.Errorf("missing switch expert weights")}- down.Weight = downW- return gate, up, down++ switchMLP := &SwitchMLP{}+ if gateUpW.Scales != nil {+ switchMLP.GateUpWeightQ = gateUpW.Weight+ switchMLP.GateUpScales = gateUpW.Scales+ switchMLP.GateUpBias = gateUpW.Biases+ switchMLP.GateUpBits = gateUpW.Bits+ switchMLP.GateUpGroupSize = gateUpW.GroupSize+ switchMLP.GateUpMode = gateUpW.Mode+ } else {+ switchMLP.GateUpWeight = transposeExpertWeightForGatherMM(gateUpW.Weight)+ }+ if downW.Scales != nil {+ switchMLP.DownWeightQ = downW.Weight+ switchMLP.DownScales = downW.Scales+ switchMLP.DownBiases = downW.Biases+ switchMLP.DownBits = downW.Bits+ switchMLP.DownGroupSize = downW.GroupSize+ switchMLP.DownMode = downW.Mode+ } else {+ switchMLP.DownWeight = transposeExpertWeightForGatherMM(downW.Weight)+ }+ return switchMLP, nil}func sanitizeConvWeight(w *mlx.Array) *mlx.Array {@@ -849,7 +855,6 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {}}}- moeLoadSummaries := make([]string, 0)for i := range cfg.NumHiddenLayers {layerPrefix := fmt.Sprintf("%slayers.%d", modelPrefix, i)@@ -941,86 +946,9 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {return fmt.Errorf("layer %d: missing moe gate", i)}- gateW := loadStackedProjection(tensors, cfg, useQuantizedExperts,- layerPrefix+".mlp.switch_mlp.gate_proj",- layerPrefix+".mlp.experts.gate_proj",- )- upW := loadStackedProjection(tensors, cfg, useQuantizedExperts,- layerPrefix+".mlp.switch_mlp.up_proj",- layerPrefix+".mlp.experts.up_proj",- )- downW := loadStackedProjection(tensors, cfg, useQuantizedExperts,- layerPrefix+".mlp.switch_mlp.down_proj",- layerPrefix+".mlp.experts.down_proj",- )- if gateW == nil || upW == nil || downW == nil {- g2, u2, d2 := splitGateUpProjection(tensors, cfg, useQuantizedExperts, layerPrefix)- if gateW == nil {- gateW = g2- }- if upW == nil {- upW = u2- }- if downW == nil {- downW = d2- }- }- if gateW == nil || upW == nil || downW == nil {- gateW = collectPerExpertProjection(tensors, cfg, useQuantizedExperts, layerPrefix, "gate_proj", cfg.NumExperts)- upW = collectPerExpertProjection(tensors, cfg, useQuantizedExperts, layerPrefix, "up_proj", cfg.NumExperts)- downW = collectPerExpertProjection(tensors, cfg, useQuantizedExperts, layerPrefix, "down_proj", cfg.NumExperts)- }-- if gateW == nil || upW == nil || downW == nil {- return fmt.Errorf("layer %d: missing switch expert weights", i)- }-- switchMLP := &SwitchMLP{}- if gateW.Scales != nil && upW.Scales != nil && downW.Scales != nil {- switchMLP.UseQuantized = true- switchMLP.GateWeightQ = gateW.Weight- switchMLP.GateScales = gateW.Scales- switchMLP.GateBiases = gateW.Biases- switchMLP.GateBits = gateW.Bits- switchMLP.GateGroupSize = gateW.GroupSize- switchMLP.GateMode = gateW.Mode- switchMLP.UpWeightQ = upW.Weight- switchMLP.UpScales = upW.Scales- switchMLP.UpBiases = upW.Biases- switchMLP.UpBits = upW.Bits- switchMLP.UpGroupSize = upW.GroupSize- switchMLP.UpMode = upW.Mode- switchMLP.DownWeightQ = downW.Weight- switchMLP.DownScales = downW.Scales- switchMLP.DownBiases = downW.Biases- switchMLP.DownBits = downW.Bits- switchMLP.DownGroupSize = downW.GroupSize- switchMLP.DownMode = downW.Mode- } else {- switchMLP.GateWeight = transposeExpertWeightForGatherMM(gateW.Weight)- switchMLP.UpWeight = transposeExpertWeightForGatherMM(upW.Weight)- switchMLP.DownWeight = transposeExpertWeightForGatherMM(downW.Weight)- moeLoadSummaries = append(moeLoadSummaries,- fmt.Sprintf(- "layer=%d moe_fallback reason=%s %s %s %s",- i,- summarizeMoEFallbackReason(gateW, upW, downW),- describeMoEProjection("gate", gateW),- describeMoEProjection("up", upW),- describeMoEProjection("down", downW),- ),- )- }- if switchMLP.UseQuantized {- moeLoadSummaries = append(moeLoadSummaries,- fmt.Sprintf(- "layer=%d moe_quantized %s %s %s",- i,- describeMoEProjection("gate", gateW),- describeMoEProjection("up", upW),- describeMoEProjection("down", downW),- ),- )+ switchMLP, err := loadSwitchMLP(tensors, cfg, useQuantizedExperts, layerPrefix)+ if err != nil {+ return fmt.Errorf("layer %d: %w", i, err)}moe.SwitchMLP = switchMLP@@ -1051,9 +979,6 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {m.Layers[i] = layer}- for _, summary := range moeLoadSummaries {- slog.Debug("qwen3.5 moe load", "summary", summary)- }return nil}@@ -1222,19 +1147,19 @@ func (s *SwitchMLP) Forward(x *mlx.Array, indices *mlx.Array, cfg *Config) *mlx.idxFlat = mlx.Reshape(mlx.Take(idxAll, order, 0), n, 1)}- var gate, up, hidden, down *mlx.Array- if s.UseQuantized {- gate = mlx.GatherQMM(xFlat, s.GateWeightQ, s.GateScales, s.GateBiases,- nil, idxFlat, true, s.GateGroupSize, s.GateBits, s.GateMode, doSort)- up = mlx.GatherQMM(xFlat, s.UpWeightQ, s.UpScales, s.UpBiases,- nil, idxFlat, true, s.UpGroupSize, s.UpBits, s.UpMode, doSort)- hidden = mlx.SwiGLU(gate, up)+ var gateUp, down *mlx.Array+ if s.GateUpWeightQ != nil {+ gateUp = mlx.GatherQMM(xFlat, s.GateUpWeightQ, s.GateUpScales, s.GateUpBias,+ nil, idxFlat, true, s.GateUpGroupSize, s.GateUpBits, s.GateUpMode, doSort)+ } else {+ gateUp = mlx.GatherMM(xFlat, s.GateUpWeight, nil, idxFlat, doSort)+ }+ gate, up := splitLastAxisHalves(gateUp)+ hidden := mlx.SwiGLU(gate, up)+ if s.DownWeightQ != nil {down = mlx.GatherQMM(hidden, s.DownWeightQ, s.DownScales, s.DownBiases,nil, idxFlat, true, s.DownGroupSize, s.DownBits, s.DownMode, doSort)} else {- gate = mlx.GatherMM(xFlat, s.GateWeight, nil, idxFlat, doSort)- up = mlx.GatherMM(xFlat, s.UpWeight, nil, idxFlat, doSort)- hidden = mlx.SwiGLU(gate, up)down = mlx.GatherMM(hidden, s.DownWeight, nil, idxFlat, doSort)}