Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
qwen3_5: decode each expert tensor with its own quantization format
Jesse Grossd6f69da04da9
diff --git a/x/models/qwen3_5/qwen3_5.go b/x/models/qwen3_5/qwen3_5.goindex 1dcdfe819..16324053f 100644--- a/x/models/qwen3_5/qwen3_5.go+++ b/x/models/qwen3_5/qwen3_5.go@@ -171,6 +171,10 @@ type SwitchMLP struct {UpGroupSize intDownGroupSize int+ GateMode string+ UpMode string+ DownMode string+UseQuantized bool}@@ -979,16 +983,19 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {switchMLP.GateBiases = gateW.BiasesswitchMLP.GateBits = gateW.BitsswitchMLP.GateGroupSize = gateW.GroupSize+ switchMLP.GateMode = gateW.ModeswitchMLP.UpWeightQ = upW.WeightswitchMLP.UpScales = upW.ScalesswitchMLP.UpBiases = upW.BiasesswitchMLP.UpBits = upW.BitsswitchMLP.UpGroupSize = upW.GroupSize+ switchMLP.UpMode = upW.ModeswitchMLP.DownWeightQ = downW.WeightswitchMLP.DownScales = downW.ScalesswitchMLP.DownBiases = downW.BiasesswitchMLP.DownBits = downW.BitsswitchMLP.DownGroupSize = downW.GroupSize+ switchMLP.DownMode = downW.Mode} else {switchMLP.GateWeight = transposeExpertWeightForGatherMM(gateW.Weight)switchMLP.UpWeight = transposeExpertWeightForGatherMM(upW.Weight)@@ -1218,12 +1225,12 @@ func (s *SwitchMLP) Forward(x *mlx.Array, indices *mlx.Array, cfg *Config) *mlx.var gate, up, hidden, down *mlx.Arrayif s.UseQuantized {gate = mlx.GatherQMM(xFlat, s.GateWeightQ, s.GateScales, s.GateBiases,- nil, idxFlat, true, s.GateGroupSize, s.GateBits, cfg.QuantMode, doSort)+ nil, idxFlat, true, s.GateGroupSize, s.GateBits, s.GateMode, doSort)up = mlx.GatherQMM(xFlat, s.UpWeightQ, s.UpScales, s.UpBiases,- nil, idxFlat, true, s.UpGroupSize, s.UpBits, cfg.QuantMode, doSort)+ nil, idxFlat, true, s.UpGroupSize, s.UpBits, s.UpMode, doSort)hidden = mlx.SwiGLU(gate, up)down = mlx.GatherQMM(hidden, s.DownWeightQ, s.DownScales, s.DownBiases,- nil, idxFlat, true, s.DownGroupSize, s.DownBits, cfg.QuantMode, doSort)+ nil, idxFlat, true, s.DownGroupSize, s.DownBits, s.DownMode, doSort)} else {gate = mlx.GatherMM(xFlat, s.GateWeight, nil, idxFlat, doSort)up = mlx.GatherMM(xFlat, s.UpWeight, nil, idxFlat, doSort)