Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlx: read candidate logits in float32 for quantized heads
Jeffrey Morganc1c536297f5b
diff --git a/mlxrunner/model/qwen3_5/score.go b/mlxrunner/model/qwen3_5/score.goindex 58be6bdf1..5813a9338 100644--- a/mlxrunner/model/qwen3_5/score.go+++ b/mlxrunner/model/qwen3_5/score.go@@ -5,19 +5,31 @@ import ("github.com/ollama/ollama/mlxrunner/nn")-// UnembedCandidates scores one hidden position. For dense heads, gather before-// casting so the output stays in FP32 without allocating an-// FP32 copy of the full vocabulary head. This also covers tied dense embeddings.+// UnembedCandidates scores one hidden position in FP32, gathering candidate rows+// before conversion to avoid an FP32 copy of the full vocabulary head.func (m *Model) UnembedCandidates(hidden, candidates *mlx.Array) *mlx.Array {- if head, ok := m.LMHead.(*nn.Linear); ok {- weight := head.Weight.TakeAxis(candidates, 0).AsType(mlx.DTypeFloat32)- logits := hidden.AsType(mlx.DTypeFloat32).Matmul(weight.Transpose(1, 0)).Reshape(-1)- if head.Bias != nil {- logits = logits.Add(head.Bias.TakeAxis(candidates, 0).AsType(mlx.DTypeFloat32))+ var weight, bias *mlx.Array+ switch head := m.LMHead.(type) {+ case *nn.Linear:+ weight, bias = head.Weight.TakeAxis(candidates, 0), head.Bias+ case *nn.QuantizedLinear:+ var qbiases *mlx.Array+ if head.QBiases != nil {+ qbiases = head.QBiases.TakeAxis(candidates, 0)}- return logits+ scale := head.GlobalScale+ if scale != nil && scale.Size() > 1 {+ scale = scale.TakeAxis(candidates, 0)+ }+ weight = mlx.Dequantize(head.Weight.TakeAxis(candidates, 0), head.Scales.TakeAxis(candidates, 0), qbiases,+ head.GroupSize, head.Bits, head.Mode, scale)+ bias = head.Bias+ default:+ return m.Unembed(hidden).Reshape(-1).TakeAxis(candidates, 0)+ }+ logits := hidden.AsType(mlx.DTypeFloat32).Matmul(weight.AsType(mlx.DTypeFloat32).Transpose(1, 0)).Reshape(-1)+ if bias != nil {+ logits = logits.Add(bias.TakeAxis(candidates, 0).AsType(mlx.DTypeFloat32))}- // Retain existing quantization, bias, and global-scale handling. Selected-row- // dequantization can be added separately when its formats are validated.- return m.Unembed(hidden).Reshape(-1).TakeAxis(candidates, 0)+ return logits}diff --git a/mlxrunner/model/qwen3_5/score_test.go b/mlxrunner/model/qwen3_5/score_test.goindex af1e90b7c..4a9702362 100644--- a/mlxrunner/model/qwen3_5/score_test.go+++ b/mlxrunner/model/qwen3_5/score_test.go@@ -44,32 +44,35 @@ func TestUnembedCandidates(t *testing.T) {}func TestUnembedQuantizedCandidates(t *testing.T) {- for _, mode := range []string{"affine", "nvfp4"} {- mlxtest.RunSubtest(t, mode, func(t *mlxtest.T) {+ for _, tc := range []struct {+ mode string+ groupSize, bits int+ }{{"affine", 64, 4}, {"mxfp8", 32, 8}, {"nvfp4", 16, 4}} {+ mlxtest.RunSubtest(t, tc.mode, func(t *mlxtest.T) {mlx.Scoped(func() {values := make([]float32, 32*64)for i := range values {values[i] = float32(math.Sin(float64(i)))}weight := mlx.FromValues(values, 32, 64).AsType(mlx.DTypeBFloat16)- groupSize := 64- if mode == "nvfp4" {- groupSize = 16- }bias := mlx.FromValues(values[:32], 32).AsType(mlx.DTypeBFloat16)- head := nn.NewQuantizedLinear(weight, bias, groupSize, 4, mode)- if mode == "nvfp4" {+ head := nn.NewQuantizedLinear(weight, bias, tc.groupSize, tc.bits, tc.mode)+ if tc.mode == "nvfp4" {head.GlobalScale = mlx.FromValues([]float32{0.5}, 1)}m := &Model{LMHead: head}hidden := mlx.FromValues(values[:64], 1, 1, 64).AsType(mlx.DTypeBFloat16)ids := mlx.FromValues([]int32{31, 2, 0}, 3)- got := m.UnembedCandidates(hidden, ids).AsType(mlx.DTypeFloat32)- want := m.Unembed(hidden).Reshape(-1).TakeAxis(ids, 0).AsType(mlx.DTypeFloat32)+ got := m.UnembedCandidates(hidden, ids)+ dense := mlx.Dequantize(head.Weight, head.Scales, head.QBiases, tc.groupSize, tc.bits, tc.mode, head.GlobalScale).AsType(mlx.DTypeFloat32)+ want := hidden.AsType(mlx.DTypeFloat32).Matmul(dense.Transpose(1, 0)).Reshape(-1).Add(bias.AsType(mlx.DTypeFloat32)).TakeAxis(ids, 0)mlx.Eval(got, want)+ if got.DType() != mlx.DTypeFloat32 || got.Dim(0) != 3 {+ t.Fatal("quantized candidate logits must be FP32")+ }actual, expected := got.Floats(), want.Floats()for i, v := range actual {- if v != expected[i] {+ if math.Abs(float64(v-expected[i])) > 1e-4 {t.Fatalf("candidate %d: selected=%g full=%g", i, v, expected[i])}}