Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

mlx: read candidate logits in float32 for quantized heads

Jeffrey Morganc1c536297f5b
diff --git a/mlxrunner/model/qwen3_5/score.go b/mlxrunner/model/qwen3_5/score.go
index 58be6bdf1..5813a9338 100644
--- a/mlxrunner/model/qwen3_5/score.go
+++ b/mlxrunner/model/qwen3_5/score.go
@@ -5,19 +5,31 @@ import (
"github.com/ollama/ollama/mlxrunner/nn"
)
-// UnembedCandidates scores one hidden position. For dense heads, gather before
-// casting so the output stays in FP32 without allocating an
-// FP32 copy of the full vocabulary head. This also covers tied dense embeddings.
+// UnembedCandidates scores one hidden position in FP32, gathering candidate rows
+// before conversion to avoid an FP32 copy of the full vocabulary head.
func (m *Model) UnembedCandidates(hidden, candidates *mlx.Array) *mlx.Array {
- if head, ok := m.LMHead.(*nn.Linear); ok {
- weight := head.Weight.TakeAxis(candidates, 0).AsType(mlx.DTypeFloat32)
- logits := hidden.AsType(mlx.DTypeFloat32).Matmul(weight.Transpose(1, 0)).Reshape(-1)
- if head.Bias != nil {
- logits = logits.Add(head.Bias.TakeAxis(candidates, 0).AsType(mlx.DTypeFloat32))
+ var weight, bias *mlx.Array
+ switch head := m.LMHead.(type) {
+ case *nn.Linear:
+ weight, bias = head.Weight.TakeAxis(candidates, 0), head.Bias
+ case *nn.QuantizedLinear:
+ var qbiases *mlx.Array
+ if head.QBiases != nil {
+ qbiases = head.QBiases.TakeAxis(candidates, 0)
}
- return logits
+ scale := head.GlobalScale
+ if scale != nil && scale.Size() > 1 {
+ scale = scale.TakeAxis(candidates, 0)
+ }
+ weight = mlx.Dequantize(head.Weight.TakeAxis(candidates, 0), head.Scales.TakeAxis(candidates, 0), qbiases,
+ head.GroupSize, head.Bits, head.Mode, scale)
+ bias = head.Bias
+ default:
+ return m.Unembed(hidden).Reshape(-1).TakeAxis(candidates, 0)
+ }
+ logits := hidden.AsType(mlx.DTypeFloat32).Matmul(weight.AsType(mlx.DTypeFloat32).Transpose(1, 0)).Reshape(-1)
+ if bias != nil {
+ logits = logits.Add(bias.TakeAxis(candidates, 0).AsType(mlx.DTypeFloat32))
}
- // Retain existing quantization, bias, and global-scale handling. Selected-row
- // dequantization can be added separately when its formats are validated.
- return m.Unembed(hidden).Reshape(-1).TakeAxis(candidates, 0)
+ return logits
}
diff --git a/mlxrunner/model/qwen3_5/score_test.go b/mlxrunner/model/qwen3_5/score_test.go
index af1e90b7c..4a9702362 100644
--- a/mlxrunner/model/qwen3_5/score_test.go
+++ b/mlxrunner/model/qwen3_5/score_test.go
@@ -44,32 +44,35 @@ func TestUnembedCandidates(t *testing.T) {
}
func TestUnembedQuantizedCandidates(t *testing.T) {
- for _, mode := range []string{"affine", "nvfp4"} {
- mlxtest.RunSubtest(t, mode, func(t *mlxtest.T) {
+ for _, tc := range []struct {
+ mode string
+ groupSize, bits int
+ }{{"affine", 64, 4}, {"mxfp8", 32, 8}, {"nvfp4", 16, 4}} {
+ mlxtest.RunSubtest(t, tc.mode, func(t *mlxtest.T) {
mlx.Scoped(func() {
values := make([]float32, 32*64)
for i := range values {
values[i] = float32(math.Sin(float64(i)))
}
weight := mlx.FromValues(values, 32, 64).AsType(mlx.DTypeBFloat16)
- groupSize := 64
- if mode == "nvfp4" {
- groupSize = 16
- }
bias := mlx.FromValues(values[:32], 32).AsType(mlx.DTypeBFloat16)
- head := nn.NewQuantizedLinear(weight, bias, groupSize, 4, mode)
- if mode == "nvfp4" {
+ head := nn.NewQuantizedLinear(weight, bias, tc.groupSize, tc.bits, tc.mode)
+ if tc.mode == "nvfp4" {
head.GlobalScale = mlx.FromValues([]float32{0.5}, 1)
}
m := &Model{LMHead: head}
hidden := mlx.FromValues(values[:64], 1, 1, 64).AsType(mlx.DTypeBFloat16)
ids := mlx.FromValues([]int32{31, 2, 0}, 3)
- got := m.UnembedCandidates(hidden, ids).AsType(mlx.DTypeFloat32)
- want := m.Unembed(hidden).Reshape(-1).TakeAxis(ids, 0).AsType(mlx.DTypeFloat32)
+ got := m.UnembedCandidates(hidden, ids)
+ dense := mlx.Dequantize(head.Weight, head.Scales, head.QBiases, tc.groupSize, tc.bits, tc.mode, head.GlobalScale).AsType(mlx.DTypeFloat32)
+ want := hidden.AsType(mlx.DTypeFloat32).Matmul(dense.Transpose(1, 0)).Reshape(-1).Add(bias.AsType(mlx.DTypeFloat32)).TakeAxis(ids, 0)
mlx.Eval(got, want)
+ if got.DType() != mlx.DTypeFloat32 || got.Dim(0) != 3 {
+ t.Fatal("quantized candidate logits must be FP32")
+ }
actual, expected := got.Floats(), want.Floats()
for i, v := range actual {
- if v != expected[i] {
+ if math.Abs(float64(v-expected[i])) > 1e-4 {
t.Fatalf("candidate %d: selected=%g full=%g", i, v, expected[i])
}
}