Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlx: allow the embedding layer to use the nvfp4 global scale (#16527)
Patrick Devine3ef69ef7849e
diff --git a/x/mlxrunner/model/embedding.go b/x/mlxrunner/model/embedding.goindex d45d48e2f..1b3c14ef9 100644--- a/x/mlxrunner/model/embedding.go+++ b/x/mlxrunner/model/embedding.go@@ -35,7 +35,23 @@ func MakeEmbeddingLayer(scales,)- return nn.NewQuantizedEmbedding(w, scales, qbiases, groupSize, bits, mode)+ // Check for per-tensor global scale (NVIDIA double-scale nvfp4).+ // NVIDIA ModelOpt stores this as "weight_scale_2"; our import+ // pipeline maps it to "weight.global_scale".+ globalScale := tensors[path+".weight.global_scale"]+ if globalScale == nil {+ globalScale = tensors[path+".weight_scale_2"]+ }++ return &nn.QuantizedEmbedding{+ Weight: w,+ Scales: scales,+ QBiases: qbiases,+ GlobalScale: globalScale,+ GroupSize: groupSize,+ Bits: bits,+ Mode: mode,+ }}return nn.NewEmbedding(w)diff --git a/x/mlxrunner/model/embedding_test.go b/x/mlxrunner/model/embedding_test.goindex 2935bdc3f..975e4f83b 100644--- a/x/mlxrunner/model/embedding_test.go+++ b/x/mlxrunner/model/embedding_test.go@@ -76,3 +76,51 @@ func TestMakeEmbeddingLayerQuantized(t *testing.T) {t.Fatalf("AsLinear type = %T, want *nn.QuantizedLinear", emb.AsLinear())}}++func TestMakeEmbeddingLayerQuantizedGlobalScale(t *testing.T) {+ weight := &mlx.Array{}+ scales := &mlx.Array{}+ globalScale := &mlx.Array{}++ emb := MakeEmbeddingLayer(map[string]*mlx.Array{+ "model.embed_tokens.weight": weight,+ "model.embed_tokens.weight_scale": scales,+ "model.embed_tokens.weight.global_scale": globalScale,+ }, "model.embed_tokens", 16, 4, "nvfp4", nil)++ qemb, ok := emb.(*nn.QuantizedEmbedding)+ if !ok {+ t.Fatalf("embedding type = %T, want *nn.QuantizedEmbedding", emb)+ }+ if qemb.GlobalScale != globalScale {+ t.Fatalf("GlobalScale = %p, want %p", qemb.GlobalScale, globalScale)+ }++ linear, ok := qemb.AsLinear().(*nn.QuantizedLinear)+ if !ok {+ t.Fatalf("AsLinear type = %T, want *nn.QuantizedLinear", qemb.AsLinear())+ }+ if linear.GlobalScale != globalScale {+ t.Fatalf("AsLinear GlobalScale = %p, want %p", linear.GlobalScale, globalScale)+ }+}++func TestMakeEmbeddingLayerQuantizedGlobalScaleFallback(t *testing.T) {+ weight := &mlx.Array{}+ scales := &mlx.Array{}+ globalScale := &mlx.Array{}++ emb := MakeEmbeddingLayer(map[string]*mlx.Array{+ "model.embed_tokens.weight": weight,+ "model.embed_tokens.weight_scale": scales,+ "model.embed_tokens.weight_scale_2": globalScale,+ }, "model.embed_tokens", 16, 4, "nvfp4", nil)++ qemb, ok := emb.(*nn.QuantizedEmbedding)+ if !ok {+ t.Fatalf("embedding type = %T, want *nn.QuantizedEmbedding", emb)+ }+ if qemb.GlobalScale != globalScale {+ t.Fatalf("GlobalScale = %p, want %p", qemb.GlobalScale, globalScale)+ }+}diff --git a/x/models/nn/nn.go b/x/models/nn/nn.goindex 4410848b7..9aa333bc9 100644--- a/x/models/nn/nn.go+++ b/x/models/nn/nn.go@@ -166,23 +166,13 @@ func (e *Embedding) AsLinear() LinearLayer {// QuantizedEmbedding performs row-wise embedding lookup from affine/nvfp4/etc.// packed weights and dequantizes only the selected rows.type QuantizedEmbedding struct {- Weight *mlx.Array- Scales *mlx.Array- QBiases *mlx.Array- GroupSize int- Bits int- Mode string-}--func NewQuantizedEmbedding(weight, scales, qbiases *mlx.Array, groupSize, bits int, mode string) *QuantizedEmbedding {- return &QuantizedEmbedding{- Weight: weight,- Scales: scales,- QBiases: qbiases,- GroupSize: groupSize,- Bits: bits,- Mode: mode,- }+ Weight *mlx.Array+ Scales *mlx.Array+ QBiases *mlx.Array+ GlobalScale *mlx.Array // Per-tensor global scale for double-scale nvfp4 (nil for standard)+ GroupSize int+ Bits int+ Mode string}func (qe *QuantizedEmbedding) Forward(indices *mlx.Array) *mlx.Array {@@ -192,17 +182,22 @@ func (qe *QuantizedEmbedding) Forward(indices *mlx.Array) *mlx.Array {if qe.QBiases != nil && qe.QBiases.Valid() {qbiases = qe.QBiases.TakeAxis(indices, 0)}- return mlx.Dequantize(weight, scales, qbiases, qe.GroupSize, qe.Bits, qe.Mode)+ out := mlx.Dequantize(weight, scales, qbiases, qe.GroupSize, qe.Bits, qe.Mode)+ if qe.GlobalScale != nil {+ out = mlx.Mul(out, qe.GlobalScale)+ }+ return out}func (qe *QuantizedEmbedding) AsLinear() LinearLayer {return &QuantizedLinear{- Weight: qe.Weight,- Scales: qe.Scales,- QBiases: qe.QBiases,- GroupSize: qe.GroupSize,- Bits: qe.Bits,- Mode: qe.Mode,+ Weight: qe.Weight,+ Scales: qe.Scales,+ QBiases: qe.QBiases,+ GlobalScale: qe.GlobalScale,+ GroupSize: qe.GroupSize,+ Bits: qe.Bits,+ Mode: qe.Mode,}}diff --git a/x/models/nn/nn_test.go b/x/models/nn/nn_test.goindex 8ec4b7575..8c475600e 100644--- a/x/models/nn/nn_test.go+++ b/x/models/nn/nn_test.go@@ -185,3 +185,40 @@ func TestQuantizedLinearMXFP4MatchesDequantizedWeight(t *testing.T) {}}}++func TestQuantizedEmbeddingAsLinearPreservesGlobalScale(t *testing.T) {+ weight := &mlx.Array{}+ scales := &mlx.Array{}+ qbiases := &mlx.Array{}+ globalScale := &mlx.Array{}++ embedding := &QuantizedEmbedding{+ Weight: weight,+ Scales: scales,+ QBiases: qbiases,+ GlobalScale: globalScale,+ GroupSize: 16,+ Bits: 4,+ Mode: "nvfp4",+ }++ linear, ok := embedding.AsLinear().(*QuantizedLinear)+ if !ok {+ t.Fatalf("AsLinear type = %T, want *QuantizedLinear", embedding.AsLinear())+ }+ if linear.Weight != weight {+ t.Fatalf("AsLinear Weight = %p, want %p", linear.Weight, weight)+ }+ if linear.Scales != scales {+ t.Fatalf("AsLinear Scales = %p, want %p", linear.Scales, scales)+ }+ if linear.QBiases != qbiases {+ t.Fatalf("AsLinear QBiases = %p, want %p", linear.QBiases, qbiases)+ }+ if linear.GlobalScale != globalScale {+ t.Fatalf("AsLinear GlobalScale = %p, want %p", linear.GlobalScale, globalScale)+ }+ if linear.GroupSize != 16 || linear.Bits != 4 || linear.Mode != "nvfp4" {+ t.Fatalf("AsLinear quant params = (%d, %d, %q), want (16, 4, %q)", linear.GroupSize, linear.Bits, linear.Mode, "nvfp4")+ }+}