Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
model: add gemma3 to the mlxrunner (#14276)
Patrick Devine041fb7763945
diff --git a/x/mlxrunner/imports.go b/x/mlxrunner/imports.goindex e8950eff8..1daad3e4d 100644--- a/x/mlxrunner/imports.go+++ b/x/mlxrunner/imports.go@@ -3,5 +3,6 @@package mlxrunnerimport (+ _ "github.com/ollama/ollama/x/models/gemma3"_ "github.com/ollama/ollama/x/models/glm4_moe_lite")diff --git a/x/mlxrunner/model/quant.go b/x/mlxrunner/model/quant.gonew file mode 100644index 000000000..3a17ab485--- /dev/null+++ b/x/mlxrunner/model/quant.go@@ -0,0 +1,130 @@+//go:build mlx++package model++import (+ "strings"++ "github.com/ollama/ollama/x/mlxrunner/mlx"+)++// QuantizationParams returns default groupSize, bits, and mode for a quantization type.+func QuantizationParams(quantization string) (groupSize, bits int, mode string) {+ switch strings.ToUpper(quantization) {+ case "NVFP4":+ return 16, 4, "nvfp4"+ case "FP4", "Q4", "INT4":+ return 32, 4, "affine"+ case "MXFP8":+ return 32, 8, "mxfp8"+ case "FP8", "Q8", "INT8", "":+ return 64, 8, "affine"+ default:+ return 32, 8, "affine"+ }+}++// TensorQuantParams resolves quant params for a tensor using per-tensor metadata+// when available, otherwise falling back to the provided model defaults.+func TensorQuantParams(+ defaultGroupSize, defaultBits int,+ defaultMode string,+ tensorQuant map[string]*TensorQuantInfo,+ tensorName string,+) (groupSize, bits int, mode string, fromTensor bool) {+ if tensorQuant != nil {+ if tq := tensorQuant[tensorName]; tq != nil {+ groupSize, bits, mode = QuantizationParams(tq.QuantType)+ if tq.GroupSize > 0 {+ groupSize = tq.GroupSize+ }+ return groupSize, bits, mode, true+ }+ }+ return defaultGroupSize, defaultBits, defaultMode, false+}++// ResolveLinearQuantParams resolves quantization params for a quantized linear+// tensor, preferring per-tensor metadata and falling back to shape-based+// inference for affine packed tensors.+func ResolveLinearQuantParams(+ defaultGroupSize, defaultBits int,+ defaultMode string,+ tensorQuant map[string]*TensorQuantInfo,+ tensorName string,+ weight, scales *mlx.Array,+) (groupSize, bits int, mode string) {+ groupSize, bits, mode, fromTensor := TensorQuantParams(+ defaultGroupSize,+ defaultBits,+ defaultMode,+ tensorQuant,+ tensorName,+ )++ if mode == "affine" {+ if inferredGroupSize, inferredBits, ok := InferAffineQuantParamsFromShapes(weight, scales, bits); ok {+ if !fromTensor || groupSize == 0 || bits == 0 {+ groupSize = inferredGroupSize+ bits = inferredBits+ }+ }+ }++ return groupSize, bits, mode+}++// InferAffineQuantParamsFromShapes infers (groupSize,bits) for affine quantized+// tensors from packed weight and scale shapes.+func InferAffineQuantParamsFromShapes(weight, scales *mlx.Array, hintBits int) (groupSize, bits int, ok bool) {+ if weight == nil || scales == nil {+ return 0, 0, false+ }++ weightShape := weight.Dims()+ scaleShape := scales.Dims()+ if len(weightShape) == 0 || len(scaleShape) == 0 {+ return 0, 0, false+ }++ weightCols := weightShape[len(weightShape)-1]+ scalesCols := scaleShape[len(scaleShape)-1]+ if weightCols <= 0 || scalesCols <= 0 {+ return 0, 0, false+ }++ groupSize4 := weightCols * 8 / scalesCols+ groupSize8 := weightCols * 4 / scalesCols++ switch {+ case groupSize4 == 32:+ return 32, 4, true+ case groupSize8 == 64:+ return 64, 8, true+ case groupSize4 == 64 && groupSize8 == 32:+ if hintBits == 8 {+ return 32, 8, true+ }+ if hintBits == 4 {+ return 64, 4, true+ }+ }++ if isCommonGroupSize(groupSize4) && !isCommonGroupSize(groupSize8) {+ return groupSize4, 4, true+ }+ if isCommonGroupSize(groupSize8) && !isCommonGroupSize(groupSize4) {+ return groupSize8, 8, true+ }++ return 0, 0, false+}++func isCommonGroupSize(v int) bool {+ switch v {+ case 16, 32, 64, 128:+ return true+ default:+ return false+ }+}diff --git a/x/mlxrunner/model/root.go b/x/mlxrunner/model/root.goindex 885647ab3..c912f7f4c 100644--- a/x/mlxrunner/model/root.go+++ b/x/mlxrunner/model/root.go@@ -8,42 +8,63 @@ import ("fmt""io""os"+ "sort"+ "strconv""strings""github.com/ollama/ollama/x/imagegen/manifest")+// TensorQuantInfo describes per-tensor quantization metadata.+type TensorQuantInfo struct {+ QuantType string+ GroupSize int+}+// Root wraps a ModelManifest with pre-scanned quantization metadata.type Root struct {- Manifest *manifest.ModelManifest+ Manifest *manifest.ModelManifest++ // Backwards-compatible model-level quant metadata (first tensor blob).quantType stringgroupSize int++ // Per-tensor quantization metadata.+ tensorQuant map[string]*TensorQuantInfo}-// Open loads a manifest for the given model name and pre-scans the first-// tensor blob for quantization metadata (quant_type, group_size).+// Open loads a manifest for the given model name and scans tensor blobs for+// quantization metadata.func Open(modelName string) (*Root, error) {m, err := manifest.LoadManifest(modelName)if err != nil {return nil, err}- root := &Root{Manifest: m}+ root := &Root{+ Manifest: m,+ tensorQuant: make(map[string]*TensorQuantInfo),+ }- // Pre-scan first tensor blob for quantization metadatafor _, layer := range m.GetTensorLayers("") {blobPath := m.BlobPath(layer.Digest)- meta, err := readBlobMetadata(blobPath)- if err != nil || meta == nil {++ infos, blobQuantType, blobGroupSize, err := readBlobTensorQuantInfo(blobPath)+ if err != nil {continue}- if qt := meta["quant_type"]; qt != "" {- root.quantType = strings.ToUpper(qt)++ for name, info := range infos {+ root.tensorQuant[name] = info}- if gs := meta["group_size"]; gs != "" {- fmt.Sscanf(gs, "%d", &root.groupSize)++ if root.quantType == "" && blobQuantType != "" {+ root.quantType = strings.ToUpper(blobQuantType)+ root.groupSize = blobGroupSize+ if root.groupSize == 0 {+ root.groupSize = defaultGroupSize(root.quantType)+ }}- break // only check the first tensor blob}return root, nil@@ -52,46 +73,180 @@ func Open(modelName string) (*Root, error) {// Close is a no-op for now (future: release resources).func (r *Root) Close() {}-// QuantType returns the quantization type detected from tensor metadata.+// QuantType returns the quantization type detected from the first tensor blob metadata.func (r *Root) QuantType() string { return r.quantType }-// GroupSize returns the quantization group size detected from tensor metadata.+// GroupSize returns the quantization group size detected from the first tensor blob metadata.func (r *Root) GroupSize() int { return r.groupSize }-// readBlobMetadata reads the __metadata__ from a safetensors blob header.-func readBlobMetadata(path string) (map[string]string, error) {+// TensorQuant returns per-tensor quantization metadata if available.+func (r *Root) TensorQuant(name string) *TensorQuantInfo {+ if r == nil {+ return nil+ }+ return r.tensorQuant[name]+}++// AllTensorQuant returns a copy of the per-tensor quantization metadata.+func (r *Root) AllTensorQuant() map[string]*TensorQuantInfo {+ out := make(map[string]*TensorQuantInfo, len(r.tensorQuant))+ for k, v := range r.tensorQuant {+ if v == nil {+ continue+ }+ copy := *v+ out[k] = ©+ }+ return out+}++func defaultGroupSize(quantType string) int {+ groupSize, _, _ := QuantizationParams(quantType)+ return groupSize+}++func readBlobTensorQuantInfo(path string) (map[string]*TensorQuantInfo, string, int, error) {f, err := os.Open(path)if err != nil {- return nil, err+ return nil, "", 0, err}defer f.Close()var headerSize uint64if err := binary.Read(f, binary.LittleEndian, &headerSize); err != nil {- return nil, err+ return nil, "", 0, err}- if headerSize > 1024*1024 {- return nil, fmt.Errorf("header too large: %d", headerSize)+ if headerSize > 100*1024*1024 {+ return nil, "", 0, fmt.Errorf("header too large: %d", headerSize)}data := make([]byte, headerSize)if _, err := io.ReadFull(f, data); err != nil {- return nil, err+ return nil, "", 0, err}var header map[string]json.RawMessageif err := json.Unmarshal(data, &header); err != nil {- return nil, err+ return nil, "", 0, err}+ globalQuantType, globalGroupSize := parseGlobalQuantMetadata(header)+ globalQuantType = strings.ToUpper(globalQuantType)++ mainNames := mainTensorNames(header)+ infos := make(map[string]*TensorQuantInfo)+ for _, name := range mainNames {+ if _, ok := header[name+".scale"]; !ok {+ continue+ }++ quantType := globalQuantType+ groupSize := globalGroupSize++ inferredType, inferredGroup := inferQuantTypeFromShapes(header, name, quantType)+ if quantType == "" {+ quantType = inferredType+ }+ if groupSize == 0 {+ groupSize = inferredGroup+ }+ if quantType == "" {+ continue+ }+ if groupSize == 0 {+ groupSize = defaultGroupSize(quantType)+ }++ infos[name] = &TensorQuantInfo{QuantType: quantType, GroupSize: groupSize}+ }++ return infos, globalQuantType, globalGroupSize, nil+}++func parseGlobalQuantMetadata(header map[string]json.RawMessage) (quantType string, groupSize int) {metaRaw, ok := header["__metadata__"]if !ok {- return nil, nil+ return "", 0}var meta map[string]stringif err := json.Unmarshal(metaRaw, &meta); err != nil {- return nil, err+ return "", 0+ }++ quantType = meta["quant_type"]+ if gs := meta["group_size"]; gs != "" {+ groupSize, _ = strconv.Atoi(gs)+ }+ return quantType, groupSize+}++func mainTensorNames(header map[string]json.RawMessage) []string {+ names := make([]string, 0, len(header))+ for name := range header {+ if name == "__metadata__" || strings.HasSuffix(name, ".scale") || strings.HasSuffix(name, ".bias") {+ continue+ }+ names = append(names, name)+ }+ sort.Strings(names)+ return names+}++func inferQuantTypeFromShapes(header map[string]json.RawMessage, tensorName string, hintQuantType string) (string, int) {+ type tensorShape struct {+ Shape []int64 `json:"shape"`+ }++ mainRaw, ok := header[tensorName]+ if !ok {+ return "", 0}- return meta, nil+ scaleRaw, ok := header[tensorName+".scale"]+ if !ok {+ return "", 0+ }++ var mainInfo tensorShape+ if err := json.Unmarshal(mainRaw, &mainInfo); err != nil || len(mainInfo.Shape) == 0 {+ return "", 0+ }++ var scaleInfo tensorShape+ if err := json.Unmarshal(scaleRaw, &scaleInfo); err != nil || len(scaleInfo.Shape) == 0 {+ return "", 0+ }++ weightCols := int(mainInfo.Shape[len(mainInfo.Shape)-1])+ scalesCols := int(scaleInfo.Shape[len(scaleInfo.Shape)-1])+ if weightCols <= 0 || scalesCols <= 0 {+ return "", 0+ }++ groupSize4 := weightCols * 8 / scalesCols+ groupSize8 := weightCols * 4 / scalesCols++ switch {+ case groupSize4 == 32:+ return "INT4", 32+ case groupSize8 == 64:+ return "INT8", 64+ case groupSize4 == 64 && groupSize8 == 32:+ h := strings.ToUpper(hintQuantType)+ if strings.Contains(h, "8") {+ return "INT8", 32+ }+ if strings.Contains(h, "4") {+ return "INT4", 64+ }+ }++ if isCommonGroupSize(groupSize4) && !isCommonGroupSize(groupSize8) {+ return "INT4", groupSize4+ }+ if isCommonGroupSize(groupSize8) && !isCommonGroupSize(groupSize4) {+ return "INT8", groupSize8+ }++ return "", 0}diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.goindex b7650b68d..cd4d78620 100644--- a/x/mlxrunner/pipeline.go+++ b/x/mlxrunner/pipeline.go@@ -24,9 +24,13 @@ func (r *Runner) TextGenerationPipeline(request Request) error {caches, tokens := r.FindNearestCache(inputs)if len(caches) == 0 {- caches = make([]cache.Cache, r.Model.NumLayers())- for i := range caches {- caches[i] = cache.NewKVCache()+ if cacheFactory, ok := r.Model.(interface{ NewCaches() []cache.Cache }); ok {+ caches = cacheFactory.NewCaches()+ } else {+ caches = make([]cache.Cache, r.Model.NumLayers())+ for i := range caches {+ caches[i] = cache.NewKVCache()+ }}}diff --git a/x/models/gemma3/gemma3.go b/x/models/gemma3/gemma3.gonew file mode 100644index 000000000..f35ef2a75--- /dev/null+++ b/x/models/gemma3/gemma3.go@@ -0,0 +1,521 @@+//go:build mlx++// Package gemma3 provides the Gemma 3 text model implementation for MLX.+package gemma3++import (+ "encoding/json"+ "fmt"+ "math"++ "github.com/ollama/ollama/x/imagegen/tokenizer"+ "github.com/ollama/ollama/x/mlxrunner/cache"+ "github.com/ollama/ollama/x/mlxrunner/mlx"+ "github.com/ollama/ollama/x/mlxrunner/model"+ "github.com/ollama/ollama/x/mlxrunner/model/base"+ "github.com/ollama/ollama/x/models/nn"+)++func init() {+ base.Register("Gemma3ForCausalLM", newModel)+ base.Register("Gemma3ForConditionalGeneration", newModel)+}++// TextConfig holds configuration for the Gemma 3 text model.+type TextConfig struct {+ HiddenSize int32 `json:"hidden_size"`+ NumHiddenLayers int32 `json:"num_hidden_layers"`+ IntermediateSize int32 `json:"intermediate_size"`+ NumAttentionHeads int32 `json:"num_attention_heads"`+ NumKeyValueHeads int32 `json:"num_key_value_heads"`+ HeadDim int32 `json:"head_dim"`+ VocabSize int32 `json:"vocab_size"`+ RMSNormEps float32 `json:"rms_norm_eps"`+ RopeTheta float32 `json:"rope_theta"`+ RopeLocalBaseFreq float32 `json:"rope_local_base_freq"`+ MaxPositionEmbeddings int32 `json:"max_position_embeddings"`+ SlidingWindow int32 `json:"sliding_window"`+ SlidingWindowPattern int32 `json:"sliding_window_pattern"`+ LayerTypes []string `json:"layer_types"`+ TieWordEmbeddings bool `json:"tie_word_embeddings"`++ // Quantization parameters (set during load based on model quantization).+ QuantGroupSize int `json:"-"`+ QuantBits int `json:"-"`+ QuantMode string `json:"-"`+ TensorQuant map[string]*model.TensorQuantInfo `json:"-"`++ // Computed fields.+ Scale float32 `json:"-"`+}++// Attention implements Gemma 3 attention with Q/K normalization.+type Attention struct {+ QProj nn.LinearLayer+ KProj nn.LinearLayer+ VProj nn.LinearLayer+ OProj nn.LinearLayer++ QNorm *nn.RMSNorm+ KNorm *nn.RMSNorm++ // Precomputed (1 + weight) for Gemma-style RMSNorm.+ QNormScaled *mlx.Array+ KNormScaled *mlx.Array+}++// MLP is the feed-forward network with GELU activation.+type MLP struct {+ GateProj nn.LinearLayer+ UpProj nn.LinearLayer+ DownProj nn.LinearLayer+}++// DecoderLayer is a single transformer block.+type DecoderLayer struct {+ InputNorm *nn.RMSNorm+ Attention *Attention+ PostAttnNorm *nn.RMSNorm+ PreFFNorm *nn.RMSNorm+ MLP *MLP+ PostFFNorm *nn.RMSNorm++ // Precomputed (1 + weight) for Gemma-style RMSNorm.+ InputNormScaled *mlx.Array+ PostAttnNormScaled *mlx.Array+ PreFFNormScaled *mlx.Array+ PostFFNormScaled *mlx.Array++ // Layer metadata.+ IsSliding bool+ LayerIdx int32+}++// Model is the Gemma 3 text-only model.+type Model struct {+ EmbedTokens *nn.Embedding+ Layers []*DecoderLayer+ Norm *nn.RMSNorm+ LMHead nn.LinearLayer++ // Precomputed (1 + weight) for Gemma-style RMSNorm.+ NormScaled *mlx.Array++ tok *tokenizer.Tokenizer+ *TextConfig++ weightPrefix string+}++func defaultHeads(numLayers int32) (numHeads, numKVHeads int32) {+ switch numLayers {+ case 34:+ return 8, 4+ case 48:+ return 16, 8+ case 62:+ return 32, 16+ default:+ return 8, 4+ }+}++func parseTextConfig(configData []byte) (TextConfig, bool, error) {+ var cfg TextConfig+ if err := json.Unmarshal(configData, &cfg); err != nil {+ return TextConfig{}, false, fmt.Errorf("parse config: %w", err)+ }++ var wrapped struct {+ TextConfig *TextConfig `json:"text_config"`+ }+ if err := json.Unmarshal(configData, &wrapped); err != nil {+ return TextConfig{}, false, fmt.Errorf("parse nested text config: %w", err)+ }++ fromConditional := wrapped.TextConfig != nil+ if fromConditional {+ cfg = *wrapped.TextConfig++ if cfg.HeadDim == 0 {+ cfg.HeadDim = 256+ }+ if cfg.NumAttentionHeads == 0 {+ cfg.NumAttentionHeads, cfg.NumKeyValueHeads = defaultHeads(cfg.NumHiddenLayers)+ }+ if cfg.NumKeyValueHeads == 0 {+ _, cfg.NumKeyValueHeads = defaultHeads(cfg.NumHiddenLayers)+ }+ if cfg.VocabSize == 0 {+ cfg.VocabSize = 262208+ }+ if cfg.SlidingWindowPattern == 0 && len(cfg.LayerTypes) == 0 {+ cfg.SlidingWindowPattern = 6+ }+ if cfg.MaxPositionEmbeddings == 0 {+ cfg.MaxPositionEmbeddings = 131072+ }+ }++ if cfg.HeadDim == 0 {+ cfg.HeadDim = 256+ }+ if cfg.NumAttentionHeads == 0 {+ cfg.NumAttentionHeads, cfg.NumKeyValueHeads = defaultHeads(cfg.NumHiddenLayers)+ }+ if cfg.NumKeyValueHeads == 0 {+ cfg.NumKeyValueHeads = max(1, cfg.NumAttentionHeads/2)+ }+ if cfg.RopeTheta == 0 {+ cfg.RopeTheta = 1000000+ }+ if cfg.RopeLocalBaseFreq == 0 {+ cfg.RopeLocalBaseFreq = 10000+ }+ if cfg.RMSNormEps == 0 {+ cfg.RMSNormEps = 1e-6+ }+ if cfg.VocabSize == 0 {+ cfg.VocabSize = 262208+ }++ cfg.Scale = float32(1.0 / math.Sqrt(float64(cfg.HeadDim)))++ return cfg, fromConditional, nil+}++func resolveWeightPrefix(tensors map[string]*mlx.Array) string {+ for _, prefix := range []string{"", "language_model."} {+ if tensors[prefix+"model.embed_tokens.weight"] != nil {+ return prefix+ }+ }+ return ""+}++func isLayerSliding(layerIdx int32, cfg *TextConfig) bool {+ if len(cfg.LayerTypes) > 0 && int(layerIdx) < len(cfg.LayerTypes) {+ return cfg.LayerTypes[layerIdx] == "sliding_attention"+ }+ if cfg.SlidingWindowPattern <= 0 {+ return false+ }+ return (layerIdx+1)%cfg.SlidingWindowPattern != 0+}++func precomputeGemmaScaledWeights(m *Model) {+ if m.Norm != nil {+ m.NormScaled = mlx.AddScalar(m.Norm.Weight, 1.0)+ }++ var scaled []*mlx.Array+ if m.NormScaled != nil {+ scaled = append(scaled, m.NormScaled)+ }++ for _, layer := range m.Layers {+ if layer == nil || layer.Attention == nil {+ continue+ }++ if layer.InputNorm != nil {+ layer.InputNormScaled = mlx.AddScalar(layer.InputNorm.Weight, 1.0)+ scaled = append(scaled, layer.InputNormScaled)+ }+ if layer.PostAttnNorm != nil {+ layer.PostAttnNormScaled = mlx.AddScalar(layer.PostAttnNorm.Weight, 1.0)+ scaled = append(scaled, layer.PostAttnNormScaled)+ }+ if layer.PreFFNorm != nil {+ layer.PreFFNormScaled = mlx.AddScalar(layer.PreFFNorm.Weight, 1.0)+ scaled = append(scaled, layer.PreFFNormScaled)+ }+ if layer.PostFFNorm != nil {+ layer.PostFFNormScaled = mlx.AddScalar(layer.PostFFNorm.Weight, 1.0)+ scaled = append(scaled, layer.PostFFNormScaled)+ }++ if layer.Attention.QNorm != nil {+ layer.Attention.QNormScaled = mlx.AddScalar(layer.Attention.QNorm.Weight, 1.0)+ scaled = append(scaled, layer.Attention.QNormScaled)+ }+ if layer.Attention.KNorm != nil {+ layer.Attention.KNormScaled = mlx.AddScalar(layer.Attention.KNorm.Weight, 1.0)+ scaled = append(scaled, layer.Attention.KNormScaled)+ }+ }++ if len(scaled) > 0 {+ mlx.Eval(scaled...)+ }+}++func newModel(root *model.Root) (base.Model, error) {+ configData, err := root.Manifest.ReadConfig("config.json")+ if err != nil {+ return nil, fmt.Errorf("load config: %w", err)+ }++ cfg, _, err := parseTextConfig(configData)+ if err != nil {+ return nil, err+ }++ if qt := root.QuantType(); qt != "" {+ cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode = model.QuantizationParams(qt)+ if gs := root.GroupSize(); gs > 0 {+ cfg.QuantGroupSize = gs+ }+ } else {+ cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode = model.QuantizationParams("")+ }+ cfg.TensorQuant = root.AllTensorQuant()++ tokData, err := root.Manifest.ReadConfig("tokenizer.json")+ if err != nil {+ return nil, fmt.Errorf("load tokenizer config: %w", err)+ }++ tokConfig := &tokenizer.TokenizerConfig{ConfigJSON: configData}+ if genConfigData, err := root.Manifest.ReadConfig("generation_config.json"); err == nil {+ tokConfig.GenerationConfigJSON = genConfigData+ }+ if tokConfigData, err := root.Manifest.ReadConfig("tokenizer_config.json"); err == nil {+ tokConfig.TokenizerConfigJSON = tokConfigData+ }++ tok, err := tokenizer.LoadFromBytesWithConfig(tokData, tokConfig)+ if err != nil {+ return nil, fmt.Errorf("parse tokenizer: %w", err)+ }++ m := &Model{+ Layers: make([]*DecoderLayer, cfg.NumHiddenLayers),+ TextConfig: &cfg,+ tok: tok,+ }++ for i := range m.Layers {+ m.Layers[i] = &DecoderLayer{+ LayerIdx: int32(i),+ IsSliding: isLayerSliding(int32(i), m.TextConfig),+ }+ }++ return m, nil+}++// LoadWeights receives all tensors loaded from the manifest and assigns them+// to model fields.+func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {+ m.weightPrefix = resolveWeightPrefix(tensors)+ prefix := m.weightPrefix+ linears := model.NewLinearFactory(tensors, m.QuantGroupSize, m.QuantBits, m.QuantMode, m.TensorQuant)++ embedWeight := tensors[prefix+"model.embed_tokens.weight"]+ if embedWeight == nil {+ return fmt.Errorf("missing embedding weight: %smodel.embed_tokens.weight", prefix)+ }+ m.EmbedTokens = nn.NewEmbedding(embedWeight)++ normWeight := tensors[prefix+"model.norm.weight"]+ if normWeight == nil {+ return fmt.Errorf("missing final norm weight: %smodel.norm.weight", prefix)+ }+ m.Norm = nn.NewRMSNorm(normWeight, m.RMSNormEps)++ if lmHead := linears.Make(prefix + "lm_head"); lmHead != nil {+ m.LMHead = lmHead+ } else if lmHead := linears.Make("lm_head"); lmHead != nil {+ m.LMHead = lmHead+ } else {+ // Gemma usually ties output projection to embeddings.+ m.LMHead = nn.NewLinear(embedWeight, nil)+ }++ for i := int32(0); i < m.NumHiddenLayers; i++ {+ layerPrefix := fmt.Sprintf("%smodel.layers.%d", prefix, i)++ layer := &DecoderLayer{+ LayerIdx: i,+ IsSliding: isLayerSliding(i, m.TextConfig),+ Attention: &Attention{},+ MLP: &MLP{},+ }++ if w := tensors[layerPrefix+".input_layernorm.weight"]; w != nil {+ layer.InputNorm = nn.NewRMSNorm(w, m.RMSNormEps)+ }+ if w := tensors[layerPrefix+".post_attention_layernorm.weight"]; w != nil {+ layer.PostAttnNorm = nn.NewRMSNorm(w, m.RMSNormEps)+ }+ if w := tensors[layerPrefix+".pre_feedforward_layernorm.weight"]; w != nil {+ layer.PreFFNorm = nn.NewRMSNorm(w, m.RMSNormEps)+ }+ if w := tensors[layerPrefix+".post_feedforward_layernorm.weight"]; w != nil {+ layer.PostFFNorm = nn.NewRMSNorm(w, m.RMSNormEps)+ }++ layer.Attention.QProj = linears.Make(layerPrefix + ".self_attn.q_proj")+ layer.Attention.KProj = linears.Make(layerPrefix + ".self_attn.k_proj")+ layer.Attention.VProj = linears.Make(layerPrefix + ".self_attn.v_proj")+ layer.Attention.OProj = linears.Make(layerPrefix + ".self_attn.o_proj")++ if w := tensors[layerPrefix+".self_attn.q_norm.weight"]; w != nil {+ layer.Attention.QNorm = nn.NewRMSNorm(w, m.RMSNormEps)+ }+ if w := tensors[layerPrefix+".self_attn.k_norm.weight"]; w != nil {+ layer.Attention.KNorm = nn.NewRMSNorm(w, m.RMSNormEps)+ }++ layer.MLP.GateProj = linears.Make(layerPrefix + ".mlp.gate_proj")+ layer.MLP.UpProj = linears.Make(layerPrefix + ".mlp.up_proj")+ layer.MLP.DownProj = linears.Make(layerPrefix + ".mlp.down_proj")++ if layer.InputNorm == nil {+ return fmt.Errorf("layer %d: missing input_layernorm", i)+ }+ if layer.PostAttnNorm == nil {+ return fmt.Errorf("layer %d: missing post_attention_layernorm", i)+ }+ if layer.PreFFNorm == nil {+ return fmt.Errorf("layer %d: missing pre_feedforward_layernorm", i)+ }+ if layer.PostFFNorm == nil {+ return fmt.Errorf("layer %d: missing post_feedforward_layernorm", i)+ }+ if layer.Attention.QProj == nil || layer.Attention.KProj == nil || layer.Attention.VProj == nil || layer.Attention.OProj == nil {+ return fmt.Errorf("layer %d: missing attention projections", i)+ }+ if layer.Attention.QNorm == nil || layer.Attention.KNorm == nil {+ return fmt.Errorf("layer %d: missing attention q/k norms", i)+ }+ if layer.MLP.GateProj == nil || layer.MLP.UpProj == nil || layer.MLP.DownProj == nil {+ return fmt.Errorf("layer %d: missing mlp projections", i)+ }++ m.Layers[i] = layer+ }++ precomputeGemmaScaledWeights(m)+ if m.NormScaled == nil {+ return fmt.Errorf("missing precomputed final norm weight")+ }+ collected := mlx.Collect(m)+ mlx.Eval(collected...)++ return nil+}++func (m *Model) Forward(tokens *mlx.Array, caches []cache.Cache) *mlx.Array {+ dims := tokens.Dims()+ B, L := int32(dims[0]), int32(dims[1])++ h := m.EmbedTokens.Forward(tokens)+ h = mlx.MulScalar(h, float32(math.Sqrt(float64(m.HiddenSize))))++ for i, layer := range m.Layers {+ var c cache.Cache+ if caches != nil && i < len(caches) {+ c = caches[i]+ }+ h = layer.Forward(h, c, B, L, m.TextConfig)+ }++ return mlx.RMSNormFn(h, m.NormScaled, m.RMSNormEps)+}++func (m *Model) Unembed(x *mlx.Array) *mlx.Array {+ return m.LMHead.Forward(x)+}++func (m *Model) NumLayers() int {+ return len(m.Layers)+}++func (m *Model) Tokenizer() *tokenizer.Tokenizer {+ return m.tok+}++// NewCaches creates cache objects for all layers.+func (m *Model) NewCaches() []cache.Cache {+ caches := make([]cache.Cache, len(m.Layers))+ for i, layer := range m.Layers {+ if m.SlidingWindow > 0 && layer.IsSliding {+ caches[i] = cache.NewRotatingKVCache(int(m.SlidingWindow))+ } else {+ caches[i] = cache.NewKVCache()+ }+ }+ return caches+}++// FormatPrompt applies the Gemma 3 chat template.+func (m *Model) FormatPrompt(prompt string) string {+ return fmt.Sprintf("<start_of_turn>user\n%s<end_of_turn>\n<start_of_turn>model\n", prompt)+}++func (l *DecoderLayer) Forward(x *mlx.Array, c cache.Cache, B, L int32, cfg *TextConfig) *mlx.Array {+ normed := mlx.RMSNormFn(x, l.InputNormScaled, cfg.RMSNormEps)++ attnOut := l.Attention.Forward(normed, c, B, L, l.IsSliding, cfg)+ attnOut = mlx.RMSNormFn(attnOut, l.PostAttnNormScaled, cfg.RMSNormEps)+ h := mlx.Add(x, attnOut)++ normed = mlx.RMSNormFn(h, l.PreFFNormScaled, cfg.RMSNormEps)++ mlpOut := l.MLP.Forward(normed)+ mlpOut = mlx.RMSNormFn(mlpOut, l.PostFFNormScaled, cfg.RMSNormEps)++ return mlx.Add(h, mlpOut)+}++func (a *Attention) Forward(x *mlx.Array, c cache.Cache, B, L int32, isSliding bool, cfg *TextConfig) *mlx.Array {+ q := a.QProj.Forward(x)+ k := a.KProj.Forward(x)+ v := a.VProj.Forward(x)++ q = mlx.Reshape(q, B, L, cfg.NumAttentionHeads, cfg.HeadDim)+ q = mlx.Transpose(q, 0, 2, 1, 3)++ k = mlx.Reshape(k, B, L, cfg.NumKeyValueHeads, cfg.HeadDim)+ k = mlx.Transpose(k, 0, 2, 1, 3)++ v = mlx.Reshape(v, B, L, cfg.NumKeyValueHeads, cfg.HeadDim)+ v = mlx.Transpose(v, 0, 2, 1, 3)++ q = mlx.RMSNormFn(q, a.QNormScaled, cfg.RMSNormEps)+ k = mlx.RMSNormFn(k, a.KNormScaled, cfg.RMSNormEps)++ ropeTheta := cfg.RopeTheta+ if isSliding {+ ropeTheta = cfg.RopeLocalBaseFreq+ }++ offset := 0+ if c != nil {+ offset = c.Offset()+ }+ q = mlx.RoPEWithBase(q, int(cfg.HeadDim), false, ropeTheta, 1.0, offset)+ k = mlx.RoPEWithBase(k, int(cfg.HeadDim), false, ropeTheta, 1.0, offset)++ if c != nil {+ k, v = c.Update(k, v)+ }++ repeatFactor := cfg.NumAttentionHeads / cfg.NumKeyValueHeads+ if repeatFactor > 1 {+ k = nn.RepeatKV(k, repeatFactor)+ v = nn.RepeatKV(v, repeatFactor)+ }++ out := mlx.ScaledDotProductAttentionCausal(q, k, v, cfg.Scale, L > 1)+ out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim)+ return a.OProj.Forward(out)+}++func (m *MLP) Forward(x *mlx.Array) *mlx.Array {+ gate := mlx.GELUApprox(m.GateProj.Forward(x))+ up := m.UpProj.Forward(x)+ return m.DownProj.Forward(mlx.Mul(gate, up))+}diff --git a/x/models/glm4_moe_lite/glm4_moe_lite.go b/x/models/glm4_moe_lite/glm4_moe_lite.goindex 974213196..65e26244d 100644--- a/x/models/glm4_moe_lite/glm4_moe_lite.go+++ b/x/models/glm4_moe_lite/glm4_moe_lite.go@@ -8,7 +8,6 @@ import ("encoding/json""fmt""math"- "strings""github.com/ollama/ollama/x/imagegen/tokenizer""github.com/ollama/ollama/x/mlxrunner/cache"@@ -64,9 +63,10 @@ type Config struct {RopeScaling *RopeScaling `json:"rope_scaling"`// Quantization parameters (set during load based on model quantization)- QuantGroupSize int `json:"-"` // Group size for quantization (default 64)- QuantBits int `json:"-"` // Bits per weight (4 or 8)- QuantMode string `json:"-"` // Quantization mode ("affine", etc.)+ QuantGroupSize int `json:"-"` // Group size for quantization (default 64)+ QuantBits int `json:"-"` // Bits per weight (4 or 8)+ QuantMode string `json:"-"` // Quantization mode ("affine", etc.)+ TensorQuant map[string]*model.TensorQuantInfo `json:"-"`// Computed fieldsQHeadDim int32 `json:"-"` // qk_nope_head_dim + qk_rope_head_dim@@ -372,22 +372,6 @@ func supportsGatherQMM(mode string, bits int) bool {return mode == "affine" && (bits == 4 || bits == 8)}-// quantizationParams returns groupSize, bits, mode for a quantization type string.-func quantizationParams(quantization string) (groupSize, bits int, mode string) {- switch strings.ToUpper(quantization) {- case "NVFP4":- return 16, 4, "nvfp4"- case "FP4", "Q4", "INT4":- return 32, 4, "affine"- case "MXFP8":- return 32, 8, "mxfp8"- case "FP8", "Q8", "INT8", "":- return 64, 8, "affine"- default:- return 32, 8, "affine"- }-}-// ExpertWeight holds a single expert's weight with optional quantization components.type ExpertWeight struct {Weight *mlx.Array@@ -408,7 +392,15 @@ func loadExpertWeight(tensors map[string]*mlx.Array, path string, useQuantized bif scales != nil {qbiases := tensors[path+".weight_qbias"]- groupSize, bits, mode := cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode+ groupSize, bits, mode := model.ResolveLinearQuantParams(+ cfg.QuantGroupSize,+ cfg.QuantBits,+ cfg.QuantMode,+ cfg.TensorQuant,+ path+".weight",+ w,+ scales,+ )if useQuantized && supportsGatherQMM(mode, bits) {return &ExpertWeight{Weight: w, Scales: scales, Biases: qbiases, Bits: bits, GroupSize: groupSize}@@ -492,7 +484,16 @@ func sanitizeMLAWeights(tensors map[string]*mlx.Array, prefix string, cfg *Confi// Check if quantized and dequantizeif scales := tensors[path+".weight_scale"]; scales != nil {qbiases := tensors[path+".weight_qbias"]- w = mlx.Dequantize(w, scales, qbiases, cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode)+ groupSize, bits, mode := model.ResolveLinearQuantParams(+ cfg.QuantGroupSize,+ cfg.QuantBits,+ cfg.QuantMode,+ cfg.TensorQuant,+ path+".weight",+ w,+ scales,+ )+ w = mlx.Dequantize(w, scales, qbiases, groupSize, bits, mode)}headDim := cfg.QKNopeHeadDim + cfg.VHeadDim@@ -507,32 +508,6 @@ func sanitizeMLAWeights(tensors map[string]*mlx.Array, prefix string, cfg *Confireturn embedQ, unembedOut}-// makeLinear creates a Linear or QuantizedLinear layer from the tensor map.-func makeLinear(tensors map[string]*mlx.Array, path string, cfg *Config) nn.LinearLayer {- w := tensors[path+".weight"]- if w == nil {- return nil- }-- scales := tensors[path+".weight_scale"]- if scales != nil {- qbiases := tensors[path+".weight_qbias"]- bias := tensors[path+".bias"]- return &nn.QuantizedLinear{- Weight: w,- Scales: scales,- QBiases: qbiases,- Bias: bias,- GroupSize: cfg.QuantGroupSize,- Bits: cfg.QuantBits,- Mode: cfg.QuantMode,- }- }-- bias := tensors[path+".bias"]- return nn.NewLinear(w, bias)-}-// newModel creates a new GLM4-MoE-Lite model from a Root (config + tokenizer,// no weights loaded yet). Called by the registry via base.New().func newModel(root *model.Root) (base.Model, error) {@@ -551,13 +526,14 @@ func newModel(root *model.Root) (base.Model, error) {// Set up quantization parameters from pre-scanned metadataif qt := root.QuantType(); qt != "" {- _, cfg.QuantBits, cfg.QuantMode = quantizationParams(qt)+ cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode = model.QuantizationParams(qt)if gs := root.GroupSize(); gs > 0 {cfg.QuantGroupSize = gs- } else {- cfg.QuantGroupSize, _, _ = quantizationParams(qt)}+ } else {+ cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode = model.QuantizationParams("")}+ cfg.TensorQuant = root.AllTensorQuant()// Load tokenizertokData, err := root.Manifest.ReadConfig("tokenizer.json")@@ -596,7 +572,20 @@ func newModel(root *model.Root) (base.Model, error) {// layer creation.func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {cfg := m.Config+ linears := model.NewLinearFactory(tensors, cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode, cfg.TensorQuant)useQuantized := supportsGatherQMM(cfg.QuantMode, cfg.QuantBits)+ if !useQuantized && cfg.TensorQuant != nil {+ for _, tq := range cfg.TensorQuant {+ if tq == nil {+ continue+ }+ _, bits, mode := model.QuantizationParams(tq.QuantType)+ if supportsGatherQMM(mode, bits) {+ useQuantized = true+ break+ }+ }+ }// Load embeddingif w := tensors["model.embed_tokens.weight"]; w != nil {@@ -609,7 +598,7 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {}// Load LM head- m.LMHead = makeLinear(tensors, "lm_head", cfg)+ m.LMHead = linears.Make("lm_head")// Load layersfor i := int32(0); i < cfg.NumHiddenLayers; i++ {@@ -617,16 +606,16 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {// Load attention (same for both block types)attn := &MLAAttention{}- attn.QAProj = makeLinear(tensors, prefix+".self_attn.q_a_proj", cfg)+ attn.QAProj = linears.Make(prefix + ".self_attn.q_a_proj")if w := tensors[prefix+".self_attn.q_a_layernorm.weight"]; w != nil {attn.QALayerNorm = nn.NewRMSNorm(w, cfg.RMSNormEps)}- attn.QBProj = makeLinear(tensors, prefix+".self_attn.q_b_proj", cfg)- attn.KVAProjWithMQA = makeLinear(tensors, prefix+".self_attn.kv_a_proj_with_mqa", cfg)+ attn.QBProj = linears.Make(prefix + ".self_attn.q_b_proj")+ attn.KVAProjWithMQA = linears.Make(prefix + ".self_attn.kv_a_proj_with_mqa")if w := tensors[prefix+".self_attn.kv_a_layernorm.weight"]; w != nil {attn.KVALayerNorm = nn.NewRMSNorm(w, cfg.RMSNormEps)}- attn.OProj = makeLinear(tensors, prefix+".self_attn.o_proj", cfg)+ attn.OProj = linears.Make(prefix + ".self_attn.o_proj")// Sanitize MLA weights for absorbed attentionembedQ, unembedOut := sanitizeMLAWeights(tensors, prefix, cfg)@@ -647,9 +636,9 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {}block.MLP = &DenseMLP{- GateProj: makeLinear(tensors, prefix+".mlp.gate_proj", cfg),- UpProj: makeLinear(tensors, prefix+".mlp.up_proj", cfg),- DownProj: makeLinear(tensors, prefix+".mlp.down_proj", cfg),+ GateProj: linears.Make(prefix + ".mlp.gate_proj"),+ UpProj: linears.Make(prefix + ".mlp.up_proj"),+ DownProj: linears.Make(prefix + ".mlp.down_proj"),}m.Layers[i] = block@@ -690,7 +679,7 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {}moeGate := &MoEGate{}- moeGate.Gate = makeLinear(tensors, prefix+".mlp.gate", cfg)+ moeGate.Gate = linears.Make(prefix + ".mlp.gate")if bias := tensors[prefix+".mlp.gate.e_score_correction_bias"]; bias != nil {moeGate.EScoreCorrectionBias = bias}@@ -703,9 +692,9 @@ func (m *Model) LoadWeights(tensors map[string]*mlx.Array) error {// Load shared experts if presentif cfg.NSharedExperts > 0 {block.MoE.SharedExperts = &SharedExperts{- GateProj: makeLinear(tensors, prefix+".mlp.shared_experts.gate_proj", cfg),- UpProj: makeLinear(tensors, prefix+".mlp.shared_experts.up_proj", cfg),- DownProj: makeLinear(tensors, prefix+".mlp.shared_experts.down_proj", cfg),+ GateProj: linears.Make(prefix + ".mlp.shared_experts.gate_proj"),+ UpProj: linears.Make(prefix + ".mlp.shared_experts.up_proj"),+ DownProj: linears.Make(prefix + ".mlp.shared_experts.down_proj"),}}