Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
chore: remove old imagegen LLMs models (#14597)
Patrick Devine110eff01a9f8
diff --git a/cmd/cmd.go b/cmd/cmd.goindex 031b200a8..518458d07 100644--- a/cmd/cmd.go+++ b/cmd/cmd.go@@ -606,17 +606,6 @@ func RunHandler(cmd *cobra.Command, args []string) error {}opts.WordWrap = !nowrap- useImagegen := false- if cmd.Flags().Lookup("imagegen") != nil {- useImagegen, err = cmd.Flags().GetBool("imagegen")- if err != nil {- return err- }- }- if useImagegen {- opts.Options["use_imagegen_runner"] = true- }-// Fill out the rest of the options based on information about the// model.client, err := api.ClientFromEnvironment()diff --git a/server/routes.go b/server/routes.goindex 43f76ea08..053b2fdc1 100644--- a/server/routes.go+++ b/server/routes.go@@ -161,7 +161,7 @@ func (s *Server) scheduleRunner(ctx context.Context, name string, caps []model.Creturn nil, nil, nil, fmt.Errorf("%s %w", name, err)}- useImagegen, _ := requestOpts["use_imagegen_runner"].(bool)+ // Deprecated runner override option; ignore if present.delete(requestOpts, "use_imagegen_runner")opts, err := s.modelOptions(model, requestOpts)@@ -169,7 +169,7 @@ func (s *Server) scheduleRunner(ctx context.Context, name string, caps []model.Creturn nil, nil, nil, err}- runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive, useImagegen)+ runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive)var runner *runnerRefselect {case runner = <-runnerCh:diff --git a/server/sched.go b/server/sched.goindex 4a64223e5..3d0dac863 100644--- a/server/sched.go+++ b/server/sched.go@@ -33,7 +33,6 @@ type LlmRequest struct {successCh chan *runnerReferrCh chan errorschedAttempts uint- useImagegen bool}type Scheduler struct {@@ -106,7 +105,7 @@ func schedulerModelKey(m *Model) string {}// context must be canceled to decrement ref count and release the runner-func (s *Scheduler) GetRunner(c context.Context, m *Model, opts api.Options, sessionDuration *api.Duration, useImagegen bool) (chan *runnerRef, chan error) {+func (s *Scheduler) GetRunner(c context.Context, m *Model, opts api.Options, sessionDuration *api.Duration) (chan *runnerRef, chan error) {if opts.NumCtx < 4 {opts.NumCtx = 4}@@ -123,7 +122,6 @@ func (s *Scheduler) GetRunner(c context.Context, m *Model, opts api.Options, sessessionDuration: sessionDuration,successCh: make(chan *runnerRef, 1),errCh: make(chan error, 1),- useImagegen: useImagegen,}key := schedulerModelKey(req.model)@@ -593,20 +591,15 @@ iGPUScan:return false}-// loadMLX loads an experimental safetensors model using the unified MLX runner.-// This supports both LLM (completion) and image generation models.+// loadMLX loads an experimental safetensors model using MLX runners.+// Image models use x/imagegen; LLM models use x/mlxrunner.func (s *Scheduler) loadMLX(req *LlmRequest) bool {modelName := req.model.ShortNamevar server llm.LlamaServervar err error- isImagegen := falseif slices.Contains(req.model.Config.Capabilities, "image") {- server, err = imagegen.NewServer(modelName, imagegen.ModeImageGen)- isImagegen = true- } else if req.useImagegen {- server, err = imagegen.NewServer(modelName, imagegen.ModeLLM)- isImagegen = true+ server, err = imagegen.NewServer(modelName)} else {server, err = mlxrunner.NewClient(modelName)}@@ -628,7 +621,7 @@ func (s *Scheduler) loadMLX(req *LlmRequest) bool {llama: server,Options: &req.opts,loading: false,- isImagegen: isImagegen,+ isImagegen: slices.Contains(req.model.Config.Capabilities, "image"),sessionDuration: sessionDuration,totalSize: totalSize,vramSize: vramSize,@@ -737,8 +730,8 @@ func (runner *runnerRef) needsReload(ctx context.Context, req *LlmRequest) boolrunner.refMu.Lock()defer runner.refMu.Unlock()- // Check if runner type (imagegen vs mlxrunner) matches what's requested- wantImagegen := req.useImagegen || slices.Contains(req.model.Config.Capabilities, "image")+ // Check if runner type (imagegen vs mlxrunner) matches what's requested.+ wantImagegen := slices.Contains(req.model.Config.Capabilities, "image")if runner.isImagegen != wantImagegen {return true}diff --git a/server/sched_test.go b/server/sched_test.goindex a21f0a709..0b79c7834 100644--- a/server/sched_test.go+++ b/server/sched_test.go@@ -408,10 +408,10 @@ func TestSchedGetRunner(t *testing.T) {s.getSystemInfoFn = getSystemInfoFns.newServerFn = a.newServerslog.Info("a")- successCh1a, errCh1a := s.GetRunner(a.ctx, a.req.model, a.req.opts, a.req.sessionDuration, false)+ successCh1a, errCh1a := s.GetRunner(a.ctx, a.req.model, a.req.opts, a.req.sessionDuration)require.Len(t, s.pendingReqCh, 1)slog.Info("b")- successCh1b, errCh1b := s.GetRunner(b.ctx, b.req.model, b.req.opts, b.req.sessionDuration, false)+ successCh1b, errCh1b := s.GetRunner(b.ctx, b.req.model, b.req.opts, b.req.sessionDuration)require.Len(t, s.pendingReqCh, 1)require.Empty(t, successCh1b)require.Len(t, errCh1b, 1)@@ -435,7 +435,7 @@ func TestSchedGetRunner(t *testing.T) {c.req.model.ModelPath = "bad path"slog.Info("c")- successCh1c, errCh1c := s.GetRunner(c.ctx, c.req.model, c.req.opts, c.req.sessionDuration, false)+ successCh1c, errCh1c := s.GetRunner(c.ctx, c.req.model, c.req.opts, c.req.sessionDuration)// Starts in pending channel, then should be quickly processed to return an errortime.Sleep(50 * time.Millisecond) // Long enough for the "a" model to expire and unloadrequire.Empty(t, successCh1c)@@ -470,7 +470,7 @@ func TestSchedGetRunnerUsesDigestKeyWhenModelPathEmpty(t *testing.T) {s.loadedMu.Unlock()reqModel := &Model{Name: "safetensors-b", Digest: "sha-b"}- successCh, errCh := s.GetRunner(ctx, reqModel, opts, nil, false)+ successCh, errCh := s.GetRunner(ctx, reqModel, opts, nil)require.Empty(t, successCh)require.Empty(t, errCh)@@ -499,7 +499,7 @@ func TestSchedGetRunnerReusesSameDigestWhenModelPathEmpty(t *testing.T) {s.loadedMu.Unlock()reqCtx, cancelReq := context.WithCancel(ctx)- successCh, errCh := s.GetRunner(reqCtx, &Model{Name: "safetensors-a-copy", Digest: "sha-a"}, opts, nil, false)+ successCh, errCh := s.GetRunner(reqCtx, &Model{Name: "safetensors-a-copy", Digest: "sha-a"}, opts, nil)cancelReq()select {@@ -574,7 +574,7 @@ func TestSchedPrematureExpired(t *testing.T) {s.getGpuFn = getGpuFns.getSystemInfoFn = getSystemInfoFns.newServerFn = scenario1a.newServer- successCh1a, errCh1a := s.GetRunner(scenario1a.ctx, scenario1a.req.model, scenario1a.req.opts, scenario1a.req.sessionDuration, false)+ successCh1a, errCh1a := s.GetRunner(scenario1a.ctx, scenario1a.req.model, scenario1a.req.opts, scenario1a.req.sessionDuration)require.Len(t, s.pendingReqCh, 1)s.Run(ctx)select {diff --git a/x/imagegen/cmd/engine/README.md b/x/imagegen/cmd/engine/README.mdindex 02dc0b979..3991c02a8 100644--- a/x/imagegen/cmd/engine/README.md+++ b/x/imagegen/cmd/engine/README.md@@ -10,17 +10,7 @@ go build -tags mlx -o engine ./x/imagegen/cmd/engine## Text Generation-```bash-./engine -model /path/to/model -prompt "Hello" -max-tokens 100-```--Options:--- `-temperature` - sampling temperature (default 0.7)-- `-top-p` - nucleus sampling (default 0.9)-- `-top-k` - top-k sampling (default 40)--Supports: Llama, Gemma3, GPT-OSS+Text generation models are no longer supported by this engine.## Image Generationdiff --git a/x/imagegen/cmd/engine/main.go b/x/imagegen/cmd/engine/main.goindex f0e705d1c..6ec7de9e1 100644--- a/x/imagegen/cmd/engine/main.go+++ b/x/imagegen/cmd/engine/main.go@@ -18,9 +18,6 @@ import ("github.com/ollama/ollama/x/imagegen""github.com/ollama/ollama/x/imagegen/mlx""github.com/ollama/ollama/x/imagegen/models/flux2"- "github.com/ollama/ollama/x/imagegen/models/gemma3"- "github.com/ollama/ollama/x/imagegen/models/gpt_oss"- "github.com/ollama/ollama/x/imagegen/models/llama""github.com/ollama/ollama/x/imagegen/models/zimage""github.com/ollama/ollama/x/imagegen/safetensors")@@ -170,11 +167,11 @@ func main() {log.Fatal(err)}- // Load image if provided and model supports it+ // Load image if provided and model supports it.var image *mlx.Arrayif *imagePath != "" {if mm, ok := m.(interface{ ImageSize() int32 }); ok {- image, err = gemma3.ProcessImage(*imagePath, mm.ImageSize())+ image, err = imagegen.ProcessImage(*imagePath, mm.ImageSize())if err != nil {log.Fatal("load image:", err)}@@ -236,14 +233,8 @@ func load(modelPath string) (Model, error) {}switch kind {- case "gpt_oss":- return gpt_oss.Load(modelPath)- case "gemma3":- return gemma3.Load(modelPath)- case "gemma3_text":- return gemma3.LoadText(modelPath)default:- return llama.Load(modelPath)+ return nil, fmt.Errorf("model type %q is not supported by x/imagegen/cmd/engine", kind)}}diff --git a/x/imagegen/models/gemma3/image.go b/x/imagegen/image_processor.gosimilarity index 73%rename from x/imagegen/models/gemma3/image.gorename to x/imagegen/image_processor.goindex 9532d852d..7a562feb5 100644--- a/x/imagegen/models/gemma3/image.go+++ b/x/imagegen/image_processor.go@@ -1,6 +1,6 @@//go:build mlx-package gemma3+package imagegenimport ("fmt"@@ -13,8 +13,8 @@ import ("golang.org/x/image/draw")-// ProcessImage loads and preprocesses an image for the vision tower-// Returns [1, H, W, C] tensor in NHWC format normalized for SigLIP+// ProcessImage loads and preprocesses an image for multimodal vision towers.+// Returns [1, H, W, C] tensor in NHWC format normalized for SigLIP.func ProcessImage(path string, imageSize int32) (*mlx.Array, error) {f, err := os.Open(path)if err != nil {@@ -30,20 +30,20 @@ func ProcessImage(path string, imageSize int32) (*mlx.Array, error) {return ProcessImageData(img, imageSize)}-// ProcessImageData preprocesses an image.Image for the vision tower+// ProcessImageData preprocesses an image.Image for multimodal vision towers.func ProcessImageData(img image.Image, imageSize int32) (*mlx.Array, error) {- // Resize to target size using bilinear interpolation+ // Resize to target size using bilinear interpolation.resized := image.NewRGBA(image.Rect(0, 0, int(imageSize), int(imageSize)))draw.BiLinear.Scale(resized, resized.Bounds(), img, img.Bounds(), draw.Over, nil)- // Convert to float32 array [H, W, C] and normalize- // SigLIP normalization: (pixel / 255.0 - 0.5) / 0.5 = pixel / 127.5 - 1.0+ // Convert to float32 array [H, W, C] and normalize.+ // SigLIP normalization: (pixel / 255.0 - 0.5) / 0.5 = pixel / 127.5 - 1.0.data := make([]float32, imageSize*imageSize*3)idx := 0for y := int32(0); y < imageSize; y++ {for x := int32(0); x < imageSize; x++ {r, g, b, _ := resized.At(int(x), int(y)).RGBA()- // RGBA returns 16-bit values, convert to 8-bit+ // RGBA returns 16-bit values, convert to 8-bit.data[idx] = float32(r>>8)/127.5 - 1.0data[idx+1] = float32(g>>8)/127.5 - 1.0data[idx+2] = float32(b>>8)/127.5 - 1.0@@ -51,8 +51,8 @@ func ProcessImageData(img image.Image, imageSize int32) (*mlx.Array, error) {}}- // Create MLX array [1, H, W, C] for NHWC layout+ // Create MLX array [1, H, W, C] for NHWC layout.arr := mlx.NewArrayFloat32(data, []int32{1, imageSize, imageSize, 3})- mlx.Eval(arr) // Materialize to prevent use-after-free+ mlx.Eval(arr) // Materialize to prevent use-after-free.return arr, nil}diff --git a/x/imagegen/llm.go b/x/imagegen/llm.godeleted file mode 100644index eda3b64b1..000000000--- a/x/imagegen/llm.go+++ /dev/null@@ -1,420 +0,0 @@-//go:build mlx--package imagegen--import (- "encoding/json"- "errors"- "fmt"- "log/slog"- "net/http"- "strings"- "sync"- "time"-- "github.com/ollama/ollama/x/imagegen/cache"- "github.com/ollama/ollama/x/imagegen/manifest"- "github.com/ollama/ollama/x/imagegen/mlx"- "github.com/ollama/ollama/x/imagegen/models/glm4_moe_lite"- "github.com/ollama/ollama/x/imagegen/tokenizer"-)--// TextModel is the interface for LLM text generation models.-type TextModel interface {- Forward(tokens *mlx.Array, caches []cache.Cache) *mlx.Array- NewCache(maxSeqLen int32) []cache.Cache- Tokenizer() *tokenizer.Tokenizer- VocabSize() int32- MaxContextLength() int32- NumLayers() int-}--// llmState holds the state for LLM generation-type llmState struct {- model TextModel-}--var llmMu sync.Mutex--// Dedicated stream for generation (like mlx-lm's generation_stream)-var generationStream *mlx.Stream--// withStream runs fn with the generation stream as default-func withStream(fn func()) {- // Lazy initialization of generationStream- if generationStream == nil {- generationStream = mlx.NewStream()- }- orig := mlx.GetDefaultStream()- mlx.SetDefaultStream(generationStream)- fn()- mlx.SetDefaultStream(orig)-}--// Decoder wraps model + cache for autoregressive generation.-// This matches the pattern from cmd/engine/generate.go-type Decoder struct {- model TextModel- caches []cache.Cache- vocabSize int32- temp float32- token *mlx.Array // Current token (kept across iterations)- oldCacheState []*mlx.Array // Preallocated slice for old cache state-}--func NewDecoder(m TextModel, temp float32) *Decoder {- caches := m.NewCache(0)- return &Decoder{- model: m,- caches: caches,- vocabSize: m.VocabSize(),- temp: temp,- oldCacheState: make([]*mlx.Array, 0, len(caches)*2),- }-}--func (d *Decoder) prefill(inputIDs []int32) int {- processed := 0-- // Track old cache state to free after each chunk- var oldCacheState []*mlx.Array-- // Process all-but-1 tokens in chunks, eval cache state for memory management- for len(inputIDs) > 1 {- chunkSize := min(2048, len(inputIDs)-1)- if chunkSize <= 0 {- break- }- chunk := inputIDs[:chunkSize]-- // Save old cache state before forward- oldCacheState = oldCacheState[:0]- for _, c := range d.caches {- oldCacheState = append(oldCacheState, c.State()...)- }-- var cacheState []*mlx.Array- withStream(func() {- x := mlx.NewArrayInt32(chunk, []int32{1, int32(len(chunk))})- d.model.Forward(x, d.caches)- for _, c := range d.caches {- cacheState = append(cacheState, c.State()...)- }- })- mlx.Eval(cacheState...)-- // Free old cache state- for _, arr := range oldCacheState {- if arr != nil {- arr.Free()- }- }-- inputIDs = inputIDs[chunkSize:]- processed += chunkSize- }-- // Save old cache state before final step- oldCacheState = oldCacheState[:0]- for _, c := range d.caches {- oldCacheState = append(oldCacheState, c.State()...)- }-- // Final token + sampling- withStream(func() {- x := mlx.NewArrayInt32(inputIDs, []int32{1, int32(len(inputIDs))})- mlx.Eval(x) // Materialize before any other evals- logits := d.model.Forward(x, d.caches)- d.token = sample(logits, d.temp, d.vocabSize)- })- // Keep cache state (token auto-kept by AsyncEval)- for _, c := range d.caches {- mlx.Keep(c.State()...)- }- mlx.AsyncEval(d.token)-- // Free old cache state from before final step- for _, arr := range oldCacheState {- if arr != nil {- arr.Free()- }- }-- mlx.ClearCache()-- return processed + len(inputIDs)-}--func (d *Decoder) step() int32 {- prevToken := d.token-- // Save old cache state (reuse preallocated slice)- d.oldCacheState = d.oldCacheState[:0]- for _, c := range d.caches {- d.oldCacheState = append(d.oldCacheState, c.State()...)- }-- withStream(func() {- logits := d.model.Forward(mlx.Reshape(prevToken, 1, 1), d.caches)- d.token = sample(logits, d.temp, d.vocabSize)- })- // Keep token and new cache state so they survive cleanup- mlx.Keep(d.token)- for _, c := range d.caches {- mlx.Keep(c.State()...)- }- mlx.AsyncEval(d.token)-- // Sync on previous token (GPU already working on next step)- val := prevToken.ItemInt32()-- // Free old token and old cache state- prevToken.Free()- for _, arr := range d.oldCacheState {- arr.Free()- }- return val-}--// sample samples from logits using temperature scaling-func sample(logits *mlx.Array, temp float32, vocabSize int32) *mlx.Array {- // Get last position logits: [1, L, vocab] -> [vocab]- shape := logits.Shape()- seqLen := shape[1]- lastLogits := mlx.Slice(logits, []int32{0, seqLen - 1, 0}, []int32{1, seqLen, vocabSize})- lastLogits = mlx.Reshape(lastLogits, vocabSize)-- if temp <= 0 || temp < 0.01 {- // Greedy decoding- return mlx.Argmax(lastLogits, -1, false)- }-- // Apply temperature scaling- scaled := mlx.DivScalar(lastLogits, temp)- return mlx.RandomCategorical(scaled, -1, 1)-}--// loadLLMModel loads a safetensors LLM model and its tokenizer from manifest storage.-func (s *server) loadLLMModel() error {- // Load the manifest to get model information- modelManifest, err := manifest.LoadManifest(s.modelName)- if err != nil {- return fmt.Errorf("failed to load manifest: %w", err)- }-- // Detect model architecture from config.json- configData, err := modelManifest.ReadConfig("config.json")- if err != nil {- return fmt.Errorf("failed to read config.json: %w", err)- }-- var modelConfig struct {- Architectures []string `json:"architectures"`- ModelType string `json:"model_type"`- }- if err := json.Unmarshal(configData, &modelConfig); err != nil {- return fmt.Errorf("failed to parse config.json: %w", err)- }-- arch := ""- if len(modelConfig.Architectures) > 0 {- arch = modelConfig.Architectures[0]- }- if arch == "" {- arch = modelConfig.ModelType- }-- slog.Info("detected LLM architecture", "architecture", arch, "model_type", modelConfig.ModelType)-- // Load the appropriate model based on architecture- var model TextModel- archLower := strings.ToLower(arch)-- switch {- case strings.Contains(archLower, "glm4moelite"):- m, err := glm4_moe_lite.LoadFromManifest(modelManifest)- if err != nil {- return fmt.Errorf("failed to load glm4-moe-lite model: %w", err)- }- model = m- slog.Info("loaded glm4-moe-lite model", "vocab_size", m.VocabSize(), "layers", m.NumLayers())-- default:- return fmt.Errorf("LLM architecture %q is not yet supported. "+- "Supported architectures: glm4-moe-lite. "+- "Please convert your model to GGUF format or use a supported architecture", arch)- }-- s.llmModel = &llmState{- model: model,- }-- return nil-}--// handleLLMCompletion handles LLM text generation requests.-func (s *server) handleLLMCompletion(w http.ResponseWriter, r *http.Request, req Request) {- if s.llmModel == nil {- http.Error(w, "LLM model not loaded", http.StatusInternalServerError)- return- }-- // Serialize generation requests- llmMu.Lock()- defer llmMu.Unlock()-- if err := s.llmGenerate(w, r, req); err != nil {- slog.Error("LLM generation failed", "error", err)- // Don't send error if we've already started streaming- }-}--// llmGenerate runs the generation loop using the Decoder pattern from cmd/engine-func (s *server) llmGenerate(w http.ResponseWriter, r *http.Request, req Request) error {- state := s.llmModel-- // Set up streaming response- w.Header().Set("Content-Type", "application/x-ndjson")- w.Header().Set("Transfer-Encoding", "chunked")- flusher, ok := w.(http.Flusher)- if !ok {- return errors.New("streaming not supported")- }-- tok := state.model.Tokenizer()-- // The prompt is already formatted by the server using the model's renderer- // (see server/prompt.go renderPrompt), so we don't apply FormatPrompt here.- prompt := req.Prompt-- // Tokenize the prompt- inputIDs := tok.Encode(prompt, true)- slog.Debug("tokenized prompt", "num_tokens", len(inputIDs))-- // Generation parameters- maxTokens := int(state.model.MaxContextLength())- if maxTokens <= 0 {- maxTokens = 4096- }- if req.Options != nil && req.Options.NumPredict > 0 {- maxTokens = req.Options.NumPredict- }-- temperature := float32(0.7)- if req.Options != nil && req.Options.Temperature > 0 {- temperature = float32(req.Options.Temperature)- }-- // Enable MLX compilation for better performance- mlx.EnableCompile()-- // Create decoder with fresh caches- dec := NewDecoder(state.model, temperature)-- prefillStart := time.Now()- prefillTokens := dec.prefill(inputIDs)- // Prefill measurement includes time to first token- firstToken := dec.step()- prefillDuration := time.Since(prefillStart)- promptEvalDuration := prefillDuration-- enc := json.NewEncoder(w)- ctx := r.Context()- generated := 0- stopReason := "max_tokens"-- // Handle first token- generated++- if tok.IsEOS(firstToken) {- resp := Response{- Done: true,- StopReason: fmt.Sprintf("first_token_eos:%d", firstToken),- PromptEvalCount: prefillTokens,- PromptEvalDuration: int(promptEvalDuration.Nanoseconds()),- }- enc.Encode(resp)- flusher.Flush()- return nil- }-- text := tok.Decode([]int32{firstToken})- resp := Response{Content: text}- enc.Encode(resp)- flusher.Flush()-- genStart := time.Now()-- // Generation loop- for n := 1; n < maxTokens; n++ {- // Check for cancellation- select {- case <-ctx.Done():- stopReason = fmt.Sprintf("context_cancelled:%d", generated)- break- default:- }- if stopReason != "max_tokens" {- break- }-- token := dec.step()- generated++-- if tok.IsEOS(token) {- stopReason = fmt.Sprintf("eos_token:%d", token)- break- }-- text := tok.Decode([]int32{token})-- // Check for stop sequences- if req.Options != nil && len(req.Options.Stop) > 0 {- shouldStop := false- var matchedStop string- for _, stop := range req.Options.Stop {- if strings.Contains(text, stop) {- text = strings.Split(text, stop)[0]- shouldStop = true- matchedStop = stop- break- }- }- if shouldStop {- if text != "" {- resp := Response{Content: text}- enc.Encode(resp)- flusher.Flush()- }- stopReason = fmt.Sprintf("stop_sequence:%s", matchedStop)- break- }- }-- resp := Response{Content: text}- enc.Encode(resp)- flusher.Flush()-- // Periodically clear MLX cache- if n%256 == 0 {- mlx.ClearCache()- }- }-- // Clean up- mlx.ClearCache()-- // Send final response with stats- evalDuration := time.Since(genStart)- resp = Response{- Done: true,- StopReason: fmt.Sprintf("%s:generated=%d", stopReason, generated),- PromptEvalCount: prefillTokens,- PromptEvalDuration: int(promptEvalDuration.Nanoseconds()),- EvalCount: generated,- EvalDuration: int(evalDuration.Nanoseconds()),- }- enc.Encode(resp)- flusher.Flush()-- return nil-}diff --git a/x/imagegen/models/gemma3/gemma3.go b/x/imagegen/models/gemma3/gemma3.godeleted file mode 100644index b56adc797..000000000--- a/x/imagegen/models/gemma3/gemma3.go+++ /dev/null@@ -1,614 +0,0 @@-//go:build mlx--package gemma3--import (- "encoding/json"- "fmt"- "math"- "os"- "path/filepath"-- "github.com/ollama/ollama/x/imagegen/cache"- "github.com/ollama/ollama/x/imagegen/mlx"- "github.com/ollama/ollama/x/imagegen/nn"- "github.com/ollama/ollama/x/imagegen/safetensors"- "github.com/ollama/ollama/x/imagegen/tokenizer"-)--// TextConfig holds configuration for the text model-type TextConfig struct {- HiddenSize int32 `json:"hidden_size"`- NumHiddenLayers int32 `json:"num_hidden_layers"`- IntermediateSize int32 `json:"intermediate_size"`- NumAttentionHeads int32 `json:"num_attention_heads"`- NumKeyValueHeads int32 `json:"num_key_value_heads"`- HeadDim int32 `json:"head_dim"`- VocabSize int32 `json:"vocab_size"`- RMSNormEps float32 `json:"rms_norm_eps"`- RopeTheta float32 `json:"rope_theta"`- RopeLocalBaseFreq float32 `json:"rope_local_base_freq"`- MaxPositionEmbeddings int32 `json:"max_position_embeddings"`- SlidingWindow int32 `json:"sliding_window"`- SlidingWindowPattern int32 `json:"sliding_window_pattern"`-- // Computed fields- Scale float32 `json:"-"`-}--// TextModel is the Gemma 3 text-only model-type TextModel struct {- EmbedTokens *nn.Embedding `weight:"model.embed_tokens"`- Layers []*DecoderLayer `weight:"model.layers"`- Norm *nn.RMSNorm `weight:"model.norm"`- Output *nn.Linear `weight:"-"` // Tied to EmbedTokens, set manually-- // Precomputed (1 + weight) for Gemma-style RMSNorm to avoid allocation per forward- NormScaled *mlx.Array `weight:"-"`-- tok *tokenizer.Tokenizer- *TextConfig-}--// DecoderLayer is a single transformer block-type DecoderLayer struct {- InputNorm *nn.RMSNorm `weight:"input_layernorm"`- Attention *Attention- PostAttnNorm *nn.RMSNorm `weight:"post_attention_layernorm"`- PreFFNorm *nn.RMSNorm `weight:"pre_feedforward_layernorm"`- MLP *MLP- PostFFNorm *nn.RMSNorm `weight:"post_feedforward_layernorm"`-- // Precomputed (1 + weight) for Gemma-style RMSNorm- InputNormScaled *mlx.Array `weight:"-"`- PostAttnNormScaled *mlx.Array `weight:"-"`- PreFFNormScaled *mlx.Array `weight:"-"`- PostFFNormScaled *mlx.Array `weight:"-"`-- // Whether this layer uses sliding window attention- IsSliding bool- LayerIdx int32-}--// Attention implements Gemma 3 attention with Q/K normalization-type Attention struct {- QProj *nn.Linear `weight:"self_attn.q_proj"`- KProj *nn.Linear `weight:"self_attn.k_proj"`- VProj *nn.Linear `weight:"self_attn.v_proj"`- OProj *nn.Linear `weight:"self_attn.o_proj"`- QNorm *nn.RMSNorm `weight:"self_attn.q_norm"`- KNorm *nn.RMSNorm `weight:"self_attn.k_norm"`-- // Precomputed (1 + weight) for Gemma-style RMSNorm- QNormScaled *mlx.Array `weight:"-"`- KNormScaled *mlx.Array `weight:"-"`-}--// MLP is the feed-forward network with GELU activation-type MLP struct {- GateProj *nn.Linear `weight:"mlp.gate_proj"`- UpProj *nn.Linear `weight:"mlp.up_proj"`- DownProj *nn.Linear `weight:"mlp.down_proj"`-}--// LoadText loads the text-only Gemma 3 model-func LoadText(modelPath string) (*TextModel, error) {- data, err := os.ReadFile(filepath.Join(modelPath, "config.json"))- if err != nil {- return nil, fmt.Errorf("load config: %w", err)- }- var cfg TextConfig- if err := json.Unmarshal(data, &cfg); err != nil {- return nil, fmt.Errorf("parse config: %w", err)- }-- // Compute scale- cfg.Scale = float32(1.0 / math.Sqrt(float64(cfg.HeadDim)))-- // Set defaults if not specified- if cfg.RopeTheta == 0 {- cfg.RopeTheta = 1000000- }- if cfg.RopeLocalBaseFreq == 0 {- cfg.RopeLocalBaseFreq = 10000- }- if cfg.RMSNormEps == 0 {- cfg.RMSNormEps = 1e-6- }-- weights, err := safetensors.LoadModelWeights(modelPath)- if err != nil {- return nil, fmt.Errorf("load weights: %w", err)- }-- tok, err := tokenizer.Load(filepath.Join(modelPath, "tokenizer.json"))- if err != nil {- return nil, fmt.Errorf("load tokenizer: %w", err)- }-- m := &TextModel{- Layers: make([]*DecoderLayer, cfg.NumHiddenLayers),- TextConfig: &cfg,- tok: tok,- }-- // Initialize layer metadata- for i := range m.Layers {- m.Layers[i] = &DecoderLayer{- LayerIdx: int32(i),- IsSliding: isLayerSliding(int32(i), cfg.SlidingWindowPattern),- }- }-- if err := safetensors.LoadModule(m, weights, ""); err != nil {- return nil, err- }-- // Tied embeddings for output- m.Output = nn.NewLinear(m.EmbedTokens.Weight, nil)-- mlx.Eval(mlx.Collect(m)...)- weights.ReleaseAll()-- // Precompute (1 + weight) for Gemma-style RMSNorm to avoid per-forward allocation- precomputeGemmaScaledWeights(m)-- return m, nil-}--// precomputeGemmaScaledWeights computes (1 + weight) for all RMSNorm layers-// This avoids creating temporary arrays on every forward pass-func precomputeGemmaScaledWeights(m *TextModel) {- m.NormScaled = mlx.AddScalar(m.Norm.Weight, 1.0)-- for _, layer := range m.Layers {- layer.InputNormScaled = mlx.AddScalar(layer.InputNorm.Weight, 1.0)- layer.PostAttnNormScaled = mlx.AddScalar(layer.PostAttnNorm.Weight, 1.0)- layer.PreFFNormScaled = mlx.AddScalar(layer.PreFFNorm.Weight, 1.0)- layer.PostFFNormScaled = mlx.AddScalar(layer.PostFFNorm.Weight, 1.0)-- layer.Attention.QNormScaled = mlx.AddScalar(layer.Attention.QNorm.Weight, 1.0)- layer.Attention.KNormScaled = mlx.AddScalar(layer.Attention.KNorm.Weight, 1.0)- }-- // Eval all the precomputed weights- var scaled []*mlx.Array- scaled = append(scaled, m.NormScaled)- for _, layer := range m.Layers {- scaled = append(scaled, layer.InputNormScaled, layer.PostAttnNormScaled,- layer.PreFFNormScaled, layer.PostFFNormScaled,- layer.Attention.QNormScaled, layer.Attention.KNormScaled)- }- mlx.Eval(scaled...)-}--// isLayerSliding determines if a layer uses sliding window attention-// Pattern N means: layers 0 to N-1 sliding, N full, N+1 to 2N-1 sliding, 2N full, etc.-func isLayerSliding(layerIdx, pattern int32) bool {- if pattern <= 0 {- return false // No sliding window- }- // Layer is full attention if (layerIdx + 1) % pattern == 0- return (layerIdx+1)%pattern != 0-}--// Forward runs the text model forward pass-func (m *TextModel) Forward(tokens *mlx.Array, caches []cache.Cache) *mlx.Array {- B, L := tokens.Shape()[0], tokens.Shape()[1]-- // Get embeddings and scale by sqrt(hidden_size)- h := m.EmbedTokens.Forward(tokens)- h = mlx.MulScalar(h, float32(math.Sqrt(float64(m.HiddenSize))))-- for i, layer := range m.Layers {- h = layer.Forward(h, caches[i], B, L, m.TextConfig)- }-- // Final norm and output projection- return m.Output.Forward(mlx.RMSNorm(h, m.NormScaled, m.RMSNormEps))-}--// Forward runs a decoder layer-func (l *DecoderLayer) Forward(x *mlx.Array, c cache.Cache, B, L int32, cfg *TextConfig) *mlx.Array {- // Pre-attention norm (use precomputed scaled weight)- normed := mlx.RMSNorm(x, l.InputNormScaled, cfg.RMSNormEps)-- // Attention- attnOut := l.Attention.Forward(normed, c, B, L, l.IsSliding, cfg)-- // Post-attention norm and residual- attnOut = mlx.RMSNorm(attnOut, l.PostAttnNormScaled, cfg.RMSNormEps)- h := mlx.Add(x, attnOut)-- // Pre-FFN norm- normed = mlx.RMSNorm(h, l.PreFFNormScaled, cfg.RMSNormEps)-- // MLP- mlpOut := l.MLP.Forward(normed)-- // Post-FFN norm and residual- mlpOut = mlx.RMSNorm(mlpOut, l.PostFFNormScaled, cfg.RMSNormEps)- return mlx.Add(h, mlpOut)-}--// Forward runs attention with Q/K normalization-func (a *Attention) Forward(x *mlx.Array, c cache.Cache, B, L int32, isSliding bool, cfg *TextConfig) *mlx.Array {- q := a.QProj.Forward(x)- k := a.KProj.Forward(x)- v := a.VProj.Forward(x)-- // Reshape to [B, num_heads, L, head_dim]- q = mlx.AsStrided(q, []int32{B, cfg.NumAttentionHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumAttentionHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumAttentionHeads * cfg.HeadDim), 1}, 0)- k = mlx.AsStrided(k, []int32{B, cfg.NumKeyValueHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumKeyValueHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumKeyValueHeads * cfg.HeadDim), 1}, 0)- v = mlx.AsStrided(v, []int32{B, cfg.NumKeyValueHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumKeyValueHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumKeyValueHeads * cfg.HeadDim), 1}, 0)-- // Q/K normalization after reshaping (use precomputed scaled weight)- q = mlx.RMSNorm(q, a.QNormScaled, cfg.RMSNormEps)- k = mlx.RMSNorm(k, a.KNormScaled, cfg.RMSNormEps)-- // Apply RoPE with appropriate theta- ropeTheta := cfg.RopeTheta- if isSliding {- ropeTheta = cfg.RopeLocalBaseFreq- }- q = mlx.RoPE(q, int(cfg.HeadDim), false, ropeTheta, 1.0, c.Offset())- k = mlx.RoPE(k, int(cfg.HeadDim), false, ropeTheta, 1.0, c.Offset())-- // Update cache- k, v = c.Update(k, v, int(L))-- // Repeat K/V for GQA if needed- repeatFactor := cfg.NumAttentionHeads / cfg.NumKeyValueHeads- if repeatFactor > 1 {- k = nn.RepeatKV(k, repeatFactor)- v = nn.RepeatKV(v, repeatFactor)- }-- // Attention- out := mlx.ScaledDotProductAttention(q, k, v, cfg.Scale, L > 1)- out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim)- return a.OProj.Forward(out)-}--// compiledGeluApprox is a singleton compiled GELU function shared across all layers-var compiledGeluApprox *mlx.CompiledFunc--// getCompiledGeluApprox returns the compiled GELU function, creating it once if needed-func getCompiledGeluApprox() *mlx.CompiledFunc {- if compiledGeluApprox == nil {- compiledGeluApprox = mlx.CompileShapeless(func(inputs []*mlx.Array) []*mlx.Array {- return []*mlx.Array{geluApproxImpl(inputs[0])}- }, true)- }- return compiledGeluApprox-}--// Forward runs the MLP with GELU approximation (tanh variant)-func (m *MLP) Forward(x *mlx.Array) *mlx.Array {- gate := getCompiledGeluApprox().Call(m.GateProj.Forward(x))[0]- return m.DownProj.Forward(mlx.Mul(gate, m.UpProj.Forward(x)))-}--// geluApproxImpl computes GELU using the tanh approximation (gelu_pytorch_tanh):-// 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))-func geluApproxImpl(x *mlx.Array) *mlx.Array {- // Constants- const sqrt2OverPi = 0.7978845608028654 // sqrt(2/pi)- const coeff = 0.044715-- // x^3- x3 := mlx.Mul(mlx.Mul(x, x), x)- // x + 0.044715 * x^3- inner := mlx.Add(x, mlx.MulScalar(x3, coeff))- // sqrt(2/pi) * (x + 0.044715 * x^3)- scaled := mlx.MulScalar(inner, sqrt2OverPi)- // tanh(...)- tanh := mlx.Tanh(scaled)- // 1 + tanh(...)- onePlusTanh := mlx.AddScalar(tanh, 1.0)- // 0.5 * x * (1 + tanh(...))- return mlx.Mul(mlx.MulScalar(x, 0.5), onePlusTanh)-}--// gemmaRMSNorm applies Gemma-style RMS normalization: x * rsqrt(mean(x^2) + eps) * (1 + weight)-// Uses mlx.RMSNorm fast kernel with pre-computed (1 + weight)-func gemmaRMSNorm(x, weight *mlx.Array, eps float32) *mlx.Array {- // Gemma uses (1 + weight) instead of weight- scaledWeight := mlx.AddScalar(weight, 1.0)- return mlx.RMSNorm(x, scaledWeight, eps)-}--// Interface methods-func (m *TextModel) NumLayers() int { return len(m.Layers) }-func (m *TextModel) MaxContextLength() int32 { return m.MaxPositionEmbeddings }-func (m *TextModel) VocabSize() int32 { return m.TextConfig.VocabSize }--// Tokenizer returns the tokenizer wrapped to add BOS and apply chat template-func (m *TextModel) Tokenizer() *tokenizer.Tokenizer {- return m.tok-}--// FormatPrompt applies the Gemma 3 chat template to a prompt-func (m *TextModel) FormatPrompt(prompt string) string {- // Gemma 3 chat format: <start_of_turn>user\n{prompt}<end_of_turn>\n<start_of_turn>model\n- return fmt.Sprintf("<start_of_turn>user\n%s<end_of_turn>\n<start_of_turn>model\n", prompt)-}--func (m *TextModel) NewCache(maxSeqLen int32) []cache.Cache {- caches := make([]cache.Cache, len(m.Layers))- for i := range caches {- if m.Layers[i].IsSliding {- // Use rotating cache for sliding window layers- caches[i] = cache.NewRotatingKVCache(int(m.SlidingWindow))- } else {- // Use regular cache for global attention layers- caches[i] = cache.NewKVCache()- }- }- return caches-}--// Config holds config for the full multimodal model-type Config struct {- TextConfig TextConfig `json:"text_config"`- VisionConfig VisionConfig `json:"vision_config"`-- // Image token config (from config.json)- BOITokenIndex int32 `json:"boi_token_index"` // <start_of_image> = 255999- EOITokenIndex int32 `json:"eoi_token_index"` // <end_of_image> = 256000- ImageTokenIndex int32 `json:"image_token_index"` // <image_soft_token> = 262144- MMTokensPerImage int32 `json:"mm_tokens_per_image"` // 256-}--// Model is the full Gemma 3 multimodal model-type Model struct {- VisionTower *VisionTower `weight:"vision_tower"`- Projector *MultiModalProjector `weight:"multi_modal_projector"`- TextModel *TextModel `weight:"language_model"`- Config *Config- tok *tokenizer.Tokenizer-}--// Load loads the full multimodal Gemma 3 model-func Load(modelPath string) (*Model, error) {- data, err := os.ReadFile(filepath.Join(modelPath, "config.json"))- if err != nil {- return nil, fmt.Errorf("load config: %w", err)- }-- var cfg Config- if err := json.Unmarshal(data, &cfg); err != nil {- return nil, fmt.Errorf("parse config: %w", err)- }-- // Set defaults for text config (multimodal config often has incomplete text_config)- // These defaults match transformers.Gemma3TextConfig defaults- tc := &cfg.TextConfig- if tc.HeadDim == 0 {- tc.HeadDim = 256 // Gemma 3 uses head_dim=256- }- if tc.NumAttentionHeads == 0 {- // Gemma 3 4B uses 8 attention heads (cannot infer from hidden_size/head_dim)- tc.NumAttentionHeads = 8- }- if tc.NumKeyValueHeads == 0 {- // Gemma 3 4B uses 4 KV heads (GQA with 2:1 ratio)- tc.NumKeyValueHeads = 4- }- if tc.VocabSize == 0 {- tc.VocabSize = 262208 // Gemma 3 vocab size (not 262144!)- }- if tc.RopeTheta == 0 {- tc.RopeTheta = 1000000- }- if tc.RopeLocalBaseFreq == 0 {- tc.RopeLocalBaseFreq = 10000- }- if tc.RMSNormEps == 0 {- tc.RMSNormEps = 1e-6- }- if tc.SlidingWindowPattern == 0 {- tc.SlidingWindowPattern = 6- }- if tc.MaxPositionEmbeddings == 0 {- tc.MaxPositionEmbeddings = 131072 // Gemma 3 4B default- }-- // Compute text model scale- tc.Scale = float32(1.0 / math.Sqrt(float64(tc.HeadDim)))-- // Set defaults for image token config- if cfg.BOITokenIndex == 0 {- cfg.BOITokenIndex = 255999 // <start_of_image>- }- if cfg.EOITokenIndex == 0 {- cfg.EOITokenIndex = 256000 // <end_of_image>- }- if cfg.ImageTokenIndex == 0 {- cfg.ImageTokenIndex = 262144 // <image_soft_token>- }- if cfg.MMTokensPerImage == 0 {- cfg.MMTokensPerImage = 256- }-- weights, err := safetensors.LoadModelWeights(modelPath)- if err != nil {- return nil, fmt.Errorf("load weights: %w", err)- }-- tok, err := tokenizer.Load(filepath.Join(modelPath, "tokenizer.json"))- if err != nil {- return nil, fmt.Errorf("load tokenizer: %w", err)- }-- m := &Model{- VisionTower: &VisionTower{- Embeddings: &VisionEmbeddings{},- Encoder: make([]*VisionEncoderLayer, cfg.VisionConfig.NumHiddenLayers),- Config: &cfg.VisionConfig,- },- Projector: &MultiModalProjector{},- TextModel: &TextModel{- Layers: make([]*DecoderLayer, cfg.TextConfig.NumHiddenLayers),- TextConfig: &cfg.TextConfig,- },- Config: &cfg,- tok: tok,- }-- // Initialize text layer metadata- for i := range m.TextModel.Layers {- m.TextModel.Layers[i] = &DecoderLayer{- LayerIdx: int32(i),- IsSliding: isLayerSliding(int32(i), cfg.TextConfig.SlidingWindowPattern),- }- }-- // Initialize vision encoder layers- for i := range m.VisionTower.Encoder {- m.VisionTower.Encoder[i] = &VisionEncoderLayer{}- }-- if err := safetensors.LoadModule(m, weights, ""); err != nil {- return nil, err- }-- // Tied embeddings for text output- m.TextModel.Output = nn.NewLinear(m.TextModel.EmbedTokens.Weight, nil)- m.TextModel.tok = tok-- mlx.Eval(mlx.Collect(m)...)- weights.ReleaseAll()-- // Precompute (1 + weight) for Gemma-style RMSNorm- precomputeGemmaScaledWeights(m.TextModel)-- // Precompute projector's scaled weight- m.Projector.SoftEmbNormScaled = mlx.AddScalar(m.Projector.SoftEmbNorm.Weight, 1.0)- mlx.Eval(m.Projector.SoftEmbNormScaled)-- return m, nil-}--// Forward runs the text-only forward pass-func (m *Model) Forward(tokens *mlx.Array, caches []cache.Cache) *mlx.Array {- return m.TextModel.Forward(tokens, caches)-}--// ForwardWithImage runs the multimodal forward pass-// tokens: [B, L] input token IDs (with image placeholder tokens)-// image: [B, H, W, C] preprocessed image tensor-func (m *Model) ForwardWithImage(tokens *mlx.Array, image *mlx.Array, caches []cache.Cache) *mlx.Array {- B, L := tokens.Shape()[0], tokens.Shape()[1]- cfg := m.Config.TextConfig-- // Find image token position FIRST before any eval that might free tokens- imageStartPos := int32(-1)- if image != nil && B == 1 {- tokenData := tokens.DataInt32() // This evals tokens- for i, t := range tokenData {- if t == m.Config.ImageTokenIndex {- imageStartPos = int32(i)- break- }- }- }-- // Get text embeddings and scale- h := m.TextModel.EmbedTokens.Forward(tokens)- h = mlx.MulScalar(h, float32(math.Sqrt(float64(cfg.HiddenSize))))-- // Process image if provided- if image != nil && imageStartPos >= 0 {- // Vision tower: [B, H, W, C] -> [B, num_patches, vision_hidden]- visionFeatures := m.VisionTower.Forward(image)-- // Project to text space: [B, num_patches, vision_hidden] -> [B, 256, text_hidden]- imageEmbeds := m.Projector.Forward(visionFeatures, cfg.RMSNormEps)-- // Eval h and imageEmbeds together so neither gets freed- mlx.Eval(h, imageEmbeds)-- // Cast imageEmbeds to match text embeddings dtype (bf16)- if imageEmbeds.Dtype() != h.Dtype() {- imageEmbeds = mlx.AsType(imageEmbeds, h.Dtype())- mlx.Eval(imageEmbeds)- }-- // Insert image embeddings at the known position- h = m.insertImageEmbeddingsAt(h, imageEmbeds, imageStartPos)- }-- // Run through text model layers- for i, layer := range m.TextModel.Layers {- h = layer.Forward(h, caches[i], B, L, m.TextModel.TextConfig)- }-- // Final norm and output projection- return m.TextModel.Output.Forward(mlx.RMSNorm(h, m.TextModel.NormScaled, cfg.RMSNormEps))-}--// insertImageEmbeddingsAt replaces image placeholder tokens with actual image embeddings-// at a known position (to avoid re-scanning tokens after eval)-// textEmbeds: [B, L, hidden_size] text embeddings-// imageEmbeds: [B, 256, hidden_size] image embeddings from projector-// startPos: starting position of image tokens in the sequence-func (m *Model) insertImageEmbeddingsAt(textEmbeds, imageEmbeds *mlx.Array, startPos int32) *mlx.Array {- numImageTokens := imageEmbeds.Shape()[1]- L := textEmbeds.Shape()[1]-- // Split text embeddings: [0:startPos] + imageEmbeds + [startPos+256:L]- afterStart := startPos + numImageTokens-- // Slice before image tokens: textEmbeds[:, 0:startPos, :]- before := mlx.SliceAxis(textEmbeds, 1, 0, startPos)-- // Slice after image tokens: textEmbeds[:, startPos+256:L, :]- after := mlx.SliceAxis(textEmbeds, 1, afterStart, L)-- // Concatenate: before + imageEmbeds + after along axis 1- return mlx.Concatenate([]*mlx.Array{before, imageEmbeds, after}, 1)-}--// Interface methods for Model-func (m *Model) NumLayers() int { return len(m.TextModel.Layers) }-func (m *Model) MaxContextLength() int32 { return m.Config.TextConfig.MaxPositionEmbeddings }-func (m *Model) VocabSize() int32 { return m.Config.TextConfig.VocabSize }-func (m *Model) Tokenizer() *tokenizer.Tokenizer { return m.tok }-func (m *Model) NewCache(maxSeqLen int32) []cache.Cache { return m.TextModel.NewCache(maxSeqLen) }-func (m *Model) ImageSize() int32 { return m.Config.VisionConfig.ImageSize }--// FormatPrompt applies the Gemma 3 multimodal chat template-func (m *Model) FormatPrompt(prompt string) string {- return fmt.Sprintf("<start_of_turn>user\n%s<end_of_turn>\n<start_of_turn>model\n", prompt)-}--// FormatPromptWithImage applies the Gemma 3 multimodal chat template with image-func (m *Model) FormatPromptWithImage(prompt string) string {- return fmt.Sprintf("<start_of_turn>user\n<start_of_image>%s<end_of_turn>\n<start_of_turn>model\n", prompt)-}--// ExpandImageTokens expands <start_of_image> into 256 image placeholder tokens-// Input tokens containing boi_token (255999) are expanded to:-// boi_token + 256 * image_token + eoi_token-func (m *Model) ExpandImageTokens(tokens []int32) []int32 {- result := make([]int32, 0, len(tokens)+int(m.Config.MMTokensPerImage)+1)-- for _, t := range tokens {- if t == m.Config.BOITokenIndex {- // Expand: boi + 256 * image_token + eoi- result = append(result, m.Config.BOITokenIndex)- for i := int32(0); i < m.Config.MMTokensPerImage; i++ {- result = append(result, m.Config.ImageTokenIndex)- }- result = append(result, m.Config.EOITokenIndex)- } else {- result = append(result, t)- }- }-- return result-}diff --git a/x/imagegen/models/gemma3/projector.go b/x/imagegen/models/gemma3/projector.godeleted file mode 100644index ecdbe6941..000000000--- a/x/imagegen/models/gemma3/projector.go+++ /dev/null@@ -1,50 +0,0 @@-//go:build mlx--package gemma3--import (- "github.com/ollama/ollama/x/imagegen/mlx"- "github.com/ollama/ollama/x/imagegen/nn"-)--// MultiModalProjector projects vision features to text embedding space-type MultiModalProjector struct {- // mm_input_projection_weight: [vision_hidden, text_hidden]- InputProjection *mlx.Array `weight:"mm_input_projection_weight"`- SoftEmbNorm *nn.RMSNorm `weight:"mm_soft_emb_norm"`-- // Precomputed (1 + weight) for Gemma-style RMSNorm- SoftEmbNormScaled *mlx.Array `weight:"-"`-}--// Forward projects vision features to text space-// Input: [B, num_patches, vision_hidden] (e.g., [1, 4096, 1152])-// Output: [B, num_image_tokens, text_hidden] (e.g., [1, 256, 2560])-func (p *MultiModalProjector) Forward(visionFeatures *mlx.Array, eps float32) *mlx.Array {- // Average pool 4x4: [B, 4096, 1152] -> [B, 256, 1152]- // 4096 patches = 64x64 grid, pool to 16x16 = 256 tokens- B := visionFeatures.Shape()[0]- visionHidden := visionFeatures.Shape()[2]-- // Reshape to [B, 64, 64, hidden]- gridSize := int32(64) // sqrt(4096)- pooledSize := int32(16) // 64/4- h := mlx.Reshape(visionFeatures, B, gridSize, gridSize, visionHidden)-- // Reshape to [B, 16, 4, 16, 4, hidden] for 4x4 pooling- h = mlx.Reshape(h, B, pooledSize, 4, pooledSize, 4, visionHidden)-- // Average over pooling dimensions (axes 2 and 4)- h = mlx.Mean(h, 4, false)- h = mlx.Mean(h, 2, false)-- // h is now [B, 16, 16, hidden], reshape to [B, 256, hidden]- numTokens := pooledSize * pooledSize- h = mlx.Reshape(h, B, numTokens, visionHidden)-- // Apply Gemma-style RMS norm (use precomputed 1 + weight)- h = mlx.RMSNorm(h, p.SoftEmbNormScaled, eps)-- // Project to text space: [B, 256, vision_hidden] @ [vision_hidden, text_hidden]- return mlx.Linear(h, p.InputProjection)-}diff --git a/x/imagegen/models/gemma3/vision.go b/x/imagegen/models/gemma3/vision.godeleted file mode 100644index 1c4d8e54f..000000000--- a/x/imagegen/models/gemma3/vision.go+++ /dev/null@@ -1,138 +0,0 @@-//go:build mlx--package gemma3--import (- "math"-- "github.com/ollama/ollama/x/imagegen/mlx"- "github.com/ollama/ollama/x/imagegen/nn"-)--// VisionConfig holds configuration for the SigLIP vision tower-type VisionConfig struct {- HiddenSize int32 `json:"hidden_size"`- ImageSize int32 `json:"image_size"`- IntermediateSize int32 `json:"intermediate_size"`- NumAttentionHeads int32 `json:"num_attention_heads"`- NumHiddenLayers int32 `json:"num_hidden_layers"`- PatchSize int32 `json:"patch_size"`-}--// VisionTower is the SigLIP vision encoder-type VisionTower struct {- Embeddings *VisionEmbeddings `weight:"vision_model.embeddings"`- Encoder []*VisionEncoderLayer `weight:"vision_model.encoder.layers"`- PostLayerNorm *nn.LayerNorm `weight:"vision_model.post_layernorm"`- Config *VisionConfig-}--// VisionEmbeddings handles patch and position embeddings-type VisionEmbeddings struct {- // PatchWeight: [O, C, kH, kW] from PyTorch, transposed to [O, kH, kW, C] for MLX- PatchWeight *mlx.Array `weight:"patch_embedding.weight"`- PatchBias *mlx.Array `weight:"patch_embedding.bias"`- PosEmbed *nn.Embedding `weight:"position_embedding"`-}--// VisionEncoderLayer is a single transformer encoder layer-type VisionEncoderLayer struct {- LayerNorm1 *nn.LayerNorm `weight:"layer_norm1"`- Attention *VisionAttention `weight:"self_attn"`- LayerNorm2 *nn.LayerNorm `weight:"layer_norm2"`- MLP *VisionMLP `weight:"mlp"`-}--// VisionAttention implements multi-head self-attention-type VisionAttention struct {- QProj *nn.Linear `weight:"q_proj"`- KProj *nn.Linear `weight:"k_proj"`- VProj *nn.Linear `weight:"v_proj"`- OutProj *nn.Linear `weight:"out_proj"`-}--// VisionMLP is the feed-forward network-type VisionMLP struct {- FC1 *nn.Linear `weight:"fc1"`- FC2 *nn.Linear `weight:"fc2"`-}--// Forward runs the vision tower on preprocessed images-// Input: [B, H, W, C] normalized image tensor (NHWC layout for MLX)-// Output: [B, num_patches, hidden_size]-func (v *VisionTower) Forward(x *mlx.Array) *mlx.Array {- // Patch embedding conv: input [B, H, W, C], weight [O, kH, kW, C] -> [B, grid, grid, O]- // Weight comes as [O, C, kH, kW] from PyTorch, transpose to [O, kH, kW, C]- weight := mlx.Transpose(v.Embeddings.PatchWeight, 0, 2, 3, 1)- h := mlx.Conv2d(x, weight, v.Config.PatchSize, 0) // stride=patch_size, no padding-- // Add bias: [O] -> [1, 1, 1, O] for broadcasting- bias := mlx.Reshape(v.Embeddings.PatchBias, 1, 1, 1, v.Embeddings.PatchBias.Shape()[0])- h = mlx.Add(h, bias)-- // h is [B, grid, grid, hidden], flatten to [B, num_patches, hidden]- B := h.Shape()[0]- gridH, gridW := h.Shape()[1], h.Shape()[2]- hidden := h.Shape()[3]- numPatches := gridH * gridW- h = mlx.Reshape(h, B, numPatches, hidden)-- // Add position embeddings- posIds := mlx.ArangeInt(0, numPatches, 1, mlx.DtypeInt32)- posEmbed := v.Embeddings.PosEmbed.Forward(posIds)- h = mlx.Add(h, posEmbed)-- // Encoder layers- headDim := float32(v.Config.HiddenSize / v.Config.NumAttentionHeads)- scale := float32(1.0 / math.Sqrt(float64(headDim)))- for _, layer := range v.Encoder {- h = layer.Forward(h, v.Config, scale)- }-- // Final layer norm- h = v.PostLayerNorm.Forward(h)-- return h-}--// Forward runs a vision encoder layer-func (l *VisionEncoderLayer) Forward(x *mlx.Array, cfg *VisionConfig, scale float32) *mlx.Array {- // Pre-norm attention- h := l.LayerNorm1.Forward(x)- h = l.Attention.Forward(h, cfg, scale)- x = mlx.Add(x, h)-- // Pre-norm MLP- h = l.LayerNorm2.Forward(x)- h = l.MLP.Forward(h)- return mlx.Add(x, h)-}--// Forward runs multi-head self-attention-func (a *VisionAttention) Forward(x *mlx.Array, cfg *VisionConfig, scale float32) *mlx.Array {- B, L := x.Shape()[0], x.Shape()[1]- headDim := cfg.HiddenSize / cfg.NumAttentionHeads-- q := a.QProj.Forward(x)- k := a.KProj.Forward(x)- v := a.VProj.Forward(x)-- // Reshape to [B, num_heads, L, head_dim]- q = mlx.Transpose(mlx.Reshape(q, B, L, cfg.NumAttentionHeads, headDim), 0, 2, 1, 3)- k = mlx.Transpose(mlx.Reshape(k, B, L, cfg.NumAttentionHeads, headDim), 0, 2, 1, 3)- v = mlx.Transpose(mlx.Reshape(v, B, L, cfg.NumAttentionHeads, headDim), 0, 2, 1, 3)-- // Scaled dot-product attention (no causal mask for vision)- out := mlx.ScaledDotProductAttention(q, k, v, scale, false)-- // Reshape back: [B, num_heads, L, head_dim] -> [B, L, hidden]- out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.HiddenSize)-- return a.OutProj.Forward(out)-}--// Forward runs the MLP with GELU activation-func (m *VisionMLP) Forward(x *mlx.Array) *mlx.Array {- h := mlx.GELU(m.FC1.Forward(x))- return m.FC2.Forward(h)-}diff --git a/x/imagegen/models/glm4_moe_lite/glm4_moe_lite.go b/x/imagegen/models/glm4_moe_lite/glm4_moe_lite.godeleted file mode 100644index 3931693b8..000000000--- a/x/imagegen/models/glm4_moe_lite/glm4_moe_lite.go+++ /dev/null@@ -1,840 +0,0 @@-//go:build mlx--// Package glm4_moe_lite provides the GLM4-MoE-Lite implementation for MLX.-// This model uses Multi-head Latent Attention (MLA) and Mixture of Experts (MoE).-package glm4_moe_lite--import (- "encoding/json"- "fmt"- "math"-- "github.com/ollama/ollama/x/imagegen/cache"- "github.com/ollama/ollama/x/imagegen/manifest"- "github.com/ollama/ollama/x/imagegen/mlx"- "github.com/ollama/ollama/x/imagegen/nn"- "github.com/ollama/ollama/x/imagegen/safetensors"- "github.com/ollama/ollama/x/imagegen/tokenizer"-)--// RopeScaling holds RoPE scaling configuration-type RopeScaling struct {- Factor float32 `json:"factor"`- MscaleAllDim float32 `json:"mscale_all_dim"`-}--// Config holds GLM4-MoE-Lite model configuration-type Config struct {- HiddenSize int32 `json:"hidden_size"`- NumHiddenLayers int32 `json:"num_hidden_layers"`- IntermediateSize int32 `json:"intermediate_size"`- MoEIntermediateSize int32 `json:"moe_intermediate_size"`- NumAttentionHeads int32 `json:"num_attention_heads"`- NumKeyValueHeads int32 `json:"num_key_value_heads"`- VocabSize int32 `json:"vocab_size"`- RMSNormEps float32 `json:"rms_norm_eps"`- RopeTheta float32 `json:"rope_theta"`- MaxPositionEmbeddings int32 `json:"max_position_embeddings"`- AttentionBias bool `json:"attention_bias"`-- // MLA (Multi-head Latent Attention) parameters- QLoraRank int32 `json:"q_lora_rank"`- KVLoraRank int32 `json:"kv_lora_rank"`- QKRopeHeadDim int32 `json:"qk_rope_head_dim"`- QKNopeHeadDim int32 `json:"qk_nope_head_dim"`- VHeadDim int32 `json:"v_head_dim"`-- // MoE parameters- NRoutedExperts int32 `json:"n_routed_experts"`- NSharedExperts int32 `json:"n_shared_experts"`- NumExpertsPerTok int32 `json:"num_experts_per_tok"`- RoutedScalingFactor float32 `json:"routed_scaling_factor"`- NormTopKProb bool `json:"norm_topk_prob"`- FirstKDenseReplace int32 `json:"first_k_dense_replace"`- NGroup int32 `json:"n_group"`- TopKGroup int32 `json:"topk_group"`-- // RoPE scaling- RopeScaling *RopeScaling `json:"rope_scaling"`-- // Quantization parameters (set during load based on model quantization)- QuantGroupSize int `json:"-"` // Group size for quantization (default 64)- QuantBits int `json:"-"` // Bits per weight (4 or 8)- QuantMode string `json:"-"` // Quantization mode ("affine", etc.)-- // Computed fields- QHeadDim int32 `json:"-"` // qk_nope_head_dim + qk_rope_head_dim- Scale float32 `json:"-"` // 1/sqrt(QHeadDim) with mscale adjustment-}--// MLAAttention implements Multi-head Latent Attention with absorption.-// This uses absorbed MLA which operates in latent space for reduced KV cache.-type MLAAttention struct {- // Low-rank query projections- QAProj nn.LinearLayer `weight:"self_attn.q_a_proj"`- QALayerNorm *nn.RMSNorm `weight:"self_attn.q_a_layernorm"`- QBProj nn.LinearLayer `weight:"self_attn.q_b_proj"`-- // Low-rank KV projections (with shared rope component)- KVAProjWithMQA nn.LinearLayer `weight:"self_attn.kv_a_proj_with_mqa"`- KVALayerNorm *nn.RMSNorm `weight:"self_attn.kv_a_layernorm"`-- // Absorbed MLA projections (derived from kv_b_proj)- // EmbedQ: projects q_nope to latent space [num_heads, kv_lora_rank, qk_nope_head_dim]- // UnembedOut: projects attention output from latent space [num_heads, v_head_dim, kv_lora_rank]- EmbedQ *nn.MultiLinear `weight:"-"`- UnembedOut *nn.MultiLinear `weight:"-"`-- // Output projection- OProj nn.LinearLayer `weight:"self_attn.o_proj"`-}--// Forward computes absorbed MLA attention output.-// This operates in latent space for reduced KV cache memory.-func (a *MLAAttention) Forward(x *mlx.Array, c cache.Cache, B, L int32, cfg *Config) *mlx.Array {- // Query path: q_a_proj -> layernorm -> q_b_proj- q := a.QAProj.Forward(x)- q = a.QALayerNorm.Forward(q, cfg.RMSNormEps)- q = a.QBProj.Forward(q)-- // Reshape Q: [B, L, num_heads * q_head_dim] -> [B, num_heads, L, q_head_dim]- q = mlx.Reshape(q, B, L, cfg.NumAttentionHeads, cfg.QHeadDim)- q = mlx.Transpose(q, 0, 2, 1, 3)-- // Split Q into nope and rope parts- qNope := mlx.Slice(q, []int32{0, 0, 0, 0}, []int32{B, cfg.NumAttentionHeads, L, cfg.QKNopeHeadDim})- qPE := mlx.Slice(q, []int32{0, 0, 0, cfg.QKNopeHeadDim}, []int32{B, cfg.NumAttentionHeads, L, cfg.QHeadDim})-- // KV path: get compressed KV and k_pe- compressedKV := a.KVAProjWithMQA.Forward(x)-- // Split into compressed_kv and k_pe (shared rope component)- kvCompressed := mlx.Slice(compressedKV, []int32{0, 0, 0}, []int32{B, L, cfg.KVLoraRank})- kPE := mlx.Slice(compressedKV, []int32{0, 0, cfg.KVLoraRank}, []int32{B, L, cfg.KVLoraRank + cfg.QKRopeHeadDim})-- // k_pe is shared across heads (MQA-style): [B, L, rope_dim] -> [B, 1, L, rope_dim]- kPE = mlx.Reshape(kPE, B, L, 1, cfg.QKRopeHeadDim)- kPE = mlx.Transpose(kPE, 0, 2, 1, 3)-- // Apply layernorm to get kv latent representation- kvLatent := a.KVALayerNorm.Forward(kvCompressed, cfg.RMSNormEps)- // kvLatent: [B, L, kv_lora_rank] -> [B, 1, L, kv_lora_rank] for broadcasting- kvLatent = mlx.ExpandDims(kvLatent, 1)-- // Apply RoPE to the rope parts- offset := 0- if c != nil {- offset = c.Offset()- }- qPE = mlx.RoPE(qPE, int(cfg.QKRopeHeadDim), true, cfg.RopeTheta, 1.0, offset)- kPE = mlx.RoPE(kPE, int(cfg.QKRopeHeadDim), true, cfg.RopeTheta, 1.0, offset)-- // ABSORBED MLA: project q_nope to latent space- // qNope: [B, num_heads, L, qk_nope_head_dim]- // EmbedQ: [num_heads, kv_lora_rank, qk_nope_head_dim]- // Result: [B, num_heads, L, kv_lora_rank]- qLatent := a.EmbedQ.Forward(qNope)-- // Keys = concat(kvLatent, kPE)- // kvLatent: [B, 1, L, kv_lora_rank]- // kPE: [B, 1, L, qk_rope_head_dim]- // keys: [B, 1, L, kv_lora_rank + qk_rope_head_dim]- keys := mlx.Concatenate([]*mlx.Array{kvLatent, kPE}, 3)-- // Cache the smaller latent representation- // We cache keys (latent + rope) and use empty values since values are derived from keys- cachedL := L- if c != nil {- // Create placeholder values with 0 dims for cache (we don't actually use cached values)- placeholderValues := mlx.Zeros([]int32{B, 1, L, 0}, mlx.DtypeFloat32)- keys, _ = c.Update(keys, placeholderValues, int(L))- cachedL = int32(keys.Shape()[2])- }-- // Values are the first kv_lora_rank dims of keys (slice off rope part)- values := mlx.Slice(keys, []int32{0, 0, 0, 0}, []int32{B, 1, cachedL, cfg.KVLoraRank})-- // Queries = concat(qLatent, qPE)- // qLatent: [B, num_heads, L, kv_lora_rank]- // qPE: [B, num_heads, L, qk_rope_head_dim]- // queries: [B, num_heads, L, kv_lora_rank + qk_rope_head_dim]- queries := mlx.Concatenate([]*mlx.Array{qLatent, qPE}, 3)-- // Attention in latent space- // queries: [B, num_heads, L, kv_lora_rank + rope_dim]- // keys: [B, 1, cachedL, kv_lora_rank + rope_dim]- // values: [B, 1, cachedL, kv_lora_rank]- out := mlx.ScaledDotProductAttention(queries, keys, values, cfg.Scale, L > 1)-- // ABSORBED MLA: unembed from latent space- // out: [B, num_heads, L, kv_lora_rank]- // UnembedOut: [num_heads, v_head_dim, kv_lora_rank]- // Result: [B, num_heads, L, v_head_dim]- out = a.UnembedOut.Forward(out)-- // Reshape back: [B, num_heads, L, v_head_dim] -> [B, L, num_heads * v_head_dim]- out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.VHeadDim)-- return a.OProj.Forward(out)-}--// DenseMLP implements the standard SwiGLU MLP for dense layers-type DenseMLP struct {- GateProj nn.LinearLayer `weight:"mlp.gate_proj"`- UpProj nn.LinearLayer `weight:"mlp.up_proj"`- DownProj nn.LinearLayer `weight:"mlp.down_proj"`-}--// Forward applies the SwiGLU MLP-func (m *DenseMLP) Forward(x *mlx.Array) *mlx.Array {- gate := mlx.SiLU(m.GateProj.Forward(x))- up := m.UpProj.Forward(x)- return m.DownProj.Forward(mlx.Mul(gate, up))-}--// MoEGate implements the expert gating mechanism-type MoEGate struct {- Gate nn.LinearLayer `weight:"mlp.gate"`- EScoreCorrectionBias *mlx.Array `weight:"mlp.gate.e_score_correction_bias,optional"`-}--// Forward computes expert selection indices and scores-func (g *MoEGate) Forward(x *mlx.Array, cfg *Config) (*mlx.Array, *mlx.Array) {- // Compute gate logits through linear layer (handles both quantized and non-quantized)- gates := g.Gate.Forward(x)-- // Sigmoid scoring- scores := mlx.Sigmoid(gates)- origScores := scores-- // Add correction bias if present- if g.EScoreCorrectionBias != nil {- scores = mlx.Add(scores, g.EScoreCorrectionBias)- }-- // Group-wise expert selection (simplified for n_group=1)- // Select top-k experts- topK := cfg.NumExpertsPerTok- negScores := mlx.Neg(scores)- inds := mlx.Argpartition(negScores, int(topK)-1, -1)-- shape := inds.Shape()- inds = mlx.Slice(inds, []int32{0, 0, 0}, []int32{shape[0], shape[1], topK})-- // Get scores for selected experts- scores = mlx.TakeAlongAxis(origScores, inds, -1)-- // Normalize if configured- if topK > 1 && cfg.NormTopKProb {- sumScores := mlx.Sum(scores, -1, true)- scores = mlx.Div(scores, sumScores)- }-- // Apply routing scaling factor- scores = mlx.MulScalar(scores, cfg.RoutedScalingFactor)-- return inds, scores-}--// SwitchMLP implements the MoE expert computation using stacked weights-// Note: No weight tags - these are populated manually by stacking expert weights-type SwitchMLP struct {- // Dequantized weights (used when GatherQMM not available)- GateWeight *mlx.Array- UpWeight *mlx.Array- DownWeight *mlx.Array-- // Quantized weights (used with GatherQMM for 4/8-bit affine)- GateWeightQ, GateScales, GateBiases *mlx.Array- UpWeightQ, UpScales, UpBiases *mlx.Array- DownWeightQ, DownScales, DownBiases *mlx.Array-- // Quantization bits per projection (supports mixed precision Q4/Q8)- GateBits int- UpBits int- DownBits int-- // Quantization group size per projection (detected from tensor shapes)- GateGroupSize int- UpGroupSize int- DownGroupSize int-- // If true, use GatherQMM with quantized weights- UseQuantized bool-}--// Forward applies the switched expert MLP-func (s *SwitchMLP) Forward(x *mlx.Array, indices *mlx.Array, cfg *Config) *mlx.Array {- shape := x.Shape()- B, L := shape[0], shape[1]- topK := cfg.NumExpertsPerTok-- // Expand x for expert computation: [B, L, D] -> [B, L, 1, 1, D]- xExpanded := mlx.ExpandDims(mlx.ExpandDims(x, -2), -2)-- // Flatten for gather_mm: [B*L, 1, 1, D]- xFlat := mlx.Reshape(xExpanded, B*L, 1, 1, cfg.HiddenSize)-- // Flatten indices: [B, L, topK] -> [B*L, topK]- idxFlat := mlx.Reshape(indices, B*L, topK)-- // Sort for efficient gather (when we have many tokens)- doSort := B*L >= 64- var invOrder *mlx.Array- n := B * L * topK-- if doSort {- idxAll := mlx.Flatten(idxFlat)- order := mlx.Argsort(idxAll, 0)- invOrder = mlx.Argsort(order, 0)- // Reorder x based on sorted indices- xFlat = mlx.ExpandDims(mlx.Take(mlx.Squeeze(xFlat, 1), mlx.FloorDivideScalar(order, topK), 0), 1)- idxFlat = mlx.Reshape(mlx.Take(idxAll, order, 0), n, 1)- }-- var gate, up, hidden, down *mlx.Array-- if s.UseQuantized {- // Use GatherQMM for quantized weights (faster, keeps weights quantized)- // Each projection may have different bits and group sizes (mixed precision: Q4 for gate/up, Q8 for down)- gate = mlx.GatherQMM(xFlat, s.GateWeightQ, s.GateScales, s.GateBiases,- nil, idxFlat, true, s.GateGroupSize, s.GateBits, cfg.QuantMode, doSort)- up = mlx.GatherQMM(xFlat, s.UpWeightQ, s.UpScales, s.UpBiases,- nil, idxFlat, true, s.UpGroupSize, s.UpBits, cfg.QuantMode, doSort)-- hidden = mlx.Mul(mlx.SiLU(gate), up)-- down = mlx.GatherQMM(hidden, s.DownWeightQ, s.DownScales, s.DownBiases,- nil, idxFlat, true, s.DownGroupSize, s.DownBits, cfg.QuantMode, doSort)- } else {- // Use GatherMM for dequantized/non-quantized weights- gate = mlx.GatherMM(xFlat, mlx.Transpose(s.GateWeight, 0, 2, 1), nil, idxFlat, doSort)- up = mlx.GatherMM(xFlat, mlx.Transpose(s.UpWeight, 0, 2, 1), nil, idxFlat, doSort)-- hidden = mlx.Mul(mlx.SiLU(gate), up)-- down = mlx.GatherMM(hidden, mlx.Transpose(s.DownWeight, 0, 2, 1), nil, idxFlat, doSort)- }-- // Unsort if we sorted- if doSort {- down = mlx.Reshape(mlx.Take(mlx.Squeeze(mlx.Squeeze(down, 2), 1), invOrder, 0), B*L, topK, cfg.HiddenSize)- } else {- down = mlx.Squeeze(down, 2)- }-- return mlx.Reshape(down, B, L, topK, cfg.HiddenSize)-}--// SharedExperts implements the shared expert MLP-type SharedExperts struct {- GateProj nn.LinearLayer `weight:"mlp.shared_experts.gate_proj"`- UpProj nn.LinearLayer `weight:"mlp.shared_experts.up_proj"`- DownProj nn.LinearLayer `weight:"mlp.shared_experts.down_proj"`-}--// Forward applies the shared expert MLP-func (s *SharedExperts) Forward(x *mlx.Array) *mlx.Array {- gate := mlx.SiLU(s.GateProj.Forward(x))- up := s.UpProj.Forward(x)- return s.DownProj.Forward(mlx.Mul(gate, up))-}--// MoE implements the full Mixture of Experts layer-type MoE struct {- Gate *MoEGate- SwitchMLP *SwitchMLP- SharedExperts *SharedExperts-}--// Forward applies the MoE layer-func (m *MoE) Forward(x *mlx.Array, cfg *Config) *mlx.Array {- shape := x.Shape()- B, L := shape[0], shape[1]-- // Get expert indices and scores- inds, scores := m.Gate.Forward(x, cfg)-- // Apply routed experts- expertOut := m.SwitchMLP.Forward(x, inds, cfg)-- // Weight by scores: [B, L, topK, D] * [B, L, topK, 1] -> sum over topK- scoresExpanded := mlx.ExpandDims(scores, -1)- y := mlx.Sum(mlx.Mul(expertOut, scoresExpanded), 2, false)-- // Add shared experts if present- if m.SharedExperts != nil {- y = mlx.Add(y, m.SharedExperts.Forward(x))- }-- return mlx.Reshape(y, B, L, cfg.HiddenSize)-}--// DenseBlock represents a dense transformer block (for first_k_dense_replace layers)-type DenseBlock struct {- Attention *MLAAttention- MLP *DenseMLP- InputLayerNorm *nn.RMSNorm `weight:"input_layernorm"`- PostAttentionLayerNorm *nn.RMSNorm `weight:"post_attention_layernorm"`-}--// Forward applies the dense block-func (b *DenseBlock) Forward(x *mlx.Array, c cache.Cache, B, L int32, cfg *Config) *mlx.Array {- // Pre-norm attention with residual- r := b.Attention.Forward(b.InputLayerNorm.Forward(x, cfg.RMSNormEps), c, B, L, cfg)- h := mlx.Add(x, r)-- // Pre-norm MLP with residual- r = b.MLP.Forward(b.PostAttentionLayerNorm.Forward(h, cfg.RMSNormEps))- return mlx.Add(h, r)-}--// MoEBlock represents a MoE transformer block-type MoEBlock struct {- Attention *MLAAttention- MoE *MoE- InputLayerNorm *nn.RMSNorm `weight:"input_layernorm"`- PostAttentionLayerNorm *nn.RMSNorm `weight:"post_attention_layernorm"`-}--// Forward applies the MoE block-func (b *MoEBlock) Forward(x *mlx.Array, c cache.Cache, B, L int32, cfg *Config) *mlx.Array {- // Pre-norm attention with residual- r := b.Attention.Forward(b.InputLayerNorm.Forward(x, cfg.RMSNormEps), c, B, L, cfg)- h := mlx.Add(x, r)-- // Pre-norm MoE with residual- r = b.MoE.Forward(b.PostAttentionLayerNorm.Forward(h, cfg.RMSNormEps), cfg)- return mlx.Add(h, r)-}--// Block interface for both dense and MoE blocks-type Block interface {- Forward(x *mlx.Array, c cache.Cache, B, L int32, cfg *Config) *mlx.Array-}--// Model represents the complete GLM4-MoE-Lite model-type Model struct {- EmbedTokens *nn.Embedding `weight:"model.embed_tokens"`- Layers []Block `weight:"-"` // Loaded manually due to different block types- Norm *nn.RMSNorm `weight:"model.norm"`- LMHead nn.LinearLayer `weight:"lm_head"`-- tok *tokenizer.Tokenizer- *Config-}--// computeScale computes the attention scale.-// Uses the full key head dimension (qkNopeHeadDim + qkRopeHeadDim) to match the Ollama runner.-func computeScale(cfg *Config) float32 {- keyLength := cfg.QKNopeHeadDim + cfg.QKRopeHeadDim- scale := float32(1.0 / math.Sqrt(float64(keyLength)))- if cfg.RopeScaling != nil && cfg.RopeScaling.MscaleAllDim > 0 && cfg.RopeScaling.Factor > 1 {- s := 0.1*cfg.RopeScaling.MscaleAllDim*float32(math.Log(float64(cfg.RopeScaling.Factor))) + 1.0- scale *= s * s- }- return scale-}--// supportsGatherQMM returns true if the quantization mode has GatherQMM kernel support.-// Currently only 4-bit and 8-bit affine quantization are supported.-func supportsGatherQMM(mode string, bits int) bool {- return mode == "affine" && (bits == 4 || bits == 8)-}--// ExpertWeight holds a single expert's weight with optional quantization components.-type ExpertWeight struct {- Weight *mlx.Array // Quantized weight (if quantized) or dequantized weight- Scales *mlx.Array // Quantization scales (nil if not quantized)- Biases *mlx.Array // Quantization biases (nil if not quantized or mode doesn't use biases)- Bits int // Quantization bits (4 or 8), 0 if not quantized- GroupSize int // Quantization group size, 0 if not quantized-}--// getQuantParams returns quantization parameters from model metadata.-// Returns groupSize, bits, and mode for the model's quantization type.-func getQuantParams(weights safetensors.WeightSource) (groupSize, bits int, mode string) {- groupSize, bits, mode = safetensors.QuantizationParams(weights.Quantization())- // Use metadata group_size if available (overrides default)- if gs := weights.GroupSize(); gs > 0 {- groupSize = gs- }- return groupSize, bits, mode-}--// loadExpertWeight loads an expert weight.-// If useQuantized is true and the weight is quantized with a supported mode, returns quantized components.-// Otherwise dequantizes and returns only the weight.-func loadExpertWeight(weights safetensors.WeightSource, path string, useQuantized bool, cfg *Config) *ExpertWeight {- w, _ := weights.GetTensor(path + ".weight")- if w == nil {- return nil- }-- // Check if this is a quantized weight by looking for scales- scalePath := path + ".weight_scale"- if weights.HasTensor(scalePath) {- scales, _ := weights.GetTensor(scalePath)- var qbiases *mlx.Array- qbiasPath := path + ".weight_qbias"- if weights.HasTensor(qbiasPath) {- qbiases, _ = weights.GetTensor(qbiasPath)- }-- // Get quantization params from metadata- groupSize, bits, mode := getQuantParams(weights)-- // Update config with group size (for GatherQMM calls)- if cfg.QuantGroupSize == 0 {- cfg.QuantGroupSize = groupSize- }-- // If GatherQMM is supported and requested, return quantized components- if useQuantized && supportsGatherQMM(mode, bits) {- return &ExpertWeight{Weight: w, Scales: scales, Biases: qbiases, Bits: bits, GroupSize: groupSize}- }-- // Otherwise dequantize- return &ExpertWeight{Weight: mlx.Dequantize(w, scales, qbiases, groupSize, bits, mode)}- }-- return &ExpertWeight{Weight: w}-}--// sanitizeMLAWeights transforms kv_b_proj weights into absorbed MLA format.-// Returns embed_q and unembed_out weights for per-head projections.-//-// kv_b_proj.weight shape: [num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank]-// Output:-// - embed_q: [num_heads, kv_lora_rank, qk_nope_head_dim] - projects q_nope to latent-// - unembed_out: [num_heads, v_head_dim, kv_lora_rank] - projects latent to output-func sanitizeMLAWeights(weights safetensors.WeightSource, prefix string, cfg *Config) (*mlx.Array, *mlx.Array) {- path := prefix + ".self_attn.kv_b_proj"- w, err := weights.GetTensor(path + ".weight")- if err != nil || w == nil {- return nil, nil- }-- // Check if quantized and dequantize- scalePath := path + ".weight_scale"- if weights.HasTensor(scalePath) {- scales, _ := weights.GetTensor(scalePath)- var qbiases *mlx.Array- qbiasPath := path + ".weight_qbias"- if weights.HasTensor(qbiasPath) {- qbiases, _ = weights.GetTensor(qbiasPath)- }-- groupSize, bits, mode := getQuantParams(weights)- w = mlx.Dequantize(w, scales, qbiases, groupSize, bits, mode)- }-- // w: [num_heads * (qk_nope_head_dim + v_head_dim), kv_lora_rank]- // Reshape to [num_heads, qk_nope_head_dim + v_head_dim, kv_lora_rank]- headDim := cfg.QKNopeHeadDim + cfg.VHeadDim- w = mlx.Reshape(w, cfg.NumAttentionHeads, headDim, cfg.KVLoraRank)-- // Split into wk and wv- // wk: [num_heads, qk_nope_head_dim, kv_lora_rank]- // wv: [num_heads, v_head_dim, kv_lora_rank]- wk := mlx.Slice(w, []int32{0, 0, 0}, []int32{cfg.NumAttentionHeads, cfg.QKNopeHeadDim, cfg.KVLoraRank})- wv := mlx.Slice(w, []int32{0, cfg.QKNopeHeadDim, 0}, []int32{cfg.NumAttentionHeads, headDim, cfg.KVLoraRank})-- // Transform for absorbed MLA:- // embed_q: transpose(wk) -> [num_heads, kv_lora_rank, qk_nope_head_dim]- // This allows: q_nope @ embed_q.T = q_nope @ wk (absorbed key projection)- embedQ := mlx.Transpose(wk, 0, 2, 1)-- // unembed_out: wv stays [num_heads, v_head_dim, kv_lora_rank]- // This allows: latent_out @ unembed_out.T = latent_out @ wv.T (absorbed value projection)- unembedOut := wv-- return embedQ, unembedOut-}--// StackedExpertWeights holds stacked weights for all experts.-type StackedExpertWeights struct {- Weight *mlx.Array // Stacked weights [num_experts, out, in] or [num_experts, out, in_packed]- Scales *mlx.Array // Stacked scales (nil if not quantized)- Biases *mlx.Array // Stacked biases (nil if not quantized)- Bits int // Quantization bits (4 or 8), 0 if not quantized- GroupSize int // Quantization group size, 0 if not quantized-}--// collectAndStackExpertWeights loads and stacks expert weights for one projection type.-func collectAndStackExpertWeights(- weights safetensors.WeightSource,- prefix string,- projName string,- numExperts int32,- useQuantized bool,- cfg *Config,-) *StackedExpertWeights {- var w, s, b []*mlx.Array- var bits, groupSize int-- for e := int32(0); e < numExperts; e++ {- path := fmt.Sprintf("%s.mlp.experts.%d.%s", prefix, e, projName)- ew := loadExpertWeight(weights, path, useQuantized, cfg)- if ew == nil {- continue- }- w = append(w, ew.Weight)- if ew.Scales != nil {- s = append(s, ew.Scales)- }- if ew.Biases != nil {- b = append(b, ew.Biases)- }- if e == 0 {- bits = ew.Bits- groupSize = ew.GroupSize- }- }-- result := &StackedExpertWeights{Bits: bits, GroupSize: groupSize}- if len(w) > 0 {- result.Weight = mlx.Stack(w, 0)- if len(s) > 0 {- result.Scales = mlx.Stack(s, 0)- }- if len(b) > 0 {- result.Biases = mlx.Stack(b, 0)- }- }- return result-}--// sanitizeExpertWeights stacks individual expert weights into tensors.-// If useQuantized is true and weights support GatherQMM, returns quantized components.-// Otherwise returns dequantized weights with nil scales/biases.-// Bits and GroupSize are detected per-weight to support mixed-precision (Q4 for gate/up, Q8 for down).-func sanitizeExpertWeights(weights safetensors.WeightSource, prefix string, numExperts int32, useQuantized bool, cfg *Config) (gate, up, down *StackedExpertWeights) {- gate = collectAndStackExpertWeights(weights, prefix, "gate_proj", numExperts, useQuantized, cfg)- up = collectAndStackExpertWeights(weights, prefix, "up_proj", numExperts, useQuantized, cfg)- down = collectAndStackExpertWeights(weights, prefix, "down_proj", numExperts, useQuantized, cfg)- return gate, up, down-}--// LoadFromManifest loads a GLM4-MoE-Lite model from a manifest (Ollama blob storage).-func LoadFromManifest(modelManifest *manifest.ModelManifest) (*Model, error) {- // Read config from manifest- configData, err := modelManifest.ReadConfig("config.json")- if err != nil {- return nil, fmt.Errorf("load config: %w", err)- }-- var cfg Config- if err := json.Unmarshal(configData, &cfg); err != nil {- return nil, fmt.Errorf("parse config: %w", err)- }-- // Compute derived fields- cfg.QHeadDim = cfg.QKNopeHeadDim + cfg.QKRopeHeadDim- cfg.Scale = computeScale(&cfg)-- // Load weights from manifest blobs- weights, err := manifest.LoadWeightsFromManifest(modelManifest, "")- if err != nil {- return nil, fmt.Errorf("load weights: %w", err)- }-- if err := weights.Load(0); err != nil {- return nil, fmt.Errorf("load weight data: %w", err)- }-- // Set up quantization parameters (only if model is actually quantized)- // Note: QuantGroupSize will be detected dynamically from tensor shapes during weight loading- quantization := weights.Quantization()- useQuantized := false- if quantization != "" {- _, cfg.QuantBits, cfg.QuantMode = safetensors.QuantizationParams(quantization)- useQuantized = supportsGatherQMM(cfg.QuantMode, cfg.QuantBits)- }-- // Load tokenizer from manifest with config files for EOS token detection- tokData, err := modelManifest.ReadConfig("tokenizer.json")- if err != nil {- return nil, fmt.Errorf("load tokenizer config: %w", err)- }-- // Build tokenizer config with companion files for EOS/BOS token loading- tokConfig := &tokenizer.TokenizerConfig{- ConfigJSON: configData, // Already loaded above, contains eos_token_id- }-- // Try to load generation_config.json if available (preferred source for EOS)- if genConfigData, err := modelManifest.ReadConfig("generation_config.json"); err == nil {- tokConfig.GenerationConfigJSON = genConfigData- }-- // Try to load tokenizer_config.json if available- if tokConfigData, err := modelManifest.ReadConfig("tokenizer_config.json"); err == nil {- tokConfig.TokenizerConfigJSON = tokConfigData- }-- tok, err := tokenizer.LoadFromBytesWithConfig(tokData, tokConfig)- if err != nil {- return nil, fmt.Errorf("parse tokenizer: %w", err)- }-- m := &Model{- Layers: make([]Block, cfg.NumHiddenLayers),- Config: &cfg,- tok: tok,- }-- // Load embedding, norm, and lm_head- if err := safetensors.LoadModule(m, weights, ""); err != nil {- return nil, err- }-- // Load layers manually due to different block types- for i := int32(0); i < cfg.NumHiddenLayers; i++ {- prefix := fmt.Sprintf("model.layers.%d", i)-- // Load attention (same for both block types)- attn := &MLAAttention{}- if err := safetensors.LoadModule(attn, weights, prefix); err != nil {- return nil, fmt.Errorf("layer %d attention: %w", i, err)- }-- // Sanitize MLA weights for absorbed attention- embedQ, unembedOut := sanitizeMLAWeights(weights, prefix, &cfg)- attn.EmbedQ = nn.NewMultiLinear(embedQ)- attn.UnembedOut = nn.NewMultiLinear(unembedOut)-- if i < cfg.FirstKDenseReplace {- // Dense block- block := &DenseBlock{Attention: attn}- if err := safetensors.LoadModule(block, weights, prefix); err != nil {- return nil, fmt.Errorf("layer %d dense: %w", i, err)- }- m.Layers[i] = block- } else {- // MoE block- block := &MoEBlock{Attention: attn}- if err := safetensors.LoadModule(block, weights, prefix); err != nil {- return nil, fmt.Errorf("layer %d moe block: %w", i, err)- }-- // Stack expert weights (pass cfg so group sizes can be detected)- gate, up, down := sanitizeExpertWeights(weights, prefix, cfg.NRoutedExperts, useQuantized, &cfg)-- switchMLP := &SwitchMLP{UseQuantized: useQuantized}- if useQuantized {- switchMLP.GateWeightQ = gate.Weight- switchMLP.GateScales = gate.Scales- switchMLP.GateBiases = gate.Biases- switchMLP.GateBits = gate.Bits- switchMLP.GateGroupSize = gate.GroupSize- switchMLP.UpWeightQ = up.Weight- switchMLP.UpScales = up.Scales- switchMLP.UpBiases = up.Biases- switchMLP.UpBits = up.Bits- switchMLP.UpGroupSize = up.GroupSize- switchMLP.DownWeightQ = down.Weight- switchMLP.DownScales = down.Scales- switchMLP.DownBiases = down.Biases- switchMLP.DownBits = down.Bits- switchMLP.DownGroupSize = down.GroupSize- } else {- switchMLP.GateWeight = gate.Weight- switchMLP.UpWeight = up.Weight- switchMLP.DownWeight = down.Weight- }-- block.MoE = &MoE{- Gate: &MoEGate{},- SwitchMLP: switchMLP,- }-- // Load gate weights- if err := safetensors.LoadModule(block.MoE.Gate, weights, prefix); err != nil {- return nil, fmt.Errorf("layer %d gate: %w", i, err)- }-- // Load shared experts if present- if cfg.NSharedExperts > 0 {- block.MoE.SharedExperts = &SharedExperts{}- if err := safetensors.LoadModule(block.MoE.SharedExperts, weights, prefix); err != nil {- return nil, fmt.Errorf("layer %d shared experts: %w", i, err)- }- }-- m.Layers[i] = block- }- }-- mlx.Eval(mlx.Collect(m)...)- weights.ReleaseAll()-- return m, nil-}--// Forward computes the forward pass of the model-func (m *Model) Forward(tokens *mlx.Array, caches []cache.Cache) *mlx.Array {- B, L := tokens.Shape()[0], tokens.Shape()[1]-- h := m.EmbedTokens.Forward(tokens)-- for i, layer := range m.Layers {- var c cache.Cache- if caches != nil {- c = caches[i]- }- h = layer.Forward(h, c, B, L, m.Config)- }-- h = m.Norm.Forward(h, m.RMSNormEps)- return m.LMHead.Forward(h)-}--// Interface methods--// NumLayers returns the number of transformer layers-func (m *Model) NumLayers() int { return len(m.Layers) }--// MaxContextLength returns the maximum context length-func (m *Model) MaxContextLength() int32 { return m.MaxPositionEmbeddings }--// VocabSize returns the vocabulary size-func (m *Model) VocabSize() int32 { return m.Config.VocabSize }--// Tokenizer returns the model's tokenizer-func (m *Model) Tokenizer() *tokenizer.Tokenizer { return m.tok }--// NewCache creates a new KV cache for the model-func (m *Model) NewCache(maxSeqLen int32) []cache.Cache {- caches := make([]cache.Cache, len(m.Layers))- for i := range caches {- caches[i] = cache.NewKVCache()- }- return caches-}--// FormatPrompt applies the GLM-4 chat template with thinking enabled by default.-// This follows the GLM-4.7 format with <think> tag for reasoning mode.-func (m *Model) FormatPrompt(prompt string) string {- return "[gMASK]<sop><|user|>" + prompt + "<|assistant|><think>"-}--// FormatPromptWithThinking applies the GLM-4 chat template with explicit thinking control.-// When think is true, the prompt ends with <think> to enable reasoning mode.-// When think is false, the prompt ends with </think> to skip reasoning.-func (m *Model) FormatPromptWithThinking(prompt string, think bool) string {- if think {- return "[gMASK]<sop><|user|>" + prompt + "<|assistant|><think>"- }- return "[gMASK]<sop><|user|>" + prompt + "<|assistant|></think>"-}--// NewRenderer returns a new Renderer for formatting multi-turn conversations.-func (m *Model) NewRenderer() *Renderer {- return &Renderer{}-}--// NewParser returns a new Parser for extracting thinking and tool calls from output.-func (m *Model) NewParser() *Parser {- return &Parser{}-}diff --git a/x/imagegen/models/glm4_moe_lite/parser.go b/x/imagegen/models/glm4_moe_lite/parser.godeleted file mode 100644index c81ec5a40..000000000--- a/x/imagegen/models/glm4_moe_lite/parser.go+++ /dev/null@@ -1,479 +0,0 @@-//go:build mlx--package glm4_moe_lite--import (- "context"- "encoding/json"- "encoding/xml"- "fmt"- "log/slog"- "strings"- "unicode"-- "github.com/ollama/ollama/api"- "github.com/ollama/ollama/logutil"-)--type parserState int--const (- parserState_LookingForThinkingOpen parserState = iota- parserState_ThinkingStartedEatingWhitespace- parserState_CollectingThinking- parserState_ThinkingDoneEatingWhitespace- parserState_CollectingContent- parserState_ToolStartedEatingWhitespace- parserState_CollectingToolContent-)--const (- thinkingOpenTag = "<think>"- thinkingCloseTag = "</think>"- toolOpenTag = "<tool_call>"- toolCloseTag = "</tool_call>"-)--// Parser parses GLM4-MoE-Lite model output to extract thinking and tool calls.-// GLM-4's prompt ends with <think> when thinking is enabled, so the parser-// must start in CollectingThinking state (the model outputs thinking content directly).-type Parser struct {- state parserState- buffer strings.Builder- tools []api.Tool-}--// HasToolSupport returns true as GLM4 supports tool calling.-func (p *Parser) HasToolSupport() bool {- return true-}--// HasThinkingSupport returns true as GLM4 supports thinking mode.-func (p *Parser) HasThinkingSupport() bool {- return true-}--// Init initializes the parser with tools and thinking configuration.-func (p *Parser) Init(tools []api.Tool, lastMessage *api.Message, thinkValue *api.ThinkValue) []api.Tool {- p.tools = tools- // When thinking is enabled (nil or true), the prompt ends with <think>,- // so model output starts directly with thinking content (no opening tag).- if thinkValue == nil || thinkValue.Bool() {- p.state = parserState_CollectingThinking- }- return tools-}--type parserEvent interface {- isParserEvent()-}--type eventContent struct {- content string-}--func (eventContent) isParserEvent() {}--type eventRawToolCall struct {- raw string-}--func (eventRawToolCall) isParserEvent() {}--type eventThinkingContent struct {- content string-}--func (eventThinkingContent) isParserEvent() {}--// Add processes new output text and returns parsed content, thinking, and tool calls.-func (p *Parser) Add(s string, done bool) (content string, thinking string, calls []api.ToolCall, err error) {- p.buffer.WriteString(s)- events := p.parseEvents()-- var toolCalls []api.ToolCall- var contentSb strings.Builder- var thinkingSb strings.Builder-- for _, event := range events {- switch event := event.(type) {- case eventRawToolCall:- toolCall, err := parseToolCall(event, p.tools)- if err != nil {- slog.Warn("glm-4 tool call parsing failed", "error", err)- return "", "", nil, err- }- toolCalls = append(toolCalls, toolCall)- case eventThinkingContent:- thinkingSb.WriteString(event.content)- case eventContent:- contentSb.WriteString(event.content)- }- }-- return contentSb.String(), thinkingSb.String(), toolCalls, nil-}--func (p *Parser) parseEvents() []parserEvent {- var all []parserEvent-- keepLooping := true- for keepLooping {- var events []parserEvent- events, keepLooping = p.eat()- if len(events) > 0 {- all = append(all, events...)- }- }-- if len(all) > 0 {- slog.Log(context.TODO(), logutil.LevelTrace, "glm-4 events parsed", "events", all, "state", p.state, "buffer", p.buffer.String())- }-- return all-}--// eatLeadingWhitespaceAndTransitionTo consumes leading whitespace from the buffer-// and transitions to the next state. Returns (nil, false) if only whitespace remains-// in the buffer (needs more input), or (nil, true) if we successfully transitioned.-func (p *Parser) eatLeadingWhitespaceAndTransitionTo(nextState parserState) ([]parserEvent, bool) {- trimmed := strings.TrimLeftFunc(p.buffer.String(), unicode.IsSpace)- p.buffer.Reset()- if trimmed == "" {- return nil, false // Still only whitespace, keep waiting for more input- }- p.state = nextState- p.buffer.WriteString(trimmed)- return nil, true // Successfully transitioned-}--// splitAtTag splits the buffer at the given tag, returns the content before (trimmed of trailing whitespace),-// the content after (optionally trimmed of leading whitespace), and updates the buffer-func (p *Parser) splitAtTag(tag string, trimAfter bool) (string, string) {- split := strings.SplitN(p.buffer.String(), tag, 2)- before := split[0]- before = strings.TrimRightFunc(before, unicode.IsSpace)- after := split[1]- if trimAfter {- after = strings.TrimLeftFunc(after, unicode.IsSpace)- }- p.buffer.Reset()- p.buffer.WriteString(after)- return before, after-}--func (p *Parser) eat() ([]parserEvent, bool) {- var events []parserEvent-- switch p.state {- case parserState_LookingForThinkingOpen:- trimmed := strings.TrimLeftFunc(p.buffer.String(), unicode.IsSpace)- if strings.HasPrefix(trimmed, thinkingOpenTag) {- // Found <think> opening tag- after := strings.TrimPrefix(trimmed, thinkingOpenTag)- after = strings.TrimLeftFunc(after, unicode.IsSpace)- p.buffer.Reset()- p.buffer.WriteString(after)- if after == "" {- p.state = parserState_ThinkingStartedEatingWhitespace- } else {- p.state = parserState_CollectingThinking- }- return events, true- } else if strings.HasPrefix(thinkingOpenTag, trimmed) {- // Partial opening tag seen, keep accumulating- return events, false- } else if trimmed == "" {- // Only whitespace, keep accumulating- return events, false- } else {- // No thinking tag found, skip to content collection- p.state = parserState_CollectingContent- // Don't trim - we want to keep the original content- return events, true- }-- case parserState_ThinkingStartedEatingWhitespace:- return p.eatLeadingWhitespaceAndTransitionTo(parserState_CollectingThinking)-- case parserState_CollectingThinking:- acc := p.buffer.String()- if strings.Contains(acc, thinkingCloseTag) {- thinking, remaining := p.splitAtTag(thinkingCloseTag, true)- if len(thinking) > 0 {- events = append(events, eventThinkingContent{content: thinking})- }- if remaining == "" {- p.state = parserState_ThinkingDoneEatingWhitespace- } else {- p.state = parserState_CollectingContent- }- return events, true- } else if overlapLen := overlap(acc, thinkingCloseTag); overlapLen > 0 {- // Partial closing tag - withhold it along with any trailing whitespace before it- beforePartialTag := acc[:len(acc)-overlapLen]- trailingWsLen := trailingWhitespaceLen(beforePartialTag)- ambiguousStart := len(beforePartialTag) - trailingWsLen-- unambiguous := acc[:ambiguousStart]- ambiguous := acc[ambiguousStart:]- p.buffer.Reset()- p.buffer.WriteString(ambiguous)- if len(unambiguous) > 0 {- events = append(events, eventThinkingContent{content: unambiguous})- }- return events, false- } else {- // Pure thinking content - withhold trailing whitespace (might precede closing tag)- whitespaceLen := trailingWhitespaceLen(acc)- ambiguousStart := len(acc) - whitespaceLen-- unambiguous := acc[:ambiguousStart]- ambiguous := acc[ambiguousStart:]- p.buffer.Reset()- p.buffer.WriteString(ambiguous)- if len(unambiguous) > 0 {- events = append(events, eventThinkingContent{content: unambiguous})- }- return events, false- }-- case parserState_ThinkingDoneEatingWhitespace:- return p.eatLeadingWhitespaceAndTransitionTo(parserState_CollectingContent)-- case parserState_CollectingContent:- if strings.Contains(p.buffer.String(), toolOpenTag) {- before, after := p.splitAtTag(toolOpenTag, true)- if len(before) > 0 {- events = append(events, eventContent{content: before})- }- if after == "" {- p.state = parserState_ToolStartedEatingWhitespace- } else {- p.state = parserState_CollectingToolContent- }- return events, true- } else if overlapLen := overlap(p.buffer.String(), toolOpenTag); overlapLen > 0 {- beforePartialTag := p.buffer.String()[:len(p.buffer.String())-overlapLen]- trailingWsLen := trailingWhitespaceLen(beforePartialTag)- ambiguousStart := len(beforePartialTag) - trailingWsLen-- unambiguous := p.buffer.String()[:ambiguousStart]- ambiguous := p.buffer.String()[ambiguousStart:]- p.buffer.Reset()- p.buffer.WriteString(ambiguous)- if len(unambiguous) > 0 {- events = append(events, eventContent{content: unambiguous})- }- return events, false- } else {- whitespaceLen := trailingWhitespaceLen(p.buffer.String())- ambiguousStart := len(p.buffer.String()) - whitespaceLen-- unambiguous := p.buffer.String()[:ambiguousStart]- ambiguous := p.buffer.String()[ambiguousStart:]- p.buffer.Reset()- p.buffer.WriteString(ambiguous)- if len(unambiguous) > 0 {- events = append(events, eventContent{content: unambiguous})- }- return events, false- }-- case parserState_ToolStartedEatingWhitespace:- return p.eatLeadingWhitespaceAndTransitionTo(parserState_CollectingToolContent)-- case parserState_CollectingToolContent:- acc := p.buffer.String()- if strings.Contains(acc, toolCloseTag) {- toolContent, _ := p.splitAtTag(toolCloseTag, true)- if len(toolContent) == 0 {- slog.Warn("glm4 tool call closing tag found but no content before it")- }- events = append(events, eventRawToolCall{raw: toolContent})- p.state = parserState_CollectingContent- return events, true- } else {- // Keep accumulating - tool calls are not streamed- // We just wait for the closing tag- return events, false- }-- default:- panic("unreachable")- }-}--// overlap returns the length of the overlap between the end of s and the start of tag.-func overlap(s, tag string) int {- for i := 1; i <= len(tag) && i <= len(s); i++ {- if strings.HasSuffix(s, tag[:i]) {- return i- }- }- return 0-}--// trailingWhitespaceLen returns the length of trailing whitespace in s.-func trailingWhitespaceLen(s string) int {- trimmed := strings.TrimRightFunc(s, unicode.IsSpace)- return len(s) - len(trimmed)-}--// ToolCallXML represents the structure of a GLM-4 tool call for XML parsing-type ToolCallXML struct {- XMLName xml.Name `xml:"tool_call"`- Content string `xml:",chardata"` // Function name (text nodes between tags)- Keys []string `xml:"arg_key"` // All arg_key elements in document order- Values []string `xml:"arg_value"` // All arg_value elements in document order-}--// escapeContent escapes XML entities in text content while preserving arg_key/arg_value tags-func escapeContent(s string) string {- var result strings.Builder- inTag := false-- for i := range len(s) {- ch := s[i]-- if ch == '<' {- // Check if this is a known tag- if strings.HasPrefix(s[i:], "<arg_key>") ||- strings.HasPrefix(s[i:], "</arg_key>") ||- strings.HasPrefix(s[i:], "<arg_value>") ||- strings.HasPrefix(s[i:], "</arg_value>") {- inTag = true- }- }-- if inTag {- result.WriteByte(ch)- if ch == '>' {- inTag = false- }- } else {- // Escape special characters in text content- switch ch {- case '&':- result.WriteString("&")- case '<':- result.WriteString("<")- case '>':- result.WriteString(">")- default:- result.WriteByte(ch)- }- }- }-- return result.String()-}--func parseToolCall(raw eventRawToolCall, tools []api.Tool) (api.ToolCall, error) {- // Escape any unescaped entities in text content- escaped := escapeContent(raw.raw)-- // Wrap the content in a root element to make it valid XML- xmlString := "<tool_call>" + escaped + "</tool_call>"-- // Parse XML into struct- var parsed ToolCallXML- if err := xml.Unmarshal([]byte(xmlString), &parsed); err != nil {- return api.ToolCall{}, fmt.Errorf("failed to parse XML: %w", err)- }-- // Extract and trim function name- functionName := strings.TrimSpace(parsed.Content)- if functionName == "" {- return api.ToolCall{}, fmt.Errorf("empty function name")- }-- // Verify keys and values are paired correctly- if len(parsed.Keys) != len(parsed.Values) {- return api.ToolCall{}, fmt.Errorf("mismatched arg_key and arg_value counts: %d keys, %d values", len(parsed.Keys), len(parsed.Values))- }-- // Find the matching tool to get parameter types- var matchedTool *api.Tool- for i := range tools {- if tools[i].Function.Name == functionName {- matchedTool = &tools[i]- break- }- }-- // Build arguments map by pairing keys and values- toolCall := api.ToolCall{- Function: api.ToolCallFunction{- Name: functionName,- Arguments: api.NewToolCallFunctionArguments(),- },- }-- for i := range parsed.Keys {- key := strings.TrimSpace(parsed.Keys[i])- value := parsed.Values[i] // Don't trim here - parseValue handles it-- // Look up parameter type- var paramType api.PropertyType- if matchedTool != nil && matchedTool.Function.Parameters.Properties != nil {- if prop, ok := matchedTool.Function.Parameters.Properties.Get(key); ok {- // Handle anyOf by collecting all types from the union- if len(prop.AnyOf) > 0 {- for _, anyOfProp := range prop.AnyOf {- paramType = append(paramType, anyOfProp.Type...)- }- } else {- paramType = prop.Type- }- }- }-- // Parse value with type coercion- toolCall.Function.Arguments.Set(key, parseValue(value, paramType))- }-- return toolCall, nil-}--// parseValue parses a string value and coerces it to the appropriate type based on paramType.-func parseValue(value string, paramType api.PropertyType) any {- value = strings.TrimSpace(value)-- // If no type specified, return as string- if len(paramType) == 0 {- return value- }-- // Try to parse based on specified types- for _, t := range paramType {- switch t {- case "boolean":- if value == "true" {- return true- }- if value == "false" {- return false- }- case "integer":- var i int64- if _, err := fmt.Sscanf(value, "%d", &i); err == nil {- return i- }- case "number":- var f float64- if _, err := fmt.Sscanf(value, "%f", &f); err == nil {- return f- }- case "array", "object":- // Try to parse as JSON- var result any- if err := json.Unmarshal([]byte(value), &result); err == nil {- return result- }- }- }-- // Default to string- return value-}diff --git a/x/imagegen/models/glm4_moe_lite/parser_test.go b/x/imagegen/models/glm4_moe_lite/parser_test.godeleted file mode 100644index 0ce382709..000000000--- a/x/imagegen/models/glm4_moe_lite/parser_test.go+++ /dev/null@@ -1,192 +0,0 @@-//go:build mlx--package glm4_moe_lite--import (- "testing"-- "github.com/ollama/ollama/api"-)--func TestParserThinking(t *testing.T) {- tests := []struct {- name string- input string- thinkEnabled bool- wantContent string- wantThinking string- wantToolCalls int- }{- {- name: "thinking enabled - simple thinking then content",- input: "Let me think about this...</think>Here is my answer.",- thinkEnabled: true,- wantThinking: "Let me think about this...",- wantContent: "Here is my answer.",- },- {- name: "thinking enabled - only thinking",- input: "I need to consider multiple factors...",- thinkEnabled: true,- wantThinking: "I need to consider multiple factors...",- wantContent: "",- },- {- name: "thinking disabled - direct content",- input: "Here is my direct answer.",- thinkEnabled: false,- wantThinking: "",- wantContent: "Here is my direct answer.",- },- {- name: "thinking with tool call",- input: "Let me search for that...</think>I'll use a tool.<tool_call>search<arg_key>query</arg_key><arg_value>test</arg_value></tool_call>",- thinkEnabled: true,- wantThinking: "Let me search for that...",- wantContent: "I'll use a tool.",- wantToolCalls: 1,- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- p := &Parser{}-- var thinkValue *api.ThinkValue- if tt.thinkEnabled {- thinkValue = &api.ThinkValue{Value: true}- } else {- thinkValue = &api.ThinkValue{Value: false}- }-- // Define tools for tool call tests- props := api.NewToolPropertiesMap()- props.Set("query", api.ToolProperty{Type: api.PropertyType{"string"}})- tools := []api.Tool{- {- Function: api.ToolFunction{- Name: "search",- Parameters: api.ToolFunctionParameters{- Properties: props,- },- },- },- }-- p.Init(tools, nil, thinkValue)-- content, thinking, calls, err := p.Add(tt.input, true)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }-- if thinking != tt.wantThinking {- t.Errorf("thinking = %q, want %q", thinking, tt.wantThinking)- }- if content != tt.wantContent {- t.Errorf("content = %q, want %q", content, tt.wantContent)- }- if len(calls) != tt.wantToolCalls {- t.Errorf("len(calls) = %d, want %d", len(calls), tt.wantToolCalls)- }- })- }-}--func TestParserToolCall(t *testing.T) {- p := &Parser{}-- props := api.NewToolPropertiesMap()- props.Set("location", api.ToolProperty{Type: api.PropertyType{"string"}})- props.Set("unit", api.ToolProperty{Type: api.PropertyType{"string"}})- tools := []api.Tool{- {- Function: api.ToolFunction{- Name: "get_weather",- Parameters: api.ToolFunctionParameters{- Properties: props,- },- },- },- }-- // Initialize with thinking disabled- tv := &api.ThinkValue{Value: false}- p.Init(tools, nil, tv)-- input := "<tool_call>get_weather<arg_key>location</arg_key><arg_value>San Francisco</arg_value><arg_key>unit</arg_key><arg_value>celsius</arg_value></tool_call>"-- _, _, calls, err := p.Add(input, true)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }-- if len(calls) != 1 {- t.Fatalf("expected 1 tool call, got %d", len(calls))- }-- call := calls[0]- if call.Function.Name != "get_weather" {- t.Errorf("function name = %q, want %q", call.Function.Name, "get_weather")- }-- location, ok := call.Function.Arguments.Get("location")- if !ok || location != "San Francisco" {- t.Errorf("location = %v, want %q", location, "San Francisco")- }-- unit, ok := call.Function.Arguments.Get("unit")- if !ok || unit != "celsius" {- t.Errorf("unit = %v, want %q", unit, "celsius")- }-}--func TestOverlap(t *testing.T) {- tests := []struct {- s string- tag string- want int- }{- {"hello<", "</think>", 1},- {"hello</", "</think>", 2},- {"hello</t", "</think>", 3},- {"hello</th", "</think>", 4},- {"hello</thi", "</think>", 5},- {"hello</thin", "</think>", 6},- {"hello</think", "</think>", 7},- {"hello</think>", "</think>", 8}, // Complete tag at end returns full length- {"hello", "</think>", 0},- {"", "</think>", 0},- }-- for _, tt := range tests {- t.Run(tt.s+"_"+tt.tag, func(t *testing.T) {- got := overlap(tt.s, tt.tag)- if got != tt.want {- t.Errorf("overlap(%q, %q) = %d, want %d", tt.s, tt.tag, got, tt.want)- }- })- }-}--func TestTrailingWhitespaceLen(t *testing.T) {- tests := []struct {- s string- want int- }{- {"hello ", 3},- {"hello\n\t ", 3},- {"hello", 0},- {"", 0},- {" ", 3},- }-- for _, tt := range tests {- t.Run(tt.s, func(t *testing.T) {- got := trailingWhitespaceLen(tt.s)- if got != tt.want {- t.Errorf("trailingWhitespaceLen(%q) = %d, want %d", tt.s, got, tt.want)- }- })- }-}diff --git a/x/imagegen/models/glm4_moe_lite/render.go b/x/imagegen/models/glm4_moe_lite/render.godeleted file mode 100644index 4998604bf..000000000--- a/x/imagegen/models/glm4_moe_lite/render.go+++ /dev/null@@ -1,175 +0,0 @@-//go:build mlx--package glm4_moe_lite--import (- "encoding/json"- "fmt"- "strings"-- "github.com/ollama/ollama/api"-)--// Renderer renders messages for GLM4-MoE-Lite models.-//-// GLM-4 Thinking Modes (ref: https://docs.z.ai/guides/capabilities/thinking-mode):-//-// 1. INTERLEAVED THINKING-// The model thinks between tool calls and after receiving tool results.-// This enables complex step-by-step reasoning: interpreting each tool output-// before deciding what to do next. Thinking blocks are preserved and returned-// with tool results to maintain reasoning continuity.-//-// 2. PRESERVED THINKING-// The model retains reasoning content from previous assistant turns in context.-// This preserves reasoning continuity across multi-turn conversations. The-// upstream API has a "clear_thinking" parameter to control this:-// - clear_thinking=true: clears reasoning from previous turns (outputs </think>)-// - clear_thinking=false: preserves <think>...</think> blocks from previous turns-//-// 3. TURN-LEVEL THINKING-// Controls whether the model should reason on each turn. The upstream API-// uses "enable_thinking" parameter:-// - enable_thinking=true: outputs <think> to start reasoning-// - enable_thinking=false: outputs </think> to skip reasoning-//-// OLLAMA DEFAULTS:-// - Thinking is ENABLED by default (thinkValue=nil or true outputs <think>)-// - Thinking is PRESERVED by default (reasoning content from previous turns is always-// included in <think>...</think> blocks, equivalent to clear_thinking=false)-// - Users can disable thinking per-turn via thinkValue=false-type Renderer struct{}--// Render renders messages into the GLM4 chat format.-func (r *Renderer) Render(messages []api.Message, tools []api.Tool, thinkValue *api.ThinkValue) (string, error) {- var sb strings.Builder-- sb.WriteString("[gMASK]<sop>")-- if len(tools) > 0 {- sb.WriteString("<|system|>\n")- sb.WriteString("# Tools\n\n")- sb.WriteString("You may call one or more functions to assist with the user query.\n\n")- sb.WriteString("You are provided with function signatures within <tools></tools> XML tags:\n")- sb.WriteString("<tools>\n")- for _, tool := range tools {- d, _ := json.Marshal(tool)- sb.WriteString(formatToolJSON(d))- sb.WriteString("\n")- }- sb.WriteString("</tools>\n\n")- sb.WriteString("For each function call, output the function name and arguments within the following XML format:\n")- sb.WriteString("<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>")- }-- think := true- if thinkValue != nil && !thinkValue.Bool() {- think = false- }-- for i, message := range messages {- switch message.Role {- case "user":- sb.WriteString("<|user|>")- sb.WriteString(message.Content)- case "assistant":- sb.WriteString("<|assistant|>")- if message.Thinking != "" {- sb.WriteString("<think>" + message.Thinking + "</think>")- } else {- sb.WriteString("</think>")- }- if message.Content != "" {- sb.WriteString(message.Content)- }- if len(message.ToolCalls) > 0 {- for _, toolCall := range message.ToolCalls {- sb.WriteString("<tool_call>" + toolCall.Function.Name)- sb.WriteString(renderToolArguments(toolCall.Function.Arguments))- sb.WriteString("</tool_call>")- }- }- case "tool":- if i == 0 || messages[i-1].Role != "tool" {- sb.WriteString("<|observation|>")- }- sb.WriteString("<tool_response>")- sb.WriteString(message.Content)- sb.WriteString("</tool_response>")- case "system":- sb.WriteString("<|system|>")- sb.WriteString(message.Content)- }- }-- sb.WriteString("<|assistant|>")- if think {- sb.WriteString("<think>")- } else {- sb.WriteString("</think>")- }-- return sb.String(), nil-}--// renderToolArguments converts tool call arguments to GLM4 XML format.-func renderToolArguments(args api.ToolCallFunctionArguments) string {- var sb strings.Builder- for key, value := range args.All() {- sb.WriteString("<arg_key>" + key + "</arg_key>")- var valueStr string- if str, ok := value.(string); ok {- valueStr = str- } else {- jsonBytes, err := json.Marshal(value)- if err != nil {- valueStr = fmt.Sprintf("%v", value)- } else {- valueStr = string(jsonBytes)- }- }-- sb.WriteString("<arg_value>" + valueStr + "</arg_value>")- }-- return sb.String()-}--// formatToolJSON formats JSON for GLM4 tool definitions by adding spaces after : and ,-func formatToolJSON(raw []byte) string {- var sb strings.Builder- sb.Grow(len(raw) + len(raw)/10)-- inString := false- escaped := false- for i := range raw {- ch := raw[i]- sb.WriteByte(ch)-- if inString {- if escaped {- escaped = false- continue- }- if ch == '\\' {- escaped = true- continue- }- if ch == '"' {- inString = false- }- continue- }-- if ch == '"' {- inString = true- continue- }-- if ch == ':' || ch == ',' {- sb.WriteByte(' ')- }- }-- return sb.String()-}diff --git a/x/imagegen/models/glm4_moe_lite/render_test.go b/x/imagegen/models/glm4_moe_lite/render_test.godeleted file mode 100644index f0d576bec..000000000--- a/x/imagegen/models/glm4_moe_lite/render_test.go+++ /dev/null@@ -1,205 +0,0 @@-//go:build mlx--package glm4_moe_lite--import (- "strings"- "testing"-- "github.com/ollama/ollama/api"-)--func TestRendererSimple(t *testing.T) {- r := &Renderer{}-- messages := []api.Message{- {Role: "user", Content: "Hello"},- }-- // Thinking enabled (default)- result, err := r.Render(messages, nil, nil)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }-- expected := "[gMASK]<sop><|user|>Hello<|assistant|><think>"- if result != expected {- t.Errorf("result = %q, want %q", result, expected)- }-}--func TestRendererThinkingDisabled(t *testing.T) {- r := &Renderer{}-- messages := []api.Message{- {Role: "user", Content: "Hello"},- }-- tv := &api.ThinkValue{Value: false}-- result, err := r.Render(messages, nil, tv)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }-- expected := "[gMASK]<sop><|user|>Hello<|assistant|></think>"- if result != expected {- t.Errorf("result = %q, want %q", result, expected)- }-}--func TestRendererMultiTurn(t *testing.T) {- r := &Renderer{}-- messages := []api.Message{- {Role: "user", Content: "What is 2+2?"},- {Role: "assistant", Content: "4", Thinking: "Let me calculate: 2+2=4"},- {Role: "user", Content: "And 3+3?"},- }-- result, err := r.Render(messages, nil, nil)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }-- // Check key parts- if !strings.Contains(result, "[gMASK]<sop>") {- t.Error("missing [gMASK]<sop> prefix")- }- if !strings.Contains(result, "<|user|>What is 2+2?") {- t.Error("missing first user message")- }- if !strings.Contains(result, "<|assistant|><think>Let me calculate: 2+2=4</think>4") {- t.Error("missing assistant message with thinking")- }- if !strings.Contains(result, "<|user|>And 3+3?") {- t.Error("missing second user message")- }- if !strings.HasSuffix(result, "<|assistant|><think>") {- t.Errorf("should end with <|assistant|><think>, got suffix: %q", result[len(result)-30:])- }-}--func TestRendererWithSystem(t *testing.T) {- r := &Renderer{}-- messages := []api.Message{- {Role: "system", Content: "You are a helpful assistant."},- {Role: "user", Content: "Hello"},- }-- result, err := r.Render(messages, nil, nil)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }-- if !strings.Contains(result, "<|system|>You are a helpful assistant.") {- t.Error("missing system message")- }-}--func TestRendererWithTools(t *testing.T) {- r := &Renderer{}-- messages := []api.Message{- {Role: "user", Content: "What's the weather?"},- }-- props := api.NewToolPropertiesMap()- props.Set("location", api.ToolProperty{Type: api.PropertyType{"string"}, Description: "The city"})- tools := []api.Tool{- {- Function: api.ToolFunction{- Name: "get_weather",- Description: "Get the weather for a location",- Parameters: api.ToolFunctionParameters{- Type: "object",- Properties: props,- Required: []string{"location"},- },- },- },- }-- result, err := r.Render(messages, tools, nil)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }-- // Check for tool system prompt- if !strings.Contains(result, "<|system|>") {- t.Error("missing system tag for tools")- }- if !strings.Contains(result, "# Tools") {- t.Error("missing tools header")- }- if !strings.Contains(result, "<tools>") {- t.Error("missing tools tag")- }- if !strings.Contains(result, "get_weather") {- t.Error("missing tool name")- }- if !strings.Contains(result, "</tools>") {- t.Error("missing closing tools tag")- }-}--func TestRendererWithToolCalls(t *testing.T) {- r := &Renderer{}-- args := api.NewToolCallFunctionArguments()- args.Set("location", "San Francisco")-- messages := []api.Message{- {Role: "user", Content: "What's the weather in SF?"},- {- Role: "assistant",- ToolCalls: []api.ToolCall{- {- Function: api.ToolCallFunction{- Name: "get_weather",- Arguments: args,- },- },- },- },- {Role: "tool", Content: "Sunny, 72F"},- }-- result, err := r.Render(messages, nil, nil)- if err != nil {- t.Fatalf("unexpected error: %v", err)- }-- if !strings.Contains(result, "<tool_call>get_weather") {- t.Error("missing tool call")- }- if !strings.Contains(result, "<arg_key>location</arg_key>") {- t.Error("missing arg_key")- }- if !strings.Contains(result, "<arg_value>San Francisco</arg_value>") {- t.Error("missing arg_value")- }- if !strings.Contains(result, "</tool_call>") {- t.Error("missing tool call closing tag")- }- if !strings.Contains(result, "<|observation|>") {- t.Error("missing observation tag")- }- if !strings.Contains(result, "<tool_response>Sunny, 72F</tool_response>") {- t.Error("missing tool response")- }-}--func TestFormatToolJSON(t *testing.T) {- input := []byte(`{"name":"test","value":123}`)- result := formatToolJSON(input)-- // Should add spaces after : and ,- if !strings.Contains(result, ": ") {- t.Error("should add space after colon")- }- if !strings.Contains(result, ", ") {- t.Error("should add space after comma")- }-}diff --git a/x/imagegen/models/gpt_oss/gpt_oss.go b/x/imagegen/models/gpt_oss/gpt_oss.godeleted file mode 100644index bbf01370f..000000000--- a/x/imagegen/models/gpt_oss/gpt_oss.go+++ /dev/null@@ -1,487 +0,0 @@-//go:build mlx--package gpt_oss--import (- "encoding/json"- "fmt"- "math"- "os"- "path/filepath"-- "github.com/ollama/ollama/x/imagegen/cache"- "github.com/ollama/ollama/x/imagegen/mlx"- "github.com/ollama/ollama/x/imagegen/nn"- "github.com/ollama/ollama/x/imagegen/safetensors"- "github.com/ollama/ollama/x/imagegen/tokenizer"-)--// RopeScaling holds YaRN or other RoPE scaling configuration-type RopeScaling struct {- RopeType string `json:"rope_type"`- Factor float32 `json:"factor"`- OriginalMaxPositionEmbeddings int32 `json:"original_max_position_embeddings"`- BetaFast float32 `json:"beta_fast"`- BetaSlow float32 `json:"beta_slow"`-}--type Config struct {- HiddenSize int32 `json:"hidden_size"`- NumHiddenLayers int32 `json:"num_hidden_layers"`- IntermediateSize int32 `json:"intermediate_size"`- NumAttentionHeads int32 `json:"num_attention_heads"`- NumKeyValueHeads int32 `json:"num_key_value_heads"`- VocabSize int32 `json:"vocab_size"`- RMSNormEps float32 `json:"rms_norm_eps"`- RopeTheta float32 `json:"rope_theta"`- HeadDim int32 `json:"head_dim"`- SlidingWindow int32 `json:"sliding_window"`- NumLocalExperts int32 `json:"num_local_experts"`- NumExpertsPerTok int32 `json:"num_experts_per_tok"`- LayerTypes []string `json:"layer_types"`- SwiGLULimit float32 `json:"swiglu_limit"`- RopeScaling *RopeScaling `json:"rope_scaling"`- Scale float32 `json:"-"` // computed: 1/sqrt(HeadDim)-}--type Attention struct {- QProj *nn.Linear `weight:"self_attn.q_proj"`- KProj *nn.Linear `weight:"self_attn.k_proj"`- VProj *nn.Linear `weight:"self_attn.v_proj"`- OProj *nn.Linear `weight:"self_attn.o_proj"`- Sinks *mlx.Array `weight:"self_attn.sinks,optional"`- YarnFreqs *mlx.Array // computed- YarnMscale float32-}--// swiGLU applies the GPT-OSS custom SwiGLU activation.-// Formula: (gate * sigmoid(alpha * gate)) * (up + 1)-// with clipping: gate to [None, limit], up to [-limit, limit]-func swiGLU(gate, up *mlx.Array, alpha, limit float32) *mlx.Array {- // Clip gate to [None, limit]- gateClipped := mlx.ClipScalar(gate, 0, limit, false, true)-- // Clip up to [-limit, limit]- upClipped := mlx.ClipScalar(up, -limit, limit, true, true)-- // glu_scaled = alpha * gate_clipped- gluScaled := mlx.MulScalar(gateClipped, alpha)-- // sig = sigmoid(glu_scaled)- sig := mlx.Sigmoid(gluScaled)-- // out_glu = gate_clipped * sig- outGlu := mlx.Mul(gateClipped, sig)-- // result = out_glu * (up_clipped + 1)- return mlx.Mul(outGlu, mlx.AddScalar(upClipped, 1.0))-}--// compiledSwiGLU is a singleton compiled SwiGLU function shared across all layers-var compiledSwiGLU *mlx.CompiledFunc--// getCompiledSwiGLU returns the compiled SwiGLU function, creating it once if needed-func getCompiledSwiGLU() *mlx.CompiledFunc {- if compiledSwiGLU == nil {- const alpha float32 = 1.702- const limit float32 = 7.0- compiledSwiGLU = mlx.CompileShapeless(func(inputs []*mlx.Array) []*mlx.Array {- return []*mlx.Array{swiGLU(inputs[0], inputs[1], alpha, limit)}- }, true) // shapeless=true so it works for any input size- }- return compiledSwiGLU-}--// ComputeYarnFreqs computes YaRN-modified RoPE frequencies-// Based on mlx-lm's YarnRoPE implementation-func ComputeYarnFreqs(dims int32, base, scalingFactor float32, origMaxPos int32, betaFast, betaSlow float32) (*mlx.Array, float32) {- // yarn_find_correction_dim- yarnFindCorrectionDim := func(numRotations float64) float64 {- return float64(dims) * math.Log(float64(origMaxPos)/(numRotations*2*math.Pi)) / (2 * math.Log(float64(base)))- }-- // yarn_find_correction_range- low := int(math.Floor(yarnFindCorrectionDim(float64(betaFast))))- high := int(math.Ceil(yarnFindCorrectionDim(float64(betaSlow))))- if low < 0 {- low = 0- }- if high > int(dims)-1 {- high = int(dims) - 1- }-- // yarn_get_mscale- yarnGetMscale := func(scale, mscale float64) float64 {- if scale <= 1 {- return 1.0- }- return 0.1*mscale*math.Log(scale) + 1.0- }- mscale := float32(yarnGetMscale(float64(scalingFactor), 1.0) / yarnGetMscale(float64(scalingFactor), 0.0))-- // Compute frequencies- // freq_extra = base ** (arange(0, dims, 2) / dims)- // freq_inter = scaling_factor * freq_extra- halfDims := dims / 2- freqData := make([]float32, halfDims)- for i := int32(0); i < halfDims; i++ {- exp := float64(2*i) / float64(dims)- freqExtra := math.Pow(float64(base), exp)- freqInter := float64(scalingFactor) * freqExtra-- // linear ramp mask- var freqMask float64- if low == high {- freqMask = 0.0- } else {- t := (float64(i) - float64(low)) / float64(high-low)- if t < 0 {- t = 0- }- if t > 1 {- t = 1- }- freqMask = 1.0 - t- }-- // Combined frequency: (inter * extra) / (inter * mask + extra * (1 - mask))- freqData[i] = float32((freqInter * freqExtra) / (freqInter*freqMask + freqExtra*(1-freqMask)))- }-- return mlx.NewArray(freqData, []int32{halfDims}), mscale-}--// initYarn initializes YaRN RoPE if configured-func (a *Attention) initYarn(cfg *Config) {- a.YarnMscale = 1.0- if cfg.RopeScaling != nil && cfg.RopeScaling.RopeType == "yarn" {- a.YarnFreqs, a.YarnMscale = ComputeYarnFreqs(- cfg.HeadDim,- cfg.RopeTheta,- cfg.RopeScaling.Factor,- cfg.RopeScaling.OriginalMaxPositionEmbeddings,- cfg.RopeScaling.BetaFast,- cfg.RopeScaling.BetaSlow,- )- }-}--func (a *Attention) Forward(x *mlx.Array, c cache.Cache, B, L int32, mask *mlx.Array, maskMode string, cfg *Config) *mlx.Array {- q := a.QProj.Forward(x)- k := a.KProj.Forward(x)- v := a.VProj.Forward(x)-- // Reshape via AsStrided: [B, L, n_heads * head_dim] -> [B, n_heads, L, head_dim]- q = mlx.AsStrided(q, []int32{B, cfg.NumAttentionHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumAttentionHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumAttentionHeads * cfg.HeadDim), 1}, 0)- k = mlx.AsStrided(k, []int32{B, cfg.NumKeyValueHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumKeyValueHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumKeyValueHeads * cfg.HeadDim), 1}, 0)- v = mlx.AsStrided(v, []int32{B, cfg.NumKeyValueHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumKeyValueHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumKeyValueHeads * cfg.HeadDim), 1}, 0)-- offset := 0- if c != nil {- offset = c.Offset()- }- if a.YarnFreqs != nil {- if a.YarnMscale != 1.0 {- q = mlx.MulScalar(q, a.YarnMscale)- }- q = mlx.RoPEWithFreqs(q, a.YarnFreqs, int(cfg.HeadDim), false, 1.0, offset)- k = mlx.RoPEWithFreqs(k, a.YarnFreqs, int(cfg.HeadDim), false, 1.0, offset)- } else {- q = mlx.RoPE(q, int(cfg.HeadDim), false, cfg.RopeTheta, 1.0, offset)- k = mlx.RoPE(k, int(cfg.HeadDim), false, cfg.RopeTheta, 1.0, offset)- }-- if c != nil {- k, v = c.Update(k, v, int(L))- }-- out := mlx.ScaledDotProductAttentionWithSinks(q, k, v, cfg.Scale, maskMode, mask, a.Sinks)- out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim)- return a.OProj.Forward(out)-}--// CreateSlidingWindowMask creates a causal mask with sliding window-// Mirrors mlx-lm's create_causal_mask with window_size-func CreateSlidingWindowMask(seqLen, queryStart, keyStart, keyLen, windowSize int) *mlx.Array {- // Build mask aligned to actual cache length (may be rotated)- // rinds covers existing keys: [keyStart, keyStart+keyLen)- // linds covers new queries: [queryStart, queryStart+seqLen)- rinds := mlx.Arange(float32(keyStart), float32(keyStart+keyLen), 1) // [keyLen]- linds := mlx.Arange(float32(queryStart), float32(queryStart+seqLen), 1) // [seqLen]-- linds = mlx.ExpandDims(linds, 1) // [seqLen, 1]- rinds = mlx.ExpandDims(rinds, 0) // [1, keyLen]-- causalMask := mlx.GreaterEqual(linds, rinds) // [seqLen, keyLen]- windowLimit := mlx.AddScalar(rinds, float32(windowSize))- windowMask := mlx.LessArray(linds, windowLimit) // [seqLen, keyLen]-- return mlx.LogicalAnd(causalMask, windowMask)-}--// MoE represents the Mixture of Experts SwiGLU layer with quantized experts.-type MoE struct {- Router *nn.Linear `weight:"mlp.router"`- TopK int32- HiddenSize int32- GroupSize int- Bits int- // Expert weights (loaded manually via sanitizeExpertWeights)- GateBlocks, GateScales, GateBias *mlx.Array- UpBlocks, UpScales, UpBias *mlx.Array- DownBlocks, DownScales, DownBias *mlx.Array-}--func (moe *MoE) Forward(x *mlx.Array, B, L int32) *mlx.Array {- logits := moe.Router.Forward(x)- neg := mlx.Neg(logits)- part := mlx.Argpartition(neg, int(moe.TopK)-1, -1)- topKIdx := mlx.Slice(part, []int32{0, 0, 0}, []int32{B, L, moe.TopK})- topKVal := mlx.TakeAlongAxis(logits, topKIdx, -1)- weights := mlx.Softmax(topKVal, -1)-- xFlat := mlx.Reshape(x, B*L, 1, 1, moe.HiddenSize)- idxFlat := mlx.Reshape(topKIdx, B*L, moe.TopK)-- doSort := B*L >= 64- var invOrder *mlx.Array- sorted := false- n := B * L * moe.TopK-- if doSort {- idxAll := mlx.Flatten(idxFlat)- order := mlx.Argsort(idxAll, 0)- invOrder = mlx.Argsort(order, 0)- xFlat = mlx.ExpandDims(mlx.Take(mlx.Squeeze(xFlat, 1), mlx.FloorDivideScalar(order, moe.TopK), 0), 1)- idxFlat = mlx.Reshape(mlx.Take(idxAll, order, 0), n, 1)- sorted = true- }-- gate := mlx.GatherQMM(xFlat, moe.GateBlocks, moe.GateScales, nil, nil, idxFlat, true, moe.GroupSize, moe.Bits, "mxfp4", sorted)- up := mlx.GatherQMM(xFlat, moe.UpBlocks, moe.UpScales, nil, nil, idxFlat, true, moe.GroupSize, moe.Bits, "mxfp4", sorted)-- if moe.GateBias != nil {- gate = mlx.Add(gate, mlx.ExpandDims(mlx.Take(moe.GateBias, idxFlat, 0), 2))- }- if moe.UpBias != nil {- up = mlx.Add(up, mlx.ExpandDims(mlx.Take(moe.UpBias, idxFlat, 0), 2))- }-- hidden := getCompiledSwiGLU().Call(gate, up)[0]-- down := mlx.GatherQMM(hidden, moe.DownBlocks, moe.DownScales, nil, nil, idxFlat, true, moe.GroupSize, moe.Bits, "mxfp4", sorted)- if moe.DownBias != nil {- down = mlx.Add(down, mlx.ExpandDims(mlx.Take(moe.DownBias, idxFlat, 0), 2))- }-- if doSort {- down = mlx.Reshape(mlx.Take(mlx.Squeeze(mlx.Squeeze(down, 2), 1), invOrder, 0), B*L, moe.TopK, moe.HiddenSize)- } else {- down = mlx.Squeeze(down, 2)- }-- ewFlat := mlx.Reshape(weights, B*L, moe.TopK, 1)- return mlx.Reshape(mlx.Sum(mlx.Mul(down, ewFlat), 1, false), B, L, moe.HiddenSize)-}--type Block struct {- Attention *Attention- MLP *MoE- InputNorm *nn.RMSNorm `weight:"input_layernorm"`- PostAttnNorm *nn.RMSNorm `weight:"post_attention_layernorm"`- LayerType string // "sliding_attention" or "full_attention"-}--func (b *Block) Forward(x *mlx.Array, c cache.Cache, B, L int32, mask *mlx.Array, maskMode string, cfg *Config) *mlx.Array {- h := mlx.Add(x, b.Attention.Forward(b.InputNorm.Forward(x, cfg.RMSNormEps), c, B, L, mask, maskMode, cfg))- return mlx.Add(h, b.MLP.Forward(b.PostAttnNorm.Forward(h, cfg.RMSNormEps), B, L))-}--type Model struct {- EmbedTokens *nn.Embedding `weight:"model.embed_tokens"`- Layers []*Block `weight:"-"` // loaded manually due to MoE sanitization- Norm *nn.RMSNorm `weight:"model.norm"`- LMHead *nn.Linear `weight:"lm_head"`-- tok *tokenizer.Tokenizer- *Config-}--func (m *Model) Tokenizer() *tokenizer.Tokenizer { return m.tok }-func (m *Model) NumLayers() int { return len(m.Layers) }-func (m *Model) VocabSize() int32 { return m.Config.VocabSize }--func (m *Model) NewCache(int32) []cache.Cache {- caches := make([]cache.Cache, len(m.Layers))- for i, layer := range m.Layers {- if layer.LayerType == "sliding_attention" && m.SlidingWindow > 0 {- caches[i] = cache.NewRotatingKVCache(int(m.SlidingWindow))- } else {- caches[i] = cache.NewKVCache()- }- }- return caches-}--func (m *Model) Forward(tokens *mlx.Array, caches []cache.Cache) *mlx.Array {- B, L := tokens.Shape()[0], tokens.Shape()[1]- x := m.EmbedTokens.Forward(tokens)-- // Find representative cache indices for sliding window attention- var swaIdx int = -1- for i, layer := range m.Layers {- if layer.LayerType == "sliding_attention" {- swaIdx = i- break- }- }-- // Create masks once at model level- var fullMask, swaMask *mlx.Array- var fullMaskMode, swaMaskMode string-- if L > 1 {- fullMaskMode = "causal"- if swaIdx >= 0 && m.SlidingWindow > 0 && caches != nil {- c := caches[swaIdx]- offset := c.Offset()- windowSize := int(m.SlidingWindow)- cacheLen := min(int(L), windowSize)- if offset > 0 {- cacheLen = min(c.Len()+int(L), windowSize)- }- if int(L) > windowSize {- swaMask = CreateSlidingWindowMask(int(L), offset, offset+int(L)-cacheLen, cacheLen, windowSize)- } else {- swaMaskMode = "causal"- }- } else {- swaMaskMode = "causal"- }- }-- for i, layer := range m.Layers {- var c cache.Cache- if caches != nil {- c = caches[i]- }- mask, maskMode := fullMask, fullMaskMode- if layer.LayerType == "sliding_attention" {- mask, maskMode = swaMask, swaMaskMode- }- x = layer.Forward(x, c, B, L, mask, maskMode, m.Config)- }-- return m.LMHead.Forward(m.Norm.Forward(x, m.RMSNormEps))-}--// sanitizeExpertWeights splits merged gate_up weights into separate gate/up arrays.-// MXFP4 quantized weights require contiguous memory - strided views give wrong results.-func sanitizeExpertWeights(weights *safetensors.ModelWeights, prefix string) (moe *MoE) {- gateUpBlocks, _ := weights.GetTensor(prefix + ".mlp.experts.gate_up_proj_blocks")- gateUpScales, _ := weights.GetTensor(prefix + ".mlp.experts.gate_up_proj_scales")- gateUpBias, _ := weights.GetTensor(prefix + ".mlp.experts.gate_up_proj_bias")- downBlocks, _ := weights.GetTensor(prefix + ".mlp.experts.down_proj_blocks")- downScales, _ := weights.GetTensor(prefix + ".mlp.experts.down_proj_scales")- downBias, _ := weights.GetTensor(prefix + ".mlp.experts.down_proj_bias")-- moe = &MoE{GroupSize: 32, Bits: 4, DownScales: downScales, DownBias: downBias}-- if gateUpBlocks != nil {- gub := mlx.FlattenRange(mlx.View(gateUpBlocks, int(mlx.DtypeUint32)), -2, -1)- s := gub.Shape()- moe.GateBlocks = mlx.Contiguous(mlx.SliceStride(gub, []int32{0, 0, 0}, []int32{s[0], s[1], s[2]}, []int32{1, 2, 1}))- moe.UpBlocks = mlx.Contiguous(mlx.SliceStride(gub, []int32{0, 1, 0}, []int32{s[0], s[1], s[2]}, []int32{1, 2, 1}))- }- if gateUpScales != nil {- s := gateUpScales.Shape()- moe.GateScales = mlx.Contiguous(mlx.SliceStride(gateUpScales, []int32{0, 0, 0}, []int32{s[0], s[1], s[2]}, []int32{1, 2, 1}))- moe.UpScales = mlx.Contiguous(mlx.SliceStride(gateUpScales, []int32{0, 1, 0}, []int32{s[0], s[1], s[2]}, []int32{1, 2, 1}))- }- if gateUpBias != nil {- s := gateUpBias.Shape()- moe.GateBias = mlx.Contiguous(mlx.SliceStride(gateUpBias, []int32{0, 0}, []int32{s[0], s[1]}, []int32{1, 2}))- moe.UpBias = mlx.Contiguous(mlx.SliceStride(gateUpBias, []int32{0, 1}, []int32{s[0], s[1]}, []int32{1, 2}))- }- if downBlocks != nil {- moe.DownBlocks = mlx.FlattenRange(mlx.View(downBlocks, int(mlx.DtypeUint32)), -2, -1)- }- return moe-}--func Load(modelPath string) (*Model, error) {- data, err := os.ReadFile(filepath.Join(modelPath, "config.json"))- if err != nil {- return nil, fmt.Errorf("load config: %w", err)- }- var cfg Config- if err := json.Unmarshal(data, &cfg); err != nil {- return nil, fmt.Errorf("parse config: %w", err)- }- cfg.Scale = float32(1.0 / math.Sqrt(float64(cfg.HeadDim)))-- weights, err := safetensors.LoadModelWeights(modelPath)- if err != nil {- return nil, fmt.Errorf("load weights: %w", err)- }-- tok, err := tokenizer.Load(filepath.Join(modelPath, "tokenizer.json"))- if err != nil {- return nil, fmt.Errorf("load tokenizer: %w", err)- }-- m := &Model{- Layers: make([]*Block, cfg.NumHiddenLayers),- Config: &cfg,- tok: tok,- }-- // Load simple weights via struct tags- if err := safetensors.LoadModule(m, weights, ""); err != nil {- return nil, err- }-- // Load layers with custom MoE handling- for i := int32(0); i < cfg.NumHiddenLayers; i++ {- prefix := fmt.Sprintf("model.layers.%d", i)- layer := &Block{}- if err := safetensors.LoadModule(layer, weights, prefix); err != nil {- return nil, fmt.Errorf("layer %d: %w", i, err)- }-- // Initialize attention YaRN- layer.Attention.initYarn(&cfg)-- // Load MoE with weight sanitization- moe := sanitizeExpertWeights(weights, prefix)- moe.Router = layer.MLP.Router // Router was loaded by LoadModule- moe.TopK = cfg.NumExpertsPerTok- moe.HiddenSize = cfg.HiddenSize- layer.MLP = moe-- // Set layer type- layer.LayerType = "full_attention"- if int(i) < len(cfg.LayerTypes) {- layer.LayerType = cfg.LayerTypes[i]- }-- m.Layers[i] = layer- }-- // Release safetensors BEFORE eval - lazy arrays have captured data,- // this reduces peak memory by freeing mmap during materialization- weights.ReleaseAll()- mlx.Eval(mlx.Collect(m)...)-- return m, nil-}--func (m *Model) MaxContextLength() int32 {- if m.RopeScaling != nil && m.RopeScaling.OriginalMaxPositionEmbeddings > 0 {- return m.RopeScaling.OriginalMaxPositionEmbeddings- }- return 131072-}diff --git a/x/imagegen/models/llama/llama.go b/x/imagegen/models/llama/llama.godeleted file mode 100644index 2b695f78e..000000000--- a/x/imagegen/models/llama/llama.go+++ /dev/null@@ -1,152 +0,0 @@-//go:build mlx--package llama--import (- "encoding/json"- "fmt"- "math"- "os"- "path/filepath"-- "github.com/ollama/ollama/x/imagegen/cache"- "github.com/ollama/ollama/x/imagegen/mlx"- "github.com/ollama/ollama/x/imagegen/nn"- "github.com/ollama/ollama/x/imagegen/safetensors"- "github.com/ollama/ollama/x/imagegen/tokenizer"-)--type Config struct {- HiddenSize int32 `json:"hidden_size"`- NumHiddenLayers int32 `json:"num_hidden_layers"`- IntermediateSize int32 `json:"intermediate_size"`- NumAttentionHeads int32 `json:"num_attention_heads"`- NumKeyValueHeads int32 `json:"num_key_value_heads"`- VocabSize int32 `json:"vocab_size"`- RMSNormEps float32 `json:"rms_norm_eps"`- RopeTheta float32 `json:"rope_theta"`- MaxPositionEmbeddings int32 `json:"max_position_embeddings"`- HeadDim int32 `json:"-"`- Scale float32 `json:"-"`-}--type Model struct {- EmbedTokens *nn.Embedding `weight:"model.embed_tokens"`- Layers []*Layer `weight:"model.layers"`- Norm *nn.RMSNorm `weight:"model.norm"`- Output *nn.Linear `weight:"lm_head,optional"`-- tok *tokenizer.Tokenizer- *Config-}--type Layer struct {- Attention *Attention- MLP *MLP- AttentionNorm *nn.RMSNorm `weight:"input_layernorm"`- MLPNorm *nn.RMSNorm `weight:"post_attention_layernorm"`-}--type Attention struct {- QProj *nn.Linear `weight:"self_attn.q_proj"`- KProj *nn.Linear `weight:"self_attn.k_proj"`- VProj *nn.Linear `weight:"self_attn.v_proj"`- OProj *nn.Linear `weight:"self_attn.o_proj"`-}--type MLP struct {- GateProj *nn.Linear `weight:"mlp.gate_proj"`- UpProj *nn.Linear `weight:"mlp.up_proj"`- DownProj *nn.Linear `weight:"mlp.down_proj"`-}--func Load(modelPath string) (*Model, error) {- data, err := os.ReadFile(filepath.Join(modelPath, "config.json"))- if err != nil {- return nil, fmt.Errorf("load config: %w", err)- }- var cfg Config- if err := json.Unmarshal(data, &cfg); err != nil {- return nil, fmt.Errorf("parse config: %w", err)- }- cfg.HeadDim = cfg.HiddenSize / cfg.NumAttentionHeads- cfg.Scale = float32(1.0 / math.Sqrt(float64(cfg.HeadDim)))-- weights, err := safetensors.LoadModelWeights(modelPath)- if err != nil {- return nil, fmt.Errorf("load weights: %w", err)- }-- tok, err := tokenizer.Load(filepath.Join(modelPath, "tokenizer.json"))- if err != nil {- return nil, fmt.Errorf("load tokenizer: %w", err)- }-- m := &Model{- Layers: make([]*Layer, cfg.NumHiddenLayers),- Config: &cfg,- tok: tok,- }- if err := safetensors.LoadModule(m, weights, ""); err != nil {- return nil, err- }- m.Output = nn.NewLinear(m.EmbedTokens.Weight, nil)-- mlx.Eval(mlx.Collect(m)...)- weights.ReleaseAll()-- return m, nil-}--func (m *Model) Forward(tokens *mlx.Array, caches []cache.Cache) *mlx.Array {- B, L := tokens.Shape()[0], tokens.Shape()[1]- h := m.EmbedTokens.Forward(tokens)- for i, layer := range m.Layers {- h = layer.Forward(h, caches[i], B, L, m.Config)- }- return m.Output.Forward(m.Norm.Forward(h, m.RMSNormEps))-}--func (l *Layer) Forward(x *mlx.Array, c cache.Cache, B, L int32, cfg *Config) *mlx.Array {- h := mlx.Add(x, l.Attention.Forward(l.AttentionNorm.Forward(x, cfg.RMSNormEps), c, B, L, cfg))- return mlx.Add(h, l.MLP.Forward(l.MLPNorm.Forward(h, cfg.RMSNormEps)))-}--func (a *Attention) Forward(x *mlx.Array, c cache.Cache, B, L int32, cfg *Config) *mlx.Array {- q := a.QProj.Forward(x)- k := a.KProj.Forward(x)- v := a.VProj.Forward(x)-- q = mlx.AsStrided(q, []int32{B, cfg.NumAttentionHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumAttentionHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumAttentionHeads * cfg.HeadDim), 1}, 0)- k = mlx.AsStrided(k, []int32{B, cfg.NumKeyValueHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumKeyValueHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumKeyValueHeads * cfg.HeadDim), 1}, 0)- v = mlx.AsStrided(v, []int32{B, cfg.NumKeyValueHeads, L, cfg.HeadDim},- []int64{int64(L * cfg.NumKeyValueHeads * cfg.HeadDim), int64(cfg.HeadDim), int64(cfg.NumKeyValueHeads * cfg.HeadDim), 1}, 0)-- q = mlx.RoPE(q, int(cfg.HeadDim), false, cfg.RopeTheta, 1.0, c.Offset())- k = mlx.RoPE(k, int(cfg.HeadDim), false, cfg.RopeTheta, 1.0, c.Offset())-- k, v = c.Update(k, v, int(L))- out := mlx.ScaledDotProductAttention(q, k, v, cfg.Scale, L > 1)- out = mlx.Reshape(mlx.Transpose(out, 0, 2, 1, 3), B, L, cfg.NumAttentionHeads*cfg.HeadDim)- return a.OProj.Forward(out)-}--func (m *MLP) Forward(x *mlx.Array) *mlx.Array {- return m.DownProj.Forward(mlx.Mul(mlx.SiLU(m.GateProj.Forward(x)), m.UpProj.Forward(x)))-}--// Interface methods-func (m *Model) NumLayers() int { return len(m.Layers) }-func (m *Model) MaxContextLength() int32 { return m.MaxPositionEmbeddings }-func (m *Model) VocabSize() int32 { return m.Config.VocabSize }-func (m *Model) Tokenizer() *tokenizer.Tokenizer { return m.tok }--func (m *Model) NewCache(maxSeqLen int32) []cache.Cache {- caches := make([]cache.Cache, len(m.Layers))- for i := range caches {- caches[i] = cache.NewKVCache()- }- return caches-}diff --git a/x/imagegen/runner.go b/x/imagegen/runner.goindex e24383ad5..0409c4bf7 100644--- a/x/imagegen/runner.go+++ b/x/imagegen/runner.go@@ -39,19 +39,23 @@ func Execute(args []string) error {return fmt.Errorf("--port is required")}- // Initialize MLX+ // Detect model type from capabilities+ mode := detectModelMode(*modelName)+ slog.Info("starting mlx runner", "model", *modelName, "port", *port, "mode", mode)++ if mode != ModeImageGen {+ return fmt.Errorf("imagegen runner only supports image generation models")+ }++ // Initialize MLX only for image generation mode.if err := mlx.InitMLX(); err != nil {slog.Error("unable to initialize MLX", "error", err)return err}slog.Info("MLX library initialized")- // Detect model type from capabilities- mode := detectModelMode(*modelName)- slog.Info("starting mlx runner", "model", *modelName, "port", *port, "mode", mode)-// Create and start server- server, err := newServer(*modelName, *port, mode)+ server, err := newServer(*modelName, *port)if err != nil {return fmt.Errorf("failed to create server: %w", err)}@@ -61,12 +65,6 @@ func Execute(args []string) error {mux.HandleFunc("/health", server.healthHandler)mux.HandleFunc("/completion", server.completionHandler)- // LLM-specific endpoints- if mode == ModeLLM {- mux.HandleFunc("/tokenize", server.tokenizeHandler)- mux.HandleFunc("/embedding", server.embeddingHandler)- }-httpServer := &http.Server{Addr: fmt.Sprintf("127.0.0.1:%d", *port),Handler: mux,@@ -112,34 +110,22 @@ func detectModelMode(modelName string) ModelMode {// server holds the model and handles HTTP requests.type server struct {- mode ModelModemodelName stringport int- // Image generation model (when mode == ModeImageGen)+ // Image generation model.imageModel ImageModel-- // LLM model (when mode == ModeLLM)- llmModel *llmState}-// newServer creates a new server instance and loads the appropriate model.-func newServer(modelName string, port int, mode ModelMode) (*server, error) {+// newServer creates a new server instance for image generation models.+func newServer(modelName string, port int) (*server, error) {s := &server{- mode: mode,modelName: modelName,port: port,}- switch mode {- case ModeImageGen:- if err := s.loadImageModel(); err != nil {- return nil, fmt.Errorf("failed to load image model: %w", err)- }- case ModeLLM:- if err := s.loadLLMModel(); err != nil {- return nil, fmt.Errorf("failed to load LLM model: %w", err)- }+ if err := s.loadImageModel(); err != nil {+ return nil, fmt.Errorf("failed to load image model: %w", err)}return s, nil@@ -163,41 +149,5 @@ func (s *server) completionHandler(w http.ResponseWriter, r *http.Request) {return}- switch s.mode {- case ModeImageGen:- s.handleImageCompletion(w, r, req)- case ModeLLM:- s.handleLLMCompletion(w, r, req)- }-}--func (s *server) tokenizeHandler(w http.ResponseWriter, r *http.Request) {- if s.llmModel == nil {- http.Error(w, "LLM model not loaded", http.StatusInternalServerError)- return- }-- var req struct {- Content string `json:"content"`- }- if err := json.NewDecoder(r.Body).Decode(&req); err != nil {- http.Error(w, err.Error(), http.StatusBadRequest)- return- }-- tok := s.llmModel.model.Tokenizer()- tokens := tok.Encode(req.Content, false)-- // Convert int32 to int for JSON response- intTokens := make([]int, len(tokens))- for i, t := range tokens {- intTokens[i] = int(t)- }-- w.Header().Set("Content-Type", "application/json")- json.NewEncoder(w).Encode(map[string][]int{"tokens": intTokens})-}--func (s *server) embeddingHandler(w http.ResponseWriter, r *http.Request) {- http.Error(w, "embeddings not yet implemented for MLX models", http.StatusNotImplemented)+ s.handleImageCompletion(w, r, req)}diff --git a/x/imagegen/server.go b/x/imagegen/server.goindex 2beb66bda..102cb0c55 100644--- a/x/imagegen/server.go+++ b/x/imagegen/server.go@@ -30,13 +30,12 @@ import (// Server wraps an MLX runner subprocess to implement llm.LlamaServer.//// This implementation is compatible with Ollama's scheduler and can be loaded/unloaded-// like any other model. It supports both LLM (safetensors) and image generation models.+// like any other model. It is used for image generation models.type Server struct {mu sync.Mutexcmd *exec.Cmdport intmodelName string- mode ModelModevramSize uint64done chan errorclient *http.Client@@ -45,7 +44,7 @@ type Server struct {}// NewServer spawns a new MLX runner subprocess and waits until it's ready.-func NewServer(modelName string, mode ModelMode) (*Server, error) {+func NewServer(modelName string) (*Server, error) {// Validate platform support before attempting to startif err := CheckPlatformSupport(); err != nil {return nil, err@@ -119,7 +118,6 @@ func NewServer(modelName string, mode ModelMode) (*Server, error) {cmd: cmd,port: port,modelName: modelName,- mode: mode,vramSize: vramSize,done: make(chan error, 1),client: &http.Client{Timeout: 10 * time.Minute},@@ -145,7 +143,7 @@ func NewServer(modelName string, mode ModelMode) (*Server, error) {}}()- slog.Info("starting mlx runner subprocess", "exe", exe, "model", modelName, "port", port, "mode", mode)+ slog.Info("starting mlx runner subprocess", "exe", exe, "model", modelName, "port", port)if err := cmd.Start(); err != nil {return nil, fmt.Errorf("failed to start mlx runner: %w", err)}@@ -396,36 +394,7 @@ func (s *Server) Embedding(ctx context.Context, input string) ([]float32, int, e// Tokenize tokenizes the input content.func (s *Server) Tokenize(ctx context.Context, content string) ([]int, error) {- body, err := json.Marshal(map[string]string{"content": content})- if err != nil {- return nil, err- }-- url := fmt.Sprintf("http://127.0.0.1:%d/tokenize", s.port)- req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(body))- if err != nil {- return nil, err- }- req.Header.Set("Content-Type", "application/json")-- resp, err := s.client.Do(req)- if err != nil {- return nil, err- }- defer resp.Body.Close()-- if resp.StatusCode != http.StatusOK {- return nil, fmt.Errorf("tokenize failed: %d", resp.StatusCode)- }-- var result struct {- Tokens []int `json:"tokens"`- }- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {- return nil, err- }-- return result.Tokens, nil+ return nil, errors.New("tokenization not supported for image generation models")}// Detokenize converts tokens back to text.