Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlxrunner: reject media requests the model cannot serve
Jesse Grossaf5b62767293
diff --git a/x/mlxrunner/client.go b/x/mlxrunner/client.goindex 07fd49092..d09873aa0 100644--- a/x/mlxrunner/client.go+++ b/x/mlxrunner/client.go@@ -112,6 +112,7 @@ func (c *Client) WaitUntilRunning(ctx context.Context) error {type CompletionRequest struct {Prompt string+ Media []llm.MediaDataOptions api.OptionsLogprobs boolTopLogprobs int@@ -155,6 +156,7 @@ func (c *Client) Close() error {func (c *Client) Completion(ctx context.Context, req llm.CompletionRequest, fn func(llm.CompletionResponse)) error {creq := CompletionRequest{Prompt: req.Prompt,+ Media: req.Media,Logprobs: req.Logprobs,TopLogprobs: req.TopLogprobs,}diff --git a/x/mlxrunner/model/base/media.go b/x/mlxrunner/model/base/media.gonew file mode 100644index 000000000..513add66f--- /dev/null+++ b/x/mlxrunner/model/base/media.go@@ -0,0 +1,77 @@+package base++import (+ // Every model's PrepareMedia decodes through image.Decode; the decoder+ // set is registered once here so all models accept the same formats.+ _ "image/gif"+ _ "image/jpeg"+ _ "image/png"++ _ "golang.org/x/image/webp"++ "github.com/ollama/ollama/x/mlxrunner/mlx"+)++// Segment is one run of the prompt in stream order: either a tokenized text+// run (Tokens set) or a single media item (Kind and Data set).+type Segment struct {+ Tokens []int32+ Kind string+ Data []byte+}++// PreparedItem describes one media occurrence in the prepared stream. A+// model chooses item granularity: one per media segment, or several when+// parts encode and evaluate independently (e.g. per tile).+type PreparedItem struct {+ // Range is the expansion's token range [start, end) in Tokens;+ // non-empty, since cache identity enters through these positions.+ Range [2]int++ // Source is the index of the segment this item was prepared from; the+ // item's prefix-cache identity is keyed on that segment's bytes.+ Source int++ // MediaData is the preprocessed encoder input with shape Dims. Dims+ // enters the cache keys too: geometry changes features under+ // identical bytes.+ MediaData []float32+ Dims []int++ // Opaque carries model-private preprocessing state to EncodeMedia+ // and Forward.+ Opaque any++ // Causal marks an expansion whose tokens attend causally, so chunks+ // may split it. Unset, the first evaluation covers the whole+ // expansion in one forward, as bidirectional runs require.+ Causal bool+}++// PreparedRequest is the expanded input stream, every media segment's+// expansion spliced in place, with the items in stream order.+type PreparedRequest struct {+ Tokens []int32+ Items []PreparedItem++ // Layout is an opaque request-scoped value computed in the one pass+ // that sees every splice position; immutable, carried unread by the+ // runner to every forward. Delivered only when Items is non-empty.+ // Nil when the model derives nothing from it.+ Layout any+}++// MediaModel is implemented by models that accept media inputs.+type MediaModel interface {+ // PrepareMedia runs once per request on the request goroutine, CPU+ // only, and returns the expanded stream. It must be deterministic for+ // given segments: prefix-cache restores splice cached state with+ // recomputed state.+ PrepareMedia(segments []Segment) (*PreparedRequest, error)++ // EncodeMedia builds one item's lazy feature graph on the MLX thread;+ // it must not evaluate — the consuming forward's evaluation pulls it.+ // Read the pixels from data: the runner frees the item's MediaData+ // once its expansion is evaluated.+ EncodeMedia(item *PreparedItem, data *mlx.Array) *mlx.Array+}diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.goindex 4373b2c38..1a3df8e95 100644--- a/x/mlxrunner/pipeline.go+++ b/x/mlxrunner/pipeline.go@@ -14,6 +14,7 @@ import ("github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/cache""github.com/ollama/ollama/x/mlxrunner/mlx"+ "github.com/ollama/ollama/x/mlxrunner/model/base"sampler "github.com/ollama/ollama/x/mlxrunner/sample""github.com/ollama/ollama/x/tokenizer")@@ -30,6 +31,16 @@ func (r *Runner) Prepare(request *Request) error {return errors.New("model not loaded")}+ if len(request.Media) > 0 {+ if _, ok := r.Model.(base.MediaModel); !ok {+ kind := string(request.Media[0].Kind)+ if kind == "" {+ kind = "media"+ }+ return fmt.Errorf("this model does not support %s input", kind)+ }+ }+tokens := r.Tokenizer.Encode(request.Prompt, r.Tokenizer.AddBOS())if len(tokens) == 0 {return errors.New("empty prompt")diff --git a/x/mlxrunner/pipeline_test.go b/x/mlxrunner/pipeline_test.gonew file mode 100644index 000000000..f27568db7--- /dev/null+++ b/x/mlxrunner/pipeline_test.go@@ -0,0 +1,42 @@+package mlxrunner++import (+ "strings"+ "testing"++ "github.com/ollama/ollama/llm"+ "github.com/ollama/ollama/x/mlxrunner/batch"+ "github.com/ollama/ollama/x/mlxrunner/cache"+ "github.com/ollama/ollama/x/mlxrunner/mlx"+ "github.com/ollama/ollama/x/tokenizer"+)++// textOnlyModel satisfies base.Model but not base.MediaModel.+type textOnlyModel struct{}++func (textOnlyModel) LoadWeights(map[string]*mlx.Array) error { return nil }+func (textOnlyModel) NewCaches() []cache.Cache { return nil }+func (textOnlyModel) Forward(*batch.Batch, []cache.Cache) (*mlx.Array, *mlx.Array) {+ return nil, nil+}+func (textOnlyModel) Unembed(x *mlx.Array) *mlx.Array { return x }+func (textOnlyModel) Tokenizer() *tokenizer.Tokenizer { return nil }+func (textOnlyModel) MaxContextLength() int { return 0 }++func TestPrepareRejectsMediaWithoutSupport(t *testing.T) {+ r := &Runner{Model: textOnlyModel{}}+ req := &Request{+ CompletionRequest: CompletionRequest{+ Prompt: "[img-0] what is this?",+ Media: []llm.MediaData{{ID: 0, Kind: llm.MediaKindImage, Data: []byte{1}}},+ },+ }++ err := r.Prepare(req)+ if err == nil {+ t.Fatal("expected error for media on a text-only model")+ }+ if !strings.Contains(err.Error(), "does not support image input") {+ t.Fatalf("unexpected error: %v", err)+ }+}