Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

mlxrunner: reject media requests the model cannot serve

Jesse Grossaf5b62767293
diff --git a/x/mlxrunner/client.go b/x/mlxrunner/client.go
index 07fd49092..d09873aa0 100644
--- a/x/mlxrunner/client.go
+++ b/x/mlxrunner/client.go
@@ -112,6 +112,7 @@ func (c *Client) WaitUntilRunning(ctx context.Context) error {
type CompletionRequest struct {
Prompt string
+ Media []llm.MediaData
Options api.Options
Logprobs bool
TopLogprobs int
@@ -155,6 +156,7 @@ func (c *Client) Close() error {
func (c *Client) Completion(ctx context.Context, req llm.CompletionRequest, fn func(llm.CompletionResponse)) error {
creq := CompletionRequest{
Prompt: req.Prompt,
+ Media: req.Media,
Logprobs: req.Logprobs,
TopLogprobs: req.TopLogprobs,
}
diff --git a/x/mlxrunner/model/base/media.go b/x/mlxrunner/model/base/media.go
new file mode 100644
index 000000000..513add66f
--- /dev/null
+++ b/x/mlxrunner/model/base/media.go
@@ -0,0 +1,77 @@
+package base
+
+import (
+ // Every model's PrepareMedia decodes through image.Decode; the decoder
+ // set is registered once here so all models accept the same formats.
+ _ "image/gif"
+ _ "image/jpeg"
+ _ "image/png"
+
+ _ "golang.org/x/image/webp"
+
+ "github.com/ollama/ollama/x/mlxrunner/mlx"
+)
+
+// Segment is one run of the prompt in stream order: either a tokenized text
+// run (Tokens set) or a single media item (Kind and Data set).
+type Segment struct {
+ Tokens []int32
+ Kind string
+ Data []byte
+}
+
+// PreparedItem describes one media occurrence in the prepared stream. A
+// model chooses item granularity: one per media segment, or several when
+// parts encode and evaluate independently (e.g. per tile).
+type PreparedItem struct {
+ // Range is the expansion's token range [start, end) in Tokens;
+ // non-empty, since cache identity enters through these positions.
+ Range [2]int
+
+ // Source is the index of the segment this item was prepared from; the
+ // item's prefix-cache identity is keyed on that segment's bytes.
+ Source int
+
+ // MediaData is the preprocessed encoder input with shape Dims. Dims
+ // enters the cache keys too: geometry changes features under
+ // identical bytes.
+ MediaData []float32
+ Dims []int
+
+ // Opaque carries model-private preprocessing state to EncodeMedia
+ // and Forward.
+ Opaque any
+
+ // Causal marks an expansion whose tokens attend causally, so chunks
+ // may split it. Unset, the first evaluation covers the whole
+ // expansion in one forward, as bidirectional runs require.
+ Causal bool
+}
+
+// PreparedRequest is the expanded input stream, every media segment's
+// expansion spliced in place, with the items in stream order.
+type PreparedRequest struct {
+ Tokens []int32
+ Items []PreparedItem
+
+ // Layout is an opaque request-scoped value computed in the one pass
+ // that sees every splice position; immutable, carried unread by the
+ // runner to every forward. Delivered only when Items is non-empty.
+ // Nil when the model derives nothing from it.
+ Layout any
+}
+
+// MediaModel is implemented by models that accept media inputs.
+type MediaModel interface {
+ // PrepareMedia runs once per request on the request goroutine, CPU
+ // only, and returns the expanded stream. It must be deterministic for
+ // given segments: prefix-cache restores splice cached state with
+ // recomputed state.
+ PrepareMedia(segments []Segment) (*PreparedRequest, error)
+
+ // EncodeMedia builds one item's lazy feature graph on the MLX thread;
+ // it must not evaluate — the consuming forward's evaluation pulls it.
+ // Read the pixels from data: the runner frees the item's MediaData
+ // once its expansion is evaluated.
+ EncodeMedia(item *PreparedItem, data *mlx.Array) *mlx.Array
+}
diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.go
index 4373b2c38..1a3df8e95 100644
--- a/x/mlxrunner/pipeline.go
+++ b/x/mlxrunner/pipeline.go
@@ -14,6 +14,7 @@ import (
"github.com/ollama/ollama/x/mlxrunner/batch"
"github.com/ollama/ollama/x/mlxrunner/cache"
"github.com/ollama/ollama/x/mlxrunner/mlx"
+ "github.com/ollama/ollama/x/mlxrunner/model/base"
sampler "github.com/ollama/ollama/x/mlxrunner/sample"
"github.com/ollama/ollama/x/tokenizer"
)
@@ -30,6 +31,16 @@ func (r *Runner) Prepare(request *Request) error {
return errors.New("model not loaded")
}
+ if len(request.Media) > 0 {
+ if _, ok := r.Model.(base.MediaModel); !ok {
+ kind := string(request.Media[0].Kind)
+ if kind == "" {
+ kind = "media"
+ }
+ return fmt.Errorf("this model does not support %s input", kind)
+ }
+ }
+
tokens := r.Tokenizer.Encode(request.Prompt, r.Tokenizer.AddBOS())
if len(tokens) == 0 {
return errors.New("empty prompt")
diff --git a/x/mlxrunner/pipeline_test.go b/x/mlxrunner/pipeline_test.go
new file mode 100644
index 000000000..f27568db7
--- /dev/null
+++ b/x/mlxrunner/pipeline_test.go
@@ -0,0 +1,42 @@
+package mlxrunner
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/ollama/ollama/llm"
+ "github.com/ollama/ollama/x/mlxrunner/batch"
+ "github.com/ollama/ollama/x/mlxrunner/cache"
+ "github.com/ollama/ollama/x/mlxrunner/mlx"
+ "github.com/ollama/ollama/x/tokenizer"
+)
+
+// textOnlyModel satisfies base.Model but not base.MediaModel.
+type textOnlyModel struct{}
+
+func (textOnlyModel) LoadWeights(map[string]*mlx.Array) error { return nil }
+func (textOnlyModel) NewCaches() []cache.Cache { return nil }
+func (textOnlyModel) Forward(*batch.Batch, []cache.Cache) (*mlx.Array, *mlx.Array) {
+ return nil, nil
+}
+func (textOnlyModel) Unembed(x *mlx.Array) *mlx.Array { return x }
+func (textOnlyModel) Tokenizer() *tokenizer.Tokenizer { return nil }
+func (textOnlyModel) MaxContextLength() int { return 0 }
+
+func TestPrepareRejectsMediaWithoutSupport(t *testing.T) {
+ r := &Runner{Model: textOnlyModel{}}
+ req := &Request{
+ CompletionRequest: CompletionRequest{
+ Prompt: "[img-0] what is this?",
+ Media: []llm.MediaData{{ID: 0, Kind: llm.MediaKindImage, Data: []byte{1}}},
+ },
+ }
+
+ err := r.Prepare(req)
+ if err == nil {
+ t.Fatal("expected error for media on a text-only model")
+ }
+ if !strings.Contains(err.Error(), "does not support image input") {
+ t.Fatalf("unexpected error: %v", err)
+ }
+}