Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlx: return exact types and evaluate arrays in the value readers
Jesse Gross7623501fc2c9
diff --git a/x/mlxrunner/dflash_test.go b/x/mlxrunner/dflash_test.goindex 89aa11814..f88d38cd7 100644--- a/x/mlxrunner/dflash_test.go+++ b/x/mlxrunner/dflash_test.go@@ -61,11 +61,7 @@ func (d *fakeBlockDraft) Forward(b *batch.Batch, _, draftCaches []cache.Cache) (}if b.InputIDs != nil {mlx.Eval(b.InputIDs)- ids := b.InputIDs.Ints()- call.block = make([]int32, len(ids))- for i, id := range ids {- call.block[i] = int32(id)- }+ call.block = b.InputIDs.Ints()}d.calls = append(d.calls, call)@@ -212,7 +208,7 @@ func TestDFlashProposeBounds(t *testing.T) {t.Fatalf("propose declined with context and budget")}mlx.Eval(cand.tokens)- if got := cand.tokens.Ints(); !slices.Equal(got, []int{2, 3, 4}) {+ if got := cand.tokens.Ints(); !slices.Equal(got, []int32{2, 3, 4}) {t.Fatalf("draft tokens = %v, want [2 3 4]", got)}if got, want := draft.calls[0].block, []int32{1, 6, 6, 6}; !slices.Equal(got, want) {diff --git a/x/mlxrunner/mlx/array.go b/x/mlxrunner/mlx/array.goindex 5271b619f..f87d147d5 100644--- a/x/mlxrunner/mlx/array.go+++ b/x/mlxrunner/mlx/array.go@@ -236,52 +236,31 @@ func (t *Array) DType() DType {// data utilities-func (t *Array) Int() int {- switch dt := t.DType(); dt {- case DTypeInt32:- var item C.int32_t- C.mlx_array_item_int32(&item, t.ctx)- return int(item)- case DTypeUint32:- var item C.uint32_t- C.mlx_array_item_uint32(&item, t.ctx)- return int(item)- case DTypeInt64:- var item C.int64_t- C.mlx_array_item_int64(&item, t.ctx)- return int(item)- case DTypeUint64:- var item C.uint64_t- C.mlx_array_item_uint64(&item, t.ctx)- return int(item)- default:- panic(fmt.Sprintf("mlx: Int requires an integer array, got %v", dt))+func (t *Array) Int() int32 {+ if dt := t.DType(); dt != DTypeInt32 {+ panic(fmt.Sprintf("mlx: Int requires a DTypeInt32 array, got %v", dt))}+ var item C.int32_t+ C.mlx_array_item_int32(&item, t.ctx)+ return int32(item)}-func (t *Array) Float() float64 {- switch dt := t.DType(); dt {- case DTypeFloat32:- var item C.float- C.mlx_array_item_float32(&item, t.ctx)- return float64(item)- case DTypeFloat64:- var item C.double- C.mlx_array_item_float64(&item, t.ctx)- return float64(item)- default:- panic(fmt.Sprintf("mlx: Float requires a float32 or float64 array, got %v", dt))+func (t *Array) Float() float32 {+ if dt := t.DType(); dt != DTypeFloat32 {+ panic(fmt.Sprintf("mlx: Float requires a DTypeFloat32 array, got %v", dt))}+ var item C.float+ C.mlx_array_item_float32(&item, t.ctx)+ return float32(item)}-func (t *Array) Ints() []int {+func (t *Array) Ints() []int32 {if dt := t.DType(); dt != DTypeInt32 {panic(fmt.Sprintf("mlx: Ints requires DTypeInt32, got %v", dt))}- ints := make([]int, t.Size())- for i, f := range unsafe.Slice(C.mlx_array_data_int32(t.ctx), len(ints)) {- ints[i] = int(f)- }+ Eval(t)+ ints := make([]int32, t.Size())+ copy(ints, unsafe.Slice((*int32)(unsafe.Pointer(C.mlx_array_data_int32(t.ctx))), len(ints)))return ints}@@ -289,10 +268,9 @@ func (t *Array) Floats() []float32 {if dt := t.DType(); dt != DTypeFloat32 {panic(fmt.Sprintf("mlx: Floats requires DTypeFloat32, got %v", dt))}+ Eval(t)floats := make([]float32, t.Size())- for i, f := range unsafe.Slice(C.mlx_array_data_float32(t.ctx), len(floats)) {- floats[i] = float32(f)- }+ copy(floats, unsafe.Slice((*float32)(unsafe.Pointer(C.mlx_array_data_float32(t.ctx))), len(floats)))return floats}diff --git a/x/mlxrunner/mlx/array_test.go b/x/mlxrunner/mlx/array_test.goindex 927853c0c..7386f3335 100644--- a/x/mlxrunner/mlx/array_test.go+++ b/x/mlxrunner/mlx/array_test.go@@ -58,13 +58,13 @@ func testComparisonOpsAndBernoulli(t *testing.T) {Eval(eq, gt, le, bern)for name, tc := range map[string]struct {- got []int- want []int+ got []int32+ want []int32}{- "equal": {eq.Ints(), []int{1, 0, 0}},- "greater": {gt.Ints(), []int{0, 1, 0}},- "lessEqual": {le.Ints(), []int{1, 0, 1}},- "bernoulli": {bern.Ints(), []int{1, 0}},+ "equal": {eq.Ints(), []int32{1, 0, 0}},+ "greater": {gt.Ints(), []int32{0, 1, 0}},+ "lessEqual": {le.Ints(), []int32{1, 0, 1}},+ "bernoulli": {bern.Ints(), []int32{1, 0}},} {t.Run(name, func(t *testing.T) {if len(tc.got) != len(tc.want) {diff --git a/x/mlxrunner/mtp_test.go b/x/mlxrunner/mtp_test.goindex 80460e3da..276e35e53 100644--- a/x/mlxrunner/mtp_test.go+++ b/x/mlxrunner/mtp_test.go@@ -71,17 +71,13 @@ func (m *fakeMTPModel) Forward(b *batch.Batch, caches []cache.Cache) (hidden, aubreak}if rc, ok := c.(*fakeRewindableCache); ok {- seg := make([]int32, len(ids))- for i, id := range ids {- seg[i] = int32(id)- }- rc.feed(seg)+ rc.feed(ids)}}preds := make([]int32, len(ids))for i, id := range ids {- preds[i] = m.predict[int32(id)]+ preds[i] = m.predict[id]}out := oneHotLogits(preds)return out, out@@ -121,7 +117,7 @@ func (d *fakeMTPDraft) NewCaches() []cache.Cache { return nil }func (d *fakeMTPDraft) Forward(b *batch.Batch, _, _ []cache.Cache) (hidden, auxHidden *mlx.Array) {d.layouts = append(d.layouts, b.Layout)mlx.Eval(b.InputIDs)- prev := int32(b.InputIDs.Ints()[0])+ prev := b.InputIDs.Ints()[0]d.calls = append(d.calls, draftCall{position: b.SeqOffsets[0], from: prev})return oneHotLogits([]int32{d.predict[prev]}), mlx.Zeros(mlx.DTypeFloat32, 1, 1, mtpTestVocab)}@@ -160,11 +156,7 @@ func (d *fakeKVDraft) NewCaches() []cache.Cache { return d.draftCaches }func (d *fakeKVDraft) Forward(b *batch.Batch, _, draftCaches []cache.Cache) (hidden, auxHidden *mlx.Array) {mlx.Eval(b.InputIDs, b.Hidden)- rawIDs := b.InputIDs.Ints()- ids := make([]int32, len(rawIDs))- for i, id := range rawIDs {- ids[i] = int32(id)- }+ ids := b.InputIDs.Ints()hot := make([]int32, b.Hidden.Dim(1))flat := b.Hidden.Floats()@@ -262,8 +254,8 @@ func collectResponses(ch chan CompletionResponse) (content string, final Complet}// resultIDs reads the token id of each result.-func resultIDs(results []sampler.Result) []int {- ids := make([]int, 0, len(results))+func resultIDs(results []sampler.Result) []int32 {+ ids := make([]int32, 0, len(results))for _, res := range results {ids = append(ids, res.Token.Int())}@@ -295,7 +287,7 @@ func TestAcceptMTPDraftsGreedyAcceptAll(t *testing.T) {t.Fatalf("accepted = %d, observed = %d, want 3 and 3", accepted, observed)}// The run is the accepted drafts followed by the bonus token (5).- if got := resultIDs(results); !slices.Equal(got, []int{2, 3, 4, 5}) {+ if got := resultIDs(results); !slices.Equal(got, []int32{2, 3, 4, 5}) {t.Fatalf("results = %v, want [2 3 4 5]", got)}if position != 4 {@@ -332,7 +324,7 @@ func TestAcceptMTPDraftsGreedyMismatch(t *testing.T) {}// The run is the one accepted draft (2) followed by the target's own// prediction at the rejection point (3).- if got := resultIDs(results); !slices.Equal(got, []int{2, 3}) {+ if got := resultIDs(results); !slices.Equal(got, []int32{2, 3}) {t.Fatalf("results = %v, want [2 3]", got)}if position != 2 {@@ -370,7 +362,7 @@ func TestAcceptMTPDraftsGreedyEOS(t *testing.T) {}// The EOS ends generation, so the run is exactly the accepted tokens// with no bonus appended.- if got := resultIDs(results); !slices.Equal(got, []int{2, int(eos)}) {+ if got := resultIDs(results); !slices.Equal(got, []int32{2, eos}) {t.Fatalf("results = %v, want [2 %d]", got, eos)}if len(results) != accepted {@@ -1204,12 +1196,12 @@ func TestDecodeParkedDraftResume(t *testing.T) {d := spec.decoder(mlx.FromValues([]int32{1}, 1), 0).(*speculativeDecoder)// Two parked calls arrive pipelined, one token each.- for _, want := range []int{2, 3} {+ for _, want := range []int32{2, 3} {results, err := d.next(20)if err != nil {t.Fatalf("parked next: %v", err)}- if got := resultIDs(results); !slices.Equal(got, []int{want}) {+ if got := resultIDs(results); !slices.Equal(got, []int32{want}) {t.Fatalf("parked results = %v, want [%d]", got, want)}}@@ -1223,7 +1215,7 @@ func TestDecodeParkedDraftResume(t *testing.T) {if err != nil {t.Fatalf("resume next: %v", err)}- if got := resultIDs(results); !slices.Equal(got, []int{4}) {+ if got := resultIDs(results); !slices.Equal(got, []int32{4}) {t.Fatalf("resume results = %v, want [4]", got)}if len(model.forwards) != forwards {@@ -1236,7 +1228,7 @@ func TestDecodeParkedDraftResume(t *testing.T) {if err != nil {t.Fatalf("round next: %v", err)}- if got := resultIDs(results); !slices.Equal(got, []int{5, 6, int(eos)}) {+ if got := resultIDs(results); !slices.Equal(got, []int32{5, 6, eos}) {t.Fatalf("round results = %v, want [5 6 %d]", got, eos)}wantForwards := []forwardCall{{0, 1}, {1, 1}, {2, 1}, {3, 3}}diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.goindex 911f345ae..6cf00c1cb 100644--- a/x/mlxrunner/pipeline.go+++ b/x/mlxrunner/pipeline.go@@ -240,7 +240,7 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSessdefer func() {results, _ := d.drain()for _, res := range results {- session.outputs = append(session.outputs, int32(res.Token.Int()))+ session.outputs = append(session.outputs, res.Token.Int())}}()@@ -275,9 +275,7 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSessdone := falsestream := len(results)for i, res := range results {- // Int evaluates the array before reading it; a raw data read- // on a lazy array races its evaluation and returns garbage.- id := int32(res.Token.Int())+ id := res.Token.Int()session.outputs = append(session.outputs, id)if done {continue@@ -400,7 +398,7 @@ type detokenizer struct {}func (d *detokenizer) detokenize(res sampler.Result) (CompletionResponse, bool) {- output := int32(res.Token.Int())+ output := res.Token.Int()d.buf.WriteString(d.tokenizer.Decode([]int32{output}))d.logprobs = append(d.logprobs, buildLogprob(res, d.wantLogprobs, d.wantTopLogprobs, d.tokenizer.Decode)...)@@ -426,7 +424,7 @@ func buildLogprob(sample sampler.Result, wantLogprobs bool, wantTopLogprobs int,out := llm.Logprob{TokenLogprob: llm.TokenLogprob{- Token: tok(int32(sample.Token.Int())),+ Token: tok(sample.Token.Int()),Logprob: float64(sample.Logprob.Floats()[0]),},}@@ -437,7 +435,7 @@ func buildLogprob(sample sampler.Result, wantLogprobs bool, wantTopLogprobs int,pairs := make([]llm.TokenLogprob, len(ids))for i, id := range ids {pairs[i] = llm.TokenLogprob{- Token: tok(int32(id)),+ Token: tok(id),Logprob: float64(vals[i]),}}diff --git a/x/mlxrunner/sample/logprob_test.go b/x/mlxrunner/sample/logprob_test.goindex 9e37dd817..d29550663 100644--- a/x/mlxrunner/sample/logprob_test.go+++ b/x/mlxrunner/sample/logprob_test.go@@ -13,7 +13,7 @@ import (// logprobEntry is the (token id, logprob) pair returned by the sampler's// top-K extraction, used after the test-side descending sort.type logprobEntry struct {- id int+ id int32logprob float64}@@ -21,7 +21,7 @@ type logprobEntry struct {// and returns the greedily-sampled token id, its logprob, and the top-K// entries sorted descending by logprob. Logits must be a [vocab]-shaped// slice; the helper reshapes it to [1, vocab] before calling the sampler.-func runSampleLogprobs(t *testing.T, logits []float32, topK int) (int, float64, []logprobEntry) {+func runSampleLogprobs(t *testing.T, logits []float32, topK int) (int32, float64, []logprobEntry) {t.Helper()s := New(128)@@ -62,7 +62,7 @@ func TestSampleLogprobsBasic(t *testing.T) {name stringlogits []float32topK int- wantSelectedID int+ wantSelectedID int32wantTopLen int}{{@@ -215,10 +215,10 @@ func TestSampleLogprobsSelectedTokenCorrectness(t *testing.T) {logits := []float32{3.0, 1.0, 2.0, 0.5}- maxIdx := 0+ maxIdx := int32(0)for i, v := range logits[1:] {if v > logits[maxIdx] {- maxIdx = i + 1+ maxIdx = int32(i + 1)}}@@ -279,7 +279,7 @@ func TestSampleLogprobsTopKOrdering(t *testing.T) {// Logits chosen so argmax order differs from index order.logits := []float32{2.0, 5.0, 1.0, 4.0, 3.0}- wantOrder := []int{1, 3, 4, 0, 2}+ wantOrder := []int32{1, 3, 4, 0, 2}_, _, top := runSampleLogprobs(t, logits, len(logits))diff --git a/x/mlxrunner/sample/sample_test.go b/x/mlxrunner/sample/sample_test.goindex 3a906ad4e..8d59c5a18 100644--- a/x/mlxrunner/sample/sample_test.go+++ b/x/mlxrunner/sample/sample_test.go@@ -39,7 +39,7 @@ func batchLogits(rows ...[]float32) *mlx.Array {// sampleOne runs Sample on a freshly-added single slot and returns the// sampled token id. Used both for the single-slot options table and as the// reference oracle for the batched-equivalence test.-func sampleOne(t *testing.T, opts Options, priorTokens []int32, values []float32) int {+func sampleOne(t *testing.T, opts Options, priorTokens []int32, values []float32) int32 {t.Helper()s := New(128)t.Cleanup(func() {@@ -69,7 +69,7 @@ func TestSampleSingleSlotOptions(t *testing.T) {opts Optionspriors []int32logits []float32- want int+ want int32}{{name: "presence penalty",@@ -197,7 +197,7 @@ func TestDistributionResidualUsesTargetSupport(t *testing.T) {ids := residual.IDs.Ints()probs := residual.Probs.Floats()- want := map[int]float64{2: 0.625, 5: 0.375}+ want := map[int32]float64{2: 0.625, 5: 0.375}if len(ids) != 2 || len(probs) != 2 {t.Fatalf("residual = ids %v probs %v, want 2 sparse entries", ids, probs)}@@ -215,7 +215,7 @@ func TestDistributionResidualUsesTargetSupport(t *testing.T) {func TestSeededSamplingIsReproducible(t *testing.T) {skipIfNoMLX(t)- seededSequence := func(seed int) []int {+ seededSequence := func(seed int) []int32 {s := New(128)t.Cleanup(func() {s.Free()@@ -224,7 +224,7 @@ func TestSeededSamplingIsReproducible(t *testing.T) {s.Add(0, Options{Temperature: 1, TopK: 4, Seed: seed, UseSeed: true}, nil)logits := slotLogits([]float32{0, 0, 0, 0})- out := make([]int, 32)+ out := make([]int32, 32)for i := range out {token := s.Sample([]int{0}, logits).Tokenmlx.Eval(token)@@ -248,7 +248,7 @@ func TestSeededSamplingIsReproducible(t *testing.T) {func TestSeededBernoulliIsReproducible(t *testing.T) {skipIfNoMLX(t)- seededMask := func() []int {+ seededMask := func() []int32 {s := New(128)t.Cleanup(func() {s.Free()@@ -322,7 +322,7 @@ func TestSpeculativeScoresUsesDraftHistoryWithoutCommit(t *testing.T) {tokens := scores.Argmax(-1, false).AsType(mlx.DTypeInt32)mlx.Eval(tokens)- if got, want := tokens.Ints(), []int{3, 4, 2}; len(got) != len(want) {+ if got, want := tokens.Ints(), []int32{3, 4, 2}; len(got) != len(want) {t.Fatalf("tokens = %v, want %v", got, want)} else {for i := range want {@@ -391,7 +391,7 @@ func TestDistributionMultiRowWithoutChain(t *testing.T) {), nil)top := dist.IDs.Slice(mlx.Slice(), mlx.Slice(0, 1))mlx.Eval(top)- if got, want := top.Ints(), []int{2, 1, 2}; !slices.Equal(got, want) {+ if got, want := top.Ints(), []int32{2, 1, 2}; !slices.Equal(got, want) {t.Fatalf("seq %d top tokens = %v, want %v", seqID, got, want)}}@@ -412,7 +412,7 @@ func TestCommitBatchesRingWrites(t *testing.T) {mlx.Eval(s.history)got := s.history.Ints()- want := []int{32, 33, 34, 31}+ want := []int32{32, 33, 34, 31}for i := range want {if got[i] != want[i] {t.Fatalf("history = %v, want %v", got, want)@@ -484,7 +484,7 @@ func TestBatchSamplingPreservesPerSlotBehavior(t *testing.T) {for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {// Per-slot reference for each sampled seq.- want := make([]int, len(tc.sample))+ want := make([]int32, len(tc.sample))for i, id := range tc.sample {var spec slotfor _, s := range tc.slots {diff --git a/x/mlxrunner/speculate.go b/x/mlxrunner/speculate.goindex 88b7cfe1d..006025d36 100644--- a/x/mlxrunner/speculate.go+++ b/x/mlxrunner/speculate.go@@ -451,8 +451,8 @@ func (s *speculationSession) accept(position *int, current sampler.Result, candikeep := accepteddone := falsefor i, id := range draftIDs[:accepted] {- commitIDs = append(commitIDs, int32(id))- if r.Tokenizer.IsEOS(int32(id)) {+ commitIDs = append(commitIDs, id)+ if r.Tokenizer.IsEOS(id) {done = trueaccepted = i + 1observed = accepted@@ -471,7 +471,7 @@ func (s *speculationSession) accept(position *int, current sampler.Result, candi// Report the validated run (current plus kept drafts) to the drafter before// returning, so a cancelled emission still leaves it matching the caches. A// done generation's final token is uncommitted; it reaches finish instead.- runIDs := append([]int32{int32(current.Token.Int())}, commitIDs[:keep]...)+ runIDs := append([]int32{current.Token.Int()}, commitIDs[:keep]...)s.drafter.committed(mlx.FromValues(runIDs, 1, len(runIDs)),auxHiddenSeq.Slice(mlx.Slice(), mlx.Slice(0, len(runIDs)), mlx.Slice()),@@ -485,9 +485,9 @@ func (s *speculationSession) accept(position *int, current sampler.Result, candivar nextID int32if accepted < draftCount {- nextID = int32(residualTokens.Ints()[accepted])+ nextID = residualTokens.Ints()[accepted]} else {- nextID = int32(bonusToken.Int())+ nextID = bonusToken.Int()}commitIDs = append(commitIDs, nextID)r.Sampler.Commit(pipelineSlot, commitIDs)@@ -509,10 +509,10 @@ func (r *Runner) sampleTokenAt(dist sampler.Distribution, index int) *mlx.Array// draftResults wraps accepted draft ids as sampler results; drafts carry no// logprobs, so only the token id is set.-func draftResults(ids []int) []sampler.Result {+func draftResults(ids []int32) []sampler.Result {results := make([]sampler.Result, len(ids))for i, id := range ids {- results[i] = sampler.Result{Token: mlx.FromValues([]int32{int32(id)}, 1)}+ results[i] = sampler.Result{Token: mlx.FromValues([]int32{id}, 1)}}return results}diff --git a/x/models/gemma4/gemma4_moe_test.go b/x/models/gemma4/gemma4_moe_test.goindex 4fae76067..e8113ef8a 100644--- a/x/models/gemma4/gemma4_moe_test.go+++ b/x/models/gemma4/gemma4_moe_test.go@@ -307,7 +307,7 @@ func legacyRouterForward(r *Router, x *mlx.Array, cfg *TextConfig) (*mlx.Array,return scores, inds}-func intSlicesEqual(a, b []int) bool {+func intSlicesEqual(a, b []int32) bool {if len(a) != len(b) {return false}