Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

mlx: return exact types and evaluate arrays in the value readers

Jesse Gross7623501fc2c9
diff --git a/x/mlxrunner/dflash_test.go b/x/mlxrunner/dflash_test.go
index 89aa11814..f88d38cd7 100644
--- a/x/mlxrunner/dflash_test.go
+++ b/x/mlxrunner/dflash_test.go
@@ -61,11 +61,7 @@ func (d *fakeBlockDraft) Forward(b *batch.Batch, _, draftCaches []cache.Cache) (
}
if b.InputIDs != nil {
mlx.Eval(b.InputIDs)
- ids := b.InputIDs.Ints()
- call.block = make([]int32, len(ids))
- for i, id := range ids {
- call.block[i] = int32(id)
- }
+ call.block = b.InputIDs.Ints()
}
d.calls = append(d.calls, call)
@@ -212,7 +208,7 @@ func TestDFlashProposeBounds(t *testing.T) {
t.Fatalf("propose declined with context and budget")
}
mlx.Eval(cand.tokens)
- if got := cand.tokens.Ints(); !slices.Equal(got, []int{2, 3, 4}) {
+ if got := cand.tokens.Ints(); !slices.Equal(got, []int32{2, 3, 4}) {
t.Fatalf("draft tokens = %v, want [2 3 4]", got)
}
if got, want := draft.calls[0].block, []int32{1, 6, 6, 6}; !slices.Equal(got, want) {
diff --git a/x/mlxrunner/mlx/array.go b/x/mlxrunner/mlx/array.go
index 5271b619f..f87d147d5 100644
--- a/x/mlxrunner/mlx/array.go
+++ b/x/mlxrunner/mlx/array.go
@@ -236,52 +236,31 @@ func (t *Array) DType() DType {
// data utilities
-func (t *Array) Int() int {
- switch dt := t.DType(); dt {
- case DTypeInt32:
- var item C.int32_t
- C.mlx_array_item_int32(&item, t.ctx)
- return int(item)
- case DTypeUint32:
- var item C.uint32_t
- C.mlx_array_item_uint32(&item, t.ctx)
- return int(item)
- case DTypeInt64:
- var item C.int64_t
- C.mlx_array_item_int64(&item, t.ctx)
- return int(item)
- case DTypeUint64:
- var item C.uint64_t
- C.mlx_array_item_uint64(&item, t.ctx)
- return int(item)
- default:
- panic(fmt.Sprintf("mlx: Int requires an integer array, got %v", dt))
+func (t *Array) Int() int32 {
+ if dt := t.DType(); dt != DTypeInt32 {
+ panic(fmt.Sprintf("mlx: Int requires a DTypeInt32 array, got %v", dt))
}
+ var item C.int32_t
+ C.mlx_array_item_int32(&item, t.ctx)
+ return int32(item)
}
-func (t *Array) Float() float64 {
- switch dt := t.DType(); dt {
- case DTypeFloat32:
- var item C.float
- C.mlx_array_item_float32(&item, t.ctx)
- return float64(item)
- case DTypeFloat64:
- var item C.double
- C.mlx_array_item_float64(&item, t.ctx)
- return float64(item)
- default:
- panic(fmt.Sprintf("mlx: Float requires a float32 or float64 array, got %v", dt))
+func (t *Array) Float() float32 {
+ if dt := t.DType(); dt != DTypeFloat32 {
+ panic(fmt.Sprintf("mlx: Float requires a DTypeFloat32 array, got %v", dt))
}
+ var item C.float
+ C.mlx_array_item_float32(&item, t.ctx)
+ return float32(item)
}
-func (t *Array) Ints() []int {
+func (t *Array) Ints() []int32 {
if dt := t.DType(); dt != DTypeInt32 {
panic(fmt.Sprintf("mlx: Ints requires DTypeInt32, got %v", dt))
}
- ints := make([]int, t.Size())
- for i, f := range unsafe.Slice(C.mlx_array_data_int32(t.ctx), len(ints)) {
- ints[i] = int(f)
- }
+ Eval(t)
+ ints := make([]int32, t.Size())
+ copy(ints, unsafe.Slice((*int32)(unsafe.Pointer(C.mlx_array_data_int32(t.ctx))), len(ints)))
return ints
}
@@ -289,10 +268,9 @@ func (t *Array) Floats() []float32 {
if dt := t.DType(); dt != DTypeFloat32 {
panic(fmt.Sprintf("mlx: Floats requires DTypeFloat32, got %v", dt))
}
+ Eval(t)
floats := make([]float32, t.Size())
- for i, f := range unsafe.Slice(C.mlx_array_data_float32(t.ctx), len(floats)) {
- floats[i] = float32(f)
- }
+ copy(floats, unsafe.Slice((*float32)(unsafe.Pointer(C.mlx_array_data_float32(t.ctx))), len(floats)))
return floats
}
diff --git a/x/mlxrunner/mlx/array_test.go b/x/mlxrunner/mlx/array_test.go
index 927853c0c..7386f3335 100644
--- a/x/mlxrunner/mlx/array_test.go
+++ b/x/mlxrunner/mlx/array_test.go
@@ -58,13 +58,13 @@ func testComparisonOpsAndBernoulli(t *testing.T) {
Eval(eq, gt, le, bern)
for name, tc := range map[string]struct {
- got []int
- want []int
+ got []int32
+ want []int32
}{
- "equal": {eq.Ints(), []int{1, 0, 0}},
- "greater": {gt.Ints(), []int{0, 1, 0}},
- "lessEqual": {le.Ints(), []int{1, 0, 1}},
- "bernoulli": {bern.Ints(), []int{1, 0}},
+ "equal": {eq.Ints(), []int32{1, 0, 0}},
+ "greater": {gt.Ints(), []int32{0, 1, 0}},
+ "lessEqual": {le.Ints(), []int32{1, 0, 1}},
+ "bernoulli": {bern.Ints(), []int32{1, 0}},
} {
t.Run(name, func(t *testing.T) {
if len(tc.got) != len(tc.want) {
diff --git a/x/mlxrunner/mtp_test.go b/x/mlxrunner/mtp_test.go
index 80460e3da..276e35e53 100644
--- a/x/mlxrunner/mtp_test.go
+++ b/x/mlxrunner/mtp_test.go
@@ -71,17 +71,13 @@ func (m *fakeMTPModel) Forward(b *batch.Batch, caches []cache.Cache) (hidden, au
break
}
if rc, ok := c.(*fakeRewindableCache); ok {
- seg := make([]int32, len(ids))
- for i, id := range ids {
- seg[i] = int32(id)
- }
- rc.feed(seg)
+ rc.feed(ids)
}
}
preds := make([]int32, len(ids))
for i, id := range ids {
- preds[i] = m.predict[int32(id)]
+ preds[i] = m.predict[id]
}
out := oneHotLogits(preds)
return out, out
@@ -121,7 +117,7 @@ func (d *fakeMTPDraft) NewCaches() []cache.Cache { return nil }
func (d *fakeMTPDraft) Forward(b *batch.Batch, _, _ []cache.Cache) (hidden, auxHidden *mlx.Array) {
d.layouts = append(d.layouts, b.Layout)
mlx.Eval(b.InputIDs)
- prev := int32(b.InputIDs.Ints()[0])
+ prev := b.InputIDs.Ints()[0]
d.calls = append(d.calls, draftCall{position: b.SeqOffsets[0], from: prev})
return oneHotLogits([]int32{d.predict[prev]}), mlx.Zeros(mlx.DTypeFloat32, 1, 1, mtpTestVocab)
}
@@ -160,11 +156,7 @@ func (d *fakeKVDraft) NewCaches() []cache.Cache { return d.draftCaches }
func (d *fakeKVDraft) Forward(b *batch.Batch, _, draftCaches []cache.Cache) (hidden, auxHidden *mlx.Array) {
mlx.Eval(b.InputIDs, b.Hidden)
- rawIDs := b.InputIDs.Ints()
- ids := make([]int32, len(rawIDs))
- for i, id := range rawIDs {
- ids[i] = int32(id)
- }
+ ids := b.InputIDs.Ints()
hot := make([]int32, b.Hidden.Dim(1))
flat := b.Hidden.Floats()
@@ -262,8 +254,8 @@ func collectResponses(ch chan CompletionResponse) (content string, final Complet
}
// resultIDs reads the token id of each result.
-func resultIDs(results []sampler.Result) []int {
- ids := make([]int, 0, len(results))
+func resultIDs(results []sampler.Result) []int32 {
+ ids := make([]int32, 0, len(results))
for _, res := range results {
ids = append(ids, res.Token.Int())
}
@@ -295,7 +287,7 @@ func TestAcceptMTPDraftsGreedyAcceptAll(t *testing.T) {
t.Fatalf("accepted = %d, observed = %d, want 3 and 3", accepted, observed)
}
// The run is the accepted drafts followed by the bonus token (5).
- if got := resultIDs(results); !slices.Equal(got, []int{2, 3, 4, 5}) {
+ if got := resultIDs(results); !slices.Equal(got, []int32{2, 3, 4, 5}) {
t.Fatalf("results = %v, want [2 3 4 5]", got)
}
if position != 4 {
@@ -332,7 +324,7 @@ func TestAcceptMTPDraftsGreedyMismatch(t *testing.T) {
}
// The run is the one accepted draft (2) followed by the target's own
// prediction at the rejection point (3).
- if got := resultIDs(results); !slices.Equal(got, []int{2, 3}) {
+ if got := resultIDs(results); !slices.Equal(got, []int32{2, 3}) {
t.Fatalf("results = %v, want [2 3]", got)
}
if position != 2 {
@@ -370,7 +362,7 @@ func TestAcceptMTPDraftsGreedyEOS(t *testing.T) {
}
// The EOS ends generation, so the run is exactly the accepted tokens
// with no bonus appended.
- if got := resultIDs(results); !slices.Equal(got, []int{2, int(eos)}) {
+ if got := resultIDs(results); !slices.Equal(got, []int32{2, eos}) {
t.Fatalf("results = %v, want [2 %d]", got, eos)
}
if len(results) != accepted {
@@ -1204,12 +1196,12 @@ func TestDecodeParkedDraftResume(t *testing.T) {
d := spec.decoder(mlx.FromValues([]int32{1}, 1), 0).(*speculativeDecoder)
// Two parked calls arrive pipelined, one token each.
- for _, want := range []int{2, 3} {
+ for _, want := range []int32{2, 3} {
results, err := d.next(20)
if err != nil {
t.Fatalf("parked next: %v", err)
}
- if got := resultIDs(results); !slices.Equal(got, []int{want}) {
+ if got := resultIDs(results); !slices.Equal(got, []int32{want}) {
t.Fatalf("parked results = %v, want [%d]", got, want)
}
}
@@ -1223,7 +1215,7 @@ func TestDecodeParkedDraftResume(t *testing.T) {
if err != nil {
t.Fatalf("resume next: %v", err)
}
- if got := resultIDs(results); !slices.Equal(got, []int{4}) {
+ if got := resultIDs(results); !slices.Equal(got, []int32{4}) {
t.Fatalf("resume results = %v, want [4]", got)
}
if len(model.forwards) != forwards {
@@ -1236,7 +1228,7 @@ func TestDecodeParkedDraftResume(t *testing.T) {
if err != nil {
t.Fatalf("round next: %v", err)
}
- if got := resultIDs(results); !slices.Equal(got, []int{5, 6, int(eos)}) {
+ if got := resultIDs(results); !slices.Equal(got, []int32{5, 6, eos}) {
t.Fatalf("round results = %v, want [5 6 %d]", got, eos)
}
wantForwards := []forwardCall{{0, 1}, {1, 1}, {2, 1}, {3, 3}}
diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.go
index 911f345ae..6cf00c1cb 100644
--- a/x/mlxrunner/pipeline.go
+++ b/x/mlxrunner/pipeline.go
@@ -240,7 +240,7 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSess
defer func() {
results, _ := d.drain()
for _, res := range results {
- session.outputs = append(session.outputs, int32(res.Token.Int()))
+ session.outputs = append(session.outputs, res.Token.Int())
}
}()
@@ -275,9 +275,7 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSess
done := false
stream := len(results)
for i, res := range results {
- // Int evaluates the array before reading it; a raw data read
- // on a lazy array races its evaluation and returns garbage.
- id := int32(res.Token.Int())
+ id := res.Token.Int()
session.outputs = append(session.outputs, id)
if done {
continue
@@ -400,7 +398,7 @@ type detokenizer struct {
}
func (d *detokenizer) detokenize(res sampler.Result) (CompletionResponse, bool) {
- output := int32(res.Token.Int())
+ output := res.Token.Int()
d.buf.WriteString(d.tokenizer.Decode([]int32{output}))
d.logprobs = append(d.logprobs, buildLogprob(res, d.wantLogprobs, d.wantTopLogprobs, d.tokenizer.Decode)...)
@@ -426,7 +424,7 @@ func buildLogprob(sample sampler.Result, wantLogprobs bool, wantTopLogprobs int,
out := llm.Logprob{
TokenLogprob: llm.TokenLogprob{
- Token: tok(int32(sample.Token.Int())),
+ Token: tok(sample.Token.Int()),
Logprob: float64(sample.Logprob.Floats()[0]),
},
}
@@ -437,7 +435,7 @@ func buildLogprob(sample sampler.Result, wantLogprobs bool, wantTopLogprobs int,
pairs := make([]llm.TokenLogprob, len(ids))
for i, id := range ids {
pairs[i] = llm.TokenLogprob{
- Token: tok(int32(id)),
+ Token: tok(id),
Logprob: float64(vals[i]),
}
}
diff --git a/x/mlxrunner/sample/logprob_test.go b/x/mlxrunner/sample/logprob_test.go
index 9e37dd817..d29550663 100644
--- a/x/mlxrunner/sample/logprob_test.go
+++ b/x/mlxrunner/sample/logprob_test.go
@@ -13,7 +13,7 @@ import (
// logprobEntry is the (token id, logprob) pair returned by the sampler's
// top-K extraction, used after the test-side descending sort.
type logprobEntry struct {
- id int
+ id int32
logprob float64
}
@@ -21,7 +21,7 @@ type logprobEntry struct {
// and returns the greedily-sampled token id, its logprob, and the top-K
// entries sorted descending by logprob. Logits must be a [vocab]-shaped
// slice; the helper reshapes it to [1, vocab] before calling the sampler.
-func runSampleLogprobs(t *testing.T, logits []float32, topK int) (int, float64, []logprobEntry) {
+func runSampleLogprobs(t *testing.T, logits []float32, topK int) (int32, float64, []logprobEntry) {
t.Helper()
s := New(128)
@@ -62,7 +62,7 @@ func TestSampleLogprobsBasic(t *testing.T) {
name string
logits []float32
topK int
- wantSelectedID int
+ wantSelectedID int32
wantTopLen int
}{
{
@@ -215,10 +215,10 @@ func TestSampleLogprobsSelectedTokenCorrectness(t *testing.T) {
logits := []float32{3.0, 1.0, 2.0, 0.5}
- maxIdx := 0
+ maxIdx := int32(0)
for i, v := range logits[1:] {
if v > logits[maxIdx] {
- maxIdx = i + 1
+ maxIdx = int32(i + 1)
}
}
@@ -279,7 +279,7 @@ func TestSampleLogprobsTopKOrdering(t *testing.T) {
// Logits chosen so argmax order differs from index order.
logits := []float32{2.0, 5.0, 1.0, 4.0, 3.0}
- wantOrder := []int{1, 3, 4, 0, 2}
+ wantOrder := []int32{1, 3, 4, 0, 2}
_, _, top := runSampleLogprobs(t, logits, len(logits))
diff --git a/x/mlxrunner/sample/sample_test.go b/x/mlxrunner/sample/sample_test.go
index 3a906ad4e..8d59c5a18 100644
--- a/x/mlxrunner/sample/sample_test.go
+++ b/x/mlxrunner/sample/sample_test.go
@@ -39,7 +39,7 @@ func batchLogits(rows ...[]float32) *mlx.Array {
// sampleOne runs Sample on a freshly-added single slot and returns the
// sampled token id. Used both for the single-slot options table and as the
// reference oracle for the batched-equivalence test.
-func sampleOne(t *testing.T, opts Options, priorTokens []int32, values []float32) int {
+func sampleOne(t *testing.T, opts Options, priorTokens []int32, values []float32) int32 {
t.Helper()
s := New(128)
t.Cleanup(func() {
@@ -69,7 +69,7 @@ func TestSampleSingleSlotOptions(t *testing.T) {
opts Options
priors []int32
logits []float32
- want int
+ want int32
}{
{
name: "presence penalty",
@@ -197,7 +197,7 @@ func TestDistributionResidualUsesTargetSupport(t *testing.T) {
ids := residual.IDs.Ints()
probs := residual.Probs.Floats()
- want := map[int]float64{2: 0.625, 5: 0.375}
+ want := map[int32]float64{2: 0.625, 5: 0.375}
if len(ids) != 2 || len(probs) != 2 {
t.Fatalf("residual = ids %v probs %v, want 2 sparse entries", ids, probs)
}
@@ -215,7 +215,7 @@ func TestDistributionResidualUsesTargetSupport(t *testing.T) {
func TestSeededSamplingIsReproducible(t *testing.T) {
skipIfNoMLX(t)
- seededSequence := func(seed int) []int {
+ seededSequence := func(seed int) []int32 {
s := New(128)
t.Cleanup(func() {
s.Free()
@@ -224,7 +224,7 @@ func TestSeededSamplingIsReproducible(t *testing.T) {
s.Add(0, Options{Temperature: 1, TopK: 4, Seed: seed, UseSeed: true}, nil)
logits := slotLogits([]float32{0, 0, 0, 0})
- out := make([]int, 32)
+ out := make([]int32, 32)
for i := range out {
token := s.Sample([]int{0}, logits).Token
mlx.Eval(token)
@@ -248,7 +248,7 @@ func TestSeededSamplingIsReproducible(t *testing.T) {
func TestSeededBernoulliIsReproducible(t *testing.T) {
skipIfNoMLX(t)
- seededMask := func() []int {
+ seededMask := func() []int32 {
s := New(128)
t.Cleanup(func() {
s.Free()
@@ -322,7 +322,7 @@ func TestSpeculativeScoresUsesDraftHistoryWithoutCommit(t *testing.T) {
tokens := scores.Argmax(-1, false).AsType(mlx.DTypeInt32)
mlx.Eval(tokens)
- if got, want := tokens.Ints(), []int{3, 4, 2}; len(got) != len(want) {
+ if got, want := tokens.Ints(), []int32{3, 4, 2}; len(got) != len(want) {
t.Fatalf("tokens = %v, want %v", got, want)
} else {
for i := range want {
@@ -391,7 +391,7 @@ func TestDistributionMultiRowWithoutChain(t *testing.T) {
), nil)
top := dist.IDs.Slice(mlx.Slice(), mlx.Slice(0, 1))
mlx.Eval(top)
- if got, want := top.Ints(), []int{2, 1, 2}; !slices.Equal(got, want) {
+ if got, want := top.Ints(), []int32{2, 1, 2}; !slices.Equal(got, want) {
t.Fatalf("seq %d top tokens = %v, want %v", seqID, got, want)
}
}
@@ -412,7 +412,7 @@ func TestCommitBatchesRingWrites(t *testing.T) {
mlx.Eval(s.history)
got := s.history.Ints()
- want := []int{32, 33, 34, 31}
+ want := []int32{32, 33, 34, 31}
for i := range want {
if got[i] != want[i] {
t.Fatalf("history = %v, want %v", got, want)
@@ -484,7 +484,7 @@ func TestBatchSamplingPreservesPerSlotBehavior(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
// Per-slot reference for each sampled seq.
- want := make([]int, len(tc.sample))
+ want := make([]int32, len(tc.sample))
for i, id := range tc.sample {
var spec slot
for _, s := range tc.slots {
diff --git a/x/mlxrunner/speculate.go b/x/mlxrunner/speculate.go
index 88b7cfe1d..006025d36 100644
--- a/x/mlxrunner/speculate.go
+++ b/x/mlxrunner/speculate.go
@@ -451,8 +451,8 @@ func (s *speculationSession) accept(position *int, current sampler.Result, candi
keep := accepted
done := false
for i, id := range draftIDs[:accepted] {
- commitIDs = append(commitIDs, int32(id))
- if r.Tokenizer.IsEOS(int32(id)) {
+ commitIDs = append(commitIDs, id)
+ if r.Tokenizer.IsEOS(id) {
done = true
accepted = i + 1
observed = accepted
@@ -471,7 +471,7 @@ func (s *speculationSession) accept(position *int, current sampler.Result, candi
// Report the validated run (current plus kept drafts) to the drafter before
// returning, so a cancelled emission still leaves it matching the caches. A
// done generation's final token is uncommitted; it reaches finish instead.
- runIDs := append([]int32{int32(current.Token.Int())}, commitIDs[:keep]...)
+ runIDs := append([]int32{current.Token.Int()}, commitIDs[:keep]...)
s.drafter.committed(
mlx.FromValues(runIDs, 1, len(runIDs)),
auxHiddenSeq.Slice(mlx.Slice(), mlx.Slice(0, len(runIDs)), mlx.Slice()),
@@ -485,9 +485,9 @@ func (s *speculationSession) accept(position *int, current sampler.Result, candi
var nextID int32
if accepted < draftCount {
- nextID = int32(residualTokens.Ints()[accepted])
+ nextID = residualTokens.Ints()[accepted]
} else {
- nextID = int32(bonusToken.Int())
+ nextID = bonusToken.Int()
}
commitIDs = append(commitIDs, nextID)
r.Sampler.Commit(pipelineSlot, commitIDs)
@@ -509,10 +509,10 @@ func (r *Runner) sampleTokenAt(dist sampler.Distribution, index int) *mlx.Array
// draftResults wraps accepted draft ids as sampler results; drafts carry no
// logprobs, so only the token id is set.
-func draftResults(ids []int) []sampler.Result {
+func draftResults(ids []int32) []sampler.Result {
results := make([]sampler.Result, len(ids))
for i, id := range ids {
- results[i] = sampler.Result{Token: mlx.FromValues([]int32{int32(id)}, 1)}
+ results[i] = sampler.Result{Token: mlx.FromValues([]int32{id}, 1)}
}
return results
}
diff --git a/x/models/gemma4/gemma4_moe_test.go b/x/models/gemma4/gemma4_moe_test.go
index 4fae76067..e8113ef8a 100644
--- a/x/models/gemma4/gemma4_moe_test.go
+++ b/x/models/gemma4/gemma4_moe_test.go
@@ -307,7 +307,7 @@ func legacyRouterForward(r *Router, x *mlx.Array, cfg *TextConfig) (*mlx.Array,
return scores, inds
}
-func intSlicesEqual(a, b []int) bool {
+func intSlicesEqual(a, b []int32) bool {
if len(a) != len(b) {
return false
}