Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlxrunner: resolve each speculative round in one host sync
Jesse Gross114875133b8b
diff --git a/x/mlxrunner/speculate.go b/x/mlxrunner/speculate.goindex f0039303c..84cefb0f2 100644--- a/x/mlxrunner/speculate.go+++ b/x/mlxrunner/speculate.go@@ -463,7 +463,15 @@ func (s *speculationSession) accept(position *int, current sampler.Result, canditargetDist := r.Sampler.Distribution(pipelineSlot, r.Model.Unembed(hiddenSeq), candidates.tokens)draftDist := candidates.distacceptedMask := r.sampleAcceptedMask(targetDist.SliceRows(0, draftCount), draftDist, candidates.tokens)- mlx.Eval(candidates.tokens, acceptedMask)++ // The next token is sampled for every possible outcome before anything+ // is evaluated — the residual at each rejection point in one batched+ // draw, plus the bonus row — so a single Eval covers acceptance and the+ // next token instead of a second host round trip after the rejection+ // point is known.+ residualTokens := r.Sampler.SampleDistribution(pipelineSlot, targetDist.SliceRows(0, draftCount).ResidualAgainst(draftDist))+ bonusToken := r.sampleTokenAt(targetDist, draftCount)+ mlx.Eval(candidates.tokens, acceptedMask, residualTokens, bonusToken)draftIDs := candidates.tokens.Ints()acceptedFlags := acceptedMask.Ints()@@ -516,18 +524,16 @@ func (s *speculationSession) accept(position *int, current sampler.Result, candireturn results, accepted, nil}- var nextToken *mlx.Array- if accepted == draftCount {- nextToken = r.sampleTokenAt(targetDist, draftCount)+ var nextID int32+ if accepted < draftCount {+ nextID = int32(residualTokens.Ints()[accepted])} else {- nextToken = r.sampleResidualToken(targetDist, draftDist, accepted)+ nextID = int32(bonusToken.Int())}- mlx.Eval(nextToken)- nextID := int32(nextToken.Int())commitIDs = append(commitIDs, nextID)r.Sampler.Commit(pipelineSlot, commitIDs)- results = append(results, sampler.Result{Token: nextToken})+ results = append(results, sampler.Result{Token: mlx.FromValues([]int32{nextID}, 1)})return results, accepted, nil}@@ -542,11 +548,6 @@ func (r *Runner) sampleTokenAt(dist sampler.Distribution, index int) *mlx.Arrayreturn r.Sampler.SampleDistribution(pipelineSlot, dist.SliceRows(index, index+1))}-func (r *Runner) sampleResidualToken(targetDist, draftDist sampler.Distribution, index int) *mlx.Array {- residual := targetDist.SliceRows(index, index+1).ResidualAgainst(draftDist.SliceRows(index, index+1))- return tokenVector(r.Sampler.SampleDistribution(pipelineSlot, residual))-}-// draftResults wraps accepted draft ids as sampler results; drafts carry no// logprobs, so only the token id is set.func draftResults(ids []int) []sampler.Result {@@ -569,14 +570,3 @@ func tokenInput(token *mlx.Array) *mlx.Array {panic(fmt.Sprintf("token must be rank 0, 1, or 2, got rank %d", token.NumDims()))}}--func tokenVector(token *mlx.Array) *mlx.Array {- switch token.NumDims() {- case 0:- return token.Reshape(1)- case 1:- return token- default:- panic(fmt.Sprintf("sampled token must be rank 0 or 1, got rank %d", token.NumDims()))- }-}