Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

mlxrunner: Release freed KV buffers during speculative decode

Jesse Grossec3cc23076c4
diff --git a/mlxrunner/pipeline.go b/mlxrunner/pipeline.go
index eabe2e0d4..827dc0bc2 100644
--- a/mlxrunner/pipeline.go
+++ b/mlxrunner/pipeline.go
@@ -278,7 +278,9 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSess
now := time.Now()
// Release MLX's cached free buffers every clearCacheInterval tokens so the
- // allocator's pool does not grow unbounded over a long generation.
+ // allocator's pool does not grow unbounded over a long generation. A
+ // speculative round emits several tokens at once, so the clear fires on
+ // crossing a multiple of the interval, not on landing exactly on one.
const clearCacheInterval = 256
generated := 0
@@ -289,6 +291,7 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSess
var done bool
var err error
+ before := generated
mlx.Scoped(func() {
var results []sampler.Result
results, err = d.next(request.Options.NumPredict - generated)
@@ -348,7 +351,7 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSess
break
}
- if generated%clearCacheInterval == 0 {
+ if generated/clearCacheInterval != before/clearCacheInterval {
mlx.ClearCache()
}
}