Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlxrunner: Release freed KV buffers during speculative decode
Jesse Grossec3cc23076c4
diff --git a/mlxrunner/pipeline.go b/mlxrunner/pipeline.goindex eabe2e0d4..827dc0bc2 100644--- a/mlxrunner/pipeline.go+++ b/mlxrunner/pipeline.go@@ -278,7 +278,9 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSessnow := time.Now()// Release MLX's cached free buffers every clearCacheInterval tokens so the- // allocator's pool does not grow unbounded over a long generation.+ // allocator's pool does not grow unbounded over a long generation. A+ // speculative round emits several tokens at once, so the clear fires on+ // crossing a multiple of the interval, not on landing exactly on one.const clearCacheInterval = 256generated := 0@@ -289,6 +291,7 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSessvar done boolvar err error+ before := generatedmlx.Scoped(func() {var results []sampler.Resultresults, err = d.next(request.Options.NumPredict - generated)@@ -348,7 +351,7 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSessbreak}- if generated%clearCacheInterval == 0 {+ if generated/clearCacheInterval != before/clearCacheInterval {mlx.ClearCache()}}