Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlxrunner: Fix panic on full KV cache hit
Jesse Grossdd5eb6337dab
diff --git a/x/mlxrunner/cache.go b/x/mlxrunner/cache.goindex 0d858d91b..48f953b2c 100644--- a/x/mlxrunner/cache.go+++ b/x/mlxrunner/cache.go@@ -78,6 +78,12 @@ func (c *kvCache) findRemaining(tokens []int32) []int32 {prefix++}+ // Always keep at least one token to re-evaluate so the+ // pipeline can seed token generation from it.+ if prefix == len(tokens) && prefix > 0 {+ prefix--+ }+if prefix < len(c.tokens) {trim := len(c.tokens) - prefixfor _, kv := range c.caches {diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.goindex 945f94755..ed068ed6b 100644--- a/x/mlxrunner/pipeline.go+++ b/x/mlxrunner/pipeline.go@@ -47,6 +47,10 @@ func (r *Runner) TextGenerationPipeline(request Request) error {mlx.ResetPeakMemory()inputs := r.Tokenizer.Encode(request.Prompt, true)+ if len(inputs) == 0 {+ return errors.New("empty prompt")+ }+session := r.cache.begin(r.Model, inputs)defer session.close()@@ -54,7 +58,6 @@ func (r *Runner) TextGenerationPipeline(request Request) error {tokens := session.remainingtotal, processed := len(tokens), 0- slog.Info("Prompt processing progress", "processed", processed, "total", total)for total-processed > 1 {if err := request.Ctx.Err(); err != nil {return err@@ -104,7 +107,6 @@ func (r *Runner) TextGenerationPipeline(request Request) error {nextSample, nextLogprobs = step(sample)if i == 0 {- slog.Info("Prompt processing progress", "processed", total, "total", total)mlx.Eval(sample)final.PromptTokensDuration = time.Since(now)now = time.Now()