Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlxrunner: Simplify KV cache to single-entry prefix matching
Jesse Gross5c73c4e2eede
diff --git a/x/mlxrunner/cache.go b/x/mlxrunner/cache.goindex 49ddd04b6..750d556b4 100644--- a/x/mlxrunner/cache.go+++ b/x/mlxrunner/cache.go@@ -3,94 +3,65 @@package mlxrunnerimport (+ "fmt""log/slog"+ "github.com/ollama/ollama/logutil""github.com/ollama/ollama/x/mlxrunner/cache"+ "github.com/ollama/ollama/x/mlxrunner/mlx")+// CacheEntry stores a single sequencetype CacheEntry struct {- Caches []cache.Cache- Count int- Entries map[int32]*CacheEntry+ Tokens []int32+ Caches []cache.Cache}-func (s Runner) FindNearestCache(tokens []int32) ([]cache.Cache, []int32) {- current := &CacheEntry{Entries: s.CacheEntries}- index, cacheIndex := 0, -1- for _, token := range tokens {- if _, ok := current.Entries[token]; !ok {- break- }-- current = current.Entries[token]- if len(current.Caches) > 0 {- cacheIndex = index- }+// FindNearestCache finds the longest common prefix between tokens and the cached sequence+func (r *Runner) FindNearestCache(tokens []int32) ([]cache.Cache, []int32) {+ if r.cache == nil {+ slog.Info("Cache miss", "left", len(tokens))+ return nil, tokens+ }- index += 1+ // Find longest common prefix+ prefix := 0+ for prefix < len(tokens) && prefix < len(r.cache.Tokens) && tokens[prefix] == r.cache.Tokens[prefix] {+ prefix++}- if cacheIndex == len(tokens)-1 {- slog.Info("Cache hit", "type", "exact", "total", len(tokens), "cached", len(tokens), "left", len(tokens))- return current.Caches, []int32{}- } else if cacheIndex > 1 {- slog.Info("Cache hit", "type", "partial", "total", len(tokens), "cached", cacheIndex+1, "left", len(tokens[cacheIndex+1:]))- return current.Caches, tokens[cacheIndex+1:]- } else if index > 0 && cacheIndex < 0 {- type stackItem struct {- entry *CacheEntry- tokens []int32+ switch {+ case prefix == 0:+ for _, c := range r.cache.Caches {+ c.Free()}-- var best, item stackItem- stack := []stackItem{{entry: current, tokens: []int32{}}}- for len(stack) > 0 {- item, stack = stack[len(stack)-1], stack[:len(stack)-1]- if len(item.entry.Caches) > 0 {- if len(best.tokens) == 0 || len(item.tokens) < len(best.tokens) {- best = item- }- } else {- for token, entry := range item.entry.Entries {- stack = append(stack, stackItem{- entry: entry,- tokens: append(item.tokens, token),- })- }- }+ r.cache = nil+ slog.Info("Cache miss", "left", len(tokens))+ return nil, tokens+ case prefix < len(r.cache.Tokens):+ trim := len(r.cache.Tokens) - prefix+ for _, c := range r.cache.Caches {+ c.Trim(trim)}-- prefix := min(len(tokens)-1, index)- caches := make([]cache.Cache, len(best.entry.Caches))- trim := len(best.tokens)+1- for i := range caches {- caches[i] = best.entry.Caches[i].Clone()- caches[i].Trim(trim)- }-- slog.Info("Cache hit", "type", "prefix", "total", len(tokens), "cached", prefix, "left", len(tokens[prefix:]), "trimmed", trim)- return caches, tokens[prefix:]+ r.cache.Tokens = r.cache.Tokens[:prefix]}- slog.Info("Cache miss", "left", len(tokens))- return nil, tokens+ slog.Info("Cache hit", "total", len(tokens), "cached", prefix, "left", len(tokens[prefix:]))+ return r.cache.Caches, tokens[prefix:]}-func (s *Runner) InsertCache(tokens []int32, caches []cache.Cache) {- current := &CacheEntry{Entries: s.CacheEntries}- for _, token := range tokens {- if _, ok := current.Entries[token]; !ok {- current.Entries[token] = &CacheEntry{- Entries: make(map[int32]*CacheEntry),- }- }-- current = current.Entries[token]+func (r *Runner) InsertCache(tokens []int32, caches []cache.Cache) {+ r.cache = &CacheEntry{+ Tokens: tokens,+ Caches: caches,}+}- if len(current.Caches) > 0 {- current.Count += 1- } else {- current.Caches = caches+func (c *CacheEntry) LogCache() {+ var totalBytes int+ for _, kv := range c.Caches {+ k, v := kv.State()+ totalBytes += k.NumBytes() + v.NumBytes()}+ logutil.Trace(fmt.Sprintf("kv cache tokens: %d, size: %s", c.Caches[0].Offset(), mlx.PrettyBytes(totalBytes)))}diff --git a/x/mlxrunner/cache/cache.go b/x/mlxrunner/cache/cache.goindex 3196b9e2a..274bdffe1 100644--- a/x/mlxrunner/cache/cache.go+++ b/x/mlxrunner/cache/cache.go@@ -13,6 +13,7 @@ type Cache interface {State() (keys, values *mlx.Array)Trim(int) intClone() Cache+ Free()Offset() intLen() int}@@ -84,6 +85,11 @@ func (c *KVCache) Clone() Cache {return clone}+func (c *KVCache) Free() {+ mlx.Unpin(c.keys, c.values)+ c.keys, c.values = nil, nil+}+func (c *KVCache) Offset() int { return c.offset }func (c *KVCache) Len() int { return c.offset }diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.goindex 618d7ec9e..e16a6c9a6 100644--- a/x/mlxrunner/pipeline.go+++ b/x/mlxrunner/pipeline.go@@ -125,6 +125,9 @@ func (r *Runner) TextGenerationPipeline(request Request) error {if slog.Default().Enabled(context.TODO(), logutil.LevelTrace) {mlx.LogArrays()+ if r.cache != nil {+ r.cache.LogCache()+ }}return nildiff --git a/x/mlxrunner/runner.go b/x/mlxrunner/runner.goindex 0b24fdb3d..effaf0847 100644--- a/x/mlxrunner/runner.go+++ b/x/mlxrunner/runner.go@@ -58,10 +58,10 @@ type Response struct {}type Runner struct {- Model base.Model- Tokenizer *tokenizer.Tokenizer- Requests chan Request- CacheEntries map[int32]*CacheEntry+ Model base.Model+ Tokenizer *tokenizer.Tokenizer+ Requests chan Request+ cache *CacheEntry}func (r *Runner) Load(modelName string) error {diff --git a/x/mlxrunner/server.go b/x/mlxrunner/server.goindex ef1e0dd1c..09b71f3c8 100644--- a/x/mlxrunner/server.go+++ b/x/mlxrunner/server.go@@ -40,8 +40,7 @@ func Execute(args []string) error {flagSet.Parse(args)runner := Runner{- Requests: make(chan Request),- CacheEntries: make(map[int32]*CacheEntry),+ Requests: make(chan Request),}if err := runner.Load(modelName); err != nil {