Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlx: compact restored recurrent state after speculative rollback (#18955)
Daniel Hiltgena5b659e61b2c
diff --git a/mlx/array_test.go b/mlx/array_test.goindex 06870e6df..5e65f6e23 100644--- a/mlx/array_test.go+++ b/mlx/array_test.go@@ -1,11 +1,56 @@package mlximport (+ "math""testing""github.com/ollama/ollama/mlx/mlxthread/mlxthreadtest")+func TestContiguousDetachesLargeView(t *testing.T) {+ withMLXThread(t, func(t *mlxthreadtest.T) {+ ClearCache()+ baseline := ActiveMemory()+ holder := NewScope()+ defer holder.Close()+ var compact *Array+ const stateElements = 16 * 128 * 128+ const stateBytes = stateElements * 4++ Scoped(func() {+ values := make([]float32, 3*stateElements)+ values[stateElements] = math.Float32frombits(0x80000000)+ values[stateElements+1] = math.Float32frombits(0x7fc01234)+ values[stateElements+2] = 1+ source := FromValues(values, 3, 1, 16, 128, 128)+ Eval(source)+ view := SliceStartStop(source,+ []int32{1, 0, 0, 0, 0}, []int32{2, 1, 16, 128, 128}).Reshape(1, 16, 128, 128)+ compact = Contiguous(view.Clone(), false)+ Eval(compact)+ holder.Attach(compact)+ })+ copied := compact.Floats()+ for i, want := range []uint32{0x80000000, 0x7fc01234, 0x3f800000} {+ if got := math.Float32bits(copied[i]); got != want {+ t.Fatalf("copied state bits[%d] = %08x, want %08x", i, got, want)+ }+ }++ // A dependent GPU operation drains any in-flight use of the source.+ // A shallow clone still keeps the full 3 MiB after this point.+ Scoped(func() {+ Eval(Add(compact, FromValues([]float32{2}, 1)))+ })+ ClearCache()+ // The compact state and dependent output can each occupy one state.+ // A shared view would also keep the three-state source buffer.+ if retained := ActiveMemory() - baseline; retained > 2*stateBytes+stateBytes/2 {+ t.Fatalf("one-state copy retained %d bytes of its 3 MiB source", retained)+ }+ })+}+func TestFromValue(t *testing.T) {withMLXThread(t, func(t *mlxthreadtest.T) {for got, want := range map[*Array]DType{diff --git a/mlxrunner/cache/recurrent.go b/mlxrunner/cache/recurrent.goindex ea8ee3980..4cf3f5a49 100644--- a/mlxrunner/cache/recurrent.go+++ b/mlxrunner/cache/recurrent.go@@ -18,10 +18,11 @@ import (// Conv state shape: [B, convTail, convDim]// Delta state shape: [B, numVHeads, headVDim, headKDim]type RecurrentCache struct {- convState *mlx.Array- deltaState *mlx.Array- scope *mlx.Scope- offset int+ convState *mlx.Array+ deltaState *mlx.Array+ scope *mlx.Scope+ offset int+ needsCompact boolconvTail intconvDim int@@ -41,7 +42,7 @@ type RecurrentCache struct {func (c *RecurrentCache) PrepareSnapshots(offsets []int) {c.snapshots.prepare(c.offset, offsets)// The current offset is a valid boundary right now, so capture it.- c.captureBoundary(c.offset, c.convState, c.deltaState)+ c.captureBoundary(c.offset, c.convState, c.deltaState, c.needsCompact)}func (c *RecurrentCache) TakeSnapshots() []Snapshot { return c.snapshots.take() }@@ -66,13 +67,13 @@ func (c *RecurrentCache) SnapshotSplits(forwardLen int) []int {// captureBoundary snapshots the boundary state (conv, delta) at reached if// reached is a scheduled offset — the live state at a forward boundary, or a// kernel segment state at an interior split.-func (c *RecurrentCache) captureBoundary(reached int, conv, delta *mlx.Array) {+func (c *RecurrentCache) captureBoundary(reached int, conv, delta *mlx.Array, mayShareBacking bool) {c.snapshots.captureReached(reached, func(int) Snapshot {// Nothing exists to page out yet; a zero-width capture is a nil entry.if conv == nil && delta == nil {return nil}- return newRecurrentSnapshot(conv, delta, reached)+ return newRecurrentSnapshot(conv, delta, reached, mayShareBacking)})}@@ -144,23 +145,29 @@ func (c *RecurrentCache) Put(b *batch.Batch, convStates, deltaStates []*mlx.Arrapanic(fmt.Sprintf("recurrent cache: %d interior splits but %d boundary states", len(splits), len(convStates)))}+ // Per-token capture returns views of one allocation holding every interior+ // delta state. With one split, a two-token capture has only one state in+ // that allocation; a longer forward computes the split as a separate output.+ // The final state is always a separate output.+ mayShareBacking := len(splits) > 1 && len(splits) == b.InputIDs.Dim(1)-1// Leading entries are the interior split boundaries; capture each as a// snapshot at its scheduled offset.for i, s := range splits {- c.captureBoundary(start+s, convStates[i], deltaStates[i])+ c.captureBoundary(start+s, convStates[i], deltaStates[i], mayShareBacking)}// The final entry is the forward-end state — the committed live state.last := len(convStates) - 1c.convState = c.setState(c.convState, convStates[last])c.deltaState = c.setState(c.deltaState, deltaStates[last])+ c.needsCompact = falsec.offset += int(b.SeqQueryLens[0])- c.captureBoundary(c.offset, c.convState, c.deltaState)+ c.captureBoundary(c.offset, c.convState, c.deltaState, false)}func (c *RecurrentCache) State() []*mlx.Array {state := []*mlx.Array{c.convState, c.deltaState}- // Captured snapshots own compact copies still needing eval; fold them into+ // Captured snapshots own array handles still needing eval; fold them into// the caller's batched State eval instead of an async eval per snapshot per// layer. They drain at TakeSnapshots.for _, s := range c.snapshots.captured {@@ -172,30 +179,50 @@ func (c *RecurrentCache) State() []*mlx.Array {return state}+// PrepareCompaction returns nil unless the live delta state may share a+// multi-state backing allocation. The caller must evaluate a returned copy+// while the source remains in scope, then commit it. A normal Put replaces+// the restored state with the kernel's final state and needs no copy.+func (c *RecurrentCache) PrepareCompaction() *mlx.Array {+ if !c.needsCompact {+ return nil+ }+ // For an oversized per-token state buffer, Contiguous materializes the+ // selected view instead of retaining that buffer.+ return mlx.Contiguous(c.deltaState, false)+}++// CommitCompaction replaces the live state only after the copy has evaluated.+func (c *RecurrentCache) CommitCompaction(compact *mlx.Array) {+ c.deltaState = c.setState(c.deltaState, compact)+ c.needsCompact = false+}+// recurrentSnapshot holds paged-out recurrent state. Self-contained —// does not depend on any parent state.type recurrentSnapshot struct {convState, deltaState *mlx.Arrayscope *mlx.Scopeoffset int+ mayShareBacking bool}func (s *recurrentSnapshot) Size() int { return s.convState.NumBytes() + s.deltaState.NumBytes() }func (s *recurrentSnapshot) Close() { s.scope.Close() }-// SetMaterializeHook is a no-op: recurrent snapshots own their compact copy from-// construction.+// SetMaterializeHook is a no-op: recurrent snapshots retain their source handles.func (s *recurrentSnapshot) SetMaterializeHook(func(int)) {}-// newRecurrentSnapshot clones conv/delta into an owned snapshot at-// offset. It does not schedule the eval — capture-path snapshots ride the-// cache's State into the caller's batched eval.-func newRecurrentSnapshot(conv, delta *mlx.Array, offset int) *recurrentSnapshot {+// newRecurrentSnapshot retains shallow conv/delta handles at offset. It does+// not schedule the eval — capture-path snapshots ride the cache's State into+// the caller's batched eval.+func newRecurrentSnapshot(conv, delta *mlx.Array, offset int, mayShareBacking bool) *recurrentSnapshot {snap := &recurrentSnapshot{- convState: conv.Clone(),- deltaState: delta.Clone(),- scope: mlx.NewScope(),- offset: offset,+ convState: conv.Clone(),+ deltaState: delta.Clone(),+ scope: mlx.NewScope(),+ offset: offset,+ mayShareBacking: mayShareBacking,}snap.scope.Attach(snap.convState, snap.deltaState)return snap@@ -209,7 +236,7 @@ func (c *RecurrentCache) Snapshot(fromOffset int) Snapshot {// Page-out snapshots go straight to the trie and never ride State, so// schedule the eval here instead of through the batched State eval.- snap := newRecurrentSnapshot(c.convState, c.deltaState, c.offset)+ snap := newRecurrentSnapshot(c.convState, c.deltaState, c.offset, c.needsCompact)mlx.AsyncEval(snap.convState, snap.deltaState)return snap}@@ -235,6 +262,7 @@ func (c *RecurrentCache) Restore(snapshot Snapshot, target int) bool {c.deltaState = c.setState(c.deltaState, snap.deltaState)})c.offset = snap.offset+ c.needsCompact = snap.mayShareBackingreturn true}@@ -257,6 +285,7 @@ func (c *RecurrentCache) Free() {c.scope.Close()c.convState, c.deltaState = nil, nilc.offset = 0+ c.needsCompact = falsec.snapshots = pendingSnapshots{}}diff --git a/mlxrunner/cache/recurrent_test.go b/mlxrunner/cache/recurrent_test.goindex 56b6d775a..3b81a5178 100644--- a/mlxrunner/cache/recurrent_test.go+++ b/mlxrunner/cache/recurrent_test.go@@ -54,6 +54,79 @@ func TestRecurrentCacheRestoreExactOffset(t *testing.T) {})}+func TestRecurrentCacheCompactsRestoredState(t *testing.T) {+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewRecurrentCache(3, 12, 4, 8, 8)+ defer c.Free()+ b := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 3), SeqQueryLens: []int32{3}}+ c.Get(b, mlx.DTypeFloat16)+ state := c.State()+ c.Put(b, []*mlx.Array{state[0]}, []*mlx.Array{state[1]})+ if got := c.PrepareCompaction(); got != nil {+ t.Fatal("normal Put should not require compaction")+ }++ compactSnap := c.Snapshot(0)+ if !c.Restore(compactSnap, 3) {+ t.Fatal("compact snapshot restore failed")+ }+ compactSnap.Close()+ if got := c.PrepareCompaction(); got != nil {+ t.Fatal("restoring a normal snapshot should not copy it")+ }++ c.PrepareSnapshots([]int{3, 4, 5})+ shared := mlx.Zeros(mlx.DTypeFloat32, 2, 1, 4, 8, 8)+ interior := make([]*mlx.Array, 2)+ for i := range interior {+ interior[i] = mlx.SliceStartStop(shared,+ []int32{int32(i), 0, 0, 0, 0},+ []int32{int32(i + 1), 1, 4, 8, 8}).Reshape(1, 4, 8, 8)+ }+ conv := []*mlx.Array{+ mlx.Zeros(mlx.DTypeFloat16, 1, 3, 12),+ mlx.Zeros(mlx.DTypeFloat16, 1, 3, 12),+ mlx.Zeros(mlx.DTypeFloat16, 1, 3, 12),+ }+ delta := []*mlx.Array{interior[0], interior[1], mlx.Zeros(mlx.DTypeFloat32, 1, 4, 8, 8)}+ c.Put(b, conv, delta)+ snaps := c.TakeSnapshots()+ if got := c.PrepareCompaction(); got != nil {+ t.Fatal("the final kernel state should not require compaction")+ }+ if !c.Restore(snaps[1], 4) {+ t.Fatal("interior snapshot restore failed")+ }+ mlx.Scoped(func() {+ compact := c.PrepareCompaction()+ if compact == nil {+ t.Fatal("restored state was not compacted")+ }+ mlx.Eval(compact)+ c.CommitCompaction(compact)+ if got := c.PrepareCompaction(); got != nil {+ t.Fatal("state was compacted twice")+ }+ })+ if got := c.Offset(); got != 4 {+ t.Fatalf("offset after compaction = %d, want 4", got)+ }+ if !c.Restore(snaps[1], 4) {+ t.Fatal("second restore failed")+ }+ state = c.State()+ c.Put(b, []*mlx.Array{state[0]}, []*mlx.Array{state[1]})+ if got := c.PrepareCompaction(); got != nil {+ t.Fatal("Put should replace the restored state without compaction")+ }+ for _, snap := range snaps {+ if snap != nil {+ snap.Close()+ }+ }+ })+}+func TestRecurrentCacheGetLazyInit(t *testing.T) {mlxtest.Run(t, func(t *mlxtest.T) {c := NewRecurrentCache(3, 4, 2, 4, 4)diff --git a/mlxrunner/prefix_cache.go b/mlxrunner/prefix_cache.goindex 02259ba63..9df9acb35 100644--- a/mlxrunner/prefix_cache.go+++ b/mlxrunner/prefix_cache.go@@ -588,6 +588,34 @@ func (s *cacheSession) close() {// PrepareSnapshots would otherwise overwrite, leaking them.s.attachPrefillSnapshots()+ // A speculative rollback may leave the selected recurrent state as a view+ // of the whole per-token capture buffer. Copy only states that still need+ // detaching at request close, after decoding has settled and before the+ // trie can retain a snapshot of them. Evaluate as one batch while the+ // source handles remain in scope.+ mlx.Scoped(func() {+ type prepared struct {+ cache *cache.RecurrentCache+ state *mlx.Array+ }+ var copies []prepared+ var arrays []*mlx.Array+ for _, kv := range s.caches {+ if recurrent, ok := kv.(*cache.RecurrentCache); ok {+ if state := recurrent.PrepareCompaction(); state != nil {+ copies = append(copies, prepared{recurrent, state})+ arrays = append(arrays, state)+ }+ }+ }+ if len(arrays) > 0 {+ mlx.Eval(arrays...)+ for _, copy := range copies {+ copy.cache.CommitCompaction(copy.state)+ }+ }+ })+offset := s.cache.minCacheOffset()if offset <= 0 {returndiff --git a/mlxrunner/prefix_cache_test.go b/mlxrunner/prefix_cache_test.goindex e163b0370..0ab18211d 100644--- a/mlxrunner/prefix_cache_test.go+++ b/mlxrunner/prefix_cache_test.go@@ -6,10 +6,48 @@ import ("time""github.com/ollama/ollama/mlx"+ "github.com/ollama/ollama/mlx/mlxtest"+ "github.com/ollama/ollama/mlxrunner/batch""github.com/ollama/ollama/mlxrunner/cache""github.com/ollama/ollama/mlxrunner/model")+func TestSessionCloseCompactsRestoredRecurrentStateWithoutNewFrontier(t *testing.T) {+ mlxtest.Run(t, func(t *mlxtest.T) {+ recurrent := cache.NewRecurrentCache(3, 12, 4, 8, 8)+ defer recurrent.Free()+ b := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 1), SeqQueryLens: []int32{1}}+ recurrent.Get(b, mlx.DTypeFloat16)+ recurrent.PrepareSnapshots([]int{0, 1, 2})+ forward := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 3), SeqQueryLens: []int32{3}}+ conv := mlx.Zeros(mlx.DTypeFloat16, 1, 3, 12)+ delta := mlx.Zeros(mlx.DTypeFloat32, 1, 4, 8, 8)+ recurrent.Put(forward, []*mlx.Array{conv, conv, conv}, []*mlx.Array{delta, delta, delta})+ snaps := recurrent.TakeSnapshots()+ if !recurrent.Restore(snaps[1], 1) {+ t.Fatal("interior restore failed")+ }+ for _, snap := range snaps {+ if snap != nil {+ snap.Close()+ }+ }+ if state := recurrent.PrepareCompaction(); state == nil {+ t.Fatal("restored interior state should need compaction before close")+ }++ pc := newPrefixCache([]cache.Cache{recurrent})+ pc.ensureRoot()+ frontier := pc.root.appendChild([]trieKey{7}, 1)+ pc.activePath = []*trieNode{pc.root, frontier}+ session := &cacheSession{cache: pc, caches: pc.caches, effInputs: []uint32{7}}+ session.close()+ if state := recurrent.PrepareCompaction(); state != nil {+ t.Fatal("restored state was not compacted when the frontier did not advance")+ }+ })+}+// snapshotTracker records every fakeSnapshot created and every Close() call// so tests can detect leaked (created but never closed) or double-closed snapshots.type snapshotTracker struct {