Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

mlx: compact restored recurrent state after speculative rollback (#18955)

Daniel Hiltgena5b659e61b2c
diff --git a/mlx/array_test.go b/mlx/array_test.go
index 06870e6df..5e65f6e23 100644
--- a/mlx/array_test.go
+++ b/mlx/array_test.go
@@ -1,11 +1,56 @@
package mlx
import (
+ "math"
"testing"
"github.com/ollama/ollama/mlx/mlxthread/mlxthreadtest"
)
+func TestContiguousDetachesLargeView(t *testing.T) {
+ withMLXThread(t, func(t *mlxthreadtest.T) {
+ ClearCache()
+ baseline := ActiveMemory()
+ holder := NewScope()
+ defer holder.Close()
+ var compact *Array
+ const stateElements = 16 * 128 * 128
+ const stateBytes = stateElements * 4
+
+ Scoped(func() {
+ values := make([]float32, 3*stateElements)
+ values[stateElements] = math.Float32frombits(0x80000000)
+ values[stateElements+1] = math.Float32frombits(0x7fc01234)
+ values[stateElements+2] = 1
+ source := FromValues(values, 3, 1, 16, 128, 128)
+ Eval(source)
+ view := SliceStartStop(source,
+ []int32{1, 0, 0, 0, 0}, []int32{2, 1, 16, 128, 128}).Reshape(1, 16, 128, 128)
+ compact = Contiguous(view.Clone(), false)
+ Eval(compact)
+ holder.Attach(compact)
+ })
+ copied := compact.Floats()
+ for i, want := range []uint32{0x80000000, 0x7fc01234, 0x3f800000} {
+ if got := math.Float32bits(copied[i]); got != want {
+ t.Fatalf("copied state bits[%d] = %08x, want %08x", i, got, want)
+ }
+ }
+
+ // A dependent GPU operation drains any in-flight use of the source.
+ // A shallow clone still keeps the full 3 MiB after this point.
+ Scoped(func() {
+ Eval(Add(compact, FromValues([]float32{2}, 1)))
+ })
+ ClearCache()
+ // The compact state and dependent output can each occupy one state.
+ // A shared view would also keep the three-state source buffer.
+ if retained := ActiveMemory() - baseline; retained > 2*stateBytes+stateBytes/2 {
+ t.Fatalf("one-state copy retained %d bytes of its 3 MiB source", retained)
+ }
+ })
+}
+
func TestFromValue(t *testing.T) {
withMLXThread(t, func(t *mlxthreadtest.T) {
for got, want := range map[*Array]DType{
diff --git a/mlxrunner/cache/recurrent.go b/mlxrunner/cache/recurrent.go
index ea8ee3980..4cf3f5a49 100644
--- a/mlxrunner/cache/recurrent.go
+++ b/mlxrunner/cache/recurrent.go
@@ -18,10 +18,11 @@ import (
// Conv state shape: [B, convTail, convDim]
// Delta state shape: [B, numVHeads, headVDim, headKDim]
type RecurrentCache struct {
- convState *mlx.Array
- deltaState *mlx.Array
- scope *mlx.Scope
- offset int
+ convState *mlx.Array
+ deltaState *mlx.Array
+ scope *mlx.Scope
+ offset int
+ needsCompact bool
convTail int
convDim int
@@ -41,7 +42,7 @@ type RecurrentCache struct {
func (c *RecurrentCache) PrepareSnapshots(offsets []int) {
c.snapshots.prepare(c.offset, offsets)
// The current offset is a valid boundary right now, so capture it.
- c.captureBoundary(c.offset, c.convState, c.deltaState)
+ c.captureBoundary(c.offset, c.convState, c.deltaState, c.needsCompact)
}
func (c *RecurrentCache) TakeSnapshots() []Snapshot { return c.snapshots.take() }
@@ -66,13 +67,13 @@ func (c *RecurrentCache) SnapshotSplits(forwardLen int) []int {
// captureBoundary snapshots the boundary state (conv, delta) at reached if
// reached is a scheduled offset — the live state at a forward boundary, or a
// kernel segment state at an interior split.
-func (c *RecurrentCache) captureBoundary(reached int, conv, delta *mlx.Array) {
+func (c *RecurrentCache) captureBoundary(reached int, conv, delta *mlx.Array, mayShareBacking bool) {
c.snapshots.captureReached(reached, func(int) Snapshot {
// Nothing exists to page out yet; a zero-width capture is a nil entry.
if conv == nil && delta == nil {
return nil
}
- return newRecurrentSnapshot(conv, delta, reached)
+ return newRecurrentSnapshot(conv, delta, reached, mayShareBacking)
})
}
@@ -144,23 +145,29 @@ func (c *RecurrentCache) Put(b *batch.Batch, convStates, deltaStates []*mlx.Arra
panic(fmt.Sprintf("recurrent cache: %d interior splits but %d boundary states", len(splits), len(convStates)))
}
+ // Per-token capture returns views of one allocation holding every interior
+ // delta state. With one split, a two-token capture has only one state in
+ // that allocation; a longer forward computes the split as a separate output.
+ // The final state is always a separate output.
+ mayShareBacking := len(splits) > 1 && len(splits) == b.InputIDs.Dim(1)-1
// Leading entries are the interior split boundaries; capture each as a
// snapshot at its scheduled offset.
for i, s := range splits {
- c.captureBoundary(start+s, convStates[i], deltaStates[i])
+ c.captureBoundary(start+s, convStates[i], deltaStates[i], mayShareBacking)
}
// The final entry is the forward-end state — the committed live state.
last := len(convStates) - 1
c.convState = c.setState(c.convState, convStates[last])
c.deltaState = c.setState(c.deltaState, deltaStates[last])
+ c.needsCompact = false
c.offset += int(b.SeqQueryLens[0])
- c.captureBoundary(c.offset, c.convState, c.deltaState)
+ c.captureBoundary(c.offset, c.convState, c.deltaState, false)
}
func (c *RecurrentCache) State() []*mlx.Array {
state := []*mlx.Array{c.convState, c.deltaState}
- // Captured snapshots own compact copies still needing eval; fold them into
+ // Captured snapshots own array handles still needing eval; fold them into
// the caller's batched State eval instead of an async eval per snapshot per
// layer. They drain at TakeSnapshots.
for _, s := range c.snapshots.captured {
@@ -172,30 +179,50 @@ func (c *RecurrentCache) State() []*mlx.Array {
return state
}
+// PrepareCompaction returns nil unless the live delta state may share a
+// multi-state backing allocation. The caller must evaluate a returned copy
+// while the source remains in scope, then commit it. A normal Put replaces
+// the restored state with the kernel's final state and needs no copy.
+func (c *RecurrentCache) PrepareCompaction() *mlx.Array {
+ if !c.needsCompact {
+ return nil
+ }
+ // For an oversized per-token state buffer, Contiguous materializes the
+ // selected view instead of retaining that buffer.
+ return mlx.Contiguous(c.deltaState, false)
+}
+
+// CommitCompaction replaces the live state only after the copy has evaluated.
+func (c *RecurrentCache) CommitCompaction(compact *mlx.Array) {
+ c.deltaState = c.setState(c.deltaState, compact)
+ c.needsCompact = false
+}
+
// recurrentSnapshot holds paged-out recurrent state. Self-contained —
// does not depend on any parent state.
type recurrentSnapshot struct {
convState, deltaState *mlx.Array
scope *mlx.Scope
offset int
+ mayShareBacking bool
}
func (s *recurrentSnapshot) Size() int { return s.convState.NumBytes() + s.deltaState.NumBytes() }
func (s *recurrentSnapshot) Close() { s.scope.Close() }
-// SetMaterializeHook is a no-op: recurrent snapshots own their compact copy from
-// construction.
+// SetMaterializeHook is a no-op: recurrent snapshots retain their source handles.
func (s *recurrentSnapshot) SetMaterializeHook(func(int)) {}
-// newRecurrentSnapshot clones conv/delta into an owned snapshot at
-// offset. It does not schedule the eval — capture-path snapshots ride the
-// cache's State into the caller's batched eval.
-func newRecurrentSnapshot(conv, delta *mlx.Array, offset int) *recurrentSnapshot {
+// newRecurrentSnapshot retains shallow conv/delta handles at offset. It does
+// not schedule the eval — capture-path snapshots ride the cache's State into
+// the caller's batched eval.
+func newRecurrentSnapshot(conv, delta *mlx.Array, offset int, mayShareBacking bool) *recurrentSnapshot {
snap := &recurrentSnapshot{
- convState: conv.Clone(),
- deltaState: delta.Clone(),
- scope: mlx.NewScope(),
- offset: offset,
+ convState: conv.Clone(),
+ deltaState: delta.Clone(),
+ scope: mlx.NewScope(),
+ offset: offset,
+ mayShareBacking: mayShareBacking,
}
snap.scope.Attach(snap.convState, snap.deltaState)
return snap
@@ -209,7 +236,7 @@ func (c *RecurrentCache) Snapshot(fromOffset int) Snapshot {
// Page-out snapshots go straight to the trie and never ride State, so
// schedule the eval here instead of through the batched State eval.
- snap := newRecurrentSnapshot(c.convState, c.deltaState, c.offset)
+ snap := newRecurrentSnapshot(c.convState, c.deltaState, c.offset, c.needsCompact)
mlx.AsyncEval(snap.convState, snap.deltaState)
return snap
}
@@ -235,6 +262,7 @@ func (c *RecurrentCache) Restore(snapshot Snapshot, target int) bool {
c.deltaState = c.setState(c.deltaState, snap.deltaState)
})
c.offset = snap.offset
+ c.needsCompact = snap.mayShareBacking
return true
}
@@ -257,6 +285,7 @@ func (c *RecurrentCache) Free() {
c.scope.Close()
c.convState, c.deltaState = nil, nil
c.offset = 0
+ c.needsCompact = false
c.snapshots = pendingSnapshots{}
}
diff --git a/mlxrunner/cache/recurrent_test.go b/mlxrunner/cache/recurrent_test.go
index 56b6d775a..3b81a5178 100644
--- a/mlxrunner/cache/recurrent_test.go
+++ b/mlxrunner/cache/recurrent_test.go
@@ -54,6 +54,79 @@ func TestRecurrentCacheRestoreExactOffset(t *testing.T) {
})
}
+func TestRecurrentCacheCompactsRestoredState(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ c := NewRecurrentCache(3, 12, 4, 8, 8)
+ defer c.Free()
+ b := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 3), SeqQueryLens: []int32{3}}
+ c.Get(b, mlx.DTypeFloat16)
+ state := c.State()
+ c.Put(b, []*mlx.Array{state[0]}, []*mlx.Array{state[1]})
+ if got := c.PrepareCompaction(); got != nil {
+ t.Fatal("normal Put should not require compaction")
+ }
+
+ compactSnap := c.Snapshot(0)
+ if !c.Restore(compactSnap, 3) {
+ t.Fatal("compact snapshot restore failed")
+ }
+ compactSnap.Close()
+ if got := c.PrepareCompaction(); got != nil {
+ t.Fatal("restoring a normal snapshot should not copy it")
+ }
+
+ c.PrepareSnapshots([]int{3, 4, 5})
+ shared := mlx.Zeros(mlx.DTypeFloat32, 2, 1, 4, 8, 8)
+ interior := make([]*mlx.Array, 2)
+ for i := range interior {
+ interior[i] = mlx.SliceStartStop(shared,
+ []int32{int32(i), 0, 0, 0, 0},
+ []int32{int32(i + 1), 1, 4, 8, 8}).Reshape(1, 4, 8, 8)
+ }
+ conv := []*mlx.Array{
+ mlx.Zeros(mlx.DTypeFloat16, 1, 3, 12),
+ mlx.Zeros(mlx.DTypeFloat16, 1, 3, 12),
+ mlx.Zeros(mlx.DTypeFloat16, 1, 3, 12),
+ }
+ delta := []*mlx.Array{interior[0], interior[1], mlx.Zeros(mlx.DTypeFloat32, 1, 4, 8, 8)}
+ c.Put(b, conv, delta)
+ snaps := c.TakeSnapshots()
+ if got := c.PrepareCompaction(); got != nil {
+ t.Fatal("the final kernel state should not require compaction")
+ }
+ if !c.Restore(snaps[1], 4) {
+ t.Fatal("interior snapshot restore failed")
+ }
+ mlx.Scoped(func() {
+ compact := c.PrepareCompaction()
+ if compact == nil {
+ t.Fatal("restored state was not compacted")
+ }
+ mlx.Eval(compact)
+ c.CommitCompaction(compact)
+ if got := c.PrepareCompaction(); got != nil {
+ t.Fatal("state was compacted twice")
+ }
+ })
+ if got := c.Offset(); got != 4 {
+ t.Fatalf("offset after compaction = %d, want 4", got)
+ }
+ if !c.Restore(snaps[1], 4) {
+ t.Fatal("second restore failed")
+ }
+ state = c.State()
+ c.Put(b, []*mlx.Array{state[0]}, []*mlx.Array{state[1]})
+ if got := c.PrepareCompaction(); got != nil {
+ t.Fatal("Put should replace the restored state without compaction")
+ }
+ for _, snap := range snaps {
+ if snap != nil {
+ snap.Close()
+ }
+ }
+ })
+}
+
func TestRecurrentCacheGetLazyInit(t *testing.T) {
mlxtest.Run(t, func(t *mlxtest.T) {
c := NewRecurrentCache(3, 4, 2, 4, 4)
diff --git a/mlxrunner/prefix_cache.go b/mlxrunner/prefix_cache.go
index 02259ba63..9df9acb35 100644
--- a/mlxrunner/prefix_cache.go
+++ b/mlxrunner/prefix_cache.go
@@ -588,6 +588,34 @@ func (s *cacheSession) close() {
// PrepareSnapshots would otherwise overwrite, leaking them.
s.attachPrefillSnapshots()
+ // A speculative rollback may leave the selected recurrent state as a view
+ // of the whole per-token capture buffer. Copy only states that still need
+ // detaching at request close, after decoding has settled and before the
+ // trie can retain a snapshot of them. Evaluate as one batch while the
+ // source handles remain in scope.
+ mlx.Scoped(func() {
+ type prepared struct {
+ cache *cache.RecurrentCache
+ state *mlx.Array
+ }
+ var copies []prepared
+ var arrays []*mlx.Array
+ for _, kv := range s.caches {
+ if recurrent, ok := kv.(*cache.RecurrentCache); ok {
+ if state := recurrent.PrepareCompaction(); state != nil {
+ copies = append(copies, prepared{recurrent, state})
+ arrays = append(arrays, state)
+ }
+ }
+ }
+ if len(arrays) > 0 {
+ mlx.Eval(arrays...)
+ for _, copy := range copies {
+ copy.cache.CommitCompaction(copy.state)
+ }
+ }
+ })
+
offset := s.cache.minCacheOffset()
if offset <= 0 {
return
diff --git a/mlxrunner/prefix_cache_test.go b/mlxrunner/prefix_cache_test.go
index e163b0370..0ab18211d 100644
--- a/mlxrunner/prefix_cache_test.go
+++ b/mlxrunner/prefix_cache_test.go
@@ -6,10 +6,48 @@ import (
"time"
"github.com/ollama/ollama/mlx"
+ "github.com/ollama/ollama/mlx/mlxtest"
+ "github.com/ollama/ollama/mlxrunner/batch"
"github.com/ollama/ollama/mlxrunner/cache"
"github.com/ollama/ollama/mlxrunner/model"
)
+func TestSessionCloseCompactsRestoredRecurrentStateWithoutNewFrontier(t *testing.T) {
+ mlxtest.Run(t, func(t *mlxtest.T) {
+ recurrent := cache.NewRecurrentCache(3, 12, 4, 8, 8)
+ defer recurrent.Free()
+ b := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 1), SeqQueryLens: []int32{1}}
+ recurrent.Get(b, mlx.DTypeFloat16)
+ recurrent.PrepareSnapshots([]int{0, 1, 2})
+ forward := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 3), SeqQueryLens: []int32{3}}
+ conv := mlx.Zeros(mlx.DTypeFloat16, 1, 3, 12)
+ delta := mlx.Zeros(mlx.DTypeFloat32, 1, 4, 8, 8)
+ recurrent.Put(forward, []*mlx.Array{conv, conv, conv}, []*mlx.Array{delta, delta, delta})
+ snaps := recurrent.TakeSnapshots()
+ if !recurrent.Restore(snaps[1], 1) {
+ t.Fatal("interior restore failed")
+ }
+ for _, snap := range snaps {
+ if snap != nil {
+ snap.Close()
+ }
+ }
+ if state := recurrent.PrepareCompaction(); state == nil {
+ t.Fatal("restored interior state should need compaction before close")
+ }
+
+ pc := newPrefixCache([]cache.Cache{recurrent})
+ pc.ensureRoot()
+ frontier := pc.root.appendChild([]trieKey{7}, 1)
+ pc.activePath = []*trieNode{pc.root, frontier}
+ session := &cacheSession{cache: pc, caches: pc.caches, effInputs: []uint32{7}}
+ session.close()
+ if state := recurrent.PrepareCompaction(); state != nil {
+ t.Fatal("restored state was not compacted when the frontier did not advance")
+ }
+ })
+}
+
// snapshotTracker records every fakeSnapshot created and every Close() call
// so tests can detect leaked (created but never closed) or double-closed snapshots.
type snapshotTracker struct {