Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlxrunner: support draft heads that maintain draft caches
Jesse Gross28fbbb06d56a
diff --git a/x/mlxrunner/batch/batch.go b/x/mlxrunner/batch/batch.goindex 409f53076..011b31103 100644--- a/x/mlxrunner/batch/batch.go+++ b/x/mlxrunner/batch/batch.go@@ -17,6 +17,10 @@ type Batch struct {// Length equals the batch dimension of InputIDs.SeqQueryLens []int32+ // Hidden is the target hidden state a draft model fuses with its input+ // embedding for this step. It is nil for ordinary forward passes.+ Hidden *mlx.Array+// Memo is per-forward memoization used to cache results, such as masks,// which are often the same across layers.Memo Memodiff --git a/x/mlxrunner/model/base/base.go b/x/mlxrunner/model/base/base.goindex 4f43d74b6..9a0569561 100644--- a/x/mlxrunner/model/base/base.go+++ b/x/mlxrunner/model/base/base.go@@ -27,8 +27,21 @@ type Model interface {LoadWeights(tensors map[string]*mlx.Array) error}-// DraftModel is an auxiliary model stored alongside a target model.+// DraftModel is an auxiliary model alongside a target that proposes speculative+// tokens.type DraftModel interface {+ // Draft fuses b.Hidden (the target hidden state) into its own forward and+ // returns the head's hidden plus the projected hidden seeding the next step.+ Draft(b *batch.Batch, caches []cache.Cache) (hidden, projected *mlx.Array)++ // Unembed projects a hidden state to vocabulary logits.+ Unembed(x *mlx.Array) *mlx.Array++ // DraftCaches selects the draft model's own KV caches from the full+ // per-request slice — any subset, or nil when the draft keeps no KV.+ DraftCaches(caches []cache.Cache) []cache.Cache++ // LoadWeights assigns manifest tensors to the draft head's fields.LoadWeights(tensors map[string]*mlx.Array) error}@@ -46,15 +59,10 @@ type MTPDefaultsProvider interface {MTPDraftDefaults(sample bool) MTPDefaults}-// MTPDraftModel is a draft model capable of Gemma-style multi-token-// prediction from target token embeddings, target hidden states, and target KV.-type MTPDraftModel interface {- Draft(inputEmbeds *mlx.Array, position int32, caches []cache.Cache) (logits, hidden *mlx.Array)-}--// MTPEmbeddingModel exposes the target token embedding path used by MTP drafts.-type MTPEmbeddingModel interface {- TokenEmbeddings(inputIDs *mlx.Array) *mlx.Array+// SelfDraft is implemented by models whose draft head ships inline with the+// target weights; it returns the head, or nil when the checkpoint shipped none.+type SelfDraft interface {+ SelfDraft() DraftModel}var (diff --git a/x/mlxrunner/mtp.go b/x/mlxrunner/mtp.goindex 52381f921..ba788d82d 100644--- a/x/mlxrunner/mtp.go+++ b/x/mlxrunner/mtp.go@@ -1,7 +1,9 @@package mlxrunnerimport (- "github.com/ollama/ollama/x/mlxrunner/cache"+ "fmt"++ "github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/mlx""github.com/ollama/ollama/x/mlxrunner/model/base"sampler "github.com/ollama/ollama/x/mlxrunner/sample"@@ -12,6 +14,11 @@ const (mtpDefaultMaxDraftTokens = 16)+// mtpPendingFlushTokens caps how many committed look-ahead tokens wait in the+// pending buffer before a batched flush, bounding the pinned hidden states+// regardless of what else triggers a flush.+const mtpPendingFlushTokens = 32+func (r *Runner) mtpDefaults(sample bool) base.MTPDefaults {defaults := base.MTPDefaults{InitialDraftTokens: mtpDefaultInitialDraftTokens,@@ -30,85 +37,231 @@ func (r *Runner) mtpDefaults(sample bool) base.MTPDefaults {return defaults}-// mtpDrafter drafts with a model's multi-token-prediction head: a small-// head trained to continue the target's hidden states, fed through the-// committed-stream reports. It retains the hidden at the last committed-// slot and tracks the committed frontier, which anchors its drafting.+// mtpDrafter drafts with a model's multi-token-prediction head, fed through+// the committed-stream reports. The draft KV pairs each slot S with the+// look-ahead token at S+1 fused with the target hidden at S, so a pair+// completes only when the next token arrives.type mtpDrafter struct {- spec *speculation- draft base.MTPDraftModel- target base.MTPEmbeddingModel- caches []cache.Cache-- // frontier is the slot after the last committed token; the hidden- // reported for slot frontier-1 is retained (pinned) as the fusion- // input for the next proposal chain.+ spec *speculation++ // frontier is the slot after the last reported token; frontierHidden is+ // the pinned target hidden at frontier-1, fused into the next pair.frontier intfrontierHidden *mlx.Array++ // committedDraftOffset is the slot after the last pair written to the+ // draft caches; later pairs wait pinned in the pending lists until+ // flushed. pendingCount is the look-ahead tokens those lists hold, summed+ // across the buffered runs.+ committedDraftOffset int+ pendingTokens []*mlx.Array+ pendingHiddens []*mlx.Array+ pendingCount int++ // heldHidden is the frontier row's pre-unembed hidden and heldProjected+ // its fusion hidden, carried from the last flush so the first proposal+ // reuses them without a head call.+ heldHidden *mlx.Array+ heldProjected *mlx.Array}-// newMTPDrafter returns the MTP drafter for this request's caches, or nil-// when the model carries no MTP head.-func newMTPDrafter(s *speculation, caches []cache.Cache) *mtpDrafter {- draft, ok := s.draft.(base.MTPDraftModel)- if !ok {- return nil+// newMTPDrafter returns the MTP drafter cursor for this request, syncing its+// pairing frontier to the draft caches' restored offset.+func newMTPDrafter(s *speculation) *mtpDrafter {+ d := &mtpDrafter{spec: s}+ if len(s.draftKV) > 0 {+ // A restored prefix arrives with the draft caches already written;+ // pairing resumes from their absolute offset.+ d.committedDraftOffset = s.draftKV[0].Offset()+ d.frontier = d.committedDraftOffset}- target, ok := s.r.Model.(base.MTPEmbeddingModel)- if !ok {- return nil- }- return &mtpDrafter{spec: s, draft: draft, target: target, caches: caches}+ return d}func (d *mtpDrafter) committed(tokens, hiddens *mlx.Array, position int) {- d.frontier = position + tokens.Dim(1)- h := lastHiddenRow(hiddens)- mlx.Pin(h)- if d.frontierHidden != nil {- mlx.Unpin(d.frontierHidden)+ n := tokens.Dim(1)+ if len(d.spec.draftKV) > 0 {+ // The pair at slot S fuses token[S+1] with hidden[S], so a run pairs its+ // tokens with its own hiddens shifted one slot back: the first writable+ // token takes the carried frontier hidden, each later token the row+ // before it. Leading tokens whose slot is already buffered or written+ // through (a proposal consumed the run's first token, or a restored+ // prefix sits at the run start) are skipped; a slot below the frontier+ // is a gap bug.+ start := d.committedDraftOffset + d.pendingCount - position + 1+ if start < 0 {+ panic(fmt.Sprintf("mtp: committed run at %d leaves a pair gap at %d", position, d.committedDraftOffset+d.pendingCount))+ }+ if start < n {+ ids := tokens.Slice(mlx.Slice(), mlx.Slice(start, n))+ var h *mlx.Array+ if start == 0 {+ h = mlx.Concatenate([]*mlx.Array{d.frontierHidden, hiddens.Slice(mlx.Slice(), mlx.Slice(0, n-1), mlx.Slice())}, 1)+ } else {+ h = hiddens.Slice(mlx.Slice(), mlx.Slice(start-1, n-1), mlx.Slice())+ }+ d.queueCacheWrites(ids, h)+ }}- d.frontierHidden = h++ d.frontier = position + n+ d.setFrontierHidden(lastHiddenRow(hiddens))+}++// finish settles the drafter when generation ends: current completes the+// frontier pair, leveling the draft caches with the target's resting offset.+//+// TODO: leveling the draft to the target writes a boundary entry whose+// look-ahead token is outside the stored prefix (here, the never-committed+// current). When a later request restores this prefix and diverges at the+// boundary, that entry is stale and lowers draft acceptance. EAGLE keeps the+// draft one slot behind the target so the unconfirmed boundary entry is never+// written (its bigram partner token[S+1] does not exist yet); we should do the+// same rather than level here. Regenerating hidden[S] to re-pair the boundary+// on the next request is a separate, re-prefill-bound concern for recurrent+// targets.+func (d *mtpDrafter) finish(current *mlx.Array) {+ if len(d.spec.draftKV) == 0 {+ return+ }+ d.settle(current)+}++// settle completes any open frontier pair with current, then flushes.+func (d *mtpDrafter) settle(current *mlx.Array) {+ if d.frontierHidden != nil && d.frontier-1 == d.committedDraftOffset+d.pendingCount {+ d.queueCacheWrites(current.ExpandDims(-1), d.frontierHidden)+ }+ d.flush()}func (d *mtpDrafter) close() {- if d.frontierHidden != nil {- mlx.Unpin(d.frontierHidden)- d.frontierHidden = nil+ d.flush()+ d.setFrontierHidden(nil)+ d.setHeld(nil, nil)+}++// queueCacheWrites buffers completed draft-cache writes — look-ahead tokens+// fused with their target hiddens — flushing once the buffer reaches the token+// cap so the pinned hiddens stay bounded. flush coalesces the buffered writes+// into one head forward, so a contiguous run lands in a single draft-cache extend.+func (d *mtpDrafter) queueCacheWrites(tokens, hiddens *mlx.Array) {+ mlx.Pin(tokens, hiddens)+ d.pendingTokens = append(d.pendingTokens, tokens)+ d.pendingHiddens = append(d.pendingHiddens, hiddens)+ d.pendingCount += tokens.Dim(1)+ if d.pendingCount >= mtpPendingFlushTokens {+ d.flush()}}-// propose drafts a token chain Gemma-style ("single-position"): the head is-// trained to draft every speculative token as if it sat at the last-// committed slot, re-attending the target caches read-only. Each step fuses-// the previous token's target embedding with the hidden chain — the-// reported hidden first, then the head's own projections — and the-// RoPE/cache anchor stays at the last committed slot while the proposed-// tokens advance.+// flush writes the pending pairs to the draft caches in one head forward,+// dropping speculative entries past the committed range first and holding+// the last row's logits and projected hidden for the next proposal chain.+func (d *mtpDrafter) flush() {+ if len(d.pendingTokens) == 0 {+ return+ }+ for _, c := range d.spec.draftKV {+ if c.Offset() > d.committedDraftOffset {+ if !c.Restore(nil, d.committedDraftOffset) {+ panic(fmt.Sprintf("mtp: draft cache rewind to %d failed", d.committedDraftOffset))+ }+ }+ }++ ids := mlx.Concatenate(d.pendingTokens, 1)+ hiddens := mlx.Concatenate(d.pendingHiddens, 1)+ hidden, projected := d.spec.draft.Draft(&batch.Batch{+ InputIDs: ids,+ SeqOffsets: []int32{int32(d.committedDraftOffset)},+ SeqQueryLens: []int32{int32(ids.Dim(1))},+ Hidden: hiddens,+ }, d.spec.caches)+ d.setHeld(lastHiddenRow(hidden), lastHiddenRow(projected))+ d.committedDraftOffset += ids.Dim(1)++ // Force the draft writes: a session that never drafts would otherwise+ // leave the flush chain unevaluated, pinning every hidden until close.+ state := make([]*mlx.Array, 0, 2*len(d.spec.draftKV))+ for _, c := range d.spec.draftKV {+ state = append(state, c.State()...)+ }+ mlx.AsyncEval(state...)++ mlx.Unpin(d.pendingTokens...)+ mlx.Unpin(d.pendingHiddens...)+ d.pendingTokens, d.pendingHiddens = nil, nil+ d.pendingCount = 0+}++func (d *mtpDrafter) setFrontierHidden(h *mlx.Array) {+ mlx.Pin(h)+ mlx.Unpin(d.frontierHidden)+ d.frontierHidden = h+}++// setHeld replaces the held flush outputs, pinned until the next flush or close.+func (d *mtpDrafter) setHeld(hidden, projected *mlx.Array) {+ mlx.Pin(hidden, projected)+ mlx.Unpin(d.heldHidden, d.heldProjected)+ d.heldHidden, d.heldProjected = hidden, projected+}++// propose drafts a token chain after the not-yet-validated current token.+// A head with draft caches settles the frontier pair first, so its first step+// reuses the held frontier row with no head call; a cacheless head re-attends+// the target caches read-only, anchored at the last committed slot.func (d *mtpDrafter) propose(current *mlx.Array, maxTokens int) *draftCandidates {if maxTokens <= 0 || d.frontierHidden == nil {return nil}r := d.spec.r- anchor := int32(d.frontier - 1)+ if len(d.spec.draftKV) > 0 {+ d.settle(current)+ if d.heldHidden == nil {+ return nil+ }+ }+lastToken := current.ExpandDims(-1)lastHidden := d.frontierHidden- draftTokens := make([]*mlx.Array, 0, maxTokens)draftDists := make([]sampler.Distribution, 0, maxTokens)var prefix *mlx.Array- for range maxTokens {- tokenEmbedding := d.target.TokenEmbeddings(lastToken)- inputs := tokenEmbedding.Concatenate(-1, lastHidden)- logits, projected := d.draft.Draft(inputs, anchor, d.caches)- stepLogits := lastLogits(logits)+ for i := range maxTokens {+ var hidden, projected *mlx.Array+ if i == 0 && len(d.spec.draftKV) > 0 {+ // The settle flush already produced the frontier row; reuse it+ // instead of re-running the head.+ hidden, projected = d.heldHidden, d.heldProjected+ } else {+ // A head with draft caches writes each draft token to the next+ // draft-cache slot, advancing one per step from the last committed+ // slot (the held i==0 step stands in for that slot). A cacheless+ // head stays at the last committed slot every step, re-attending+ // the committed prefix read-only ("single-position").+ pos := d.frontier - 1+ if len(d.spec.draftKV) > 0 {+ pos = d.frontier - 1 + i+ }+ hidden, projected = d.spec.draft.Draft(&batch.Batch{+ InputIDs: lastToken,+ SeqOffsets: []int32{int32(pos)},+ SeqQueryLens: []int32{1},+ Hidden: lastHidden,+ }, d.spec.caches)+ }+ // Unembed only the row being sampled, never the batch.+ stepLogits := d.spec.draft.Unembed(hidden).Squeeze(1)+ lastHidden = projected+ // The chain's earlier drafts ride along as the row's history, so+ // penalties shape proposals the same way they shape validation.dist := r.Sampler.Distribution(pipelineSlot, stepLogits, prefix)nextToken := r.Sampler.SampleDistribution(pipelineSlot, dist)lastToken = nextToken.ExpandDims(-1)- lastHidden = projected- draftTokens = append(draftTokens, lastToken)draftDists = append(draftDists, dist)if prefix == nil {prefix = lastToken@@ -117,7 +270,7 @@ func (d *mtpDrafter) propose(current *mlx.Array, maxTokens int) *draftCandidates}}return &draftCandidates{- tokens: mlx.Concatenate(draftTokens, 1),+ tokens: prefix,dist: sampler.ConcatenateDistributions(draftDists),}}diff --git a/x/mlxrunner/mtp_test.go b/x/mlxrunner/mtp_test.goindex c449b3e73..bde9e607e 100644--- a/x/mlxrunner/mtp_test.go+++ b/x/mlxrunner/mtp_test.go@@ -5,6 +5,7 @@ import ("encoding/json""errors""fmt"+ "reflect""slices""strings""testing"@@ -60,7 +61,10 @@ func (m *fakeMTPModel) Forward(b *batch.Batch, caches []cache.Cache) *mlx.Arraymlx.Eval(b.InputIDs)ids := b.InputIDs.Ints()m.forwards = append(m.forwards, forwardCall{offset: b.SeqOffsets[0], n: int32(len(ids))})- for _, c := range caches {+ for i, c := range caches {+ if i >= m.NumLayers() {+ break+ }if rc, ok := c.(*fakeRewindableCache); ok {seg := make([]int32, len(ids))for i, id := range ids {@@ -85,25 +89,10 @@ func (m *fakeMTPModel) LoadWeights(map[string]*mlx.Array) error {return nil}-// TokenEmbeddings returns a width-1 embedding holding the token id as a float,-// so a draft can recover which token it is extending from inputs[...,0].-func (m *fakeMTPModel) TokenEmbeddings(inputIDs *mlx.Array) *mlx.Array {- mlx.Eval(inputIDs)- ids := inputIDs.Ints()- data := make([]float32, len(ids))- for i, id := range ids {- data[i] = float32(id)- }- return mlx.FromValues(data, inputIDs.Dim(0), inputIDs.Dim(1), 1)-}--var (- _ base.Model = (*fakeMTPModel)(nil)- _ base.MTPEmbeddingModel = (*fakeMTPModel)(nil)-)+var _ base.Model = (*fakeMTPModel)(nil)-// fakeMTPDraft extends the token in inputEmbeds through predict; a map (not a-// step counter) keeps drafting consistent regardless of batching.+// fakeMTPDraft is a cacheless draft that extends b.InputIDs through predict;+// a map (not a step counter) keeps drafting consistent regardless of batching.type fakeMTPDraft struct {predict map[int32]int32// calls records each Draft call so tests can assert the position convention.@@ -117,17 +106,82 @@ type draftCall struct {func (d *fakeMTPDraft) LoadWeights(map[string]*mlx.Array) error { return nil }-func (d *fakeMTPDraft) Draft(inputEmbeds *mlx.Array, position int32, caches []cache.Cache) (logits, hidden *mlx.Array) {- mlx.Eval(inputEmbeds)- prev := int32(inputEmbeds.Floats()[0])- d.calls = append(d.calls, draftCall{position: position, from: prev})+func (d *fakeMTPDraft) DraftCaches([]cache.Cache) []cache.Cache { return nil }++func (d *fakeMTPDraft) Draft(b *batch.Batch, caches []cache.Cache) (hidden, projected *mlx.Array) {+ mlx.Eval(b.InputIDs)+ prev := int32(b.InputIDs.Ints()[0])+ d.calls = append(d.calls, draftCall{position: b.SeqOffsets[0], from: prev})return oneHotLogits([]int32{d.predict[prev]}), mlx.Zeros(mlx.DTypeFloat32, 1, 1, mtpTestVocab)}-var (- _ base.DraftModel = (*fakeMTPDraft)(nil)- _ base.MTPDraftModel = (*fakeMTPDraft)(nil)-)+// Unembed is the identity: the fake's hidden already is its one-hot logits.+func (d *fakeMTPDraft) Unembed(x *mlx.Array) *mlx.Array { return x }++var _ base.DraftModel = (*fakeMTPDraft)(nil)++// fakeKVDraft is a draft head with its own KV cache: it claims the trailing+// cache slot, writes its input ids there on every Draft call (advancing the+// offset like a real KV write), and records each call's offset, ids, and+// the identity of every fused hidden row. A target hidden row is one-hot+// (its hot index identifies which position it came from); the head's own+// projected hidden is all-zero and records as -1.+type fakeKVDraft struct {+ predict map[int32]int32+ extends []extendCall+}++// extendCall is one recorded Draft call: the absolute slot of the first+// entry written, the look-ahead token ids, and the hot index of each fused+// hidden row (-1 for the head's own projected hidden).+type extendCall struct {+ offset int32+ ids []int32+ hiddens []int32+}++func (d *fakeKVDraft) LoadWeights(map[string]*mlx.Array) error { return nil }++func (d *fakeKVDraft) DraftCaches(caches []cache.Cache) []cache.Cache {+ return caches[len(caches)-1:]+}++func (d *fakeKVDraft) Draft(b *batch.Batch, caches []cache.Cache) (hidden, projected *mlx.Array) {+ mlx.Eval(b.InputIDs, b.Hidden)+ rawIDs := b.InputIDs.Ints()+ ids := make([]int32, len(rawIDs))+ for i, id := range rawIDs {+ ids[i] = int32(id)+ }++ hot := make([]int32, b.Hidden.Dim(1))+ flat := b.Hidden.Floats()+ for r := range hot {+ hot[r] = -1+ for v := range mtpTestVocab {+ if flat[r*mtpTestVocab+v] != 0 {+ hot[r] = int32(v)+ break+ }+ }+ }+ d.extends = append(d.extends, extendCall{offset: b.SeqOffsets[0], ids: ids, hiddens: hot})++ if rc, ok := d.DraftCaches(caches)[0].(*fakeRewindableCache); ok {+ rc.feed(ids)+ }++ preds := make([]int32, len(ids))+ for i, id := range ids {+ preds[i] = d.predict[id]+ }+ return oneHotLogits(preds), mlx.Zeros(mlx.DTypeFloat32, 1, 1, mtpTestVocab)+}++// Unembed is the identity: the fake's hidden already is its one-hot logits.+func (d *fakeKVDraft) Unembed(x *mlx.Array) *mlx.Array { return x }++var _ base.DraftModel = (*fakeKVDraft)(nil)// newTestTokenizer builds a byte-level BPE tokenizer over single-character// tokens "0".."7" with the given EOS ids, so Decode(id) yields that digit and@@ -215,7 +269,7 @@ func TestAcceptMTPDraftsGreedyAcceptAll(t *testing.T) {spec := testSpeculationSession(r, caches)current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}- results, accepted, err := spec.accept(&position, current, oneHotLogits([]int32{2}), baseLogits, candidates)+ results, accepted, err := spec.accept(&position, current, oneHotLogits([]int32{1}), baseLogits, candidates)if err != nil {t.Fatalf("accept: %v", err)}@@ -250,7 +304,7 @@ func TestAcceptMTPDraftsGreedyMismatch(t *testing.T) {spec := testSpeculationSession(r, caches)current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}- results, accepted, err := spec.accept(&position, current, oneHotLogits([]int32{2}), baseLogits, candidates)+ results, accepted, err := spec.accept(&position, current, oneHotLogits([]int32{1}), baseLogits, candidates)if err != nil {t.Fatalf("accept: %v", err)}@@ -273,8 +327,8 @@ func TestAcceptMTPDraftsGreedyMismatch(t *testing.T) {func TestAcceptMTPDraftsGreedyEOS(t *testing.T) {skipIfNoMLX(t)// The second accepted draft token is EOS: it is recorded but stops- // generation and no bonus token is produced. Both accepted tokens are- // committed, so the caches hold the EOS's KV.+ // generation and no bonus token is produced. The EOS's own KV is rolled+ // back so the caches rest one token behind the recorded outputs.const eos int32 = 6predict := map[int32]int32{1: 2, 2: eos, eos: 0}r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})@@ -287,7 +341,7 @@ func TestAcceptMTPDraftsGreedyEOS(t *testing.T) {spec := testSpeculationSession(r, caches)current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}- results, accepted, err := spec.accept(&position, current, oneHotLogits([]int32{2}), baseLogits, candidates)+ results, accepted, err := spec.accept(&position, current, oneHotLogits([]int32{1}), baseLogits, candidates)if err != nil {t.Fatalf("accept: %v", err)}@@ -302,11 +356,11 @@ func TestAcceptMTPDraftsGreedyEOS(t *testing.T) {if len(results) != accepted {t.Fatalf("results has %d tokens, want exactly the %d accepted with no bonus after EOS", len(results), accepted)}- if position != 2 {- t.Fatalf("position = %d, want 2 (both accepted tokens committed)", position)+ if position != 1 {+ t.Fatalf("position = %d, want 1 (kept draft committed, EOS dropped)", position)}- if got := caches[0].Offset(); got != 2 {- t.Fatalf("cache offset = %d, want 2 (both accepted tokens committed)", got)+ if got := caches[0].Offset(); got != 1 {+ t.Fatalf("cache offset = %d, want 1 (one behind the recorded outputs, EOS dropped)", got)}}@@ -392,7 +446,11 @@ func TestRunMTPDecodeSampled(t *testing.T) {CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},SamplerOpts: sampler.Options{Temperature: 1, Seed: 42, UseSeed: true},}- d := testDecoder(r, req, caches, []int32{1}, position)+ spec := r.spec.open(req, caches)+ if spec == nil || !spec.enabled {+ t.Fatalf("newSpeculationSession rejected a sampled request")+ }+ d := spec.decoder([]int32{1}, position)if err := r.decode(context.Background(), req, session, d, 0); err != nil {t.Fatalf("decode: %v", err)}@@ -518,7 +576,299 @@ func testDecoder(r *Runner, req Request, caches []cache.Cache, seed []int32, posif spec := r.spec.open(req, caches); spec != nil {return spec.decoder(seed, position)}- return r.pipelinedDecoder(caches, seed, position)+ return r.pipelinedDecoder(nil, caches, seed, position)+}++func TestDecodeKVDraft(t *testing.T) {+ skipIfNoMLX(t)+ // A draft with its own KV cache mirroring the target chain+ // 1->2->3->4->5->6->EOS.+ // The decode seed catch-up writes the draft pair for the prompt token,+ // the first proposal comes from the catch-up's held logits (no draft+ // call), speculative entries are written at advancing slots, and the+ // post-accept rebuild rewrites the committed range from target hiddens.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict}+ r.spec = newSpeculation(r, draft)++ caches, _ := newMTPTestCaches(2) // caches[0] target, caches[1] draft KV+ session, ch := newMTPTestSession(caches)+ position := 0++ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, caches)+ if spec == nil || !spec.enabled || len(spec.spec.targets) != 1 {+ t.Fatalf("speculation engine not built around the draft caches")+ }+ defer spec.close()+ d := spec.decoder([]int32{1}, position)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()++ content, final := collectResponses(ch)+ if content != "23456" {+ t.Fatalf("content = %q, want %q", content, "23456")+ }+ if final.EvalCount != 5 || final.DoneReason != 0 {+ t.Fatalf("EvalCount = %d, DoneReason = %d, want 5 and 0", final.EvalCount, final.DoneReason)+ }+ if got := []int32{2, 3, 4, 5, 6, eos}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }++ // All caches rest together at the trie frontier: the prompt token plus+ // five generated tokens; the EOS's KV is never committed.+ if got := caches[0].Offset(); got != 6 {+ t.Fatalf("target offset = %d, want 6", got)+ }+ if got := caches[1].Offset(); got != 6 {+ t.Fatalf("draft cache offset = %d, want 6 (lockstep with target)", got)+ }++ // Every committed draft slot S fuses look-ahead token x_{S+1} with the+ // target hidden at S (whose hot index equals the look-ahead id on this+ // mirrored chain); speculative steps fuse the head's own projections+ // (-1), seeded by the catch-up flush's held projection. The first+ // proposal of the round consumes the catch-up's held logits, so the+ // four-token draft makes only three head calls before the rebuild.+ wantExtends := []extendCall{+ {offset: 0, ids: []int32{2}, hiddens: []int32{2}}, // frontier pair flushed at the first proposal+ {offset: 1, ids: []int32{3}, hiddens: []int32{-1}}, // speculative step 2 (held projection)+ {offset: 2, ids: []int32{4}, hiddens: []int32{-1}}, // speculative step 3 (projection)+ {offset: 3, ids: []int32{5}, hiddens: []int32{-1}}, // speculative step 4 (projection)+ {offset: 1, ids: []int32{3, 4, 5, 6, eos}, hiddens: []int32{3, 4, 5, 6, eos}}, // committed pairs from the validated run + finish+ }+ if !reflect.DeepEqual(draft.extends, wantExtends) {+ t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)+ }++ // The committed draft KV holds the look-ahead tokens, one per slot.+ if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, eos}; !slices.Equal(got, want) {+ t.Fatalf("draft cache = %v, want %v", got, want)+ }+}++func TestDecodeKVDraftRejectionRebuildsFromTarget(t *testing.T) {+ skipIfNoMLX(t)+ // The draft mispredicts mid-chain: it proposes 6 where the target's own+ // next token is 4, so the round is accepted only up to the rejection and the+ // loop re-proposes from the target's correction. The speculative draft KV+ // written for the rejected proposals is rolled back and rewritten from the+ // validated run's target hiddens, so the committed draft cache holds the+ // target chain with no trace of the rejected tokens.+ const eos int32 = 7+ target := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: eos, eos: 0}+ r := mtpTestRunner(t, target, []int32{eos}, sampler.Options{})+ // The draft mirrors the target except at 3, where it proposes 6 (absent from+ // the target chain); once the target corrects 3->4 the next proposal+ // re-aligns on the shared chain.+ draft := &fakeKVDraft{predict: map[int32]int32{1: 2, 2: 3, 3: 6, 6: 0, 4: 5, 5: eos, eos: 0}}+ r.spec = newSpeculation(r, draft)++ caches, _ := newMTPTestCaches(2) // caches[0] target, caches[1] draft KV+ session, ch := newMTPTestSession(caches)+ position := 0++ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, caches)+ spec.limit = 4+ defer spec.close()+ d := spec.decoder([]int32{1}, position)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()++ content, final := collectResponses(ch)+ if content != "2345" {+ t.Fatalf("content = %q, want %q", content, "2345")+ }+ if final.DoneReason != 0 {+ t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)+ }+ if got := []int32{2, 3, 4, 5, eos}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }++ // The divergent token was drafted into the speculative KV...+ drafted := false+ for _, e := range draft.extends {+ if slices.Contains(e.ids, 6) {+ drafted = true+ }+ }+ if !drafted {+ t.Fatalf("draft never proposed the divergent token 6; extends = %+v", draft.extends)+ }+ // ...but the rejection rolled it back: both caches rest in lockstep at the+ // target chain, and the committed draft KV holds only the validated+ // look-ahead tokens.+ if got := caches[0].Offset(); got != 5 {+ t.Fatalf("target offset = %d, want 5", got)+ }+ if got := caches[1].Offset(); got != 5 {+ t.Fatalf("draft cache offset = %d, want 5 (lockstep with target)", got)+ }+ if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, eos}; !slices.Equal(got, want) {+ t.Fatalf("draft cache = %v, want %v (rejected proposals rolled back)", got, want)+ }+}++func TestDecodeMaintainsDraftCacheWithoutDrafting(t *testing.T) {+ skipIfNoMLX(t)+ // A request that cannot speculate (logprobs) on a model whose draft has+ // its own KV cache still maintains it: the pipelined decoder+ // reports each forwarded token, the pairs wait in the pending list, and+ // one batched extend at close — its final pair completed by the decoder's+ // discarded in-flight sample — leaves the draft level with the target.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}+ opts := sampler.Options{Logprobs: true}+ r := mtpTestRunner(t, predict, []int32{eos}, opts)+ draft := &fakeKVDraft{predict: predict}+ r.spec = newSpeculation(r, draft)++ caches, _ := newMTPTestCaches(2)+ session, ch := newMTPTestSession(caches)+ position := 0++ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: opts,+ }+ spec := r.spec.open(req, caches)+ if spec == nil || spec.enabled {+ t.Fatalf("want a maintain-only speculationSession, got %+v", spec)+ }+ d := spec.decoder([]int32{1}, position)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()+ spec.close() // the pipeline defers this before session close++ content, _ := collectResponses(ch)+ if content != "23456" {+ t.Fatalf("content = %q, want %q", content, "23456")+ }++ // The pipelined decoder forwards every emitted token, including the+ // ending EOS, so both caches rest at the full recorded path.+ if got := caches[0].Offset(); got != 7 {+ t.Fatalf("target offset = %d, want 7", got)+ }+ if got := caches[1].Offset(); got != 7 {+ t.Fatalf("draft cache offset = %d, want 7 (lockstep with target)", got)+ }++ // A session that never drafts defers every committed pair: the whole+ // generation arrives in one batched flush at finish, with the EOS slot's+ // pair completed by the in-flight sample (predict[eos] = 0) that decoding+ // discarded.+ wantExtends := []extendCall{+ {offset: 0, ids: []int32{2, 3, 4, 5, 6, eos, 0}, hiddens: []int32{2, 3, 4, 5, 6, eos, 0}},+ }+ if !reflect.DeepEqual(draft.extends, wantExtends) {+ t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)+ }+ if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, eos, 0}; !slices.Equal(got, want) {+ t.Fatalf("draft cache = %v, want %v", got, want)+ }+}++func TestFlushLevelsDraftCacheWithPrefill(t *testing.T) {+ skipIfNoMLX(t)+ // Prefill attaches its scheduled snapshots only at offsets every cache+ // has crossed, so the pipeline flushes the drafter's buffered pairs+ // first: after a prefill-sized committed report and a flush, the+ // draft cache covers every completed pair, one slot behind the target+ // (the frontier pair still awaits its look-ahead token).+ const eos int32 = 7+ predict := map[int32]int32{2: 3, 3: 4, 4: 5}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict}+ r.spec = newSpeculation(r, draft)++ caches, _ := newMTPTestCaches(2)+ req := Request{+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, caches)+ defer spec.close()++ // The prompt's only chunk: tokens 1..4 at slots 0..3 with their hiddens.+ spec.committed(mlx.FromValues([]int32{1, 2, 3, 4}, 1, 4), oneHotLogits([]int32{1, 2, 3, 4}), 0)+ spec.flush()++ if got := caches[1].Offset(); got != 3 {+ t.Fatalf("draft cache offset after flush = %d, want 3 (all completed pairs)", got)+ }+ wantExtends := []extendCall{+ {offset: 0, ids: []int32{2, 3, 4}, hiddens: []int32{1, 2, 3}},+ }+ if !reflect.DeepEqual(draft.extends, wantExtends) {+ t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)+ }+}++func TestCommittedRunBatchesPastFlushCap(t *testing.T) {+ skipIfNoMLX(t)+ // A committed run longer than the pending-flush cap still writes the draft+ // caches in a single head forward: the run's completed pairs coalesce into+ // one batched extend at the run's start, rather than splitting at the cap.+ const n = mtpPendingFlushTokens + 8+ predict := map[int32]int32{}+ tokens := make([]int32, n)+ for i := range tokens {+ tokens[i] = int32(i%7 + 1)+ predict[tokens[i]] = int32((i+1)%7 + 1)+ }++ r := mtpTestRunner(t, predict, []int32{0}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict}+ r.spec = newSpeculation(r, draft)++ caches, _ := newMTPTestCaches(2)+ req := Request{+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, caches)+ defer spec.close()++ // One prefill-sized chunk: n tokens at slots 0..n-1 with their hiddens.+ spec.committed(mlx.FromValues(tokens, 1, n), oneHotLogits(tokens), 0)+ spec.flush()++ if got := len(draft.extends); got != 1 {+ t.Fatalf("draft extends = %d calls, want 1 batched extend", got)+ }+ if got, want := len(draft.extends[0].ids), n-1; got != want {+ t.Fatalf("batched extend ids = %d, want %d (every completed pair)", got, want)+ }+ if got := draft.extends[0].offset; got != 0 {+ t.Fatalf("batched extend offset = %d, want 0", got)+ }+ if got := caches[1].Offset(); got != n-1 {+ t.Fatalf("draft cache offset = %d, want %d (all completed pairs)", got, n-1)+ }}// newMTPTestCaches returns n rewindable fake caches sharing one snapshot@@ -539,19 +889,16 @@ func newMTPTestSession(caches []cache.Cache) (*cacheSession, chan CompletionRespreturn &cacheSession{caches: caches}, ch}-// testSpeculationSession wires a speculation engine around the runner's drafter and-// caches for tests that drive accept directly. A runner without a draft-// model gets a no-op drafter, since the engine requires one.+// testSpeculationSession wires a speculation engine around a drafter and caches for+// tests that drive accept directly. A runner without a draft model gets a+// no-op drafter, since the engine requires one.func testSpeculationSession(r *Runner, caches []cache.Cache) *speculationSession {- s := r.spec- if s == nil {- s = &speculation{r: r}- }- var d drafter = nopDrafter{}- if md := newMTPDrafter(s, caches); md != nil {- d = md+ if r.spec != nil {+ r.spec.bind(caches)+ return &speculationSession{spec: r.spec, drafter: newMTPDrafter(r.spec)}}- return &speculationSession{spec: s, drafter: d, caches: caches}+ s := &speculation{r: r, caches: caches, targets: caches}+ return &speculationSession{spec: s, drafter: nopDrafter{}}}// nopDrafter satisfies drafter for engine tests that supply candidates@@ -560,6 +907,8 @@ type nopDrafter struct{}func (nopDrafter) propose(*mlx.Array, int) *draftCandidates { return nil }func (nopDrafter) committed(_, _ *mlx.Array, _ int) {}+func (nopDrafter) finish(*mlx.Array) {}+func (nopDrafter) flush() {}func (nopDrafter) close() {}// scriptedCandidates builds draft candidates by running the real drafter@@ -573,7 +922,8 @@ func scriptedCandidates(r *Runner, tokens []int32) *draftCandidates {chain[prev] = tokprev = tok}- d := &mtpDrafter{spec: &speculation{r: r}, draft: &fakeMTPDraft{predict: chain}, target: r.Model.(base.MTPEmbeddingModel)}+ s := &speculation{r: r, draft: &fakeMTPDraft{predict: chain}}+ d := &mtpDrafter{spec: s}d.committed(mlx.FromValues([]int32{0}, 1, 1), mlx.Zeros(mlx.DTypeFloat32, 1, 1, mtpTestVocab), 0)return d.propose(mlx.FromValues([]int32{0}, 1), len(tokens))}diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.goindex 3f5f31e2d..677fc3112 100644--- a/x/mlxrunner/pipeline.go+++ b/x/mlxrunner/pipeline.go@@ -75,7 +75,12 @@ func (r *Runner) TextGenerationPipeline(ctx context.Context, request Request) erdefer session.close()caches := session.caches- seed, position, promptEval, err := r.prefill(ctx, session)+ // Built before prefill so a drafter with draft caches follows the prompt+ // through prefill alongside the target.+ spec := r.spec.open(request, caches)+ defer spec.close()++ seed, position, promptEval, err := r.prefill(ctx, session, spec)if err != nil {return err}@@ -83,14 +88,11 @@ func (r *Runner) TextGenerationPipeline(ctx context.Context, request Request) er// Register the sampler after prefill completes.r.Sampler.Add(pipelineSlot, request.SamplerOpts, inputs)- spec := r.spec.open(request, caches)- defer spec.close()-var d decoderif spec != nil {d = spec.decoder(seed, position)} else {- d = r.pipelinedDecoder(caches, seed, position)+ d = r.pipelinedDecoder(nil, caches, seed, position)}defer d.close()return r.decode(ctx, request, session, d, promptEval)@@ -100,7 +102,7 @@ func (r *Runner) TextGenerationPipeline(ctx context.Context, request Request) er// one token for decode to seed from, and schedules the prompt's periodic// snapshots. It returns the seed tokens, the resume position, and the// prompt-evaluation duration.-func (r *Runner) prefill(ctx context.Context, session *cacheSession) ([]int32, int, time.Duration, error) {+func (r *Runner) prefill(ctx context.Context, session *cacheSession, spec *speculationSession) ([]int32, int, time.Duration, error) {start := time.Now()inputs := session.inputstokens := session.remaining@@ -143,11 +145,13 @@ func (r *Runner) prefill(ctx context.Context, session *cacheSession) ([]int32, in := min(prefillChunk, total-processed-1)- r.Model.Forward(&batch.Batch{- InputIDs: mlx.FromValues(tokens[processed:processed+n], 1, n),+ chunkIDs := mlx.FromValues(tokens[processed:processed+n], 1, n)+ hidden := r.Model.Forward(&batch.Batch{+ InputIDs: chunkIDs,SeqOffsets: []int32{int32(position)},SeqQueryLens: []int32{int32(n)},}, caches)+ spec.committed(chunkIDs, hidden, position)mlx.Sweep()materializeCaches()processed += n@@ -158,7 +162,10 @@ func (r *Runner) prefill(ctx context.Context, session *cacheSession) ([]int32, imlx.ClearCache()}- // Attach the snapshots captured during prefill to the trie.+ // Flush before attaching: snapshots attach only at offsets every cache+ // has crossed, and a drafter with draft caches keeps buffered pairs that+ // would otherwise hold those caches short of the scheduled offsets.+ spec.flush()session.attachPrefillSnapshots()return tokens[processed:], position, time.Since(start), nil@@ -263,15 +270,18 @@ func (r *Runner) decode(ctx context.Context, request Request, session *cacheSess// the next token's chain is dispatched before the returned one is// synchronized, so the device runs ahead of host emission.type pipelinedDecoder struct {- r *Runner+ r *Runner+ // spec, when non-nil, receives every forwarded token and settles its+ // drafter at close, keeping a non-drafting session's draft KV level.+ spec *speculationSessioncaches []cache.Cacheposition intsample sampler.Result // in flight: sampled, not yet forwardedemitted sampler.Result // last call's result, pinned until the next call}-func (r *Runner) pipelinedDecoder(caches []cache.Cache, seed []int32, position int) *pipelinedDecoder {- t := &pipelinedDecoder{r: r, caches: caches, position: position}+func (r *Runner) pipelinedDecoder(spec *speculationSession, caches []cache.Cache, seed []int32, position int) *pipelinedDecoder {+ t := &pipelinedDecoder{r: r, spec: spec, caches: caches, position: position}t.sample = t.dispatch(mlx.FromValues(seed, 1, len(seed)))return t}@@ -285,8 +295,10 @@ func (t *pipelinedDecoder) dispatch(token *mlx.Array) sampler.Result {SeqOffsets: []int32{int32(t.position)},SeqQueryLens: []int32{int32(token.Dim(1))},}, t.caches)+ t.spec.committed(token, hidden, t.position)t.position += token.Dim(1)- next := r.Sampler.Sample([]int{pipelineSlot}, lastLogits(r.Model.Unembed(hidden)))+ logits := r.Model.Unembed(hidden)+ next := r.Sampler.Sample([]int{pipelineSlot}, logits.Slice(mlx.Slice(), mlx.Slice(logits.Dim(1)-1), mlx.Slice()).Squeeze(1))mlx.Pin(next.Arrays()...)mlx.Sweep()mlx.AsyncEval(next.Arrays()...)@@ -300,6 +312,9 @@ func (t *pipelinedDecoder) next(int) ([]sampler.Result, error) {}func (t *pipelinedDecoder) close() {+ // The in-flight sample's forward was never dispatched; its report lets+ // the drafter settle level with the caches' resting offset.+ t.spec.finish(t.sample.Token)mlx.Unpin(t.emitted.Arrays()...)mlx.Unpin(t.sample.Arrays()...)}diff --git a/x/mlxrunner/runner.go b/x/mlxrunner/runner.goindex 189bfda9c..c76e3e4d8 100644--- a/x/mlxrunner/runner.go+++ b/x/mlxrunner/runner.go@@ -81,6 +81,9 @@ func (r *Runner) Load(modelName string) error {return err}draftModel = draft+ } else if sd, ok := m.(base.SelfDraft); ok {+ // Inline draft head: already loaded with the target; nil if none shipped.+ draftModel = sd.SelfDraft()}collected := mlx.Collect(m)diff --git a/x/mlxrunner/speculate.go b/x/mlxrunner/speculate.goindex 2b9a46a29..baf9811e3 100644--- a/x/mlxrunner/speculate.go+++ b/x/mlxrunner/speculate.go@@ -4,6 +4,7 @@ import ("fmt""log/slog""os"+ "slices""strconv""strings"@@ -23,10 +24,18 @@ type drafter interface {// committed reports a run of tokens committed to the target caches:// tokens[i] sits at slot position+i and hiddens row i is the target- // hidden state at that slot. Runs arrive in slot order — the decode- // seed, then each round's validated tokens.+ // hidden state at that slot. Runs arrive in slot order — prefill+ // chunks, the decode seed, then each round's validated tokens.committed(tokens, hiddens *mlx.Array, position int)+ // finish reports generation ended with current sampled but never+ // committed, so the drafter can settle state tracking the target caches.+ finish(current *mlx.Array)++ // flush writes any buffered committed reports through to the draft+ // caches. A drafter without draft caches has nothing to write.+ flush()+close()}@@ -100,6 +109,15 @@ func positiveEnvInt(key string) int {type speculation struct {r *Runnerdraft base.DraftModel++ // caches is the whole persistent slice, passed to every forward; draftKV+ // are the draft head's own caches and targets are the rest — the caches the+ // target forward writes, which speculation snapshots and rollback cover.+ // Bound the first time the caches exist (the Runner reuses one cache slice+ // for its life) and stable thereafter.+ caches []cache.Cache+ draftKV []cache.Cache+ targets []cache.Cache}// newSpeculation builds the speculative-decoding subsystem for a loaded model,@@ -111,41 +129,68 @@ func newSpeculation(r *Runner, draft base.DraftModel) *speculation {return &speculation{r: r, draft: draft}}+// bind computes the draft/target cache partition the first time the persistent+// caches exist; later requests reuse the same slice, so it runs once.+func (s *speculation) bind(caches []cache.Cache) {+ if s.caches != nil {+ if !slices.Equal(s.caches, caches) {+ panic("speculation: cache slice changed between requests")+ }+ return+ }+ draftKV := s.draft.DraftCaches(caches)++ // Partition caches into target slots (everything not in draftKV) in one+ // pass. The count check rejects a draft slot that isn't a member of caches.+ targets := make([]cache.Cache, 0, len(caches))+ for _, c := range caches {+ if !slices.Contains(draftKV, c) {+ targets = append(targets, c)+ }+ }+ if len(caches)-len(targets) != len(draftKV) {+ panic("speculation: DraftCaches must select slots of the cache slice")+ }++ s.caches = caches+ s.draftKV = draftKV+ s.targets = targets+}+// speculationSession is the speculation cursor for one request over a speculation. A// nil speculationSession is a plain decode.type speculationSession struct {spec *speculationdrafter drafter- caches []cache.Cache+ // enabled selects speculative rounds; a maintain-only engine decodes+ // plainly while streaming committed runs to keep draft KV prefix-cached.+ enabled boolopts specOptionslimit int // current draft lengthstats specStats}-// open returns the speculation cursor for this request, or nil when the model-// carries no draft head or the request cannot speculate. Logprobs are not yet-// supported. A nil receiver (no draft head) decodes plainly.+// open returns the speculation cursor for this request or nil when the model ships+// no draft head (a nil receiver), which decodes plainly.func (s *speculation) open(request Request, caches []cache.Cache) *speculationSession {if s == nil {return nil}- d := newMTPDrafter(s, caches)+ s.bind(caches)+ d := newMTPDrafter(s)if d == nil {return nil}+opts := request.SamplerOpts- if !s.r.mtpDefaults(opts.Temperature != 0).Enabled {- return nil- }- if opts.Logprobs || opts.TopLogprobs > 0 {- return nil- }+ enabled := s.r.mtpDefaults(opts.Temperature != 0).Enabled &&+ !opts.Logprobs && opts.TopLogprobs == 0specOpts := s.r.loadSpecOptions(opts.Temperature != 0)return &speculationSession{spec: s,drafter: d,- caches: caches,+ enabled: enabled,opts: specOpts,limit: specOpts.initialDraftTokens,stats: specStats{maxDraft: specOpts.initialDraftTokens},@@ -159,6 +204,23 @@ func (s *speculationSession) committed(tokens, hiddens *mlx.Array, position int)s.drafter.committed(tokens, hiddens, position)}+// finish reports the end of generation to the drafter: current was sampled+// after the last committed slot and will never be committed.+func (s *speculationSession) finish(current *mlx.Array) {+ if s == nil {+ return+ }+ s.drafter.finish(current)+}++// flush writes the drafter's buffered committed reports to the draft caches.+func (s *speculationSession) flush() {+ if s == nil {+ return+ }+ s.drafter.flush()+}+func (s *speculationSession) close() {if s == nil {return@@ -178,7 +240,13 @@ type speculativeDecoder struct {current sampler.Result // emitted (or the seed), not yet forwarded}-func (s *speculationSession) decoder(seed []int32, position int) *speculativeDecoder {+// decoder returns the decoder for this engine's session. A maintain-only+// engine decodes plainly via the pipelined decoder, which reports every+// forwarded token to keep draft KV level with the target.+func (s *speculationSession) decoder(seed []int32, position int) decoder {+ if !s.enabled {+ return s.spec.r.pipelinedDecoder(s, s.spec.caches, seed, position)+ }current := sampler.Result{Token: mlx.FromValues(seed, len(seed))}mlx.Pin(current.Arrays()...)return &speculativeDecoder{s: s, position: position, current: current}@@ -192,7 +260,7 @@ func (st *speculativeDecoder) next(remaining int) ([]sampler.Result, error) {InputIDs: tokenInput(st.current.Token),SeqOffsets: []int32{int32(st.position)},SeqQueryLens: []int32{1},- }, s.caches)+ }, s.spec.caches)st.position++results, next, err := s.round(&st.position, st.current, hidden, remaining)@@ -212,6 +280,10 @@ func (st *speculativeDecoder) next(remaining int) ([]sampler.Result, error) {}func (st *speculativeDecoder) close() {+ // Generation always ends with a final token that was emitted but never+ // forwarded; its report lets the drafter settle level with the caches'+ // resting offset.+ st.s.finish(st.current.Token)mlx.Unpin(st.current.Arrays()...)st.s.logStats()}@@ -368,7 +440,7 @@ func (s *speculationSession) accept(position *int, current sampler.Result, hidder := s.spec.rbefore := *positiondraftCount := candidates.tokens.Dim(1)- scheduleSpeculation(s.caches, before, draftCount)+ scheduleSpeculation(s.spec.targets, before, draftCount)// Every exit between schedule and commit must drain the snapshot// schedule and roll the speculative writes back out of the live caches:@@ -380,7 +452,7 @@ func (s *speculationSession) accept(position *int, current sampler.Result, hiddereturn}committed = true- commitSpeculation(s.caches, keep, draftCount, before)+ commitSpeculation(s.spec.targets, keep, draftCount, before)}defer commit(0)@@ -388,7 +460,7 @@ func (s *speculationSession) accept(position *int, current sampler.Result, hiddeInputIDs: candidates.tokens,SeqOffsets: []int32{int32(before)},SeqQueryLens: []int32{int32(draftCount)},- }, s.caches)+ }, s.spec.caches)targetDist := r.Sampler.Distribution(pipelineSlot, validationLogits(r, baseLogits, hiddenSeq), candidates.tokens)draftDist := candidates.dist@@ -419,18 +491,24 @@ func (s *speculationSession) accept(position *int, current sampler.Result, hiddebreak}}+ // The final token of a generation is recorded and streamed but its KV+ // is never committed: the caches rest at the trie frontier.+ keep := accepted+ if done {+ keep--+ }+ commit(keep)+ *position = before + keep- commit(accepted)- *position = before + accepted-- // Report the validated run — current plus the accepted drafts, with the+ // Report the validated run — current plus the kept drafts, with the// hidden state at each token's own slot — to the drafter before// returning, so even a cancelled emission leaves the drafter's state- // describing exactly what the target caches hold.- runIDs := append([]int32{int32(current.Token.Int())}, commitIDs...)+ // describing exactly what the target caches hold. A done generation's+ // final token is never committed; it reaches the drafter through finish.+ runIDs := append([]int32{int32(current.Token.Int())}, commitIDs[:keep]...)runHiddens := lastHiddenRow(hidden)- if accepted > 0 {- runHiddens = runHiddens.Concatenate(1, hiddenSeq.Slice(mlx.Slice(), mlx.Slice(0, accepted), mlx.Slice()))+ if keep > 0 {+ runHiddens = runHiddens.Concatenate(1, hiddenSeq.Slice(mlx.Slice(), mlx.Slice(0, keep), mlx.Slice()))}s.drafter.committed(mlx.FromValues(runIDs, 1, len(runIDs)), runHiddens, before-1)diff --git a/x/models/gemma4/assistant.go b/x/models/gemma4/assistant.goindex 20a67560e..36b675049 100644--- a/x/models/gemma4/assistant.go+++ b/x/models/gemma4/assistant.go@@ -12,10 +12,7 @@ import ("github.com/ollama/ollama/x/models/nn")-var (- _ base.DraftModel = (*AssistantModel)(nil)- _ base.MTPDraftModel = (*AssistantModel)(nil)-)+var _ base.DraftModel = (*AssistantModel)(nil)type AssistantConfig struct {TextConfig TextConfig `json:"text_config"`@@ -37,6 +34,10 @@ type AssistantModel struct {NormScaled *mlx.Array+ // target supplies the scaled token embeddings the MTP head fuses with+ // the target hidden state.+ target *Model+*AssistantConfigtensorPrefix string@@ -145,6 +146,7 @@ func newAssistantModel(root *model.Root, target base.Model) (base.DraftModel, erm := &AssistantModel{AssistantConfig: &cfg,tensorPrefix: tensorPrefix,+ target: targetGemma,Layers: make([]*AssistantLayer, cfg.TextConfig.NumHiddenLayers),TensorQuant: root.AllTensorQuant(),}@@ -267,26 +269,42 @@ func (m *AssistantModel) precomputeScaledWeights() {}}-func (m *AssistantModel) Draft(inputsEmbeds *mlx.Array, position int32, caches []cache.Cache) (logits, hidden *mlx.Array) {+// DraftCaches returns nil: the assistant keeps no KV and re-attends the+// target's caches read-only each step.+func (m *AssistantModel) DraftCaches([]cache.Cache) []cache.Cache { return nil }++// Draft is single-position: the head drafts every token as if it sat at the+// last target-seen position, so it anchors RoPE and the mask there — from the+// non-moving target full-attention cache — regardless of the advancing offset+// in b. The input fuses the target's scaled token embedding with b.Hidden.+func (m *AssistantModel) Draft(b *batch.Batch, caches []cache.Cache) (hidden, projected *mlx.Array) {+ inputsEmbeds := m.target.TokenEmbeddings(b.InputIDs).Concatenate(-1, b.Hidden)dims := inputsEmbeds.Dims()B, L := int32(dims[0]), int32(dims[1])- b := &batch.Batch{++ anchor := int32(0)+ if len(caches) > 0 {+ if full := caches[len(caches)-1]; full != nil {+ anchor = int32(full.Offset() - 1)+ }+ }+ ab := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, int(B), int(L)),- SeqOffsets: []int32{position},+ SeqOffsets: []int32{anchor},SeqQueryLens: []int32{L},}- sliding, full := m.sharedHistories(b, caches)+ sliding, full := m.sharedHistories(ab, caches)h := m.PreProjection.Forward(inputsEmbeds)- positions := mlx.FromValues([]int32{position}, 1)+ positions := mlx.FromValues([]int32{anchor}, 1)for _, layer := range m.Layers {- h = layer.Forward(h, b, positions, B, L, &m.TextConfig, sliding, full)+ h = layer.Forward(h, ab, positions, B, L, &m.TextConfig, sliding, full)}hidden = mlx.RMSNormFn(h, m.NormScaled, m.TextConfig.RMSNormEps)- projected := m.PostProjection.Forward(hidden)- return m.unembed(hidden), projected+ projected = m.PostProjection.Forward(hidden)+ return hidden, projected}func (m *AssistantModel) sharedHistories(b *batch.Batch, caches []cache.Cache) (sliding, full *nn.KVHistory) {@@ -302,7 +320,7 @@ func (m *AssistantModel) sharedHistories(b *batch.Batch, caches []cache.Cache) (return sliding, full}-func (m *AssistantModel) unembed(hidden *mlx.Array) *mlx.Array {+func (m *AssistantModel) Unembed(hidden *mlx.Array) *mlx.Array {if m.UseOrderedEmbeddings {return m.applyCentroidMasking(hidden)}