Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
qwen3next: fix issue in delta net (#14075)
Jeffrey Morgan255579aaa7d2
diff --git a/model/models/qwen3next/deltanet.go b/model/models/qwen3next/deltanet.goindex 958d1e937..e0a6f7b25 100644--- a/model/models/qwen3next/deltanet.go+++ b/model/models/qwen3next/deltanet.go@@ -406,8 +406,10 @@ func (gdn *GatedDeltaNet) deltaNetChunked(gDiff := gCumsum.Neg(ctx).Add(ctx, gLast)gDiffExp := gDiff.Exp(ctx)- // key_gdiff = k * exp(g_diff)- keyGDiff := k.Mul(ctx, gDiffExp)+ // Reshapes g_diff_exp to [1, chunkSize, nChunks, ...]+ gDiffExpReshaped := gDiffExp.Reshape(ctx, 1, chunkSize, nChunks, numVHeads*nSeqs)+ keyGDiff := k.Mul(ctx, gDiffExpReshaped)+ keyGDiffT := keyGDiff.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)// Process chunks and update statevar coreAttnOut ml.Tensor@@ -444,12 +446,9 @@ func (gdn *GatedDeltaNet) deltaNetChunked(coreAttnOut = coreAttnOut.Concat(ctx, coreAttnOutChunk, 1)}- // Update state for next chunk using pre-computed values+ // Update state for next chunkgExpLastChunk := gLastExp.Slice(ctx, 2, chunk, chunk+1, 1)- kGDiffChunk := keyGDiff.Slice(ctx, 2, chunk, chunk+1, 1)-- // kgdmulvnew = key_gdiff^T @ v_new- kGDiffChunkT := kGDiffChunk.Permute(ctx, 1, 0, 2, 3).Contiguous(ctx)+ kGDiffChunkT := keyGDiffT.Slice(ctx, 2, chunk, chunk+1, 1)kgdMulVNew := vNewT.Mulmat(ctx, kGDiffChunkT)// state = state * g_last + kgdmulvnew