Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
ci: wire up MLX unit tests for PR runs (#17022)
Daniel Hiltgene5e437711540
diff --git a/.github/scripts/prepare_mlx_darwin.sh b/.github/scripts/prepare_mlx_darwin.shnew file mode 100755index 000000000..6b2d083f1--- /dev/null+++ b/.github/scripts/prepare_mlx_darwin.sh@@ -0,0 +1,159 @@+#!/usr/bin/env bash++# Prepare prebuilt MLX Metal runtime libraries for macOS CI unit tests.+#+# Building MLX is expensive, so to enable the MLX-specific unit tests this+# helper finds the newest Ollama release whose MLX_VERSION and MLX_C_VERSION+# match the current checkout, downloads that release's ollama-darwin.tgz, and+# extracts only mlx_metal_v* into build/lib/ollama+#+# If no matching release artifact exists, the helper only emits a warning. That+# covers the expected window after an MLX update lands and before the next release+# publishes a matching Darwin payload.++set -euo pipefail++repo="${OLLAMA_MLX_RELEASE_REPO:-ollama/ollama}"+scan_limit="${OLLAMA_MLX_RELEASE_SCAN_LIMIT:-50}"+cache_dir="${OLLAMA_MLX_DARWIN_CACHE:-.cache/mlx-darwin-release}"+target_dir="${OLLAMA_MLX_DARWIN_TARGET:-build/lib/ollama}"+tarball="${cache_dir}/ollama-darwin.tgz"+tag_file="${cache_dir}/matched-tag"+pins_file="${cache_dir}/matched-pins"+target_pins_file="${target_dir}/.mlx-release-pins"+tmpdir=""+tmp_tarball=""++cleanup() {+ [ -z "${tmpdir}" ] || rm -rf "${tmpdir}"+ [ -z "${tmp_tarball}" ] || rm -f "${tmp_tarball}"+}+trap cleanup EXIT++warn() {+ if [ -n "${GITHUB_ACTIONS:-}" ]; then+ echo "::warning::$*"+ else+ echo "warning: $*" >&2+ fi+}++read_pin() {+ tr -d '[:space:]' <"$1"+}++has_payload() {+ local variant+ for variant in "${target_dir}"/mlx_metal_v*; do+ [ -d "${variant}" ] || continue+ [ -f "${variant}/libmlx.dylib" ] && [ -f "${variant}/libmlxc.dylib" ] && return 0+ done+ return 1+}++has_matching_payload() {+ [ -f "${target_pins_file}" ] || return 1+ [ "$(cat "${target_pins_file}")" = "${current_pins}" ] || return 1+ has_payload+}++extract_payload() {+ local tag="$1"+ tmpdir="$(mktemp -d)"++ tar -xzf "${tarball}" -C "${tmpdir}"+ mkdir -p "${target_dir}"++ rm -rf "${target_dir}"/mlx_metal_v*++ local found=false+ local src dest+ for src in "${tmpdir}"/mlx_metal_v*; do+ [ -d "${src}" ] || continue+ found=true+ dest="${target_dir}/$(basename "${src}")"+ rm -rf "${dest}"+ cp -R "${src}" "${dest}"+ done++ if [ "${found}" != true ] || ! has_payload; then+ echo "Downloaded ${tarball} did not contain a usable MLX Metal payload" >&2+ exit 1+ fi++ echo "${current_pins}" >"${target_pins_file}"+ echo "Prepared MLX Darwin payload from ${repo} ${tag}:"+ find "${target_dir}" -maxdepth 2 -type f \( -name 'libmlx.dylib' -o -name 'libmlxc.dylib' -o -name '*.metallib' \) -print++ rm -rf "${tmpdir}"+ tmpdir=""+}++if [ "$(uname -s)" != "Darwin" ]; then+ warn "MLX Darwin payload setup is only supported on macOS"+ exit 0+fi++current_mlx="$(read_pin MLX_VERSION)"+current_mlxc="$(read_pin MLX_C_VERSION)"+current_pins="${current_mlx} ${current_mlxc}"++if has_matching_payload; then+ echo "MLX Darwin payload already present in ${target_dir}"+ exit 0+fi++mkdir -p "${cache_dir}"++if [ -s "${tarball}" ] && [ -f "${tag_file}" ] && [ "$(cat "${pins_file}" 2>/dev/null || true)" = "${current_pins}" ]; then+ extract_payload "$(cat "${tag_file}")"+ exit 0+fi++matched_tag=""+matched_url=""++while read -r tag; do+ [ -n "${tag}" ] || continue++ if ! tag_mlx="$(curl -fsSL "https://raw.githubusercontent.com/${repo}/${tag}/MLX_VERSION" | tr -d '[:space:]')"; then+ continue+ fi+ if [ "${tag_mlx}" != "${current_mlx}" ]; then+ continue+ fi++ if ! tag_mlxc="$(curl -fsSL "https://raw.githubusercontent.com/${repo}/${tag}/MLX_C_VERSION" | tr -d '[:space:]')"; then+ continue+ fi+ if [ "${tag_mlxc}" != "${current_mlxc}" ]; then+ continue+ fi++ url="https://github.com/${repo}/releases/download/${tag}/ollama-darwin.tgz"+ if curl -fsIL "${url}" >/dev/null; then+ matched_tag="${tag}"+ matched_url="${url}"+ break+ fi++ echo "MLX pins match ${tag}, but ${url} is not available"+done < <(+ git ls-remote --tags --refs --sort=-version:refname "https://github.com/${repo}.git" 'v*' |+ awk -v limit="${scan_limit}" '{ sub("refs/tags/", "", $2); print $2; if (limit > 0 && NR >= limit) exit }'+)++if [ -z "${matched_tag}" ]; then+ warn "MLX unit tests are temporarily disabled until a release build publishes ollama-darwin.tgz for MLX_VERSION=${current_mlx} MLX_C_VERSION=${current_mlxc}"+ exit 0+fi++tmp_tarball="${tarball}.tmp"+rm -f "${tmp_tarball}"+curl -fL --retry 3 --retry-delay 2 -o "${tmp_tarball}" "${matched_url}"+mv "${tmp_tarball}" "${tarball}"+tmp_tarball=""+echo "${matched_tag}" >"${tag_file}"+echo "${current_pins}" >"${pins_file}"++extract_payload "${matched_tag}"diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yamlindex 8a0fec6c1..733d39e64 100644--- a/.github/workflows/test.yaml+++ b/.github/workflows/test.yaml@@ -414,6 +414,15 @@ jobs:- uses: actions/setup-node@v4with:node-version: '20'+ - name: Cache MLX Darwin release payload+ if: ${{ startsWith(matrix.os, 'macos') }}+ uses: actions/cache@v4+ with:+ path: .cache/mlx-darwin-release+ key: mlx-darwin-${{ hashFiles('MLX_VERSION', 'MLX_C_VERSION') }}+ - name: Prepare MLX Darwin release payload+ if: ${{ startsWith(matrix.os, 'macos') }}+ run: .github/scripts/prepare_mlx_darwin.sh- name: Install UI dependenciesworking-directory: ./app/ui/apprun: npm ci@@ -438,12 +447,57 @@ jobs:- name: Run go generaterun: go generate ./...+ - name: Verify UI generated types are current+ if: ${{ startsWith(matrix.os, 'ubuntu') }}+ run: git diff --exit-code -- app/ui/app/codegen/gotypes.gen.ts+- name: go testif: always()- run: go test -count=1 -benchtime=1x ./...+ # Smoke-run each benchmark once to catch panics and bit rot; this does+ # not assert timings. -benchtime without -bench is inert.+ run: go test -count=1 -bench=. -benchtime=1x ./...- name: go test app with live updater tagif: ${{ needs.changes.outputs.app_changed == 'True' && contains(fromJSON('["macos-latest","windows-latest"]'), matrix.os) }}run: go test -count=1 -tags updater_live ./app/...- uses: golangci/golangci-lint-action@v9++ race:+ strategy:+ matrix:+ os: [ubuntu-latest, macos-latest]+ runs-on: ${{ matrix.os }}+ env:+ CGO_ENABLED: '1'+ steps:+ - uses: actions/checkout@v4+ - uses: actions/setup-go@v5+ with:+ go-version-file: 'go.mod'+ cache-dependency-path: |+ go.sum+ LLAMA_CPP_VERSION+ MLX_VERSION+ MLX_C_VERSION+ - name: Cache MLX Darwin release payload+ if: ${{ startsWith(matrix.os, 'macos') }}+ uses: actions/cache@v4+ with:+ path: .cache/mlx-darwin-release+ key: mlx-darwin-${{ hashFiles('MLX_VERSION', 'MLX_C_VERSION') }}+ - name: Prepare MLX Darwin release payload+ if: ${{ startsWith(matrix.os, 'macos') }}+ run: .github/scripts/prepare_mlx_darwin.sh+ - uses: actions/setup-node@v4+ with:+ node-version: '20'+ # app/ui embeds app/dist, so the UI has to be built before app/... will+ # even compile.+ - name: Build UI+ working-directory: ./app/ui/app+ run: |+ npm ci+ npm run build+ - name: go test -race+ run: go test -race -count=1 ./...diff --git a/app/server/server_test.go b/app/server/server_test.goindex cb9674229..fb29fc169 100644--- a/app/server/server_test.go+++ b/app/server/server_test.go@@ -365,7 +365,7 @@ time=2025-06-30T09:25:56.197-07:00 level=DEBUG source=ggml.go:155 msg="key not fif err != nil {t.Fatalf("failed to write log file %s: %s", serverLogPath, err)}- ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond)+ ctx, cancel := context.WithTimeout(t.Context(), time.Second)defer cancel()info, err := GetInferenceInfo(ctx)if err != nil {diff --git a/app/ui/ui_test.go b/app/ui/ui_test.goindex a63bd608f..95cc88a5a 100644--- a/app/ui/ui_test.go+++ b/app/ui/ui_test.go@@ -997,7 +997,7 @@ func TestSettingsToggleAutoUpdateOn_WithPendingUpdate_ShowsNotification(t *testi}}-func TestSettingsToggleAutoUpdateOn_NoPendingUpdate_TriggersCheck(t *testing.T) {+func TestSettingsToggleAutoUpdateOn_NoPendingUpdate_DoesNotNotify(t *testing.T) {testStore := &store.Store{DBPath: filepath.Join(t.TempDir(), "db.sqlite"),}@@ -1027,12 +1027,6 @@ func TestSettingsToggleAutoUpdateOn_NoPendingUpdate_TriggersCheck(t *testing.T)}}defer upd.Store.Close()- // Initialize the checkNow channel by starting (and immediately stopping) the checker- // so TriggerImmediateCheck doesn't panic on nil channel- ctx, cancel := context.WithCancel(t.Context())- upd.StartBackgroundUpdaterChecker(ctx, func(string) error { return nil })- defer cancel()-var notificationCalled atomic.Boolserver := &Server{Store: testStore,diff --git a/app/updater/updater.go b/app/updater/updater.goindex e042807fe..53051c0e8 100644--- a/app/updater/updater.go+++ b/app/updater/updater.go@@ -353,11 +353,23 @@ func (u *Updater) TriggerImmediateCheck() {}func (u *Updater) StartBackgroundUpdaterChecker(ctx context.Context, cb func(string) error) {+ u.startBackgroundUpdaterChecker(ctx, cb)+}++func (u *Updater) startBackgroundUpdaterChecker(ctx context.Context, cb func(string) error) <-chan struct{} {u.checkNow = make(chan struct{}, 1)u.checkNow <- struct{}{} // Trigger first check after initial delay+ done := make(chan struct{})go func() {+ defer close(done)// Don't blast an update message immediately after startup- time.Sleep(UpdateCheckInitialDelay)+ initialDelay := time.NewTimer(UpdateCheckInitialDelay)+ defer initialDelay.Stop()+ select {+ case <-ctx.Done():+ return+ case <-initialDelay.C:+ }slog.Info("beginning update checker", "interval", UpdateCheckInterval)ticker := time.NewTicker(UpdateCheckInterval)defer ticker.Stop()@@ -406,4 +418,5 @@ func (u *Updater) StartBackgroundUpdaterChecker(ctx context.Context, cb func(str}}}()+ return done}diff --git a/app/updater/updater_test.go b/app/updater/updater_test.goindex 7a27599cd..f31ea20f1 100644--- a/app/updater/updater_test.go+++ b/app/updater/updater_test.go@@ -190,6 +190,20 @@ func TestDownloadNewReleaseDoesNotUseRawETagAsPathComponent(t *testing.T) {}}+// stopChecker cancels the background update checker and waits for its+// goroutine to return. Tests must join it before returning: the goroutine+// reads package-level knobs (UpdateCheckURLBase, UpdateCheckInterval, ...)+// that the next test rewrites.+func stopChecker(t *testing.T, cancel context.CancelFunc, done <-chan struct{}) {+ t.Helper()+ cancel()+ select {+ case <-done:+ case <-time.After(10 * time.Second):+ t.Error("background update checker did not stop")+ }+}+// waitDownloadIdle blocks until no download is in flight, so staged-file// handles close before t.TempDir cleanup removes the stage directory. After// the context is cancelled a new download can't write (it aborts at the HEAD@@ -289,11 +303,12 @@ func TestBackgroundCheckerSkipsAlreadyStagedETagDownload(t *testing.T) {defer cancel()callbacks := make(chan string, 4)- updater.StartBackgroundUpdaterChecker(ctx, func(ver string) error {+ checkerDone := updater.startBackgroundUpdaterChecker(ctx, func(ver string) error {callbacks <- verreturn nil})- t.Cleanup(updater.waitDownloadIdle)+ defer updater.waitDownloadIdle()+ defer stopChecker(t, cancel, checkerDone)for range 2 {select {@@ -334,10 +349,16 @@ func TestBackgoundChecker(t *testing.T) {UpdateStageDir = t.TempDir()haveUpdate := falseverified := false- done := make(chan int)+ // Buffered + non-blocking send: the checker keeps calling cb every+ // UpdateCheckInterval, and a blocking send would wedge its goroutine once+ // the test stops receiving.+ done := make(chan int, 1)cb := func(ver string) error {haveUpdate = true- done <- 0+ select {+ case done <- 0:+ default:+ }return nil}stallTimer := time.NewTimer(5 * time.Second)@@ -381,8 +402,9 @@ func TestBackgoundChecker(t *testing.T) {t.Fatal(err)}- updater.StartBackgroundUpdaterChecker(ctx, cb)- t.Cleanup(updater.waitDownloadIdle)+ checkerDone := updater.startBackgroundUpdaterChecker(ctx, cb)+ defer updater.waitDownloadIdle()+ defer stopChecker(t, cancel, checkerDone)select {case <-stallTimer.C:t.Fatal("stalled")@@ -440,12 +462,13 @@ func TestAutoUpdateDisabledSkipsDownload(t *testing.T) {}cb := func(ver string) error {- t.Fatal("callback should not be called when auto-update is disabled")+ t.Error("callback should not be called when auto-update is disabled")return nil}- updater.StartBackgroundUpdaterChecker(ctx, cb)- t.Cleanup(updater.waitDownloadIdle)+ checkerDone := updater.startBackgroundUpdaterChecker(ctx, cb)+ defer updater.waitDownloadIdle()+ defer stopChecker(t, cancel, checkerDone)// Wait enough time for multiple check cyclestime.Sleep(50 * time.Millisecond)@@ -507,8 +530,9 @@ func TestAutoUpdateReenabledDownloadsUpdate(t *testing.T) {return nil}- upd.StartBackgroundUpdaterChecker(ctx, cb)- t.Cleanup(upd.waitDownloadIdle)+ checkerDone := upd.startBackgroundUpdaterChecker(ctx, cb)+ defer upd.waitDownloadIdle()+ defer stopChecker(t, cancel, checkerDone)// Wait for a few cycles with auto-update disabled - no download should happentime.Sleep(50 * time.Millisecond)@@ -641,8 +665,9 @@ func TestTriggerImmediateCheck(t *testing.T) {return nil}- updater.StartBackgroundUpdaterChecker(ctx, cb)- t.Cleanup(updater.waitDownloadIdle)+ checkerDone := updater.startBackgroundUpdaterChecker(ctx, cb)+ defer updater.waitDownloadIdle()+ defer stopChecker(t, cancel, checkerDone)// Wait for the initial check that fires after the initial delayselect {diff --git a/x/internal/mlxtest/mlxtest.go b/x/internal/mlxtest/mlxtest.goindex ef006abb2..d19fa1892 100644--- a/x/internal/mlxtest/mlxtest.go+++ b/x/internal/mlxtest/mlxtest.go@@ -3,39 +3,51 @@package mlxtestimport (- "runtime"+ "sync""testing"+ "github.com/ollama/ollama/x/internal/mlxthreadtest""github.com/ollama/ollama/x/mlxrunner/mlx")+var testThread = sync.OnceValues(func() (*mlxthreadtest.Thread, error) {+ return mlxthreadtest.Start("mlx-test", func() error {+ if err := mlx.CheckInit(); err != nil {+ return err+ }+ if mlx.GPUIsAvailable() {+ mlx.SetDefaultDeviceGPU()+ }+ return nil+ })+})++// T is the test state available to callbacks running on the MLX thread.+type T = mlxthreadtest.T+// SkipIfUnavailable skips the test when the MLX dynamic library cannot be// loaded (e.g. no MLX backend built for this platform).func SkipIfUnavailable(t *testing.T) {t.Helper()- if err := mlx.CheckInit(); err != nil {+ if _, err := testThread(); err != nil {t.Skipf("MLX not available: %v", err)}}-// Setup prepares a test that calls into MLX natively: it skips when MLX is-// unavailable and pins the test goroutine to its OS thread for the duration-// of the test.-//-// The thread pin is load-bearing, not defensive: MLX's default stream cache-// is thread-local, and anything that migrates the goroutine mid-test (the-// race detector's scheduler in particular) otherwise panics with-// "There is no Stream(gpu, 0) in current thread".-//-// Setup deliberately does not switch devices or sweep caches: switching the-// default device re-creates the process-wide default stream, and sweeping the-// allocator cache between tests changes allocator reuse — both perturbed-// tests that share lazy arrays with subtests running on other threads.-func Setup(t *testing.T) {+// Run executes fn on the MLX thread shared by the package's test binary.+func Run(t *testing.T, fn func(*T)) {t.Helper()- SkipIfUnavailable(t)+ thread, err := testThread()+ if err != nil {+ t.Skipf("MLX not available: %v", err)+ }- runtime.LockOSThread()- t.Cleanup(runtime.UnlockOSThread)+ mlxthreadtest.Run(t, thread, fn)+}++// RunSubtest runs a named subtest on the shared MLX test thread.+func RunSubtest(t *testing.T, name string, fn func(*T)) {+ t.Helper()+ t.Run(name, func(t *testing.T) { Run(t, fn) })}diff --git a/x/internal/mlxthreadtest/mlxthreadtest.go b/x/internal/mlxthreadtest/mlxthreadtest.gonew file mode 100644index 000000000..5e43ea384--- /dev/null+++ b/x/internal/mlxthreadtest/mlxthreadtest.go@@ -0,0 +1,211 @@+// Package mlxthreadtest runs tests on a persistent MLX worker thread.+package mlxthreadtest++import (+ "context"+ "testing"++ "github.com/ollama/ollama/x/internal/mlxthread"+)++// Thread is a pinned worker used by MLX tests.+type Thread struct {+ worker *mlxthread.Thread+ id uint64+}++// Start creates a pinned test worker.+func Start(name string, init func() error) (*Thread, error) {+ t := &Thread{}+ thread, err := mlxthread.Start(name, func() error {+ t.id = currentThreadID()+ if init != nil {+ return init()+ }+ return nil+ })+ if err != nil {+ return nil, err+ }+ t.worker = thread+ return t, nil+}++// Do runs fn on the pinned worker.+func (t *Thread) Do(ctx context.Context, fn func() error) error {+ if t.onWorker() {+ panic("mlxthreadtest.Thread.Do called from its pinned worker")+ }+ return t.worker.Do(ctx, fn)+}++// Stop shuts down the pinned worker after running cleanup on it.+func (t *Thread) Stop(ctx context.Context, cleanup func()) error {+ if t.onWorker() {+ panic("mlxthreadtest.Thread.Stop called from its pinned worker")+ }+ return t.worker.Stop(ctx, cleanup)+}++func (t *Thread) onWorker() bool {+ id := currentThreadID()+ return id != 0 && id == t.id+}++// T is the subset of testing.T supported by MLX test bodies. Operations that+// end a test are replayed by Run on the test goroutine so the MLX worker remains+// alive.+type T struct {+ testReporter+ cleanups []func()+ skipped bool+ aborted bool+}++// abortPanic unwinds a test body without terminating the worker goroutine.+var abortPanic = new(struct{ marker byte })++type testReporter interface {+ Error(...any)+ Errorf(string, ...any)+ Fail()+ Failed() bool+ Helper()+ Log(...any)+ Logf(string, ...any)+}++// Run executes fn on thread. The callback must use its T argument; calling+// FailNow or SkipNow on a captured *testing.T terminates the pinned worker.+func Run(t *testing.T, thread *Thread, fn func(*T)) {+ t.Helper()+ if thread.onWorker() {+ panic("mlxthreadtest.Run called recursively from its pinned worker")+ }++ mt := &T{testReporter: t}+ result := make(chan runResult, 1)+ go func() {+ defer func() {+ if v := recover(); v != nil {+ result <- runResult{panicValue: v}+ }+ }()+ err := thread.Do(context.Background(), func() error {+ returned := false+ defer func() {+ if v := recover(); v != nil {+ panic(v)+ }+ if !returned {+ result <- runResult{goexit: true}+ }+ }()++ mt.run(fn)+ returned = true+ return nil+ })+ result <- runResult{err: err}+ }()++ res := <-result+ if res.goexit {+ panic("pinned test body called runtime.Goexit; use the test value passed to the callback")+ }+ if res.panicValue != nil {+ panic(res.panicValue)+ }+ if res.err != nil {+ t.Fatal(res.err)+ }+ if mt.skipped {+ t.SkipNow()+ }+ if mt.aborted {+ t.FailNow()+ }+}++type runResult struct {+ err error+ panicValue any+ goexit bool+}++// Cleanup registers fn to run on the MLX thread after the current body.+func (t *T) Cleanup(fn func()) {+ t.cleanups = append(t.cleanups, fn)+}++func (t *T) FailNow() {+ t.Fail()+ t.aborted = true+ panic(abortPanic)+}++func (t *T) Fatal(args ...any) {+ t.Helper()+ t.Error(args...)+ t.aborted = true+ panic(abortPanic)+}++func (t *T) Fatalf(format string, args ...any) {+ t.Helper()+ t.Errorf(format, args...)+ t.aborted = true+ panic(abortPanic)+}++func (t *T) Skip(args ...any) {+ t.Helper()+ t.Log(args...)+ t.SkipNow()+}++func (t *T) Skipf(format string, args ...any) {+ t.Helper()+ t.Logf(format, args...)+ t.SkipNow()+}++func (t *T) SkipNow() {+ t.skipped = true+ t.aborted = true+ panic(abortPanic)+}++func (t *T) Skipped() bool {+ return t.skipped+}++func (t *T) run(fn func(*T)) {+ defer func() {+ if v := recover(); v != nil && v != abortPanic {+ panic(v)+ }+ }()+ defer t.runCleanups()+ fn(t)+}++func (t *T) runCleanups() {+ var panicValue any+ for len(t.cleanups) > 0 {+ last := len(t.cleanups) - 1+ cleanup := t.cleanups[last]+ t.cleanups = t.cleanups[:last]+ func() {+ defer func() {+ if v := recover(); v != nil && v != abortPanic && panicValue == nil {+ panicValue = v+ }+ }()+ cleanup()+ }()+ }+ if panicValue != nil {+ panic(panicValue)+ }+}diff --git a/x/internal/mlxthreadtest/mlxthreadtest_test.go b/x/internal/mlxthreadtest/mlxthreadtest_test.gonew file mode 100644index 000000000..4009a02c1--- /dev/null+++ b/x/internal/mlxthreadtest/mlxthreadtest_test.go@@ -0,0 +1,379 @@+package mlxthreadtest++import (+ "context"+ "fmt"+ "os"+ "os/exec"+ "path/filepath"+ "reflect"+ "strings"+ "testing"+ "time"+)++type fakeReporter struct{ failed bool }++func (t *fakeReporter) Error(...any) { t.failed = true }+func (t *fakeReporter) Errorf(string, ...any) { t.failed = true }+func (t *fakeReporter) Fail() { t.failed = true }+func (t *fakeReporter) Failed() bool { return t.failed }+func (*fakeReporter) Helper() {}+func (*fakeReporter) Log(...any) {}+func (*fakeReporter) Logf(string, ...any) {}++func TestControlsDoNotStopWorker(t *testing.T) {+ thread, err := Start("test", nil)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ if err := thread.Stop(context.Background(), nil); err != nil {+ t.Error(err)+ }+ })++ tests := []struct {+ name string+ body func(*T)+ wantFailed bool+ wantSkipped bool+ }{+ {name: "fail now", body: func(t *T) { t.FailNow() }, wantFailed: true},+ {name: "fatal", body: func(t *T) { t.Fatal("stop") }, wantFailed: true},+ {name: "fatalf", body: func(t *T) { t.Fatalf("%s", "stop") }, wantFailed: true},+ {name: "skip now", body: func(t *T) { t.SkipNow() }, wantSkipped: true},+ {name: "skip", body: func(t *T) { t.Skip("stop") }, wantSkipped: true},+ {name: "skipf", body: func(t *T) { t.Skipf("%s", "stop") }, wantSkipped: true},+ {+ name: "failed skip",+ body: func(t *T) {+ t.Error("failed")+ t.Skip("stop")+ },+ wantFailed: true,+ wantSkipped: true,+ },+ {+ name: "cleanup fail now",+ body: func(t *T) {+ t.Cleanup(func() { t.FailNow() })+ },+ wantFailed: true,+ },+ {+ name: "cleanup fatal",+ body: func(t *T) {+ t.Cleanup(func() { t.Fatal("stop") })+ },+ wantFailed: true,+ },+ {+ name: "cleanup skip now",+ body: func(t *T) {+ t.Cleanup(func() { t.SkipNow() })+ },+ wantSkipped: true,+ },+ {+ name: "cleanup skip",+ body: func(t *T) {+ t.Cleanup(func() { t.Skip("stop") })+ },+ wantSkipped: true,+ },+ }+ for _, tt := range tests {+ reporter := &fakeReporter{}+ mt := &T{testReporter: reporter}+ if err := thread.Do(context.Background(), func() error {+ mt.run(tt.body)+ return nil+ }); err != nil {+ t.Fatalf("%s: %v", tt.name, err)+ }+ if mt.Failed() != tt.wantFailed || mt.skipped != tt.wantSkipped {+ t.Fatalf("%s: failed:%v skipped:%v", tt.name, mt.Failed(), mt.skipped)+ }++ ran := false+ if err := thread.Do(context.Background(), func() error {+ ran = true+ return nil+ }); err != nil {+ t.Fatalf("%s follow-up: %v", tt.name, err)+ }+ if !ran {+ t.Fatalf("worker did not run after %s", tt.name)+ }+ }++ reached := false+ t.Run("replay skip", func(t *testing.T) {+ Run(t, thread, func(t *T) { t.Skip("stop") })+ reached = true+ })+ if reached {+ t.Fatal("Run returned after Skip")+ }+ if err := thread.Do(context.Background(), func() error { return nil }); err != nil {+ t.Fatalf("worker did not survive replayed Skip: %v", err)+ }+}++func TestRunReplaysFatalWithoutStoppingWorker(t *testing.T) {+ if marker := os.Getenv("OLLAMA_TEST_MLX_FATAL_MARKER"); marker != "" {+ thread, err := Start("test", nil)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ if err := thread.Stop(context.Background(), nil); err != nil {+ t.Error(err)+ }+ })+ t.Cleanup(func() {+ if err := thread.Do(context.Background(), func() error { return nil }); err != nil {+ t.Errorf("worker did not survive replayed Fatal: %v", err)+ return+ }+ if err := os.WriteFile(marker, nil, 0o600); err != nil {+ t.Error(err)+ }+ })++ Run(t, thread, func(t *T) { t.Fatal("stop") })+ return+ }++ marker := filepath.Join(t.TempDir(), "worker-survived")+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)+ defer cancel()+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunReplaysFatalWithoutStoppingWorker$", "-test.timeout=2s")+ cmd.Env = append(os.Environ(), "OLLAMA_TEST_MLX_FATAL_MARKER="+marker)+ output, err := cmd.CombinedOutput()+ if err == nil {+ t.Fatal("subprocess succeeded, want Fatal failure")+ }+ if ctx.Err() != nil {+ t.Fatalf("subprocess hung: %v", ctx.Err())+ }+ if !strings.Contains(string(output), "stop") {+ t.Fatalf("missing Fatal diagnostic:\n%s", output)+ }+ if _, err := os.Stat(marker); err != nil {+ t.Fatalf("worker survival check did not complete: %v\n%s", err, output)+ }+}++func TestRunRejectsRecursiveDispatch(t *testing.T) {+ if currentThreadID() == 0 {+ t.Skip("OS thread IDs are not available on this platform")+ }++ thread, err := Start("test", nil)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ if err := thread.Stop(context.Background(), nil); err != nil {+ t.Error(err)+ }+ })++ tests := []struct {+ name string+ want string+ body func(*testing.T)+ }{+ {+ name: "Run",+ want: "called recursively",+ body: func(t *testing.T) { Run(t, thread, func(*T) {}) },+ },+ {+ name: "Do",+ want: "Thread.Do called from its pinned worker",+ body: func(*testing.T) {+ _ = thread.Do(context.Background(), func() error { return nil })+ },+ },+ {+ name: "Stop",+ want: "Thread.Stop called from its pinned worker",+ body: func(*testing.T) { _ = thread.Stop(context.Background(), nil) },+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ var got any+ func() {+ defer func() { got = recover() }()+ Run(t, thread, func(*T) { tt.body(t) })+ }()+ if !strings.Contains(fmt.Sprint(got), tt.want) {+ t.Fatalf("got panic %v, want %q", got, tt.want)+ }+ if err := thread.Do(context.Background(), func() error { return nil }); err != nil {+ t.Fatalf("worker did not survive recursive %s: %v", tt.name, err)+ }+ })+ }+}++func TestRunRejectsTestingTGoexit(t *testing.T) {+ if method := os.Getenv("OLLAMA_TEST_MLX_GOEXIT"); method != "" {+ thread, err := Start("test", nil)+ if err != nil {+ t.Fatal(err)+ }+ Run(t, thread, func(*T) {+ switch method {+ case "fail":+ t.FailNow()+ case "skip":+ t.SkipNow()+ default:+ panic("unknown Goexit test method")+ }+ })+ return+ }++ for _, method := range []string{"fail", "skip"} {+ t.Run(method, func(t *testing.T) {+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)+ defer cancel()+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunRejectsTestingTGoexit$", "-test.timeout=2s")+ cmd.Env = append(os.Environ(), "OLLAMA_TEST_MLX_GOEXIT="+method)+ output, err := cmd.CombinedOutput()+ if err == nil {+ t.Fatal("subprocess succeeded, want misuse failure")+ }+ if ctx.Err() != nil {+ t.Fatalf("subprocess hung: %v", ctx.Err())+ }+ if !strings.Contains(string(output), "pinned test body called runtime.Goexit") {+ t.Fatalf("missing Goexit diagnostic:\n%s", output)+ }+ })+ }+}++func TestCleanupOrder(t *testing.T) {+ reporter := &fakeReporter{}+ mt := &T{testReporter: reporter}+ var got []string++ mt.run(func(t *T) {+ t.Cleanup(func() { got = append(got, "cleanup 1") })+ t.Cleanup(func() {+ got = append(got, "cleanup 2")+ t.Cleanup(func() { got = append(got, "cleanup 3") })+ })+ got = append(got, "body")+ })++ want := []string{"body", "cleanup 2", "cleanup 3", "cleanup 1"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("got %v, want %v", got, want)+ }+}++func TestCleanupFailureDoesNotHideBodyPanic(t *testing.T) {+ reporter := &fakeReporter{}+ mt := &T{testReporter: reporter}+ var got any+ func() {+ defer func() { got = recover() }()+ mt.run(func(t *T) {+ t.Cleanup(func() { t.Fatal("cleanup failure") })+ panic("body panic")+ })+ }()+ if got != "body panic" {+ t.Fatalf("got panic %v, want body panic", got)+ }+ if !reporter.failed {+ t.Fatal("cleanup Fatal did not fail the test")+ }+}++func TestCleanupPanicRunsRemainingCleanups(t *testing.T) {+ reporter := &fakeReporter{}+ mt := &T{testReporter: reporter}+ var cleanups []string+ var got any+ func() {+ defer func() { got = recover() }()+ mt.run(func(t *T) {+ t.Cleanup(func() { cleanups = append(cleanups, "first") })+ t.Cleanup(func() {+ cleanups = append(cleanups, "second")+ panic("cleanup panic")+ })+ })+ }()+ if got != "cleanup panic" {+ t.Fatalf("got panic %v, want cleanup panic", got)+ }+ if want := []string{"second", "first"}; !reflect.DeepEqual(cleanups, want) {+ t.Fatalf("got cleanups %v, want %v", cleanups, want)+ }+}++func TestCleanupRunsOnWorker(t *testing.T) {+ thread, err := Start("test", nil)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ if err := thread.Stop(context.Background(), nil); err != nil {+ t.Error(err)+ }+ })++ var bodyID, cleanupID uint64+ Run(t, thread, func(t *T) {+ bodyID = currentThreadID()+ t.Cleanup(func() { cleanupID = currentThreadID() })+ })+ if bodyID != thread.id || cleanupID != thread.id {+ t.Fatalf("body thread %d, cleanup thread %d, want worker thread %d", bodyID, cleanupID, thread.id)+ }+}++func panicAtTestBody() {+ panic("test panic")+}++func TestPanicIncludesTestBodyStack(t *testing.T) {+ thread, err := Start("test", nil)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ if err := thread.Stop(context.Background(), nil); err != nil {+ t.Error(err)+ }+ })++ var cleanupRan bool+ var got any+ func() {+ defer func() { got = recover() }()+ Run(t, thread, func(t *T) {+ t.Cleanup(func() { cleanupRan = true })+ panicAtTestBody()+ })+ }()+ if !cleanupRan {+ t.Fatal("cleanup did not run after panic")+ }+ if !strings.Contains(fmt.Sprint(got), "panicAtTestBody") {+ t.Fatalf("panic stack does not include test body:\n%v", got)+ }+ if err := thread.Do(context.Background(), func() error { return nil }); err != nil {+ t.Fatalf("worker did not survive panic: %v", err)+ }+}diff --git a/x/internal/mlxthreadtest/threadid_darwin.go b/x/internal/mlxthreadtest/threadid_darwin.gonew file mode 100644index 000000000..48d3eb685--- /dev/null+++ b/x/internal/mlxthreadtest/threadid_darwin.go@@ -0,0 +1,10 @@+//go:build darwin++package mlxthreadtest++import "syscall"++func currentThreadID() uint64 {+ id, _, _ := syscall.RawSyscall(syscall.SYS_THREAD_SELFID, 0, 0, 0)+ return uint64(id)+}diff --git a/x/internal/mlxthreadtest/threadid_linux.go b/x/internal/mlxthreadtest/threadid_linux.gonew file mode 100644index 000000000..5c81aa74c--- /dev/null+++ b/x/internal/mlxthreadtest/threadid_linux.go@@ -0,0 +1,9 @@+//go:build linux++package mlxthreadtest++import "syscall"++func currentThreadID() uint64 {+ return uint64(syscall.Gettid())+}diff --git a/x/internal/mlxthreadtest/threadid_other.go b/x/internal/mlxthreadtest/threadid_other.gonew file mode 100644index 000000000..4f87279c3--- /dev/null+++ b/x/internal/mlxthreadtest/threadid_other.go@@ -0,0 +1,7 @@+//go:build !darwin && !linux && !windows++package mlxthreadtest++func currentThreadID() uint64 {+ return 0+}diff --git a/x/internal/mlxthreadtest/threadid_windows.go b/x/internal/mlxthreadtest/threadid_windows.gonew file mode 100644index 000000000..69454f1a1--- /dev/null+++ b/x/internal/mlxthreadtest/threadid_windows.go@@ -0,0 +1,9 @@+//go:build windows++package mlxthreadtest++import "golang.org/x/sys/windows"++func currentThreadID() uint64 {+ return uint64(windows.GetCurrentThreadId())+}diff --git a/x/mlxrunner/cache/cache_test.go b/x/mlxrunner/cache/cache_test.goindex 22ecbdbfa..408aa7da6 100644--- a/x/mlxrunner/cache/cache_test.go+++ b/x/mlxrunner/cache/cache_test.go@@ -3,17 +3,11 @@ package cacheimport ("testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/mlx")-func skipIfNoMLX(t *testing.T) {- t.Helper()- if err := mlx.CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }-}-// newKVBatch builds a B=1 batch at SeqOffsets=off with all-real// queries (SeqQueryLens=L) — the standard single-sequence cache// test shape.@@ -26,258 +20,265 @@ func newKVBatch(off, L int) *batch.Batch {}func TestKVCacheSnapshotRestoreNeedBase(t *testing.T) {- skipIfNoMLX(t)- c := NewKVCache()-- for range 10 {- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }-- // Snapshot [5, 10).- snap := c.Snapshot(5)-- // Free the cache completely — offset is now 0.- c.Free()-- // Restore should fail because cache doesn't have data up to fromOffset=5.- if c.Restore(snap, 10) {- t.Fatal("expected Restore to fail with no base data")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewKVCache()++ for range 10 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }++ // Snapshot [5, 10).+ snap := c.Snapshot(5)++ // Free the cache completely — offset is now 0.+ c.Free()++ // Restore should fail because cache doesn't have data up to fromOffset=5.+ if c.Restore(snap, 10) {+ t.Fatal("expected Restore to fail with no base data")+ }+ })}// TestKVCacheDataSurvivesSnapshotRestore verifies that actual array data// is preserved through a snapshot→free→restore cycle.func TestKVCacheDataSurvivesSnapshotRestore(t *testing.T) {- skipIfNoMLX(t)- c := NewKVCache()-- for range 10 {- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }-- snap := c.Snapshot(0)- if snap == nil {- t.Fatal("Snapshot returned nil")- }-- // Free and restore to a fresh cache.- c2 := NewKVCache()- if !c2.Restore(snap, 10) {- t.Fatal("Restore failed")- }- if c2.Offset() != 10 {- t.Fatalf("offset = %d, want 10", c2.Offset())- }-- // Verify State() returns arrays with correct sequence dimension.- state := c2.State()- if len(state) != 2 {- t.Fatalf("State() returned %d arrays, want 2", len(state))- }- // keys shape: [B, H, seqLen, Dk]- if state[0].Dim(2) != 10 {- t.Fatalf("keys seq dim = %d, want 10", state[0].Dim(2))- }- if state[1].Dim(2) != 10 {- t.Fatalf("values seq dim = %d, want 10", state[1].Dim(2))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewKVCache()++ for range 10 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }++ snap := c.Snapshot(0)+ if snap == nil {+ t.Fatal("Snapshot returned nil")+ }++ // Free and restore to a fresh cache.+ c2 := NewKVCache()+ if !c2.Restore(snap, 10) {+ t.Fatal("Restore failed")+ }+ if c2.Offset() != 10 {+ t.Fatalf("offset = %d, want 10", c2.Offset())+ }++ // Verify State() returns arrays with correct sequence dimension.+ state := c2.State()+ if len(state) != 2 {+ t.Fatalf("State() returned %d arrays, want 2", len(state))+ }+ // keys shape: [B, H, seqLen, Dk]+ if state[0].Dim(2) != 10 {+ t.Fatalf("keys seq dim = %d, want 10", state[0].Dim(2))+ }+ if state[1].Dim(2) != 10 {+ t.Fatalf("values seq dim = %d, want 10", state[1].Dim(2))+ }+ })}// TestKVCacheSplitPreservesData verifies that split produces two snapshots// that can be sequentially restored to rebuild the original cache state.func TestKVCacheSplitPreservesData(t *testing.T) {- skipIfNoMLX(t)- c := NewKVCache()-- for range 10 {- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }-- snap := c.Snapshot(0)- parent, child := c.Split(snap, 5)- if parent == nil || child == nil {- t.Fatal("Split returned nil")- }-- // Restore parent → offset=5, seq dim=5.- c2 := NewKVCache()- if !c2.Restore(parent, 5) {- t.Fatal("Restore(parent) failed")- }- if c2.Offset() != 5 {- t.Fatalf("offset after parent = %d, want 5", c2.Offset())- }- state := c2.State()- if state[0].Dim(2) != 5 {- t.Fatalf("keys seq dim after parent = %d, want 5", state[0].Dim(2))- }-- // Restore child on top → offset=10, seq dim=10.- if !c2.Restore(child, 10) {- t.Fatal("Restore(child) failed")- }- if c2.Offset() != 10 {- t.Fatalf("offset after child = %d, want 10", c2.Offset())- }- state = c2.State()- if state[0].Dim(2) != 10 {- t.Fatalf("keys seq dim after child = %d, want 10", state[0].Dim(2))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewKVCache()++ for range 10 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }++ snap := c.Snapshot(0)+ parent, child := c.Split(snap, 5)+ if parent == nil || child == nil {+ t.Fatal("Split returned nil")+ }++ // Restore parent → offset=5, seq dim=5.+ c2 := NewKVCache()+ if !c2.Restore(parent, 5) {+ t.Fatal("Restore(parent) failed")+ }+ if c2.Offset() != 5 {+ t.Fatalf("offset after parent = %d, want 5", c2.Offset())+ }+ state := c2.State()+ if state[0].Dim(2) != 5 {+ t.Fatalf("keys seq dim after parent = %d, want 5", state[0].Dim(2))+ }++ // Restore child on top → offset=10, seq dim=10.+ if !c2.Restore(child, 10) {+ t.Fatal("Restore(child) failed")+ }+ if c2.Offset() != 10 {+ t.Fatalf("offset after child = %d, want 10", c2.Offset())+ }+ state = c2.State()+ if state[0].Dim(2) != 10 {+ t.Fatalf("keys seq dim after child = %d, want 10", state[0].Dim(2))+ }+ })}// TestKVCacheSplitMergeRoundTripData verifies that splitting and merging back// produces a snapshot equivalent to the original.func TestKVCacheSplitMergeRoundTripData(t *testing.T) {- skipIfNoMLX(t)- c := NewKVCache()-- for range 10 {- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }-- snap := c.Snapshot(0)- parent, child := c.Split(snap, 6)- merged := c.Merge(parent, child)- if merged == nil {- t.Fatal("Merge returned nil")- }-- c2 := NewKVCache()- if !c2.Restore(merged, 10) {- t.Fatal("Restore(merged) failed")- }- if c2.Offset() != 10 {- t.Fatalf("offset = %d, want 10", c2.Offset())- }-- state := c2.State()- if state[0].Dim(2) != 10 {- t.Fatalf("keys seq dim = %d, want 10", state[0].Dim(2))- }- if state[1].Dim(2) != 10 {- t.Fatalf("values seq dim = %d, want 10", state[1].Dim(2))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewKVCache()++ for range 10 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }++ snap := c.Snapshot(0)+ parent, child := c.Split(snap, 6)+ merged := c.Merge(parent, child)+ if merged == nil {+ t.Fatal("Merge returned nil")+ }++ c2 := NewKVCache()+ if !c2.Restore(merged, 10) {+ t.Fatal("Restore(merged) failed")+ }+ if c2.Offset() != 10 {+ t.Fatalf("offset = %d, want 10", c2.Offset())+ }++ state := c2.State()+ if state[0].Dim(2) != 10 {+ t.Fatalf("keys seq dim = %d, want 10", state[0].Dim(2))+ }+ if state[1].Dim(2) != 10 {+ t.Fatalf("values seq dim = %d, want 10", state[1].Dim(2))+ }+ })}func TestRotatingKVCacheRestoreOutsideWindow(t *testing.T) {- skipIfNoMLX(t)- c := NewRotatingKVCache(4)-- // Feed 10 tokens (window size 4, so positions 0-5 are evicted).- for range 10 {- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }-- // Offset 3 is outside the window.- if c.Restore(nil, 3) {- t.Fatal("Restore(nil, 3) should fail when outside window")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewRotatingKVCache(4)++ // Feed 10 tokens (window size 4, so positions 0-5 are evicted).+ for range 10 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }++ // Offset 3 is outside the window.+ if c.Restore(nil, 3) {+ t.Fatal("Restore(nil, 3) should fail when outside window")+ }+ })}// TestRotatingKVCacheSnapshotPreservesWindow verifies that after restoring// from a snapshot, the rotating cache has the correct window of data.func TestRotatingKVCacheSnapshotPreservesWindow(t *testing.T) {- skipIfNoMLX(t)- c := NewRotatingKVCache(4)-- // Feed 10 tokens one at a time. Window size 4, so only last 4 are kept.- for range 10 {- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }-- snap := c.Snapshot(0)- if snap == nil {- t.Fatal("Snapshot returned nil")- }-- // Feed 5 more tokens.- for range 5 {- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }-- // Restore to offset 10.- if !c.Restore(snap, 10) {- t.Fatal("Restore failed")- }- if c.Offset() != 10 {- t.Fatalf("offset = %d, want 10", c.Offset())- }-- state := c.State()- if len(state) != 2 {- t.Fatalf("State() returned %d arrays, want 2", len(state))- }- // Seq dim should be min(offset, maxSize) = min(10, 4) = 4.- seqDim := state[0].Dim(2)- if seqDim != 4 {- t.Fatalf("keys seq dim = %d, want 4 (window size)", seqDim)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewRotatingKVCache(4)++ // Feed 10 tokens one at a time. Window size 4, so only last 4 are kept.+ for range 10 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }++ snap := c.Snapshot(0)+ if snap == nil {+ t.Fatal("Snapshot returned nil")+ }++ // Feed 5 more tokens.+ for range 5 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }++ // Restore to offset 10.+ if !c.Restore(snap, 10) {+ t.Fatal("Restore failed")+ }+ if c.Offset() != 10 {+ t.Fatalf("offset = %d, want 10", c.Offset())+ }++ state := c.State()+ if len(state) != 2 {+ t.Fatalf("State() returned %d arrays, want 2", len(state))+ }+ // Seq dim should be min(offset, maxSize) = min(10, 4) = 4.+ seqDim := state[0].Dim(2)+ if seqDim != 4 {+ t.Fatalf("keys seq dim = %d, want 4 (window size)", seqDim)+ }+ })}// TestRotatingKVCacheRestoreFromSnapshot verifies that restoring from a// snapshot correctly preserves the write position (idx), so subsequent// single-token updates land in the right buffer slot.func TestRotatingKVCacheRestoreFromSnapshot(t *testing.T) {- skipIfNoMLX(t)- c := NewRotatingKVCache(4)-- // Fill the window: 6 tokens into a size-4 window.- // After this, idx has wrapped and the buffer has rotated.- for range 6 {+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewRotatingKVCache(4)++ // Fill the window: 6 tokens into a size-4 window.+ // After this, idx has wrapped and the buffer has rotated.+ for range 6 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }+ if c.Offset() != 6 {+ t.Fatalf("offset = %d, want 6", c.Offset())+ }++ snap := c.Snapshot(0)++ // Mutate the cache further so live state diverges from snapshot.+ for range 3 {+ k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }++ // Restore to snapshot state.+ if !c.Restore(snap, 6) {+ t.Fatal("Restore failed")+ }+ if c.Offset() != 6 {+ t.Fatalf("offset after restore = %d, want 6", c.Offset())+ }++ // Feed one more token. If idx was restored correctly, this should+ // produce a valid window of size 4 at offset 7.k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }- if c.Offset() != 6 {- t.Fatalf("offset = %d, want 6", c.Offset())- }-- snap := c.Snapshot(0)- // Mutate the cache further so live state diverges from snapshot.- for range 3 {- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }-- // Restore to snapshot state.- if !c.Restore(snap, 6) {- t.Fatal("Restore failed")- }- if c.Offset() != 6 {- t.Fatalf("offset after restore = %d, want 6", c.Offset())- }-- // Feed one more token. If idx was restored correctly, this should- // produce a valid window of size 4 at offset 7.- k := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- v := mlx.Zeros(mlx.DTypeFloat16, 1, 4, 1, 8)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)-- if c.Offset() != 7 {- t.Fatalf("offset after post-restore update = %d, want 7", c.Offset())- }- state := c.State()- if len(state) != 2 {- t.Fatalf("State() returned %d arrays, want 2", len(state))- }- seqDim := state[0].Dim(2)- if seqDim != 4 {- t.Fatalf("keys seq dim = %d, want 4 (window size)", seqDim)- }+ if c.Offset() != 7 {+ t.Fatalf("offset after post-restore update = %d, want 7", c.Offset())+ }+ state := c.State()+ if len(state) != 2 {+ t.Fatalf("State() returned %d arrays, want 2", len(state))+ }+ seqDim := state[0].Dim(2)+ if seqDim != 4 {+ t.Fatalf("keys seq dim = %d, want 4 (window size)", seqDim)+ }+ })}diff --git a/x/mlxrunner/cache/lazy_test.go b/x/mlxrunner/cache/lazy_test.goindex 22147e44f..4739e7e42 100644--- a/x/mlxrunner/cache/lazy_test.go+++ b/x/mlxrunner/cache/lazy_test.go@@ -3,6 +3,7 @@ package cacheimport ("testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx")@@ -44,53 +45,53 @@ func settledActiveMemory() int {// snapshot stays lazy and is discarded before any overwrite. Compare against the// bytes an eager per-token copy would cost.func TestKVSpeculationCaptureAllocatesNothing(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ const before, draft, H, D = 16, 8, 4, 8- const before, draft, H, D = 16, 8, 4, 8+ c := NewKVCache()+ fillKV(c, before)- c := NewKVCache()- fillKV(c, before)+ offsets := make([]int, draft)+ for i := range offsets {+ offsets[i] = before + i+ }+ c.PrepareSnapshots(offsets)- offsets := make([]int, draft)- for i := range offsets {- offsets[i] = before + i- }- c.PrepareSnapshots(offsets)+ k, v := batchKV(draft)+ c.Update(newKVBatch(before, draft), k, v)- k, v := batchKV(draft)- c.Update(newKVBatch(before, draft), k, v)+ baseline := settledActiveMemory()- baseline := settledActiveMemory()+ snaps := c.TakeSnapshots()+ // Every captured snapshot is a lazy snapshot (no owned buffer).+ for i, s := range snaps {+ if s == nil {+ continue+ }+ if ks := s.(*kvSnapshot); ks.keys != nil {+ t.Fatalf("snaps[%d] owns a buffer at capture; want a lazy snapshot", i)+ }+ }- snaps := c.TakeSnapshots()- // Every captured snapshot is a lazy snapshot (no owned buffer).- for i, s := range snaps {- if s == nil {- continue+ // MTP commit: rewind to a partial accept, then discard all snapshots.+ if !c.Restore(nil, before+draft/2) {+ t.Fatal("live rewind failed")}- if ks := s.(*kvSnapshot); ks.keys != nil {- t.Fatalf("snaps[%d] owns a buffer at capture; want a lazy snapshot", i)+ for _, s := range snaps {+ if s != nil {+ s.Close()+ }}- }- // MTP commit: rewind to a partial accept, then discard all snapshots.- if !c.Restore(nil, before+draft/2) {- t.Fatal("live rewind failed")- }- for _, s := range snaps {- if s != nil {- s.Close()+ after := settledActiveMemory()+ // Lazy snapshots allocate nothing; allow a tiny slack for allocator noise but+ // well under one per-token copy (draft tokens * keys+values).+ perToken := (c.keys.NumBytes() + c.values.NumBytes()) / c.keys.Dim(2)+ if after-baseline > perToken {+ t.Fatalf("capture allocated %d bytes (> one token %d); lazy snapshots should allocate nothing",+ after-baseline, perToken)}- }-- after := settledActiveMemory()- // Lazy snapshots allocate nothing; allow a tiny slack for allocator noise but- // well under one per-token copy (draft tokens * keys+values).- perToken := (c.keys.NumBytes() + c.values.NumBytes()) / c.keys.Dim(2)- if after-baseline > perToken {- t.Fatalf("capture allocated %d bytes (> one token %d); lazy snapshots should allocate nothing",- after-baseline, perToken)- }+ })}// TestKVLazySnapshotSizeZeroUntilMaterialized verifies the accounting contract:@@ -98,40 +99,40 @@ func TestKVSpeculationCaptureAllocatesNothing(t *testing.T) {// destructive write triggers copyOut the materialize hook fires with the// newly-allocated bytes and Size() reports the owned arrays.func TestKVLazySnapshotSizeZeroUntilMaterialized(t *testing.T) {- skipIfNoMLX(t)-- const H, D = 4, 8+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, D = 4, 8- c := NewKVCache()- k, v := distinctKV(0, 10, H, D)- c.Update(newKVBatch(0, 10), k, v)+ c := NewKVCache()+ k, v := distinctKV(0, 10, H, D)+ c.Update(newKVBatch(0, 10), k, v)- snap := c.Snapshot(5).(*kvSnapshot)- defer snap.Close()- if snap.Size() != 0 {- t.Fatalf("lazy snapshot Size = %d, want 0", snap.Size())- }+ snap := c.Snapshot(5).(*kvSnapshot)+ defer snap.Close()+ if snap.Size() != 0 {+ t.Fatalf("lazy snapshot Size = %d, want 0", snap.Size())+ }- var hookDelta int- snap.SetMaterializeHook(func(delta int) { hookDelta = delta })+ var hookDelta int+ snap.SetMaterializeHook(func(delta int) { hookDelta = delta })- // Rewind and overwrite to force copyOut.- if !c.Restore(nil, 5) {- t.Fatal("rewind failed")- }- nk, nv := distinctKV(100, 3, H, D)- c.Update(newKVBatch(5, 3), nk, nv)+ // Rewind and overwrite to force copyOut.+ if !c.Restore(nil, 5) {+ t.Fatal("rewind failed")+ }+ nk, nv := distinctKV(100, 3, H, D)+ c.Update(newKVBatch(5, 3), nk, nv)- if snap.keys == nil {- t.Fatal("snapshot was not materialized by overwriting write")- }- want := snap.keys.NumBytes() + snap.values.NumBytes()- if hookDelta != want {- t.Fatalf("hook fired with delta %d, want %d (owned bytes)", hookDelta, want)- }- if snap.Size() != want {- t.Fatalf("materialized Size = %d, want %d", snap.Size(), want)- }+ if snap.keys == nil {+ t.Fatal("snapshot was not materialized by overwriting write")+ }+ want := snap.keys.NumBytes() + snap.values.NumBytes()+ if hookDelta != want {+ t.Fatalf("hook fired with delta %d, want %d (owned bytes)", hookDelta, want)+ }+ if snap.Size() != want {+ t.Fatalf("materialized Size = %d, want %d", snap.Size(), want)+ }+ })}// TestKVLazySnapshotCopiedOutOnOverwrite verifies that after a rewind, a write@@ -139,110 +140,110 @@ func TestKVLazySnapshotSizeZeroUntilMaterialized(t *testing.T) {// still reads the pre-overwrite data — both keys and values, across the whole// captured range (guarding the Slice+Contiguous copy-out representation).func TestKVLazySnapshotCopiedOutOnOverwrite(t *testing.T) {- skipIfNoMLX(t)-- const H, D = 4, 8-- c := NewKVCache()- // Fill [0,10) with position-encoded values (keys p, values -p).- k, v := distinctKV(0, 10, H, D)- c.Update(newKVBatch(0, 10), k, v)-- // Lazy snapshot [5,10).- snap := c.Snapshot(5).(*kvSnapshot)- if snap.keys != nil {- t.Fatal("Snapshot should return a lazy snapshot")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, D = 4, 8++ c := NewKVCache()+ // Fill [0,10) with position-encoded values (keys p, values -p).+ k, v := distinctKV(0, 10, H, D)+ c.Update(newKVBatch(0, 10), k, v)++ // Lazy snapshot [5,10).+ snap := c.Snapshot(5).(*kvSnapshot)+ if snap.keys != nil {+ t.Fatal("Snapshot should return a lazy snapshot")+ }- // Rewind to 5 and overwrite [5,8) with different data.- if !c.Restore(nil, 5) {- t.Fatal("rewind failed")- }- nk, nv := distinctKV(100, 3, H, D) // positions 100,101,102- c.Update(newKVBatch(5, 3), nk, nv)+ // Rewind to 5 and overwrite [5,8) with different data.+ if !c.Restore(nil, 5) {+ t.Fatal("rewind failed")+ }+ nk, nv := distinctKV(100, 3, H, D) // positions 100,101,102+ c.Update(newKVBatch(5, 3), nk, nv)- // The overwrite must have copied the lazy snapshot out beforehand, preserving- // the pre-overwrite keys and values across the whole [5,10) range.- if snap.keys == nil {- t.Fatal("lazy snapshot was not copied out before the overwriting write")- }- mlx.Eval(snap.keys, snap.values)- keys := snap.keys.Floats()- vals := snap.values.Floats()- for l := range snap.toOffset - snap.fromOffset {- wantK := float32(snap.fromOffset + l)- if got := keys[l*D]; got != wantK {- t.Fatalf("snapshot keys[%d] = %v, want %v (pre-overwrite data)", l, got, wantK)+ // The overwrite must have copied the lazy snapshot out beforehand, preserving+ // the pre-overwrite keys and values across the whole [5,10) range.+ if snap.keys == nil {+ t.Fatal("lazy snapshot was not copied out before the overwriting write")}- if got := vals[l*D]; got != -wantK {- t.Fatalf("snapshot values[%d] = %v, want %v (pre-overwrite data)", l, got, -wantK)+ mlx.Eval(snap.keys, snap.values)+ keys := snap.keys.Floats()+ vals := snap.values.Floats()+ for l := range snap.toOffset - snap.fromOffset {+ wantK := float32(snap.fromOffset + l)+ if got := keys[l*D]; got != wantK {+ t.Fatalf("snapshot keys[%d] = %v, want %v (pre-overwrite data)", l, got, wantK)+ }+ if got := vals[l*D]; got != -wantK {+ t.Fatalf("snapshot values[%d] = %v, want %v (pre-overwrite data)", l, got, -wantK)+ }}- }- snap.Close()+ snap.Close()+ })}// TestKVLazySnapshotCopiedOutOnFree verifies Free copies out outstanding lazy snapshots// so they survive after the cache buffer is gone.func TestKVLazySnapshotCopiedOutOnFree(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, D = 4, 8- const H, D = 4, 8+ c := NewKVCache()+ k, v := distinctKV(0, 10, H, D)+ c.Update(newKVBatch(0, 10), k, v)- c := NewKVCache()- k, v := distinctKV(0, 10, H, D)- c.Update(newKVBatch(0, 10), k, v)+ snap := c.Snapshot(5).(*kvSnapshot)+ c.Free()- snap := c.Snapshot(5).(*kvSnapshot)- c.Free()-- if snap.keys == nil {- t.Fatal("lazy snapshot was not copied out on Free")- }- mlx.Eval(snap.keys)- if got := firstKeyAt(snap.keys, 0, D); got != 5 {- t.Fatalf("snapshot[0] key = %v, want 5 (data preserved through Free)", got)- }- snap.Close()+ if snap.keys == nil {+ t.Fatal("lazy snapshot was not copied out on Free")+ }+ mlx.Eval(snap.keys)+ if got := firstKeyAt(snap.keys, 0, D); got != 5 {+ t.Fatalf("snapshot[0] key = %v, want 5 (data preserved through Free)", got)+ }+ snap.Close()+ })}// TestKVLazySnapshotSplitMergeNoCopy verifies Split of a lazy snapshot and Merge of two// adjacent lazy snapshots are pure arithmetic — they produce lazy snapshots and allocate// nothing — while still tracking the correct offsets and data.func TestKVLazySnapshotSplitMergeNoCopy(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, D = 4, 8- const H, D = 4, 8+ c := NewKVCache()+ k, v := distinctKV(0, 10, H, D)+ c.Update(newKVBatch(0, 10), k, v)- c := NewKVCache()- k, v := distinctKV(0, 10, H, D)- c.Update(newKVBatch(0, 10), k, v)+ base := settledActiveMemory()- base := settledActiveMemory()-- // Lazy snapshot [2,10), split at 5.- snap := c.Snapshot(2)- p, ch := c.Split(snap, 5)- ps, cs := p.(*kvSnapshot), ch.(*kvSnapshot)- if ps.keys != nil || cs.keys != nil {- t.Fatal("Split of a lazy snapshot should yield lazy snapshots (no copy)")- }- if ps.fromOffset != 2 || ps.toOffset != 5 || cs.fromOffset != 5 || cs.toOffset != 10 {- t.Fatalf("split ranges = [%d,%d)/[%d,%d), want [2,5)/[5,10)", ps.fromOffset, ps.toOffset, cs.fromOffset, cs.toOffset)- }+ // Lazy snapshot [2,10), split at 5.+ snap := c.Snapshot(2)+ p, ch := c.Split(snap, 5)+ ps, cs := p.(*kvSnapshot), ch.(*kvSnapshot)+ if ps.keys != nil || cs.keys != nil {+ t.Fatal("Split of a lazy snapshot should yield lazy snapshots (no copy)")+ }+ if ps.fromOffset != 2 || ps.toOffset != 5 || cs.fromOffset != 5 || cs.toOffset != 10 {+ t.Fatalf("split ranges = [%d,%d)/[%d,%d), want [2,5)/[5,10)", ps.fromOffset, ps.toOffset, cs.fromOffset, cs.toOffset)+ }- // Merge them back into [2,10).- merged := c.Merge(p, ch).(*kvSnapshot)- if merged.keys != nil {- t.Fatal("Merge of adjacent lazy snapshots should yield a lazy snapshot (no Concatenate)")- }- if merged.fromOffset != 2 || merged.toOffset != 10 {- t.Fatalf("merged range = [%d,%d), want [2,10)", merged.fromOffset, merged.toOffset)- }+ // Merge them back into [2,10).+ merged := c.Merge(p, ch).(*kvSnapshot)+ if merged.keys != nil {+ t.Fatal("Merge of adjacent lazy snapshots should yield a lazy snapshot (no Concatenate)")+ }+ if merged.fromOffset != 2 || merged.toOffset != 10 {+ t.Fatalf("merged range = [%d,%d), want [2,10)", merged.fromOffset, merged.toOffset)+ }- if after := settledActiveMemory(); after > base {- t.Fatalf("Split/Merge of lazy snapshots allocated %d bytes; want 0", after-base)- }- merged.Close()+ if after := settledActiveMemory(); after > base {+ t.Fatalf("Split/Merge of lazy snapshots allocated %d bytes; want 0", after-base)+ }+ merged.Close()+ })}// TestKVLazySnapshotSurvivesPathSwitch reproduces the switchToPath sequence that@@ -251,41 +252,41 @@ func TestKVLazySnapshotSplitMergeNoCopy(t *testing.T) {// different path (Restore feeds new data via appendKV, overwriting the old// leaf's slots). The paged-out snapshot must still hold the original tokens.func TestKVLazySnapshotSurvivesPathSwitch(t *testing.T) {- skipIfNoMLX(t)-- const H, D = 4, 8+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, D = 4, 8- c := NewKVCache()- // Active path tokens [0,10): positions 0..9.- k, v := distinctKV(0, 10, H, D)- c.Update(newKVBatch(0, 10), k, v)+ c := NewKVCache()+ // Active path tokens [0,10): positions 0..9.+ k, v := distinctKV(0, 10, H, D)+ c.Update(newKVBatch(0, 10), k, v)- // Page out the diverging leaf [4,10) as a lazy snapshot (what switchToPath does- // before rewinding).- leaf := c.Snapshot(4).(*kvSnapshot)+ // Page out the diverging leaf [4,10) as a lazy snapshot (what switchToPath does+ // before rewinding).+ leaf := c.Snapshot(4).(*kvSnapshot)- // Rewind to the common ancestor at 4 (Restore(nil): offset move only).- if !c.Restore(nil, 4) {- t.Fatal("rewind to ancestor failed")- }+ // Rewind to the common ancestor at 4 (Restore(nil): offset move only).+ if !c.Restore(nil, 4) {+ t.Fatal("rewind to ancestor failed")+ }- // Page in the new path [4,9): positions 200..204, overwriting the old leaf's- // slots. appendKV must copy the leaf lazy snapshot out first.- nk, nv := distinctKV(200, 5, H, D)- c.Update(newKVBatch(4, 5), nk, nv)+ // Page in the new path [4,9): positions 200..204, overwriting the old leaf's+ // slots. appendKV must copy the leaf lazy snapshot out first.+ nk, nv := distinctKV(200, 5, H, D)+ c.Update(newKVBatch(4, 5), nk, nv)- if leaf.keys == nil {- t.Fatal("leaf lazy snapshot was not copied out during page-in")- }- mlx.Eval(leaf.keys)- // leaf covers [4,10): its keys are the original positions 4..9.- keys := leaf.keys.Floats()- for l := range leaf.toOffset - leaf.fromOffset {- if got, want := keys[l*D], float32(leaf.fromOffset+l); got != want {- t.Fatalf("paged-out leaf key[%d] = %v, want %v (original path data)", l, got, want)+ if leaf.keys == nil {+ t.Fatal("leaf lazy snapshot was not copied out during page-in")}- }- leaf.Close()+ mlx.Eval(leaf.keys)+ // leaf covers [4,10): its keys are the original positions 4..9.+ keys := leaf.keys.Floats()+ for l := range leaf.toOffset - leaf.fromOffset {+ if got, want := keys[l*D], float32(leaf.fromOffset+l); got != want {+ t.Fatalf("paged-out leaf key[%d] = %v, want %v (original path data)", l, got, want)+ }+ }+ leaf.Close()+ })}// TestKVRestoreLiveLazySnapshotIsOffsetMove verifies the same-path rewind/rematch@@ -294,46 +295,46 @@ func TestKVLazySnapshotSurvivesPathSwitch(t *testing.T) {// the offset without cloning or replaying. This is the switchToPath sequence// where a paged-out leaf is restored before any write displaced it.func TestKVRestoreLiveLazySnapshotIsOffsetMove(t *testing.T) {- skipIfNoMLX(t)-- const H, D = 4, 8+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, D = 4, 8- c := NewKVCache()- k, v := distinctKV(0, 10, H, D)- c.Update(newKVBatch(0, 10), k, v)+ c := NewKVCache()+ k, v := distinctKV(0, 10, H, D)+ c.Update(newKVBatch(0, 10), k, v)- // Page out the leaf [5,10) as a lazy snapshot, then rewind (offset move only).- snap := c.Snapshot(5).(*kvSnapshot)- if !c.Restore(nil, 5) {- t.Fatal("rewind failed")- }+ // Page out the leaf [5,10) as a lazy snapshot, then rewind (offset move only).+ snap := c.Snapshot(5).(*kvSnapshot)+ if !c.Restore(nil, 5) {+ t.Fatal("rewind failed")+ }- base := settledActiveMemory()+ base := settledActiveMemory()- // Restore the snapshot back to 10. Its slots [5,10) were never overwritten,- // so it is still lazy and the data is already in the buffer — a pure offset- // move, no allocation.- if !c.Restore(snap, 10) {- t.Fatal("restore failed")- }- if snap.keys != nil {- t.Fatal("snapshot was copied out; expected the offset-move fast path")- }- if c.Offset() != 10 {- t.Fatalf("offset after restore = %d, want 10", c.Offset())- }- if after := settledActiveMemory(); after > base {- t.Fatalf("restore of a live lazy snapshot allocated %d bytes; want 0 (offset move)", after-base)- }+ // Restore the snapshot back to 10. Its slots [5,10) were never overwritten,+ // so it is still lazy and the data is already in the buffer — a pure offset+ // move, no allocation.+ if !c.Restore(snap, 10) {+ t.Fatal("restore failed")+ }+ if snap.keys != nil {+ t.Fatal("snapshot was copied out; expected the offset-move fast path")+ }+ if c.Offset() != 10 {+ t.Fatalf("offset after restore = %d, want 10", c.Offset())+ }+ if after := settledActiveMemory(); after > base {+ t.Fatalf("restore of a live lazy snapshot allocated %d bytes; want 0 (offset move)", after-base)+ }- // The buffer still holds the original positions 0..9.- st := c.State()- mlx.Eval(st[0])- keys := st[0].Floats()- for l := range 10 {- if got := keys[l*D]; got != float32(l) {- t.Fatalf("restored key[%d] = %v, want %v", l, got, float32(l))+ // The buffer still holds the original positions 0..9.+ st := c.State()+ mlx.Eval(st[0])+ keys := st[0].Floats()+ for l := range 10 {+ if got := keys[l*D]; got != float32(l) {+ t.Fatalf("restored key[%d] = %v, want %v", l, got, float32(l))+ }}- }- snap.Close()+ snap.Close()+ })}diff --git a/x/mlxrunner/cache/recurrent_test.go b/x/mlxrunner/cache/recurrent_test.goindex ce1e12c4f..4a4624d51 100644--- a/x/mlxrunner/cache/recurrent_test.go+++ b/x/mlxrunner/cache/recurrent_test.go@@ -4,6 +4,7 @@ import ("math""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/mlx""github.com/ollama/ollama/x/models/nn"@@ -13,66 +14,68 @@ import (// only succeeds when target exactly matches the snapshot's offset. Recurrent// state is cumulative, so it can't be rewound or fast-forwarded.func TestRecurrentCacheRestoreExactOffset(t *testing.T) {- skipIfNoMLX(t)- c := NewRecurrentCache(3, 12, 4, 8, 8)- b1 := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 1)}- c.Get(b1, mlx.DTypeFloat16) // lazy-init-- keep := func() ([]*mlx.Array, []*mlx.Array) {- s := c.State()- return []*mlx.Array{s[0]}, []*mlx.Array{s[1]}- }-- b10 := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 10), SeqQueryLens: []int32{10}}- cs, ds := keep()- c.Put(b10, cs, ds) // advance to 10-- snap := c.Snapshot(0) // snap.offset == 10-- b5 := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 5), SeqQueryLens: []int32{5}}- cs, ds = keep()- c.Put(b5, cs, ds) // cache now at 15-- // target < snap.offset: fails (can't rewind past snapshot)- if c.Restore(snap, 5) {- t.Fatal("Restore(snap, 5) should fail — target != snap.offset")- }-- // target > snap.offset: fails (can't advance without feeding tokens)- if c.Restore(snap, 15) {- t.Fatal("Restore(snap, 15) should fail — target != snap.offset")- }-- // target == snap.offset: succeeds- if !c.Restore(snap, 10) {- t.Fatal("Restore(snap, 10) should succeed — target == snap.offset")- }- if c.Offset() != 10 {- t.Fatalf("offset = %d, want 10", c.Offset())- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewRecurrentCache(3, 12, 4, 8, 8)+ b1 := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 1)}+ c.Get(b1, mlx.DTypeFloat16) // lazy-init++ keep := func() ([]*mlx.Array, []*mlx.Array) {+ s := c.State()+ return []*mlx.Array{s[0]}, []*mlx.Array{s[1]}+ }++ b10 := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 10), SeqQueryLens: []int32{10}}+ cs, ds := keep()+ c.Put(b10, cs, ds) // advance to 10++ snap := c.Snapshot(0) // snap.offset == 10++ b5 := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 5), SeqQueryLens: []int32{5}}+ cs, ds = keep()+ c.Put(b5, cs, ds) // cache now at 15++ // target < snap.offset: fails (can't rewind past snapshot)+ if c.Restore(snap, 5) {+ t.Fatal("Restore(snap, 5) should fail — target != snap.offset")+ }++ // target > snap.offset: fails (can't advance without feeding tokens)+ if c.Restore(snap, 15) {+ t.Fatal("Restore(snap, 15) should fail — target != snap.offset")+ }++ // target == snap.offset: succeeds+ if !c.Restore(snap, 10) {+ t.Fatal("Restore(snap, 10) should succeed — target == snap.offset")+ }+ if c.Offset() != 10 {+ t.Fatalf("offset = %d, want 10", c.Offset())+ }+ })}func TestRecurrentCacheGetLazyInit(t *testing.T) {- skipIfNoMLX(t)- c := NewRecurrentCache(3, 4, 2, 4, 4)- b := &batch.Batch{- InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 1),- SeqOffsets: []int32{0},- SeqQueryLens: []int32{1},- }- h := c.Get(b, mlx.DTypeBFloat16)- if c.Offset() != 0 {- t.Fatalf("Get should not advance; got offset %d", c.Offset())- }- if h.ConvState() == nil || h.DeltaState() == nil {- t.Fatal("history should expose conv/delta tensors")- }- if got := h.ConvState().DType(); got != mlx.DTypeBFloat16 {- t.Fatalf("conv state dtype = %v, want %v", got, mlx.DTypeBFloat16)- }- if got := h.DeltaState().DType(); got != mlx.DTypeFloat32 {- t.Fatalf("delta state dtype = %v, want %v", got, mlx.DTypeFloat32)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewRecurrentCache(3, 4, 2, 4, 4)+ b := &batch.Batch{+ InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 1),+ SeqOffsets: []int32{0},+ SeqQueryLens: []int32{1},+ }+ h := c.Get(b, mlx.DTypeBFloat16)+ if c.Offset() != 0 {+ t.Fatalf("Get should not advance; got offset %d", c.Offset())+ }+ if h.ConvState() == nil || h.DeltaState() == nil {+ t.Fatal("history should expose conv/delta tensors")+ }+ if got := h.ConvState().DType(); got != mlx.DTypeBFloat16 {+ t.Fatalf("conv state dtype = %v, want %v", got, mlx.DTypeBFloat16)+ }+ if got := h.DeltaState().DType(); got != mlx.DTypeFloat32 {+ t.Fatalf("delta state dtype = %v, want %v", got, mlx.DTypeFloat32)+ }+ })}// TestRecurrentCachePaddedRoundTrip runs Get → CausalConv1D →@@ -83,138 +86,140 @@ func TestRecurrentCacheGetLazyInit(t *testing.T) {// Pins the recurrent contract: a forward with padding produces the// same end-state as a forward with the real-prefix-only input.func TestRecurrentCachePaddedRoundTrip(t *testing.T) {- skipIfNoMLX(t)- const convTail, convDim = 2, 6- const numVHeads, headVDim, headKDim = 1, 4, 6- const L = 4- const qLen = 2-- // Distinct values for the real prefix and large junk in the padded- // tail so any leak from padded positions is visible.- const packedDim = 2*headKDim + numVHeads*headVDim- mkPacked := func(seed float32, T int) (packed, ba *mlx.Array) {- pv := make([]float32, T*packedDim)- bv := make([]float32, T*2*numVHeads)- for i := range pv {- pv[i] = seed + 0.05*float32(i)- }- for i := range bv {- bv[i] = seed - 0.02*float32(i)- }- return mlx.FromValues(pv, 1, T, packedDim), mlx.FromValues(bv, 1, T, 2*numVHeads)- }- mkPackedPadded := func() (packed, ba *mlx.Array) {- pReal, baReal := mkPacked(0.3, qLen)- pPad := mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, L-qLen, packedDim), 99)- baPad := mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, L-qLen, 2*numVHeads), 99)- return mlx.Concatenate([]*mlx.Array{pReal, pPad}, 1), mlx.Concatenate([]*mlx.Array{baReal, baPad}, 1)- }- dtBias := mlx.FromValues([]float32{0.3}, numVHeads)- aExp := mlx.FromValues([]float32{0.12}, numVHeads)-- // The conv input dimension must match the cache's convDim.- mkConvInput := func(seed float32, T int) *mlx.Array {- vals := make([]float32, 1*T*convDim)- for i := range vals {- vals[i] = seed + 0.05*float32(i)- }- return mlx.FromValues(vals, 1, T, convDim)- }- mkWeight := func(seed float32) *mlx.Array {- vals := make([]float32, convDim*(convTail+1))- for i := range vals {- vals[i] = seed + 0.1*float32(i)- }- return mlx.FromValues(vals, convDim, convTail+1)- }- weight := mkWeight(0.2)- // Build the depthwise causal Conv1d as the model does at load time: the- // [C, K] kernel becomes [C, K, 1] and the conv is grouped per channel.- conv := nn.NewConv1d(mlx.ExpandDims(weight, 2), nil, 1, 0, 1, convDim)-- runForward := func(c *RecurrentCache, b *batch.Batch, T int) (*mlx.Array, *mlx.Array) {- var convInput *mlx.Array- if T == L {- realPart := mkConvInput(0.4, qLen)- padPart := mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, T-qLen, convDim), 99)- convInput = mlx.Concatenate([]*mlx.Array{realPart, padPart}, 1)- } else {- convInput = mkConvInput(0.4, T)- }-- history := c.Get(b, mlx.DTypeFloat32)- _, convStates := nn.CausalConv1D(b, convInput, conv, convTail,- nn.WithRecurrentHistory(history))-- var packed, ba *mlx.Array- if T == L {- packed, ba = mkPackedPadded()- } else {- packed, ba = mkPacked(0.3, T)- }- _, deltaStates := nn.GatedDelta(b, packed, ba, dtBias, aExp, nn.WithRecurrentHistory(history))-- c.Put(b, convStates, deltaStates)- return convStates[len(convStates)-1], deltaStates[len(deltaStates)-1]- }-- // Padded forward.- cPad := NewRecurrentCache(convTail, convDim, numVHeads, headVDim, headKDim)- bPad := &batch.Batch{- InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, L),- SeqOffsets: []int32{0},- SeqQueryLens: []int32{int32(qLen)},- }- nextConvPad, deltaPad := runForward(cPad, bPad, L)- mlx.Eval(nextConvPad, deltaPad)- if got := cPad.Offset(); got != qLen {- t.Fatalf("padded forward: Offset() = %d, want %d (must advance by SeqQueryLens, not L)", got, qLen)- }-- // Unpadded reference.- cRef := NewRecurrentCache(convTail, convDim, numVHeads, headVDim, headKDim)- bRef := &batch.Batch{- InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, qLen),- SeqOffsets: []int32{0},- SeqQueryLens: []int32{int32(qLen)},- }- nextConvRef, deltaRef := runForward(cRef, bRef, qLen)- mlx.Eval(nextConvRef, deltaRef)- if got := cRef.Offset(); got != qLen {- t.Fatalf("unpadded forward: Offset() = %d, want %d", got, qLen)- }-- gp := nextConvPad.Floats()- gr := nextConvRef.Floats()- if len(gp) != len(gr) {- t.Fatalf("nextConv shape mismatch: padded %d vs unpadded %d", len(gp), len(gr))- }- for i := range gp {- if math.Abs(float64(gp[i]-gr[i])) > 1e-4 {- t.Fatalf("nextConv[%d]: padded=%v unpadded=%v (padding leaked into conv state)", i, gp[i], gr[i])- }- }-- dp := deltaPad.Floats()- dr := deltaRef.Floats()- if len(dp) != len(dr) {- t.Fatalf("delta state shape mismatch: padded %d vs unpadded %d", len(dp), len(dr))- }- for i := range dp {- if math.Abs(float64(dp[i]-dr[i])) > 1e-3 {- t.Fatalf("delta state[%d]: padded=%v unpadded=%v (padding leaked into recurrent state)", i, dp[i], dr[i])- }- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const convTail, convDim = 2, 6+ const numVHeads, headVDim, headKDim = 1, 4, 6+ const L = 4+ const qLen = 2++ // Distinct values for the real prefix and large junk in the padded+ // tail so any leak from padded positions is visible.+ const packedDim = 2*headKDim + numVHeads*headVDim+ mkPacked := func(seed float32, T int) (packed, ba *mlx.Array) {+ pv := make([]float32, T*packedDim)+ bv := make([]float32, T*2*numVHeads)+ for i := range pv {+ pv[i] = seed + 0.05*float32(i)+ }+ for i := range bv {+ bv[i] = seed - 0.02*float32(i)+ }+ return mlx.FromValues(pv, 1, T, packedDim), mlx.FromValues(bv, 1, T, 2*numVHeads)+ }+ mkPackedPadded := func() (packed, ba *mlx.Array) {+ pReal, baReal := mkPacked(0.3, qLen)+ pPad := mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, L-qLen, packedDim), 99)+ baPad := mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, L-qLen, 2*numVHeads), 99)+ return mlx.Concatenate([]*mlx.Array{pReal, pPad}, 1), mlx.Concatenate([]*mlx.Array{baReal, baPad}, 1)+ }+ dtBias := mlx.FromValues([]float32{0.3}, numVHeads)+ aExp := mlx.FromValues([]float32{0.12}, numVHeads)++ // The conv input dimension must match the cache's convDim.+ mkConvInput := func(seed float32, T int) *mlx.Array {+ vals := make([]float32, 1*T*convDim)+ for i := range vals {+ vals[i] = seed + 0.05*float32(i)+ }+ return mlx.FromValues(vals, 1, T, convDim)+ }+ mkWeight := func(seed float32) *mlx.Array {+ vals := make([]float32, convDim*(convTail+1))+ for i := range vals {+ vals[i] = seed + 0.1*float32(i)+ }+ return mlx.FromValues(vals, convDim, convTail+1)+ }+ weight := mkWeight(0.2)+ // Build the depthwise causal Conv1d as the model does at load time: the+ // [C, K] kernel becomes [C, K, 1] and the conv is grouped per channel.+ conv := nn.NewConv1d(mlx.ExpandDims(weight, 2), nil, 1, 0, 1, convDim)++ runForward := func(c *RecurrentCache, b *batch.Batch, T int) (*mlx.Array, *mlx.Array) {+ var convInput *mlx.Array+ if T == L {+ realPart := mkConvInput(0.4, qLen)+ padPart := mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, T-qLen, convDim), 99)+ convInput = mlx.Concatenate([]*mlx.Array{realPart, padPart}, 1)+ } else {+ convInput = mkConvInput(0.4, T)+ }++ history := c.Get(b, mlx.DTypeFloat32)+ _, convStates := nn.CausalConv1D(b, convInput, conv, convTail,+ nn.WithRecurrentHistory(history))++ var packed, ba *mlx.Array+ if T == L {+ packed, ba = mkPackedPadded()+ } else {+ packed, ba = mkPacked(0.3, T)+ }+ _, deltaStates := nn.GatedDelta(b, packed, ba, dtBias, aExp, nn.WithRecurrentHistory(history))++ c.Put(b, convStates, deltaStates)+ return convStates[len(convStates)-1], deltaStates[len(deltaStates)-1]+ }++ // Padded forward.+ cPad := NewRecurrentCache(convTail, convDim, numVHeads, headVDim, headKDim)+ bPad := &batch.Batch{+ InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, L),+ SeqOffsets: []int32{0},+ SeqQueryLens: []int32{int32(qLen)},+ }+ nextConvPad, deltaPad := runForward(cPad, bPad, L)+ mlx.Eval(nextConvPad, deltaPad)+ if got := cPad.Offset(); got != qLen {+ t.Fatalf("padded forward: Offset() = %d, want %d (must advance by SeqQueryLens, not L)", got, qLen)+ }++ // Unpadded reference.+ cRef := NewRecurrentCache(convTail, convDim, numVHeads, headVDim, headKDim)+ bRef := &batch.Batch{+ InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, qLen),+ SeqOffsets: []int32{0},+ SeqQueryLens: []int32{int32(qLen)},+ }+ nextConvRef, deltaRef := runForward(cRef, bRef, qLen)+ mlx.Eval(nextConvRef, deltaRef)+ if got := cRef.Offset(); got != qLen {+ t.Fatalf("unpadded forward: Offset() = %d, want %d", got, qLen)+ }++ gp := nextConvPad.Floats()+ gr := nextConvRef.Floats()+ if len(gp) != len(gr) {+ t.Fatalf("nextConv shape mismatch: padded %d vs unpadded %d", len(gp), len(gr))+ }+ for i := range gp {+ if math.Abs(float64(gp[i]-gr[i])) > 1e-4 {+ t.Fatalf("nextConv[%d]: padded=%v unpadded=%v (padding leaked into conv state)", i, gp[i], gr[i])+ }+ }++ dp := deltaPad.Floats()+ dr := deltaRef.Floats()+ if len(dp) != len(dr) {+ t.Fatalf("delta state shape mismatch: padded %d vs unpadded %d", len(dp), len(dr))+ }+ for i := range dp {+ if math.Abs(float64(dp[i]-dr[i])) > 1e-3 {+ t.Fatalf("delta state[%d]: padded=%v unpadded=%v (padding leaked into recurrent state)", i, dp[i], dr[i])+ }+ }+ })}func TestRecurrentCachePutAdvances(t *testing.T) {- skipIfNoMLX(t)- c := NewRecurrentCache(3, 4, 2, 4, 4)- b := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 2), SeqQueryLens: []int32{2}}- newConv := mlx.Zeros(mlx.DTypeFloat16, 1, 3, 4)- newDelta := mlx.Zeros(mlx.DTypeFloat16, 1, 2, 4, 4)- c.Put(b, []*mlx.Array{newConv}, []*mlx.Array{newDelta})- if c.Offset() != 2 {- t.Fatalf("cache offset not advanced: %d", c.Offset())- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewRecurrentCache(3, 4, 2, 4, 4)+ b := &batch.Batch{InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 2), SeqQueryLens: []int32{2}}+ newConv := mlx.Zeros(mlx.DTypeFloat16, 1, 3, 4)+ newDelta := mlx.Zeros(mlx.DTypeFloat16, 1, 2, 4, 4)+ c.Put(b, []*mlx.Array{newConv}, []*mlx.Array{newDelta})+ if c.Offset() != 2 {+ t.Fatalf("cache offset not advanced: %d", c.Offset())+ }+ })}diff --git a/x/mlxrunner/cache/rotating_attention_test.go b/x/mlxrunner/cache/rotating_attention_test.goindex a7fbdae74..6d2d6d318 100644--- a/x/mlxrunner/cache/rotating_attention_test.go+++ b/x/mlxrunner/cache/rotating_attention_test.go@@ -4,6 +4,7 @@ import ("math""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/mlx""github.com/ollama/ollama/x/models/nn"@@ -19,7 +20,6 @@ import (// applier's gather composing the caller's logical mask back into// storage order) must equal the logical-order reference.func TestRotatingKVCacheDecodeParity(t *testing.T) {- skipIfNoMLX(t)const H, D = 1, 4const window = 4const totalWrites = 7 // past wrap (window=4); last write is the L=1 decode@@ -39,46 +39,49 @@ func TestRotatingKVCacheDecodeParity(t *testing.T) {return}- q := mlx.FromValues([]float32{0.7, -0.4, 0.2, 0.9}, 1, H, 1, D)- mlx.Eval(q)+ var q, kLogical, vLogical, logicalMask *mlx.Array+ var b *batch.Batch+ var history *nn.KVHistory+ mlxtest.Run(t, func(t *mlxtest.T) {+ q = mlx.FromValues([]float32{0.7, -0.4, 0.2, 0.9}, 1, H, 1, D)- // Drive the cache: write positions 0..totalWrites-2 as a "history",- // then position totalWrites-1 is the actual L=1 decode under test.- c := NewRotatingKVCache(window)- for pos := range totalWrites - 1 {- k, v := perPosKV(pos)- c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)- }+ // Drive the cache: write positions 0..totalWrites-2 as a "history",+ // then position totalWrites-1 is the actual L=1 decode under test.+ c := NewRotatingKVCache(window)+ for pos := range totalWrites - 1 {+ k, v := perPosKV(pos)+ c.Update(newKVBatch(c.Offset(), k.Dim(2)), k, v)+ }- finalPos := totalWrites - 1- kFinal, vFinal := perPosKV(finalPos)- b := &batch.Batch{- InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 1),- SeqOffsets: []int32{int32(finalPos)},- SeqQueryLens: []int32{1},- }- history := c.Update(b, kFinal, vFinal)-- // Reference: the in-window logical-position-ordered K and V are- // the last `window` per-position values (positions- // [finalPos-window+1, finalPos]). Build them in that order.- startPos := max(finalPos-window+1, 0)- logicalKs := make([]*mlx.Array, 0, window)- logicalVs := make([]*mlx.Array, 0, window)- for pos := startPos; pos <= finalPos; pos++ {- kp, vp := perPosKV(pos)- logicalKs = append(logicalKs, kp)- logicalVs = append(logicalVs, vp)- }- kLogical := mlx.Concatenate(logicalKs, 2)- vLogical := mlx.Concatenate(logicalVs, 2)+ finalPos := totalWrites - 1+ kFinal, vFinal := perPosKV(finalPos)+ b = &batch.Batch{+ InputIDs: mlx.Zeros(mlx.DTypeInt32, 1, 1),+ SeqOffsets: []int32{int32(finalPos)},+ SeqQueryLens: []int32{1},+ }+ history = c.Update(b, kFinal, vFinal)++ // Reference: the in-window logical-position-ordered K and V are+ // the last `window` per-position values (positions+ // [finalPos-window+1, finalPos]). Build them in that order.+ startPos := max(finalPos-window+1, 0)+ logicalKs := make([]*mlx.Array, 0, window)+ logicalVs := make([]*mlx.Array, 0, window)+ for pos := startPos; pos <= finalPos; pos++ {+ kp, vp := perPosKV(pos)+ logicalKs = append(logicalKs, kp)+ logicalVs = append(logicalVs, vp)+ }+ kLogical = mlx.Concatenate(logicalKs, 2)+ vLogical = mlx.Concatenate(logicalVs, 2)- // A logical-order ArrayMask with distinct, non-trivial values per- // key column. Picked so each column's contribution to softmax is- // distinct — the test fails if the cache's gather permutes the- // columns wrong before the kernel sees them.- maskVals := []float32{0.1, -0.3, 0.7, -0.2}- logicalMask := mlx.FromValues(maskVals, 1, 1, 1, window)+ // A logical-order ArrayMask with distinct, non-trivial values per+ // key column. Picked so each column's contribution to softmax is+ // distinct — the test fails if the cache's gather permutes the+ // columns wrong before the kernel sees them.+ logicalMask = mlx.FromValues([]float32{0.1, -0.3, 0.7, -0.2}, 1, 1, 1, window)+ })cases := []struct {name string@@ -95,7 +98,7 @@ func TestRotatingKVCacheDecodeParity(t *testing.T) {}for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tc.name, func(t *mlxtest.T) {got := nn.ScaledDotProductAttention(b, q, scale,nn.WithKVHistory(history),nn.WithMask(tc.model))@@ -115,45 +118,57 @@ func TestRotatingKVCacheDecodeParity(t *testing.T) {}func TestAssistantSharedHistoryL1MasksMatchNoMask(t *testing.T) {- skipIfNoMLX(t)- if !mlx.MetalIsAvailable() {- t.Skip("MLX Metal not available")- }const H, D = 1, 4const window = 4const total = 7const scale = 1.0- q := mlx.FromValues([]float32{0.7, -0.4, 0.2, 0.9}, 1, H, 1, D)- mlx.Eval(q)-- full := NewKVCache()- sliding := NewRotatingKVCache(window)- for pos := range total {- kVals := make([]float32, H*D)- vVals := make([]float32, H*D)- for i := range kVals {- kVals[i] = 0.1*float32(pos+1) + 0.01*float32(i)- vVals[i] = -0.1*float32(pos+1) + 0.01*float32(i)- }- k := mlx.FromValues(kVals, 1, H, 1, D)- v := mlx.FromValues(vVals, 1, H, 1, D)- full.Update(newKVBatch(full.Offset(), 1), k, v)- sliding.Update(newKVBatch(sliding.Offset(), 1), k, v)- }-- b := newKVBatch(total-1, 1)- cases := []struct {+ var available bool+ var q *mlx.Array+ var b *batch.Batch+ var cases []struct {name stringh *nn.KVHistorymask nn.AttentionMask- }{- {name: "full", h: full.View(b), mask: nn.CausalMask()},- {name: "sliding", h: sliding.View(b), mask: nn.CausalMask()},+ }+ mlxtest.Run(t, func(t *mlxtest.T) {+ available = mlx.MetalIsAvailable()+ if !available {+ return+ }++ q = mlx.FromValues([]float32{0.7, -0.4, 0.2, 0.9}, 1, H, 1, D)+ full := NewKVCache()+ sliding := NewRotatingKVCache(window)+ for pos := range total {+ kVals := make([]float32, H*D)+ vVals := make([]float32, H*D)+ for i := range kVals {+ kVals[i] = 0.1*float32(pos+1) + 0.01*float32(i)+ vVals[i] = -0.1*float32(pos+1) + 0.01*float32(i)+ }+ k := mlx.FromValues(kVals, 1, H, 1, D)+ v := mlx.FromValues(vVals, 1, H, 1, D)+ full.Update(newKVBatch(full.Offset(), 1), k, v)+ sliding.Update(newKVBatch(sliding.Offset(), 1), k, v)+ }++ b = newKVBatch(total-1, 1)+ cases = []struct {+ name string+ h *nn.KVHistory+ mask nn.AttentionMask+ }{+ {name: "full", h: full.View(b), mask: nn.CausalMask()},+ {name: "sliding", h: sliding.View(b), mask: nn.CausalMask()},+ }+ })+ if !available {+ t.Skip("MLX Metal not available")}for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tc.name, func(t *mlxtest.T) {got := nn.ScaledDotProductAttention(b, q, scale, nn.WithKVHistory(tc.h), nn.WithMask(tc.mask))want := mlx.FastScaledDotProductAttention(q, tc.h.K(), tc.h.V(), scale, "", nil)@@ -173,23 +188,26 @@ func TestAssistantSharedHistoryL1MasksMatchNoMask(t *testing.T) {// matches a reference computed from the same K/V with the model mask// and window restriction composed manually.func TestRotatingKVCachePrefillParity(t *testing.T) {- skipIfNoMLX(t)const H, L, D = 1, 6, 4const window = 4const scale = 1.0- qVals := make([]float32, 1*H*L*D)- kVals := make([]float32, 1*H*L*D)- vVals := make([]float32, 1*H*L*D)- for i := range qVals {- qVals[i] = 0.5 + 0.05*float32(i)- kVals[i] = -0.3 + 0.07*float32(i)- vVals[i] = 0.3 + 0.03*float32(i)- }- q := mlx.FromValues(qVals, 1, H, L, D)- k := mlx.FromValues(kVals, 1, H, L, D)- v := mlx.FromValues(vVals, 1, H, L, D)- b := newKVBatch(0, L)+ var q, k, v *mlx.Array+ var b *batch.Batch+ mlxtest.Run(t, func(t *mlxtest.T) {+ qVals := make([]float32, 1*H*L*D)+ kVals := make([]float32, 1*H*L*D)+ vVals := make([]float32, 1*H*L*D)+ for i := range qVals {+ qVals[i] = 0.5 + 0.05*float32(i)+ kVals[i] = -0.3 + 0.07*float32(i)+ vVals[i] = 0.3 + 0.03*float32(i)+ }+ q = mlx.FromValues(qVals, 1, H, L, D)+ k = mlx.FromValues(kVals, 1, H, L, D)+ v = mlx.FromValues(vVals, 1, H, L, D)+ b = newKVBatch(0, L)+ })cases := []struct {name string@@ -205,7 +223,7 @@ func TestRotatingKVCachePrefillParity(t *testing.T) {negInf := float32(math.Inf(-1))for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tc.name, func(t *mlxtest.T) {c := NewRotatingKVCache(window)history := c.Update(b, k, v)@@ -257,78 +275,79 @@ func TestRotatingKVCachePrefillParity(t *testing.T) {// an identical write with no snapshots scheduled. Capture happens after the// write via lazy snapshots, so it must not perturb the write itself.func TestRotatingKVCacheScheduledSnapshotParity(t *testing.T) {- skipIfNoMLX(t)- const H, D = 1, 4- const window = 4- const before = 5 // past wrap before the batched write- const draft = 4- const scale = 1.0-- perPosKV := func(pos int) (k, v *mlx.Array) {- kVals := make([]float32, H*D)- vVals := make([]float32, H*D)- for i := range kVals {- kVals[i] = 0.1*float32(pos+1) + 0.01*float32(i)- vVals[i] = -0.1*float32(pos+1) + 0.01*float32(i)+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, D = 1, 4+ const window = 4+ const before = 5 // past wrap before the batched write+ const draft = 4+ const scale = 1.0++ perPosKV := func(pos int) (k, v *mlx.Array) {+ kVals := make([]float32, H*D)+ vVals := make([]float32, H*D)+ for i := range kVals {+ kVals[i] = 0.1*float32(pos+1) + 0.01*float32(i)+ vVals[i] = -0.1*float32(pos+1) + 0.01*float32(i)+ }+ return mlx.FromValues(kVals, 1, H, 1, D), mlx.FromValues(vVals, 1, H, 1, D)}- return mlx.FromValues(kVals, 1, H, 1, D), mlx.FromValues(vVals, 1, H, 1, D)- }-- // Build the batched K/V for offsets [before, before+draft).- batchK := make([]*mlx.Array, draft)- batchV := make([]*mlx.Array, draft)- for i := range draft {- batchK[i], batchV[i] = perPosKV(before + i)- }- kBatch := mlx.Concatenate(batchK, 2)- vBatch := mlx.Concatenate(batchV, 2)- qVals := make([]float32, H*draft*D)- for i := range qVals {- qVals[i] = 0.5 + 0.05*float32(i)- }- q := mlx.FromValues(qVals, 1, H, draft, D)- b := newKVBatch(before, draft)+ // Build the batched K/V for offsets [before, before+draft).+ batchK := make([]*mlx.Array, draft)+ batchV := make([]*mlx.Array, draft)+ for i := range draft {+ batchK[i], batchV[i] = perPosKV(before + i)+ }+ kBatch := mlx.Concatenate(batchK, 2)+ vBatch := mlx.Concatenate(batchV, 2)- // Run the same write twice: once with snapshots scheduled, once without.- run := func(schedule bool) []float32 {- c := NewRotatingKVCache(window)- for pos := range before {- k, v := perPosKV(pos)- c.Update(newKVBatch(c.Offset(), 1), k, v)+ qVals := make([]float32, H*draft*D)+ for i := range qVals {+ qVals[i] = 0.5 + 0.05*float32(i)}- if schedule {- offsets := make([]int, draft)- for i := range offsets {- offsets[i] = before + i+ q := mlx.FromValues(qVals, 1, H, draft, D)+ b := newKVBatch(before, draft)++ // Run the same write twice: once with snapshots scheduled, once without.+ run := func(schedule bool) []float32 {+ c := NewRotatingKVCache(window)+ for pos := range before {+ k, v := perPosKV(pos)+ c.Update(newKVBatch(c.Offset(), 1), k, v)}- c.PrepareSnapshots(offsets)- }- history := c.Update(b, kBatch, vBatch)- out := nn.ScaledDotProductAttention(b, q, scale,- nn.WithKVHistory(history),- nn.WithMask(nn.CausalMask()))- if schedule {- for _, s := range c.TakeSnapshots() {- if s != nil {- s.Close()+ if schedule {+ offsets := make([]int, draft)+ for i := range offsets {+ offsets[i] = before + i}+ c.PrepareSnapshots(offsets)}+ history := c.Update(b, kBatch, vBatch)+ out := nn.ScaledDotProductAttention(b, q, scale,+ nn.WithKVHistory(history),+ nn.WithMask(nn.CausalMask()))+ if schedule {+ for _, s := range c.TakeSnapshots() {+ if s != nil {+ s.Close()+ }+ }+ }+ mlx.Eval(out)+ return out.Floats()}- mlx.Eval(out)- return out.Floats()- }- withSnap := run(true)- noSnap := run(false)- if len(withSnap) != len(noSnap) {- t.Fatalf("output length %d vs %d", len(withSnap), len(noSnap))- }- for i := range noSnap {- if math.Abs(float64(withSnap[i]-noSnap[i])) > 1e-5 {- t.Fatalf("index %d: scheduled=%v, unscheduled=%v", i, withSnap[i], noSnap[i])+ withSnap := run(true)+ noSnap := run(false)+ if len(withSnap) != len(noSnap) {+ t.Fatalf("output length %d vs %d", len(withSnap), len(noSnap))}- }+ for i := range noSnap {+ if math.Abs(float64(withSnap[i]-noSnap[i])) > 1e-5 {+ t.Fatalf("index %d: scheduled=%v, unscheduled=%v", i, withSnap[i], noSnap[i])+ }+ }+ })}// TestRotatingKVCacheMLAParity drives a rotating cache with the MLA@@ -337,35 +356,36 @@ func TestRotatingKVCacheScheduledSnapshotParity(t *testing.T) {// a manual reference. Pins the cache+MLA integration that// glm4_moe_lite uses in production.func TestRotatingKVCacheMLAParity(t *testing.T) {- skipIfNoMLX(t)- const H, L, D, valueDim = 1, 3, 6, 4- const scale = 1.0- const window = 8 // window >= L so no window restriction+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, L, D, valueDim = 1, 3, 6, 4+ const scale = 1.0+ const window = 8 // window >= L so no window restriction- kVals := make([]float32, 1*H*L*D)- for i := range kVals {- kVals[i] = 0.1 * float32(i+1)- }- k := mlx.FromValues(kVals, 1, H, L, D)- v := mlx.Zeros(mlx.DTypeFloat32, 1, H, L, 0)-- q := mlx.Zeros(mlx.DTypeFloat32, 1, H, L, D)- b := newKVBatch(0, L)-- c := NewRotatingKVCache(window)- history := c.Update(b, k, v)- got := nn.ScaledDotProductAttention(b, q, scale,- nn.WithMLAHistory(history, valueDim),- nn.WithMask(nn.CausalMask()))-- vRef := k.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(0, valueDim))- want := mlx.FastScaledDotProductAttention(q, k, vRef, scale, "causal", nil)-- mlx.Eval(got, want)- gs, ws := got.Floats(), want.Floats()- for i := range ws {- if math.Abs(float64(gs[i]-ws[i])) > 1e-5 {- t.Fatalf("index %d: got %v, want %v", i, gs[i], ws[i])+ kVals := make([]float32, 1*H*L*D)+ for i := range kVals {+ kVals[i] = 0.1 * float32(i+1)}- }+ k := mlx.FromValues(kVals, 1, H, L, D)+ v := mlx.Zeros(mlx.DTypeFloat32, 1, H, L, 0)++ q := mlx.Zeros(mlx.DTypeFloat32, 1, H, L, D)+ b := newKVBatch(0, L)++ c := NewRotatingKVCache(window)+ history := c.Update(b, k, v)+ got := nn.ScaledDotProductAttention(b, q, scale,+ nn.WithMLAHistory(history, valueDim),+ nn.WithMask(nn.CausalMask()))++ vRef := k.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(0, valueDim))+ want := mlx.FastScaledDotProductAttention(q, k, vRef, scale, "causal", nil)++ mlx.Eval(got, want)+ gs, ws := got.Floats(), want.Floats()+ for i := range ws {+ if math.Abs(float64(gs[i]-ws[i])) > 1e-5 {+ t.Fatalf("index %d: got %v, want %v", i, gs[i], ws[i])+ }+ }+ })}diff --git a/x/mlxrunner/cache/rotating_multiturn_test.go b/x/mlxrunner/cache/rotating_multiturn_test.goindex 03991a1b9..2b6ce30c0 100644--- a/x/mlxrunner/cache/rotating_multiturn_test.go+++ b/x/mlxrunner/cache/rotating_multiturn_test.go@@ -3,6 +3,7 @@ package cacheimport ("testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx")@@ -27,7 +28,7 @@ func multiTokenKV(ids []float32) (*mlx.Array, *mlx.Array) {// stateIDs returns the ids currently in the cache in slot order (logical// after a concat, physical/rotated after a single-token update).-func stateIDs(t *testing.T, c *RotatingKVCache) []float32 {+func stateIDs(t *mlxtest.T, c *RotatingKVCache) []float32 {t.Helper()state := c.State()if state == nil {@@ -75,32 +76,32 @@ func feedSingle(c *RotatingKVCache, id float32) {// pre-existing tokens in logical order so the first Q of the new batch// has a full sliding window.func TestRotatingKVCacheConcatMidRotationPreservesContext(t *testing.T) {- skipIfNoMLX(t)-- const window = 4- c := NewRotatingKVCache(window)-- nextID := feedMulti(c, 1, 3)- for range 6 {- feedSingle(c, nextID)- nextID++- }- if c.Offset() != 9 {- t.Fatalf("setup: offset=%d want 9", c.Offset())- }- if c.idx >= c.maxSize {- t.Fatalf("setup: expected mid-rotation idx (<%d), got %d", c.maxSize, c.idx)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 4+ c := NewRotatingKVCache(window)++ nextID := feedMulti(c, 1, 3)+ for range 6 {+ feedSingle(c, nextID)+ nextID+++ }+ if c.Offset() != 9 {+ t.Fatalf("setup: offset=%d want 9", c.Offset())+ }+ if c.idx >= c.maxSize {+ t.Fatalf("setup: expected mid-rotation idx (<%d), got %d", c.maxSize, c.idx)+ }- feedMulti(c, 10, 2)- got := stateIDs(t, c)- want := []float32{7, 8, 9, 10, 11}- if !equalSlice(got, want) {- t.Fatalf("post-concat window=%v want %v", got, want)- }- if c.Offset() != 11 {- t.Fatalf("offset=%d want 11", c.Offset())- }+ feedMulti(c, 10, 2)+ got := stateIDs(t, c)+ want := []float32{7, 8, 9, 10, 11}+ if !equalSlice(got, want) {+ t.Fatalf("post-concat window=%v want %v", got, want)+ }+ if c.Offset() != 11 {+ t.Fatalf("offset=%d want 11", c.Offset())+ }+ })}// TestRotatingKVCacheConcatAlignedInvariant: with an aligned buffer@@ -108,38 +109,38 @@ func TestRotatingKVCacheConcatMidRotationPreservesContext(t *testing.T) {// tokens plus the full new batch. This is the chunked-prefill contract// x/mlxrunner/pipeline.go relies on.func TestRotatingKVCacheConcatAlignedInvariant(t *testing.T) {- skipIfNoMLX(t)-- const window = 4- c := NewRotatingKVCache(window)-- // Chunk 1 fills past maxSize, leaving Dim == maxSize aligned.- feedMulti(c, 1, 6)- // Chunk 2: the buffer is intentionally oversized to (maxSize-1) + L- // so the first new Q has its full window in scope for this forward.- feedMulti(c, 7, 3)- got := stateIDs(t, c)- want := []float32{4, 5, 6, 7, 8, 9}- if !equalSlice(got, want) {- t.Fatalf("post-chunk-2 buffer=%v want %v", got, want)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 4+ c := NewRotatingKVCache(window)++ // Chunk 1 fills past maxSize, leaving Dim == maxSize aligned.+ feedMulti(c, 1, 6)+ // Chunk 2: the buffer is intentionally oversized to (maxSize-1) + L+ // so the first new Q has its full window in scope for this forward.+ feedMulti(c, 7, 3)+ got := stateIDs(t, c)+ want := []float32{4, 5, 6, 7, 8, 9}+ if !equalSlice(got, want) {+ t.Fatalf("post-chunk-2 buffer=%v want %v", got, want)+ }- // The next decode trims oversize back to maxSize; order may be- // physical (rotated), so check as a set.- feedSingle(c, 10)- got = stateIDs(t, c)- if len(got) != window {- t.Fatalf("post-decode Dim=%d want %d", len(got), window)- }- seen := map[float32]bool{}- for _, v := range got {- seen[v] = true- }- for _, w := range []float32{7, 8, 9, 10} {- if !seen[w] {- t.Fatalf("post-decode window missing %v (got %v)", w, got)+ // The next decode trims oversize back to maxSize; order may be+ // physical (rotated), so check as a set.+ feedSingle(c, 10)+ got = stateIDs(t, c)+ if len(got) != window {+ t.Fatalf("post-decode Dim=%d want %d", len(got), window)}- }+ seen := map[float32]bool{}+ for _, v := range got {+ seen[v] = true+ }+ for _, w := range []float32{7, 8, 9, 10} {+ if !seen[w] {+ t.Fatalf("post-decode window missing %v (got %v)", w, got)+ }+ }+ })}// TestRotatingKVCacheConcatAfterDecodeGrowsBuffer: update() grows the@@ -148,20 +149,20 @@ func TestRotatingKVCacheConcatAlignedInvariant(t *testing.T) {// has not wrapped. Those trailing slots are zero padding and must not// be pulled back into the live window on the next concat.func TestRotatingKVCacheConcatAfterDecodeGrowsBuffer(t *testing.T) {- skipIfNoMLX(t)-- const window = 512- c := NewRotatingKVCache(window)-- feedMulti(c, 1, 3)- feedSingle(c, 4)- feedMulti(c, 5, 3)-- got := stateIDs(t, c)- want := []float32{1, 2, 3, 4, 5, 6, 7}- if !equalSlice(got, want) {- t.Fatalf("growing-buffer concat=%v want %v", got, want)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 512+ c := NewRotatingKVCache(window)++ feedMulti(c, 1, 3)+ feedSingle(c, 4)+ feedMulti(c, 5, 3)++ got := stateIDs(t, c)+ want := []float32{1, 2, 3, 4, 5, 6, 7}+ if !equalSlice(got, want) {+ t.Fatalf("growing-buffer concat=%v want %v", got, want)+ }+ })}// TestRotatingKVCacheConcatAfterLiveRewind: x/mlxrunner/cache.go calls@@ -171,53 +172,53 @@ func TestRotatingKVCacheConcatAfterDecodeGrowsBuffer(t *testing.T) {// tokens. A subsequent concat must drop those, not treat them as wrapped// window content.func TestRotatingKVCacheConcatAfterLiveRewind(t *testing.T) {- skipIfNoMLX(t)-- const window = 8- c := NewRotatingKVCache(window)-- // Grow the buffer to exactly maxSize without wrapping.- feedMulti(c, 1, 2)- for id := float32(3); id <= 8; id++ {- feedSingle(c, id)- }- if c.Offset() != window {- t.Fatalf("setup: offset=%d want %d", c.Offset(), window)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 8+ c := NewRotatingKVCache(window)++ // Grow the buffer to exactly maxSize without wrapping.+ feedMulti(c, 1, 2)+ for id := float32(3); id <= 8; id++ {+ feedSingle(c, id)+ }+ if c.Offset() != window {+ t.Fatalf("setup: offset=%d want %d", c.Offset(), window)+ }- if !c.Restore(nil, 2) {- t.Fatalf("live rewind to 2 failed")- }- if c.Offset() != 2 {- t.Fatalf("post-rewind offset=%d want 2", c.Offset())- }+ if !c.Restore(nil, 2) {+ t.Fatalf("live rewind to 2 failed")+ }+ if c.Offset() != 2 {+ t.Fatalf("post-rewind offset=%d want 2", c.Offset())+ }- feedMulti(c, 9, 3)- got := stateIDs(t, c)- want := []float32{1, 2, 9, 10, 11}- if !equalSlice(got, want) {- t.Fatalf("post-rewind concat=%v want %v", got, want)- }- if c.Offset() != 5 {- t.Fatalf("offset=%d want 5", c.Offset())- }+ feedMulti(c, 9, 3)+ got := stateIDs(t, c)+ want := []float32{1, 2, 9, 10, 11}+ if !equalSlice(got, want) {+ t.Fatalf("post-rewind concat=%v want %v", got, want)+ }+ if c.Offset() != 5 {+ t.Fatalf("offset=%d want 5", c.Offset())+ }+ })}// TestRotatingKVCacheConcatGrowingBuffer: when oldLen < maxSize the trim// formula drops to non-positive and all pre-existing tokens are kept.func TestRotatingKVCacheConcatGrowingBuffer(t *testing.T) {- skipIfNoMLX(t)-- const window = 4- c := NewRotatingKVCache(window)-- feedMulti(c, 1, 2)- feedMulti(c, 3, 2)- got := stateIDs(t, c)- want := []float32{1, 2, 3, 4}- if !equalSlice(got, want) {- t.Fatalf("growing buffer=%v want %v", got, want)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 4+ c := NewRotatingKVCache(window)++ feedMulti(c, 1, 2)+ feedMulti(c, 3, 2)+ got := stateIDs(t, c)+ want := []float32{1, 2, 3, 4}+ if !equalSlice(got, want) {+ t.Fatalf("growing buffer=%v want %v", got, want)+ }+ })}// TestRotatingKVCacheRunnerChunkedPrefill mirrors the@@ -225,114 +226,114 @@ func TestRotatingKVCacheConcatGrowingBuffer(t *testing.T) {// repeated L>1 Update() calls on a single cache. Scaled-down proxy for// the Gemma 4 26B case (sliding_window=1024, prefillChunkSize=2048).func TestRotatingKVCacheRunnerChunkedPrefill(t *testing.T) {- skipIfNoMLX(t)-- const window = 4- c := NewRotatingKVCache(window)+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 4+ c := NewRotatingKVCache(window)- feedMulti(c, 1, 8)- if c.Offset() != 8 {- t.Fatalf("chunk 1: offset=%d want 8", c.Offset())- }+ feedMulti(c, 1, 8)+ if c.Offset() != 8 {+ t.Fatalf("chunk 1: offset=%d want 8", c.Offset())+ }- feedMulti(c, 9, 8)- got := stateIDs(t, c)- want := []float32{6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}- if !equalSlice(got, want) {- t.Fatalf("chunk 2: buffer=%v want %v", got, want)- }+ feedMulti(c, 9, 8)+ got := stateIDs(t, c)+ want := []float32{6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}+ if !equalSlice(got, want) {+ t.Fatalf("chunk 2: buffer=%v want %v", got, want)+ }- feedMulti(c, 17, 4)- got = stateIDs(t, c)- want = []float32{14, 15, 16, 17, 18, 19, 20}- if !equalSlice(got, want) {- t.Fatalf("chunk 3: buffer=%v want %v", got, want)- }+ feedMulti(c, 17, 4)+ got = stateIDs(t, c)+ want = []float32{14, 15, 16, 17, 18, 19, 20}+ if !equalSlice(got, want) {+ t.Fatalf("chunk 3: buffer=%v want %v", got, want)+ }- // Decode trims oversize back to maxSize; order may be physical.- feedSingle(c, 21)- got = stateIDs(t, c)- if len(got) != window {- t.Fatalf("post-decode Dim=%d want %d", len(got), window)- }- seen := map[float32]bool{}- for _, v := range got {- seen[v] = true- }- for _, w := range []float32{18, 19, 20, 21} {- if !seen[w] {- t.Fatalf("post-decode window missing %v (got %v)", w, got)+ // Decode trims oversize back to maxSize; order may be physical.+ feedSingle(c, 21)+ got = stateIDs(t, c)+ if len(got) != window {+ t.Fatalf("post-decode Dim=%d want %d", len(got), window)}- }+ seen := map[float32]bool{}+ for _, v := range got {+ seen[v] = true+ }+ for _, w := range []float32{18, 19, 20, 21} {+ if !seen[w] {+ t.Fatalf("post-decode window missing %v (got %v)", w, got)+ }+ }+ })}// TestRotatingKVCacheMultiTurnChatSimulation walks a prefill → decode →// prefill sequence and checks that each new prefill retains the last// (maxSize-1) pre-existing tokens in logical order.func TestRotatingKVCacheMultiTurnChatSimulation(t *testing.T) {- skipIfNoMLX(t)-- const window = 4- c := NewRotatingKVCache(window)-- nextID := feedMulti(c, 1, 2)- for range 5 {- feedSingle(c, nextID)- nextID++- }- if c.Offset() != 7 {- t.Fatalf("turn 1: offset=%d want 7", c.Offset())- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 4+ c := NewRotatingKVCache(window)++ nextID := feedMulti(c, 1, 2)+ for range 5 {+ feedSingle(c, nextID)+ nextID+++ }+ if c.Offset() != 7 {+ t.Fatalf("turn 1: offset=%d want 7", c.Offset())+ }- feedMulti(c, nextID, 3)- nextID += 3- got := stateIDs(t, c)- want := []float32{5, 6, 7, 8, 9, 10}- if !equalSlice(got, want) {- t.Fatalf("turn 2 prefill buffer=%v want %v", got, want)- }+ feedMulti(c, nextID, 3)+ nextID += 3+ got := stateIDs(t, c)+ want := []float32{5, 6, 7, 8, 9, 10}+ if !equalSlice(got, want) {+ t.Fatalf("turn 2 prefill buffer=%v want %v", got, want)+ }- for range 4 {- feedSingle(c, nextID)- nextID++- }- if c.Offset() != 14 {- t.Fatalf("turn 2 decode: offset=%d want 14", c.Offset())- }+ for range 4 {+ feedSingle(c, nextID)+ nextID+++ }+ if c.Offset() != 14 {+ t.Fatalf("turn 2 decode: offset=%d want 14", c.Offset())+ }- feedMulti(c, nextID, 2)- got = stateIDs(t, c)- want = []float32{12, 13, 14, 15, 16}- if !equalSlice(got, want) {- t.Fatalf("turn 3 prefill buffer=%v want %v", got, want)- }+ feedMulti(c, nextID, 2)+ got = stateIDs(t, c)+ want = []float32{12, 13, 14, 15, 16}+ if !equalSlice(got, want) {+ t.Fatalf("turn 3 prefill buffer=%v want %v", got, want)+ }+ })}// TestRotatingKVCacheOffsetTracking: Offset() is the monotonic logical// token count through any mix of Update() calls — Gemma 4 uses// donorEntry.Offset - L for the consumer's RoPE offset.func TestRotatingKVCacheOffsetTracking(t *testing.T) {- skipIfNoMLX(t)-- c := NewRotatingKVCache(4)- nextID := feedMulti(c, 1, 3)- if c.Offset() != 3 {- t.Fatalf("after prefill 3: offset=%d want 3", c.Offset())- }- for i := range 5 {- feedSingle(c, nextID)- nextID++- if c.Offset() != 3+i+1 {- t.Fatalf("after decode %d: offset=%d want %d", i, c.Offset(), 3+i+1)+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewRotatingKVCache(4)+ nextID := feedMulti(c, 1, 3)+ if c.Offset() != 3 {+ t.Fatalf("after prefill 3: offset=%d want 3", c.Offset())}- }- nextID = feedMulti(c, nextID, 2)- if c.Offset() != 10 {- t.Fatalf("after turn-2 prefill: offset=%d want 10", c.Offset())- }- // L > maxSize concat.- feedMulti(c, nextID, 7)- if c.Offset() != 17 {- t.Fatalf("after large prefill: offset=%d want 17", c.Offset())- }+ for i := range 5 {+ feedSingle(c, nextID)+ nextID+++ if c.Offset() != 3+i+1 {+ t.Fatalf("after decode %d: offset=%d want %d", i, c.Offset(), 3+i+1)+ }+ }+ nextID = feedMulti(c, nextID, 2)+ if c.Offset() != 10 {+ t.Fatalf("after turn-2 prefill: offset=%d want 10", c.Offset())+ }+ // L > maxSize concat.+ feedMulti(c, nextID, 7)+ if c.Offset() != 17 {+ t.Fatalf("after large prefill: offset=%d want 17", c.Offset())+ }+ })}diff --git a/x/mlxrunner/cache/snapshot_capture_test.go b/x/mlxrunner/cache/snapshot_capture_test.goindex eaa47835e..eb81df1a3 100644--- a/x/mlxrunner/cache/snapshot_capture_test.go+++ b/x/mlxrunner/cache/snapshot_capture_test.go@@ -4,6 +4,7 @@ import ("slices""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx")@@ -55,7 +56,7 @@ func fillTagged(c Attention, n int) {// windowTags reads c's logical window and returns the per-position tag (the// absolute offset each slot holds, recovered as value-1). It uses element 0 of// each token, which taggedKV set uniformly. Returns nil if the window is empty.-func windowTags(t *testing.T, c *RotatingKVCache) []int {+func windowTags(t *mlxtest.T, c *RotatingKVCache) []int {t.Helper()state := c.State()if len(state) == 0 {@@ -100,68 +101,68 @@ func wantWindowTags(offset, window int) []int {// speculation commit path — a live rewind, since KV is append-only — restores// the cache to each accepted offset with the prefix intact.func TestKVCachePerTokenSnapshotRestore(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ const before = 6+ const draft = 4- const before = 6- const draft = 4-- for accepted := 0; accepted <= draft; accepted++ {- c := NewKVCache()- fillKV(c, before)+ for accepted := 0; accepted <= draft; accepted++ {+ c := NewKVCache()+ fillKV(c, before)- offsets := make([]int, draft)- for i := range offsets {- offsets[i] = before + i- }- c.PrepareSnapshots(offsets)+ offsets := make([]int, draft)+ for i := range offsets {+ offsets[i] = before + i+ }+ c.PrepareSnapshots(offsets)- k, v := batchKV(draft)- c.Update(newKVBatch(before, draft), k, v)- if c.Offset() != before+draft {- t.Fatalf("accepted=%d: offset after write = %d, want %d", accepted, c.Offset(), before+draft)- }+ k, v := batchKV(draft)+ c.Update(newKVBatch(before, draft), k, v)+ if c.Offset() != before+draft {+ t.Fatalf("accepted=%d: offset after write = %d, want %d", accepted, c.Offset(), before+draft)+ }- snaps := c.TakeSnapshots()- if len(snaps) != draft {- t.Fatalf("accepted=%d: got %d snapshots, want %d", accepted, len(snaps), draft)- }+ snaps := c.TakeSnapshots()+ if len(snaps) != draft {+ t.Fatalf("accepted=%d: got %d snapshots, want %d", accepted, len(snaps), draft)+ }- // Captures are edge-local: offset before is zero-width (nil), and each- // later offset holds exactly the single token [before+i-1, before+i).- if snaps[0] != nil {- t.Fatalf("accepted=%d: snaps[0] = %v, want nil (zero-width base)", accepted, snaps[0])- }- for i := 1; i < draft; i++ {- ks := snaps[i].(*kvSnapshot)- if ks.fromOffset != before+i-1 || ks.toOffset != before+i {- t.Fatalf("accepted=%d: snaps[%d] = [%d,%d), want [%d,%d)", accepted, i, ks.fromOffset, ks.toOffset, before+i-1, before+i)+ // Captures are edge-local: offset before is zero-width (nil), and each+ // later offset holds exactly the single token [before+i-1, before+i).+ if snaps[0] != nil {+ t.Fatalf("accepted=%d: snaps[0] = %v, want nil (zero-width base)", accepted, snaps[0])+ }+ for i := 1; i < draft; i++ {+ ks := snaps[i].(*kvSnapshot)+ if ks.fromOffset != before+i-1 || ks.toOffset != before+i {+ t.Fatalf("accepted=%d: snaps[%d] = [%d,%d), want [%d,%d)", accepted, i, ks.fromOffset, ks.toOffset, before+i-1, before+i)+ }}- }- // Commit rolls back via a live rewind (Restore(nil)) — the append-only- // buffer still holds [0, before+draft), so the edge captures go unused.- if accepted < draft {- if !c.Restore(nil, before+accepted) {- t.Fatalf("accepted=%d: live rewind failed", accepted)+ // Commit rolls back via a live rewind (Restore(nil)) — the append-only+ // buffer still holds [0, before+draft), so the edge captures go unused.+ if accepted < draft {+ if !c.Restore(nil, before+accepted) {+ t.Fatalf("accepted=%d: live rewind failed", accepted)+ }}- }- for _, s := range snaps {- if s != nil {- s.Close()+ for _, s := range snaps {+ if s != nil {+ s.Close()+ }}- }- want := before + draft- if accepted < draft {- want = before + accepted- }- if c.Offset() != want {- t.Fatalf("accepted=%d: offset after commit = %d, want %d", accepted, c.Offset(), want)- }- if st := c.State(); len(st) == 2 && st[0].Dim(2) != want {- t.Fatalf("accepted=%d: state seq dim = %d, want %d", accepted, st[0].Dim(2), want)+ want := before + draft+ if accepted < draft {+ want = before + accepted+ }+ if c.Offset() != want {+ t.Fatalf("accepted=%d: offset after commit = %d, want %d", accepted, c.Offset(), want)+ }+ if st := c.State(); len(st) == 2 && st[0].Dim(2) != want {+ t.Fatalf("accepted=%d: state seq dim = %d, want %d", accepted, st[0].Dim(2), want)+ }}- }+ })}// TestKVCaptureMergeSplit verifies that two adjacent edge-local captures merge@@ -169,46 +170,46 @@ func TestKVCachePerTokenSnapshotRestore(t *testing.T) {// trie performs on stored snapshots (mergeWithChild / splitNode) — proving they// work on captured snapshots, not just freshly-taken ones.func TestKVCaptureMergeSplit(t *testing.T) {- skipIfNoMLX(t)-- const before = 6- c := NewKVCache()- fillKV(c, before)-- // Schedule two interior offsets so the write captures [before, before+1) and- // [before+1, before+2).- c.PrepareSnapshots([]int{before + 1, before + 2})- k, v := batchKV(3)- c.Update(newKVBatch(before, 3), k, v)-- snaps := c.TakeSnapshots()- a := snaps[0].(*kvSnapshot)- b := snaps[1].(*kvSnapshot)- if a.fromOffset != before || a.toOffset != before+1 {- t.Fatalf("snaps[0] = [%d,%d), want [%d,%d)", a.fromOffset, a.toOffset, before, before+1)- }- if b.fromOffset != before+1 || b.toOffset != before+2 {- t.Fatalf("snaps[1] = [%d,%d), want [%d,%d)", b.fromOffset, b.toOffset, before+1, before+2)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const before = 6+ c := NewKVCache()+ fillKV(c, before)- // Merge the adjacent edges into [before, before+2).- merged := c.Merge(snaps[0], snaps[1]).(*kvSnapshot)- if merged.fromOffset != before || merged.toOffset != before+2 {- t.Fatalf("merged = [%d,%d), want [%d,%d)", merged.fromOffset, merged.toOffset, before, before+2)- }+ // Schedule two interior offsets so the write captures [before, before+1) and+ // [before+1, before+2).+ c.PrepareSnapshots([]int{before + 1, before + 2})+ k, v := batchKV(3)+ c.Update(newKVBatch(before, 3), k, v)- // Split back at before+1 and confirm the halves match the originals.- p, ch := c.Split(merged, before+1)- ps := p.(*kvSnapshot)- cs := ch.(*kvSnapshot)- if ps.fromOffset != before || ps.toOffset != before+1 {- t.Fatalf("split parent = [%d,%d), want [%d,%d)", ps.fromOffset, ps.toOffset, before, before+1)- }- if cs.fromOffset != before+1 || cs.toOffset != before+2 {- t.Fatalf("split child = [%d,%d), want [%d,%d)", cs.fromOffset, cs.toOffset, before+1, before+2)- }- p.Close()- ch.Close()+ snaps := c.TakeSnapshots()+ a := snaps[0].(*kvSnapshot)+ b := snaps[1].(*kvSnapshot)+ if a.fromOffset != before || a.toOffset != before+1 {+ t.Fatalf("snaps[0] = [%d,%d), want [%d,%d)", a.fromOffset, a.toOffset, before, before+1)+ }+ if b.fromOffset != before+1 || b.toOffset != before+2 {+ t.Fatalf("snaps[1] = [%d,%d), want [%d,%d)", b.fromOffset, b.toOffset, before+1, before+2)+ }++ // Merge the adjacent edges into [before, before+2).+ merged := c.Merge(snaps[0], snaps[1]).(*kvSnapshot)+ if merged.fromOffset != before || merged.toOffset != before+2 {+ t.Fatalf("merged = [%d,%d), want [%d,%d)", merged.fromOffset, merged.toOffset, before, before+2)+ }++ // Split back at before+1 and confirm the halves match the originals.+ p, ch := c.Split(merged, before+1)+ ps := p.(*kvSnapshot)+ cs := ch.(*kvSnapshot)+ if ps.fromOffset != before || ps.toOffset != before+1 {+ t.Fatalf("split parent = [%d,%d), want [%d,%d)", ps.fromOffset, ps.toOffset, before, before+1)+ }+ if cs.fromOffset != before+1 || cs.toOffset != before+2 {+ t.Fatalf("split child = [%d,%d), want [%d,%d)", cs.fromOffset, cs.toOffset, before+1, before+2)+ }+ p.Close()+ ch.Close()+ })}// TestRotatingPerTokenSnapshotRestore exercises per-token capture on a@@ -219,7 +220,7 @@ func TestKVCaptureMergeSplit(t *testing.T) {// observable as wrong positions, not just wrong shapes; the wrapped regime// forces concat's linearize branch to run on entry.func TestRotatingPerTokenSnapshotRestore(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)cases := []struct {name string@@ -233,7 +234,7 @@ func TestRotatingPerTokenSnapshotRestore(t *testing.T) {const draft = 4for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tc.name, func(t *mlxtest.T) {for accepted := 0; accepted <= draft; accepted++ {c := NewRotatingKVCache(tc.window)fillTagged(c, tc.before)@@ -294,7 +295,7 @@ func TestRotatingPerTokenSnapshotRestore(t *testing.T) {// same window data. Covers the restored content, a following decode write, and// the snapshot copying out correctly afterward, across wrap regimes.func TestRotatingRestoreLazyOwnSnapshotSlices(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)cases := []struct {name string@@ -308,7 +309,7 @@ func TestRotatingRestoreLazyOwnSnapshotSlices(t *testing.T) {const draft, accepted = 4, 2for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tc.name, func(t *mlxtest.T) {c := NewRotatingKVCache(tc.window)fillTagged(c, tc.before)@@ -386,61 +387,61 @@ func TestRotatingRestoreLazyOwnSnapshotSlices(t *testing.T) {// when a following write would destroy the window and copies the snapshot out —// the lazy mechanism paying for itself only when the data is about to be lost.func TestRotatingRestoreHookedSnapshotStaysLazy(t *testing.T) {- skipIfNoMLX(t)-- const window, before, draft, accepted = 4, 10, 4, 2+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window, before, draft, accepted = 4, 10, 4, 2- c := NewRotatingKVCache(window)- fillTagged(c, before)+ c := NewRotatingKVCache(window)+ fillTagged(c, before)- offsets := make([]int, draft)- for i := range offsets {- offsets[i] = before + i- }- c.PrepareSnapshots(offsets)+ offsets := make([]int, draft)+ for i := range offsets {+ offsets[i] = before + i+ }+ c.PrepareSnapshots(offsets)- k, v := taggedKV(before, draft)- c.Update(newKVBatch(before, draft), k, v)+ k, v := taggedKV(before, draft)+ c.Update(newKVBatch(before, draft), k, v)- snaps := c.TakeSnapshots()- snap := snaps[accepted].(*rotatingSnapshot)- // Simulate trie ownership: a node sets a materialize hook on attach.- fired := 0- snap.SetMaterializeHook(func(int) { fired++ })+ snaps := c.TakeSnapshots()+ snap := snaps[accepted].(*rotatingSnapshot)+ // Simulate trie ownership: a node sets a materialize hook on attach.+ fired := 0+ snap.SetMaterializeHook(func(int) { fired++ })- if !c.Restore(snap, before+accepted) {- t.Fatal("restore failed")- }- // Re-pointed, not copied out: still lazy, hook unfired.- if snap.keys != nil {- t.Fatal("hooked snapshot was copied out; expected the re-point fast path")- }- if fired != 0 {- t.Fatalf("materialize hook fired %d times on restore, want 0 (still lazy)", fired)- }+ if !c.Restore(snap, before+accepted) {+ t.Fatal("restore failed")+ }+ // Re-pointed, not copied out: still lazy, hook unfired.+ if snap.keys != nil {+ t.Fatal("hooked snapshot was copied out; expected the re-point fast path")+ }+ if fired != 0 {+ t.Fatalf("materialize hook fired %d times on restore, want 0 (still lazy)", fired)+ }- got := windowTags(t, c)- want := wantWindowTags(before+accepted, window)- if !slices.Equal(got, want) {- t.Fatalf("window tags = %v, want %v", got, want)- }+ got := windowTags(t, c)+ want := wantWindowTags(before+accepted, window)+ if !slices.Equal(got, want) {+ t.Fatalf("window tags = %v, want %v", got, want)+ }- // A following decode write destroys the window's slots, so it copies the- // snapshot out — firing the hook exactly once.- wk, wv := taggedKV(c.Offset(), 1)- c.Update(newKVBatch(c.Offset(), 1), wk, wv)- if snap.keys == nil {- t.Fatal("following write did not copy out the snapshot")- }- if fired != 1 {- t.Fatalf("materialize hook fired %d times, want 1", fired)- }+ // A following decode write destroys the window's slots, so it copies the+ // snapshot out — firing the hook exactly once.+ wk, wv := taggedKV(c.Offset(), 1)+ c.Update(newKVBatch(c.Offset(), 1), wk, wv)+ if snap.keys == nil {+ t.Fatal("following write did not copy out the snapshot")+ }+ if fired != 1 {+ t.Fatalf("materialize hook fired %d times, want 1", fired)+ }- for _, s := range snaps {- if s != nil {- s.Close()+ for _, s := range snaps {+ if s != nil {+ s.Close()+ }}- }+ })}// TestRotatingSnapshotSingleTokenWrite mirrors the tail of chunked prefill: the@@ -451,7 +452,7 @@ func TestRotatingRestoreHookedSnapshotStaysLazy(t *testing.T) {// not-yet-wrapped and wrapped regimes; the wrapped one exercises the ring-clone// fallback.func TestRotatingSnapshotSingleTokenWrite(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)cases := []struct {name string@@ -463,7 +464,7 @@ func TestRotatingSnapshotSingleTokenWrite(t *testing.T) {}for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tc.name, func(t *mlxtest.T) {c := NewRotatingKVCache(tc.window)fillTagged(c, tc.before)@@ -495,46 +496,46 @@ func TestRotatingSnapshotSingleTokenWrite(t *testing.T) {// batched write trims/rewrites the buffer the snapshot's slots lived in. This is// the case that forces a lazy snapshot to copy out before the later write destroys it.func TestRotatingSnapshotSurvivesLaterChunk(t *testing.T) {- skipIfNoMLX(t)-- const window = 6- c := NewRotatingKVCache(window)+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 6+ c := NewRotatingKVCache(window)- // Schedule an offset in the first chunk and one in the second, then write- // both chunks before taking — the second chunk's concat trims past the first- // snapshot's window.- c.PrepareSnapshots([]int{4, 12})+ // Schedule an offset in the first chunk and one in the second, then write+ // both chunks before taking — the second chunk's concat trims past the first+ // snapshot's window.+ c.PrepareSnapshots([]int{4, 12})- k1, v1 := taggedKV(0, 8) // chunk 1: [0, 8)- c.Update(newKVBatch(0, 8), k1, v1)- k2, v2 := taggedKV(8, 8) // chunk 2: [8, 16)- c.Update(newKVBatch(8, 8), k2, v2)+ k1, v1 := taggedKV(0, 8) // chunk 1: [0, 8)+ c.Update(newKVBatch(0, 8), k1, v1)+ k2, v2 := taggedKV(8, 8) // chunk 2: [8, 16)+ c.Update(newKVBatch(8, 8), k2, v2)- snaps := c.TakeSnapshots()- if len(snaps) != 2 || snaps[0] == nil || snaps[1] == nil {- t.Fatalf("snapshots not captured: %v", snaps)- }+ snaps := c.TakeSnapshots()+ if len(snaps) != 2 || snaps[0] == nil || snaps[1] == nil {+ t.Fatalf("snapshots not captured: %v", snaps)+ }- // Restore the first-chunk snapshot (offset 4): its window predates the- // second chunk entirely, so the data must have survived the chunk-2 write.- if !c.Restore(snaps[0], 4) {- t.Fatal("restore to offset 4 failed")- }- if got, want := windowTags(t, c), wantWindowTags(4, window); !slices.Equal(got, want) {- t.Fatalf("offset 4 window tags = %v, want %v", got, want)- }+ // Restore the first-chunk snapshot (offset 4): its window predates the+ // second chunk entirely, so the data must have survived the chunk-2 write.+ if !c.Restore(snaps[0], 4) {+ t.Fatal("restore to offset 4 failed")+ }+ if got, want := windowTags(t, c), wantWindowTags(4, window); !slices.Equal(got, want) {+ t.Fatalf("offset 4 window tags = %v, want %v", got, want)+ }- // Restore the second-chunk snapshot (offset 12) on a fresh cache.- c2 := NewRotatingKVCache(window)- if !c2.Restore(snaps[1], 12) {- t.Fatal("restore to offset 12 failed")- }- if got, want := windowTags(t, c2), wantWindowTags(12, window); !slices.Equal(got, want) {- t.Fatalf("offset 12 window tags = %v, want %v", got, want)- }- for _, s := range snaps {- s.Close()- }+ // Restore the second-chunk snapshot (offset 12) on a fresh cache.+ c2 := NewRotatingKVCache(window)+ if !c2.Restore(snaps[1], 12) {+ t.Fatal("restore to offset 12 failed")+ }+ if got, want := windowTags(t, c2), wantWindowTags(12, window); !slices.Equal(got, want) {+ t.Fatalf("offset 12 window tags = %v, want %v", got, want)+ }+ for _, s := range snaps {+ s.Close()+ }+ })}// TestRotatingLazySnapshotSizeZeroUntilMaterialized verifies the speculation@@ -544,211 +545,211 @@ func TestRotatingSnapshotSurvivesLaterChunk(t *testing.T) {// lazy snapshot reports Size() == 0 until a destructive write copies it out, at// which point the materialize hook fires with the newly-allocated bytes.func TestRotatingLazySnapshotSizeZeroUntilMaterialized(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ const window = 4+ const before = 10 // wrapped+ const draft = 4+ c := NewRotatingKVCache(window)+ fillTagged(c, before)- const window = 4- const before = 10 // wrapped- const draft = 4- c := NewRotatingKVCache(window)- fillTagged(c, before)+ offsets := make([]int, draft)+ for i := range offsets {+ offsets[i] = before + i+ }+ c.PrepareSnapshots(offsets)- offsets := make([]int, draft)- for i := range offsets {- offsets[i] = before + i- }- c.PrepareSnapshots(offsets)+ k, v := taggedKV(before, draft)+ c.Update(newKVBatch(before, draft), k, v)- k, v := taggedKV(before, draft)- c.Update(newKVBatch(before, draft), k, v)+ snaps := c.TakeSnapshots()+ defer func() {+ for _, s := range snaps {+ if s != nil {+ s.Close()+ }+ }+ }()- snaps := c.TakeSnapshots()- defer func() {- for _, s := range snaps {- if s != nil {- s.Close()+ // Interior captures (offsets after the start boundary) stay lazy: no write+ // destroyed their slots, so keys is still nil and the issuing cache is live.+ for i := 1; i < draft; i++ {+ rs := snaps[i].(*rotatingSnapshot)+ if rs.keys != nil {+ t.Fatalf("snaps[%d] copied out (keys != nil); expected a live lazy snapshot", i)+ }+ if rs.cache == nil {+ t.Fatalf("snaps[%d] has no issuing cache; expected a live lazy snapshot", i)}}- }()- // Interior captures (offsets after the start boundary) stay lazy: no write- // destroyed their slots, so keys is still nil and the issuing cache is live.- for i := 1; i < draft; i++ {- rs := snaps[i].(*rotatingSnapshot)- if rs.keys != nil {- t.Fatalf("snaps[%d] copied out (keys != nil); expected a live lazy snapshot", i)+ lazy := snaps[1].(*rotatingSnapshot)+ if lazy.Size() != 0 {+ t.Fatalf("lazy rotating snapshot Size = %d, want 0", lazy.Size())}- if rs.cache == nil {- t.Fatalf("snaps[%d] has no issuing cache; expected a live lazy snapshot", i)- }- }-- lazy := snaps[1].(*rotatingSnapshot)- if lazy.Size() != 0 {- t.Fatalf("lazy rotating snapshot Size = %d, want 0", lazy.Size())- }- var hookDelta int- lazy.SetMaterializeHook(func(delta int) { hookDelta = delta })+ var hookDelta int+ lazy.SetMaterializeHook(func(delta int) { hookDelta = delta })- // Free the cache to force every outstanding lazy snapshot to copy out.- c.Free()+ // Free the cache to force every outstanding lazy snapshot to copy out.+ c.Free()- if lazy.keys == nil {- t.Fatal("Free did not materialize the lazy snapshot")- }- want := lazy.keys.NumBytes() + lazy.values.NumBytes()- if hookDelta != want {- t.Fatalf("hook fired with delta %d, want %d", hookDelta, want)- }- if lazy.Size() != want {- t.Fatalf("materialized Size = %d, want %d", lazy.Size(), want)- }+ if lazy.keys == nil {+ t.Fatal("Free did not materialize the lazy snapshot")+ }+ want := lazy.keys.NumBytes() + lazy.values.NumBytes()+ if hookDelta != want {+ t.Fatalf("hook fired with delta %d, want %d", hookDelta, want)+ }+ if lazy.Size() != want {+ t.Fatalf("materialized Size = %d, want %d", lazy.Size(), want)+ }+ })}// TestPerTokenSnapshotPersistsAcrossWrites verifies that scheduled offsets// survive multiple writes until TakeSnapshots — the property prefill would rely// on to snapshot interior offsets without splitting its forward.func TestPerTokenSnapshotPersistsAcrossWrites(t *testing.T) {- skipIfNoMLX(t)-- c := NewKVCache()- fillKV(c, 2)+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewKVCache()+ fillKV(c, 2)- // Schedule offsets that span two separate writes. Offset 2 equals the- // schedule-time position, so it captures a zero-width range (nil); the rest- // are edge-local.- c.PrepareSnapshots([]int{2, 3, 5})+ // Schedule offsets that span two separate writes. Offset 2 equals the+ // schedule-time position, so it captures a zero-width range (nil); the rest+ // are edge-local.+ c.PrepareSnapshots([]int{2, 3, 5})- k1, v1 := batchKV(2) // reaches offsets 2,3- c.Update(newKVBatch(2, 2), k1, v1)- k2, v2 := batchKV(2) // reaches offsets 4,5- c.Update(newKVBatch(4, 2), k2, v2)+ k1, v1 := batchKV(2) // reaches offsets 2,3+ c.Update(newKVBatch(2, 2), k1, v1)+ k2, v2 := batchKV(2) // reaches offsets 4,5+ c.Update(newKVBatch(4, 2), k2, v2)- snaps := c.TakeSnapshots()- if len(snaps) != 3 {- t.Fatalf("got %d snapshots, want 3", len(snaps))- }- if snaps[0] != nil {- t.Fatalf("snaps[0] = %v, want nil (zero-width base)", snaps[0])- }- if s := snaps[1].(*kvSnapshot); s.fromOffset != 2 || s.toOffset != 3 {- t.Fatalf("snaps[1] = [%d,%d), want [2,3)", s.fromOffset, s.toOffset)- }- // Offset 5 was scheduled across the second write; its edge starts at the- // previous scheduled offset (3), confirming the base cursor only advances- // on capture so the snapshot range matches the trie edge between scheduled- // offsets — write boundaries between captures must not move it.- if s := snaps[2].(*kvSnapshot); s.fromOffset != 3 || s.toOffset != 5 {- t.Fatalf("snaps[2] = [%d,%d), want [3,5)", s.fromOffset, s.toOffset)- }+ snaps := c.TakeSnapshots()+ if len(snaps) != 3 {+ t.Fatalf("got %d snapshots, want 3", len(snaps))+ }+ if snaps[0] != nil {+ t.Fatalf("snaps[0] = %v, want nil (zero-width base)", snaps[0])+ }+ if s := snaps[1].(*kvSnapshot); s.fromOffset != 2 || s.toOffset != 3 {+ t.Fatalf("snaps[1] = [%d,%d), want [2,3)", s.fromOffset, s.toOffset)+ }+ // Offset 5 was scheduled across the second write; its edge starts at the+ // previous scheduled offset (3), confirming the base cursor only advances+ // on capture so the snapshot range matches the trie edge between scheduled+ // offsets — write boundaries between captures must not move it.+ if s := snaps[2].(*kvSnapshot); s.fromOffset != 3 || s.toOffset != 5 {+ t.Fatalf("snaps[2] = [%d,%d), want [3,5)", s.fromOffset, s.toOffset)+ }- // Restore from the [2,3) edge snapshot to offset 3.- if !c.Restore(snaps[1], 3) {- t.Fatal("restore to offset 3 failed")- }- if c.Offset() != 3 {- t.Fatalf("offset = %d, want 3", c.Offset())- }- for _, s := range snaps {- if s != nil {- s.Close()+ // Restore from the [2,3) edge snapshot to offset 3.+ if !c.Restore(snaps[1], 3) {+ t.Fatal("restore to offset 3 failed")}- }+ if c.Offset() != 3 {+ t.Fatalf("offset = %d, want 3", c.Offset())+ }+ for _, s := range snaps {+ if s != nil {+ s.Close()+ }+ }+ })}// TestRecurrentSnapshotSplitsAndSegmentedCapture verifies that SnapshotSplits// reports the interior scheduled offsets and PutSegmented captures them from the// per-boundary states, so each accepted count restores to a distinct state.func TestRecurrentSnapshotSplitsAndSegmentedCapture(t *testing.T) {- skipIfNoMLX(t)-- const convTail, convDim, nv, vd, kd = 3, 8, 2, 4, 4- c := NewRecurrentCache(convTail, convDim, nv, vd, kd)- c.Get(newKVBatch(0, 1), mlx.DTypeFloat16)- // Advance to offset 5 so the speculative forward starts there.- c.Put(newKVBatch(0, 5),- []*mlx.Array{mlx.Zeros(mlx.DTypeFloat16, 1, convTail, convDim)},- []*mlx.Array{mlx.Zeros(mlx.DTypeFloat32, 1, nv, vd, kd)})-- const before, draft = 5, 4- offsets := []int{before, before + 1, before + 2, before + 3}- c.PrepareSnapshots(offsets)-- splits := c.SnapshotSplits(draft)- want := []int{1, 2, 3}- if len(splits) != len(want) {- t.Fatalf("SnapshotSplits = %v, want %v", splits, want)- }- for i := range want {- if splits[i] != want[i] {+ mlxtest.Run(t, func(t *mlxtest.T) {+ const convTail, convDim, nv, vd, kd = 3, 8, 2, 4, 4+ c := NewRecurrentCache(convTail, convDim, nv, vd, kd)+ c.Get(newKVBatch(0, 1), mlx.DTypeFloat16)+ // Advance to offset 5 so the speculative forward starts there.+ c.Put(newKVBatch(0, 5),+ []*mlx.Array{mlx.Zeros(mlx.DTypeFloat16, 1, convTail, convDim)},+ []*mlx.Array{mlx.Zeros(mlx.DTypeFloat32, 1, nv, vd, kd)})++ const before, draft = 5, 4+ offsets := []int{before, before + 1, before + 2, before + 3}+ c.PrepareSnapshots(offsets)++ splits := c.SnapshotSplits(draft)+ want := []int{1, 2, 3}+ if len(splits) != len(want) {t.Fatalf("SnapshotSplits = %v, want %v", splits, want)}- }-- // Distinct per-boundary states (3 interior splits + the end) so restore- // targets are distinguishable.- mkConv := func(s float32) *mlx.Array { return mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat16, 1, convTail, convDim), s) }- mkDelta := func(s float32) *mlx.Array { return mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, nv, vd, kd), s) }- convStates := []*mlx.Array{mkConv(1), mkConv(2), mkConv(3), mkConv(4)}- deltaStates := []*mlx.Array{mkDelta(1), mkDelta(2), mkDelta(3), mkDelta(4)}+ for i := range want {+ if splits[i] != want[i] {+ t.Fatalf("SnapshotSplits = %v, want %v", splits, want)+ }+ }- c.Put(newKVBatch(before, draft), convStates, deltaStates)- if c.Offset() != before+draft {- t.Fatalf("offset after segmented put = %d, want %d", c.Offset(), before+draft)- }+ // Distinct per-boundary states (3 interior splits + the end) so restore+ // targets are distinguishable.+ mkConv := func(s float32) *mlx.Array { return mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat16, 1, convTail, convDim), s) }+ mkDelta := func(s float32) *mlx.Array { return mlx.AddScalar(mlx.Zeros(mlx.DTypeFloat32, 1, nv, vd, kd), s) }+ convStates := []*mlx.Array{mkConv(1), mkConv(2), mkConv(3), mkConv(4)}+ deltaStates := []*mlx.Array{mkDelta(1), mkDelta(2), mkDelta(3), mkDelta(4)}- snaps := c.TakeSnapshots()- if len(snaps) != draft {- t.Fatalf("got %d snapshots, want %d", len(snaps), draft)- }- for i, s := range snaps {- if s == nil {- t.Fatalf("snapshot %d not captured", i)+ c.Put(newKVBatch(before, draft), convStates, deltaStates)+ if c.Offset() != before+draft {+ t.Fatalf("offset after segmented put = %d, want %d", c.Offset(), before+draft)}- }- // Full accept (no restore): the live state is the committed end boundary- // (value 4) at offset before+draft.- st := c.State()- mlx.Eval(st[1])- if got := st[1].Floats()[0]; got != 4 {- t.Fatalf("full-accept delta state = %v, want end boundary value 4", got)- }-- // Each partial accept restores to offset before+accepted and must recover the- // distinct boundary state captured there: snaps[0] is the pre-forward state- // (value 0); snaps[i>=1] is the interior split boundary (value i). Recurrent- // snapshots are self-contained, so restores need not run in order.- for accepted := range draft {- if !c.Restore(snaps[accepted], before+accepted) {- t.Fatalf("accepted=%d: restore to before+%d failed", accepted, accepted)+ snaps := c.TakeSnapshots()+ if len(snaps) != draft {+ t.Fatalf("got %d snapshots, want %d", len(snaps), draft)}- if c.Offset() != before+accepted {- t.Fatalf("accepted=%d: offset after restore = %d, want %d", accepted, c.Offset(), before+accepted)+ for i, s := range snaps {+ if s == nil {+ t.Fatalf("snapshot %d not captured", i)+ }}++ // Full accept (no restore): the live state is the committed end boundary+ // (value 4) at offset before+draft.st := c.State()mlx.Eval(st[1])- if got := st[1].Floats()[0]; got != float32(accepted) {- t.Fatalf("accepted=%d: restored delta state = %v, want boundary value %d", accepted, got, accepted)+ if got := st[1].Floats()[0]; got != 4 {+ t.Fatalf("full-accept delta state = %v, want end boundary value 4", got)}- }- for _, s := range snaps {- s.Close()- }++ // Each partial accept restores to offset before+accepted and must recover the+ // distinct boundary state captured there: snaps[0] is the pre-forward state+ // (value 0); snaps[i>=1] is the interior split boundary (value i). Recurrent+ // snapshots are self-contained, so restores need not run in order.+ for accepted := range draft {+ if !c.Restore(snaps[accepted], before+accepted) {+ t.Fatalf("accepted=%d: restore to before+%d failed", accepted, accepted)+ }+ if c.Offset() != before+accepted {+ t.Fatalf("accepted=%d: offset after restore = %d, want %d", accepted, c.Offset(), before+accepted)+ }+ st := c.State()+ mlx.Eval(st[1])+ if got := st[1].Floats()[0]; got != float32(accepted) {+ t.Fatalf("accepted=%d: restored delta state = %v, want boundary value %d", accepted, got, accepted)+ }+ }+ for _, s := range snaps {+ s.Close()+ }+ })}// TestPrepareSnapshotsPastOffsetPanics verifies scheduling an already-passed offset// is rejected.func TestPrepareSnapshotsPastOffsetPanics(t *testing.T) {- skipIfNoMLX(t)-- c := NewKVCache()- fillKV(c, 5)+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := NewKVCache()+ fillKV(c, 5)- defer func() {- if recover() == nil {- t.Fatal("expected panic for already-passed offset")- }- }()- c.PrepareSnapshots([]int{3})+ defer func() {+ if recover() == nil {+ t.Fatal("expected panic for already-passed offset")+ }+ }()+ c.PrepareSnapshots([]int{3})+ })}diff --git a/x/mlxrunner/dflash_test.go b/x/mlxrunner/dflash_test.goindex f9c515809..3f4a89f1c 100644--- a/x/mlxrunner/dflash_test.go+++ b/x/mlxrunner/dflash_test.go@@ -6,6 +6,7 @@ import ("testing""github.com/ollama/ollama/api"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/cache""github.com/ollama/ollama/x/mlxrunner/mlx"@@ -92,7 +93,7 @@ var _ base.BlockDraft = (*fakeBlockDraft)(nil)// newBlockTestSession wires a runner around a fakeBlockDraft and opens one// request's drafting session, returning the concrete session for// internal-state assertions.-func newBlockTestSession(t *testing.T, predict map[int32]int32, blockSize int) (*Runner, *fakeBlockDraft, *dflashDraftSession, []cache.Cache) {+func newBlockTestSession(t *mlxtest.T, predict map[int32]int32, blockSize int) (*Runner, *fakeBlockDraft, *dflashDraftSession, []cache.Cache) {t.Helper()r := mtpTestRunner(t, predict, []int32{7}, sampler.Options{})caches, _ := newMTPTestCaches(2) // caches[0] target, caches[1] draft context@@ -108,236 +109,243 @@ func draftTokensOf(caches []cache.Cache) []int32 {}func TestDFlashCommittedBuffersPastFlushCap(t *testing.T) {- skipIfNoMLX(t)- _, draft, session, caches := newBlockTestSession(t, nil, 4)-- // One prefill-sized run at the flush cap writes through immediately in a- // single context-only Draft call.- n := dflashPendingFlushTokens- ids := make([]int32, n)- for i := range ids {- ids[i] = int32(i % mtpTestVocab)- }- session.committed(mlx.FromValues(ids, 1, n), oneHotLogits(ids), 0, nil)- if got := len(draft.calls); got != 1 {- t.Fatalf("draft calls after cap-sized run = %d, want 1", got)- }- if got := caches[1].Offset(); got != n {- t.Fatalf("draft cache offset = %d, want %d", got, n)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ _, draft, session, caches := newBlockTestSession(t, nil, 4)++ // One prefill-sized run at the flush cap writes through immediately in a+ // single context-only Draft call.+ n := dflashPendingFlushTokens+ ids := make([]int32, n)+ for i := range ids {+ ids[i] = int32(i % mtpTestVocab)+ }+ session.committed(mlx.FromValues(ids, 1, n), oneHotLogits(ids), 0, nil)+ if got := len(draft.calls); got != 1 {+ t.Fatalf("draft calls after cap-sized run = %d, want 1", got)+ }+ if got := caches[1].Offset(); got != n {+ t.Fatalf("draft cache offset = %d, want %d", got, n)+ }- // A run below the cap only buffers; settle writes it through, skipping- // the leading rows the flush already covered.- tail := []int32{1, 2, 3}- session.committed(mlx.FromValues(tail, 1, 3), oneHotLogits(tail), n-1, nil)- if got := len(draft.calls); got != 1 {- t.Fatalf("draft calls after buffered run = %d, want 1 (buffered)", got)- }- session.settle(nil)- want := blockCall{offset: int32(n), ctx: []int32{2, 3}}- if got := draft.calls[1]; got.offset != want.offset || !slices.Equal(got.ctx, want.ctx) || got.block != nil {- t.Fatalf("settle flush = %+v, want %+v", got, want)- }- if got := caches[1].Offset(); got != n+2 {- t.Fatalf("draft cache offset = %d, want %d (level with reports)", got, n+2)- }+ // A run below the cap only buffers; settle writes it through, skipping+ // the leading rows the flush already covered.+ tail := []int32{1, 2, 3}+ session.committed(mlx.FromValues(tail, 1, 3), oneHotLogits(tail), n-1, nil)+ if got := len(draft.calls); got != 1 {+ t.Fatalf("draft calls after buffered run = %d, want 1 (buffered)", got)+ }+ session.settle(nil)+ want := blockCall{offset: int32(n), ctx: []int32{2, 3}}+ if got := draft.calls[1]; got.offset != want.offset || !slices.Equal(got.ctx, want.ctx) || got.block != nil {+ t.Fatalf("settle flush = %+v, want %+v", got, want)+ }+ if got := caches[1].Offset(); got != n+2 {+ t.Fatalf("draft cache offset = %d, want %d (level with reports)", got, n+2)+ }+ })}func TestDFlashCommittedGapPanics(t *testing.T) {- skipIfNoMLX(t)- _, _, session, _ := newBlockTestSession(t, nil, 4)+ mlxtest.Run(t, func(t *mlxtest.T) {+ _, _, session, _ := newBlockTestSession(t, nil, 4)- session.committed(mlx.FromValues([]int32{1}, 1, 1), oneHotLogits([]int32{1}), 0, nil)- defer func() {- if recover() == nil {- t.Fatalf("committed run past the frontier did not panic")- }- }()- // The frontier is at slot 1; a run starting at 3 leaves slot 1..2 unfed.- session.committed(mlx.FromValues([]int32{4}, 1, 1), oneHotLogits([]int32{4}), 3, nil)+ session.committed(mlx.FromValues([]int32{1}, 1, 1), oneHotLogits([]int32{1}), 0, nil)+ defer func() {+ if recover() == nil {+ t.Fatalf("committed run past the frontier did not panic")+ }+ }()+ // The frontier is at slot 1; a run starting at 3 leaves slot 1..2 unfed.+ session.committed(mlx.FromValues([]int32{4}, 1, 1), oneHotLogits([]int32{4}), 3, nil)+ })}func TestDFlashRestoredPrefixResumes(t *testing.T) {- skipIfNoMLX(t)- r := mtpTestRunner(t, nil, []int32{7}, sampler.Options{})- caches, _ := newMTPTestCaches(2)- draft := &fakeBlockDraft{blockSize: 4, maskToken: 6, draftCaches: caches[1:]}- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])-- // A restored prefix arrives with the draft caches already written.- restored := []int32{1, 2, 3, 4, 5}- caches[1].(*fakeRewindableCache).feed(restored)- session := r.spec.drafter.open(nil).(*dflashDraftSession)- if session.ctxOffset != len(restored) {- t.Fatalf("ctxOffset = %d, want %d (synced to restored offset)", session.ctxOffset, len(restored))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ r := mtpTestRunner(t, nil, []int32{7}, sampler.Options{})+ caches, _ := newMTPTestCaches(2)+ draft := &fakeBlockDraft{blockSize: 4, maskToken: 6, draftCaches: caches[1:]}+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])++ // A restored prefix arrives with the draft caches already written.+ restored := []int32{1, 2, 3, 4, 5}+ caches[1].(*fakeRewindableCache).feed(restored)+ session := r.spec.drafter.open(nil).(*dflashDraftSession)+ if session.ctxOffset != len(restored) {+ t.Fatalf("ctxOffset = %d, want %d (synced to restored offset)", session.ctxOffset, len(restored))+ }- // The resumed prefill's run overlaps the restore point; only the rows- // past the frontier are buffered and written.- run := []int32{2, 3, 0, 1}- session.committed(mlx.FromValues(run, 1, 4), oneHotLogits(run), 3, nil)- session.settle(nil)- want := blockCall{offset: 5, ctx: []int32{0, 1}}- if got := draft.calls[0]; got.offset != want.offset || !slices.Equal(got.ctx, want.ctx) || got.block != nil {- t.Fatalf("resume flush = %+v, want %+v", got, want)- }- if got, wantTok := draftTokensOf(caches), append(restored, 0, 1); !slices.Equal(got, wantTok) {- t.Fatalf("draft cache = %v, want %v", got, wantTok)- }+ // The resumed prefill's run overlaps the restore point; only the rows+ // past the frontier are buffered and written.+ run := []int32{2, 3, 0, 1}+ session.committed(mlx.FromValues(run, 1, 4), oneHotLogits(run), 3, nil)+ session.settle(nil)+ want := blockCall{offset: 5, ctx: []int32{0, 1}}+ if got := draft.calls[0]; got.offset != want.offset || !slices.Equal(got.ctx, want.ctx) || got.block != nil {+ t.Fatalf("resume flush = %+v, want %+v", got, want)+ }+ if got, wantTok := draftTokensOf(caches), append(restored, 0, 1); !slices.Equal(got, wantTok) {+ t.Fatalf("draft cache = %v, want %v", got, wantTok)+ }+ })}func TestDFlashProposeBounds(t *testing.T) {- skipIfNoMLX(t)- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5}- _, draft, session, _ := newBlockTestSession(t, predict, 4)- current := mlx.FromValues([]int32{1}, 1)-- // Nothing committed yet: no context to draft from.- if session.propose(current, 4) != nil {- t.Fatalf("propose with no context did not decline")- }- session.committed(mlx.FromValues([]int32{1}, 1, 1), oneHotLogits([]int32{1}), 0, nil)- if session.propose(current, 0) != nil {- t.Fatalf("propose with no budget did not decline")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5}+ _, draft, session, _ := newBlockTestSession(t, predict, 4)+ current := mlx.FromValues([]int32{1}, 1)++ // Nothing committed yet: no context to draft from.+ if session.propose(current, 4) != nil {+ t.Fatalf("propose with no context did not decline")+ }+ session.committed(mlx.FromValues([]int32{1}, 1, 1), oneHotLogits([]int32{1}), 0, nil)+ if session.propose(current, 0) != nil {+ t.Fatalf("propose with no budget did not decline")+ }- // The block caps the draft at blockSize-1 mask rows regardless of budget.- cand := session.propose(current, 10)- if cand == nil {- t.Fatalf("propose declined with context and budget")- }- mlx.Eval(cand.tokens)- if got := cand.tokens.Ints(); !slices.Equal(got, []int32{2, 3, 4}) {- t.Fatalf("draft tokens = %v, want [2 3 4]", got)- }- if got, want := draft.calls[0].block, []int32{1, 6, 6, 6}; !slices.Equal(got, want) {- t.Fatalf("block = %v, want %v (anchor plus blockSize-1 masks)", got, want)- }+ // The block caps the draft at blockSize-1 mask rows regardless of budget.+ cand := session.propose(current, 10)+ if cand == nil {+ t.Fatalf("propose declined with context and budget")+ }+ mlx.Eval(cand.tokens)+ if got := cand.tokens.Ints(); !slices.Equal(got, []int32{2, 3, 4}) {+ t.Fatalf("draft tokens = %v, want [2 3 4]", got)+ }+ if got, want := draft.calls[0].block, []int32{1, 6, 6, 6}; !slices.Equal(got, want) {+ t.Fatalf("block = %v, want %v (anchor plus blockSize-1 masks)", got, want)+ }+ })}func TestDFlashBlockRewoundBeforeContextWrites(t *testing.T) {- skipIfNoMLX(t)- predict := map[int32]int32{2: 3, 3: 4, 4: 5}- _, draft, session, caches := newBlockTestSession(t, predict, 4)+ mlxtest.Run(t, func(t *mlxtest.T) {+ predict := map[int32]int32{2: 3, 3: 4, 4: 5}+ _, draft, session, caches := newBlockTestSession(t, predict, 4)- session.committed(mlx.FromValues([]int32{1}, 1, 1), oneHotLogits([]int32{1}), 0, nil)- if session.propose(mlx.FromValues([]int32{2}, 1), 3) == nil {- t.Fatalf("propose declined")- }- // The proposal's block sits in the caches until the next write.- if got, want := draftTokensOf(caches), []int32{1, 2, 6, 6, 6}; !slices.Equal(got, want) {- t.Fatalf("draft cache after propose = %v, want %v", got, want)- }+ session.committed(mlx.FromValues([]int32{1}, 1, 1), oneHotLogits([]int32{1}), 0, nil)+ if session.propose(mlx.FromValues([]int32{2}, 1), 3) == nil {+ t.Fatalf("propose declined")+ }+ // The proposal's block sits in the caches until the next write.+ if got, want := draftTokensOf(caches), []int32{1, 2, 6, 6, 6}; !slices.Equal(got, want) {+ t.Fatalf("draft cache after propose = %v, want %v", got, want)+ }- // The next round's report rewinds the block before appending context, so- // the accepted tokens' rows land at their true slots.- run := []int32{2, 3, 4}- session.committed(mlx.FromValues(run, 1, 3), oneHotLogits(run), 1, nil)- session.settle(nil)- if got, want := draftTokensOf(caches), []int32{1, 2, 3, 4}; !slices.Equal(got, want) {- t.Fatalf("draft cache after settle = %v, want %v (block rewound)", got, want)- }- if got := caches[1].Offset(); got != 4 {- t.Fatalf("draft cache offset = %d, want 4 (level with reports)", got)- }- want := blockCall{offset: 1, ctx: []int32{2, 3, 4}}- if got := draft.calls[1]; got.offset != want.offset || !slices.Equal(got.ctx, want.ctx) || got.block != nil {- t.Fatalf("context flush = %+v, want %+v", got, want)- }+ // The next round's report rewinds the block before appending context, so+ // the accepted tokens' rows land at their true slots.+ run := []int32{2, 3, 4}+ session.committed(mlx.FromValues(run, 1, 3), oneHotLogits(run), 1, nil)+ session.settle(nil)+ if got, want := draftTokensOf(caches), []int32{1, 2, 3, 4}; !slices.Equal(got, want) {+ t.Fatalf("draft cache after settle = %v, want %v (block rewound)", got, want)+ }+ if got := caches[1].Offset(); got != 4 {+ t.Fatalf("draft cache offset = %d, want 4 (level with reports)", got)+ }+ want := blockCall{offset: 1, ctx: []int32{2, 3, 4}}+ if got := draft.calls[1]; got.offset != want.offset || !slices.Equal(got.ctx, want.ctx) || got.block != nil {+ t.Fatalf("context flush = %+v, want %+v", got, want)+ }+ })}func TestDFlashCloseDrainsOutstandingBlock(t *testing.T) {- skipIfNoMLX(t)- predict := map[int32]int32{2: 3, 3: 4, 4: 5}- _, _, session, caches := newBlockTestSession(t, predict, 4)+ mlxtest.Run(t, func(t *mlxtest.T) {+ predict := map[int32]int32{2: 3, 3: 4, 4: 5}+ _, _, session, caches := newBlockTestSession(t, predict, 4)- session.committed(mlx.FromValues([]int32{1}, 1, 1), oneHotLogits([]int32{1}), 0, nil)- if session.propose(mlx.FromValues([]int32{2}, 1), 3) == nil {- t.Fatalf("propose declined")- }- // A session that ends with a proposal in flight still leaves the caches- // level: close rewinds the block even with nothing pending to flush.- session.close()- if got, want := draftTokensOf(caches), []int32{1}; !slices.Equal(got, want) {- t.Fatalf("draft cache after close = %v, want %v", got, want)- }+ session.committed(mlx.FromValues([]int32{1}, 1, 1), oneHotLogits([]int32{1}), 0, nil)+ if session.propose(mlx.FromValues([]int32{2}, 1), 3) == nil {+ t.Fatalf("propose declined")+ }+ // A session that ends with a proposal in flight still leaves the caches+ // level: close rewinds the block even with nothing pending to flush.+ session.close()+ if got, want := draftTokensOf(caches), []int32{1}; !slices.Equal(got, want) {+ t.Fatalf("draft cache after close = %v, want %v", got, want)+ }+ })}func TestDecodeBlockDraft(t *testing.T) {- skipIfNoMLX(t)- // The block draft mirrors the target chain, so one proposal round accepts- // every draft and the bonus token is the EOS.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- caches, _ := newMTPTestCaches(2)- draft := &fakeBlockDraft{predict: predict, blockSize: 3, maskToken: 6, draftCaches: caches[1:]}- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- session, ch := newMTPTestSession(caches)-- req := Request{- Responses: ch,- Tokens: []int32{1},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- if spec == nil || !spec.enabled {- t.Fatalf("open rejected a block-draft request")- }- pinDraftLimit(spec, 4)- d := spec.decoder(mlx.FromValues([]int32{1}, 1), 0, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()- spec.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // The block draft mirrors the target chain, so one proposal round accepts+ // every draft and the bonus token is the EOS.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ caches, _ := newMTPTestCaches(2)+ draft := &fakeBlockDraft{predict: predict, blockSize: 3, maskToken: 6, draftCaches: caches[1:]}+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ session, ch := newMTPTestSession(caches)++ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ if spec == nil || !spec.enabled {+ t.Fatalf("open rejected a block-draft request")+ }+ pinDraftLimit(spec, 4)+ d := spec.decoder(mlx.FromValues([]int32{1}, 1), 0, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()+ spec.close()- content, final := collectResponses(ch)- if content != "2345" {- t.Fatalf("content = %q, want %q", content, "2345")- }- if !final.Done || final.DoneReason != 0 {- t.Fatalf("final = %+v, want Done with EOS reason", final)- }- if want := []int32{2, 3, 4, 5, eos}; !slices.Equal(session.outputs, want) {- t.Fatalf("session outputs = %v, want %v", session.outputs, want)- }+ content, final := collectResponses(ch)+ if content != "2345" {+ t.Fatalf("content = %q, want %q", content, "2345")+ }+ if !final.Done || final.DoneReason != 0 {+ t.Fatalf("final = %+v, want Done with EOS reason", final)+ }+ if want := []int32{2, 3, 4, 5, eos}; !slices.Equal(session.outputs, want) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, want)+ }- // The unprimed drafter parks the first call, so two tokens decode as- // pipelined plain forwards; the resumed round then validates the current- // token and blockSize-1 drafts in one fused forward.- wantForwards := []forwardCall{{offset: 0, n: 1}, {offset: 1, n: 1}, {offset: 2, n: 3}}- model := r.Model.(*fakeMTPModel)- if !slices.Equal(model.forwards, wantForwards) {- t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)- }+ // The unprimed drafter parks the first call, so two tokens decode as+ // pipelined plain forwards; the resumed round then validates the current+ // token and blockSize-1 drafts in one fused forward.+ wantForwards := []forwardCall{{offset: 0, n: 1}, {offset: 1, n: 1}, {offset: 2, n: 3}}+ model := r.Model.(*fakeMTPModel)+ if !slices.Equal(model.forwards, wantForwards) {+ t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)+ }- // Ending the parked stretch settles the buffered context through, so the- // proposal runs block-only; close's flush then writes the accepted rows- // after rewinding the block.- wantCalls := []blockCall{- {offset: 0, ctx: []int32{2, 3}},- {offset: 2, block: []int32{3, 6, 6}},- {offset: 2, ctx: []int32{4, 5, 7}},- }- if len(draft.calls) != len(wantCalls) {- t.Fatalf("draft calls = %+v, want %+v", draft.calls, wantCalls)- }- for i, want := range wantCalls {- got := draft.calls[i]- if got.offset != want.offset || !slices.Equal(got.ctx, want.ctx) || !slices.Equal(got.block, want.block) {- t.Fatalf("draft call %d = %+v, want %+v", i, got, want)+ // Ending the parked stretch settles the buffered context through, so the+ // proposal runs block-only; close's flush then writes the accepted rows+ // after rewinding the block.+ wantCalls := []blockCall{+ {offset: 0, ctx: []int32{2, 3}},+ {offset: 2, block: []int32{3, 6, 6}},+ {offset: 2, ctx: []int32{4, 5, 7}},+ }+ if len(draft.calls) != len(wantCalls) {+ t.Fatalf("draft calls = %+v, want %+v", draft.calls, wantCalls)+ }+ for i, want := range wantCalls {+ got := draft.calls[i]+ if got.offset != want.offset || !slices.Equal(got.ctx, want.ctx) || !slices.Equal(got.block, want.block) {+ t.Fatalf("draft call %d = %+v, want %+v", i, got, want)+ }}- }- // The draft caches end level with the target, holding only context rows.- if got, want := caches[1].Offset(), caches[0].Offset(); got != want {- t.Fatalf("draft cache offset = %d, want %d (level with target)", got, want)- }- if toks := draftTokensOf(caches); slices.Contains(toks, 6) {- t.Fatalf("draft cache retains block rows: %v", toks)- }+ // The draft caches end level with the target, holding only context rows.+ if got, want := caches[1].Offset(), caches[0].Offset(); got != want {+ t.Fatalf("draft cache offset = %d, want %d (level with target)", got, want)+ }+ if toks := draftTokensOf(caches); slices.Contains(toks, 6) {+ t.Fatalf("draft cache retains block rows: %v", toks)+ }+ })}diff --git a/x/mlxrunner/grammar_mask_test.go b/x/mlxrunner/grammar_mask_test.goindex a87ca6387..638d38cba 100644--- a/x/mlxrunner/grammar_mask_test.go+++ b/x/mlxrunner/grammar_mask_test.go@@ -4,45 +4,47 @@ import ("math""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx")func TestApplyTokenMask(t *testing.T) {- skipIfNoMLX(t)- const (- bitsPerMaskWord = 32- firstTokenID = 0 // Least-significant bit of the first mask word.- interiorTokenID = 7 // Last bit of the first mask word's low byte.- lastIDInFirstMaskWord = bitsPerMaskWord - 1 // Sign bit of the int32-backed first mask word.- lastVocabID = 40 // Final valid ID in a partially used second mask word.- vocabSize = lastVocabID + 1- )- allowedIDs := []int{firstTokenID, interiorTokenID, lastIDInFirstMaskWord, lastVocabID}- word0 := uint32(1)<<firstTokenID |- uint32(1)<<interiorTokenID |- uint32(1)<<lastIDInFirstMaskWord- packed := []int32{- int32(word0),- int32(uint32(1) << (lastVocabID - bitsPerMaskWord)),- }- e := &grammarEngine{}- e.initMask(vocabSize)- logits := mlx.Zeros(mlx.DTypeFloat32, 1, vocabSize)- masked := e.apply(logits, mlx.FromValues(packed, 1, len(packed)))- mlx.Eval(masked)- got := masked.Floats()- for id := range vocabSize {- allowed := false- for _, a := range allowedIDs {- if id == a {- allowed = true- }- }- if allowed && got[id] != 0 {- t.Fatalf("allowed token %d masked to %v", id, got[id])+ mlxtest.Run(t, func(t *mlxtest.T) {+ const (+ bitsPerMaskWord = 32+ firstTokenID = 0 // Least-significant bit of the first mask word.+ interiorTokenID = 7 // Last bit of the first mask word's low byte.+ lastIDInFirstMaskWord = bitsPerMaskWord - 1 // Sign bit of the int32-backed first mask word.+ lastVocabID = 40 // Final valid ID in a partially used second mask word.+ vocabSize = lastVocabID + 1+ )+ allowedIDs := []int{firstTokenID, interiorTokenID, lastIDInFirstMaskWord, lastVocabID}+ word0 := uint32(1)<<firstTokenID |+ uint32(1)<<interiorTokenID |+ uint32(1)<<lastIDInFirstMaskWord+ packed := []int32{+ int32(word0),+ int32(uint32(1) << (lastVocabID - bitsPerMaskWord)),}- if !allowed && !math.IsInf(float64(got[id]), -1) {- t.Fatalf("disallowed token %d = %v, want -Inf", id, got[id])+ e := &grammarEngine{}+ e.initMask(vocabSize)+ logits := mlx.Zeros(mlx.DTypeFloat32, 1, vocabSize)+ masked := e.apply(logits, mlx.FromValues(packed, 1, len(packed)))+ mlx.Eval(masked)+ got := masked.Floats()+ for id := range vocabSize {+ allowed := false+ for _, a := range allowedIDs {+ if id == a {+ allowed = true+ }+ }+ if allowed && got[id] != 0 {+ t.Fatalf("allowed token %d masked to %v", id, got[id])+ }+ if !allowed && !math.IsInf(float64(got[id]), -1) {+ t.Fatalf("disallowed token %d = %v, want -Inf", id, got[id])+ }}- }+ })}diff --git a/x/mlxrunner/media_test.go b/x/mlxrunner/media_test.goindex 291e1ab9e..b1052a5dd 100644--- a/x/mlxrunner/media_test.go+++ b/x/mlxrunner/media_test.go@@ -4,6 +4,7 @@ import ("slices""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx""github.com/ollama/ollama/x/mlxrunner/model/base")@@ -72,52 +73,52 @@ func (m encodeCountingModel) EncodeMedia(item *base.PreparedItem, data *mlx.Arra}func TestBatchMediaLifecycle(t *testing.T) {- skipIfNoMLX(t)-- calls := 0- prepared := &base.PreparedItem{- Range: [2]int{2, 6},- MediaData: []float32{1, 2},- Dims: []int{2},- Opaque: 7,- }- r := &Runner{Model: encodeCountingModel{calls: &calls}}- request := Request{- Tokens: make([]int32, 8),- MediaItems: []mediaItem{{pos: 2, length: 4, item: prepared}},- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ calls := 0+ prepared := &base.PreparedItem{+ Range: [2]int{2, 6},+ MediaData: []float32{1, 2},+ Dims: []int{2},+ Opaque: 7,+ }+ r := &Runner{Model: encodeCountingModel{calls: &calls}}+ request := Request{+ Tokens: make([]int32, 8),+ MediaItems: []mediaItem{{pos: 2, length: 4, item: prepared}},+ }- m := r.openMedia(request)- if m == nil {- t.Fatal("openMedia returned nil for a media request")- }- if m.manifest[0].Pos != 2 || m.manifest[0].Opaque != 7 {- t.Fatalf("manifest = %+v", m.manifest[0])- }+ m := r.openMedia(request)+ if m == nil {+ t.Fatal("openMedia returned nil for a media request")+ }+ if m.manifest[0].Pos != 2 || m.manifest[0].Opaque != 7 {+ t.Fatalf("manifest = %+v", m.manifest[0])+ }- if items := m.batchMedia(0, 2); items[0].Features != nil || calls != 0 {- t.Fatal("non-overlapping chunk encoded features")- }- if items := m.batchMedia(0, 4); items[0].Features == nil || calls != 1 {- t.Fatalf("overlap did not encode once (calls=%d)", calls)- }- if items := m.batchMedia(4, 2); items[0].Features == nil || calls != 1 {- t.Fatalf("second overlap re-encoded (calls=%d)", calls)- }+ if items := m.batchMedia(0, 2); items[0].Features != nil || calls != 0 {+ t.Fatal("non-overlapping chunk encoded features")+ }+ if items := m.batchMedia(0, 4); items[0].Features == nil || calls != 1 {+ t.Fatalf("overlap did not encode once (calls=%d)", calls)+ }+ if items := m.batchMedia(4, 2); items[0].Features == nil || calls != 1 {+ t.Fatalf("second overlap re-encoded (calls=%d)", calls)+ }- m.release(4)- if m.manifest[0].Features == nil {- t.Fatal("release dropped features before the expansion was evaluated")- }- m.release(6)- if m.manifest[0].Features != nil {- t.Fatal("release kept features past the expansion end")- }- m.close()+ m.release(4)+ if m.manifest[0].Features == nil {+ t.Fatal("release dropped features before the expansion was evaluated")+ }+ m.release(6)+ if m.manifest[0].Features != nil {+ t.Fatal("release kept features past the expansion end")+ }+ m.close()- if r.openMedia(Request{Tokens: make([]int32, 8)}) != nil {- t.Fatal("openMedia returned non-nil for a text-only request")- }+ if r.openMedia(Request{Tokens: make([]int32, 8)}) != nil {+ t.Fatal("openMedia returned non-nil for a text-only request")+ }+ })}// Two prompts that differ only in their image diverge at the expansion'sdiff --git a/x/mlxrunner/mlx/act_test.go b/x/mlxrunner/mlx/act_test.goindex e3c39891f..cb330ced3 100644--- a/x/mlxrunner/mlx/act_test.go+++ b/x/mlxrunner/mlx/act_test.go@@ -4,7 +4,7 @@ import ("math""testing"- "github.com/ollama/ollama/x/internal/mlxthread"+ "github.com/ollama/ollama/x/internal/mlxthreadtest")func TestGELUCompiledMatchesEager(t *testing.T) {@@ -20,7 +20,7 @@ func TestGELUCompiledMatchesEager(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {EnableCompile()input := FromValues(values, len(values)).AsType(tt.dtype)Pin(input)@@ -53,34 +53,17 @@ func BenchmarkGELUCompiled(b *testing.B) {}func benchmarkGELU(b *testing.B, fn func(*Array) *Array) {- thread, err := mlxthread.Start("mlx-gelu-benchmark", func() error {- if err := CheckInit(); err != nil {- return err- }- if GPUIsAvailable() {- SetDefaultDeviceGPU()- }- EnableCompile()- return nil- })- if err != nil {- b.Skipf("MLX not available: %v", err)- }- defer func() {- if err := thread.Stop(b.Context(), func() {- Sweep()- ClearCache()- resetDefaultStreamCache()- }); err != nil {- b.Fatal(err)- }- }()-+ thread := mlxTestThread(b)if err := thread.Do(b.Context(), func() error {+ EnableCompile()input := AddScalar(Zeros(DTypeBFloat16, 1, 4096, 8192), 1)Eval(input)Pin(input)- defer Unpin(input)+ defer func() {+ Unpin(input)+ Sweep()+ ClearCache()+ }()warmup := fn(input)Eval(warmup)@@ -100,7 +83,7 @@ func benchmarkGELU(b *testing.B, fn func(*Array) *Array) {func TestReLUSquared(t *testing.T) {var got []float32- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {x := FromValues([]float32{-2, -0, 0.5, 2}, 4)Pin(x)defer Unpin(x)diff --git a/x/mlxrunner/mlx/array_test.go b/x/mlxrunner/mlx/array_test.goindex 7386f3335..39bbc7aaa 100644--- a/x/mlxrunner/mlx/array_test.go+++ b/x/mlxrunner/mlx/array_test.go@@ -1,9 +1,13 @@package mlx-import "testing"+import (+ "testing"++ "github.com/ollama/ollama/x/internal/mlxthreadtest"+)func TestFromValue(t *testing.T) {- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {for got, want := range map[*Array]DType{FromValue(true): DTypeBool,FromValue(false): DTypeBool,@@ -20,7 +24,7 @@ func TestFromValue(t *testing.T) {}func TestFromValues(t *testing.T) {- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {for got, want := range map[*Array]DType{FromValues([]bool{true, false, true}, 3): DTypeBool,FromValues([]uint8{1, 2, 3}, 3): DTypeUint8,@@ -43,36 +47,40 @@ func TestFromValues(t *testing.T) {}func TestComparisonOpsAndBernoulli(t *testing.T) {- withMLXThread(t, func() {- testComparisonOpsAndBernoulli(t)- })-}--func testComparisonOpsAndBernoulli(t *testing.T) {- a := FromValues([]float32{1, 2, 3}, 3)- b := FromValues([]float32{1, 1, 4}, 3)- eq := a.Equal(b).AsType(DTypeInt32)- gt := a.Greater(b).AsType(DTypeInt32)- le := a.LessEqual(b).AsType(DTypeInt32)- bern := Bernoulli(FromValues([]float32{1, 0}, 2)).AsType(DTypeInt32)- Eval(eq, gt, le, bern)-- for name, tc := range map[string]struct {+ var tests []struct {+ name stringgot []int32want []int32- }{- "equal": {eq.Ints(), []int32{1, 0, 0}},- "greater": {gt.Ints(), []int32{0, 1, 0}},- "lessEqual": {le.Ints(), []int32{1, 0, 1}},- "bernoulli": {bern.Ints(), []int32{1, 0}},- } {- t.Run(name, func(t *testing.T) {- if len(tc.got) != len(tc.want) {- t.Fatalf("got %v, want %v", tc.got, tc.want)+ }+ withMLXThread(t, func(*mlxthreadtest.T) {+ a := FromValues([]float32{1, 2, 3}, 3)+ b := FromValues([]float32{1, 1, 4}, 3)+ eq := a.Equal(b).AsType(DTypeInt32)+ gt := a.Greater(b).AsType(DTypeInt32)+ le := a.LessEqual(b).AsType(DTypeInt32)+ bern := Bernoulli(FromValues([]float32{1, 0}, 2)).AsType(DTypeInt32)+ Eval(eq, gt, le, bern)++ tests = []struct {+ name string+ got []int32+ want []int32+ }{+ {name: "equal", got: eq.Ints(), want: []int32{1, 0, 0}},+ {name: "greater", got: gt.Ints(), want: []int32{0, 1, 0}},+ {name: "lessEqual", got: le.Ints(), want: []int32{1, 0, 1}},+ {name: "bernoulli", got: bern.Ints(), want: []int32{1, 0}},+ }+ })++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if len(tt.got) != len(tt.want) {+ t.Fatalf("got %v, want %v", tt.got, tt.want)}- for i := range tc.want {- if tc.got[i] != tc.want[i] {- t.Fatalf("got %v, want %v", tc.got, tc.want)+ for i := range tt.want {+ if tt.got[i] != tt.want[i] {+ t.Fatalf("got %v, want %v", tt.got, tt.want)}}})diff --git a/x/mlxrunner/mlx/compile_test.go b/x/mlxrunner/mlx/compile_test.goindex 0a3316b9e..9978cb739 100644--- a/x/mlxrunner/mlx/compile_test.go+++ b/x/mlxrunner/mlx/compile_test.go@@ -2,15 +2,17 @@ package mlximport ("testing"++ "github.com/ollama/ollama/x/internal/mlxthreadtest")func TestCompileFusion(t *testing.T) {- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testCompileFusion(t)})}-func testCompileFusion(t *testing.T) {+func testCompileFusion(t *mlxthreadtest.T) {// Compile fuses the ops inside a function body into a single kernel,// eliminating intermediate buffers. Use a diamond-shaped graph where// two branches must be materialized simultaneously without fusion,@@ -67,12 +69,12 @@ func testCompileFusion(t *testing.T) {}func TestCompileNested(t *testing.T) {- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testCompileNested(t)})}-func testCompileNested(t *testing.T) {+func testCompileNested(t *mlxthreadtest.T) {// A compiled function that calls another compiled function should// produce correct results. The inner function inlines via isTracing()// during the outer's trace.@@ -103,12 +105,12 @@ func testCompileNested(t *testing.T) {}func TestCompileCallbackPanicRecovers(t *testing.T) {- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testCompileCallbackPanicRecovers(t)})}-func testCompileCallbackPanicRecovers(t *testing.T) {+func testCompileCallbackPanicRecovers(t *mlxthreadtest.T) {boom := Compile1("boom", func(a *Array) *Array {panic("intentional test panic")})@@ -130,12 +132,12 @@ func testCompileCallbackPanicRecovers(t *testing.T) {}func TestCompileNoTrackingGrowth(t *testing.T) {- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testCompileNoTrackingGrowth(t)})}-func testCompileNoTrackingGrowth(t *testing.T) {+func testCompileNoTrackingGrowth(t *mlxthreadtest.T) {// Repeated invocations of a compiled kernel should not grow the// tracked-arrays list; the callback's traceScratch collects// intermediates during tracing and frees them when the callback returns.diff --git a/x/mlxrunner/mlx/depthwise_conv_test.go b/x/mlxrunner/mlx/depthwise_conv_test.goindex bc409d1d9..57c74c50f 100644--- a/x/mlxrunner/mlx/depthwise_conv_test.go+++ b/x/mlxrunner/mlx/depthwise_conv_test.go@@ -3,21 +3,16 @@ package mlximport ("fmt""testing"++ "github.com/ollama/ollama/x/internal/mlxthreadtest")func TestDepthwiseConvSiLUMatchesGraph(t *testing.T) {- skipIfNoMLX(t)-- // Collected, not reported inside the callback: a t.Fatal there calls- // runtime.Goexit, which the MLX worker cannot recover, so the job's result- // is never delivered and the test hangs until the binary timeout.- var mismatches []string- withMLXThread(t, func() {- mismatches = depthwiseConvSiLUMismatches()+ withMLXThread(t, func(t *mlxthreadtest.T) {+ for _, mismatch := range depthwiseConvSiLUMismatches() {+ t.Error(mismatch)+ }})- for _, m := range mismatches {- t.Error(m)- }}func depthwiseConvSiLUMismatches() []string {diff --git a/x/mlxrunner/mlx/gated_delta_test.go b/x/mlxrunner/mlx/gated_delta_test.goindex 77def9f40..81a2034a6 100644--- a/x/mlxrunner/mlx/gated_delta_test.go+++ b/x/mlxrunner/mlx/gated_delta_test.go@@ -4,6 +4,8 @@ import ("fmt""math""testing"++ "github.com/ollama/ollama/x/internal/mlxthreadtest")type gatedDeltaTestGeometry struct {@@ -36,13 +38,12 @@ func gatedDeltaReference(in gatedDeltaTestInputs, captureAll bool) (y, nextState// metal::exp a float ulp apart, which after bf16 rounding leaves rare// differing inputs (beta sigmoid at -6.84375); the lattice here avoids them.func TestGatedDeltaMatchesGraph(t *testing.T) {- skipIfNoMLX(t)- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testGatedDeltaMatchesGraph(t)})}-func testGatedDeltaMatchesGraph(t *testing.T) {+func testGatedDeltaMatchesGraph(t *mlxthreadtest.T) {geometries := []gatedDeltaTestGeometry{{Hk: 16, Dk: 128, Hv: 32, Dv: 128},{Hk: 4, Dk: 64, Hv: 8, Dv: 32},@@ -76,15 +77,14 @@ func testGatedDeltaMatchesGraph(t *testing.T) {}func TestGatedDeltaGraphRouting(t *testing.T) {- skipIfNoMLX(t)- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testGatedDeltaGraphRouting(t)})}// Contract misses — T beyond the kernel cap and a float32 packed input —// run the same step as graph ops.-func testGatedDeltaGraphRouting(t *testing.T) {+func testGatedDeltaGraphRouting(t *mlxthreadtest.T) {g := gatedDeltaTestGeometry{Hk: 4, Dk: 64, Hv: 8, Dv: 32}check := func(name string, in gatedDeltaTestInputs, captureAll bool) {@@ -144,14 +144,13 @@ func scaledGatedDeltaRow(base gatedDeltaTestInputs, scale float32) gatedDeltaTes}func TestGatedDeltaBatchedRows(t *testing.T) {- skipIfNoMLX(t)- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testGatedDeltaBatchedRows(t)})}// Each batched row must match its own single-row launch bit-for-bit.-func testGatedDeltaBatchedRows(t *testing.T) {+func testGatedDeltaBatchedRows(t *mlxthreadtest.T) {g := gatedDeltaTestGeometry{Hk: 4, Dk: 64, Hv: 8, Dv: 32}for _, T := range []int{1, 5, 11} {rows := []gatedDeltaTestInputs{gatedDeltaTestInputs36(g, T)}@@ -181,8 +180,7 @@ func testGatedDeltaBatchedRows(t *testing.T) {}func TestGatedDeltaRaggedRows(t *testing.T) {- skipIfNoMLX(t)- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testGatedDeltaRaggedRows(t)})}@@ -191,7 +189,7 @@ func TestGatedDeltaRaggedRows(t *testing.T) {// ba tail poisoned to -inf — runs identity steps with zero output there: its// full output and final state must match a launch of only its real tokens// with zeros appended, while the full-length row is unaffected.-func testGatedDeltaRaggedRows(t *testing.T) {+func testGatedDeltaRaggedRows(t *mlxthreadtest.T) {g := gatedDeltaTestGeometry{Hk: 4, Dk: 64, Hv: 8, Dv: 32}const T, realLen = 6, 4row0 := gatedDeltaTestInputs36(g, T)diff --git a/x/mlxrunner/mlx/mamba_kernels_test.go b/x/mlxrunner/mlx/mamba_kernels_test.goindex 1b18245b4..67518f4af 100644--- a/x/mlxrunner/mlx/mamba_kernels_test.go+++ b/x/mlxrunner/mlx/mamba_kernels_test.go@@ -4,12 +4,14 @@ import ("fmt""math""testing"++ "github.com/ollama/ollama/x/internal/mlxthreadtest")func TestMamba2ScanMatchesReference(t *testing.T) {- requireMamba2Metal(t)- var failures []error- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {+ requireMamba2Metal(t)+ var failures []errorin := newMamba2TestInputs(1, 3, 2, 2, 2, 32)gotY, gotState, interior := Mamba2Scan(in.hidden, in.bState, in.cState, in.dt, in.state, in.a, in.d, in.dtBias, nil, false)if len(interior) != 0 {@@ -18,15 +20,15 @@ func TestMamba2ScanMatchesReference(t *testing.T) {wantY, wantState := mamba2ScanReference(in.hidden, in.bState, in.cState, in.dt, in.state, in.a, in.d, in.dtBias)failures = appendArrayCloseError(failures, "mamba2 scan y", gotY, wantY, 1e-5)failures = appendArrayCloseError(failures, "mamba2 scan state", gotState, wantState, 1e-5)+ reportMamba2Failures(t, failures)})- reportMamba2Failures(t, failures)}// Every interior state must match, not just one boundary.func TestMamba2ScanCaptureAllMatchesPerTokenReference(t *testing.T) {- requireMamba2Metal(t)- var failures []error- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {+ requireMamba2Metal(t)+ var failures []errorconst T = 4in := newMamba2TestInputs(1, T, 2, 2, 2, 32)gotY, gotEnd, gotInterior := Mamba2Scan(in.hidden, in.bState, in.cState, in.dt, in.state, in.a, in.d, in.dtBias, nil, true)@@ -50,14 +52,14 @@ func TestMamba2ScanCaptureAllMatchesPerTokenReference(t *testing.T) {}}failures = appendArrayCloseError(failures, "captureAll end state", gotEnd, state, 1e-5)+ reportMamba2Failures(t, failures)})- reportMamba2Failures(t, failures)}func TestMamba2ScanGroupedStatesMatchRepeatedReference(t *testing.T) {- requireMamba2Metal(t)- var failures []error- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {+ requireMamba2Metal(t)+ var failures []errorin := newMamba2TestInputs(1, 2, 4, 2, 2, 32)gotY, gotState, _ := Mamba2Scan(in.hidden, in.bState, in.cState, in.dt, in.state, in.a, in.d, in.dtBias, nil, false)wantY, wantState := mamba2ScanReference(@@ -68,15 +70,15 @@ func TestMamba2ScanGroupedStatesMatchRepeatedReference(t *testing.T) {)failures = appendArrayCloseError(failures, "grouped mamba2 y", gotY, wantY, 1e-5)failures = appendArrayCloseError(failures, "grouped mamba2 state", gotState, wantState, 1e-5)+ reportMamba2Failures(t, failures)})- reportMamba2Failures(t, failures)}// A shape outside the kernel's contract must still compute the right answer// through the graph implementation rather than fail.func TestMamba2ScanUnsupportedShapeMatchesGraph(t *testing.T) {- var failures []error- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {+ var failures []errorin := newMamba2TestInputs(1, 2, 2, 2, 2, 31)if _, ok := resolveMamba2ScanDims(in.hidden, in.bState, in.cState, in.dt, in.state, in.a, in.d, in.dtBias); ok {failures = append(failures, fmt.Errorf("resolveMamba2ScanDims ok=true for unsupported S=31 shape"))@@ -85,16 +87,16 @@ func TestMamba2ScanUnsupportedShapeMatchesGraph(t *testing.T) {wantY, wantState := mamba2ScanReference(in.hidden, in.bState, in.cState, in.dt, in.state, in.a, in.d, in.dtBias)failures = appendArrayCloseError(failures, "unsupported-shape y", gotY, wantY, 1e-5)failures = appendArrayCloseError(failures, "unsupported-shape state", gotState, wantState, 1e-5)+ reportMamba2Failures(t, failures)})- reportMamba2Failures(t, failures)}// A padded position must be an identity step. Without the mask it still decays// the state by exp(dt*a).func TestMamba2ScanPaddedRowIsIdentity(t *testing.T) {- requireMamba2Metal(t)- var failures []error- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {+ requireMamba2Metal(t)+ var failures []errorconst (B = 2L = 3@@ -127,8 +129,8 @@ func TestMamba2ScanPaddedRowIsIdentity(t *testing.T) {failures = append(failures, fmt.Errorf("padded output[%d] = %v, want 0", i, v))}}+ reportMamba2Failures(t, failures)})- reportMamba2Failures(t, failures)}type mamba2TestInputs struct {@@ -187,9 +189,8 @@ func onesTest(dtype DType, shape ...int) *Array {return AddScalar(Zeros(dtype, shape...), 1)}-func requireMamba2Metal(t *testing.T) {+func requireMamba2Metal(t *mlxthreadtest.T) {t.Helper()- skipIfNoMLX(t)if !MetalIsAvailable() {t.Skip("MLX Metal not available")}@@ -210,7 +211,7 @@ func appendArrayCloseError(failures []error, name string, got, want *Array, tolreturn failures}-func reportMamba2Failures(t *testing.T, failures []error) {+func reportMamba2Failures(t *mlxthreadtest.T, failures []error) {t.Helper()for _, err := range failures {t.Error(err)diff --git a/x/mlxrunner/mlx/memory_test.go b/x/mlxrunner/mlx/memory_test.goindex 8eceb848c..5e54c39cc 100644--- a/x/mlxrunner/mlx/memory_test.go+++ b/x/mlxrunner/mlx/memory_test.go@@ -6,21 +6,19 @@ import ("math""slices""testing"++ "github.com/ollama/ollama/x/internal/mlxthreadtest")func TestSetWiredLimitRejectsOversizeWithoutChangingLimit(t *testing.T) {- skipIfNoMLX(t)- if !GPUIsAvailable() {- t.Skip("MLX GPU not available")- }-- var testErr error- withMLXThread(t, func() {- testErr = checkWiredLimitRejectsOversize()+ withMLXThread(t, func(t *mlxthreadtest.T) {+ if !GPUIsAvailable() {+ t.Skip("MLX GPU not available")+ }+ if err := checkWiredLimitRejectsOversize(); err != nil {+ t.Fatal(err)+ }})- if testErr != nil {- t.Fatal(testErr)- }}func checkWiredLimitRejectsOversize() (err error) {diff --git a/x/mlxrunner/mlx/ops_extra_test.go b/x/mlxrunner/mlx/ops_extra_test.goindex 8138420d6..648354085 100644--- a/x/mlxrunner/mlx/ops_extra_test.go+++ b/x/mlxrunner/mlx/ops_extra_test.go@@ -3,14 +3,15 @@ package mlximport ("math""testing"++ "github.com/ollama/ollama/x/internal/mlxthreadtest")// fp4Values decodes an fp4 (E2M1) code to its value.var fp4Values = [16]float32{0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6}func TestDequantizeGlobalScale(t *testing.T) {- skipIfNoMLX(t)- withMLXThread(t, func() {+ withMLXThread(t, func(t *mlxthreadtest.T) {testDequantizeGlobalScale(t)})}@@ -18,7 +19,7 @@ func TestDequantizeGlobalScale(t *testing.T) {// The quantized payload is built directly, the way an nvfp4 checkpoint ships// it: packed fp4 codes, e4m3 group-scale bytes, and a separate global scale.// Only the dequantize consumer path runs, so expectations are exact.-func testDequantizeGlobalScale(t *testing.T) {+func testDequantizeGlobalScale(t *mlxthreadtest.T) {const rows, cols, group = 4, 64, 16// Every group cycles through all 16 codes; group g of row r has scale// 2^((r+g)%4-1), a power of two so every expected product is exact.diff --git a/x/mlxrunner/mlx/thread_test.go b/x/mlxrunner/mlx/thread_test.goindex 34d15ac42..cd3a5160d 100644--- a/x/mlxrunner/mlx/thread_test.go+++ b/x/mlxrunner/mlx/thread_test.go@@ -6,20 +6,11 @@ import ("sync""testing"- "github.com/ollama/ollama/x/internal/mlxthread"+ "github.com/ollama/ollama/x/internal/mlxthreadtest")-func skipIfNoMLX(t *testing.T) {- t.Helper()- if err := CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }-}--func startMLXThread(t *testing.T) *mlxthread.Thread {- t.Helper()-- thread, err := mlxthread.Start("mlx-test", func() error {+var testThread = sync.OnceValues(func() (*mlxthreadtest.Thread, error) {+ return mlxthreadtest.Start("mlx-test", func() error {if err := CheckInit(); err != nil {return err}@@ -28,42 +19,26 @@ func startMLXThread(t *testing.T) *mlxthread.Thread {}return nil})+})++func mlxTestThread(tb testing.TB) *mlxthreadtest.Thread {+ tb.Helper()++ thread, err := testThread()if err != nil {- t.Skipf("MLX not available: %v", err)+ tb.Skipf("MLX not available: %v", err)}return thread}-func stopMLXThread(t *testing.T, thread *mlxthread.Thread) {+func withMLXThread(t *testing.T, fn func(*mlxthreadtest.T)) {t.Helper()-- if err := thread.Stop(context.Background(), func() {- Sweep()- ClearCache()- resetDefaultStreamCache()- }); err != nil {- t.Fatal(err)- }-}--func withMLXThread(t *testing.T, fn func()) {- t.Helper()-- thread := startMLXThread(t)- defer stopMLXThread(t, thread)-- if err := thread.Do(context.Background(), func() error {- fn()- return nil- }); err != nil {- t.Fatal(err)- }+ mlxthreadtest.Run(t, mlxTestThread(t), fn)}func TestThreadedMLXOperations(t *testing.T) {- thread := startMLXThread(t)- defer stopMLXThread(t, thread)+ thread := mlxTestThread(t)oldProcs := runtime.GOMAXPROCS(8)defer runtime.GOMAXPROCS(oldProcs)diff --git a/x/mlxrunner/model/embedding_test.go b/x/mlxrunner/model/embedding_test.goindex 975e4f83b..7f57a10c4 100644--- a/x/mlxrunner/model/embedding_test.go+++ b/x/mlxrunner/model/embedding_test.go@@ -3,78 +3,72 @@ package modelimport ("testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx""github.com/ollama/ollama/x/models/nn")-func skipIfNoMLX(t *testing.T) {- t.Helper()- if err := mlx.CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }-}-func TestMakeEmbeddingLayerDense(t *testing.T) {- skipIfNoMLX(t)-- weight := mlx.FromValues([]float32{- 1, 2, 3, 4,- 5, 6, 7, 8,- }, 2, 4).AsType(mlx.DTypeBFloat16)-- emb := MakeEmbeddingLayer(map[string]*mlx.Array{- "model.embed_tokens.weight": weight,- }, "model.embed_tokens", 0, 0, "", nil)-- dense, ok := emb.(*nn.Embedding)- if !ok {- t.Fatalf("embedding type = %T, want *nn.Embedding", emb)- }- if dense.Weight.DType() != mlx.DTypeBFloat16 {- t.Fatalf("embedding dtype = %v, want %v", dense.Weight.DType(), mlx.DTypeBFloat16)- }- if _, ok := emb.AsLinear().(*nn.Linear); !ok {- t.Fatalf("AsLinear type = %T, want *nn.Linear", emb.AsLinear())- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ weight := mlx.FromValues([]float32{+ 1, 2, 3, 4,+ 5, 6, 7, 8,+ }, 2, 4).AsType(mlx.DTypeBFloat16)++ emb := MakeEmbeddingLayer(map[string]*mlx.Array{+ "model.embed_tokens.weight": weight,+ }, "model.embed_tokens", 0, 0, "", nil)++ dense, ok := emb.(*nn.Embedding)+ if !ok {+ t.Fatalf("embedding type = %T, want *nn.Embedding", emb)+ }+ if dense.Weight.DType() != mlx.DTypeBFloat16 {+ t.Fatalf("embedding dtype = %v, want %v", dense.Weight.DType(), mlx.DTypeBFloat16)+ }+ if _, ok := emb.AsLinear().(*nn.Linear); !ok {+ t.Fatalf("AsLinear type = %T, want *nn.Linear", emb.AsLinear())+ }+ })}func TestMakeEmbeddingLayerQuantized(t *testing.T) {- skipIfNoMLX(t)-- denseWeight := mlx.FromValues(func() []float32 {- out := make([]float32, 2*64)- for i := range out {- out[i] = float32(i%17) / 8+ mlxtest.Run(t, func(t *mlxtest.T) {+ denseWeight := mlx.FromValues(func() []float32 {+ out := make([]float32, 2*64)+ for i := range out {+ out[i] = float32(i%17) / 8+ }+ return out+ }(), 2, 64).AsType(mlx.DTypeBFloat16)++ qw, scales, qbiases := mlx.Quantize(denseWeight, 64, 4, "affine")+ mlx.Eval(qw, scales, qbiases)++ emb := MakeEmbeddingLayer(map[string]*mlx.Array{+ "model.embed_tokens.weight": qw,+ "model.embed_tokens.weight_scale": scales,+ "model.embed_tokens.weight_qbias": qbiases,+ }, "model.embed_tokens", 64, 4, "affine", nil)++ qemb, ok := emb.(*nn.QuantizedEmbedding)+ if !ok {+ t.Fatalf("embedding type = %T, want *nn.QuantizedEmbedding", emb)+ }+ if qemb.GroupSize != 64 || qemb.Bits != 4 || qemb.Mode != "affine" {+ t.Fatalf("quant params = (%d, %d, %q), want (64, 4, %q)", qemb.GroupSize, qemb.Bits, qemb.Mode, "affine")}- return out- }(), 2, 64).AsType(mlx.DTypeBFloat16)-- qw, scales, qbiases := mlx.Quantize(denseWeight, 64, 4, "affine")- mlx.Eval(qw, scales, qbiases)-- emb := MakeEmbeddingLayer(map[string]*mlx.Array{- "model.embed_tokens.weight": qw,- "model.embed_tokens.weight_scale": scales,- "model.embed_tokens.weight_qbias": qbiases,- }, "model.embed_tokens", 64, 4, "affine", nil)-- qemb, ok := emb.(*nn.QuantizedEmbedding)- if !ok {- t.Fatalf("embedding type = %T, want *nn.QuantizedEmbedding", emb)- }- if qemb.GroupSize != 64 || qemb.Bits != 4 || qemb.Mode != "affine" {- t.Fatalf("quant params = (%d, %d, %q), want (64, 4, %q)", qemb.GroupSize, qemb.Bits, qemb.Mode, "affine")- }- indices := mlx.FromValues([]int32{1, 0}, 2)- out := emb.Forward(indices)- mlx.Eval(out)- if dims := out.Dims(); len(dims) != 2 || dims[0] != 2 || dims[1] != 64 {- t.Fatalf("embedding output dims = %v, want [2 64]", dims)- }- if _, ok := emb.AsLinear().(*nn.QuantizedLinear); !ok {- t.Fatalf("AsLinear type = %T, want *nn.QuantizedLinear", emb.AsLinear())- }+ indices := mlx.FromValues([]int32{1, 0}, 2)+ out := emb.Forward(indices)+ mlx.Eval(out)+ if dims := out.Dims(); len(dims) != 2 || dims[0] != 2 || dims[1] != 64 {+ t.Fatalf("embedding output dims = %v, want [2 64]", dims)+ }+ if _, ok := emb.AsLinear().(*nn.QuantizedLinear); !ok {+ t.Fatalf("AsLinear type = %T, want *nn.QuantizedLinear", emb.AsLinear())+ }+ })}func TestMakeEmbeddingLayerQuantizedGlobalScale(t *testing.T) {diff --git a/x/mlxrunner/mtp_test.go b/x/mlxrunner/mtp_test.goindex 7f0c2a12a..d3106e275 100644--- a/x/mlxrunner/mtp_test.go+++ b/x/mlxrunner/mtp_test.go@@ -20,15 +20,6 @@ import ("github.com/ollama/ollama/x/tokenizer")-// skipIfNoMLX skips when MLX is unavailable and pins the test to its OS-// thread. The pin is load-bearing: MLX caches its default stream per thread,-// so a goroutine that migrates mid-run panics with "There is no Stream(gpu, 0)-// in current thread".-func skipIfNoMLX(t *testing.T) {- t.Helper()- mlxtest.Setup(t)-}-// The MTP fakes make hidden state and logits the same tensor (Forward returns// one-hot logits, Unembed is the identity), so tests fully script target and// draft predictions.@@ -191,10 +182,15 @@ func (d *fakeKVDraft) Unembed(x *mlx.Array) *mlx.Array { return x }var _ base.DraftModel = (*fakeKVDraft)(nil)+type fatalTester interface {+ Helper()+ Fatalf(string, ...any)+}+// newTestTokenizer builds a byte-level BPE tokenizer over single-character// tokens "0".."7" with the given EOS ids, so Decode(id) yields that digit and// IsEOS reports membership.-func newTestTokenizer(t *testing.T, eos []int32) *tokenizer.Tokenizer {+func newTestTokenizer(t fatalTester, eos []int32) *tokenizer.Tokenizer {t.Helper()vocab := make(map[string]int32, mtpTestVocab)for i := range mtpTestVocab {@@ -222,7 +218,7 @@ func newTestTokenizer(t *testing.T, eos []int32) *tokenizer.Tokenizer {// mtpTestRunner wires a Runner with the MTP fakes and a real sampler// registered with opts.-func mtpTestRunner(t *testing.T, predict map[int32]int32, eos []int32, opts sampler.Options) *Runner {+func mtpTestRunner(t *mlxtest.T, predict map[int32]int32, eos []int32, opts sampler.Options) *Runner {t.Helper()tok := newTestTokenizer(t, eos)r := &Runner{@@ -263,491 +259,501 @@ func resultIDs(results []sampler.Result) []int32 {}func TestAcceptMTPDraftsGreedyAcceptAll(t *testing.T) {- skipIfNoMLX(t)- // Target predicts 1->2->3->4 along the accepted chain; the draft proposed- // exactly that, so every draft token is accepted and the bonus token is the- // target's prediction after the last accepted token.- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5}- r := mtpTestRunner(t, predict, nil, sampler.Options{})-- caches, _ := newMTPTestCaches(1)- candidates := scriptedCandidates(r, []int32{2, 3, 4})-- position := caches[0].Offset()-- spec := testSpeculationSession(r, caches)- current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}- unpin := pinAcceptInputs(current, candidates)- defer unpin()- results, accepted, observed, err := spec.accept(&position, current, candidates)- if err != nil {- t.Fatalf("accept: %v", err)- }- if accepted != 3 || observed != 3 {- t.Fatalf("accepted = %d, observed = %d, want 3 and 3", accepted, observed)- }- // The run is the accepted drafts followed by the bonus token (5).- if got := resultIDs(results); !slices.Equal(got, []int32{2, 3, 4, 5}) {- t.Fatalf("results = %v, want [2 3 4 5]", got)- }- if position != 4 {- t.Fatalf("position = %d, want 4 (current + accepted)", position)- }- if got := caches[0].Offset(); got != 4 {- t.Fatalf("cache offset = %d, want 4 (current + all drafts kept)", got)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Target predicts 1->2->3->4 along the accepted chain; the draft proposed+ // exactly that, so every draft token is accepted and the bonus token is the+ // target's prediction after the last accepted token.+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5}+ r := mtpTestRunner(t, predict, nil, sampler.Options{})++ caches, _ := newMTPTestCaches(1)+ candidates := scriptedCandidates(r, []int32{2, 3, 4})++ position := caches[0].Offset()++ spec := testSpeculationSession(r, caches)+ current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}+ unpin := pinAcceptInputs(current, candidates)+ defer unpin()+ results, accepted, observed, err := spec.accept(&position, current, candidates)+ if err != nil {+ t.Fatalf("accept: %v", err)+ }+ if accepted != 3 || observed != 3 {+ t.Fatalf("accepted = %d, observed = %d, want 3 and 3", accepted, observed)+ }+ // The run is the accepted drafts followed by the bonus token (5).+ if got := resultIDs(results); !slices.Equal(got, []int32{2, 3, 4, 5}) {+ t.Fatalf("results = %v, want [2 3 4 5]", got)+ }+ if position != 4 {+ t.Fatalf("position = %d, want 4 (current + accepted)", position)+ }+ if got := caches[0].Offset(); got != 4 {+ t.Fatalf("cache offset = %d, want 4 (current + all drafts kept)", got)+ }+ })}func TestAcceptMTPDraftsGreedyMismatch(t *testing.T) {- skipIfNoMLX(t)- // Target predicts 1->2->9 but the draft proposed 2 then 7: the second draft- // token mismatches, so only the first is accepted and the bonus is the- // target's own prediction (3) at the rejection point.- predict := map[int32]int32{1: 2, 2: 3, 7: 0}- r := mtpTestRunner(t, predict, nil, sampler.Options{})-- caches, _ := newMTPTestCaches(1)- candidates := scriptedCandidates(r, []int32{2, 7})-- position := caches[0].Offset()-- spec := testSpeculationSession(r, caches)- current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}- unpin := pinAcceptInputs(current, candidates)- defer unpin()- results, accepted, observed, err := spec.accept(&position, current, candidates)- if err != nil {- t.Fatalf("accept: %v", err)- }- if accepted != 1 || observed != 2 {- t.Fatalf("accepted = %d, observed = %d, want 1 and 2 (a rejection judges the full round)", accepted, observed)- }- // The run is the one accepted draft (2) followed by the target's own- // prediction at the rejection point (3).- if got := resultIDs(results); !slices.Equal(got, []int32{2, 3}) {- t.Fatalf("results = %v, want [2 3]", got)- }- if position != 2 {- t.Fatalf("position = %d, want 2 (current + accepted)", position)- }- if got := caches[0].Offset(); got != 2 {- t.Fatalf("cache offset = %d, want 2 (rolled back to accepted)", got)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Target predicts 1->2->9 but the draft proposed 2 then 7: the second draft+ // token mismatches, so only the first is accepted and the bonus is the+ // target's own prediction (3) at the rejection point.+ predict := map[int32]int32{1: 2, 2: 3, 7: 0}+ r := mtpTestRunner(t, predict, nil, sampler.Options{})++ caches, _ := newMTPTestCaches(1)+ candidates := scriptedCandidates(r, []int32{2, 7})++ position := caches[0].Offset()++ spec := testSpeculationSession(r, caches)+ current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}+ unpin := pinAcceptInputs(current, candidates)+ defer unpin()+ results, accepted, observed, err := spec.accept(&position, current, candidates)+ if err != nil {+ t.Fatalf("accept: %v", err)+ }+ if accepted != 1 || observed != 2 {+ t.Fatalf("accepted = %d, observed = %d, want 1 and 2 (a rejection judges the full round)", accepted, observed)+ }+ // The run is the one accepted draft (2) followed by the target's own+ // prediction at the rejection point (3).+ if got := resultIDs(results); !slices.Equal(got, []int32{2, 3}) {+ t.Fatalf("results = %v, want [2 3]", got)+ }+ if position != 2 {+ t.Fatalf("position = %d, want 2 (current + accepted)", position)+ }+ if got := caches[0].Offset(); got != 2 {+ t.Fatalf("cache offset = %d, want 2 (rolled back to accepted)", got)+ }+ })}func TestAcceptMTPDraftsGreedyEOS(t *testing.T) {- skipIfNoMLX(t)- // The second accepted draft token is EOS: it is recorded but stops- // generation and no bonus token is produced. The EOS's own KV is rolled- // back so the caches rest one token behind the recorded outputs.- const eos int32 = 6- predict := map[int32]int32{1: 2, 2: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})-- caches, _ := newMTPTestCaches(1)- candidates := scriptedCandidates(r, []int32{2, eos})-- position := caches[0].Offset()-- spec := testSpeculationSession(r, caches)- current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}- unpin := pinAcceptInputs(current, candidates)- defer unpin()- results, accepted, observed, err := spec.accept(&position, current, candidates)- if err != nil {- t.Fatalf("accept: %v", err)- }- if accepted != 2 || observed != 2 {- t.Fatalf("accepted = %d, observed = %d, want 2 and 2 (token + EOS)", accepted, observed)- }- // The EOS ends generation, so the run is exactly the accepted tokens- // with no bonus appended.- if got := resultIDs(results); !slices.Equal(got, []int32{2, eos}) {- t.Fatalf("results = %v, want [2 %d]", got, eos)- }- if len(results) != accepted {- t.Fatalf("results has %d tokens, want exactly the %d accepted with no bonus after EOS", len(results), accepted)- }- if position != 2 {- t.Fatalf("position = %d, want 2 (current + kept draft, EOS dropped)", position)- }- if got := caches[0].Offset(); got != 2 {- t.Fatalf("cache offset = %d, want 2 (one behind the recorded outputs, EOS dropped)", got)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // The second accepted draft token is EOS: it is recorded but stops+ // generation and no bonus token is produced. The EOS's own KV is rolled+ // back so the caches rest one token behind the recorded outputs.+ const eos int32 = 6+ predict := map[int32]int32{1: 2, 2: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})++ caches, _ := newMTPTestCaches(1)+ candidates := scriptedCandidates(r, []int32{2, eos})++ position := caches[0].Offset()++ spec := testSpeculationSession(r, caches)+ current := sampler.Result{Token: mlx.FromValues([]int32{1}, 1)}+ unpin := pinAcceptInputs(current, candidates)+ defer unpin()+ results, accepted, observed, err := spec.accept(&position, current, candidates)+ if err != nil {+ t.Fatalf("accept: %v", err)+ }+ if accepted != 2 || observed != 2 {+ t.Fatalf("accepted = %d, observed = %d, want 2 and 2 (token + EOS)", accepted, observed)+ }+ // The EOS ends generation, so the run is exactly the accepted tokens+ // with no bonus appended.+ if got := resultIDs(results); !slices.Equal(got, []int32{2, eos}) {+ t.Fatalf("results = %v, want [2 %d]", got, eos)+ }+ if len(results) != accepted {+ t.Fatalf("results has %d tokens, want exactly the %d accepted with no bonus after EOS", len(results), accepted)+ }+ if position != 2 {+ t.Fatalf("position = %d, want 2 (current + kept draft, EOS dropped)", position)+ }+ if got := caches[0].Offset(); got != 2 {+ t.Fatalf("cache offset = %d, want 2 (one behind the recorded outputs, EOS dropped)", got)+ }+ })}func TestRunMTPDecodeGreedy(t *testing.T) {- skipIfNoMLX(t)- // The seed token 1 is the last prefill token; its prediction (2) is the- // first generated token. The decode then walks 2->3->4->EOS. The draft- // proposes the correct chain so steps are accepted in a single forward.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- // The draft mirrors the target chain, so every drafted token is accepted.- draft := &fakeMTPDraft{predict: predict}- caches, _ := newMTPTestCaches(1)- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- session, ch := newMTPTestSession(caches)- position := 1 // one prefill token already processed-- req := Request{- Responses: ch,- Tokens: []int32{0},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- d := testDecoder(t, r, req, caches, []int32{1}, position)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // The seed token 1 is the last prefill token; its prediction (2) is the+ // first generated token. The decode then walks 2->3->4->EOS. The draft+ // proposes the correct chain so steps are accepted in a single forward.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ // The draft mirrors the target chain, so every drafted token is accepted.+ draft := &fakeMTPDraft{predict: predict}+ caches, _ := newMTPTestCaches(1)+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ session, ch := newMTPTestSession(caches)+ position := 1 // one prefill token already processed++ req := Request{+ Responses: ch,+ Tokens: []int32{0},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ d := testDecoder(t, r, req, caches, []int32{1}, position)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()- content, final := collectResponses(ch)- if content != "234" {- t.Fatalf("content = %q, want %q", content, "234")- }- if !final.Done {- t.Fatalf("final response not marked Done")- }- if final.DoneReason != 0 {- t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)- }- if got := []int32{2, 3, 4, eos}; !slices.Equal(session.outputs, got) {- t.Fatalf("session outputs = %v, want %v", session.outputs, got)- }+ content, final := collectResponses(ch)+ if content != "234" {+ t.Fatalf("content = %q, want %q", content, "234")+ }+ if !final.Done {+ t.Fatalf("final response not marked Done")+ }+ if final.DoneReason != 0 {+ t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)+ }+ if got := []int32{2, 3, 4, eos}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }- // The unprimed drafter parks the first call, so the seed token and the- // stretch's in-flight successor decode as pipelined plain forwards; the- // resumed round then validates the current token and the 4 drafts in one- // fused forward at offset 3.- wantForwards := []forwardCall{{offset: 1, n: 1}, {offset: 2, n: 1}, {offset: 3, n: 5}}- model := r.Model.(*fakeMTPModel)- if !slices.Equal(model.forwards, wantForwards) {- t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)- }+ // The unprimed drafter parks the first call, so the seed token and the+ // stretch's in-flight successor decode as pipelined plain forwards; the+ // resumed round then validates the current token and the 4 drafts in one+ // fused forward at offset 3.+ wantForwards := []forwardCall{{offset: 1, n: 1}, {offset: 2, n: 1}, {offset: 3, n: 5}}+ model := r.Model.(*fakeMTPModel)+ if !slices.Equal(model.forwards, wantForwards) {+ t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)+ }- // Single-position drafting anchors every Draft call in a round at the- // last committed position (offset 2 — the current token at offset 3 is- // not yet validated), while the input token advances along the proposed- // chain.- wantDraft := []draftCall{{2, 3}, {2, 4}, {2, eos}, {2, 0}}- if !slices.Equal(draft.calls, wantDraft) {- t.Fatalf("draft calls = %v, want %v", draft.calls, wantDraft)- }+ // Single-position drafting anchors every Draft call in a round at the+ // last committed position (offset 2 — the current token at offset 3 is+ // not yet validated), while the input token advances along the proposed+ // chain.+ wantDraft := []draftCall{{2, 3}, {2, 4}, {2, eos}, {2, 0}}+ if !slices.Equal(draft.calls, wantDraft) {+ t.Fatalf("draft calls = %v, want %v", draft.calls, wantDraft)+ }+ })}func TestRunMTPDecodeSampled(t *testing.T) {- skipIfNoMLX(t)- // The same chain at temperature 1: because oneHotLogits uses a large gap,- // the proposal and target distributions are effectively point masses, so the- // rejection-sampling accept path that the sampled and greedy paths now share- // accepts the mirrored draft chain deterministically.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{Temperature: 1, Seed: 42, UseSeed: true})- caches, _ := newMTPTestCaches(1)- r.cache.caches = caches- r.spec = newSpeculation(r, &fakeMTPDraft{predict: predict}, caches, nil)- session, ch := newMTPTestSession(caches)- position := 1-- req := Request{- Responses: ch,- Tokens: []int32{0},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{Temperature: 1, Seed: 42, UseSeed: true},- }- spec := r.spec.open(req, nil)- if spec == nil || !spec.enabled {- t.Fatalf("open rejected a sampled request")- }- pinDraftLimit(spec, 4)- d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // The same chain at temperature 1: because oneHotLogits uses a large gap,+ // the proposal and target distributions are effectively point masses, so the+ // rejection-sampling accept path that the sampled and greedy paths now share+ // accepts the mirrored draft chain deterministically.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{Temperature: 1, Seed: 42, UseSeed: true})+ caches, _ := newMTPTestCaches(1)+ r.cache.caches = caches+ r.spec = newSpeculation(r, &fakeMTPDraft{predict: predict}, caches, nil)+ session, ch := newMTPTestSession(caches)+ position := 1++ req := Request{+ Responses: ch,+ Tokens: []int32{0},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{Temperature: 1, Seed: 42, UseSeed: true},+ }+ spec := r.spec.open(req, nil)+ if spec == nil || !spec.enabled {+ t.Fatalf("open rejected a sampled request")+ }+ pinDraftLimit(spec, 4)+ d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()- content, final := collectResponses(ch)- if content != "234" {- t.Fatalf("content = %q, want %q", content, "234")- }- if final.DoneReason != 0 {- t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)- }- if got := []int32{2, 3, 4, eos}; !slices.Equal(session.outputs, got) {- t.Fatalf("session outputs = %v, want %v", session.outputs, got)- }+ content, final := collectResponses(ch)+ if content != "234" {+ t.Fatalf("content = %q, want %q", content, "234")+ }+ if final.DoneReason != 0 {+ t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)+ }+ if got := []int32{2, 3, 4, eos}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }+ })}func TestRunMTPDecodeWarmDrafter(t *testing.T) {- skipIfNoMLX(t)- // A drafter warmed by a prefill report proposes in the very first round:- // the last prompt token seeds the decode loop and is forwarded fused- // with the drafts, so generation runs no plain forward at all.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- draft := &fakeMTPDraft{predict: predict}- caches, _ := newMTPTestCaches(1)- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- session, ch := newMTPTestSession(caches)- position := 1 // one prefill token already processed-- req := Request{- Responses: ch,- Tokens: []int32{0},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- pinDraftLimit(spec, 4)- // The prefill chunk's committed report: token 0 at slot 0 with its- // hidden row, leaving the drafter ready to propose from slot 1.- spec.committed(mlx.FromValues([]int32{0}, 1, 1), oneHotLogits([]int32{1}), 0, nil)-- d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // A drafter warmed by a prefill report proposes in the very first round:+ // the last prompt token seeds the decode loop and is forwarded fused+ // with the drafts, so generation runs no plain forward at all.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ draft := &fakeMTPDraft{predict: predict}+ caches, _ := newMTPTestCaches(1)+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ session, ch := newMTPTestSession(caches)+ position := 1 // one prefill token already processed++ req := Request{+ Responses: ch,+ Tokens: []int32{0},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ pinDraftLimit(spec, 4)+ // The prefill chunk's committed report: token 0 at slot 0 with its+ // hidden row, leaving the drafter ready to propose from slot 1.+ spec.committed(mlx.FromValues([]int32{0}, 1, 1), oneHotLogits([]int32{1}), 0, nil)++ d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()- content, final := collectResponses(ch)- if content != "234" {- t.Fatalf("content = %q, want %q", content, "234")- }- if final.DoneReason != 0 {- t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)- }- if got := []int32{2, 3, 4, eos}; !slices.Equal(session.outputs, got) {- t.Fatalf("session outputs = %v, want %v", session.outputs, got)- }+ content, final := collectResponses(ch)+ if content != "234" {+ t.Fatalf("content = %q, want %q", content, "234")+ }+ if final.DoneReason != 0 {+ t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)+ }+ if got := []int32{2, 3, 4, eos}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }- // One fused forward validates the seed token and all 4 proposals- // together at the seed token's slot.- wantForwards := []forwardCall{{offset: 1, n: 5}}- model := r.Model.(*fakeMTPModel)- if !slices.Equal(model.forwards, wantForwards) {- t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)- }+ // One fused forward validates the seed token and all 4 proposals+ // together at the seed token's slot.+ wantForwards := []forwardCall{{offset: 1, n: 5}}+ model := r.Model.(*fakeMTPModel)+ if !slices.Equal(model.forwards, wantForwards) {+ t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)+ }- // Warm drafting anchors at the last reported slot (0) and extends the- // seed token through the proposal chain.- wantDraft := []draftCall{{0, 1}, {0, 2}, {0, 3}, {0, 4}}- if !slices.Equal(draft.calls, wantDraft) {- t.Fatalf("draft calls = %v, want %v", draft.calls, wantDraft)- }+ // Warm drafting anchors at the last reported slot (0) and extends the+ // seed token through the proposal chain.+ wantDraft := []draftCall{{0, 1}, {0, 2}, {0, 3}, {0, 4}}+ if !slices.Equal(draft.calls, wantDraft) {+ t.Fatalf("draft calls = %v, want %v", draft.calls, wantDraft)+ }+ })}func TestRunMTPDecodeEOSCutLeavesPositionsUnjudged(t *testing.T) {- skipIfNoMLX(t)- // An accepted EOS inside the draft ends the round at a terminator, not a- // target rejection. The persisted acceptance model records outcomes only- // up to the EOS: the positions past it lie beyond where the round stopped,- // and folding them in as rejections would bias the depth controller- // shallow.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- caches, _ := newMTPTestCaches(1)- r.cache.caches = caches- r.spec = newSpeculation(r, &fakeMTPDraft{predict: predict}, caches, nil)- session, ch := newMTPTestSession(caches)- position := 1-- req := Request{- Responses: ch,- Tokens: []int32{0},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- if spec == nil || !spec.enabled {- t.Fatalf("want an enabled speculationSession with a depth controller, got %+v", spec)- }- // Force a four-token first round so the EOS (third draft) is interior;- // the controller still records the round's outcomes.- spec.limit = 4- spec.committed(mlx.FromValues([]int32{0}, 1, 1), oneHotLogits([]int32{1}), 0, nil)-- d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // An accepted EOS inside the draft ends the round at a terminator, not a+ // target rejection. The persisted acceptance model records outcomes only+ // up to the EOS: the positions past it lie beyond where the round stopped,+ // and folding them in as rejections would bias the depth controller+ // shallow.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ caches, _ := newMTPTestCaches(1)+ r.cache.caches = caches+ r.spec = newSpeculation(r, &fakeMTPDraft{predict: predict}, caches, nil)+ session, ch := newMTPTestSession(caches)+ position := 1++ req := Request{+ Responses: ch,+ Tokens: []int32{0},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ if spec == nil || !spec.enabled {+ t.Fatalf("want an enabled speculationSession with a depth controller, got %+v", spec)+ }+ // Force a four-token first round so the EOS (third draft) is interior;+ // the controller still records the round's outcomes.+ spec.limit = 4+ spec.committed(mlx.FromValues([]int32{0}, 1, 1), oneHotLogits([]int32{1}), 0, nil)++ d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()- content, _ := collectResponses(ch)- if content != "23" {- t.Fatalf("content = %q, want %q", content, "23")- }- if got := []int32{2, 3, eos}; !slices.Equal(session.outputs, got) {- t.Fatalf("session outputs = %v, want %v", session.outputs, got)- }+ content, _ := collectResponses(ch)+ if content != "23" {+ t.Fatalf("content = %q, want %q", content, "23")+ }+ if got := []int32{2, 3, eos}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }- // The round drafted {2, 3, eos, 0} and the target accepted through the- // EOS at position 3; the fourth draft was cut, not judged. Positions- // 1..3 record accepts and position 4 is never grown.- acc := r.spec.depth.acc- if len(acc.seen) != 4 {- t.Fatalf("acceptance model grew to %d positions, want 4 (the cut position stays unjudged)", len(acc.seen))- }- for i := 1; i <= 3; i++ {- if acc.seen[i] != 1 || acc.rate[i] != 1 {- t.Fatalf("position %d: seen = %d, rate = %v, want 1 and 1", i, acc.seen[i], acc.rate[i])+ // The round drafted {2, 3, eos, 0} and the target accepted through the+ // EOS at position 3; the fourth draft was cut, not judged. Positions+ // 1..3 record accepts and position 4 is never grown.+ acc := r.spec.depth.acc+ if len(acc.seen) != 4 {+ t.Fatalf("acceptance model grew to %d positions, want 4 (the cut position stays unjudged)", len(acc.seen))}- }+ for i := 1; i <= 3; i++ {+ if acc.seen[i] != 1 || acc.rate[i] != 1 {+ t.Fatalf("position %d: seen = %d, rate = %v, want 1 and 1", i, acc.seen[i], acc.rate[i])+ }+ }+ })}func TestDecodePlain(t *testing.T) {- skipIfNoMLX(t)- // The same chain with no speculationSession: decode's pipelined loop runs,- // dispatching the forward that produces the next token before the- // current one is emitted.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})-- caches, _ := newMTPTestCaches(1)- session, ch := newMTPTestSession(caches)- position := 1-- req := Request{- Responses: ch,- Tokens: []int32{0},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- d := testDecoder(t, r, req, caches, []int32{1}, position)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // The same chain with no speculationSession: decode's pipelined loop runs,+ // dispatching the forward that produces the next token before the+ // current one is emitted.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})++ caches, _ := newMTPTestCaches(1)+ session, ch := newMTPTestSession(caches)+ position := 1++ req := Request{+ Responses: ch,+ Tokens: []int32{0},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ d := testDecoder(t, r, req, caches, []int32{1}, position)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()- content, final := collectResponses(ch)- if content != "234" {- t.Fatalf("content = %q, want %q", content, "234")- }- if final.DoneReason != 0 || final.EvalCount != 3 {- t.Fatalf("final DoneReason = %d, EvalCount = %d, want 0 (EOS) and 3", final.DoneReason, final.EvalCount)- }- if got := []int32{2, 3, 4, eos, 0}; !slices.Equal(session.outputs, got) {- t.Fatalf("session outputs = %v, want %v", session.outputs, got)- }+ content, final := collectResponses(ch)+ if content != "234" {+ t.Fatalf("content = %q, want %q", content, "234")+ }+ if final.DoneReason != 0 || final.EvalCount != 3 {+ t.Fatalf("final DoneReason = %d, EvalCount = %d, want 0 (EOS) and 3", final.DoneReason, final.EvalCount)+ }+ if got := []int32{2, 3, 4, eos, 0}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }- // One forward per token: the seed at offset 1, then each sampled token- // in turn — including the ending EOS, whose forward is already in- // flight when the token is checked. Every forwarded token is recorded,- // and the EOS's sampled successor is recorded without being forwarded.- wantForwards := []forwardCall{{1, 1}, {2, 1}, {3, 1}, {4, 1}, {5, 1}}- model := r.Model.(*fakeMTPModel)- if !slices.Equal(model.forwards, wantForwards) {- t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)- }+ // One forward per token: the seed at offset 1, then each sampled token+ // in turn — including the ending EOS, whose forward is already in+ // flight when the token is checked. Every forwarded token is recorded,+ // and the EOS's sampled successor is recorded without being forwarded.+ wantForwards := []forwardCall{{1, 1}, {2, 1}, {3, 1}, {4, 1}, {5, 1}}+ model := r.Model.(*fakeMTPModel)+ if !slices.Equal(model.forwards, wantForwards) {+ t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)+ }+ })}func TestDecodeCancelledMidStream(t *testing.T) {- skipIfNoMLX(t)- // Cancelling while accepted drafts stream must leave the session- // consistent: every token committed to the caches is recorded in- // session.outputs, no speculation snapshot schedule is left pending on- // the shared caches, and every captured snapshot is closed.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- caches, tr := newMTPTestCaches(1)- r.cache.caches = caches- r.spec = newSpeculation(r, &fakeMTPDraft{predict: predict}, caches, nil)- session := &cacheSession{caches: caches}- ch := make(chan CompletionResponse) // unbuffered: every send must rendezvous-- ctx, cancel := context.WithCancel(context.Background())- go func() {- <-ch // the first streamed token- cancel()- // Stop reading: the next send blocks until the emit select sees- // the cancelled context.- }()-- position := 0- req := Request{- Responses: ch,- Tokens: []int32{1},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- d := testDecoder(t, r, req, caches, []int32{1}, position)- err := r.decode(ctx, req, session, d, 0)- d.close()- if !errors.Is(err, context.Canceled) {- t.Fatalf("decode error = %v, want context.Canceled", err)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Cancelling while accepted drafts stream must leave the session+ // consistent: every token committed to the caches is recorded in+ // session.outputs, no speculation snapshot schedule is left pending on+ // the shared caches, and every captured snapshot is closed.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ caches, tr := newMTPTestCaches(1)+ r.cache.caches = caches+ r.spec = newSpeculation(r, &fakeMTPDraft{predict: predict}, caches, nil)+ session := &cacheSession{caches: caches}+ ch := make(chan CompletionResponse) // unbuffered: every send must rendezvous++ ctx, cancel := context.WithCancel(context.Background())+ go func() {+ <-ch // the first streamed token+ cancel()+ // Stop reading: the next send blocks until the emit select sees+ // the cancelled context.+ }()++ position := 0+ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ d := testDecoder(t, r, req, caches, []int32{1}, position)+ err := r.decode(ctx, req, session, d, 0)+ d.close()+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("decode error = %v, want context.Canceled", err)+ }- // Every committed token is recorded, and the caches never run ahead of- // the record: the final recorded output is the round's next token,- // emitted before it is ever forwarded, so the caches rest one short.- rc := caches[0].(*fakeRewindableCache)- if want := 1 + len(session.outputs) - 1; rc.Offset() != want {- t.Fatalf("cache offset = %d, want %d (seed + recorded outputs %v minus the unforwarded final)", rc.Offset(), want, session.outputs)- }- if rc.pending.offsets != nil {- t.Fatalf("speculation snapshot schedule left pending: %v", rc.pending.offsets)- }- for i, s := range tr.all {- if s.closeCount == 0 {- t.Fatalf("snapshot #%d [%d,%d) leaked: never closed", i, s.from, s.to)+ // Every committed token is recorded, and the caches never run ahead of+ // the record: the final recorded output is the round's next token,+ // emitted before it is ever forwarded, so the caches rest one short.+ rc := caches[0].(*fakeRewindableCache)+ if want := 1 + len(session.outputs) - 1; rc.Offset() != want {+ t.Fatalf("cache offset = %d, want %d (seed + recorded outputs %v minus the unforwarded final)", rc.Offset(), want, session.outputs)}- }+ if rc.pending.offsets != nil {+ t.Fatalf("speculation snapshot schedule left pending: %v", rc.pending.offsets)+ }+ for i, s := range tr.all {+ if s.closeCount == 0 {+ t.Fatalf("snapshot #%d [%d,%d) leaked: never closed", i, s.from, s.to)+ }+ }+ })}func TestLayoutRidesEveryForward(t *testing.T) {- skipIfNoMLX(t)- // The request's opaque layout state must reach every forward: parked- // pipelined dispatches, the fused verification forward, and the draft- // model's own forwards alike.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- caches, _ := newMTPTestCaches(1)- r.cache.caches = caches- r.spec = newSpeculation(r, &fakeMTPDraft{predict: predict}, caches, nil)- session, ch := newMTPTestSession(caches)-- req := Request{- Responses: ch,- Tokens: []int32{0},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, []any{"layout"})- pinDraftLimit(spec, 4)- d := spec.decoder(mlx.FromValues([]int32{1}, 1), 1, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // The request's opaque layout state must reach every forward: parked+ // pipelined dispatches, the fused verification forward, and the draft+ // model's own forwards alike.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ caches, _ := newMTPTestCaches(1)+ r.cache.caches = caches+ r.spec = newSpeculation(r, &fakeMTPDraft{predict: predict}, caches, nil)+ session, ch := newMTPTestSession(caches)++ req := Request{+ Responses: ch,+ Tokens: []int32{0},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, []any{"layout"})+ pinDraftLimit(spec, 4)+ d := spec.decoder(mlx.FromValues([]int32{1}, 1), 1, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()- model := r.Model.(*fakeMTPModel)- if len(model.layouts) < 3 {- t.Fatalf("forwards recorded = %d, want parked dispatches and a fused round", len(model.layouts))- }- for i, l := range model.layouts {- if len(l) != 1 || l[0] != "layout" {- t.Fatalf("forward %d layout = %v", i, l)+ model := r.Model.(*fakeMTPModel)+ if len(model.layouts) < 3 {+ t.Fatalf("forwards recorded = %d, want parked dispatches and a fused round", len(model.layouts))+ }+ for i, l := range model.layouts {+ if len(l) != 1 || l[0] != "layout" {+ t.Fatalf("forward %d layout = %v", i, l)+ }}- }- draft := r.spec.draft.(*fakeMTPDraft)- if len(draft.layouts) == 0 {- t.Fatal("no draft forwards recorded")- }- for i, l := range draft.layouts {- if len(l) != 1 || l[0] != "layout" {- t.Fatalf("draft forward %d layout = %v", i, l)+ draft := r.spec.draft.(*fakeMTPDraft)+ if len(draft.layouts) == 0 {+ t.Fatal("no draft forwards recorded")}- }+ for i, l := range draft.layouts {+ if len(l) != 1 || l[0] != "layout" {+ t.Fatalf("draft forward %d layout = %v", i, l)+ }+ }+ })}// pinDraftLimit fixes an engine's draft length for the whole run: decode@@ -761,7 +767,7 @@ func pinDraftLimit(spec *speculationSession, limit int) {// testDecoder builds the decoder TextGenerationPipeline would construct for// this request, with the draft length pinned to a fixed width; tests close it// explicitly so close-time effects are visible to assertions.-func testDecoder(t *testing.T, r *Runner, req Request, caches []cache.Cache, seed []int32, position int) decoder {+func testDecoder(t *mlxtest.T, r *Runner, req Request, caches []cache.Cache, seed []int32, position int) decoder {if spec := r.spec.open(req, nil); spec != nil {if spec.enabled {pinDraftLimit(spec, 4)@@ -772,490 +778,498 @@ func testDecoder(t *testing.T, r *Runner, req Request, caches []cache.Cache, see}func TestDecodeKVDraft(t *testing.T) {- skipIfNoMLX(t)- // A draft with its own KV cache mirroring the target chain- // 1->2->3->4->5->6->EOS.- // The unprimed drafter parks the first call, whose pipelined tokens pair- // the prompt token and the first generated token; the first proposal- // comes from the catch-up's held logits (no draft call), speculative- // entries are written at advancing slots, and the post-accept rebuild- // rewrites the committed range from target hiddens.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- draft := &fakeKVDraft{predict: predict, draftCaches: nil}- caches, _ := newMTPTestCaches(2) // caches[0] target, caches[1] draft KV- draft.draftCaches = caches[1:]- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- session, ch := newMTPTestSession(caches)- position := 0-- req := Request{- Responses: ch,- Tokens: []int32{1},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- if spec == nil || !spec.enabled || len(spec.spec.targets) != 1 {- t.Fatalf("speculation engine not built around the draft caches")- }- pinDraftLimit(spec, 4)- defer spec.close()- d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // A draft with its own KV cache mirroring the target chain+ // 1->2->3->4->5->6->EOS.+ // The unprimed drafter parks the first call, whose pipelined tokens pair+ // the prompt token and the first generated token; the first proposal+ // comes from the catch-up's held logits (no draft call), speculative+ // entries are written at advancing slots, and the post-accept rebuild+ // rewrites the committed range from target hiddens.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict, draftCaches: nil}+ caches, _ := newMTPTestCaches(2) // caches[0] target, caches[1] draft KV+ draft.draftCaches = caches[1:]+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ session, ch := newMTPTestSession(caches)+ position := 0++ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ if spec == nil || !spec.enabled || len(spec.spec.targets) != 1 {+ t.Fatalf("speculation engine not built around the draft caches")+ }+ pinDraftLimit(spec, 4)+ defer spec.close()+ d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()- content, final := collectResponses(ch)- if content != "23456" {- t.Fatalf("content = %q, want %q", content, "23456")- }- if final.EvalCount != 5 || final.DoneReason != 0 {- t.Fatalf("EvalCount = %d, DoneReason = %d, want 5 and 0", final.EvalCount, final.DoneReason)- }- if got := []int32{2, 3, 4, 5, 6, eos}; !slices.Equal(session.outputs, got) {- t.Fatalf("session outputs = %v, want %v", session.outputs, got)- }+ content, final := collectResponses(ch)+ if content != "23456" {+ t.Fatalf("content = %q, want %q", content, "23456")+ }+ if final.EvalCount != 5 || final.DoneReason != 0 {+ t.Fatalf("EvalCount = %d, DoneReason = %d, want 5 and 0", final.EvalCount, final.DoneReason)+ }+ if got := []int32{2, 3, 4, 5, 6, eos}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }- // All caches rest together at the trie frontier: the prompt token plus- // five generated tokens; the EOS's KV is never committed.- if got := caches[0].Offset(); got != 6 {- t.Fatalf("target offset = %d, want 6", got)- }- if got := caches[1].Offset(); got != 6 {- t.Fatalf("draft cache offset = %d, want 6 (lockstep with target)", got)- }+ // All caches rest together at the trie frontier: the prompt token plus+ // five generated tokens; the EOS's KV is never committed.+ if got := caches[0].Offset(); got != 6 {+ t.Fatalf("target offset = %d, want 6", got)+ }+ if got := caches[1].Offset(); got != 6 {+ t.Fatalf("draft cache offset = %d, want 6 (lockstep with target)", got)+ }- // Every committed draft slot S fuses look-ahead token x_{S+1} with the- // target hidden at S (whose hot index equals the look-ahead id on this- // mirrored chain); speculative steps fuse the head's own projections- // (-1), seeded by the catch-up flush's held projection. The unprimed- // drafter parks the first call, so two pipelined tokens precede the- // round and their pairs flush together at the first proposal, which- // consumes the catch-up's held logits — the four-token draft makes only- // three head calls before the rebuild.- wantExtends := []extendCall{- {offset: 0, ids: []int32{2, 3}, hiddens: []int32{2, 3}}, // parked pairs flushed at the first proposal- {offset: 2, ids: []int32{4}, hiddens: []int32{-1}}, // speculative step 2 (held projection)- {offset: 3, ids: []int32{5}, hiddens: []int32{-1}}, // speculative step 3 (projection)- {offset: 4, ids: []int32{6}, hiddens: []int32{-1}}, // speculative step 4 (projection)- {offset: 2, ids: []int32{4, 5, 6, eos}, hiddens: []int32{4, 5, 6, eos}}, // committed pairs from the validated run + finish- }- if !reflect.DeepEqual(draft.extends, wantExtends) {- t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)- }+ // Every committed draft slot S fuses look-ahead token x_{S+1} with the+ // target hidden at S (whose hot index equals the look-ahead id on this+ // mirrored chain); speculative steps fuse the head's own projections+ // (-1), seeded by the catch-up flush's held projection. The unprimed+ // drafter parks the first call, so two pipelined tokens precede the+ // round and their pairs flush together at the first proposal, which+ // consumes the catch-up's held logits — the four-token draft makes only+ // three head calls before the rebuild.+ wantExtends := []extendCall{+ {offset: 0, ids: []int32{2, 3}, hiddens: []int32{2, 3}}, // parked pairs flushed at the first proposal+ {offset: 2, ids: []int32{4}, hiddens: []int32{-1}}, // speculative step 2 (held projection)+ {offset: 3, ids: []int32{5}, hiddens: []int32{-1}}, // speculative step 3 (projection)+ {offset: 4, ids: []int32{6}, hiddens: []int32{-1}}, // speculative step 4 (projection)+ {offset: 2, ids: []int32{4, 5, 6, eos}, hiddens: []int32{4, 5, 6, eos}}, // committed pairs from the validated run + finish+ }+ if !reflect.DeepEqual(draft.extends, wantExtends) {+ t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)+ }- // The committed draft KV holds the look-ahead tokens, one per slot.- if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, eos}; !slices.Equal(got, want) {- t.Fatalf("draft cache = %v, want %v", got, want)- }+ // The committed draft KV holds the look-ahead tokens, one per slot.+ if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, eos}; !slices.Equal(got, want) {+ t.Fatalf("draft cache = %v, want %v", got, want)+ }+ })}func TestDecodeKVDraftRejectionRebuildsFromTarget(t *testing.T) {- skipIfNoMLX(t)- // The draft mispredicts mid-chain: it proposes 6 where the target's own- // next token is 4, so the round is accepted only up to the rejection and the- // loop re-proposes from the target's correction. The speculative draft KV- // written for the rejected proposals is rolled back and rewritten from the- // validated run's target hiddens, so the committed draft cache holds the- // target chain with no trace of the rejected tokens.- const eos int32 = 7- target := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: eos, eos: 0}- r := mtpTestRunner(t, target, []int32{eos}, sampler.Options{})- // The draft mirrors the target except at 3, where it proposes 6 (absent from- // the target chain); once the target corrects 3->4 the next proposal- // re-aligns on the shared chain.- draft := &fakeKVDraft{predict: map[int32]int32{1: 2, 2: 3, 3: 6, 6: 0, 4: 5, 5: eos, eos: 0}, draftCaches: nil}- caches, _ := newMTPTestCaches(2) // caches[0] target, caches[1] draft KV- draft.draftCaches = caches[1:]- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- session, ch := newMTPTestSession(caches)- position := 0-- req := Request{- Responses: ch,- Tokens: []int32{1},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- pinDraftLimit(spec, 4)- defer spec.close()- d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // The draft mispredicts mid-chain: it proposes 6 where the target's own+ // next token is 4, so the round is accepted only up to the rejection and the+ // loop re-proposes from the target's correction. The speculative draft KV+ // written for the rejected proposals is rolled back and rewritten from the+ // validated run's target hiddens, so the committed draft cache holds the+ // target chain with no trace of the rejected tokens.+ const eos int32 = 7+ target := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: eos, eos: 0}+ r := mtpTestRunner(t, target, []int32{eos}, sampler.Options{})+ // The draft mirrors the target except at 3, where it proposes 6 (absent from+ // the target chain); once the target corrects 3->4 the next proposal+ // re-aligns on the shared chain.+ draft := &fakeKVDraft{predict: map[int32]int32{1: 2, 2: 3, 3: 6, 6: 0, 4: 5, 5: eos, eos: 0}, draftCaches: nil}+ caches, _ := newMTPTestCaches(2) // caches[0] target, caches[1] draft KV+ draft.draftCaches = caches[1:]+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ session, ch := newMTPTestSession(caches)+ position := 0++ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ pinDraftLimit(spec, 4)+ defer spec.close()+ d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()- content, final := collectResponses(ch)- if content != "2345" {- t.Fatalf("content = %q, want %q", content, "2345")- }- if final.DoneReason != 0 {- t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)- }- if got := []int32{2, 3, 4, 5, eos}; !slices.Equal(session.outputs, got) {- t.Fatalf("session outputs = %v, want %v", session.outputs, got)- }+ content, final := collectResponses(ch)+ if content != "2345" {+ t.Fatalf("content = %q, want %q", content, "2345")+ }+ if final.DoneReason != 0 {+ t.Fatalf("DoneReason = %d, want 0 (EOS)", final.DoneReason)+ }+ if got := []int32{2, 3, 4, 5, eos}; !slices.Equal(session.outputs, got) {+ t.Fatalf("session outputs = %v, want %v", session.outputs, got)+ }- // The divergent token was drafted into the speculative KV...- drafted := false- for _, e := range draft.extends {- if slices.Contains(e.ids, 6) {- drafted = true+ // The divergent token was drafted into the speculative KV...+ drafted := false+ for _, e := range draft.extends {+ if slices.Contains(e.ids, 6) {+ drafted = true+ }}- }- if !drafted {- t.Fatalf("draft never proposed the divergent token 6; extends = %+v", draft.extends)- }- // ...but the rejection rolled it back: both caches rest in lockstep at the- // target chain, and the committed draft KV holds only the validated- // look-ahead tokens.- if got := caches[0].Offset(); got != 5 {- t.Fatalf("target offset = %d, want 5", got)- }- if got := caches[1].Offset(); got != 5 {- t.Fatalf("draft cache offset = %d, want 5 (lockstep with target)", got)- }- if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, eos}; !slices.Equal(got, want) {- t.Fatalf("draft cache = %v, want %v (rejected proposals rolled back)", got, want)- }+ if !drafted {+ t.Fatalf("draft never proposed the divergent token 6; extends = %+v", draft.extends)+ }+ // ...but the rejection rolled it back: both caches rest in lockstep at the+ // target chain, and the committed draft KV holds only the validated+ // look-ahead tokens.+ if got := caches[0].Offset(); got != 5 {+ t.Fatalf("target offset = %d, want 5", got)+ }+ if got := caches[1].Offset(); got != 5 {+ t.Fatalf("draft cache offset = %d, want 5 (lockstep with target)", got)+ }+ if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, eos}; !slices.Equal(got, want) {+ t.Fatalf("draft cache = %v, want %v (rejected proposals rolled back)", got, want)+ }+ })}func TestDecodeMaintainsDraftCacheWithoutDrafting(t *testing.T) {- skipIfNoMLX(t)- // A request that cannot speculate (logprobs) on a model whose draft has- // its own KV cache still maintains it: the speculationSession permanently parks,- // so the inner pipelined decoder reports each forwarded token, the pairs- // wait in the pending list, and one batched extend at close — its final- // pair completed by the decoder's discarded in-flight sample — leaves the- // draft level with the target.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}- opts := sampler.Options{Logprobs: true}- r := mtpTestRunner(t, predict, []int32{eos}, opts)- draft := &fakeKVDraft{predict: predict, draftCaches: nil}- caches, _ := newMTPTestCaches(2)- draft.draftCaches = caches[1:]- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- session, ch := newMTPTestSession(caches)- position := 0-- req := Request{- Responses: ch,- Tokens: []int32{1},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: opts,- }- spec := r.spec.open(req, nil)- if spec == nil || spec.enabled {- t.Fatalf("want a permanent-park speculationSession, got %+v", spec)- }- d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()- spec.close() // the pipeline defers this before session close+ mlxtest.Run(t, func(t *mlxtest.T) {+ // A request that cannot speculate (logprobs) on a model whose draft has+ // its own KV cache still maintains it: the speculationSession permanently parks,+ // so the inner pipelined decoder reports each forwarded token, the pairs+ // wait in the pending list, and one batched extend at close — its final+ // pair completed by the decoder's discarded in-flight sample — leaves the+ // draft level with the target.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}+ opts := sampler.Options{Logprobs: true}+ r := mtpTestRunner(t, predict, []int32{eos}, opts)+ draft := &fakeKVDraft{predict: predict, draftCaches: nil}+ caches, _ := newMTPTestCaches(2)+ draft.draftCaches = caches[1:]+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ session, ch := newMTPTestSession(caches)+ position := 0++ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: opts,+ }+ spec := r.spec.open(req, nil)+ if spec == nil || spec.enabled {+ t.Fatalf("want a permanent-park speculationSession, got %+v", spec)+ }+ d := spec.decoder(mlx.FromValues([]int32{1}, 1), position, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()+ spec.close() // the pipeline defers this before session close- content, _ := collectResponses(ch)- if content != "23456" {- t.Fatalf("content = %q, want %q", content, "23456")- }+ content, _ := collectResponses(ch)+ if content != "23456" {+ t.Fatalf("content = %q, want %q", content, "23456")+ }- // The pipelined decoder forwards every emitted token, including the- // ending EOS, so both caches rest at the full recorded path.- if got := caches[0].Offset(); got != 7 {- t.Fatalf("target offset = %d, want 7", got)- }- if got := caches[1].Offset(); got != 7 {- t.Fatalf("draft cache offset = %d, want 7 (lockstep with target)", got)- }+ // The pipelined decoder forwards every emitted token, including the+ // ending EOS, so both caches rest at the full recorded path.+ if got := caches[0].Offset(); got != 7 {+ t.Fatalf("target offset = %d, want 7", got)+ }+ if got := caches[1].Offset(); got != 7 {+ t.Fatalf("draft cache offset = %d, want 7 (lockstep with target)", got)+ }- // A session that never drafts defers every committed pair: the whole- // generation arrives in one batched flush at finish, with the EOS slot's- // pair completed by the in-flight sample (predict[eos] = 0) that decoding- // discarded.- wantExtends := []extendCall{- {offset: 0, ids: []int32{2, 3, 4, 5, 6, eos, 0}, hiddens: []int32{2, 3, 4, 5, 6, eos, 0}},- }- if !reflect.DeepEqual(draft.extends, wantExtends) {- t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)- }- if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, eos, 0}; !slices.Equal(got, want) {- t.Fatalf("draft cache = %v, want %v", got, want)- }+ // A session that never drafts defers every committed pair: the whole+ // generation arrives in one batched flush at finish, with the EOS slot's+ // pair completed by the in-flight sample (predict[eos] = 0) that decoding+ // discarded.+ wantExtends := []extendCall{+ {offset: 0, ids: []int32{2, 3, 4, 5, 6, eos, 0}, hiddens: []int32{2, 3, 4, 5, 6, eos, 0}},+ }+ if !reflect.DeepEqual(draft.extends, wantExtends) {+ t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)+ }+ if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, eos, 0}; !slices.Equal(got, want) {+ t.Fatalf("draft cache = %v, want %v", got, want)+ }+ })}func TestSettleLevelsDraftCacheWithPrefill(t *testing.T) {- skipIfNoMLX(t)- // Prefill attaches its scheduled snapshots only at offsets every cache- // has crossed, so the pipeline settles the drafter with the seed first:- // the completed frontier pair brings the draft cache level with the- // target, in one batched extend with the buffered pairs.- const eos int32 = 7- predict := map[int32]int32{2: 3, 3: 4, 4: 5}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- draft := &fakeKVDraft{predict: predict, draftCaches: nil}- caches, _ := newMTPTestCaches(2)- draft.draftCaches = caches[1:]- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- req := Request{- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- defer spec.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Prefill attaches its scheduled snapshots only at offsets every cache+ // has crossed, so the pipeline settles the drafter with the seed first:+ // the completed frontier pair brings the draft cache level with the+ // target, in one batched extend with the buffered pairs.+ const eos int32 = 7+ predict := map[int32]int32{2: 3, 3: 4, 4: 5}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict, draftCaches: nil}+ caches, _ := newMTPTestCaches(2)+ draft.draftCaches = caches[1:]+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ req := Request{+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ defer spec.close()- // The prompt's only chunk: tokens 1..4 at slots 0..3 with their hiddens;- // token 5 is the seed.- spec.committed(mlx.FromValues([]int32{1, 2, 3, 4}, 1, 4), oneHotLogits([]int32{1, 2, 3, 4}), 0, nil)- spec.settle(mlx.FromValues([]int32{5}, 1))+ // The prompt's only chunk: tokens 1..4 at slots 0..3 with their hiddens;+ // token 5 is the seed.+ spec.committed(mlx.FromValues([]int32{1, 2, 3, 4}, 1, 4), oneHotLogits([]int32{1, 2, 3, 4}), 0, nil)+ spec.settle(mlx.FromValues([]int32{5}, 1))- if got := caches[1].Offset(); got != 4 {- t.Fatalf("draft cache offset after settle = %d, want 4 (level with target)", got)- }- wantExtends := []extendCall{- {offset: 0, ids: []int32{2, 3, 4, 5}, hiddens: []int32{1, 2, 3, 4}},- }- if !reflect.DeepEqual(draft.extends, wantExtends) {- t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)- }+ if got := caches[1].Offset(); got != 4 {+ t.Fatalf("draft cache offset after settle = %d, want 4 (level with target)", got)+ }+ wantExtends := []extendCall{+ {offset: 0, ids: []int32{2, 3, 4, 5}, hiddens: []int32{1, 2, 3, 4}},+ }+ if !reflect.DeepEqual(draft.extends, wantExtends) {+ t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)+ }+ })}func TestFlushMediaHeldUntilEmbedded(t *testing.T) {- skipIfNoMLX(t)- // The deferred flush embeds prompt tokens after prefill has released the- // media features, so the session holds delivered feature rows itself:- // each flush carries the held rows, a row is dropped once the flush's- // embed frontier passes its delivered end, and a row a later chunk- // redelivers is held again.- predict := map[int32]int32{}- r := mtpTestRunner(t, predict, []int32{7}, sampler.Options{})- draft := &fakeKVDraft{predict: predict}- caches, _ := newMTPTestCaches(2)- draft.draftCaches = caches[1:]- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])-- d := r.spec.drafter.open(nil).(*mtpDraftSession)- item := batch.MediaItem{Pos: 1, Features: mlx.Zeros(mlx.DTypeFloat32, 4, 4)}-- // Chunk [0, 4) delivers the item; the flush embeds through the delivered- // end, so the row rides that flush and is dropped.- d.committed(mlx.FromValues([]int32{1, 2, 3, 4}, 1, 4), oneHotLogits([]int32{1, 2, 3, 4}), 0, []batch.MediaItem{item})- d.flush()- // A chunk with no delivery flushes without media.- d.committed(mlx.FromValues([]int32{5, 6}, 1, 2), oneHotLogits([]int32{5, 6}), 4, nil)- d.flush()- // A redelivered row (one arrives per chunk it spans) is held and- // flushed again.- d.committed(mlx.FromValues([]int32{2, 3}, 1, 2), oneHotLogits([]int32{2, 3}), 6, []batch.MediaItem{item})- d.flush()- d.close()-- got := make([][]int, len(draft.extends))- for i, e := range draft.extends {- got[i] = e.media- }- if want := [][]int{{1}, nil, {1}}; !reflect.DeepEqual(got, want) {- t.Fatalf("flush media = %v, want %v", got, want)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // The deferred flush embeds prompt tokens after prefill has released the+ // media features, so the session holds delivered feature rows itself:+ // each flush carries the held rows, a row is dropped once the flush's+ // embed frontier passes its delivered end, and a row a later chunk+ // redelivers is held again.+ predict := map[int32]int32{}+ r := mtpTestRunner(t, predict, []int32{7}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict}+ caches, _ := newMTPTestCaches(2)+ draft.draftCaches = caches[1:]+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])++ d := r.spec.drafter.open(nil).(*mtpDraftSession)+ item := batch.MediaItem{Pos: 1, Features: mlx.Zeros(mlx.DTypeFloat32, 4, 4)}++ // Chunk [0, 4) delivers the item; the flush embeds through the delivered+ // end, so the row rides that flush and is dropped.+ d.committed(mlx.FromValues([]int32{1, 2, 3, 4}, 1, 4), oneHotLogits([]int32{1, 2, 3, 4}), 0, []batch.MediaItem{item})+ d.flush()+ // A chunk with no delivery flushes without media.+ d.committed(mlx.FromValues([]int32{5, 6}, 1, 2), oneHotLogits([]int32{5, 6}), 4, nil)+ d.flush()+ // A redelivered row (one arrives per chunk it spans) is held and+ // flushed again.+ d.committed(mlx.FromValues([]int32{2, 3}, 1, 2), oneHotLogits([]int32{2, 3}), 6, []batch.MediaItem{item})+ d.flush()+ d.close()++ got := make([][]int, len(draft.extends))+ for i, e := range draft.extends {+ got[i] = e.media+ }+ if want := [][]int{{1}, nil, {1}}; !reflect.DeepEqual(got, want) {+ t.Fatalf("flush media = %v, want %v", got, want)+ }+ })}func TestCommittedRunBatchesPastFlushCap(t *testing.T) {- skipIfNoMLX(t)- // A committed run longer than the pending-flush cap still writes the draft- // caches in a single head forward: the run's completed pairs coalesce into- // one batched extend at the run's start, rather than splitting at the cap.- const n = mtpPendingFlushTokens + 8- predict := map[int32]int32{}- tokens := make([]int32, n)- for i := range tokens {- tokens[i] = int32(i%7 + 1)- predict[tokens[i]] = int32((i+1)%7 + 1)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // A committed run longer than the pending-flush cap still writes the draft+ // caches in a single head forward: the run's completed pairs coalesce into+ // one batched extend at the run's start, rather than splitting at the cap.+ const n = mtpPendingFlushTokens + 8+ predict := map[int32]int32{}+ tokens := make([]int32, n)+ for i := range tokens {+ tokens[i] = int32(i%7 + 1)+ predict[tokens[i]] = int32((i+1)%7 + 1)+ }- r := mtpTestRunner(t, predict, []int32{0}, sampler.Options{})- draft := &fakeKVDraft{predict: predict, draftCaches: nil}- caches, _ := newMTPTestCaches(2)- draft.draftCaches = caches[1:]- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- req := Request{- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- defer spec.close()+ r := mtpTestRunner(t, predict, []int32{0}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict, draftCaches: nil}+ caches, _ := newMTPTestCaches(2)+ draft.draftCaches = caches[1:]+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ req := Request{+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ defer spec.close()- // One prefill-sized chunk: n tokens at slots 0..n-1 with their hiddens.- // The run crosses the flush cap, so the write happens inside committed.- spec.committed(mlx.FromValues(tokens, 1, n), oneHotLogits(tokens), 0, nil)+ // One prefill-sized chunk: n tokens at slots 0..n-1 with their hiddens.+ // The run crosses the flush cap, so the write happens inside committed.+ spec.committed(mlx.FromValues(tokens, 1, n), oneHotLogits(tokens), 0, nil)- if got := len(draft.extends); got != 1 {- t.Fatalf("draft extends = %d calls, want 1 batched extend", got)- }- if got, want := len(draft.extends[0].ids), n-1; got != want {- t.Fatalf("batched extend ids = %d, want %d (every completed pair)", got, want)- }- if got := draft.extends[0].offset; got != 0 {- t.Fatalf("batched extend offset = %d, want 0", got)- }- if got := caches[1].Offset(); got != n-1 {- t.Fatalf("draft cache offset = %d, want %d (all completed pairs)", got, n-1)- }+ if got := len(draft.extends); got != 1 {+ t.Fatalf("draft extends = %d calls, want 1 batched extend", got)+ }+ if got, want := len(draft.extends[0].ids), n-1; got != want {+ t.Fatalf("batched extend ids = %d, want %d (every completed pair)", got, want)+ }+ if got := draft.extends[0].offset; got != 0 {+ t.Fatalf("batched extend offset = %d, want 0", got)+ }+ if got := caches[1].Offset(); got != n-1 {+ t.Fatalf("draft cache offset = %d, want %d (all completed pairs)", got, n-1)+ }+ })}func TestRestoredPrefixRewritesBoundaryPair(t *testing.T) {- skipIfNoMLX(t)- // A finished generation levels the draft with the target, its boundary- // pair naming the never-committed EOS. The next request restores one- // token below the match (the draft look-ahead), so the re-evaluated- // boundary token's report rewrites that pair with the token that follows.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- draft := &fakeKVDraft{predict: predict, draftCaches: nil}- caches, _ := newMTPTestCaches(2)- draft.draftCaches = caches[1:]- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- session, ch := newMTPTestSession(caches)- req := Request{- Responses: ch,- Tokens: []int32{1},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- pinDraftLimit(spec, 4)- d := spec.decoder(mlx.FromValues([]int32{1}, 1), 0, nil)- if err := r.decode(context.Background(), req, session, d, 0); err != nil {- t.Fatalf("decode: %v", err)- }- d.close()- spec.close()+ mlxtest.Run(t, func(t *mlxtest.T) {+ // A finished generation levels the draft with the target, its boundary+ // pair naming the never-committed EOS. The next request restores one+ // token below the match (the draft look-ahead), so the re-evaluated+ // boundary token's report rewrites that pair with the token that follows.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict, draftCaches: nil}+ caches, _ := newMTPTestCaches(2)+ draft.draftCaches = caches[1:]+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ session, ch := newMTPTestSession(caches)+ req := Request{+ Responses: ch,+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ pinDraftLimit(spec, 4)+ d := spec.decoder(mlx.FromValues([]int32{1}, 1), 0, nil)+ if err := r.decode(context.Background(), req, session, d, 0); err != nil {+ t.Fatalf("decode: %v", err)+ }+ d.close()+ spec.close()- // The leveled rest: the draft's last pair names the EOS.- if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, eos}; !slices.Equal(got, want) {- t.Fatalf("draft cache = %v, want %v", got, want)- }+ // The leveled rest: the draft's last pair names the EOS.+ if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, eos}; !slices.Equal(got, want) {+ t.Fatalf("draft cache = %v, want %v", got, want)+ }- // The next request matches the six stored tokens and restores at 5, as- // begin does for the draft look-ahead; its prefill re-evaluates token 6,- // and the token that follows is 1, not the EOS.- for _, c := range caches {- if !c.Restore(nil, 5) {- t.Fatal("restore to 5 failed")+ // The next request matches the six stored tokens and restores at 5, as+ // begin does for the draft look-ahead; its prefill re-evaluates token 6,+ // and the token that follows is 1, not the EOS.+ for _, c := range caches {+ if !c.Restore(nil, 5) {+ t.Fatal("restore to 5 failed")+ }}- }- spec = r.spec.open(req, nil)- spec.committed(mlx.FromValues([]int32{6, 1}, 1, 2), oneHotLogits([]int32{eos, 2}), 5, nil)- spec.close()+ spec = r.spec.open(req, nil)+ spec.committed(mlx.FromValues([]int32{6, 1}, 1, 2), oneHotLogits([]int32{eos, 2}), 5, nil)+ spec.close()- last := draft.extends[len(draft.extends)-1]- if want := (extendCall{offset: 5, ids: []int32{1}, hiddens: []int32{eos}}); !reflect.DeepEqual(last, want) {- t.Fatalf("boundary rewrite = %+v, want %+v", last, want)- }- if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, 1}; !slices.Equal(got, want) {- t.Fatalf("draft cache = %v, want %v (stale EOS pair rewritten)", got, want)- }+ last := draft.extends[len(draft.extends)-1]+ if want := (extendCall{offset: 5, ids: []int32{1}, hiddens: []int32{eos}}); !reflect.DeepEqual(last, want) {+ t.Fatalf("boundary rewrite = %+v, want %+v", last, want)+ }+ if got, want := caches[1].(*fakeRewindableCache).tokens, []int32{2, 3, 4, 5, 6, 1}; !slices.Equal(got, want) {+ t.Fatalf("draft cache = %v, want %v (stale EOS pair rewritten)", got, want)+ }+ })}func TestDecodeParkedDraftResume(t *testing.T) {- skipIfNoMLX(t)- // Leaving a parked stretch: the inner decoder's in-flight sample is- // emitted without any new forward, becomes the round's current, and the- // next call drafts from it — with the draft pairs contiguous across the- // boundary.- const eos int32 = 7- predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}- r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})- draft := &fakeKVDraft{predict: predict, draftCaches: nil}- caches, _ := newMTPTestCaches(2)- draft.draftCaches = caches[1:]- r.cache.caches = caches- r.spec = newSpeculation(r, draft, caches[:1], caches[1:])- req := Request{- Tokens: []int32{1},- CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},- SamplerOpts: sampler.Options{},- }- spec := r.spec.open(req, nil)- if spec == nil || !spec.enabled {- t.Fatalf("want a drafting speculationSession, got %+v", spec)- }- pinDraftLimit(spec, 0)- d := spec.decoder(mlx.FromValues([]int32{1}, 1), 0, nil).(*speculativeDecoder)+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Leaving a parked stretch: the inner decoder's in-flight sample is+ // emitted without any new forward, becomes the round's current, and the+ // next call drafts from it — with the draft pairs contiguous across the+ // boundary.+ const eos int32 = 7+ predict := map[int32]int32{1: 2, 2: 3, 3: 4, 4: 5, 5: 6, 6: eos, eos: 0}+ r := mtpTestRunner(t, predict, []int32{eos}, sampler.Options{})+ draft := &fakeKVDraft{predict: predict, draftCaches: nil}+ caches, _ := newMTPTestCaches(2)+ draft.draftCaches = caches[1:]+ r.cache.caches = caches+ r.spec = newSpeculation(r, draft, caches[:1], caches[1:])+ req := Request{+ Tokens: []int32{1},+ CompletionRequest: CompletionRequest{Options: api.Options{NumPredict: 20}},+ SamplerOpts: sampler.Options{},+ }+ spec := r.spec.open(req, nil)+ if spec == nil || !spec.enabled {+ t.Fatalf("want a drafting speculationSession, got %+v", spec)+ }+ pinDraftLimit(spec, 0)+ d := spec.decoder(mlx.FromValues([]int32{1}, 1), 0, nil).(*speculativeDecoder)++ // Two parked calls arrive pipelined, one token each.+ for _, want := range []int32{2, 3} {+ results, err := d.next(20)+ if err != nil {+ t.Fatalf("parked next: %v", err)+ }+ if got := resultIDs(results); !slices.Equal(got, []int32{want}) {+ t.Fatalf("parked results = %v, want [%d]", got, want)+ }+ }- // Two parked calls arrive pipelined, one token each.- for _, want := range []int32{2, 3} {+ // The controller picks a positive depth: the next call hands back the+ // in-flight sample without dispatching another forward.+ model := r.Model.(*fakeMTPModel)+ forwards := len(model.forwards)+ spec.limit = 2results, err := d.next(20)if err != nil {- t.Fatalf("parked next: %v", err)+ t.Fatalf("resume next: %v", err)}- if got := resultIDs(results); !slices.Equal(got, []int32{want}) {- t.Fatalf("parked results = %v, want [%d]", got, want)+ if got := resultIDs(results); !slices.Equal(got, []int32{4}) {+ t.Fatalf("resume results = %v, want [4]", got)+ }+ if len(model.forwards) != forwards {+ t.Fatalf("resume dispatched a forward: %v", model.forwards[forwards:])}- }-- // The controller picks a positive depth: the next call hands back the- // in-flight sample without dispatching another forward.- model := r.Model.(*fakeMTPModel)- forwards := len(model.forwards)- spec.limit = 2- results, err := d.next(20)- if err != nil {- t.Fatalf("resume next: %v", err)- }- if got := resultIDs(results); !slices.Equal(got, []int32{4}) {- t.Fatalf("resume results = %v, want [4]", got)- }- if len(model.forwards) != forwards {- t.Fatalf("resume dispatched a forward: %v", model.forwards[forwards:])- }- // The following call runs a fused round drafting from the resumed- // current token.- results, err = d.next(20)- if err != nil {- t.Fatalf("round next: %v", err)- }- if got := resultIDs(results); !slices.Equal(got, []int32{5, 6, eos}) {- t.Fatalf("round results = %v, want [5 6 %d]", got, eos)- }- wantForwards := []forwardCall{{0, 1}, {1, 1}, {2, 1}, {3, 3}}- if !slices.Equal(model.forwards, wantForwards) {- t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)- }+ // The following call runs a fused round drafting from the resumed+ // current token.+ results, err = d.next(20)+ if err != nil {+ t.Fatalf("round next: %v", err)+ }+ if got := resultIDs(results); !slices.Equal(got, []int32{5, 6, eos}) {+ t.Fatalf("round results = %v, want [5 6 %d]", got, eos)+ }+ wantForwards := []forwardCall{{0, 1}, {1, 1}, {2, 1}, {3, 3}}+ if !slices.Equal(model.forwards, wantForwards) {+ t.Fatalf("target forwards = %v, want %v", model.forwards, wantForwards)+ }- // The stretch's reports, the resume pair, and the round's validated run- // extend the draft cache contiguously: the catch-up flush at the round's- // propose, its speculative entry, and the rebuild from target hiddens at- // close.- d.close()- spec.close()- wantExtends := []extendCall{- {offset: 0, ids: []int32{2, 3, 4}, hiddens: []int32{2, 3, 4}},- {offset: 3, ids: []int32{5}, hiddens: []int32{-1}},- {offset: 3, ids: []int32{5, 6, eos}, hiddens: []int32{5, 6, eos}},- }- if !reflect.DeepEqual(draft.extends, wantExtends) {- t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)- }- if got, want := caches[0].Offset(), 6; got != want {- t.Fatalf("target offset = %d, want %d (EOS never forwarded)", got, want)- }- if got, want := caches[1].Offset(), 6; got != want {- t.Fatalf("draft cache offset = %d, want %d (lockstep with target)", got, want)- }+ // The stretch's reports, the resume pair, and the round's validated run+ // extend the draft cache contiguously: the catch-up flush at the round's+ // propose, its speculative entry, and the rebuild from target hiddens at+ // close.+ d.close()+ spec.close()+ wantExtends := []extendCall{+ {offset: 0, ids: []int32{2, 3, 4}, hiddens: []int32{2, 3, 4}},+ {offset: 3, ids: []int32{5}, hiddens: []int32{-1}},+ {offset: 3, ids: []int32{5, 6, eos}, hiddens: []int32{5, 6, eos}},+ }+ if !reflect.DeepEqual(draft.extends, wantExtends) {+ t.Fatalf("draft extends = %+v, want %+v", draft.extends, wantExtends)+ }+ if got, want := caches[0].Offset(), 6; got != want {+ t.Fatalf("target offset = %d, want %d (EOS never forwarded)", got, want)+ }+ if got, want := caches[1].Offset(), 6; got != want {+ t.Fatalf("draft cache offset = %d, want %d (lockstep with target)", got, want)+ }+ })}// newMTPTestCaches returns n rewindable fake caches sharing one snapshotdiff --git a/x/mlxrunner/sample/logprob_test.go b/x/mlxrunner/sample/logprob_test.goindex d29550663..2afb6b0f2 100644--- a/x/mlxrunner/sample/logprob_test.go+++ b/x/mlxrunner/sample/logprob_test.go@@ -1,5 +1,3 @@-//go:build mlx-package sampleimport (@@ -7,6 +5,7 @@ import ("sort""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx")@@ -21,21 +20,21 @@ type logprobEntry struct {// and returns the greedily-sampled token id, its logprob, and the top-K// entries sorted descending by logprob. Logits must be a [vocab]-shaped// slice; the helper reshapes it to [1, vocab] before calling the sampler.-func runSampleLogprobs(t *testing.T, logits []float32, topK int) (int32, float64, []logprobEntry) {+func runSampleLogprobs(t *mlxtest.T, logits []float32, topK int) (int32, float64, []logprobEntry) {t.Helper()s := New(128)- defer func() {+ t.Cleanup(func() {s.Free()mlx.Sweep()- }()+ })s.Add(0, Options{Logprobs: true, TopLogprobs: topK}, nil)tensor := mlx.FromValues(logits, 1, len(logits))res := s.Sample([]int{0}, tensor)mlx.Pin(res.Arrays()...)- defer mlx.Unpin(res.Arrays()...)+ t.Cleanup(func() { mlx.Unpin(res.Arrays()...) })mlx.Sweep()mlx.Eval(res.Arrays()...)@@ -56,7 +55,7 @@ func runSampleLogprobs(t *testing.T, logits []float32, topK int) (int32, float64}func TestSampleLogprobsBasic(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)tests := []struct {name string@@ -82,7 +81,7 @@ func TestSampleLogprobsBasic(t *testing.T) {}for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tt.name, func(t *mlxtest.T) {selected, _, top := runSampleLogprobs(t, tt.logits, tt.topK)if selected != tt.wantSelectedID {t.Errorf("selected = %d, want %d", selected, tt.wantSelectedID)@@ -95,28 +94,28 @@ func TestSampleLogprobsBasic(t *testing.T) {}func TestSampleLogprobsNumericalStability(t *testing.T) {- skipIfNoMLX(t)-- logits := []float32{1000.0, 999.0, 998.0}- _, selLP, top := runSampleLogprobs(t, logits, 3)+ mlxtest.Run(t, func(t *mlxtest.T) {+ logits := []float32{1000.0, 999.0, 998.0}+ _, selLP, top := runSampleLogprobs(t, logits, 3)- if math.IsInf(selLP, 0) || math.IsNaN(selLP) {- t.Errorf("selected logprob is not finite: %f", selLP)- }- for i, e := range top {- if math.IsInf(e.logprob, 0) || math.IsNaN(e.logprob) {- t.Errorf("top[%d] logprob is not finite: %f", i, e.logprob)+ if math.IsInf(selLP, 0) || math.IsNaN(selLP) {+ t.Errorf("selected logprob is not finite: %f", selLP)}- }- for i := 1; i < len(top); i++ {- if top[i].logprob > top[i-1].logprob {- t.Errorf("top logprobs not descending: %f > %f", top[i].logprob, top[i-1].logprob)+ for i, e := range top {+ if math.IsInf(e.logprob, 0) || math.IsNaN(e.logprob) {+ t.Errorf("top[%d] logprob is not finite: %f", i, e.logprob)+ }}- }+ for i := 1; i < len(top); i++ {+ if top[i].logprob > top[i-1].logprob {+ t.Errorf("top logprobs not descending: %f > %f", top[i].logprob, top[i-1].logprob)+ }+ }+ })}func TestSampleLogprobsProbabilityCorrectness(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)tests := []struct {name string@@ -129,7 +128,7 @@ func TestSampleLogprobsProbabilityCorrectness(t *testing.T) {}for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tt.name, func(t *mlxtest.T) {selected, selLP, top := runSampleLogprobs(t, tt.logits, len(tt.logits))if selLP > 0 {@@ -174,7 +173,7 @@ func TestSampleLogprobsProbabilityCorrectness(t *testing.T) {}func TestSampleLogprobsSoftmaxCorrectness(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)tests := []struct {name string@@ -188,7 +187,7 @@ func TestSampleLogprobsSoftmaxCorrectness(t *testing.T) {}for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tt.name, func(t *mlxtest.T) {_, _, top := runSampleLogprobs(t, tt.logits, len(tt.logits))if len(top) != len(tt.logits) {t.Fatalf("top-K length = %d, want %d", len(top), len(tt.logits))@@ -211,90 +210,90 @@ func TestSampleLogprobsSoftmaxCorrectness(t *testing.T) {}func TestSampleLogprobsSelectedTokenCorrectness(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ logits := []float32{3.0, 1.0, 2.0, 0.5}- logits := []float32{3.0, 1.0, 2.0, 0.5}-- maxIdx := int32(0)- for i, v := range logits[1:] {- if v > logits[maxIdx] {- maxIdx = int32(i + 1)+ maxIdx := int32(0)+ for i, v := range logits[1:] {+ if v > logits[maxIdx] {+ maxIdx = int32(i + 1)+ }}- }- selected, selLP, top := runSampleLogprobs(t, logits, len(logits))+ selected, selLP, top := runSampleLogprobs(t, logits, len(logits))- if selected != maxIdx {- t.Errorf("selected = %d, want argmax %d", selected, maxIdx)- }+ if selected != maxIdx {+ t.Errorf("selected = %d, want argmax %d", selected, maxIdx)+ }- if top[0].id != maxIdx {- t.Errorf("top[0].id = %d, want argmax %d", top[0].id, maxIdx)- }- if math.Abs(top[0].logprob-selLP) > 1e-6 {- t.Errorf("top[0].logprob = %f, want selected %f", top[0].logprob, selLP)- }+ if top[0].id != maxIdx {+ t.Errorf("top[0].id = %d, want argmax %d", top[0].id, maxIdx)+ }+ if math.Abs(top[0].logprob-selLP) > 1e-6 {+ t.Errorf("top[0].logprob = %f, want selected %f", top[0].logprob, selLP)+ }+ })}// TestBatchedLogprobsPerRow verifies that per-row logprobs in a batched// sample call match the per-slot reference. The numerically-stable softmax// must reduce along the last axis only, not over the whole batch.func TestBatchedLogprobsPerRow(t *testing.T) {- skipIfNoMLX(t)-- rowA := []float32{2, 1, 0}- rowB := []float32{0, 5, 0}+ mlxtest.Run(t, func(t *mlxtest.T) {+ rowA := []float32{2, 1, 0}+ rowB := []float32{0, 5, 0}- _, wantA, _ := runSampleLogprobs(t, rowA, 0)- _, wantB, _ := runSampleLogprobs(t, rowB, 0)+ _, wantA, _ := runSampleLogprobs(t, rowA, 0)+ _, wantB, _ := runSampleLogprobs(t, rowB, 0)- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })+ s.Add(1, Options{Logprobs: true}, nil)+ s.Add(2, Options{Logprobs: true}, nil)++ logits := mlx.FromValues(append(append([]float32{}, rowA...), rowB...), 2, 3)+ res := s.Sample([]int{1, 2}, logits)+ mlx.Pin(res.Arrays()...)+ t.Cleanup(func() { mlx.Unpin(res.Arrays()...) })+ mlx.Eval(res.Arrays()...)++ got := res.Logprob.Floats()+ if len(got) != 2 {+ t.Fatalf("Logprob length = %d, want 2", len(got))+ }+ if math.Abs(float64(got[0])-wantA) > 1e-5 {+ t.Errorf("row 0 logprob = %f, want %f (per-slot reference)", got[0], wantA)+ }+ if math.Abs(float64(got[1])-wantB) > 1e-5 {+ t.Errorf("row 1 logprob = %f, want %f (per-slot reference)", got[1], wantB)+ }})- s.Add(1, Options{Logprobs: true}, nil)- s.Add(2, Options{Logprobs: true}, nil)-- logits := mlx.FromValues(append(append([]float32{}, rowA...), rowB...), 2, 3)- res := s.Sample([]int{1, 2}, logits)- mlx.Pin(res.Arrays()...)- t.Cleanup(func() { mlx.Unpin(res.Arrays()...) })- mlx.Eval(res.Arrays()...)-- got := res.Logprob.Floats()- if len(got) != 2 {- t.Fatalf("Logprob length = %d, want 2", len(got))- }- if math.Abs(float64(got[0])-wantA) > 1e-5 {- t.Errorf("row 0 logprob = %f, want %f (per-slot reference)", got[0], wantA)- }- if math.Abs(float64(got[1])-wantB) > 1e-5 {- t.Errorf("row 1 logprob = %f, want %f (per-slot reference)", got[1], wantB)- }}func TestSampleLogprobsTopKOrdering(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Logits chosen so argmax order differs from index order.+ logits := []float32{2.0, 5.0, 1.0, 4.0, 3.0}+ wantOrder := []int32{1, 3, 4, 0, 2}- // Logits chosen so argmax order differs from index order.- logits := []float32{2.0, 5.0, 1.0, 4.0, 3.0}- wantOrder := []int32{1, 3, 4, 0, 2}+ _, _, top := runSampleLogprobs(t, logits, len(logits))- _, _, top := runSampleLogprobs(t, logits, len(logits))-- if len(top) != len(wantOrder) {- t.Fatalf("top-K length = %d, want %d", len(top), len(wantOrder))- }- for i, e := range top {- if e.id != wantOrder[i] {- t.Errorf("top[%d].id = %d, want %d", i, e.id, wantOrder[i])+ if len(top) != len(wantOrder) {+ t.Fatalf("top-K length = %d, want %d", len(top), len(wantOrder))}- }- for i := 1; i < len(top); i++ {- if top[i].logprob > top[i-1].logprob {- t.Errorf("top[%d].logprob (%f) > top[%d].logprob (%f)",- i, top[i].logprob, i-1, top[i-1].logprob)+ for i, e := range top {+ if e.id != wantOrder[i] {+ t.Errorf("top[%d].id = %d, want %d", i, e.id, wantOrder[i])+ }}- }+ for i := 1; i < len(top); i++ {+ if top[i].logprob > top[i-1].logprob {+ t.Errorf("top[%d].logprob (%f) > top[%d].logprob (%f)",+ i, top[i].logprob, i-1, top[i-1].logprob)+ }+ }+ })}diff --git a/x/mlxrunner/sample/sample_test.go b/x/mlxrunner/sample/sample_test.goindex 8d59c5a18..eb9446250 100644--- a/x/mlxrunner/sample/sample_test.go+++ b/x/mlxrunner/sample/sample_test.go@@ -1,5 +1,3 @@-//go:build mlx-package sampleimport (@@ -7,16 +5,10 @@ import ("slices""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx")-func skipIfNoMLX(t *testing.T) {- t.Helper()- if err := mlx.CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }-}-// slotLogits builds a [1, V] logits tensor for a single-slot Sample call.func slotLogits(values []float32) *mlx.Array {return mlx.FromValues(values, 1, len(values))@@ -39,7 +31,7 @@ func batchLogits(rows ...[]float32) *mlx.Array {// sampleOne runs Sample on a freshly-added single slot and returns the// sampled token id. Used both for the single-slot options table and as the// reference oracle for the batched-equivalence test.-func sampleOne(t *testing.T, opts Options, priorTokens []int32, values []float32) int32 {+func sampleOne(t *mlxtest.T, opts Options, priorTokens []int32, values []float32) int32 {t.Helper()s := New(128)t.Cleanup(func() {@@ -62,7 +54,7 @@ func logOf(p float64) float32 { return float32(math.Log(p)) }// hand from the math of each transform, not from a second call into the// sampler — so a regression in any single transform shows up here.func TestSampleSingleSlotOptions(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)cases := []struct {name string@@ -134,7 +126,7 @@ func TestSampleSingleSlotOptions(t *testing.T) {}for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tc.name, func(t *mlxtest.T) {if got := sampleOne(t, tc.opts, tc.priors, tc.logits); got != tc.want {t.Errorf("got %d, want %d", got, tc.want)}@@ -143,129 +135,129 @@ func TestSampleSingleSlotOptions(t *testing.T) {}func TestDistributionAppliesTopKBeforeTopP(t *testing.T) {- skipIfNoMLX(t)-- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()- })- s.Add(0, Options{Temperature: 1, TopK: 2, TopP: 0.7}, nil)+ mlxtest.Run(t, func(t *mlxtest.T) {+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })+ s.Add(0, Options{Temperature: 1, TopK: 2, TopP: 0.7}, nil)- dist := s.Distribution(0, slotLogits([]float32{logOf(0.6), logOf(0.2), logOf(0.2)}), nil)- mlx.Eval(dist.Arrays()...)+ dist := s.Distribution(0, slotLogits([]float32{logOf(0.6), logOf(0.2), logOf(0.2)}), nil)+ mlx.Eval(dist.Arrays()...)- ids := dist.IDs.Ints()- probs := dist.Probs.Floats()- if len(ids) != 2 || len(probs) != 2 {- t.Fatalf("support = ids %v probs %v, want 2 sparse entries", ids, probs)- }+ ids := dist.IDs.Ints()+ probs := dist.Probs.Floats()+ if len(ids) != 2 || len(probs) != 2 {+ t.Fatalf("support = ids %v probs %v, want 2 sparse entries", ids, probs)+ }- foundTop := false- for i, id := range ids {- switch id {- case 0:- foundTop = true- if math.Abs(float64(probs[i]-1)) > 1e-5 {- t.Fatalf("top token prob = %v, want 1; ids=%v probs=%v", probs[i], ids, probs)- }- default:- if math.Abs(float64(probs[i])) > 1e-5 {- t.Fatalf("non-top token %d prob = %v, want 0; ids=%v probs=%v", id, probs[i], ids, probs)+ foundTop := false+ for i, id := range ids {+ switch id {+ case 0:+ foundTop = true+ if math.Abs(float64(probs[i]-1)) > 1e-5 {+ t.Fatalf("top token prob = %v, want 1; ids=%v probs=%v", probs[i], ids, probs)+ }+ default:+ if math.Abs(float64(probs[i])) > 1e-5 {+ t.Fatalf("non-top token %d prob = %v, want 0; ids=%v probs=%v", id, probs[i], ids, probs)+ }}}- }- if !foundTop {- t.Fatalf("top-k support %v did not include token 0", ids)- }+ if !foundTop {+ t.Fatalf("top-k support %v did not include token 0", ids)+ }+ })}func TestDistributionResidualUsesTargetSupport(t *testing.T) {- skipIfNoMLX(t)-- target := Distribution{- IDs: mlx.NewArrayInt32([]int32{2, 5}, []int32{1, 2}),- Probs: mlx.FromValues([]float32{0.7, 0.3}, 1, 2),- }- draft := Distribution{- IDs: mlx.NewArrayInt32([]int32{2, 4}, []int32{1, 2}),- Probs: mlx.FromValues([]float32{0.2, 0.8}, 1, 2),- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ target := Distribution{+ IDs: mlx.NewArrayInt32([]int32{2, 5}, []int32{1, 2}),+ Probs: mlx.FromValues([]float32{0.7, 0.3}, 1, 2),+ }+ draft := Distribution{+ IDs: mlx.NewArrayInt32([]int32{2, 4}, []int32{1, 2}),+ Probs: mlx.FromValues([]float32{0.2, 0.8}, 1, 2),+ }- residual := target.ResidualAgainst(draft)- mlx.Eval(residual.Arrays()...)+ residual := target.ResidualAgainst(draft)+ mlx.Eval(residual.Arrays()...)- ids := residual.IDs.Ints()- probs := residual.Probs.Floats()- want := map[int32]float64{2: 0.625, 5: 0.375}- if len(ids) != 2 || len(probs) != 2 {- t.Fatalf("residual = ids %v probs %v, want 2 sparse entries", ids, probs)- }- for i, id := range ids {- w, ok := want[id]- if !ok {- t.Fatalf("residual includes token %d outside target support: ids=%v probs=%v", id, ids, probs)+ ids := residual.IDs.Ints()+ probs := residual.Probs.Floats()+ want := map[int32]float64{2: 0.625, 5: 0.375}+ if len(ids) != 2 || len(probs) != 2 {+ t.Fatalf("residual = ids %v probs %v, want 2 sparse entries", ids, probs)}- if math.Abs(float64(probs[i])-w) > 1e-5 {- t.Fatalf("residual token %d prob = %v, want %v; ids=%v probs=%v", id, probs[i], w, ids, probs)+ for i, id := range ids {+ w, ok := want[id]+ if !ok {+ t.Fatalf("residual includes token %d outside target support: ids=%v probs=%v", id, ids, probs)+ }+ if math.Abs(float64(probs[i])-w) > 1e-5 {+ t.Fatalf("residual token %d prob = %v, want %v; ids=%v probs=%v", id, probs[i], w, ids, probs)+ }}- }+ })}func TestSeededSamplingIsReproducible(t *testing.T) {- skipIfNoMLX(t)-- seededSequence := func(seed int) []int32 {- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()- })- s.Add(0, Options{Temperature: 1, TopK: 4, Seed: seed, UseSeed: true}, nil)-- logits := slotLogits([]float32{0, 0, 0, 0})- out := make([]int32, 32)- for i := range out {- token := s.Sample([]int{0}, logits).Token- mlx.Eval(token)- out[i] = token.Int()+ mlxtest.Run(t, func(t *mlxtest.T) {+ seededSequence := func(seed int) []int32 {+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })+ s.Add(0, Options{Temperature: 1, TopK: 4, Seed: seed, UseSeed: true}, nil)++ logits := slotLogits([]float32{0, 0, 0, 0})+ out := make([]int32, 32)+ for i := range out {+ token := s.Sample([]int{0}, logits).Token+ mlx.Eval(token)+ out[i] = token.Int()+ }+ return out}- return out- }- a := seededSequence(1234)- b := seededSequence(1234)- if !slices.Equal(a, b) {- t.Fatalf("same seed produced different sequences:\n%v\n%v", a, b)- }+ a := seededSequence(1234)+ b := seededSequence(1234)+ if !slices.Equal(a, b) {+ t.Fatalf("same seed produced different sequences:\n%v\n%v", a, b)+ }- c := seededSequence(5678)- if slices.Equal(a, c) {- t.Fatalf("different seeds produced the same sequence: %v", a)- }+ c := seededSequence(5678)+ if slices.Equal(a, c) {+ t.Fatalf("different seeds produced the same sequence: %v", a)+ }+ })}func TestSeededBernoulliIsReproducible(t *testing.T) {- skipIfNoMLX(t)-- seededMask := func() []int32 {- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()- })- s.Add(0, Options{Seed: 99, UseSeed: true}, nil)+ mlxtest.Run(t, func(t *mlxtest.T) {+ seededMask := func() []int32 {+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })+ s.Add(0, Options{Seed: 99, UseSeed: true}, nil)- mask := s.Bernoulli(0, mlx.FromValues([]float32{0.5, 0.5, 0.5, 0.5, 0.5, 0.5}, 6)).AsType(mlx.DTypeInt32)- mlx.Eval(mask)- return mask.Ints()- }+ mask := s.Bernoulli(0, mlx.FromValues([]float32{0.5, 0.5, 0.5, 0.5, 0.5, 0.5}, 6)).AsType(mlx.DTypeInt32)+ mlx.Eval(mask)+ return mask.Ints()+ }- a := seededMask()- b := seededMask()- if !slices.Equal(a, b) {- t.Fatalf("same seed produced different bernoulli masks:\n%v\n%v", a, b)- }+ a := seededMask()+ b := seededMask()+ if !slices.Equal(a, b) {+ t.Fatalf("same seed produced different bernoulli masks:\n%v\n%v", a, b)+ }+ })}// TestSampleHistoryWindow verifies that penalty history respects the@@ -273,154 +265,154 @@ func TestSeededBernoulliIsReproducible(t *testing.T) {// and once the ring wraps, tokens that rotate out no longer contribute// to penalties.func TestSampleHistoryWindow(t *testing.T) {- skipIfNoMLX(t)-- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()- })+ mlxtest.Run(t, func(t *mlxtest.T) {+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })- // RepeatLastN=2 with priors {1, 2, 3}: makeHistoryRow keeps only- // {2, 3}. Token 1 was trimmed — its penalty is NOT active.- s.Add(0, Options{RepeatLastN: 2, PresencePenalty: 10}, []int32{1, 2, 3})+ // RepeatLastN=2 with priors {1, 2, 3}: makeHistoryRow keeps only+ // {2, 3}. Token 1 was trimmed — its penalty is NOT active.+ s.Add(0, Options{RepeatLastN: 2, PresencePenalty: 10}, []int32{1, 2, 3})- // Step 1: logits favor token 1 (trimmed). If the trim were broken it- // would be penalized and the argmax would move.- step1 := s.Sample([]int{0}, slotLogits([]float32{0, 5, 0, 0, 0})).Token- mlx.Eval(step1)- if got := step1.Int(); got != 1 {- t.Fatalf("step 1 = %d, want 1 (token 1 trimmed from priors)", got)- }- // After step 1 the ring holds {1, 3}; token 2 has rotated out.-- // Step 2: logits favor token 2 (rotated out). If the ring wrap were- // wrong, token 2 would still be penalized.- step2 := s.Sample([]int{0}, slotLogits([]float32{0, 0, 5, 0, 0})).Token- mlx.Eval(step2)- if got := step2.Int(); got != 2 {- t.Fatalf("step 2 = %d, want 2 (token 2 rotated out of ring)", got)- }+ // Step 1: logits favor token 1 (trimmed). If the trim were broken it+ // would be penalized and the argmax would move.+ step1 := s.Sample([]int{0}, slotLogits([]float32{0, 5, 0, 0, 0})).Token+ mlx.Eval(step1)+ if got := step1.Int(); got != 1 {+ t.Fatalf("step 1 = %d, want 1 (token 1 trimmed from priors)", got)+ }+ // After step 1 the ring holds {1, 3}; token 2 has rotated out.++ // Step 2: logits favor token 2 (rotated out). If the ring wrap were+ // wrong, token 2 would still be penalized.+ step2 := s.Sample([]int{0}, slotLogits([]float32{0, 0, 5, 0, 0})).Token+ mlx.Eval(step2)+ if got := step2.Int(); got != 2 {+ t.Fatalf("step 2 = %d, want 2 (token 2 rotated out of ring)", got)+ }+ })}func TestSpeculativeScoresUsesDraftHistoryWithoutCommit(t *testing.T) {- skipIfNoMLX(t)-- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()- })+ mlxtest.Run(t, func(t *mlxtest.T) {+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })- s.Add(0, Options{RepeatLastN: 2, RepeatPenalty: 10}, []int32{1, 2})- draftTokens := mlx.NewArrayInt32([]int32{3, 4}, []int32{1, 2})- scores := s.SpeculativeScores(0, batchLogits(- []float32{0, 9, 9, 8, 0}, // history {1,2}; token 3 wins- []float32{0, 0, 9, 9, 8}, // history {2,3}; token 4 wins- []float32{0, 0, 9, 9, 8}, // history {3,4}; token 2 wins- ), draftTokens)- tokens := scores.Argmax(-1, false).AsType(mlx.DTypeInt32)- mlx.Eval(tokens)-- if got, want := tokens.Ints(), []int32{3, 4, 2}; len(got) != len(want) {- t.Fatalf("tokens = %v, want %v", got, want)- } else {- for i := range want {- if got[i] != want[i] {- t.Fatalf("tokens = %v, want %v", got, want)+ s.Add(0, Options{RepeatLastN: 2, RepeatPenalty: 10}, []int32{1, 2})+ draftTokens := mlx.NewArrayInt32([]int32{3, 4}, []int32{1, 2})+ scores := s.SpeculativeScores(0, batchLogits(+ []float32{0, 9, 9, 8, 0}, // history {1,2}; token 3 wins+ []float32{0, 0, 9, 9, 8}, // history {2,3}; token 4 wins+ []float32{0, 0, 9, 9, 8}, // history {3,4}; token 2 wins+ ), draftTokens)+ tokens := scores.Argmax(-1, false).AsType(mlx.DTypeInt32)+ mlx.Eval(tokens)++ if got, want := tokens.Ints(), []int32{3, 4, 2}; len(got) != len(want) {+ t.Fatalf("tokens = %v, want %v", got, want)+ } else {+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("tokens = %v, want %v", got, want)+ }}}- }- if s.byID[0].historyLen != 2 {- t.Fatalf("historyLen = %d, want 2", s.byID[0].historyLen)- }+ if s.byID[0].historyLen != 2 {+ t.Fatalf("historyLen = %d, want 2", s.byID[0].historyLen)+ }+ })}func TestDistributionSingleRowAppliesDraftPrefix(t *testing.T) {- skipIfNoMLX(t)-- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()- })+ mlxtest.Run(t, func(t *mlxtest.T) {+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })- // A proposal step passes one logits row with the chain's earlier drafts:- // the single row is the chain's final step, so every draft belongs to- // its history. Slot 0 exercises the batched history path (full ring),- // slot 1 the serial path (ring not yet full).- s.Add(0, Options{RepeatLastN: 2, RepeatPenalty: 10}, []int32{0, 1})- s.Add(1, Options{RepeatLastN: 8, RepeatPenalty: 10}, []int32{0, 1})- prefix := mlx.NewArrayInt32([]int32{3, 4}, []int32{1, 2})-- for _, seqID := range []int{0, 1} {- // Drafts 3 and 4 are penalized, so token 2 wins over the higher raw- // scores; with the drafts absent from the history, token 3 would.- dist := s.Distribution(seqID, batchLogits([]float32{0, 0, 9, 9, 8}), prefix)- mlx.Eval(dist.IDs)- if got := dist.IDs.Ints()[0]; got != 2 {- t.Fatalf("seq %d token = %d, want 2 (drafts 3 and 4 penalized)", seqID, got)+ // A proposal step passes one logits row with the chain's earlier drafts:+ // the single row is the chain's final step, so every draft belongs to+ // its history. Slot 0 exercises the batched history path (full ring),+ // slot 1 the serial path (ring not yet full).+ s.Add(0, Options{RepeatLastN: 2, RepeatPenalty: 10}, []int32{0, 1})+ s.Add(1, Options{RepeatLastN: 8, RepeatPenalty: 10}, []int32{0, 1})+ prefix := mlx.NewArrayInt32([]int32{3, 4}, []int32{1, 2})++ for _, seqID := range []int{0, 1} {+ // Drafts 3 and 4 are penalized, so token 2 wins over the higher raw+ // scores; with the drafts absent from the history, token 3 would.+ dist := s.Distribution(seqID, batchLogits([]float32{0, 0, 9, 9, 8}), prefix)+ mlx.Eval(dist.IDs)+ if got := dist.IDs.Ints()[0]; got != 2 {+ t.Fatalf("seq %d token = %d, want 2 (drafts 3 and 4 penalized)", seqID, got)+ }}- }+ })}func TestDistributionMultiRowWithoutChain(t *testing.T) {- skipIfNoMLX(t)-- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()- })+ mlxtest.Run(t, func(t *mlxtest.T) {+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })- // A block drafter's proposal batch samples every row from one call with- // no draft chain: each row sees the slot history unchanged. Slot 0- // exercises the batched history path (full ring), slot 1 the serial path- // (ring not yet full).- s.Add(0, Options{RepeatLastN: 2, RepeatPenalty: 10}, []int32{3, 4})- s.Add(1, Options{RepeatLastN: 8, RepeatPenalty: 10}, []int32{3, 4})-- for _, seqID := range []int{0, 1} {- // Tokens 3 and 4 are penalized in every row alike; rows 1 and 3- // share logits, so a chain alignment leaking between rows would- // split their winners.- dist := s.Distribution(seqID, batchLogits(- []float32{0, 0, 8, 9, 9},- []float32{0, 8, 0, 9, 9},- []float32{0, 0, 8, 9, 9},- ), nil)- top := dist.IDs.Slice(mlx.Slice(), mlx.Slice(0, 1))- mlx.Eval(top)- if got, want := top.Ints(), []int32{2, 1, 2}; !slices.Equal(got, want) {- t.Fatalf("seq %d top tokens = %v, want %v", seqID, got, want)+ // A block drafter's proposal batch samples every row from one call with+ // no draft chain: each row sees the slot history unchanged. Slot 0+ // exercises the batched history path (full ring), slot 1 the serial path+ // (ring not yet full).+ s.Add(0, Options{RepeatLastN: 2, RepeatPenalty: 10}, []int32{3, 4})+ s.Add(1, Options{RepeatLastN: 8, RepeatPenalty: 10}, []int32{3, 4})++ for _, seqID := range []int{0, 1} {+ // Tokens 3 and 4 are penalized in every row alike; rows 1 and 3+ // share logits, so a chain alignment leaking between rows would+ // split their winners.+ dist := s.Distribution(seqID, batchLogits(+ []float32{0, 0, 8, 9, 9},+ []float32{0, 8, 0, 9, 9},+ []float32{0, 0, 8, 9, 9},+ ), nil)+ top := dist.IDs.Slice(mlx.Slice(), mlx.Slice(0, 1))+ mlx.Eval(top)+ if got, want := top.Ints(), []int32{2, 1, 2}; !slices.Equal(got, want) {+ t.Fatalf("seq %d top tokens = %v, want %v", seqID, got, want)+ }}- }+ })}func TestCommitBatchesRingWrites(t *testing.T) {- skipIfNoMLX(t)-- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()- })+ mlxtest.Run(t, func(t *mlxtest.T) {+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })- s.Add(0, Options{RepeatLastN: 4, RepeatPenalty: 1.1}, []int32{10, 11, 12})- s.Commit(0, []int32{20, 21, 22})- s.Commit(0, []int32{30, 31, 32, 33, 34})- mlx.Eval(s.history)+ s.Add(0, Options{RepeatLastN: 4, RepeatPenalty: 1.1}, []int32{10, 11, 12})+ s.Commit(0, []int32{20, 21, 22})+ s.Commit(0, []int32{30, 31, 32, 33, 34})+ mlx.Eval(s.history)- got := s.history.Ints()- want := []int32{32, 33, 34, 31}- for i := range want {- if got[i] != want[i] {- t.Fatalf("history = %v, want %v", got, want)+ got := s.history.Ints()+ want := []int32{32, 33, 34, 31}+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("history = %v, want %v", got, want)+ }}- }- if s.byID[0].historyLen != 11 {- t.Fatalf("historyLen = %d, want 11", s.byID[0].historyLen)- }+ if s.byID[0].historyLen != 11 {+ t.Fatalf("historyLen = %d, want 11", s.byID[0].historyLen)+ }+ })}// TestBatchSamplingPreservesPerSlotBehavior is the core equivalence test:@@ -428,7 +420,7 @@ func TestCommitBatchesRingWrites(t *testing.T) {// serial on partial ring, subset/out-of-order), a batched Sample call must// produce the same token per row as running the same slot alone.func TestBatchSamplingPreservesPerSlotBehavior(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)type slot struct {id int@@ -482,7 +474,7 @@ func TestBatchSamplingPreservesPerSlotBehavior(t *testing.T) {}for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tc.name, func(t *mlxtest.T) {// Per-slot reference for each sampled seq.want := make([]int32, len(tc.sample))for i, id := range tc.sample {@@ -522,32 +514,32 @@ func TestBatchSamplingPreservesPerSlotBehavior(t *testing.T) {// recycled row must start from its own priors only — no carryover from// the removed slot's history.func TestRemoveDoesNotLeakHistory(t *testing.T) {- skipIfNoMLX(t)-- opts := Options{RepeatLastN: 1, PresencePenalty: 10}- s := New(128)- t.Cleanup(func() {- s.Free()- mlx.Sweep()+ mlxtest.Run(t, func(t *mlxtest.T) {+ opts := Options{RepeatLastN: 1, PresencePenalty: 10}+ s := New(128)+ t.Cleanup(func() {+ s.Free()+ mlx.Sweep()+ })+ s.Add(1, opts, []int32{1})+ s.Add(2, opts, []int32{2})+ s.Remove(1)+ s.Add(3, opts, []int32{0})++ // Slot 2 retains history {2}; slot 3 retains history {0}. With+ // equal logits and PresencePenalty=10 the argmax drops to the first+ // unpenalized token.+ res := s.Sample([]int{2, 3}, batchLogits(+ []float32{3, 3, 0},+ []float32{3, 3, 0},+ ))+ mlx.Eval(res.Token)+ tokens := res.Token.Ints()+ if tokens[0] != 0 {+ t.Errorf("slot 2 = %d, want 0 (token 2 penalized)", tokens[0])+ }+ if tokens[1] != 1 {+ t.Errorf("slot 3 = %d, want 1 (token 0 penalized, no slot-1 carryover)", tokens[1])+ }})- s.Add(1, opts, []int32{1})- s.Add(2, opts, []int32{2})- s.Remove(1)- s.Add(3, opts, []int32{0})-- // Slot 2 retains history {2}; slot 3 retains history {0}. With- // equal logits and PresencePenalty=10 the argmax drops to the first- // unpenalized token.- res := s.Sample([]int{2, 3}, batchLogits(- []float32{3, 3, 0},- []float32{3, 3, 0},- ))- mlx.Eval(res.Token)- tokens := res.Token.Ints()- if tokens[0] != 0 {- t.Errorf("slot 2 = %d, want 0 (token 2 penalized)", tokens[0])- }- if tokens[1] != 1 {- t.Errorf("slot 3 = %d, want 1 (token 0 penalized, no slot-1 carryover)", tokens[1])- }}diff --git a/x/models/gemma4/gemma4_moe_test.go b/x/models/gemma4/gemma4_moe_test.goindex e8113ef8a..2c3adf361 100644--- a/x/models/gemma4/gemma4_moe_test.go+++ b/x/models/gemma4/gemma4_moe_test.go@@ -1,37 +1,12 @@package gemma4import (- "runtime""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx")-func useMLXTestThread(t *testing.T) {- t.Helper()-- runtime.LockOSThread()- initialized := false- t.Cleanup(func() {- if initialized {- mlx.Sweep()- mlx.ClearCache()- if mlx.GPUIsAvailable() {- mlx.SetDefaultDeviceGPU()- }- }- runtime.UnlockOSThread()- })-- if err := mlx.CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }- initialized = true- if mlx.GPUIsAvailable() {- mlx.SetDefaultDeviceGPU()- }-}-// onesLike creates a tensor of the given shape filled with a small constant.func onesLike(shape ...int) *mlx.Array {return mlx.AddScalar(mlx.Zeros(mlx.DTypeBFloat16, shape...), 0.01)@@ -73,72 +48,72 @@ func newMoEBlock(cfg *TextConfig) *MoEBlock {}func TestMoERouterForward(t *testing.T) {- useMLXTestThread(t)-- cfg := tinyMoEConfig()- B, L := int32(1), int32(3)- x := onesLike(int(B), int(L), int(cfg.HiddenSize))- router := newRouter(cfg)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := tinyMoEConfig()+ B, L := int32(1), int32(3)+ x := onesLike(int(B), int(L), int(cfg.HiddenSize))+ router := newRouter(cfg)- scores, inds := router.Forward(x, cfg)- mlx.Eval(scores, inds)+ scores, inds := router.Forward(x, cfg)+ mlx.Eval(scores, inds)- sDims := scores.Dims()- iDims := inds.Dims()- t.Logf("scores shape: %v, inds shape: %v", sDims, iDims)+ sDims := scores.Dims()+ iDims := inds.Dims()+ t.Logf("scores shape: %v, inds shape: %v", sDims, iDims)- if len(sDims) != 2 || sDims[0] != int(B*L) || sDims[1] != int(cfg.TopKExperts) {- t.Errorf("scores shape = %v, want [%d, %d]", sDims, B*L, cfg.TopKExperts)- }- if len(iDims) != 2 || iDims[0] != int(B*L) || iDims[1] != int(cfg.TopKExperts) {- t.Errorf("inds shape = %v, want [%d, %d]", iDims, B*L, cfg.TopKExperts)- }+ if len(sDims) != 2 || sDims[0] != int(B*L) || sDims[1] != int(cfg.TopKExperts) {+ t.Errorf("scores shape = %v, want [%d, %d]", sDims, B*L, cfg.TopKExperts)+ }+ if len(iDims) != 2 || iDims[0] != int(B*L) || iDims[1] != int(cfg.TopKExperts) {+ t.Errorf("inds shape = %v, want [%d, %d]", iDims, B*L, cfg.TopKExperts)+ }+ })}func TestMoEBlockForward(t *testing.T) {- useMLXTestThread(t)-- cfg := tinyMoEConfig()- B, L := int32(1), int32(3)- x := onesLike(int(B), int(L), int(cfg.HiddenSize))- router := newRouter(cfg)- moe := newMoEBlock(cfg)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := tinyMoEConfig()+ B, L := int32(1), int32(3)+ x := onesLike(int(B), int(L), int(cfg.HiddenSize))+ router := newRouter(cfg)+ moe := newMoEBlock(cfg)- scores, inds := router.Forward(x, cfg)- mlx.Eval(scores, inds)+ scores, inds := router.Forward(x, cfg)+ mlx.Eval(scores, inds)- out := moe.Forward(x, scores, inds, cfg)- mlx.Eval(out)+ out := moe.Forward(x, scores, inds, cfg)+ mlx.Eval(out)- outDims := out.Dims()- t.Logf("MoE output shape: %v", outDims)+ outDims := out.Dims()+ t.Logf("MoE output shape: %v", outDims)- if len(outDims) != 3 || outDims[0] != int(B) || outDims[1] != int(L) || outDims[2] != int(cfg.HiddenSize) {- t.Errorf("output shape = %v, want [%d, %d, %d]", outDims, B, L, cfg.HiddenSize)- }+ if len(outDims) != 3 || outDims[0] != int(B) || outDims[1] != int(L) || outDims[2] != int(cfg.HiddenSize) {+ t.Errorf("output shape = %v, want [%d, %d, %d]", outDims, B, L, cfg.HiddenSize)+ }+ })}func TestMoEBlockSortedForward(t *testing.T) {- useMLXTestThread(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := tinyMoEConfig()+ B, L := int32(1), int32(128)+ x := onesLike(int(B), int(L), int(cfg.HiddenSize))+ router := newRouter(cfg)+ moe := newMoEBlock(cfg)- cfg := tinyMoEConfig()- B, L := int32(1), int32(128)- x := onesLike(int(B), int(L), int(cfg.HiddenSize))- router := newRouter(cfg)- moe := newMoEBlock(cfg)+ scores, inds := router.Forward(x, cfg)+ mlx.Eval(scores, inds)- scores, inds := router.Forward(x, cfg)- mlx.Eval(scores, inds)+ out := moe.Forward(x, scores, inds, cfg)+ mlx.Eval(out)- out := moe.Forward(x, scores, inds, cfg)- mlx.Eval(out)+ outDims := out.Dims()+ t.Logf("MoE sorted output shape: %v", outDims)- outDims := out.Dims()- t.Logf("MoE sorted output shape: %v", outDims)-- if len(outDims) != 3 || outDims[0] != int(B) || outDims[1] != int(L) || outDims[2] != int(cfg.HiddenSize) {- t.Errorf("output shape = %v, want [%d, %d, %d]", outDims, B, L, cfg.HiddenSize)- }+ if len(outDims) != 3 || outDims[0] != int(B) || outDims[1] != int(L) || outDims[2] != int(cfg.HiddenSize) {+ t.Errorf("output shape = %v, want [%d, %d, %d]", outDims, B, L, cfg.HiddenSize)+ }+ })}// TestLoadFusedExpertsQuantized verifies that a quantized, fused gate_up@@ -147,7 +122,7 @@ func TestMoEBlockSortedForward(t *testing.T) {// through to the dense branch and was loaded unquantized (the memory bloat// bug this fix addresses).func TestLoadFusedExpertsQuantized(t *testing.T) {- skipIfNoMLX(t)+ mlxtest.SkipIfUnavailable(t)const E, I, H = 4, 8, 16m := &Model{TextConfig: &TextConfig{QuantGroupSize: 16, QuantBits: 4, QuantMode: "nvfp4"}}@@ -156,7 +131,7 @@ func TestLoadFusedExpertsQuantized(t *testing.T) {"model.language_model.layers.0.experts", // gemma HF (bare .experts.)"model.language_model.layers.0.moe.switch_mlp", // create pipeline} {- t.Run(prefix, func(t *testing.T) {+ mlxtest.RunSubtest(t, prefix, func(t *mlxtest.T) {gateUpKey := prefix + ".gate_up_proj"downKey := prefix + ".down_proj"tensors := map[string]*mlx.Array{@@ -194,33 +169,33 @@ func TestLoadFusedExpertsQuantized(t *testing.T) {// TestLoadFusedExpertsDense verifies that a fused gate_up projection with no// scale companions is loaded onto the dense GatherMM path, kept fused.func TestLoadFusedExpertsDense(t *testing.T) {- skipIfNoMLX(t)-- const E, I, H = 4, 8, 16- m := &Model{TextConfig: &TextConfig{}}-- gateUpKey := "model.language_model.layers.0.experts.gate_up_proj"- downKey := "model.language_model.layers.0.experts.down_proj"- tensors := map[string]*mlx.Array{- gateUpKey: onesLike(E, 2*I, H),- downKey: onesLike(E, I, H),- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ const E, I, H = 4, 8, 16+ m := &Model{TextConfig: &TextConfig{}}++ gateUpKey := "model.language_model.layers.0.experts.gate_up_proj"+ downKey := "model.language_model.layers.0.experts.down_proj"+ tensors := map[string]*mlx.Array{+ gateUpKey: onesLike(E, 2*I, H),+ downKey: onesLike(E, I, H),+ }- moe := &MoEBlock{}- m.loadFusedExperts(moe, tensors, gateUpKey, tensors[gateUpKey], downKey, tensors[downKey])+ moe := &MoEBlock{}+ m.loadFusedExperts(moe, tensors, gateUpKey, tensors[gateUpKey], downKey, tensors[downKey])- if moe.UseQuantized {- t.Error("UseQuantized = true, want false (no scales present)")- }- if !moe.UseFusedGateUp {- t.Error("UseFusedGateUp = false, want true")- }- if moe.GateUpWeight == nil || moe.DownWeight == nil {- t.Error("dense fused weights not set")- }- if moe.GateUpWeightQ != nil || moe.DownWeightQ != nil {- t.Error("quantized weights set on a dense block")- }+ if moe.UseQuantized {+ t.Error("UseQuantized = true, want false (no scales present)")+ }+ if !moe.UseFusedGateUp {+ t.Error("UseFusedGateUp = false, want true")+ }+ if moe.GateUpWeight == nil || moe.DownWeight == nil {+ t.Error("dense fused weights not set")+ }+ if moe.GateUpWeightQ != nil || moe.DownWeightQ != nil {+ t.Error("quantized weights set on a dense block")+ }+ })}// TestRouterForwardMatchesLegacy verifies the optimized Router.Forward —@@ -229,54 +204,54 @@ func TestLoadFusedExpertsDense(t *testing.T) {// normalized scores as the legacy path that softmaxes over every expert// first, gathers the top-k probabilities, then renormalizes.func TestRouterForwardMatchesLegacy(t *testing.T) {- useMLXTestThread(t)-- cfg := &TextConfig{- HiddenSize: 8,- NumExperts: 4,- TopKExperts: 2,- RMSNormEps: 1e-6,- RouterScale: 0.5,- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &TextConfig{+ HiddenSize: 8,+ NumExperts: 4,+ TopKExperts: 2,+ RMSNormEps: 1e-6,+ RouterScale: 0.5,+ }- // Distinct per-expert weight rows so top-k has a well-defined ordering- // (tied scores would let argpartition pick either tied expert and make- // the index comparison below flaky).- projWeight := mlx.FromValues([]float32{- 0.10, 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, // expert 0- 0.30, 0.29, 0.28, 0.27, 0.26, 0.25, 0.24, 0.23, // expert 1- -0.05, -0.06, -0.07, -0.08, -0.09, -0.10, -0.11, -0.12, // expert 2- 0.50, 0.48, 0.46, 0.44, 0.42, 0.40, 0.38, 0.36, // expert 3- }, int(cfg.NumExperts), int(cfg.HiddenSize))-- scale := mlx.FromValues([]float32{- 1.0, 0.9, 1.1, 1.0, 1.2, 0.8, 1.0, 1.05,- }, int(cfg.HiddenSize))-- r := &Router{- Proj: linearFromWeight(projWeight),- Scale: scale,- }+ // Distinct per-expert weight rows so top-k has a well-defined ordering+ // (tied scores would let argpartition pick either tied expert and make+ // the index comparison below flaky).+ projWeight := mlx.FromValues([]float32{+ 0.10, 0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, // expert 0+ 0.30, 0.29, 0.28, 0.27, 0.26, 0.25, 0.24, 0.23, // expert 1+ -0.05, -0.06, -0.07, -0.08, -0.09, -0.10, -0.11, -0.12, // expert 2+ 0.50, 0.48, 0.46, 0.44, 0.42, 0.40, 0.38, 0.36, // expert 3+ }, int(cfg.NumExperts), int(cfg.HiddenSize))++ scale := mlx.FromValues([]float32{+ 1.0, 0.9, 1.1, 1.0, 1.2, 0.8, 1.0, 1.05,+ }, int(cfg.HiddenSize))++ r := &Router{+ Proj: linearFromWeight(projWeight),+ Scale: scale,+ }- // Varied x so different positions potentially hit different top-k.- x := mlx.FromValues([]float32{- 0.2, -0.1, 0.3, 0.0, 0.4, -0.2, 0.1, 0.05,- -0.3, 0.2, -0.1, 0.4, -0.05, 0.3, 0.0, 0.2,- 0.5, 0.4, -0.2, 0.1, -0.3, 0.0, 0.3, -0.1,- }, 1, 3, int(cfg.HiddenSize))-- gotScores, gotInds := r.Forward(x, cfg)- wantScores, wantInds := legacyRouterForward(r, x, cfg)- gotInds = gotInds.AsType(mlx.DTypeInt32)- wantInds = wantInds.AsType(mlx.DTypeInt32)- mlx.Eval(gotScores, gotInds, wantScores, wantInds)-- if got, want := gotInds.Ints(), wantInds.Ints(); !intSlicesEqual(got, want) {- t.Fatalf("indices mismatch:\n got %v\n want %v", got, want)- }- if got, want := gotScores.Floats(), wantScores.Floats(); !floatSlicesClose(got, want, 1e-5) {- t.Fatalf("scores mismatch:\n got %v\n want %v", got, want)- }+ // Varied x so different positions potentially hit different top-k.+ x := mlx.FromValues([]float32{+ 0.2, -0.1, 0.3, 0.0, 0.4, -0.2, 0.1, 0.05,+ -0.3, 0.2, -0.1, 0.4, -0.05, 0.3, 0.0, 0.2,+ 0.5, 0.4, -0.2, 0.1, -0.3, 0.0, 0.3, -0.1,+ }, 1, 3, int(cfg.HiddenSize))++ gotScores, gotInds := r.Forward(x, cfg)+ wantScores, wantInds := legacyRouterForward(r, x, cfg)+ gotInds = gotInds.AsType(mlx.DTypeInt32)+ wantInds = wantInds.AsType(mlx.DTypeInt32)+ mlx.Eval(gotScores, gotInds, wantScores, wantInds)++ if got, want := gotInds.Ints(), wantInds.Ints(); !intSlicesEqual(got, want) {+ t.Fatalf("indices mismatch:\n got %v\n want %v", got, want)+ }+ if got, want := gotScores.Floats(), wantScores.Floats(); !floatSlicesClose(got, want, 1e-5) {+ t.Fatalf("scores mismatch:\n got %v\n want %v", got, want)+ }+ })}// legacyRouterForward implements the pre-optimization router: full softmaxdiff --git a/x/models/gemma4/gemma4_test.go b/x/models/gemma4/gemma4_test.goindex 7a85dca40..385b5b6ea 100644--- a/x/models/gemma4/gemma4_test.go+++ b/x/models/gemma4/gemma4_test.go@@ -5,6 +5,7 @@ import ("slices""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx")@@ -25,8 +26,8 @@ func TestParseSuppressTokens(t *testing.T) {}func TestParseTextConfigE2B(t *testing.T) {- skipIfNoMLX(t)- data := []byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ data := []byte(`{"architectures": ["Gemma4ForConditionalGeneration"],"text_config": {"hidden_size": 1536,@@ -71,78 +72,79 @@ func TestParseTextConfigE2B(t *testing.T) {}}`)- cfg, err := parseTextConfig(data)- if err != nil {- t.Fatalf("parseTextConfig failed: %v", err)- }+ cfg, err := parseTextConfig(data)+ if err != nil {+ t.Fatalf("parseTextConfig failed: %v", err)+ }- // Basic fields.- if cfg.HiddenSize != 1536 {- t.Errorf("HiddenSize = %d, want 1536", cfg.HiddenSize)- }- if cfg.NumHiddenLayers != 35 {- t.Errorf("NumHiddenLayers = %d, want 35", cfg.NumHiddenLayers)- }- if cfg.GlobalHeadDim != 512 {- t.Errorf("GlobalHeadDim = %d, want 512", cfg.GlobalHeadDim)- }- if cfg.FinalLogitSoftcapping != 30.0 {- t.Errorf("FinalLogitSoftcapping = %f, want 30.0", cfg.FinalLogitSoftcapping)- }- if cfg.NumKVSharedLayers != 20 {- t.Errorf("NumKVSharedLayers = %d, want 20", cfg.NumKVSharedLayers)- }- if cfg.HiddenSizePerLayer != 256 {- t.Errorf("HiddenSizePerLayer = %d, want 256", cfg.HiddenSizePerLayer)- }+ // Basic fields.+ if cfg.HiddenSize != 1536 {+ t.Errorf("HiddenSize = %d, want 1536", cfg.HiddenSize)+ }+ if cfg.NumHiddenLayers != 35 {+ t.Errorf("NumHiddenLayers = %d, want 35", cfg.NumHiddenLayers)+ }+ if cfg.GlobalHeadDim != 512 {+ t.Errorf("GlobalHeadDim = %d, want 512", cfg.GlobalHeadDim)+ }+ if cfg.FinalLogitSoftcapping != 30.0 {+ t.Errorf("FinalLogitSoftcapping = %f, want 30.0", cfg.FinalLogitSoftcapping)+ }+ if cfg.NumKVSharedLayers != 20 {+ t.Errorf("NumKVSharedLayers = %d, want 20", cfg.NumKVSharedLayers)+ }+ if cfg.HiddenSizePerLayer != 256 {+ t.Errorf("HiddenSizePerLayer = %d, want 256", cfg.HiddenSizePerLayer)+ }- // RoPE settings.- if cfg.SlidingRopeDims != 256 {- t.Errorf("SlidingRopeDims = %d, want 256", cfg.SlidingRopeDims)- }- if cfg.FullRopeDims != 512 {- t.Errorf("FullRopeDims = %d, want 512 (GlobalHeadDim, partial rotation handled via custom freqs)", cfg.FullRopeDims)- }- if cfg.SlidingRopeBase != 10000 {- t.Errorf("SlidingRopeBase = %f, want 10000", cfg.SlidingRopeBase)- }- if cfg.FullRopeBase != 1000000 {- t.Errorf("FullRopeBase = %f, want 1000000", cfg.FullRopeBase)- }+ // RoPE settings.+ if cfg.SlidingRopeDims != 256 {+ t.Errorf("SlidingRopeDims = %d, want 256", cfg.SlidingRopeDims)+ }+ if cfg.FullRopeDims != 512 {+ t.Errorf("FullRopeDims = %d, want 512 (GlobalHeadDim, partial rotation handled via custom freqs)", cfg.FullRopeDims)+ }+ if cfg.SlidingRopeBase != 10000 {+ t.Errorf("SlidingRopeBase = %f, want 10000", cfg.SlidingRopeBase)+ }+ if cfg.FullRopeBase != 1000000 {+ t.Errorf("FullRopeBase = %f, want 1000000", cfg.FullRopeBase)+ }- // Attention scale.- if cfg.SlidingScale == 0 || cfg.FullScale == 0 {- t.Error("attention scales should be non-zero")- }+ // Attention scale.+ if cfg.SlidingScale == 0 || cfg.FullScale == 0 {+ t.Error("attention scales should be non-zero")+ }- // KV sharing map.- // First shared layer is 35 - 20 = 15.- if donor, ok := cfg.KVShareMap[15]; !ok || donor != 13 {- t.Errorf("KVShareMap[15] = %d, ok=%v; want 13, true", donor, ok)- }- if donor, ok := cfg.KVShareMap[19]; !ok || donor != 14 {- t.Errorf("KVShareMap[19] = %d, ok=%v; want 14, true (full attn donor)", donor, ok)- }- if donor, ok := cfg.KVShareMap[34]; !ok || donor != 14 {- t.Errorf("KVShareMap[34] = %d, ok=%v; want 14, true (full attn donor)", donor, ok)- }- // Layer 14 should not be shared.- if _, ok := cfg.KVShareMap[14]; ok {- t.Error("layer 14 should not be in KVShareMap (non-shared)")- }+ // KV sharing map.+ // First shared layer is 35 - 20 = 15.+ if donor, ok := cfg.KVShareMap[15]; !ok || donor != 13 {+ t.Errorf("KVShareMap[15] = %d, ok=%v; want 13, true", donor, ok)+ }+ if donor, ok := cfg.KVShareMap[19]; !ok || donor != 14 {+ t.Errorf("KVShareMap[19] = %d, ok=%v; want 14, true (full attn donor)", donor, ok)+ }+ if donor, ok := cfg.KVShareMap[34]; !ok || donor != 14 {+ t.Errorf("KVShareMap[34] = %d, ok=%v; want 14, true (full attn donor)", donor, ok)+ }+ // Layer 14 should not be shared.+ if _, ok := cfg.KVShareMap[14]; ok {+ t.Error("layer 14 should not be in KVShareMap (non-shared)")+ }- // Donors.- if !cfg.KVDonors[13] {- t.Error("layer 13 should be a KV donor")- }- if !cfg.KVDonors[14] {- t.Error("layer 14 should be a KV donor")- }+ // Donors.+ if !cfg.KVDonors[13] {+ t.Error("layer 13 should be a KV donor")+ }+ if !cfg.KVDonors[14] {+ t.Error("layer 14 should be a KV donor")+ }+ })}func TestParseTextConfig26B(t *testing.T) {- skipIfNoMLX(t)- data := []byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ data := []byte(`{"architectures": ["Gemma4ForConditionalGeneration"],"text_config": {"hidden_size": 2816,@@ -188,40 +190,41 @@ func TestParseTextConfig26B(t *testing.T) {}}`)- cfg, err := parseTextConfig(data)- if err != nil {- t.Fatalf("parseTextConfig failed: %v", err)- }+ cfg, err := parseTextConfig(data)+ if err != nil {+ t.Fatalf("parseTextConfig failed: %v", err)+ }- if cfg.HiddenSize != 2816 {- t.Errorf("HiddenSize = %d, want 2816", cfg.HiddenSize)- }- if !cfg.AttentionKEqV {- t.Error("AttentionKEqV should be true")- }- if cfg.NumGlobalKeyValueHeads != 2 {- t.Errorf("NumGlobalKeyValueHeads = %d, want 2", cfg.NumGlobalKeyValueHeads)- }- if !cfg.EnableMoeBlock {- t.Error("EnableMoeBlock should be true")- }- if cfg.NumExperts != 128 {- t.Errorf("NumExperts = %d, want 128", cfg.NumExperts)- }- if cfg.TopKExperts != 8 {- t.Errorf("TopKExperts = %d, want 8", cfg.TopKExperts)- }- if cfg.ExpertIntermediateSize != 704 {- t.Errorf("ExpertIntermediateSize = %d, want 704", cfg.ExpertIntermediateSize)- }- if cfg.HiddenSizePerLayer != 0 {- t.Errorf("HiddenSizePerLayer = %d, want 0 (no PLE)", cfg.HiddenSizePerLayer)- }+ if cfg.HiddenSize != 2816 {+ t.Errorf("HiddenSize = %d, want 2816", cfg.HiddenSize)+ }+ if !cfg.AttentionKEqV {+ t.Error("AttentionKEqV should be true")+ }+ if cfg.NumGlobalKeyValueHeads != 2 {+ t.Errorf("NumGlobalKeyValueHeads = %d, want 2", cfg.NumGlobalKeyValueHeads)+ }+ if !cfg.EnableMoeBlock {+ t.Error("EnableMoeBlock should be true")+ }+ if cfg.NumExperts != 128 {+ t.Errorf("NumExperts = %d, want 128", cfg.NumExperts)+ }+ if cfg.TopKExperts != 8 {+ t.Errorf("TopKExperts = %d, want 8", cfg.TopKExperts)+ }+ if cfg.ExpertIntermediateSize != 704 {+ t.Errorf("ExpertIntermediateSize = %d, want 704", cfg.ExpertIntermediateSize)+ }+ if cfg.HiddenSizePerLayer != 0 {+ t.Errorf("HiddenSizePerLayer = %d, want 0 (no PLE)", cfg.HiddenSizePerLayer)+ }+ })}func TestParseTextConfig31B(t *testing.T) {- skipIfNoMLX(t)- data := []byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ data := []byte(`{"architectures": ["Gemma4ForConditionalGeneration"],"text_config": {"hidden_size": 5376,@@ -268,168 +271,169 @@ func TestParseTextConfig31B(t *testing.T) {}}`)- cfg, err := parseTextConfig(data)- if err != nil {- t.Fatalf("parseTextConfig failed: %v", err)- }+ cfg, err := parseTextConfig(data)+ if err != nil {+ t.Fatalf("parseTextConfig failed: %v", err)+ }- if cfg.HiddenSize != 5376 {- t.Errorf("HiddenSize = %d, want 5376", cfg.HiddenSize)- }- if cfg.NumHiddenLayers != 60 {- t.Errorf("NumHiddenLayers = %d, want 60", cfg.NumHiddenLayers)- }- if !cfg.AttentionKEqV {- t.Error("AttentionKEqV should be true")- }- if cfg.NumGlobalKeyValueHeads != 4 {- t.Errorf("NumGlobalKeyValueHeads = %d, want 4", cfg.NumGlobalKeyValueHeads)- }- if cfg.NumKeyValueHeads != 16 {- t.Errorf("NumKeyValueHeads = %d, want 16", cfg.NumKeyValueHeads)- }- if cfg.NumKVSharedLayers != 0 {- t.Errorf("NumKVSharedLayers = %d, want 0", cfg.NumKVSharedLayers)- }- if cfg.HiddenSizePerLayer != 0 {- t.Errorf("HiddenSizePerLayer = %d, want 0 (no PLE)", cfg.HiddenSizePerLayer)- }- if cfg.SlidingWindow != 1024 {- t.Errorf("SlidingWindow = %d, want 1024", cfg.SlidingWindow)- }+ if cfg.HiddenSize != 5376 {+ t.Errorf("HiddenSize = %d, want 5376", cfg.HiddenSize)+ }+ if cfg.NumHiddenLayers != 60 {+ t.Errorf("NumHiddenLayers = %d, want 60", cfg.NumHiddenLayers)+ }+ if !cfg.AttentionKEqV {+ t.Error("AttentionKEqV should be true")+ }+ if cfg.NumGlobalKeyValueHeads != 4 {+ t.Errorf("NumGlobalKeyValueHeads = %d, want 4", cfg.NumGlobalKeyValueHeads)+ }+ if cfg.NumKeyValueHeads != 16 {+ t.Errorf("NumKeyValueHeads = %d, want 16", cfg.NumKeyValueHeads)+ }+ if cfg.NumKVSharedLayers != 0 {+ t.Errorf("NumKVSharedLayers = %d, want 0", cfg.NumKVSharedLayers)+ }+ if cfg.HiddenSizePerLayer != 0 {+ t.Errorf("HiddenSizePerLayer = %d, want 0 (no PLE)", cfg.HiddenSizePerLayer)+ }+ if cfg.SlidingWindow != 1024 {+ t.Errorf("SlidingWindow = %d, want 1024", cfg.SlidingWindow)+ }- // KV sharing should be empty (no shared layers).- if len(cfg.KVShareMap) != 0 {- t.Errorf("KVShareMap should be empty, got %d entries", len(cfg.KVShareMap))- }+ // KV sharing should be empty (no shared layers).+ if len(cfg.KVShareMap) != 0 {+ t.Errorf("KVShareMap should be empty, got %d entries", len(cfg.KVShareMap))+ }- // Layer types: pattern is 5 sliding + 1 full, repeating 10 times.- if !isLayerSliding(0, &cfg) {- t.Error("layer 0 should be sliding")- }- if isLayerSliding(5, &cfg) {- t.Error("layer 5 should be full attention")- }- if !isLayerSliding(6, &cfg) {- t.Error("layer 6 should be sliding")- }- if isLayerSliding(59, &cfg) {- t.Error("layer 59 should be full attention")- }+ // Layer types: pattern is 5 sliding + 1 full, repeating 10 times.+ if !isLayerSliding(0, &cfg) {+ t.Error("layer 0 should be sliding")+ }+ if isLayerSliding(5, &cfg) {+ t.Error("layer 5 should be full attention")+ }+ if !isLayerSliding(6, &cfg) {+ t.Error("layer 6 should be sliding")+ }+ if isLayerSliding(59, &cfg) {+ t.Error("layer 59 should be full attention")+ }+ })}func TestParseTextConfig12BUnified(t *testing.T) {- skipIfNoMLX(t)-- layerTypes := make([]string, 0, 48)- for i := range 48 {- if i%6 == 5 {- layerTypes = append(layerTypes, "full_attention")- } else {- layerTypes = append(layerTypes, "sliding_attention")+ mlxtest.Run(t, func(t *mlxtest.T) {+ layerTypes := make([]string, 0, 48)+ for i := range 48 {+ if i%6 == 5 {+ layerTypes = append(layerTypes, "full_attention")+ } else {+ layerTypes = append(layerTypes, "sliding_attention")+ }}- }- data, err := json.Marshal(map[string]any{- "architectures": []string{"Gemma4UnifiedForConditionalGeneration"},- "model_type": "gemma4_unified",- "text_config": map[string]any{- "hidden_size": 3840,- "num_hidden_layers": 48,- "intermediate_size": 15360,- "num_attention_heads": 16,- "num_key_value_heads": 8,- "num_global_key_value_heads": 1,- "head_dim": 256,- "global_head_dim": 512,- "vocab_size": 262144,- "rms_norm_eps": 1e-6,- "max_position_embeddings": 131072,- "sliding_window": 1024,- "final_logit_softcapping": 30.0,- "use_double_wide_mlp": false,- "num_kv_shared_layers": 0,- "hidden_size_per_layer_input": 0,- "vocab_size_per_layer_input": 262144,- "attention_k_eq_v": true,- "enable_moe_block": false,- "tie_word_embeddings": true,- "layer_types": layerTypes,- "rope_parameters": map[string]any{- "full_attention": map[string]any{- "partial_rotary_factor": 0.25,- "rope_theta": 1000000.0,- "rope_type": "proportional",- },- "sliding_attention": map[string]any{- "rope_theta": 10000.0,- "rope_type": "default",+ data, err := json.Marshal(map[string]any{+ "architectures": []string{"Gemma4UnifiedForConditionalGeneration"},+ "model_type": "gemma4_unified",+ "text_config": map[string]any{+ "hidden_size": 3840,+ "num_hidden_layers": 48,+ "intermediate_size": 15360,+ "num_attention_heads": 16,+ "num_key_value_heads": 8,+ "num_global_key_value_heads": 1,+ "head_dim": 256,+ "global_head_dim": 512,+ "vocab_size": 262144,+ "rms_norm_eps": 1e-6,+ "max_position_embeddings": 131072,+ "sliding_window": 1024,+ "final_logit_softcapping": 30.0,+ "use_double_wide_mlp": false,+ "num_kv_shared_layers": 0,+ "hidden_size_per_layer_input": 0,+ "vocab_size_per_layer_input": 262144,+ "attention_k_eq_v": true,+ "enable_moe_block": false,+ "tie_word_embeddings": true,+ "layer_types": layerTypes,+ "rope_parameters": map[string]any{+ "full_attention": map[string]any{+ "partial_rotary_factor": 0.25,+ "rope_theta": 1000000.0,+ "rope_type": "proportional",+ },+ "sliding_attention": map[string]any{+ "rope_theta": 10000.0,+ "rope_type": "default",+ },},},- },- })- if err != nil {- t.Fatalf("json.Marshal failed: %v", err)- }+ })+ if err != nil {+ t.Fatalf("json.Marshal failed: %v", err)+ }- cfg, err := parseTextConfig(data)- if err != nil {- t.Fatalf("parseTextConfig failed: %v", err)- }+ cfg, err := parseTextConfig(data)+ if err != nil {+ t.Fatalf("parseTextConfig failed: %v", err)+ }- if cfg.HiddenSize != 3840 {- t.Errorf("HiddenSize = %d, want 3840", cfg.HiddenSize)- }- if cfg.NumHiddenLayers != 48 {- t.Errorf("NumHiddenLayers = %d, want 48", cfg.NumHiddenLayers)- }- if cfg.IntermediateSize != 15360 {- t.Errorf("IntermediateSize = %d, want 15360", cfg.IntermediateSize)- }- if cfg.NumAttentionHeads != 16 {- t.Errorf("NumAttentionHeads = %d, want 16", cfg.NumAttentionHeads)- }- if cfg.NumKeyValueHeads != 8 {- t.Errorf("NumKeyValueHeads = %d, want 8", cfg.NumKeyValueHeads)- }- if cfg.NumGlobalKeyValueHeads != 1 {- t.Errorf("NumGlobalKeyValueHeads = %d, want 1", cfg.NumGlobalKeyValueHeads)- }- if !cfg.AttentionKEqV {- t.Error("AttentionKEqV should be true")- }- if cfg.EnableMoeBlock {- t.Error("EnableMoeBlock should be false")- }- if cfg.HiddenSizePerLayer != 0 {- t.Errorf("HiddenSizePerLayer = %d, want 0", cfg.HiddenSizePerLayer)- }- if cfg.NumKVSharedLayers != 0 {- t.Errorf("NumKVSharedLayers = %d, want 0", cfg.NumKVSharedLayers)- }- if len(cfg.KVShareMap) != 0 {- t.Errorf("KVShareMap should be empty, got %d entries", len(cfg.KVShareMap))- }- if cfg.FullRopeDims != 512 {- t.Errorf("FullRopeDims = %d, want 512", cfg.FullRopeDims)- }- if cfg.FullRopeFreqs == nil {- t.Error("FullRopeFreqs should be precomputed for proportional RoPE")- }- if isLayerSliding(5, &cfg) {- t.Error("layer 5 should be full attention")- }- if !isLayerSliding(6, &cfg) {- t.Error("layer 6 should be sliding")- }- if isLayerSliding(47, &cfg) {- t.Error("layer 47 should be full attention")- }+ if cfg.HiddenSize != 3840 {+ t.Errorf("HiddenSize = %d, want 3840", cfg.HiddenSize)+ }+ if cfg.NumHiddenLayers != 48 {+ t.Errorf("NumHiddenLayers = %d, want 48", cfg.NumHiddenLayers)+ }+ if cfg.IntermediateSize != 15360 {+ t.Errorf("IntermediateSize = %d, want 15360", cfg.IntermediateSize)+ }+ if cfg.NumAttentionHeads != 16 {+ t.Errorf("NumAttentionHeads = %d, want 16", cfg.NumAttentionHeads)+ }+ if cfg.NumKeyValueHeads != 8 {+ t.Errorf("NumKeyValueHeads = %d, want 8", cfg.NumKeyValueHeads)+ }+ if cfg.NumGlobalKeyValueHeads != 1 {+ t.Errorf("NumGlobalKeyValueHeads = %d, want 1", cfg.NumGlobalKeyValueHeads)+ }+ if !cfg.AttentionKEqV {+ t.Error("AttentionKEqV should be true")+ }+ if cfg.EnableMoeBlock {+ t.Error("EnableMoeBlock should be false")+ }+ if cfg.HiddenSizePerLayer != 0 {+ t.Errorf("HiddenSizePerLayer = %d, want 0", cfg.HiddenSizePerLayer)+ }+ if cfg.NumKVSharedLayers != 0 {+ t.Errorf("NumKVSharedLayers = %d, want 0", cfg.NumKVSharedLayers)+ }+ if len(cfg.KVShareMap) != 0 {+ t.Errorf("KVShareMap should be empty, got %d entries", len(cfg.KVShareMap))+ }+ if cfg.FullRopeDims != 512 {+ t.Errorf("FullRopeDims = %d, want 512", cfg.FullRopeDims)+ }+ if cfg.FullRopeFreqs == nil {+ t.Error("FullRopeFreqs should be precomputed for proportional RoPE")+ }+ if isLayerSliding(5, &cfg) {+ t.Error("layer 5 should be full attention")+ }+ if !isLayerSliding(6, &cfg) {+ t.Error("layer 6 should be sliding")+ }+ if isLayerSliding(47, &cfg) {+ t.Error("layer 47 should be full attention")+ }+ })}func TestParseTextConfigE4B(t *testing.T) {- skipIfNoMLX(t)- data := []byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ data := []byte(`{"architectures": ["Gemma4ForConditionalGeneration"],"text_config": {"hidden_size": 2560,@@ -474,73 +478,74 @@ func TestParseTextConfigE4B(t *testing.T) {}}`)- cfg, err := parseTextConfig(data)- if err != nil {- t.Fatalf("parseTextConfig failed: %v", err)- }+ cfg, err := parseTextConfig(data)+ if err != nil {+ t.Fatalf("parseTextConfig failed: %v", err)+ }- if cfg.HiddenSize != 2560 {- t.Errorf("HiddenSize = %d, want 2560", cfg.HiddenSize)- }- if cfg.NumHiddenLayers != 42 {- t.Errorf("NumHiddenLayers = %d, want 42", cfg.NumHiddenLayers)- }- if cfg.IntermediateSize != 10240 {- t.Errorf("IntermediateSize = %d, want 10240", cfg.IntermediateSize)- }- if cfg.NumKeyValueHeads != 2 {- t.Errorf("NumKeyValueHeads = %d, want 2", cfg.NumKeyValueHeads)- }- if cfg.UseDoubleWideMLP {- t.Error("UseDoubleWideMLP should be false")- }- if cfg.NumKVSharedLayers != 18 {- t.Errorf("NumKVSharedLayers = %d, want 18", cfg.NumKVSharedLayers)- }- if cfg.HiddenSizePerLayer != 256 {- t.Errorf("HiddenSizePerLayer = %d, want 256 (has PLE)", cfg.HiddenSizePerLayer)- }- if cfg.AttentionKEqV {- t.Error("AttentionKEqV should be false")- }- if cfg.EnableMoeBlock {- t.Error("EnableMoeBlock should be false")- }- if cfg.SlidingWindow != 512 {- t.Errorf("SlidingWindow = %d, want 512", cfg.SlidingWindow)- }+ if cfg.HiddenSize != 2560 {+ t.Errorf("HiddenSize = %d, want 2560", cfg.HiddenSize)+ }+ if cfg.NumHiddenLayers != 42 {+ t.Errorf("NumHiddenLayers = %d, want 42", cfg.NumHiddenLayers)+ }+ if cfg.IntermediateSize != 10240 {+ t.Errorf("IntermediateSize = %d, want 10240", cfg.IntermediateSize)+ }+ if cfg.NumKeyValueHeads != 2 {+ t.Errorf("NumKeyValueHeads = %d, want 2", cfg.NumKeyValueHeads)+ }+ if cfg.UseDoubleWideMLP {+ t.Error("UseDoubleWideMLP should be false")+ }+ if cfg.NumKVSharedLayers != 18 {+ t.Errorf("NumKVSharedLayers = %d, want 18", cfg.NumKVSharedLayers)+ }+ if cfg.HiddenSizePerLayer != 256 {+ t.Errorf("HiddenSizePerLayer = %d, want 256 (has PLE)", cfg.HiddenSizePerLayer)+ }+ if cfg.AttentionKEqV {+ t.Error("AttentionKEqV should be false")+ }+ if cfg.EnableMoeBlock {+ t.Error("EnableMoeBlock should be false")+ }+ if cfg.SlidingWindow != 512 {+ t.Errorf("SlidingWindow = %d, want 512", cfg.SlidingWindow)+ }- // Layer types: pattern is 5 sliding + 1 full, repeating 7 times = 42 layers.- if !isLayerSliding(0, &cfg) {- t.Error("layer 0 should be sliding")- }- if isLayerSliding(5, &cfg) {- t.Error("layer 5 should be full attention")- }- if !isLayerSliding(6, &cfg) {- t.Error("layer 6 should be sliding")- }- if isLayerSliding(41, &cfg) {- t.Error("layer 41 should be full attention")- }+ // Layer types: pattern is 5 sliding + 1 full, repeating 7 times = 42 layers.+ if !isLayerSliding(0, &cfg) {+ t.Error("layer 0 should be sliding")+ }+ if isLayerSliding(5, &cfg) {+ t.Error("layer 5 should be full attention")+ }+ if !isLayerSliding(6, &cfg) {+ t.Error("layer 6 should be sliding")+ }+ if isLayerSliding(41, &cfg) {+ t.Error("layer 41 should be full attention")+ }- // KV sharing: first shared = 42 - 18 = 24.- // Layer 24 is sliding, its donor should be the last non-shared sliding layer.- // Non-shared layers: 0-23. Last sliding in 0-23 is layer 22 (23=full).- if donor, ok := cfg.KVShareMap[24]; !ok {- t.Error("layer 24 should be in KVShareMap")- } else {- t.Logf("layer 24 donor = %d", donor)- }- // Layer 29 is full_attention (5th full), donor should be the last non-shared full layer.- // Non-shared full layers: 5, 11, 17, 23.- if donor, ok := cfg.KVShareMap[29]; !ok || donor != 23 {- t.Errorf("KVShareMap[29] = %d, ok=%v; want 23, true (full attn donor)", donor, ok)- }- // Layer 23 should NOT be shared (it's the last non-shared layer).- if _, ok := cfg.KVShareMap[23]; ok {- t.Error("layer 23 should not be in KVShareMap (non-shared)")- }+ // KV sharing: first shared = 42 - 18 = 24.+ // Layer 24 is sliding, its donor should be the last non-shared sliding layer.+ // Non-shared layers: 0-23. Last sliding in 0-23 is layer 22 (23=full).+ if donor, ok := cfg.KVShareMap[24]; !ok {+ t.Error("layer 24 should be in KVShareMap")+ } else {+ t.Logf("layer 24 donor = %d", donor)+ }+ // Layer 29 is full_attention (5th full), donor should be the last non-shared full layer.+ // Non-shared full layers: 5, 11, 17, 23.+ if donor, ok := cfg.KVShareMap[29]; !ok || donor != 23 {+ t.Errorf("KVShareMap[29] = %d, ok=%v; want 23, true (full attn donor)", donor, ok)+ }+ // Layer 23 should NOT be shared (it's the last non-shared layer).+ if _, ok := cfg.KVShareMap[23]; ok {+ t.Error("layer 23 should not be in KVShareMap (non-shared)")+ }+ })}func TestLayerTypeDetection(t *testing.T) {@@ -644,9 +649,7 @@ func TestNewCachesAssistantSharedHistoryOrdering(t *testing.T) {}func TestResolveWeightPrefix(t *testing.T) {- if err := mlx.CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }+ mlxtest.SkipIfUnavailable(t)tests := []struct {name string@@ -659,7 +662,7 @@ func TestResolveWeightPrefix(t *testing.T) {}for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {+ mlxtest.RunSubtest(t, tt.name, func(t *mlxtest.T) {dummy := mlx.FromValue(float32(1.0))mlx.Eval(dummy)tensors := map[string]*mlx.Array{tt.key: dummy}@@ -670,10 +673,3 @@ func TestResolveWeightPrefix(t *testing.T) {})}}--func skipIfNoMLX(t *testing.T) {- t.Helper()- if err := mlx.CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }-}diff --git a/x/models/glimmer/glimmer_test.go b/x/models/glimmer/glimmer_test.goindex a5676d67c..3827f006e 100644--- a/x/models/glimmer/glimmer_test.go+++ b/x/models/glimmer/glimmer_test.go@@ -226,21 +226,21 @@ func TestNewCachesMatchesAttentionSchedule(t *testing.T) {}func TestUnembedReturnsFloat32Logits(t *testing.T) {- mlxtest.Setup(t)-- input := mlx.FromValues([]float32{1}, 1, 1, 1).AsType(mlx.DTypeBFloat16)- weight := mlx.FromValues([]float32{1, 2}, 2, 1).AsType(mlx.DTypeBFloat16)- m := Model{- LMHead: nn.NewLinear(weight, nil),- Config: &Config{- OutputMultiplier: 0.19611613,- OutputSoftCapTemp: 20,- },- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ input := mlx.FromValues([]float32{1}, 1, 1, 1).AsType(mlx.DTypeBFloat16)+ weight := mlx.FromValues([]float32{1, 2}, 2, 1).AsType(mlx.DTypeBFloat16)+ m := Model{+ LMHead: nn.NewLinear(weight, nil),+ Config: &Config{+ OutputMultiplier: 0.19611613,+ OutputSoftCapTemp: 20,+ },+ }- if got := m.Unembed(input).DType(); got != mlx.DTypeFloat32 {- t.Fatalf("Unembed() dtype = %v, want %v", got, mlx.DTypeFloat32)- }+ if got := m.Unembed(input).DType(); got != mlx.DTypeFloat32 {+ t.Fatalf("Unembed() dtype = %v, want %v", got, mlx.DTypeFloat32)+ }+ })}func TestComputeImageSizeMatchesReference(t *testing.T) {@@ -320,9 +320,7 @@ func TestApplyVisionRoPELayouts(t *testing.T) {}for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- mlxtest.Setup(t)-+ mlxtest.RunSubtest(t, tt.name, func(t *mlxtest.T) {x := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 1, 1, 4)cos := mlx.FromValues([]float32{0.5, 0.25}, 1, 2)sin := mlx.FromValues([]float32{0.75, 0.125}, 1, 2)diff --git a/x/models/laguna/laguna_test.go b/x/models/laguna/laguna_test.goindex d342adf0d..ba6e61457 100644--- a/x/models/laguna/laguna_test.go+++ b/x/models/laguna/laguna_test.go@@ -6,14 +6,15 @@ import ("strings""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/mlx""github.com/ollama/ollama/x/models/nn")func TestParseConfigLagunaXS(t *testing.T) {- skipIfNoMLX(t)- cfg, err := parseConfig([]byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg, err := parseConfig([]byte(`{"model_type": "laguna","hidden_size": 2048,"intermediate_size": 8192,@@ -52,45 +53,46 @@ func TestParseConfigLagunaXS(t *testing.T) {"rope_type": "linear"}}`))- if err != nil {- t.Fatal(err)- }+ if err != nil {+ t.Fatal(err)+ }- if cfg.FullRopeDim != 64 {- t.Fatalf("FullRopeDim = %d, want 64", cfg.FullRopeDim)- }- if cfg.FullRopeBase != 500000 {- t.Fatalf("FullRopeBase = %v, want 500000", cfg.FullRopeBase)- }- if cfg.FullRopeScale != 1 {- t.Fatalf("FullRopeScale = %v, want explicit YaRN attention_factor", cfg.FullRopeScale)- }- if cfg.FullRopeFreqs == nil {- t.Fatal("FullRopeFreqs should be precomputed for YaRN")- }- if cfg.SlidingRopeDim != 128 {- t.Fatalf("SlidingRopeDim = %d, want 128", cfg.SlidingRopeDim)- }- if cfg.SlidingRopeBase != 10000 {- t.Fatalf("SlidingRopeBase = %v, want 10000", cfg.SlidingRopeBase)- }- if !layerIsSliding(&cfg, 1) {- t.Fatal("layer 1 should use sliding attention")- }- if layerUsesMoE(&cfg, 0) {- t.Fatal("layer 0 should be dense due to mlp_only_layers")- }- if !layerUsesMoE(&cfg, 1) {- t.Fatal("layer 1 should use MoE")- }- if got := numHeadsForLayer(&cfg, 1); got != 64 {- t.Fatalf("numHeadsForLayer(1) = %d, want 64", got)- }+ if cfg.FullRopeDim != 64 {+ t.Fatalf("FullRopeDim = %d, want 64", cfg.FullRopeDim)+ }+ if cfg.FullRopeBase != 500000 {+ t.Fatalf("FullRopeBase = %v, want 500000", cfg.FullRopeBase)+ }+ if cfg.FullRopeScale != 1 {+ t.Fatalf("FullRopeScale = %v, want explicit YaRN attention_factor", cfg.FullRopeScale)+ }+ if cfg.FullRopeFreqs == nil {+ t.Fatal("FullRopeFreqs should be precomputed for YaRN")+ }+ if cfg.SlidingRopeDim != 128 {+ t.Fatalf("SlidingRopeDim = %d, want 128", cfg.SlidingRopeDim)+ }+ if cfg.SlidingRopeBase != 10000 {+ t.Fatalf("SlidingRopeBase = %v, want 10000", cfg.SlidingRopeBase)+ }+ if !layerIsSliding(&cfg, 1) {+ t.Fatal("layer 1 should use sliding attention")+ }+ if layerUsesMoE(&cfg, 0) {+ t.Fatal("layer 0 should be dense due to mlp_only_layers")+ }+ if !layerUsesMoE(&cfg, 1) {+ t.Fatal("layer 1 should use MoE")+ }+ if got := numHeadsForLayer(&cfg, 1); got != 64 {+ t.Fatalf("numHeadsForLayer(1) = %d, want 64", got)+ }+ })}func TestParseConfigLagunaFP8RopeScaling(t *testing.T) {- skipIfNoMLX(t)- cfg, err := parseConfig([]byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg, err := parseConfig([]byte(`{"hidden_size": 2048,"intermediate_size": 8192,"num_hidden_layers": 1,@@ -106,20 +108,21 @@ func TestParseConfigLagunaFP8RopeScaling(t *testing.T) {"factor": 32}}`))- if err != nil {- t.Fatal(err)- }- if cfg.FullRopeBase != 500000 {- t.Fatalf("FullRopeBase = %v, want 500000", cfg.FullRopeBase)- }- if cfg.FullRopeDim != 64 {- t.Fatalf("FullRopeDim = %d, want 64", cfg.FullRopeDim)- }+ if err != nil {+ t.Fatal(err)+ }+ if cfg.FullRopeBase != 500000 {+ t.Fatalf("FullRopeBase = %v, want 500000", cfg.FullRopeBase)+ }+ if cfg.FullRopeDim != 64 {+ t.Fatalf("FullRopeDim = %d, want 64", cfg.FullRopeDim)+ }+ })}func TestParseConfigLagunaGASchema(t *testing.T) {- skipIfNoMLX(t)- cfg, err := parseConfig([]byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg, err := parseConfig([]byte(`{"model_type": "laguna","hidden_size": 2048,"intermediate_size": 8192,@@ -159,39 +162,40 @@ func TestParseConfigLagunaGASchema(t *testing.T) {}}}`))- if err != nil {- t.Fatal(err)- }+ if err != nil {+ t.Fatal(err)+ }- if cfg.Gating != "per-head" {- t.Fatalf("Gating = %q, want per-head", cfg.Gating)- }- if !cfg.NormTopKProb {- t.Fatal("NormTopKProb should default true")- }- if cfg.FullRopeBase != 500000 {- t.Fatalf("FullRopeBase = %v, want 500000", cfg.FullRopeBase)- }- if cfg.SlidingRopeBase != 10000 {- t.Fatalf("SlidingRopeBase = %v, want 10000", cfg.SlidingRopeBase)- }- if cfg.FullRopeDim != 64 {- t.Fatalf("FullRopeDim = %d, want 64", cfg.FullRopeDim)- }- if cfg.SlidingRopeDim != 128 {- t.Fatalf("SlidingRopeDim = %d, want 128", cfg.SlidingRopeDim)- }- if layerUsesMoE(&cfg, 0) {- t.Fatal("layer 0 should be dense due to mlp_layer_types")- }- if !layerUsesMoE(&cfg, 1) {- t.Fatal("layer 1 should use MoE")- }+ if cfg.Gating != "per-head" {+ t.Fatalf("Gating = %q, want per-head", cfg.Gating)+ }+ if !cfg.NormTopKProb {+ t.Fatal("NormTopKProb should default true")+ }+ if cfg.FullRopeBase != 500000 {+ t.Fatalf("FullRopeBase = %v, want 500000", cfg.FullRopeBase)+ }+ if cfg.SlidingRopeBase != 10000 {+ t.Fatalf("SlidingRopeBase = %v, want 10000", cfg.SlidingRopeBase)+ }+ if cfg.FullRopeDim != 64 {+ t.Fatalf("FullRopeDim = %d, want 64", cfg.FullRopeDim)+ }+ if cfg.SlidingRopeDim != 128 {+ t.Fatalf("SlidingRopeDim = %d, want 128", cfg.SlidingRopeDim)+ }+ if layerUsesMoE(&cfg, 0) {+ t.Fatal("layer 0 should be dense due to mlp_layer_types")+ }+ if !layerUsesMoE(&cfg, 1) {+ t.Fatal("layer 1 should use MoE")+ }+ })}func TestTinyLagunaLoadAndForward(t *testing.T) {- skipIfNoMLX(t)- cfg, err := parseConfig([]byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg, err := parseConfig([]byte(`{"model_type": "laguna","hidden_size": 8,"intermediate_size": 12,@@ -229,56 +233,57 @@ func TestTinyLagunaLoadAndForward(t *testing.T) {"rope_type": "linear"}}`))- if err != nil {- t.Fatal(err)- }+ if err != nil {+ t.Fatal(err)+ }- m := &Model{- Config: &cfg,- Layers: []*Layer{- {LayerIdx: 0, IsSliding: false},- {LayerIdx: 1, IsSliding: true},- },- }- tensors := tinyLagunaTensors()- if err := m.LoadWeights(tensors); err != nil {- t.Fatalf("LoadWeights failed: %v", err)- }+ m := &Model{+ Config: &cfg,+ Layers: []*Layer{+ {LayerIdx: 0, IsSliding: false},+ {LayerIdx: 1, IsSliding: true},+ },+ }+ tensors := tinyLagunaTensors()+ if err := m.LoadWeights(tensors); err != nil {+ t.Fatalf("LoadWeights failed: %v", err)+ }- tokens := mlx.FromValues([]int32{1, 2, 3}, 1, 3)- caches := m.NewCaches()- defer func() {- for _, c := range caches {- if c != nil {- c.Free()+ tokens := mlx.FromValues([]int32{1, 2, 3}, 1, 3)+ caches := m.NewCaches()+ defer func() {+ for _, c := range caches {+ if c != nil {+ c.Free()+ }}+ }()+ hidden, _ := m.Forward(&batch.Batch{+ InputIDs: tokens,+ SeqOffsets: []int32{0},+ SeqQueryLens: []int32{int32(tokens.Dim(1))},+ }, caches)+ mlx.Eval(hidden)+ if got := hidden.Dims(); len(got) != 3 || got[0] != 1 || got[1] != 3 || got[2] != 8 {+ t.Fatalf("hidden shape = %v, want [1 3 8]", got)}- }()- hidden, _ := m.Forward(&batch.Batch{- InputIDs: tokens,- SeqOffsets: []int32{0},- SeqQueryLens: []int32{int32(tokens.Dim(1))},- }, caches)- mlx.Eval(hidden)- if got := hidden.Dims(); len(got) != 3 || got[0] != 1 || got[1] != 3 || got[2] != 8 {- t.Fatalf("hidden shape = %v, want [1 3 8]", got)- }- logits := m.Unembed(hidden)- mlx.Eval(logits)- if got := logits.Dims(); len(got) != 3 || got[0] != 1 || got[1] != 3 || got[2] != 16 {- t.Fatalf("logits shape = %v, want [1 3 16]", got)- }- for i, v := range logits.Floats() {- if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {- t.Fatalf("logits[%d] is not finite: %v", i, v)+ logits := m.Unembed(hidden)+ mlx.Eval(logits)+ if got := logits.Dims(); len(got) != 3 || got[0] != 1 || got[1] != 3 || got[2] != 16 {+ t.Fatalf("logits shape = %v, want [1 3 16]", got)}- }+ for i, v := range logits.Floats() {+ if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {+ t.Fatalf("logits[%d] is not finite: %v", i, v)+ }+ }+ })}func TestTinyLagunaLoadWeightsFusesDenseGateUp(t *testing.T) {- skipIfNoMLX(t)- cfg, err := parseConfig([]byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg, err := parseConfig([]byte(`{"model_type": "laguna","hidden_size": 8,"intermediate_size": 12,@@ -302,36 +307,37 @@ func TestTinyLagunaLoadWeightsFusesDenseGateUp(t *testing.T) {"gating": "per-head","rms_norm_eps": 1e-5}`))- if err != nil {- t.Fatal(err)- }+ if err != nil {+ t.Fatal(err)+ }- m := &Model{- Config: &cfg,- Layers: []*Layer{- {LayerIdx: 0, IsSliding: false},- {LayerIdx: 1, IsSliding: true},- },- }- if err := m.LoadWeights(tinyLagunaTensors()); err != nil {- t.Fatalf("LoadWeights failed: %v", err)- }+ m := &Model{+ Config: &cfg,+ Layers: []*Layer{+ {LayerIdx: 0, IsSliding: false},+ {LayerIdx: 1, IsSliding: true},+ },+ }+ if err := m.LoadWeights(tinyLagunaTensors()); err != nil {+ t.Fatalf("LoadWeights failed: %v", err)+ }- moe, ok := m.Layers[1].MLP.(*SparseMoE)- if !ok {- t.Fatalf("layer 1 MLP type = %T, want *SparseMoE", m.Layers[1].MLP)- }- if moe.SwitchMLP.GateUpWeight == nil {- t.Fatal("expected fused GateUpWeight to be populated")- }- if got, want := moe.SwitchMLP.GateUpWeight.Dims(), []int{2, 8, 8}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] {- t.Fatalf("GateUpWeight dims = %v, want %v", got, want)- }+ moe, ok := m.Layers[1].MLP.(*SparseMoE)+ if !ok {+ t.Fatalf("layer 1 MLP type = %T, want *SparseMoE", m.Layers[1].MLP)+ }+ if moe.SwitchMLP.GateUpWeight == nil {+ t.Fatal("expected fused GateUpWeight to be populated")+ }+ if got, want := moe.SwitchMLP.GateUpWeight.Dims(), []int{2, 8, 8}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] {+ t.Fatalf("GateUpWeight dims = %v, want %v", got, want)+ }+ })}func TestTinyLagunaLoadWeightsKeepsBF16SourceLayout(t *testing.T) {- skipIfNoMLX(t)- cfg, err := parseConfig([]byte(`{+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg, err := parseConfig([]byte(`{"model_type": "laguna","hidden_size": 8,"intermediate_size": 12,@@ -355,370 +361,380 @@ func TestTinyLagunaLoadWeightsKeepsBF16SourceLayout(t *testing.T) {"gating": "per-head","rms_norm_eps": 1e-5}`))- if err != nil {- t.Fatal(err)- }+ if err != nil {+ t.Fatal(err)+ }- tensors := tinyLagunaTensors()- for key, tensor := range tensors {- if strings.Contains(key, ".mlp.experts.") && strings.HasSuffix(key, ".weight") {- tensors[key] = tensor.AsType(mlx.DTypeBFloat16)+ tensors := tinyLagunaTensors()+ for key, tensor := range tensors {+ if strings.Contains(key, ".mlp.experts.") && strings.HasSuffix(key, ".weight") {+ tensors[key] = tensor.AsType(mlx.DTypeBFloat16)+ }+ }+ m := &Model{+ Config: &cfg,+ Layers: []*Layer{+ {LayerIdx: 0, IsSliding: false},+ {LayerIdx: 1, IsSliding: true},+ },+ }+ if err := m.LoadWeights(tensors); err != nil {+ t.Fatalf("LoadWeights failed: %v", err)}- }- m := &Model{- Config: &cfg,- Layers: []*Layer{- {LayerIdx: 0, IsSliding: false},- {LayerIdx: 1, IsSliding: true},- },- }- if err := m.LoadWeights(tensors); err != nil {- t.Fatalf("LoadWeights failed: %v", err)- }- moe, ok := m.Layers[1].MLP.(*SparseMoE)- if !ok {- t.Fatalf("layer 1 MLP type = %T, want *SparseMoE", m.Layers[1].MLP)- }- if !moe.SwitchMLP.GateUpWeightsSourceLayout || !moe.SwitchMLP.DownWeightSourceLayout {- t.Fatal("expected BF16 dense SwitchMLP to keep source-layout expert weights")- }- if moe.SwitchMLP.GateUpWeight != nil {- t.Fatal("expected BF16 source-layout SwitchMLP to avoid pre-fused gate/up weights")- }- if got, want := moe.SwitchMLP.GateWeight.Dims(), []int{2, 4, 8}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] {- t.Fatalf("GateWeight dims = %v, want %v", got, want)- }+ moe, ok := m.Layers[1].MLP.(*SparseMoE)+ if !ok {+ t.Fatalf("layer 1 MLP type = %T, want *SparseMoE", m.Layers[1].MLP)+ }+ if !moe.SwitchMLP.GateUpWeightsSourceLayout || !moe.SwitchMLP.DownWeightSourceLayout {+ t.Fatal("expected BF16 dense SwitchMLP to keep source-layout expert weights")+ }+ if moe.SwitchMLP.GateUpWeight != nil {+ t.Fatal("expected BF16 source-layout SwitchMLP to avoid pre-fused gate/up weights")+ }+ if got, want := moe.SwitchMLP.GateWeight.Dims(), []int{2, 4, 8}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] {+ t.Fatalf("GateWeight dims = %v, want %v", got, want)+ }+ })}func TestTinyLagunaLoadWeightsKeepsMixedExpertPrecision(t *testing.T) {- skipIfNoMLX(t)- cfg := &Config{- HiddenSize: 8,- IntermediateSize: 12,- MoeIntermediateSize: 4,- SharedExpertIntermediate: 4,- NumHiddenLayers: 2,- NumAttentionHeads: 2,- NumAttentionHeadsPerLayer: []int32{2, 2},- NumKeyValueHeads: 1,- HeadDim: 4,- VocabSize: 16,- LayerTypes: []string{"full_attention", "sliding_attention"},- MLPOnlyLayers: []int32{0},- DecoderSparseStep: 1,- NumExperts: 2,- NumExpertsPerTok: 1,- MoeRoutedScalingFactor: 2.5,- RMSNormEps: 1e-5,- QuantGroupSize: 4,- QuantBits: 4,- QuantMode: "affine",- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{+ HiddenSize: 8,+ IntermediateSize: 12,+ MoeIntermediateSize: 4,+ SharedExpertIntermediate: 4,+ NumHiddenLayers: 2,+ NumAttentionHeads: 2,+ NumAttentionHeadsPerLayer: []int32{2, 2},+ NumKeyValueHeads: 1,+ HeadDim: 4,+ VocabSize: 16,+ LayerTypes: []string{"full_attention", "sliding_attention"},+ MLPOnlyLayers: []int32{0},+ DecoderSparseStep: 1,+ NumExperts: 2,+ NumExpertsPerTok: 1,+ MoeRoutedScalingFactor: 2.5,+ RMSNormEps: 1e-5,+ QuantGroupSize: 4,+ QuantBits: 4,+ QuantMode: "affine",+ }- tensors := tinyLagunaTensors()- for expert := range 2 {- prefix := "model.layers.1.mlp.experts." + string(rune('0'+expert))- for _, proj := range []string{"gate_proj", "up_proj"} {- key := prefix + "." + proj + ".weight"- weight, scales, biases := mlx.Quantize(tensors[key], cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode)- tensors[key] = weight- tensors[key+"_scale"] = scales- tensors[key+"_qbias"] = biases- }- downKey := prefix + ".down_proj.weight"- tensors[downKey] = tensors[downKey].AsType(mlx.DTypeBFloat16)- }+ tensors := tinyLagunaTensors()+ for expert := range 2 {+ prefix := "model.layers.1.mlp.experts." + string(rune('0'+expert))+ for _, proj := range []string{"gate_proj", "up_proj"} {+ key := prefix + "." + proj + ".weight"+ weight, scales, biases := mlx.Quantize(tensors[key], cfg.QuantGroupSize, cfg.QuantBits, cfg.QuantMode)+ tensors[key] = weight+ tensors[key+"_scale"] = scales+ tensors[key+"_qbias"] = biases+ }+ downKey := prefix + ".down_proj.weight"+ tensors[downKey] = tensors[downKey].AsType(mlx.DTypeBFloat16)+ }- m := &Model{- Config: cfg,- Layers: []*Layer{- {LayerIdx: 0, IsSliding: false},- {LayerIdx: 1, IsSliding: true},- },- }- if err := m.LoadWeights(tensors); err != nil {- t.Fatalf("LoadWeights failed: %v", err)- }+ m := &Model{+ Config: cfg,+ Layers: []*Layer{+ {LayerIdx: 0, IsSliding: false},+ {LayerIdx: 1, IsSliding: true},+ },+ }+ if err := m.LoadWeights(tensors); err != nil {+ t.Fatalf("LoadWeights failed: %v", err)+ }- moe, ok := m.Layers[1].MLP.(*SparseMoE)- if !ok {- t.Fatalf("layer 1 MLP type = %T, want *SparseMoE", m.Layers[1].MLP)- }- hasFusedGateUp := moe.SwitchMLP.GateUpWeightQ != nil && moe.SwitchMLP.GateUpScales != nil- hasSeparateGateUp := moe.SwitchMLP.GateWeightQ != nil && moe.SwitchMLP.GateScales != nil &&- moe.SwitchMLP.UpWeightQ != nil && moe.SwitchMLP.UpScales != nil- if !hasFusedGateUp && !hasSeparateGateUp {- t.Fatal("expected quantized gate/up expert weights")- }- if moe.SwitchMLP.GateUpWeight != nil || moe.SwitchMLP.GateWeight != nil || moe.SwitchMLP.UpWeight != nil {- t.Fatal("quantized gate/up expert weights fell back to dense")- }- if moe.SwitchMLP.DownWeight == nil || moe.SwitchMLP.DownWeightQ != nil || !moe.SwitchMLP.DownWeightSourceLayout {- t.Fatal("expected BF16 down expert weights to retain source layout")- }+ moe, ok := m.Layers[1].MLP.(*SparseMoE)+ if !ok {+ t.Fatalf("layer 1 MLP type = %T, want *SparseMoE", m.Layers[1].MLP)+ }+ hasFusedGateUp := moe.SwitchMLP.GateUpWeightQ != nil && moe.SwitchMLP.GateUpScales != nil+ hasSeparateGateUp := moe.SwitchMLP.GateWeightQ != nil && moe.SwitchMLP.GateScales != nil &&+ moe.SwitchMLP.UpWeightQ != nil && moe.SwitchMLP.UpScales != nil+ if !hasFusedGateUp && !hasSeparateGateUp {+ t.Fatal("expected quantized gate/up expert weights")+ }+ if moe.SwitchMLP.GateUpWeight != nil || moe.SwitchMLP.GateWeight != nil || moe.SwitchMLP.UpWeight != nil {+ t.Fatal("quantized gate/up expert weights fell back to dense")+ }+ if moe.SwitchMLP.DownWeight == nil || moe.SwitchMLP.DownWeightQ != nil || !moe.SwitchMLP.DownWeightSourceLayout {+ t.Fatal("expected BF16 down expert weights to retain source layout")+ }+ })}func TestSparseMoERouteBiasAffectsSelectionNotRoutingWeights(t *testing.T) {- skipIfNoMLX(t)- cfg := &Config{- HiddenSize: 1,- NumExperts: 2,- NumExpertsPerTok: 1,- NormTopKProb: false,- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{+ HiddenSize: 1,+ NumExperts: 2,+ NumExpertsPerTok: 1,+ NormTopKProb: false,+ }- moe := &SparseMoE{- Gate: nn.NewLinear(mlx.FromValues([]float32{-4, -3}, 2, 1).AsType(mlx.DTypeBFloat16), nil),- EScoreCorrectionBias: mlx.FromValues([]float32{0.5, 0}, 2),- }+ moe := &SparseMoE{+ Gate: nn.NewLinear(mlx.FromValues([]float32{-4, -3}, 2, 1).AsType(mlx.DTypeBFloat16), nil),+ EScoreCorrectionBias: mlx.FromValues([]float32{0.5, 0}, 2),+ }- xFlat := mlx.FromValues([]float32{1}, 1, int(cfg.HiddenSize)).AsType(mlx.DTypeBFloat16)- scores, inds, scalesFolded := moe.route(xFlat, cfg)- if scalesFolded {- t.Fatal("route folded scales without expert projection scales")- }- scores = scores.AsType(mlx.DTypeFloat32)- inds = inds.AsType(mlx.DTypeInt32)- mlx.Eval(scores, inds)-- gates := moe.Gate.Forward(xFlat).AsType(mlx.DTypeFloat32)- probs := mlx.Sigmoid(gates)- mlx.Eval(probs)- probVals := probs.Floats()- if probVals[0] >= probVals[1] {- t.Fatalf("expected unbiased sigmoid scores to prefer expert 1, got %v", probVals)- }- if probVals[0]+0.5 <= probVals[1] {- t.Fatalf("expected bias to flip selection to expert 0, got probs=%v", probVals)- }- if got := inds.Ints(); len(got) != 1 || got[0] != 0 {- t.Fatalf("selected experts = %v, want [0]", got)- }- if got := scores.Floats(); len(got) != 1 || math.Abs(float64(got[0]-probVals[0])) > 1e-6 {- t.Fatalf("routing weights = %v, want [%v] using unbiased sigmoid scores", got, probVals[0])- }+ xFlat := mlx.FromValues([]float32{1}, 1, int(cfg.HiddenSize)).AsType(mlx.DTypeBFloat16)+ scores, inds, scalesFolded := moe.route(xFlat, cfg)+ if scalesFolded {+ t.Fatal("route folded scales without expert projection scales")+ }+ scores = scores.AsType(mlx.DTypeFloat32)+ inds = inds.AsType(mlx.DTypeInt32)+ mlx.Eval(scores, inds)++ gates := moe.Gate.Forward(xFlat).AsType(mlx.DTypeFloat32)+ probs := mlx.Sigmoid(gates)+ mlx.Eval(probs)+ probVals := probs.Floats()+ if probVals[0] >= probVals[1] {+ t.Fatalf("expected unbiased sigmoid scores to prefer expert 1, got %v", probVals)+ }+ if probVals[0]+0.5 <= probVals[1] {+ t.Fatalf("expected bias to flip selection to expert 0, got probs=%v", probVals)+ }+ if got := inds.Ints(); len(got) != 1 || got[0] != 0 {+ t.Fatalf("selected experts = %v, want [0]", got)+ }+ if got := scores.Floats(); len(got) != 1 || math.Abs(float64(got[0]-probVals[0])) > 1e-6 {+ t.Fatalf("routing weights = %v, want [%v] using unbiased sigmoid scores", got, probVals[0])+ }+ })}func TestLagunaSigmoidTopK8CompiledMatchesEager(t *testing.T) {- skipIfNoMLX(t)- gates := make([]float32, 2*16)- for i := range gates {- gates[i] = float32((i%13)-6) * 0.2- }- bias := make([]float32, 16)- scaleA := make([]float32, 16)- scaleB := make([]float32, 16)- for i := range bias {- bias[i] = float32((i%5)-2) * 0.03- scaleA[i] = 0.5 + float32(i)*0.01- scaleB[i] = 0.75 + float32(i)*0.005- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ gates := make([]float32, 2*16)+ for i := range gates {+ gates[i] = float32((i%13)-6) * 0.2+ }+ bias := make([]float32, 16)+ scaleA := make([]float32, 16)+ scaleB := make([]float32, 16)+ for i := range bias {+ bias[i] = float32((i%5)-2) * 0.03+ scaleA[i] = 0.5 + float32(i)*0.01+ scaleB[i] = 0.75 + float32(i)*0.005+ }- gatesArray := mlx.FromValues(gates, 2, 16).AsType(mlx.DTypeBFloat16)- biasArray := mlx.FromValues(bias, 16)- scaleAArray := mlx.FromValues(scaleA, 16)- scaleBArray := mlx.FromValues(scaleB, 16)- got := lagunaSigmoidTopK8ScaledNormalized(gatesArray, biasArray, scaleAArray, scaleBArray)-- probs, neg := mlx.SigmoidRouter(gatesArray.AsType(mlx.DTypeFloat32), biasArray)- wantIndices := mlx.Argpartition(neg, 7, -1)- wantIndices = mlx.SliceStartStop(wantIndices, []int32{0, 0}, []int32{2, 8})- wantScores := mlx.TakeAlongAxis(probs, wantIndices, -1)- wantScores = mlx.Div(wantScores, mlx.Sum(wantScores, -1, true))- wantScores = scaleScoresByExpert(wantScores, wantIndices, scaleAArray)- wantScores = scaleScoresByExpert(wantScores, wantIndices, scaleBArray)-- gotScores := got[0].AsType(mlx.DTypeFloat32)- gotIndices := got[1].AsType(mlx.DTypeInt32)- wantScores = wantScores.AsType(mlx.DTypeFloat32)- wantIndices = wantIndices.AsType(mlx.DTypeInt32)- mlx.Eval(gotScores, gotIndices, wantScores, wantIndices)- assertFloatSlicesClose(t, gotScores.Floats(), wantScores.Floats(), 1e-6)- if got, want := gotIndices.Ints(), wantIndices.Ints(); !slices.Equal(got, want) {- t.Fatalf("indices = %v, want %v", got, want)- }+ gatesArray := mlx.FromValues(gates, 2, 16).AsType(mlx.DTypeBFloat16)+ biasArray := mlx.FromValues(bias, 16)+ scaleAArray := mlx.FromValues(scaleA, 16)+ scaleBArray := mlx.FromValues(scaleB, 16)+ got := lagunaSigmoidTopK8ScaledNormalized(gatesArray, biasArray, scaleAArray, scaleBArray)++ probs, neg := mlx.SigmoidRouter(gatesArray.AsType(mlx.DTypeFloat32), biasArray)+ wantIndices := mlx.Argpartition(neg, 7, -1)+ wantIndices = mlx.SliceStartStop(wantIndices, []int32{0, 0}, []int32{2, 8})+ wantScores := mlx.TakeAlongAxis(probs, wantIndices, -1)+ wantScores = mlx.Div(wantScores, mlx.Sum(wantScores, -1, true))+ wantScores = scaleScoresByExpert(wantScores, wantIndices, scaleAArray)+ wantScores = scaleScoresByExpert(wantScores, wantIndices, scaleBArray)++ gotScores := got[0].AsType(mlx.DTypeFloat32)+ gotIndices := got[1].AsType(mlx.DTypeInt32)+ wantScores = wantScores.AsType(mlx.DTypeFloat32)+ wantIndices = wantIndices.AsType(mlx.DTypeInt32)+ mlx.Eval(gotScores, gotIndices, wantScores, wantIndices)+ assertFloatSlicesClose(t, gotScores.Floats(), wantScores.Floats(), 1e-6)+ if got, want := gotIndices.Ints(), wantIndices.Ints(); !slices.Equal(got, want) {+ t.Fatalf("indices = %v, want %v", got, want)+ }+ })}func TestLagunaSwiGLUGatheredGateScaleCompiledMatchesEager(t *testing.T) {- skipIfNoMLX(t)- gateValues := make([]float32, 2*8*4)- upValues := make([]float32, len(gateValues))- for i := range gateValues {- gateValues[i] = float32((i%17)-8) * 0.04- upValues[i] = float32((i%11)-5) * 0.03- }- scaleValues := make([]float32, 16)- for i := range scaleValues {- scaleValues[i] = 0.5 + float32(i)*0.025- }- indices := mlx.FromValues([]int32{- 0, 3, 6, 9, 12, 15, 2, 5,- 1, 4, 7, 10, 13, 14, 8, 11,- }, 2, 8)- gate := mlx.FromValues(gateValues, 2, 8, 1, 4).AsType(mlx.DTypeBFloat16)- up := mlx.FromValues(upValues, 2, 8, 1, 4).AsType(mlx.DTypeBFloat16)- scales := mlx.FromValues(scaleValues, 16)-- got := lagunaSwiGLUGatheredGateScale(gate, up, scales, indices)[0]- want := mlx.SwiGLU(applyExpertGlobalScale(gate, scales, indices), up)- got = got.AsType(mlx.DTypeFloat32)- want = want.AsType(mlx.DTypeFloat32)- mlx.Eval(got, want)- assertFloatSlicesClose(t, got.Floats(), want.Floats(), 1e-6)+ mlxtest.Run(t, func(t *mlxtest.T) {+ gateValues := make([]float32, 2*8*4)+ upValues := make([]float32, len(gateValues))+ for i := range gateValues {+ gateValues[i] = float32((i%17)-8) * 0.04+ upValues[i] = float32((i%11)-5) * 0.03+ }+ scaleValues := make([]float32, 16)+ for i := range scaleValues {+ scaleValues[i] = 0.5 + float32(i)*0.025+ }+ indices := mlx.FromValues([]int32{+ 0, 3, 6, 9, 12, 15, 2, 5,+ 1, 4, 7, 10, 13, 14, 8, 11,+ }, 2, 8)+ gate := mlx.FromValues(gateValues, 2, 8, 1, 4).AsType(mlx.DTypeBFloat16)+ up := mlx.FromValues(upValues, 2, 8, 1, 4).AsType(mlx.DTypeBFloat16)+ scales := mlx.FromValues(scaleValues, 16)++ got := lagunaSwiGLUGatheredGateScale(gate, up, scales, indices)[0]+ want := mlx.SwiGLU(applyExpertGlobalScale(gate, scales, indices), up)+ got = got.AsType(mlx.DTypeFloat32)+ want = want.AsType(mlx.DTypeFloat32)+ mlx.Eval(got, want)+ assertFloatSlicesClose(t, got.Floats(), want.Floats(), 1e-6)+ })}func TestLagunaMoEWeightedSumCompiledMatchesEager(t *testing.T) {- skipIfNoMLX(t)- expertValues := make([]float32, 1*2*8*4)- scoreValues := make([]float32, 1*2*8)- for i := range expertValues {- expertValues[i] = float32((i%19)-9) * 0.02- }- for i := range scoreValues {- scoreValues[i] = float32(i+1) / 20- }- addAValues := []float32{0.1, -0.2, 0.3, -0.4, 0.5, -0.6, 0.7, -0.8}- addBValues := []float32{-0.8, 0.7, -0.6, 0.5, -0.4, 0.3, -0.2, 0.1}- expert := mlx.FromValues(expertValues, 1, 2, 8, 4).AsType(mlx.DTypeBFloat16)- scores := mlx.FromValues(scoreValues, 1, 2, 8)- scale := mlx.FromValue(float32(2.5))- addA := mlx.FromValues(addAValues, 1, 2, 4).AsType(mlx.DTypeBFloat16)- addB := mlx.FromValues(addBValues, 1, 2, 4).AsType(mlx.DTypeBFloat16)-- weighted := mlx.Mul(expert, mlx.ExpandDims(scores.AsType(expert.DType()), -1))- weighted = mlx.Mul(mlx.Sum(weighted, 2, false), scale.AsType(expert.DType()))- wantAdd := mlx.Add(weighted.AsType(addA.DType()), addA)- wantAdd2 := mlx.Add(wantAdd, addB)- gotAdd := lagunaMoEWeightedSumAdd(expert, scores, scale, addA)[0]- gotAdd2 := lagunaMoEWeightedSumAdd2(expert, scores, scale, addA, addB)[0]-- gotAdd = gotAdd.AsType(mlx.DTypeFloat32)- gotAdd2 = gotAdd2.AsType(mlx.DTypeFloat32)- wantAdd = wantAdd.AsType(mlx.DTypeFloat32)- wantAdd2 = wantAdd2.AsType(mlx.DTypeFloat32)- mlx.Eval(gotAdd, gotAdd2, wantAdd, wantAdd2)- assertFloatSlicesClose(t, gotAdd.Floats(), wantAdd.Floats(), 1e-6)- assertFloatSlicesClose(t, gotAdd2.Floats(), wantAdd2.Floats(), 1e-6)+ mlxtest.Run(t, func(t *mlxtest.T) {+ expertValues := make([]float32, 1*2*8*4)+ scoreValues := make([]float32, 1*2*8)+ for i := range expertValues {+ expertValues[i] = float32((i%19)-9) * 0.02+ }+ for i := range scoreValues {+ scoreValues[i] = float32(i+1) / 20+ }+ addAValues := []float32{0.1, -0.2, 0.3, -0.4, 0.5, -0.6, 0.7, -0.8}+ addBValues := []float32{-0.8, 0.7, -0.6, 0.5, -0.4, 0.3, -0.2, 0.1}+ expert := mlx.FromValues(expertValues, 1, 2, 8, 4).AsType(mlx.DTypeBFloat16)+ scores := mlx.FromValues(scoreValues, 1, 2, 8)+ scale := mlx.FromValue(float32(2.5))+ addA := mlx.FromValues(addAValues, 1, 2, 4).AsType(mlx.DTypeBFloat16)+ addB := mlx.FromValues(addBValues, 1, 2, 4).AsType(mlx.DTypeBFloat16)++ weighted := mlx.Mul(expert, mlx.ExpandDims(scores.AsType(expert.DType()), -1))+ weighted = mlx.Mul(mlx.Sum(weighted, 2, false), scale.AsType(expert.DType()))+ wantAdd := mlx.Add(weighted.AsType(addA.DType()), addA)+ wantAdd2 := mlx.Add(wantAdd, addB)+ gotAdd := lagunaMoEWeightedSumAdd(expert, scores, scale, addA)[0]+ gotAdd2 := lagunaMoEWeightedSumAdd2(expert, scores, scale, addA, addB)[0]++ gotAdd = gotAdd.AsType(mlx.DTypeFloat32)+ gotAdd2 = gotAdd2.AsType(mlx.DTypeFloat32)+ wantAdd = wantAdd.AsType(mlx.DTypeFloat32)+ wantAdd2 = wantAdd2.AsType(mlx.DTypeFloat32)+ mlx.Eval(gotAdd, gotAdd2, wantAdd, wantAdd2)+ assertFloatSlicesClose(t, gotAdd.Floats(), wantAdd.Floats(), 1e-6)+ assertFloatSlicesClose(t, gotAdd2.Floats(), wantAdd2.Floats(), 1e-6)+ })}func TestSwitchMLPFusedGateUpMatchesSeparate(t *testing.T) {- skipIfNoMLX(t)- cfg := &Config{HiddenSize: 4, NumExpertsPerTok: 2}- B, L := int32(2), int32(3)- xVals := make([]float32, int(B*L*cfg.HiddenSize))- for i := range xVals {- xVals[i] = float32((i%17)-8) * 0.01- }- x := mlx.FromValues(xVals, int(B), int(L), int(cfg.HiddenSize)).AsType(mlx.DTypeBFloat16)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{HiddenSize: 4, NumExpertsPerTok: 2}+ B, L := int32(2), int32(3)+ xVals := make([]float32, int(B*L*cfg.HiddenSize))+ for i := range xVals {+ xVals[i] = float32((i%17)-8) * 0.01+ }+ x := mlx.FromValues(xVals, int(B), int(L), int(cfg.HiddenSize)).AsType(mlx.DTypeBFloat16)- indicesVals := make([]int32, B*L*cfg.NumExpertsPerTok)- for i := 0; i < len(indicesVals); i += int(cfg.NumExpertsPerTok) {- indicesVals[i] = int32((i / int(cfg.NumExpertsPerTok)) % 2)- indicesVals[i+1] = int32(((i / int(cfg.NumExpertsPerTok)) + 1) % 2)- }- indices := mlx.FromValues(indicesVals, int(B*L), int(cfg.NumExpertsPerTok))+ indicesVals := make([]int32, B*L*cfg.NumExpertsPerTok)+ for i := 0; i < len(indicesVals); i += int(cfg.NumExpertsPerTok) {+ indicesVals[i] = int32((i / int(cfg.NumExpertsPerTok)) % 2)+ indicesVals[i+1] = int32(((i / int(cfg.NumExpertsPerTok)) + 1) % 2)+ }+ indices := mlx.FromValues(indicesVals, int(B*L), int(cfg.NumExpertsPerTok))- separate := &SwitchMLP{- GateWeight: makePatternExpertWeight(2, 4, 3, 0.011),- UpWeight: makePatternExpertWeight(2, 4, 3, 0.017),- DownWeight: makePatternExpertWeight(2, 3, 4, 0.013),- }- fused := &SwitchMLP{- GateUpWeight: fuseExpertStacks(separate.GateWeight, separate.UpWeight, 2),- DownWeight: separate.DownWeight,- }+ separate := &SwitchMLP{+ GateWeight: makePatternExpertWeight(2, 4, 3, 0.011),+ UpWeight: makePatternExpertWeight(2, 4, 3, 0.017),+ DownWeight: makePatternExpertWeight(2, 3, 4, 0.013),+ }+ fused := &SwitchMLP{+ GateUpWeight: fuseExpertStacks(separate.GateWeight, separate.UpWeight, 2),+ DownWeight: separate.DownWeight,+ }- gotSeparate := separate.Forward(x, indices, cfg)- gotFused := fused.Forward(x, indices, cfg)- mlx.Eval(gotSeparate, gotFused)+ gotSeparate := separate.Forward(x, indices, cfg)+ gotFused := fused.Forward(x, indices, cfg)+ mlx.Eval(gotSeparate, gotFused)- gotFusedF32 := gotFused.AsType(mlx.DTypeFloat32)- gotSeparateF32 := gotSeparate.AsType(mlx.DTypeFloat32)- mlx.Eval(gotFusedF32, gotSeparateF32)- assertFloatSlicesClose(t, gotFusedF32.Floats(), gotSeparateF32.Floats(), 1e-5)+ gotFusedF32 := gotFused.AsType(mlx.DTypeFloat32)+ gotSeparateF32 := gotSeparate.AsType(mlx.DTypeFloat32)+ mlx.Eval(gotFusedF32, gotSeparateF32)+ assertFloatSlicesClose(t, gotFusedF32.Floats(), gotSeparateF32.Floats(), 1e-5)+ })}func TestSwitchMLPMixedQuantizedGateUpDenseDownMatchesDense(t *testing.T) {- skipIfNoMLX(t)- cfg := &Config{HiddenSize: 32, NumExpertsPerTok: 2}- x := makePatternExpertWeight(1, 2, int(cfg.HiddenSize), 0.013)- indices := mlx.FromValues([]int32{0, 1, 1, 0}, 2, int(cfg.NumExpertsPerTok))-- gateWeight := makePatternExpertWeight(2, 32, 32, 0.011)- upWeight := makePatternExpertWeight(2, 32, 32, 0.017)- downWeight := makePatternExpertWeight(2, 32, 32, 0.013)- gateQ, gateScales, gateBiases := mlx.Quantize(gateWeight, 32, 8, "mxfp8")- upQ, upScales, upBiases := mlx.Quantize(upWeight, 32, 8, "mxfp8")- mlx.Eval(gateQ, gateScales, upQ, upScales)-- mixed := &SwitchMLP{- GateUpWeightQ: fuseExpertStacks(gateQ, upQ, 1),- GateUpScales: fuseExpertStacks(gateScales, upScales, 1),- GateUpBiases: fuseExpertStacks(gateBiases, upBiases, 1),- GateUpBits: 8,- GateUpGroupSize: 32,- GateUpMode: "mxfp8",- DownWeight: downWeight,- DownWeightSourceLayout: true,- }- dense := &SwitchMLP{- GateWeight: gateWeight,- UpWeight: upWeight,- DownWeight: downWeight,- GateUpWeightsSourceLayout: true,- DownWeightSourceLayout: true,- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{HiddenSize: 32, NumExpertsPerTok: 2}+ x := makePatternExpertWeight(1, 2, int(cfg.HiddenSize), 0.013)+ indices := mlx.FromValues([]int32{0, 1, 1, 0}, 2, int(cfg.NumExpertsPerTok))++ gateWeight := makePatternExpertWeight(2, 32, 32, 0.011)+ upWeight := makePatternExpertWeight(2, 32, 32, 0.017)+ downWeight := makePatternExpertWeight(2, 32, 32, 0.013)+ gateQ, gateScales, gateBiases := mlx.Quantize(gateWeight, 32, 8, "mxfp8")+ upQ, upScales, upBiases := mlx.Quantize(upWeight, 32, 8, "mxfp8")+ mlx.Eval(gateQ, gateScales, upQ, upScales)++ mixed := &SwitchMLP{+ GateUpWeightQ: fuseExpertStacks(gateQ, upQ, 1),+ GateUpScales: fuseExpertStacks(gateScales, upScales, 1),+ GateUpBiases: fuseExpertStacks(gateBiases, upBiases, 1),+ GateUpBits: 8,+ GateUpGroupSize: 32,+ GateUpMode: "mxfp8",+ DownWeight: downWeight,+ DownWeightSourceLayout: true,+ }+ dense := &SwitchMLP{+ GateWeight: gateWeight,+ UpWeight: upWeight,+ DownWeight: downWeight,+ GateUpWeightsSourceLayout: true,+ DownWeightSourceLayout: true,+ }- got := mixed.Forward(x, indices, cfg).AsType(mlx.DTypeFloat32)- want := dense.Forward(x, indices, cfg).AsType(mlx.DTypeFloat32)- mlx.Eval(got, want)- assertFloatSlicesClose(t, got.Floats(), want.Floats(), 0.02)+ got := mixed.Forward(x, indices, cfg).AsType(mlx.DTypeFloat32)+ want := dense.Forward(x, indices, cfg).AsType(mlx.DTypeFloat32)+ mlx.Eval(got, want)+ assertFloatSlicesClose(t, got.Floats(), want.Floats(), 0.02)+ })}func TestDenseExpertWeightForGatherMMDequantizesQuantizedWeight(t *testing.T) {- skipIfNoMLX(t)- weight := makePatternExpertWeight(2, 4, 32, 0.011)- qweight, scales, qbiases := mlx.Quantize(weight, 32, 8, "mxfp8")- mlx.Eval(qweight, scales)-- got := denseExpertWeightForGatherMM(&stackedExpertWeights{- Weight: qweight,- Scales: scales,- Biases: qbiases,- GroupSize: 32,- Bits: 8,- Mode: "mxfp8",+ mlxtest.Run(t, func(t *mlxtest.T) {+ weight := makePatternExpertWeight(2, 4, 32, 0.011)+ qweight, scales, qbiases := mlx.Quantize(weight, 32, 8, "mxfp8")+ mlx.Eval(qweight, scales)++ got := denseExpertWeightForGatherMM(&stackedExpertWeights{+ Weight: qweight,+ Scales: scales,+ Biases: qbiases,+ GroupSize: 32,+ Bits: 8,+ Mode: "mxfp8",+ })+ mlx.Eval(got)++ if got == nil {+ t.Fatal("denseExpertWeightForGatherMM returned nil")+ }+ if dims := got.Dims(); len(dims) != 3 || dims[0] != 2 || dims[1] != 32 || dims[2] != 4 {+ t.Fatalf("dense expert dims = %v, want [2 32 4]", dims)+ }+ if got.DType() == mlx.DTypeUint32 {+ t.Fatal("dense expert fallback kept packed U32 weight")+ }})- mlx.Eval(got)-- if got == nil {- t.Fatal("denseExpertWeightForGatherMM returned nil")- }- if dims := got.Dims(); len(dims) != 3 || dims[0] != 2 || dims[1] != 32 || dims[2] != 4 {- t.Fatalf("dense expert dims = %v, want [2 32 4]", dims)- }- if got.DType() == mlx.DTypeUint32 {- t.Fatal("dense expert fallback kept packed U32 weight")- }}func TestCombinedTensorGlobalScaleIgnoresInputGlobalScale(t *testing.T) {- skipIfNoMLX(t)- tensors := map[string]*mlx.Array{- "proj.weight.global_scale": mlx.FromValues([]float32{0.25}, 1),- "proj.weight.input_global_scale": mlx.FromValues([]float32{8}, 1),- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ tensors := map[string]*mlx.Array{+ "proj.weight.global_scale": mlx.FromValues([]float32{0.25}, 1),+ "proj.weight.input_global_scale": mlx.FromValues([]float32{8}, 1),+ }- got, _ := combinedTensorGlobalScale(tensors, "proj.weight")- if got == nil {- t.Fatal("combinedTensorGlobalScale returned nil")- }- mlx.Eval(got)- vals := got.Floats()- if len(vals) != 1 || vals[0] != 0.25 {- t.Fatalf("combinedTensorGlobalScale = %v, want [0.25]", vals)- }+ got, _ := combinedTensorGlobalScale(tensors, "proj.weight")+ if got == nil {+ t.Fatal("combinedTensorGlobalScale returned nil")+ }+ mlx.Eval(got)+ vals := got.Floats()+ if len(vals) != 1 || vals[0] != 0.25 {+ t.Fatalf("combinedTensorGlobalScale = %v, want [0.25]", vals)+ }+ })}func tinyLagunaTensors() map[string]*mlx.Array {@@ -770,7 +786,7 @@ func makePatternExpertWeight(numExperts, rows, cols int, scale float32) *mlx.Arrreturn makeExpertWeight(vals, numExperts, rows, cols)}-func assertFloatSlicesClose(t *testing.T, got, want []float32, tol float64) {+func assertFloatSlicesClose(t *mlxtest.T, got, want []float32, tol float64) {t.Helper()if len(got) != len(want) {t.Fatalf("length mismatch: got %d want %d", len(got), len(want))@@ -797,10 +813,3 @@ func ones(n int) *mlx.Array {}return mlx.FromValues(vals, n)}--func skipIfNoMLX(t *testing.T) {- t.Helper()- if err := mlx.CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }-}diff --git a/x/models/nemotron_h/nemotron_h_test.go b/x/models/nemotron_h/nemotron_h_test.goindex db2bd3cc9..b1fcf9c3e 100644--- a/x/models/nemotron_h/nemotron_h_test.go+++ b/x/models/nemotron_h/nemotron_h_test.go@@ -323,52 +323,52 @@ func TestSupportsGatherQMM(t *testing.T) {}func TestApplyExpertWeightGlobalScale(t *testing.T) {- mlxtest.Setup(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ weight := mlx.FromValues([]float32{+ 1, 2,+ 3, 4,+ 5, 6,+ 7, 8,+ }, 2, 2, 2)+ scale := mlx.FromValues([]float32{2, 3}, 2)- weight := mlx.FromValues([]float32{- 1, 2,- 3, 4,- 5, 6,- 7, 8,- }, 2, 2, 2)- scale := mlx.FromValues([]float32{2, 3}, 2)+ got := applyExpertWeightGlobalScale(weight, scale)+ mlx.Eval(got)- got := applyExpertWeightGlobalScale(weight, scale)- mlx.Eval(got)-- assertAllClose(t, "scaled expert weight", got.Floats(), []float32{- 2, 4,- 6, 8,- 15, 18,- 21, 24,- }, 1e-5)+ assertAllClose(t, "scaled expert weight", got.Floats(), []float32{+ 2, 4,+ 6, 8,+ 15, 18,+ 21, 24,+ }, 1e-5)+ })}// The production form leans on MLX's fused fast RMS norm, so check it against// a reference that spells out every step.func TestGatedGroupRMSNormMatchesElementwiseReference(t *testing.T) {- mlxtest.Setup(t)-- cfg := &Config{MambaNumHeads: 4, MambaHeadDim: 2, NGroups: 2, LayerNormEpsilon: 1e-5}- inner := cfg.MambaNumHeads * cfg.MambaHeadDim- groupSize := inner / cfg.NGroups- B, L := int32(2), int32(3)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{MambaNumHeads: 4, MambaHeadDim: 2, NGroups: 2, LayerNormEpsilon: 1e-5}+ inner := cfg.MambaNumHeads * cfg.MambaHeadDim+ groupSize := inner / cfg.NGroups+ B, L := int32(2), int32(3)- y := testGatedValues(0.1, int(B), int(L), int(inner))- gate := testGatedValues(-0.2, int(B), int(L), int(inner))- weight := testGatedValues(0.9, int(inner))+ y := testGatedValues(0.1, int(B), int(L), int(inner))+ gate := testGatedValues(-0.2, int(B), int(L), int(inner))+ weight := testGatedValues(0.9, int(inner))- got := gatedGroupRMSNorm(y, gate, weight, cfg, mlx.DTypeFloat32)+ got := gatedGroupRMSNorm(y, gate, weight, cfg, mlx.DTypeFloat32)- ref := mlx.Mul(y, mlx.SiLU(gate))- ref = mlx.Reshape(ref, B, L, cfg.NGroups, groupSize)- variance := mlx.Mean(mlx.Mul(ref, ref), 3, true)- ref = mlx.Mul(ref, mlx.RSqrt(mlx.AddScalar(variance, cfg.LayerNormEpsilon)))- ref = mlx.Mul(ref, mlx.Reshape(weight, 1, 1, cfg.NGroups, groupSize))- ref = mlx.Reshape(ref, B, L, inner)+ ref := mlx.Mul(y, mlx.SiLU(gate))+ ref = mlx.Reshape(ref, B, L, cfg.NGroups, groupSize)+ variance := mlx.Mean(mlx.Mul(ref, ref), 3, true)+ ref = mlx.Mul(ref, mlx.RSqrt(mlx.AddScalar(variance, cfg.LayerNormEpsilon)))+ ref = mlx.Mul(ref, mlx.Reshape(weight, 1, 1, cfg.NGroups, groupSize))+ ref = mlx.Reshape(ref, B, L, inner)- mlx.Eval(got, ref)- assertAllClose(t, "gated group rmsnorm", got.Floats(), ref.Floats(), 1e-5)+ mlx.Eval(got, ref)+ assertAllClose(t, "gated group rmsnorm", got.Floats(), ref.Floats(), 1e-5)+ })}func testGatedValues(seed float32, shape ...int) *mlx.Array {@@ -383,7 +383,7 @@ func testGatedValues(seed float32, shape ...int) *mlx.Array {return mlx.FromValues(vals, shape...)}-func assertAllClose(t *testing.T, name string, got, want []float32, tol float64) {+func assertAllClose(t *mlxtest.T, name string, got, want []float32, tol float64) {t.Helper()if len(got) != len(want) {t.Fatalf("%s length = %d, want %d", name, len(got), len(want))diff --git a/x/models/nn/nn_test.go b/x/models/nn/nn_test.goindex 9cba327bc..b9c7ca339 100644--- a/x/models/nn/nn_test.go+++ b/x/models/nn/nn_test.go@@ -8,181 +8,176 @@ import ("github.com/ollama/ollama/x/mlxrunner/mlx")-func skipIfNoMLX(t *testing.T) {- t.Helper()- mlxtest.Setup(t)-}-func approxEqual(a, b, tol float32) bool {return float32(math.Abs(float64(a-b))) < tol}// TestLayerNormNoBias verifies LayerNorm without bias against manual computation.func TestLayerNormNoBias(t *testing.T) {- skipIfNoMLX(t)-- // Input: [1, 4] — single row, 4 features- x := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 4)- weight := mlx.FromValues([]float32{1, 1, 1, 1}, 4)- mlx.Eval(x, weight)-- ln := &LayerNorm{Weight: weight, Eps: 1e-5}- out := ln.Forward(x)- mlx.Eval(out)-- data := out.Floats()- if len(data) != 4 {- t.Fatalf("expected 4 values, got %d", len(data))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Input: [1, 4] — single row, 4 features+ x := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 4)+ weight := mlx.FromValues([]float32{1, 1, 1, 1}, 4)+ mlx.Eval(x, weight)++ ln := &LayerNorm{Weight: weight, Eps: 1e-5}+ out := ln.Forward(x)+ mlx.Eval(out)++ data := out.Floats()+ if len(data) != 4 {+ t.Fatalf("expected 4 values, got %d", len(data))+ }- // Manual LayerNorm: mean=2.5, var=1.25, std=sqrt(1.25+1e-5)- // normalized = (x - mean) / std- mean := float32(2.5)- variance := float32(1.25)- std := float32(math.Sqrt(float64(variance + 1e-5)))- for i, v := range []float32{1, 2, 3, 4} {- expected := (v - mean) / std- if !approxEqual(data[i], expected, 1e-4) {- t.Errorf("index %d: expected %.6f, got %.6f", i, expected, data[i])+ // Manual LayerNorm: mean=2.5, var=1.25, std=sqrt(1.25+1e-5)+ // normalized = (x - mean) / std+ mean := float32(2.5)+ variance := float32(1.25)+ std := float32(math.Sqrt(float64(variance + 1e-5)))+ for i, v := range []float32{1, 2, 3, 4} {+ expected := (v - mean) / std+ if !approxEqual(data[i], expected, 1e-4) {+ t.Errorf("index %d: expected %.6f, got %.6f", i, expected, data[i])+ }}- }+ })}// TestLayerNormWithBias verifies LayerNorm with weight and bias.func TestLayerNormWithBias(t *testing.T) {- skipIfNoMLX(t)-- x := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 4)- weight := mlx.FromValues([]float32{2, 2, 2, 2}, 4)- bias := mlx.FromValues([]float32{10, 20, 30, 40}, 4)- mlx.Eval(x, weight, bias)-- ln := &LayerNorm{Weight: weight, Bias: bias, Eps: 1e-5}- out := ln.Forward(x)- mlx.Eval(out)-- data := out.Floats()- if len(data) != 4 {- t.Fatalf("expected 4 values, got %d", len(data))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ x := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 4)+ weight := mlx.FromValues([]float32{2, 2, 2, 2}, 4)+ bias := mlx.FromValues([]float32{10, 20, 30, 40}, 4)+ mlx.Eval(x, weight, bias)++ ln := &LayerNorm{Weight: weight, Bias: bias, Eps: 1e-5}+ out := ln.Forward(x)+ mlx.Eval(out)++ data := out.Floats()+ if len(data) != 4 {+ t.Fatalf("expected 4 values, got %d", len(data))+ }- mean := float32(2.5)- variance := float32(1.25)- std := float32(math.Sqrt(float64(variance + 1e-5)))- biases := []float32{10, 20, 30, 40}- for i, v := range []float32{1, 2, 3, 4} {- expected := ((v-mean)/std)*2 + biases[i]- if !approxEqual(data[i], expected, 1e-4) {- t.Errorf("index %d: expected %.6f, got %.6f", i, expected, data[i])+ mean := float32(2.5)+ variance := float32(1.25)+ std := float32(math.Sqrt(float64(variance + 1e-5)))+ biases := []float32{10, 20, 30, 40}+ for i, v := range []float32{1, 2, 3, 4} {+ expected := ((v-mean)/std)*2 + biases[i]+ if !approxEqual(data[i], expected, 1e-4) {+ t.Errorf("index %d: expected %.6f, got %.6f", i, expected, data[i])+ }}- }+ })}// TestLayerNormBatched verifies LayerNorm normalizes each row independently.func TestLayerNormBatched(t *testing.T) {- skipIfNoMLX(t)-- // Input: [2, 3] — two rows- x := mlx.FromValues([]float32{- 1, 2, 3,- 10, 20, 30,- }, 2, 3)- weight := mlx.FromValues([]float32{1, 1, 1}, 3)- mlx.Eval(x, weight)-- ln := &LayerNorm{Weight: weight, Eps: 1e-5}- out := ln.Forward(x)- mlx.Eval(out)-- data := out.Floats()- if len(data) != 6 {- t.Fatalf("expected 6 values, got %d", len(data))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Input: [2, 3] — two rows+ x := mlx.FromValues([]float32{+ 1, 2, 3,+ 10, 20, 30,+ }, 2, 3)+ weight := mlx.FromValues([]float32{1, 1, 1}, 3)+ mlx.Eval(x, weight)++ ln := &LayerNorm{Weight: weight, Eps: 1e-5}+ out := ln.Forward(x)+ mlx.Eval(out)++ data := out.Floats()+ if len(data) != 6 {+ t.Fatalf("expected 6 values, got %d", len(data))+ }- // Each row should be independently normalized.- // Row 0: [1,2,3] mean=2, var=2/3- // Row 1: [10,20,30] mean=20, var=200/3- // After normalization both rows should have the same pattern- // since [10,20,30] = 10*[1,2,3], the normalized values are identical.- for i := range 3 {- if !approxEqual(data[i], data[i+3], 1e-4) {- t.Errorf("row 0 elem %d (%.6f) != row 1 elem %d (%.6f); expected identical normalized values",- i, data[i], i, data[i+3])+ // Each row should be independently normalized.+ // Row 0: [1,2,3] mean=2, var=2/3+ // Row 1: [10,20,30] mean=20, var=200/3+ // After normalization both rows should have the same pattern+ // since [10,20,30] = 10*[1,2,3], the normalized values are identical.+ for i := range 3 {+ if !approxEqual(data[i], data[i+3], 1e-4) {+ t.Errorf("row 0 elem %d (%.6f) != row 1 elem %d (%.6f); expected identical normalized values",+ i, data[i], i, data[i+3])+ }}- }- // Verify the normalized values sum to ~0 (mean-centered)- sum := data[0] + data[1] + data[2]- if !approxEqual(sum, 0, 1e-4) {- t.Errorf("normalized row sum should be ~0, got %.6f", sum)- }+ // Verify the normalized values sum to ~0 (mean-centered)+ sum := data[0] + data[1] + data[2]+ if !approxEqual(sum, 0, 1e-4) {+ t.Errorf("normalized row sum should be ~0, got %.6f", sum)+ }+ })}// TestLayerNormDefaultEps verifies the default epsilon of 1e-5 is used when Eps is 0.func TestLayerNormDefaultEps(t *testing.T) {- skipIfNoMLX(t)-- x := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 4)- weight := mlx.FromValues([]float32{1, 1, 1, 1}, 4)- mlx.Eval(x, weight)-- // Eps=0 should use default 1e-5- ln0 := &LayerNorm{Weight: weight, Eps: 0}- out0 := ln0.Forward(x)- mlx.Eval(out0)-- lnExplicit := &LayerNorm{Weight: weight, Eps: 1e-5}- outExplicit := lnExplicit.Forward(x)- mlx.Eval(outExplicit)-- d0 := out0.Floats()- dE := outExplicit.Floats()- for i := range d0 {- if !approxEqual(d0[i], dE[i], 1e-6) {- t.Errorf("index %d: Eps=0 gave %.6f, Eps=1e-5 gave %.6f", i, d0[i], dE[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ x := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 4)+ weight := mlx.FromValues([]float32{1, 1, 1, 1}, 4)+ mlx.Eval(x, weight)++ // Eps=0 should use default 1e-5+ ln0 := &LayerNorm{Weight: weight, Eps: 0}+ out0 := ln0.Forward(x)+ mlx.Eval(out0)++ lnExplicit := &LayerNorm{Weight: weight, Eps: 1e-5}+ outExplicit := lnExplicit.Forward(x)+ mlx.Eval(outExplicit)++ d0 := out0.Floats()+ dE := outExplicit.Floats()+ for i := range d0 {+ if !approxEqual(d0[i], dE[i], 1e-6) {+ t.Errorf("index %d: Eps=0 gave %.6f, Eps=1e-5 gave %.6f", i, d0[i], dE[i])+ }}- }+ })}func TestQuantizedLinearMXFP4MatchesDequantizedWeight(t *testing.T) {- skipIfNoMLX(t)-- weightVals := make([]float32, 3*32)- for i := range weightVals {- weightVals[i] = float32((i%11)-5) / 7- }- inputVals := make([]float32, 2*32)- for i := range inputVals {- inputVals[i] = float32((i%7)-3) / 5- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ weightVals := make([]float32, 3*32)+ for i := range weightVals {+ weightVals[i] = float32((i%11)-5) / 7+ }+ inputVals := make([]float32, 2*32)+ for i := range inputVals {+ inputVals[i] = float32((i%7)-3) / 5+ }- weight := mlx.FromValues(weightVals, 3, 32).AsType(mlx.DTypeBFloat16)- input := mlx.FromValues(inputVals, 2, 32).AsType(mlx.DTypeBFloat16)- mlx.Eval(weight, input)+ weight := mlx.FromValues(weightVals, 3, 32).AsType(mlx.DTypeBFloat16)+ input := mlx.FromValues(inputVals, 2, 32).AsType(mlx.DTypeBFloat16)+ mlx.Eval(weight, input)- ql := NewQuantizedLinear(weight, nil, 32, 4, "mxfp4")- if ql.QBiases != nil {- t.Fatalf("mxfp4 qbiases = %v, want nil", ql.QBiases)- }+ ql := NewQuantizedLinear(weight, nil, 32, 4, "mxfp4")+ if ql.QBiases != nil {+ t.Fatalf("mxfp4 qbiases = %v, want nil", ql.QBiases)+ }- dequantizedWeight := mlx.Dequantize(ql.Weight, ql.Scales, ql.QBiases, 32, 4, "mxfp4", nil)- mlx.Eval(dequantizedWeight)+ dequantizedWeight := mlx.Dequantize(ql.Weight, ql.Scales, ql.QBiases, 32, 4, "mxfp4", nil)+ mlx.Eval(dequantizedWeight)- qOut := ql.Forward(input).AsType(mlx.DTypeFloat32)- dOut := NewLinear(dequantizedWeight, nil).Forward(input).AsType(mlx.DTypeFloat32)- mlx.Eval(qOut, dOut)+ qOut := ql.Forward(input).AsType(mlx.DTypeFloat32)+ dOut := NewLinear(dequantizedWeight, nil).Forward(input).AsType(mlx.DTypeFloat32)+ mlx.Eval(qOut, dOut)- got := qOut.Floats()- want := dOut.Floats()- if len(got) != len(want) {- t.Fatalf("output length = %d, want %d", len(got), len(want))- }+ got := qOut.Floats()+ want := dOut.Floats()+ if len(got) != len(want) {+ t.Fatalf("output length = %d, want %d", len(got), len(want))+ }- for i := range got {- if !approxEqual(got[i], want[i], 1e-3) {- t.Fatalf("output[%d] = %.6f, want %.6f", i, got[i], want[i])+ for i := range got {+ if !approxEqual(got[i], want[i], 1e-3) {+ t.Fatalf("output[%d] = %.6f, want %.6f", i, got[i], want[i])+ }}- }+ })}func TestQuantizedEmbeddingAsLinearPreservesGlobalScale(t *testing.T) {diff --git a/x/models/nn/recurrent_test.go b/x/models/nn/recurrent_test.goindex 11f10dc1f..2860ecb0a 100644--- a/x/models/nn/recurrent_test.go+++ b/x/models/nn/recurrent_test.go@@ -4,6 +4,7 @@ import ("math""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/mlx")@@ -36,30 +37,31 @@ func convFromKernel(w *mlx.Array) *Conv1d {// Guards a biased conv silently losing the fused kernel: depthwiseConvWeight// returning nil sends WithConvSiLU down separate graph ops.func TestCausalConv1DBiasTakesFusedPath(t *testing.T) {- skipIfNoMLX(t)- B, L, D, convTail := 2, 3, 4, 2- K := convTail + 1+ mlxtest.Run(t, func(t *mlxtest.T) {+ B, L, D, convTail := 2, 3, 4, 2+ K := convTail + 1- weight := fromValues(-0.3, D, K)- bias := fromValues(0.4, D)- conv := NewConv1d(mlx.ExpandDims(weight, 2), bias, 1, 0, 1, int32(D))+ weight := fromValues(-0.3, D, K)+ bias := fromValues(0.4, D)+ conv := NewConv1d(mlx.ExpandDims(weight, 2), bias, 1, 0, 1, int32(D))- if depthwiseConvWeight(conv) == nil {- t.Fatal("depthwiseConvWeight = nil for a biased depthwise conv, so the fused path is skipped")- }+ if depthwiseConvWeight(conv) == nil {+ t.Fatal("depthwiseConvWeight = nil for a biased depthwise conv, so the fused path is skipped")+ }- prior := fromValues(0.2, B, convTail, D)- input := fromValues(0.1, B, L, D)- b := &batch.Batch{- InputIDs: mlx.Zeros(mlx.DTypeInt32, B, L),- SeqOffsets: []int32{0, 0},- SeqQueryLens: []int32{int32(L), int32(L)},- }+ prior := fromValues(0.2, B, convTail, D)+ input := fromValues(0.1, B, L, D)+ b := &batch.Batch{+ InputIDs: mlx.Zeros(mlx.DTypeInt32, B, L),+ SeqOffsets: []int32{0, 0},+ SeqQueryLens: []int32{int32(L), int32(L)},+ }- got, _ := CausalConv1D(b, input, conv, convTail, WithRecurrentState(prior, nil), WithConvSiLU())- want := mlx.SiLU(conv.Forward(mlx.Concatenate([]*mlx.Array{prior, input}, 1)))- mlx.Eval(got, want)- floatsClose(t, "biased fused conv+silu", got.Floats(), want.Floats(), 1e-5)+ got, _ := CausalConv1D(b, input, conv, convTail, WithRecurrentState(prior, nil), WithConvSiLU())+ want := mlx.SiLU(conv.Forward(mlx.Concatenate([]*mlx.Array{prior, input}, 1)))+ mlx.Eval(got, want)+ floatsClose(t, "biased fused conv+silu", got.Floats(), want.Floats(), 1e-5)+ })}// TestCausalConv1DPaddedRowParity drives a B=2 batch with one short@@ -68,103 +70,104 @@ func TestCausalConv1DBiasTakesFusedPath(t *testing.T) {// short row must be the row's last convTail real positions (not the// padded tail), (c) the full row must be unaffected.func TestCausalConv1DPaddedRowParity(t *testing.T) {- skipIfNoMLX(t)- L, D, convTail := 4, 3, 2- qLenShort := 2- K := convTail + 1-- weight := fromValues(0.2, D, K)- conv := convFromKernel(weight)- priorFull := fromValues(0.5, 2, convTail, D)- priorShort := mlx.SliceStartStop(priorFull,- []int32{1, 0, 0},- []int32{2, int32(convTail), int32(D)})-- // Pad row 1 with arbitrary values past qLenShort — the wrapper- // must zero them before convolving. Distinct values let us catch- // any leak.- inputFull := fromValues(1.0, 1, L, D)- inputShortReal := mlx.FromValues([]float32{- 2.0, 2.1, 2.2,- 2.3, 2.4, 2.5,- }, 1, qLenShort, D)- inputShortPad := mlx.FromValues([]float32{- 99, 99, 99,- 99, 99, 99,- }, 1, L-qLenShort, D)- inputShortFull := mlx.Concatenate([]*mlx.Array{inputShortReal, inputShortPad}, 1)- input := mlx.Concatenate([]*mlx.Array{inputFull, inputShortFull}, 0)-- b := &batch.Batch{- InputIDs: mlx.Zeros(mlx.DTypeInt32, 2, L),- SeqOffsets: []int32{0, 0},- SeqQueryLens: []int32{int32(L), int32(qLenShort)},- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, D, convTail := 4, 3, 2+ qLenShort := 2+ K := convTail + 1++ weight := fromValues(0.2, D, K)+ conv := convFromKernel(weight)+ priorFull := fromValues(0.5, 2, convTail, D)+ priorShort := mlx.SliceStartStop(priorFull,+ []int32{1, 0, 0},+ []int32{2, int32(convTail), int32(D)})++ // Pad row 1 with arbitrary values past qLenShort — the wrapper+ // must zero them before convolving. Distinct values let us catch+ // any leak.+ inputFull := fromValues(1.0, 1, L, D)+ inputShortReal := mlx.FromValues([]float32{+ 2.0, 2.1, 2.2,+ 2.3, 2.4, 2.5,+ }, 1, qLenShort, D)+ inputShortPad := mlx.FromValues([]float32{+ 99, 99, 99,+ 99, 99, 99,+ }, 1, L-qLenShort, D)+ inputShortFull := mlx.Concatenate([]*mlx.Array{inputShortReal, inputShortPad}, 1)+ input := mlx.Concatenate([]*mlx.Array{inputFull, inputShortFull}, 0)++ b := &batch.Batch{+ InputIDs: mlx.Zeros(mlx.DTypeInt32, 2, L),+ SeqOffsets: []int32{0, 0},+ SeqQueryLens: []int32{int32(L), int32(qLenShort)},+ }- out, convStates := CausalConv1D(b, input, conv, convTail, WithRecurrentState(priorFull, nil))- nextConv := lastState(convStates)- mlx.Eval(out, nextConv)-- // Reference for row 0: B=1 unpadded length-L call.- refOut0, refConvStates0 := CausalConv1D(&batch.Batch{},- inputFull, conv, convTail,- WithRecurrentState(mlx.SliceStartStop(priorFull,- []int32{0, 0, 0},- []int32{1, int32(convTail), int32(D)}), nil))- refNextConv0 := lastState(refConvStates0)- // Reference for row 1: B=1 unpadded length-qLenShort call.- refOut1, refConvStates1 := CausalConv1D(&batch.Batch{},- inputShortReal, conv, convTail,- WithRecurrentState(priorShort, nil))- refNextConv1 := lastState(refConvStates1)- mlx.Eval(refOut0, refNextConv0, refOut1, refNextConv1)-- gotOut := out.Floats()- wantOut0 := refOut0.Floats()- wantOut1 := refOut1.Floats()-- for q := range L {- for d := range D {- i := q*D + d- if gotOut[i] != wantOut0[i] {- t.Fatalf("row 0 out[q=%d,d=%d]: got %v, want %v", q, d, gotOut[i], wantOut0[i])+ out, convStates := CausalConv1D(b, input, conv, convTail, WithRecurrentState(priorFull, nil))+ nextConv := lastState(convStates)+ mlx.Eval(out, nextConv)++ // Reference for row 0: B=1 unpadded length-L call.+ refOut0, refConvStates0 := CausalConv1D(&batch.Batch{},+ inputFull, conv, convTail,+ WithRecurrentState(mlx.SliceStartStop(priorFull,+ []int32{0, 0, 0},+ []int32{1, int32(convTail), int32(D)}), nil))+ refNextConv0 := lastState(refConvStates0)+ // Reference for row 1: B=1 unpadded length-qLenShort call.+ refOut1, refConvStates1 := CausalConv1D(&batch.Batch{},+ inputShortReal, conv, convTail,+ WithRecurrentState(priorShort, nil))+ refNextConv1 := lastState(refConvStates1)+ mlx.Eval(refOut0, refNextConv0, refOut1, refNextConv1)++ gotOut := out.Floats()+ wantOut0 := refOut0.Floats()+ wantOut1 := refOut1.Floats()++ for q := range L {+ for d := range D {+ i := q*D + d+ if gotOut[i] != wantOut0[i] {+ t.Fatalf("row 0 out[q=%d,d=%d]: got %v, want %v", q, d, gotOut[i], wantOut0[i])+ }}}- }- for q := range qLenShort {- for d := range D {- gotI := L*D + q*D + d- refI := q*D + d- if math.Abs(float64(gotOut[gotI]-wantOut1[refI])) > 1e-5 {- t.Fatalf("row 1 real out[q=%d,d=%d]: got %v, want %v", q, d, gotOut[gotI], wantOut1[refI])+ for q := range qLenShort {+ for d := range D {+ gotI := L*D + q*D + d+ refI := q*D + d+ if math.Abs(float64(gotOut[gotI]-wantOut1[refI])) > 1e-5 {+ t.Fatalf("row 1 real out[q=%d,d=%d]: got %v, want %v", q, d, gotOut[gotI], wantOut1[refI])+ }}}- }- // nextConv: row 0 unaffected, row 1 must be the row's real tail- // (positions [qLenShort - convTail, qLenShort) of the per-row- // concat, i.e. the last two real input rows in this setup).- gotTail := nextConv.Floats()- wantTail0 := refNextConv0.Floats()- wantTail1 := refNextConv1.Floats()- for k := range convTail {- for d := range D {- i := k*D + d- if gotTail[i] != wantTail0[i] {- t.Fatalf("row 0 nextConv[k=%d,d=%d]: got %v, want %v", k, d, gotTail[i], wantTail0[i])+ // nextConv: row 0 unaffected, row 1 must be the row's real tail+ // (positions [qLenShort - convTail, qLenShort) of the per-row+ // concat, i.e. the last two real input rows in this setup).+ gotTail := nextConv.Floats()+ wantTail0 := refNextConv0.Floats()+ wantTail1 := refNextConv1.Floats()+ for k := range convTail {+ for d := range D {+ i := k*D + d+ if gotTail[i] != wantTail0[i] {+ t.Fatalf("row 0 nextConv[k=%d,d=%d]: got %v, want %v", k, d, gotTail[i], wantTail0[i])+ }}}- }- for k := range convTail {- for d := range D {- gotI := convTail*D + k*D + d- refI := k*D + d- if gotTail[gotI] != wantTail1[refI] {- t.Fatalf("row 1 nextConv[k=%d,d=%d]: got %v, want %v (must come from real positions, not the padded tail)",- k, d, gotTail[gotI], wantTail1[refI])+ for k := range convTail {+ for d := range D {+ gotI := convTail*D + k*D + d+ refI := k*D + d+ if gotTail[gotI] != wantTail1[refI] {+ t.Fatalf("row 1 nextConv[k=%d,d=%d]: got %v, want %v (must come from real positions, not the padded tail)",+ k, d, gotTail[gotI], wantTail1[refI])+ }}}- }+ })}// gatedDeltaPackedInputs builds deterministic packed conv-output and@@ -194,7 +197,7 @@ func slicePrefix(a *mlx.Array, lo, hi, n int32) *mlx.Array {}// floatsClose compares two flat float slices within tolerance.-func floatsClose(t *testing.T, label string, got, want []float32, tol float64) {+func floatsClose(t *mlxtest.T, label string, got, want []float32, tol float64) {t.Helper()if len(got) != len(want) {t.Fatalf("%s: len %d, want %d", label, len(got), len(want))@@ -212,82 +215,84 @@ func floatsClose(t *testing.T, label string, got, want []float32, tol float64) {// that each boundary state equals the single-shot state over the// corresponding prefix.func TestGatedDeltaSegmentEquivalence(t *testing.T) {- skipIfNoMLX(t)- B, T, Hk, Dk, Hv, Dv := 1, 5, 1, 32, 1, 32- packed, ba, dtBias, aExp := gatedDeltaPackedInputs(B, T, Hk, Dk, Hv, Dv)- prior := mlx.Zeros(mlx.DTypeFloat32, B, Hv, Dv, Dk)- full := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{int32(T)}}-- refOut, refStates := GatedDelta(full, packed, ba, dtBias, aExp, WithRecurrentState(nil, prior))- if len(refStates) != 1 {- t.Fatalf("unsegmented call returned %d states, want 1", len(refStates))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ B, T, Hk, Dk, Hv, Dv := 1, 5, 1, 32, 1, 32+ packed, ba, dtBias, aExp := gatedDeltaPackedInputs(B, T, Hk, Dk, Hv, Dv)+ prior := mlx.Zeros(mlx.DTypeFloat32, B, Hv, Dv, Dk)+ full := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{int32(T)}}++ refOut, refStates := GatedDelta(full, packed, ba, dtBias, aExp, WithRecurrentState(nil, prior))+ if len(refStates) != 1 {+ t.Fatalf("unsegmented call returned %d states, want 1", len(refStates))+ }- cases := []struct {- name string- splits []int- }{- {"perToken", []int{1, 2, 3, 4}},- {"sparse", []int{2}},- }- for _, tc := range cases {- segOut, segStates := GatedDelta(full, packed, ba, dtBias, aExp,- WithRecurrentState(nil, prior), WithSnapshotSplits(tc.splits))- mlx.Eval(refOut, segOut)- floatsClose(t, tc.name+" out", segOut.Floats(), refOut.Floats(), 1e-4)- if len(segStates) != len(tc.splits)+1 {- t.Fatalf("%s: got %d boundary states, want %d", tc.name, len(segStates), len(tc.splits)+1)+ cases := []struct {+ name string+ splits []int+ }{+ {"perToken", []int{1, 2, 3, 4}},+ {"sparse", []int{2}},}- boundaries := append(append([]int{}, tc.splits...), T)- for i, n := range boundaries {- _, want, _ := mlx.GatedDelta(- slicePrefix(packed, 0, 1, int32(n)), slicePrefix(ba, 0, 1, int32(n)),- dtBias, aExp, prior, nil, false)- mlx.Eval(segStates[i], want)- floatsClose(t, tc.name+" boundary delta", segStates[i].Floats(), want.Floats(), 1e-4)+ for _, tc := range cases {+ segOut, segStates := GatedDelta(full, packed, ba, dtBias, aExp,+ WithRecurrentState(nil, prior), WithSnapshotSplits(tc.splits))+ mlx.Eval(refOut, segOut)+ floatsClose(t, tc.name+" out", segOut.Floats(), refOut.Floats(), 1e-4)+ if len(segStates) != len(tc.splits)+1 {+ t.Fatalf("%s: got %d boundary states, want %d", tc.name, len(segStates), len(tc.splits)+1)+ }+ boundaries := append(append([]int{}, tc.splits...), T)+ for i, n := range boundaries {+ _, want, _ := mlx.GatedDelta(+ slicePrefix(packed, 0, 1, int32(n)), slicePrefix(ba, 0, 1, int32(n)),+ dtBias, aExp, prior, nil, false)+ mlx.Eval(segStates[i], want)+ floatsClose(t, tc.name+" boundary delta", segStates[i].Floats(), want.Floats(), 1e-4)+ }}- }+ })}// TestCausalConv1DSegmentEquivalence checks the conv segmented path matches the// single-shot conv for output, final conv tail, and each boundary conv state.func TestCausalConv1DSegmentEquivalence(t *testing.T) {- skipIfNoMLX(t)- B, L, D, convTail := 1, 4, 3, 2- K := convTail + 1+ mlxtest.Run(t, func(t *mlxtest.T) {+ B, L, D, convTail := 1, 4, 3, 2+ K := convTail + 1- input := fromValues(0.5, B, L, D)- prior := fromValues(-0.3, B, convTail, D)- weight := fromValues(0.2, D, K)- conv := convFromKernel(weight)+ input := fromValues(0.5, B, L, D)+ prior := fromValues(-0.3, B, convTail, D)+ weight := fromValues(0.2, D, K)+ conv := convFromKernel(weight)- full := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{int32(L)}}+ full := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{int32(L)}}- refOut, refStates := CausalConv1D(full, input, conv, convTail, WithRecurrentState(prior, nil))- if len(refStates) != 1 {- t.Fatalf("unsegmented call returned %d states, want 1", len(refStates))- }+ refOut, refStates := CausalConv1D(full, input, conv, convTail, WithRecurrentState(prior, nil))+ if len(refStates) != 1 {+ t.Fatalf("unsegmented call returned %d states, want 1", len(refStates))+ }- segOut, segStates := CausalConv1D(full, input, conv, convTail,- WithRecurrentState(prior, nil), WithSnapshotSplits([]int{1, 2, 3}))- mlx.Eval(refOut, segOut)+ segOut, segStates := CausalConv1D(full, input, conv, convTail,+ WithRecurrentState(prior, nil), WithSnapshotSplits([]int{1, 2, 3}))+ mlx.Eval(refOut, segOut)- floatsClose(t, "conv out", segOut.Floats(), refOut.Floats(), 1e-4)- if len(segStates) != 4 {- t.Fatalf("got %d boundary conv states, want 4", len(segStates))- }- mlx.Eval(lastState(segStates), lastState(refStates))- floatsClose(t, "conv final", lastState(segStates).Floats(), lastState(refStates).Floats(), 1e-4)-- for i := range segStates {- n := int32(i + 1)- pb := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{n}}- _, want := CausalConv1D(pb,- mlx.SliceStartStop(input, []int32{0, 0, 0}, []int32{int32(B), n, int32(D)}),- conv, convTail, WithRecurrentState(prior, nil))- mlx.Eval(segStates[i], lastState(want))- floatsClose(t, "boundary conv", segStates[i].Floats(), lastState(want).Floats(), 1e-4)- }+ floatsClose(t, "conv out", segOut.Floats(), refOut.Floats(), 1e-4)+ if len(segStates) != 4 {+ t.Fatalf("got %d boundary conv states, want 4", len(segStates))+ }+ mlx.Eval(lastState(segStates), lastState(refStates))+ floatsClose(t, "conv final", lastState(segStates).Floats(), lastState(refStates).Floats(), 1e-4)++ for i := range segStates {+ n := int32(i + 1)+ pb := &batch.Batch{SeqOffsets: []int32{0}, SeqQueryLens: []int32{n}}+ _, want := CausalConv1D(pb,+ mlx.SliceStartStop(input, []int32{0, 0, 0}, []int32{int32(B), n, int32(D)}),+ conv, convTail, WithRecurrentState(prior, nil))+ mlx.Eval(segStates[i], lastState(want))+ floatsClose(t, "boundary conv", segStates[i].Floats(), lastState(want).Floats(), 1e-4)+ }+ })}// TestGatedDeltaSegmentEquivalenceBatched checks split forwards match the@@ -296,57 +301,58 @@ func TestCausalConv1DSegmentEquivalence(t *testing.T) {// row's padded positions so a short row's boundary state freezes at its// real end.func TestGatedDeltaSegmentEquivalenceBatched(t *testing.T) {- skipIfNoMLX(t)- B, T, Hk, Dk, Hv, Dv := 2, 4, 1, 32, 1, 32- packed, ba, dtBias, aExp := gatedDeltaPackedInputs(B, T, Hk, Dk, Hv, Dv)- prior := mlx.Zeros(mlx.DTypeFloat32, B, Hv, Dv, Dk)-- // Row 0 full length T; row 1 ends at 3 (so segment [3,4) is all padding- // for row 1).- rowReal := []int32{int32(T), 3}- full := &batch.Batch{- InputIDs: mlx.Zeros(mlx.DTypeInt32, B, T),- SeqOffsets: []int32{0, 0},- SeqQueryLens: rowReal,- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ B, T, Hk, Dk, Hv, Dv := 2, 4, 1, 32, 1, 32+ packed, ba, dtBias, aExp := gatedDeltaPackedInputs(B, T, Hk, Dk, Hv, Dv)+ prior := mlx.Zeros(mlx.DTypeFloat32, B, Hv, Dv, Dk)++ // Row 0 full length T; row 1 ends at 3 (so segment [3,4) is all padding+ // for row 1).+ rowReal := []int32{int32(T), 3}+ full := &batch.Batch{+ InputIDs: mlx.Zeros(mlx.DTypeInt32, B, T),+ SeqOffsets: []int32{0, 0},+ SeqQueryLens: rowReal,+ }- refOut, refStates := GatedDelta(full, packed, ba, dtBias, aExp, WithRecurrentState(nil, prior))+ refOut, refStates := GatedDelta(full, packed, ba, dtBias, aExp, WithRecurrentState(nil, prior))- cases := []struct {- name string- splits []int- }{- {"perToken", []int{1, 2, 3}},- {"sparse", []int{2}},- }- for _, tc := range cases {- segOut, segStates := GatedDelta(full, packed, ba, dtBias, aExp,- WithRecurrentState(nil, prior), WithSnapshotSplits(tc.splits))- mlx.Eval(refOut, segOut, lastState(refStates), lastState(segStates))- floatsClose(t, tc.name+" batched out", segOut.Floats(), refOut.Floats(), 1e-4)- floatsClose(t, tc.name+" batched final state", lastState(segStates).Floats(), lastState(refStates).Floats(), 1e-4)- if len(segStates) != len(tc.splits)+1 {- t.Fatalf("%s: got %d boundary states, want %d", tc.name, len(segStates), len(tc.splits)+1)+ cases := []struct {+ name string+ splits []int+ }{+ {"perToken", []int{1, 2, 3}},+ {"sparse", []int{2}},}+ for _, tc := range cases {+ segOut, segStates := GatedDelta(full, packed, ba, dtBias, aExp,+ WithRecurrentState(nil, prior), WithSnapshotSplits(tc.splits))+ mlx.Eval(refOut, segOut, lastState(refStates), lastState(segStates))+ floatsClose(t, tc.name+" batched out", segOut.Floats(), refOut.Floats(), 1e-4)+ floatsClose(t, tc.name+" batched final state", lastState(segStates).Floats(), lastState(refStates).Floats(), 1e-4)+ if len(segStates) != len(tc.splits)+1 {+ t.Fatalf("%s: got %d boundary states, want %d", tc.name, len(segStates), len(tc.splits)+1)+ }- // Each row's boundary must equal a B=1 single-shot call over that- // row's real prefix: row 0 advances the full length, row 1 freezes- // once it reaches its real length.- boundaries := append(append([]int{}, tc.splits...), T)- for i, bound := range boundaries {- for r := range B {- n := min(int32(bound), rowReal[r])- lo, hi := int32(r), int32(r)+1- rowPrior := mlx.SliceStartStop(prior, []int32{lo, 0, 0, 0}, []int32{hi, int32(Hv), int32(Dv), int32(Dk)})- _, want, _ := mlx.GatedDelta(- slicePrefix(packed, lo, hi, n), slicePrefix(ba, lo, hi, n),- dtBias, aExp, rowPrior, nil, false)- gotRow := mlx.SliceStartStop(segStates[i], []int32{lo, 0, 0, 0}, []int32{hi, int32(Hv), int32(Dv), int32(Dk)})- mlx.Eval(gotRow, want)- floatsClose(t, tc.name+" batched boundary delta", gotRow.Floats(), want.Floats(), 1e-4)+ // Each row's boundary must equal a B=1 single-shot call over that+ // row's real prefix: row 0 advances the full length, row 1 freezes+ // once it reaches its real length.+ boundaries := append(append([]int{}, tc.splits...), T)+ for i, bound := range boundaries {+ for r := range B {+ n := min(int32(bound), rowReal[r])+ lo, hi := int32(r), int32(r)+1+ rowPrior := mlx.SliceStartStop(prior, []int32{lo, 0, 0, 0}, []int32{hi, int32(Hv), int32(Dv), int32(Dk)})+ _, want, _ := mlx.GatedDelta(+ slicePrefix(packed, lo, hi, n), slicePrefix(ba, lo, hi, n),+ dtBias, aExp, rowPrior, nil, false)+ gotRow := mlx.SliceStartStop(segStates[i], []int32{lo, 0, 0, 0}, []int32{hi, int32(Hv), int32(Dv), int32(Dk)})+ mlx.Eval(gotRow, want)+ floatsClose(t, tc.name+" batched boundary delta", gotRow.Floats(), want.Floats(), 1e-4)+ }}}- }+ })}// TestCausalConv1DSegmentEquivalenceBatched is the conv analog of the gated-delta@@ -354,46 +360,47 @@ func TestGatedDeltaSegmentEquivalenceBatched(t *testing.T) {// references for a ragged B>1 batch, where a short row must freeze its tail at// its real end rather than reach into padding.func TestCausalConv1DSegmentEquivalenceBatched(t *testing.T) {- skipIfNoMLX(t)- B, L, D, convTail := 2, 4, 3, 2- K := convTail + 1-- input := fromValues(0.5, B, L, D)- prior := fromValues(-0.3, B, convTail, D)- weight := fromValues(0.2, D, K)- conv := convFromKernel(weight)-- full := &batch.Batch{SeqOffsets: []int32{0, 0}, SeqQueryLens: []int32{int32(L), 3}}-- refOut, refStates := CausalConv1D(full, input, conv, convTail, WithRecurrentState(prior, nil))- segOut, segStates := CausalConv1D(full, input, conv, convTail,- WithRecurrentState(prior, nil), WithSnapshotSplits([]int{1, 2, 3}))- mlx.Eval(refOut, segOut, lastState(refStates), lastState(segStates))-- floatsClose(t, "batched conv out", segOut.Floats(), refOut.Floats(), 1e-4)- floatsClose(t, "batched conv final", lastState(segStates).Floats(), lastState(refStates).Floats(), 1e-4)- if len(segStates) != 4 {- t.Fatalf("got %d boundary conv states, want 4", len(segStates))- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ B, L, D, convTail := 2, 4, 3, 2+ K := convTail + 1++ input := fromValues(0.5, B, L, D)+ prior := fromValues(-0.3, B, convTail, D)+ weight := fromValues(0.2, D, K)+ conv := convFromKernel(weight)++ full := &batch.Batch{SeqOffsets: []int32{0, 0}, SeqQueryLens: []int32{int32(L), 3}}++ refOut, refStates := CausalConv1D(full, input, conv, convTail, WithRecurrentState(prior, nil))+ segOut, segStates := CausalConv1D(full, input, conv, convTail,+ WithRecurrentState(prior, nil), WithSnapshotSplits([]int{1, 2, 3}))+ mlx.Eval(refOut, segOut, lastState(refStates), lastState(segStates))- // Each row's boundary i (offset i+1) must equal a B=1 single-shot conv over- // that row's real prefix: row 0 advances the full length, row 1 freezes once- // it reaches its real length 3. Per-row B=1 references avoid the ambiguity of- // redeclaring a ragged length over a uniform input slice.- rowReal := []int32{int32(L), 3}- for i := range segStates {- for r := range B {- n := min(int32(i+1), rowReal[r])- rowPrior := mlx.SliceStartStop(prior,- []int32{int32(r), 0, 0}, []int32{int32(r) + 1, int32(convTail), int32(D)})- rowInput := mlx.SliceStartStop(input,- []int32{int32(r), 0, 0}, []int32{int32(r) + 1, n, int32(D)})- _, want := CausalConv1D(&batch.Batch{}, rowInput, conv, convTail,- WithRecurrentState(rowPrior, nil))- gotRow := mlx.SliceStartStop(segStates[i],- []int32{int32(r), 0, 0}, []int32{int32(r) + 1, int32(convTail), int32(D)})- mlx.Eval(gotRow, lastState(want))- floatsClose(t, "batched boundary conv", gotRow.Floats(), lastState(want).Floats(), 1e-4)+ floatsClose(t, "batched conv out", segOut.Floats(), refOut.Floats(), 1e-4)+ floatsClose(t, "batched conv final", lastState(segStates).Floats(), lastState(refStates).Floats(), 1e-4)+ if len(segStates) != 4 {+ t.Fatalf("got %d boundary conv states, want 4", len(segStates))}- }++ // Each row's boundary i (offset i+1) must equal a B=1 single-shot conv over+ // that row's real prefix: row 0 advances the full length, row 1 freezes once+ // it reaches its real length 3. Per-row B=1 references avoid the ambiguity of+ // redeclaring a ragged length over a uniform input slice.+ rowReal := []int32{int32(L), 3}+ for i := range segStates {+ for r := range B {+ n := min(int32(i+1), rowReal[r])+ rowPrior := mlx.SliceStartStop(prior,+ []int32{int32(r), 0, 0}, []int32{int32(r) + 1, int32(convTail), int32(D)})+ rowInput := mlx.SliceStartStop(input,+ []int32{int32(r), 0, 0}, []int32{int32(r) + 1, n, int32(D)})+ _, want := CausalConv1D(&batch.Batch{}, rowInput, conv, convTail,+ WithRecurrentState(rowPrior, nil))+ gotRow := mlx.SliceStartStop(segStates[i],+ []int32{int32(r), 0, 0}, []int32{int32(r) + 1, int32(convTail), int32(D)})+ mlx.Eval(gotRow, lastState(want))+ floatsClose(t, "batched boundary conv", gotRow.Floats(), lastState(want).Floats(), 1e-4)+ }+ }+ })}diff --git a/x/models/nn/sdpa_test.go b/x/models/nn/sdpa_test.goindex 6536b9375..b225110ea 100644--- a/x/models/nn/sdpa_test.go+++ b/x/models/nn/sdpa_test.go@@ -4,6 +4,7 @@ import ("math""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/batch""github.com/ollama/ollama/x/mlxrunner/mlx")@@ -30,70 +31,73 @@ func newBatch(seqOffsets []int32, L int, qLens []int32) *batch.Batch {}func TestAttentionMaskZero(t *testing.T) {- skipIfNoMLX(t)- var m AttentionMask- if !m.IsZero() {- t.Fatal("zero value should report IsZero")- }- if m.IsCausal() {- t.Fatal("zero value should not report IsCausal")- }- b := newBatch([]int32{0}, 2, nil)- arr := m.AsArray(b, 3, mlx.DTypeFloat32)- if arr == nil {- t.Fatal("zero value AsArray should return a zeros tensor, not nil")- }- mlx.Eval(arr)- got := arr.Floats()- for i, v := range got {- if v != 0 {- t.Fatalf("zero mask should materialize all zeros; got[%d] = %v", i, v)+ mlxtest.Run(t, func(t *mlxtest.T) {+ var m AttentionMask+ if !m.IsZero() {+ t.Fatal("zero value should report IsZero")}- }+ if m.IsCausal() {+ t.Fatal("zero value should not report IsCausal")+ }+ b := newBatch([]int32{0}, 2, nil)+ arr := m.AsArray(b, 3, mlx.DTypeFloat32)+ if arr == nil {+ t.Fatal("zero value AsArray should return a zeros tensor, not nil")+ }+ mlx.Eval(arr)+ got := arr.Floats()+ for i, v := range got {+ if v != 0 {+ t.Fatalf("zero mask should materialize all zeros; got[%d] = %v", i, v)+ }+ }+ })}func TestAttentionMaskAsArrayCausal(t *testing.T) {- skipIfNoMLX(t)- L, K := 4, 6- b := newBatch([]int32{2}, L, nil)- arr := CausalMask().AsArray(b, K, mlx.DTypeFloat32)- if arr == nil {- t.Fatal("CausalMask AsArray should return a tensor")- }- dims := arr.Dims()- if len(dims) != 4 || dims[0] != 1 || dims[1] != 1 || dims[2] != L || dims[3] != K {- t.Fatalf("want shape [1,1,%d,%d], got %v", L, K, dims)- }- mlx.Eval(arr)- got := arr.Floats()- negInf := float32(math.Inf(-1))- want := make([]float32, L*K)- for q := range L {- absQ := int(b.SeqOffsets[0]) + q- for k := range K {- if k > absQ {- want[q*K+k] = negInf+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 4, 6+ b := newBatch([]int32{2}, L, nil)+ arr := CausalMask().AsArray(b, K, mlx.DTypeFloat32)+ if arr == nil {+ t.Fatal("CausalMask AsArray should return a tensor")+ }+ dims := arr.Dims()+ if len(dims) != 4 || dims[0] != 1 || dims[1] != 1 || dims[2] != L || dims[3] != K {+ t.Fatalf("want shape [1,1,%d,%d], got %v", L, K, dims)+ }+ mlx.Eval(arr)+ got := arr.Floats()+ negInf := float32(math.Inf(-1))+ want := make([]float32, L*K)+ for q := range L {+ absQ := int(b.SeqOffsets[0]) + q+ for k := range K {+ if k > absQ {+ want[q*K+k] = negInf+ }}}- }- for i := range want {- if !sameF(got[i], want[i]) {- t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ for i := range want {+ if !sameF(got[i], want[i]) {+ t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ }}- }+ })}func TestAttentionMaskRelaxLazy(t *testing.T) {- skipIfNoMLX(t)- // Relax must not materialize a tensor — the perf invariant the- // causal-flag fast path relies on. Everything else (predicates,- // AsArray contents) is exercised by the materialization tests.- m := CausalMask().- Relax(0, 1, 3, 2, 5).- Relax(0, 0, 2, 1, 4)- if m.array != nil {- t.Fatal("Relax should not materialize a tensor")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ // Relax must not materialize a tensor — the perf invariant the+ // causal-flag fast path relies on. Everything else (predicates,+ // AsArray contents) is exercised by the materialization tests.+ m := CausalMask().+ Relax(0, 1, 3, 2, 5).+ Relax(0, 0, 2, 1, 4)+ if m.array != nil {+ t.Fatal("Relax should not materialize a tensor")+ }+ })}// TestAttentionMaskRelaxNoopRectsMatchCausal pins the contract that@@ -101,340 +105,354 @@ func TestAttentionMaskRelaxLazy(t *testing.T) {// inside the causal triangle — must produce the same materialized// tensor as plain causal.func TestAttentionMaskRelaxNoopRectsMatchCausal(t *testing.T) {- skipIfNoMLX(t)- L, K := 4, 6- b := newBatch([]int32{0}, L, nil)- want := CausalMask().AsArray(b, K, mlx.DTypeFloat32)- mlx.Eval(want)- wantF := want.Floats()-- cases := []struct {- name string- qLo, qHi, kLo, kHi int- }{- {"empty Q rect", 2, 2, 0, 3},- {"empty K rect", 0, 3, 2, 2},- {"fully under causal", 5, 7, 0, 3},- }- for _, tc := range cases {- m := CausalMask().Relax(0, tc.qLo, tc.qHi, tc.kLo, tc.kHi)- arr := m.AsArray(b, K, mlx.DTypeFloat32)- mlx.Eval(arr)- got := arr.Floats()- for i := range wantF {- if !sameF(got[i], wantF[i]) {- t.Fatalf("%s index %d: want %v, got %v", tc.name, i, wantF[i], got[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 4, 6+ b := newBatch([]int32{0}, L, nil)+ want := CausalMask().AsArray(b, K, mlx.DTypeFloat32)+ mlx.Eval(want)+ wantF := want.Floats()++ cases := []struct {+ name string+ qLo, qHi, kLo, kHi int+ }{+ {"empty Q rect", 2, 2, 0, 3},+ {"empty K rect", 0, 3, 2, 2},+ {"fully under causal", 5, 7, 0, 3},+ }+ for _, tc := range cases {+ m := CausalMask().Relax(0, tc.qLo, tc.qHi, tc.kLo, tc.kHi)+ arr := m.AsArray(b, K, mlx.DTypeFloat32)+ mlx.Eval(arr)+ got := arr.Floats()+ for i := range wantF {+ if !sameF(got[i], wantF[i]) {+ t.Fatalf("%s index %d: want %v, got %v", tc.name, i, wantF[i], got[i])+ }}}- }+ })}func TestAttentionMaskAsArrayWithRelax(t *testing.T) {- skipIfNoMLX(t)- L, K := 4, 6- b := newBatch([]int32{0}, L, nil)- arr := CausalMask().Relax(0, 1, 3, 2, 5).AsArray(b, K, mlx.DTypeFloat32)- if arr == nil {- t.Fatal("expected tensor")- }- mlx.Eval(arr)- got := arr.Floats()- negInf := float32(math.Inf(-1))- want := make([]float32, L*K)- for q := range L {- for k := range K {- if k > q {- want[q*K+k] = negInf+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 4, 6+ b := newBatch([]int32{0}, L, nil)+ arr := CausalMask().Relax(0, 1, 3, 2, 5).AsArray(b, K, mlx.DTypeFloat32)+ if arr == nil {+ t.Fatal("expected tensor")+ }+ mlx.Eval(arr)+ got := arr.Floats()+ negInf := float32(math.Inf(-1))+ want := make([]float32, L*K)+ for q := range L {+ for k := range K {+ if k > q {+ want[q*K+k] = negInf+ }}}- }- for q := 1; q < 3; q++ {- for k := 2; k < 5; k++ {- want[q*K+k] = 0+ for q := 1; q < 3; q++ {+ for k := 2; k < 5; k++ {+ want[q*K+k] = 0+ }}- }- for i := range want {- if !sameF(got[i], want[i]) {- t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ for i := range want {+ if !sameF(got[i], want[i]) {+ t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ }}- }+ })}func TestAttentionMaskAsArrayPerRow(t *testing.T) {- skipIfNoMLX(t)- L, K := 3, 5- b := newBatch([]int32{0, 2}, L, nil)- m := CausalMask().- Relax(0, 0, 2, 0, 3).- Relax(1, 3, 5, 2, 5)- arr := m.AsArray(b, K, mlx.DTypeFloat32)- if arr == nil {- t.Fatal("expected tensor")- }- dims := arr.Dims()- if dims[0] != 2 {- t.Fatalf("expected batch dim 2, got %v", dims)- }- mlx.Eval(arr)- got := arr.Floats()- negInf := float32(math.Inf(-1))-- want := make([]float32, 2*L*K)- for bi, off := range b.SeqOffsets {- for q := range L {- absQ := int(off) + q- for k := range K {- if k > absQ {- want[bi*L*K+q*K+k] = negInf+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 3, 5+ b := newBatch([]int32{0, 2}, L, nil)+ m := CausalMask().+ Relax(0, 0, 2, 0, 3).+ Relax(1, 3, 5, 2, 5)+ arr := m.AsArray(b, K, mlx.DTypeFloat32)+ if arr == nil {+ t.Fatal("expected tensor")+ }+ dims := arr.Dims()+ if dims[0] != 2 {+ t.Fatalf("expected batch dim 2, got %v", dims)+ }+ mlx.Eval(arr)+ got := arr.Floats()+ negInf := float32(math.Inf(-1))++ want := make([]float32, 2*L*K)+ for bi, off := range b.SeqOffsets {+ for q := range L {+ absQ := int(off) + q+ for k := range K {+ if k > absQ {+ want[bi*L*K+q*K+k] = negInf+ }}}}- }- for q := range 2 {- for k := range 3 {- want[0*L*K+q*K+k] = 0+ for q := range 2 {+ for k := range 3 {+ want[0*L*K+q*K+k] = 0+ }}- }- for q := 1; q < 3; q++ {- for k := 2; k < 5; k++ {- want[1*L*K+q*K+k] = 0+ for q := 1; q < 3; q++ {+ for k := 2; k < 5; k++ {+ want[1*L*K+q*K+k] = 0+ }}- }- for i := range want {- if !sameF(got[i], want[i]) {- t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ for i := range want {+ if !sameF(got[i], want[i]) {+ t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ }}- }+ })}func TestQPaddingMask(t *testing.T) {- skipIfNoMLX(t)- L := 4- // Row 0 fully real; row 1 has 2 real queries.- b := newBatch([]int32{0, 0}, L, []int32{int32(L), 2})- m := QPaddingMask(b, mlx.DTypeFloat32)- if m.array == nil {- t.Fatal("expected q-padding tensor")- }- mlx.Eval(m.array)- got := m.array.Floats()- negInf := float32(math.Inf(-1))- want := make([]float32, 2*L)- // Row 0: no blocking; row 1: q >= 2 blocked.- for q := 2; q < L; q++ {- want[1*L+q] = negInf- }- for i := range want {- if !sameF(got[i], want[i]) {- t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ L := 4+ // Row 0 fully real; row 1 has 2 real queries.+ b := newBatch([]int32{0, 0}, L, []int32{int32(L), 2})+ m := QPaddingMask(b, mlx.DTypeFloat32)+ if m.array == nil {+ t.Fatal("expected q-padding tensor")+ }+ mlx.Eval(m.array)+ got := m.array.Floats()+ negInf := float32(math.Inf(-1))+ want := make([]float32, 2*L)+ // Row 0: no blocking; row 1: q >= 2 blocked.+ for q := 2; q < L; q++ {+ want[1*L+q] = negInf+ }+ for i := range want {+ if !sameF(got[i], want[i]) {+ t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ }}- }+ })}func TestKPaddingMask(t *testing.T) {- skipIfNoMLX(t)- K := 5- // Row 0 full keys; row 1 has 3 real keys.- b := newBatch([]int32{0, 0}, 4, nil)- kLens := []int32{int32(K), 3}- m := KPaddingMask(b, K, kLens, mlx.DTypeFloat32)- if m.array == nil {- t.Fatal("expected k-padding tensor")- }- mlx.Eval(m.array)- got := m.array.Floats()- negInf := float32(math.Inf(-1))- want := make([]float32, 2*K)- for k := 3; k < K; k++ {- want[1*K+k] = negInf- }- for i := range want {- if !sameF(got[i], want[i]) {- t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ K := 5+ // Row 0 full keys; row 1 has 3 real keys.+ b := newBatch([]int32{0, 0}, 4, nil)+ kLens := []int32{int32(K), 3}+ m := KPaddingMask(b, K, kLens, mlx.DTypeFloat32)+ if m.array == nil {+ t.Fatal("expected k-padding tensor")+ }+ mlx.Eval(m.array)+ got := m.array.Floats()+ negInf := float32(math.Inf(-1))+ want := make([]float32, 2*K)+ for k := 3; k < K; k++ {+ want[1*K+k] = negInf+ }+ for i := range want {+ if !sameF(got[i], want[i]) {+ t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ }}- }+ })}func TestQPaddingMaskZeroWhenFull(t *testing.T) {- skipIfNoMLX(t)- b := newBatch([]int32{0}, 4, nil)- m := QPaddingMask(b, mlx.DTypeFloat32)- if !m.IsZero() {- t.Fatal("QPaddingMask at full queries should be zero")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ b := newBatch([]int32{0}, 4, nil)+ m := QPaddingMask(b, mlx.DTypeFloat32)+ if !m.IsZero() {+ t.Fatal("QPaddingMask at full queries should be zero")+ }+ })}func TestKPaddingMaskZeroWhenFull(t *testing.T) {- skipIfNoMLX(t)- K := 4- b := newBatch([]int32{0}, 4, nil)- kLens := []int32{int32(K)}- m := KPaddingMask(b, K, kLens, mlx.DTypeFloat32)- if !m.IsZero() {- t.Fatal("KPaddingMask at full keys should be zero")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ K := 4+ b := newBatch([]int32{0}, 4, nil)+ kLens := []int32{int32(K)}+ m := KPaddingMask(b, K, kLens, mlx.DTypeFloat32)+ if !m.IsZero() {+ t.Fatal("KPaddingMask at full keys should be zero")+ }+ })}func TestAttentionMaskCombineCausal(t *testing.T) {- skipIfNoMLX(t)- var z AttentionMask- got := z.Intersect(CausalMask())- if !got.IsCausal() {- t.Fatal("zero + CausalMask should be pure causal")- }- got = CausalMask().Intersect(z)- if !got.IsCausal() {- t.Fatal("CausalMask + zero should be pure causal")- }- got = CausalMask().Intersect(CausalMask())- if !got.IsCausal() {- t.Fatal("causal + causal should stay pure causal")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ var z AttentionMask+ got := z.Intersect(CausalMask())+ if !got.IsCausal() {+ t.Fatal("zero + CausalMask should be pure causal")+ }+ got = CausalMask().Intersect(z)+ if !got.IsCausal() {+ t.Fatal("CausalMask + zero should be pure causal")+ }+ got = CausalMask().Intersect(CausalMask())+ if !got.IsCausal() {+ t.Fatal("causal + causal should stay pure causal")+ }+ })}func TestAttentionMaskCombineRelaxDroppedAgainstCausal(t *testing.T) {- skipIfNoMLX(t)- relaxed := CausalMask().Relax(0, 1, 3, 2, 5)- got := relaxed.Intersect(CausalMask())- if !got.IsCausal() {- t.Fatal("causal-with-Relax + causal should drop relaxations and stay pure causal")- }- got = CausalMask().Intersect(relaxed)- if !got.IsCausal() {- t.Fatal("causal + causal-with-Relax should drop relaxations and stay pure causal")- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ relaxed := CausalMask().Relax(0, 1, 3, 2, 5)+ got := relaxed.Intersect(CausalMask())+ if !got.IsCausal() {+ t.Fatal("causal-with-Relax + causal should drop relaxations and stay pure causal")+ }+ got = CausalMask().Intersect(relaxed)+ if !got.IsCausal() {+ t.Fatal("causal + causal-with-Relax should drop relaxations and stay pure causal")+ }- // Disjoint relaxations on two causals also drop — neither side- // agrees to release the cells the other side relaxed.- got = CausalMask().Relax(0, 1, 3, 2, 5).Intersect(CausalMask().Relax(0, 5, 7, 6, 9))- if !got.IsCausal() {- t.Fatal("disjoint relaxations on two causals should drop and stay pure causal")- }+ // Disjoint relaxations on two causals also drop — neither side+ // agrees to release the cells the other side relaxed.+ got = CausalMask().Relax(0, 1, 3, 2, 5).Intersect(CausalMask().Relax(0, 5, 7, 6, 9))+ if !got.IsCausal() {+ t.Fatal("disjoint relaxations on two causals should drop and stay pure causal")+ }+ })}func TestAttentionMaskCombineRelaxIntersect(t *testing.T) {- skipIfNoMLX(t)- L, K := 6, 6- b := newBatch([]int32{0}, L, nil)-- // Overlapping rects on two causals: the surviving relaxation is- // the geometric intersection — q in [1,3) ∩ [2,5) = [2,3),- // k in [2,5) ∩ [3,6) = [3,5).- m := CausalMask().Relax(0, 1, 3, 2, 5).Intersect(CausalMask().Relax(0, 2, 5, 3, 6))- if m.IsCausal() {- t.Fatal("overlapping relaxations should survive as their intersection, not collapse to pure causal")- }- arr := m.AsArray(b, K, mlx.DTypeFloat32)- if arr == nil {- t.Fatal("expected tensor")- }- mlx.Eval(arr)- vals := arr.Floats()- negInf := float32(math.Inf(-1))- want := make([]float32, L*K)- for q := range L {- for k := range K {- if k > q {- want[q*K+k] = negInf+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 6, 6+ b := newBatch([]int32{0}, L, nil)++ // Overlapping rects on two causals: the surviving relaxation is+ // the geometric intersection — q in [1,3) ∩ [2,5) = [2,3),+ // k in [2,5) ∩ [3,6) = [3,5).+ m := CausalMask().Relax(0, 1, 3, 2, 5).Intersect(CausalMask().Relax(0, 2, 5, 3, 6))+ if m.IsCausal() {+ t.Fatal("overlapping relaxations should survive as their intersection, not collapse to pure causal")+ }+ arr := m.AsArray(b, K, mlx.DTypeFloat32)+ if arr == nil {+ t.Fatal("expected tensor")+ }+ mlx.Eval(arr)+ vals := arr.Floats()+ negInf := float32(math.Inf(-1))+ want := make([]float32, L*K)+ for q := range L {+ for k := range K {+ if k > q {+ want[q*K+k] = negInf+ }}}- }- // Intersection rect: q ∈ [2,3), k ∈ [3,5).- for q := 2; q < 3; q++ {- for k := 3; k < 5; k++ {- want[q*K+k] = 0+ // Intersection rect: q ∈ [2,3), k ∈ [3,5).+ for q := 2; q < 3; q++ {+ for k := 3; k < 5; k++ {+ want[q*K+k] = 0+ }}- }- for i := range want {- if !sameF(vals[i], want[i]) {- t.Fatalf("index %d: want %v, got %v", i, want[i], vals[i])+ for i := range want {+ if !sameF(vals[i], want[i]) {+ t.Fatalf("index %d: want %v, got %v", i, want[i], vals[i])+ }}- }+ })}func TestAttentionMaskCombineRelaxKeptAgainstNonCausal(t *testing.T) {- skipIfNoMLX(t)- L, K := 4, 6- b := newBatch([]int32{0}, L, nil)-- // Pad q=3 — non-causal additive contribution that should leave- // the relaxation intact (the rect releases above-diagonal cells- // q in [1,3), k in [2,5) where k > q).- pad := QPaddingMask(newBatch([]int32{0}, L, []int32{3}), mlx.DTypeFloat32)- if pad.IsZero() {- t.Fatal("padding mask should be non-zero")- }- got := CausalMask().Relax(0, 1, 3, 2, 5).Intersect(pad)- arr := got.AsArray(b, K, mlx.DTypeFloat32)- if arr == nil {- t.Fatal("expected tensor")- }- mlx.Eval(arr)- vals := arr.Floats()- negInf := float32(math.Inf(-1))- want := make([]float32, L*K)- for q := range L {- for k := range K {- if k > q {- want[q*K+k] = negInf+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 4, 6+ b := newBatch([]int32{0}, L, nil)++ // Pad q=3 — non-causal additive contribution that should leave+ // the relaxation intact (the rect releases above-diagonal cells+ // q in [1,3), k in [2,5) where k > q).+ pad := QPaddingMask(newBatch([]int32{0}, L, []int32{3}), mlx.DTypeFloat32)+ if pad.IsZero() {+ t.Fatal("padding mask should be non-zero")+ }+ got := CausalMask().Relax(0, 1, 3, 2, 5).Intersect(pad)+ arr := got.AsArray(b, K, mlx.DTypeFloat32)+ if arr == nil {+ t.Fatal("expected tensor")+ }+ mlx.Eval(arr)+ vals := arr.Floats()+ negInf := float32(math.Inf(-1))+ want := make([]float32, L*K)+ for q := range L {+ for k := range K {+ if k > q {+ want[q*K+k] = negInf+ }}}- }- for q := 1; q < 3; q++ {- for k := 2; k < 5; k++ {- want[q*K+k] = 0+ for q := 1; q < 3; q++ {+ for k := 2; k < 5; k++ {+ want[q*K+k] = 0+ }}- }- for q := 3; q < L; q++ {- for k := range K {- want[q*K+k] = negInf+ for q := 3; q < L; q++ {+ for k := range K {+ want[q*K+k] = negInf+ }}- }- for i := range want {- if !sameF(vals[i], want[i]) {- t.Fatalf("index %d: want %v, got %v", i, want[i], vals[i])+ for i := range want {+ if !sameF(vals[i], want[i]) {+ t.Fatalf("index %d: want %v, got %v", i, want[i], vals[i])+ }}- }+ })}func TestAttentionMaskCombineArrays(t *testing.T) {- skipIfNoMLX(t)- a := mlx.FromValues([]float32{0, 0, 0, 0}, 1, 1, 2, 2)- bb := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 1, 2, 2)- sum := ArrayMask(a).Intersect(ArrayMask(bb))- if sum.array == nil {- t.Fatal("array + array should produce array")- }- mlx.Eval(sum.array)- got := sum.array.Floats()- want := []float32{1, 2, 3, 4}- for i := range want {- if got[i] != want[i] {- t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ a := mlx.FromValues([]float32{0, 0, 0, 0}, 1, 1, 2, 2)+ bb := mlx.FromValues([]float32{1, 2, 3, 4}, 1, 1, 2, 2)+ sum := ArrayMask(a).Intersect(ArrayMask(bb))+ if sum.array == nil {+ t.Fatal("array + array should produce array")+ }+ mlx.Eval(sum.array)+ got := sum.array.Floats()+ want := []float32{1, 2, 3, 4}+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("index %d: want %v, got %v", i, want[i], got[i])+ }}- }+ })}func TestAttentionMaskRelaxPanicOnArray(t *testing.T) {- skipIfNoMLX(t)- a := mlx.FromValues([]float32{0}, 1, 1, 1, 1)- defer func() {- if r := recover(); r == nil {- t.Fatal("Relax on ArrayMask should panic")- }- }()- ArrayMask(a).Relax(0, 0, 1, 0, 1)+ mlxtest.Run(t, func(t *mlxtest.T) {+ a := mlx.FromValues([]float32{0}, 1, 1, 1, 1)+ defer func() {+ if r := recover(); r == nil {+ t.Fatal("Relax on ArrayMask should panic")+ }+ }()+ ArrayMask(a).Relax(0, 0, 1, 0, 1)+ })}func TestAttentionMaskRelaxPanicOnZero(t *testing.T) {- skipIfNoMLX(t)- defer func() {- if r := recover(); r == nil {- t.Fatal("Relax on zero mask should panic")- }- }()- var z AttentionMask- z.Relax(0, 0, 1, 0, 1)+ mlxtest.Run(t, func(t *mlxtest.T) {+ defer func() {+ if r := recover(); r == nil {+ t.Fatal("Relax on zero mask should panic")+ }+ }()+ var z AttentionMask+ z.Relax(0, 0, 1, 0, 1)+ })}func sameF(a, b float32) bool {@@ -468,135 +486,142 @@ func sdpaInputs(L, K int) (q, k, v *mlx.Array) {}func TestSDPACausalParity(t *testing.T) {- skipIfNoMLX(t)- L, K := 4, 4- q, k, v := sdpaInputs(L, K)- b := newBatch([]int32{int32(K - L)}, L, nil)- got := ScaledDotProductAttention(b, q, 1.0,- WithKV(k, v, []int32{int32(K)}),- WithMask(CausalMask()),- )- want := mlx.FastScaledDotProductAttention(q, k, v, 1.0, "causal", nil)- mlx.Eval(got, want)- gs, ws := got.Floats(), want.Floats()- for i := range ws {- if gs[i] != ws[i] {- t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 4, 4+ q, k, v := sdpaInputs(L, K)+ b := newBatch([]int32{int32(K - L)}, L, nil)+ got := ScaledDotProductAttention(b, q, 1.0,+ WithKV(k, v, []int32{int32(K)}),+ WithMask(CausalMask()),+ )+ want := mlx.FastScaledDotProductAttention(q, k, v, 1.0, "causal", nil)+ mlx.Eval(got, want)+ gs, ws := got.Floats(), want.Floats()+ for i := range ws {+ if gs[i] != ws[i] {+ t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ }}- }+ })}func TestSDPAZeroMaskParity(t *testing.T) {- skipIfNoMLX(t)- L, K := 4, 4- q, k, v := sdpaInputs(L, K)- b := newBatch([]int32{0}, L, nil)- got := ScaledDotProductAttention(b, q, 1.0, WithKV(k, v, []int32{int32(K)}))- want := mlx.FastScaledDotProductAttention(q, k, v, 1.0, "", nil)- mlx.Eval(got, want)- gs, ws := got.Floats(), want.Floats()- for i := range ws {- if gs[i] != ws[i] {- t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 4, 4+ q, k, v := sdpaInputs(L, K)+ b := newBatch([]int32{0}, L, nil)+ got := ScaledDotProductAttention(b, q, 1.0, WithKV(k, v, []int32{int32(K)}))+ want := mlx.FastScaledDotProductAttention(q, k, v, 1.0, "", nil)+ mlx.Eval(got, want)+ gs, ws := got.Floats(), want.Floats()+ for i := range ws {+ if gs[i] != ws[i] {+ t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ }}- }+ })}func TestSDPAArrayMaskParity(t *testing.T) {- skipIfNoMLX(t)- L, K := 3, 3- q, k, v := sdpaInputs(L, K)- b := newBatch([]int32{0}, L, nil)- mask := mlx.FromValues([]float32{- 0, -1, -1,- 0, 0, -1,- 0, 0, 0,- }, 1, 1, 3, 3)- got := ScaledDotProductAttention(b, q, 1.0,- WithKV(k, v, []int32{int32(K)}),- WithMask(ArrayMask(mask)),- )- want := mlx.FastScaledDotProductAttention(q, k, v, 1.0, "array", mask)- mlx.Eval(got, want)- gs, ws := got.Floats(), want.Floats()- for i := range ws {- if gs[i] != ws[i] {- t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 3, 3+ q, k, v := sdpaInputs(L, K)+ b := newBatch([]int32{0}, L, nil)+ mask := mlx.FromValues([]float32{+ 0, -1, -1,+ 0, 0, -1,+ 0, 0, 0,+ }, 1, 1, 3, 3)+ got := ScaledDotProductAttention(b, q, 1.0,+ WithKV(k, v, []int32{int32(K)}),+ WithMask(ArrayMask(mask)),+ )+ want := mlx.FastScaledDotProductAttention(q, k, v, 1.0, "array", mask)+ mlx.Eval(got, want)+ gs, ws := got.Floats(), want.Floats()+ for i := range ws {+ if gs[i] != ws[i] {+ t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ }}- }+ })}func TestSDPARelaxMaskMaterializes(t *testing.T) {- skipIfNoMLX(t)- L, K := 3, 5- q, k, v := sdpaInputs(L, K)- b := newBatch([]int32{int32(K - L)}, L, nil)- got := ScaledDotProductAttention(b, q, 1.0,- WithKV(k, v, []int32{int32(K)}),- WithMask(CausalMask().Relax(0, 3, 5, 2, 5)),- )- ref := CausalMask().Relax(0, 3, 5, 2, 5).AsArray(b, K, k.DType())- want := mlx.FastScaledDotProductAttention(q, k, v, 1.0, "array", ref)- mlx.Eval(got, want)- gs, ws := got.Floats(), want.Floats()- for i := range ws {- if gs[i] != ws[i] {- t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, K := 3, 5+ q, k, v := sdpaInputs(L, K)+ b := newBatch([]int32{int32(K - L)}, L, nil)+ got := ScaledDotProductAttention(b, q, 1.0,+ WithKV(k, v, []int32{int32(K)}),+ WithMask(CausalMask().Relax(0, 3, 5, 2, 5)),+ )+ ref := CausalMask().Relax(0, 3, 5, 2, 5).AsArray(b, K, k.DType())+ want := mlx.FastScaledDotProductAttention(q, k, v, 1.0, "array", ref)+ mlx.Eval(got, want)+ gs, ws := got.Floats(), want.Floats()+ for i := range ws {+ if gs[i] != ws[i] {+ t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ }}- }+ })}func TestSDPAPanicsWithBothKVAndHistory(t *testing.T) {- skipIfNoMLX(t)- L := 3- q, k, v := sdpaInputs(L, L)- b := newBatch([]int32{0}, L, nil)- history := NewKVHistory(k, v, nil)- defer func() {- if r := recover(); r == nil {- t.Fatal("expected panic when both WithKV and WithKVHistory are supplied")- }- }()- ScaledDotProductAttention(b, q, 1.0, WithKV(k, v, []int32{int32(L)}), WithKVHistory(history))+ mlxtest.Run(t, func(t *mlxtest.T) {+ L := 3+ q, k, v := sdpaInputs(L, L)+ b := newBatch([]int32{0}, L, nil)+ history := NewKVHistory(k, v, nil)+ defer func() {+ if r := recover(); r == nil {+ t.Fatal("expected panic when both WithKV and WithKVHistory are supplied")+ }+ }()+ ScaledDotProductAttention(b, q, 1.0, WithKV(k, v, []int32{int32(L)}), WithKVHistory(history))+ })}func TestSDPAMLAHistorySlicesVFromK(t *testing.T) {- skipIfNoMLX(t)- L, D, valueDim := 2, 5, 3- kBuf := make([]float32, 1*1*L*D)- for i := range kBuf {- kBuf[i] = float32(i) + 1- }- k := mlx.FromValues(kBuf, 1, 1, L, D)- v := mlx.Zeros(mlx.DTypeFloat32, 1, 1, L, valueDim)- history := NewKVHistory(k, v, nil)-- q := mlx.Zeros(mlx.DTypeFloat32, 1, 1, L, D)- b := newBatch([]int32{0}, L, nil)- got := ScaledDotProductAttention(b, q, 1.0,- WithMLAHistory(history, valueDim),- )- vRef := k.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(0, valueDim))- want := mlx.FastScaledDotProductAttention(q, k, vRef, 1.0, "", nil)- mlx.Eval(got, want)- gs, ws := got.Floats(), want.Floats()- for i := range ws {- if gs[i] != ws[i] {- t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ L, D, valueDim := 2, 5, 3+ kBuf := make([]float32, 1*1*L*D)+ for i := range kBuf {+ kBuf[i] = float32(i) + 1+ }+ k := mlx.FromValues(kBuf, 1, 1, L, D)+ v := mlx.Zeros(mlx.DTypeFloat32, 1, 1, L, valueDim)+ history := NewKVHistory(k, v, nil)++ q := mlx.Zeros(mlx.DTypeFloat32, 1, 1, L, D)+ b := newBatch([]int32{0}, L, nil)+ got := ScaledDotProductAttention(b, q, 1.0,+ WithMLAHistory(history, valueDim),+ )+ vRef := k.Slice(mlx.Slice(), mlx.Slice(), mlx.Slice(), mlx.Slice(0, valueDim))+ want := mlx.FastScaledDotProductAttention(q, k, vRef, 1.0, "", nil)+ mlx.Eval(got, want)+ gs, ws := got.Floats(), want.Floats()+ for i := range ws {+ if gs[i] != ws[i] {+ t.Fatalf("index %d: want %v, got %v", i, ws[i], gs[i])+ }}- }+ })}func TestSDPAPanicsWithoutKV(t *testing.T) {- skipIfNoMLX(t)- q := mlx.FromValues(make([]float32, 4), 1, 1, 1, 4)- b := newBatch([]int32{0}, 1, nil)- defer func() {- if r := recover(); r == nil {- t.Fatal("expected panic when no K/V supplied")- }- }()- ScaledDotProductAttention(b, q, 1.0)+ mlxtest.Run(t, func(t *mlxtest.T) {+ q := mlx.FromValues(make([]float32, 4), 1, 1, 1, 4)+ b := newBatch([]int32{0}, 1, nil)+ defer func() {+ if r := recover(); r == nil {+ t.Fatal("expected panic when no K/V supplied")+ }+ }()+ ScaledDotProductAttention(b, q, 1.0)+ })}// fillTensor builds a [B, H, T, D] float32 tensor whose entries are@@ -617,62 +642,63 @@ func fillTensor(seed float32, B, H, T, D int) *mlx.Array {// central multi-sequence contract: right-padded rows must produce// per-row outputs that don't depend on the padded tails.func TestSDPAMultiSequenceParity(t *testing.T) {- skipIfNoMLX(t)- const H, D = 1, 4- const L, K = 4, 6- const qShort, kShort = 2, 2- const scale = 1.0-- q := fillTensor(0.5, 2, H, L, D)- k := fillTensor(-0.3, 2, H, K, D)- v := fillTensor(0.7, 2, H, K, D)- b := newBatch([]int32{2, 0}, L, []int32{int32(L), int32(qShort)})-- got := ScaledDotProductAttention(b, q, scale,- WithKV(k, v, []int32{int32(K), int32(kShort)}),- WithMask(CausalMask()))- mlx.Eval(got)- gotF := got.Floats()-- // Row 0: full Q at offset 2, full K. B=1 reference.- q0 := mlx.SliceStartStop(q, []int32{0, 0, 0, 0}, []int32{1, H, L, D})- k0 := mlx.SliceStartStop(k, []int32{0, 0, 0, 0}, []int32{1, H, K, D})- v0 := mlx.SliceStartStop(v, []int32{0, 0, 0, 0}, []int32{1, H, K, D})- b0 := newBatch([]int32{2}, L, nil)- ref0 := ScaledDotProductAttention(b0, q0, scale,- WithKV(k0, v0, []int32{int32(K)}),- WithMask(CausalMask()))- mlx.Eval(ref0)- ref0F := ref0.Floats()-- // Row 1: real Q at offset 0, length qShort, with kShort real keys.- q1 := mlx.SliceStartStop(q, []int32{1, 0, 0, 0}, []int32{2, H, int32(qShort), D})- k1 := mlx.SliceStartStop(k, []int32{1, 0, 0, 0}, []int32{2, H, int32(kShort), D})- v1 := mlx.SliceStartStop(v, []int32{1, 0, 0, 0}, []int32{2, H, int32(kShort), D})- b1 := newBatch([]int32{0}, qShort, nil)- ref1 := ScaledDotProductAttention(b1, q1, scale,- WithKV(k1, v1, []int32{int32(kShort)}),- WithMask(CausalMask()))- mlx.Eval(ref1)- ref1F := ref1.Floats()-- // got is [2, H, L, D] = [B=2, 1, 4, 4]. Row 0 is got[0,...] and- // must match ref0 over the full [L, D]. Row 1 is got[1,...] and- // must match ref1 over [qShort, D] only — padded positions are- // unconstrained.- rowStride := H * L * D- for i := range rowStride {- if !approxEqual(gotF[i], ref0F[i], 1e-5) {- t.Fatalf("row 0 [%d]: got %v, want %v", i, gotF[i], ref0F[i])+ mlxtest.Run(t, func(t *mlxtest.T) {+ const H, D = 1, 4+ const L, K = 4, 6+ const qShort, kShort = 2, 2+ const scale = 1.0++ q := fillTensor(0.5, 2, H, L, D)+ k := fillTensor(-0.3, 2, H, K, D)+ v := fillTensor(0.7, 2, H, K, D)+ b := newBatch([]int32{2, 0}, L, []int32{int32(L), int32(qShort)})++ got := ScaledDotProductAttention(b, q, scale,+ WithKV(k, v, []int32{int32(K), int32(kShort)}),+ WithMask(CausalMask()))+ mlx.Eval(got)+ gotF := got.Floats()++ // Row 0: full Q at offset 2, full K. B=1 reference.+ q0 := mlx.SliceStartStop(q, []int32{0, 0, 0, 0}, []int32{1, H, L, D})+ k0 := mlx.SliceStartStop(k, []int32{0, 0, 0, 0}, []int32{1, H, K, D})+ v0 := mlx.SliceStartStop(v, []int32{0, 0, 0, 0}, []int32{1, H, K, D})+ b0 := newBatch([]int32{2}, L, nil)+ ref0 := ScaledDotProductAttention(b0, q0, scale,+ WithKV(k0, v0, []int32{int32(K)}),+ WithMask(CausalMask()))+ mlx.Eval(ref0)+ ref0F := ref0.Floats()++ // Row 1: real Q at offset 0, length qShort, with kShort real keys.+ q1 := mlx.SliceStartStop(q, []int32{1, 0, 0, 0}, []int32{2, H, int32(qShort), D})+ k1 := mlx.SliceStartStop(k, []int32{1, 0, 0, 0}, []int32{2, H, int32(kShort), D})+ v1 := mlx.SliceStartStop(v, []int32{1, 0, 0, 0}, []int32{2, H, int32(kShort), D})+ b1 := newBatch([]int32{0}, qShort, nil)+ ref1 := ScaledDotProductAttention(b1, q1, scale,+ WithKV(k1, v1, []int32{int32(kShort)}),+ WithMask(CausalMask()))+ mlx.Eval(ref1)+ ref1F := ref1.Floats()++ // got is [2, H, L, D] = [B=2, 1, 4, 4]. Row 0 is got[0,...] and+ // must match ref0 over the full [L, D]. Row 1 is got[1,...] and+ // must match ref1 over [qShort, D] only — padded positions are+ // unconstrained.+ rowStride := H * L * D+ for i := range rowStride {+ if !approxEqual(gotF[i], ref0F[i], 1e-5) {+ t.Fatalf("row 0 [%d]: got %v, want %v", i, gotF[i], ref0F[i])+ }}- }- for q := range qShort {- for d := range D {- gotI := rowStride + q*D + d- refI := q*D + d- if !approxEqual(gotF[gotI], ref1F[refI], 1e-5) {- t.Fatalf("row 1 [q=%d,d=%d]: got %v, want %v", q, d, gotF[gotI], ref1F[refI])+ for q := range qShort {+ for d := range D {+ gotI := rowStride + q*D + d+ refI := q*D + d+ if !approxEqual(gotF[gotI], ref1F[refI], 1e-5) {+ t.Fatalf("row 1 [q=%d,d=%d]: got %v, want %v", q, d, gotF[gotI], ref1F[refI])+ }}}- }+ })}diff --git a/x/models/qwen3_5/gdn_projections_test.go b/x/models/qwen3_5/gdn_projections_test.goindex 73e09dfb1..0b9203c90 100644--- a/x/models/qwen3_5/gdn_projections_test.go+++ b/x/models/qwen3_5/gdn_projections_test.go@@ -4,6 +4,7 @@ import ("math""testing"+ "github.com/ollama/ollama/x/internal/mlxtest""github.com/ollama/ollama/x/mlxrunner/mlx""github.com/ollama/ollama/x/models/nn")@@ -25,7 +26,7 @@ func patternArray(rows, cols int) *mlx.Array {return mlx.FromValues(values, rows, cols).AsType(mlx.DTypeBFloat16)}-func assertBitEqual(t *testing.T, label string, got, want *mlx.Array) {+func assertBitEqual(t *mlxtest.T, label string, got, want *mlx.Array) {t.Helper()if got == nil || want == nil {if got != want {@@ -57,82 +58,85 @@ func scatterRows(packed *mlx.Array, perm []int32) *mlx.Array {}func TestPackGatedDeltaProjectionsNativeMatchesSplit(t *testing.T) {- skipIfNoMLX(t)- cfg := gdnTestConfig()- keyDim := int(cfg.LinearNumKeyHeads * cfg.LinearKeyHeadDim)- valueDim := int(cfg.LinearNumValueHeads * cfg.LinearValueHeadDim)- in := 8-- qkvW := patternArray(2*keyDim+valueDim, in)- zW := patternArray(valueDim, in)- bW := patternArray(int(cfg.LinearNumValueHeads), in)- aW := patternArray(int(cfg.LinearNumValueHeads), in)-- fromSplitQKVZ, fromSplitBA, err := packGatedDeltaProjections(- nn.NewLinear(qkvW, nil), nn.NewLinear(zW, nil),- nn.NewLinear(bW, nil), nn.NewLinear(aW, nil),- nil, nil, cfg,- )- if err != nil {- t.Fatal(err)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := gdnTestConfig()+ keyDim := int(cfg.LinearNumKeyHeads * cfg.LinearKeyHeadDim)+ valueDim := int(cfg.LinearNumValueHeads * cfg.LinearValueHeadDim)+ in := 8++ qkvW := patternArray(2*keyDim+valueDim, in)+ zW := patternArray(valueDim, in)+ bW := patternArray(int(cfg.LinearNumValueHeads), in)+ aW := patternArray(int(cfg.LinearNumValueHeads), in)++ fromSplitQKVZ, fromSplitBA, err := packGatedDeltaProjections(+ nn.NewLinear(qkvW, nil), nn.NewLinear(zW, nil),+ nn.NewLinear(bW, nil), nn.NewLinear(aW, nil),+ nil, nil, cfg,+ )+ if err != nil {+ t.Fatal(err)+ }- packedQKVZ := mlx.Concatenate([]*mlx.Array{qkvW, zW}, 0)- packedBA := mlx.Concatenate([]*mlx.Array{bW, aW}, 0)- nativeQKVZ := scatterRows(packedQKVZ, nativeQKVZPerm(cfg))- nativeBA := scatterRows(packedBA, nativeBAPerm(cfg))-- fromNativeQKVZ, fromNativeBA, err := packGatedDeltaProjections(- nil, nil, nil, nil,- nn.NewLinear(nativeQKVZ, nil), nn.NewLinear(nativeBA, nil),- cfg,- )- if err != nil {- t.Fatal(err)- }+ packedQKVZ := mlx.Concatenate([]*mlx.Array{qkvW, zW}, 0)+ packedBA := mlx.Concatenate([]*mlx.Array{bW, aW}, 0)+ nativeQKVZ := scatterRows(packedQKVZ, nativeQKVZPerm(cfg))+ nativeBA := scatterRows(packedBA, nativeBAPerm(cfg))++ fromNativeQKVZ, fromNativeBA, err := packGatedDeltaProjections(+ nil, nil, nil, nil,+ nn.NewLinear(nativeQKVZ, nil), nn.NewLinear(nativeBA, nil),+ cfg,+ )+ if err != nil {+ t.Fatal(err)+ }- assertBitEqual(t, "qkvz", fromNativeQKVZ.(*nn.Linear).Weight, fromSplitQKVZ.(*nn.Linear).Weight)- assertBitEqual(t, "ba", fromNativeBA.(*nn.Linear).Weight, fromSplitBA.(*nn.Linear).Weight)+ assertBitEqual(t, "qkvz", fromNativeQKVZ.(*nn.Linear).Weight, fromSplitQKVZ.(*nn.Linear).Weight)+ assertBitEqual(t, "ba", fromNativeBA.(*nn.Linear).Weight, fromSplitBA.(*nn.Linear).Weight)+ })}func TestConcatProjectionPairQuantized(t *testing.T) {- skipIfNoMLX(t)- in := 64- hi := nn.NewQuantizedLinear(patternArray(16, in).AsType(mlx.DTypeFloat32), nil, 32, 4, "affine")- lo := nn.NewQuantizedLinear(patternArray(8, in).AsType(mlx.DTypeFloat32), nil, 32, 4, "affine")-- packed, err := concatProjectionPair(hi, lo)- if err != nil {- t.Fatal(err)- }- q, ok := packed.(*nn.QuantizedLinear)- if !ok {- t.Fatalf("packed projection is %T, want *nn.QuantizedLinear", packed)- }- assertBitEqual(t, "weight", q.Weight, mlx.Concatenate([]*mlx.Array{hi.Weight, lo.Weight}, 0))- assertBitEqual(t, "scales", q.Scales, mlx.Concatenate([]*mlx.Array{hi.Scales, lo.Scales}, 0))+ mlxtest.Run(t, func(t *mlxtest.T) {+ in := 64+ hi := nn.NewQuantizedLinear(patternArray(16, in).AsType(mlx.DTypeFloat32), nil, 32, 4, "affine")+ lo := nn.NewQuantizedLinear(patternArray(8, in).AsType(mlx.DTypeFloat32), nil, 32, 4, "affine")++ packed, err := concatProjectionPair(hi, lo)+ if err != nil {+ t.Fatal(err)+ }+ q, ok := packed.(*nn.QuantizedLinear)+ if !ok {+ t.Fatalf("packed projection is %T, want *nn.QuantizedLinear", packed)+ }+ assertBitEqual(t, "weight", q.Weight, mlx.Concatenate([]*mlx.Array{hi.Weight, lo.Weight}, 0))+ assertBitEqual(t, "scales", q.Scales, mlx.Concatenate([]*mlx.Array{hi.Scales, lo.Scales}, 0))+ })}func TestConcatProjectionPairMixedFallsBackToDense(t *testing.T) {- skipIfNoMLX(t)- in := 64- hi := nn.NewQuantizedLinear(patternArray(16, in).AsType(mlx.DTypeFloat32), nil, 32, 4, "affine")- loW := patternArray(8, in)-- packed, err := concatProjectionPair(hi, nn.NewLinear(loW, nil))- if err != nil {- t.Fatal(err)- }- dense, ok := packed.(*nn.Linear)- if !ok {- t.Fatalf("packed projection is %T, want *nn.Linear", packed)- }- want := mlx.Concatenate([]*mlx.Array{- mlx.Dequantize(hi.Weight, hi.Scales, hi.QBiases, hi.GroupSize, hi.Bits, hi.Mode, nil),- loW.AsType(mlx.DTypeFloat16),- }, 0)- if dense.Weight.Dim(0) != 24 {- t.Fatalf("packed rows = %d, want 24", dense.Weight.Dim(0))- }- assertBitEqual(t, "weight", dense.Weight, want.AsType(dense.Weight.DType()))+ mlxtest.Run(t, func(t *mlxtest.T) {+ in := 64+ hi := nn.NewQuantizedLinear(patternArray(16, in).AsType(mlx.DTypeFloat32), nil, 32, 4, "affine")+ loW := patternArray(8, in)++ packed, err := concatProjectionPair(hi, nn.NewLinear(loW, nil))+ if err != nil {+ t.Fatal(err)+ }+ dense, ok := packed.(*nn.Linear)+ if !ok {+ t.Fatalf("packed projection is %T, want *nn.Linear", packed)+ }+ want := mlx.Concatenate([]*mlx.Array{+ mlx.Dequantize(hi.Weight, hi.Scales, hi.QBiases, hi.GroupSize, hi.Bits, hi.Mode, nil),+ loW.AsType(mlx.DTypeFloat16),+ }, 0)+ if dense.Weight.Dim(0) != 24 {+ t.Fatalf("packed rows = %d, want 24", dense.Weight.Dim(0))+ }+ assertBitEqual(t, "weight", dense.Weight, want.AsType(dense.Weight.DType()))+ })}diff --git a/x/models/qwen3_5/qwen3_5_test.go b/x/models/qwen3_5/qwen3_5_test.goindex c136ddfb0..ea35f782a 100644--- a/x/models/qwen3_5/qwen3_5_test.go+++ b/x/models/qwen3_5/qwen3_5_test.go@@ -8,33 +8,26 @@ import ("github.com/ollama/ollama/x/mlxrunner/mlx")-func skipIfNoMLX(t *testing.T) {- t.Helper()- if err := mlx.CheckInit(); err != nil {- t.Skipf("MLX not available: %v", err)- }-}-func TestSanitizeConvWeight(t *testing.T) {- mlxtest.Setup(t)-- tests := []struct {- name string- shape []int- want []int- }{- {name: "publisher layout", shape: []int{8, 1, 4}, want: []int{8, 4}},- {name: "imported layout", shape: []int{8, 4, 1}, want: []int{8, 4}},- {name: "already sanitized", shape: []int{8, 4}, want: []int{8, 4}},- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ tests := []struct {+ name string+ shape []int+ want []int+ }{+ {name: "publisher layout", shape: []int{8, 1, 4}, want: []int{8, 4}},+ {name: "imported layout", shape: []int{8, 4, 1}, want: []int{8, 4}},+ {name: "already sanitized", shape: []int{8, 4}, want: []int{8, 4}},+ }- for _, tt := range tests {- got := sanitizeConvWeight(mlx.Zeros(mlx.DTypeBFloat16, tt.shape...))- mlx.Eval(got)- if dims := got.Dims(); len(dims) != len(tt.want) || dims[0] != tt.want[0] || dims[1] != tt.want[1] {- t.Fatalf("%s: sanitizeConvWeight() shape = %v, want %v", tt.name, dims, tt.want)+ for _, tt := range tests {+ got := sanitizeConvWeight(mlx.Zeros(mlx.DTypeBFloat16, tt.shape...))+ mlx.Eval(got)+ if dims := got.Dims(); len(dims) != len(tt.want) || dims[0] != tt.want[0] || dims[1] != tt.want[1] {+ t.Fatalf("%s: sanitizeConvWeight() shape = %v, want %v", tt.name, dims, tt.want)+ }}- }+ })}func TestParseConfigNestedDefaults(t *testing.T) {@@ -209,182 +202,182 @@ func TestNewCachesLayout(t *testing.T) {}func TestLoadWeightsPreservesLinearAttentionNormWeightDType(t *testing.T) {- skipIfNoMLX(t)-- cfg := &Config{- HiddenSize: 4,- IntermediateSize: 8,- NumHiddenLayers: 2,- NumAttentionHeads: 1,- NumKeyValueHeads: 1,- HeadDim: 4,- RMSNormEps: 1e-6,- TieWordEmbeddings: true,- LayerTypes: []string{"linear", "full"},- LinearNumValueHeads: 1,- LinearNumKeyHeads: 1,- LinearKeyHeadDim: 2,- LinearValueHeadDim: 2,- LinearConvKernelDim: 4,- FullAttentionInterval: 2,- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{+ HiddenSize: 4,+ IntermediateSize: 8,+ NumHiddenLayers: 2,+ NumAttentionHeads: 1,+ NumKeyValueHeads: 1,+ HeadDim: 4,+ RMSNormEps: 1e-6,+ TieWordEmbeddings: true,+ LayerTypes: []string{"linear", "full"},+ LinearNumValueHeads: 1,+ LinearNumKeyHeads: 1,+ LinearKeyHeadDim: 2,+ LinearValueHeadDim: 2,+ LinearConvKernelDim: 4,+ FullAttentionInterval: 2,+ }- m := &Model{- Config: cfg,- Layers: make([]*Layer, cfg.NumHiddenLayers),- }+ m := &Model{+ Config: cfg,+ Layers: make([]*Layer, cfg.NumHiddenLayers),+ }- bf16 := mlx.DTypeBFloat16- f32 := mlx.DTypeFloat32- tensors := map[string]*mlx.Array{- "model.embed_tokens.weight": mlx.FromValues([]float32{1, 2, 3, 4, 5, 6, 7, 8}, 2, 4).AsType(bf16),- "model.norm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),- "model.layers.0.input_layernorm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),- "model.layers.0.post_attention_layernorm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),- "model.layers.0.linear_attn.in_proj_qkv.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- 1, 1, 0, 0,- 0, 1, 1, 0,- }, 6, 4),- "model.layers.0.linear_attn.in_proj_z.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- }, 2, 4),- "model.layers.0.linear_attn.in_proj_b.weight": mlx.FromValues([]float32{1, 0, 0, 0}, 1, 4),- "model.layers.0.linear_attn.in_proj_a.weight": mlx.FromValues([]float32{0, 1, 0, 0}, 1, 4),- "model.layers.0.linear_attn.out_proj.weight": mlx.FromValues([]float32{- 1, 0,- 0, 1,- 1, 1,- 0, 0,- }, 4, 2),- "model.layers.0.linear_attn.conv1d.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- 1, 1, 0, 0,- 0, 1, 1, 0,- }, 6, 4),- "model.layers.0.linear_attn.norm.weight": mlx.FromValues([]float32{1, 1}, 2),- "model.layers.0.linear_attn.dt_bias": mlx.FromValues([]float32{0}, 1),- "model.layers.0.linear_attn.A_log": mlx.FromValues([]float32{0}, 1),- "model.layers.0.mlp.gate_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- 1, 1, 0, 0,- 0, 1, 1, 0,- 0, 0, 1, 1,- 1, 0, 0, 1,- }, 8, 4),- "model.layers.0.mlp.up_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- 1, 1, 0, 0,- 0, 1, 1, 0,- 0, 0, 1, 1,- 1, 0, 0, 1,- }, 8, 4),- "model.layers.0.mlp.down_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0, 0, 0, 0, 0,- 0, 1, 0, 0, 0, 0, 0, 0,- 0, 0, 1, 0, 0, 0, 0, 0,- 0, 0, 0, 1, 0, 0, 0, 0,- }, 4, 8),- "model.layers.1.input_layernorm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),- "model.layers.1.post_attention_layernorm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),- "model.layers.1.self_attn.q_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- 1, 1, 0, 0,- 0, 1, 1, 0,- 0, 0, 1, 1,- 1, 0, 0, 1,- }, 8, 4),- "model.layers.1.self_attn.k_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- }, 4, 4),- "model.layers.1.self_attn.v_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- }, 4, 4),- "model.layers.1.self_attn.o_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- }, 4, 4),- "model.layers.1.self_attn.q_norm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),- "model.layers.1.self_attn.k_norm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),- "model.layers.1.mlp.gate_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- 1, 1, 0, 0,- 0, 1, 1, 0,- 0, 0, 1, 1,- 1, 0, 0, 1,- }, 8, 4),- "model.layers.1.mlp.up_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0,- 0, 1, 0, 0,- 0, 0, 1, 0,- 0, 0, 0, 1,- 1, 1, 0, 0,- 0, 1, 1, 0,- 0, 0, 1, 1,- 1, 0, 0, 1,- }, 8, 4),- "model.layers.1.mlp.down_proj.weight": mlx.FromValues([]float32{- 1, 0, 0, 0, 0, 0, 0, 0,- 0, 1, 0, 0, 0, 0, 0, 0,- 0, 0, 1, 0, 0, 0, 0, 0,- 0, 0, 0, 1, 0, 0, 0, 0,- }, 4, 8),- }+ bf16 := mlx.DTypeBFloat16+ f32 := mlx.DTypeFloat32+ tensors := map[string]*mlx.Array{+ "model.embed_tokens.weight": mlx.FromValues([]float32{1, 2, 3, 4, 5, 6, 7, 8}, 2, 4).AsType(bf16),+ "model.norm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),+ "model.layers.0.input_layernorm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),+ "model.layers.0.post_attention_layernorm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),+ "model.layers.0.linear_attn.in_proj_qkv.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ 1, 1, 0, 0,+ 0, 1, 1, 0,+ }, 6, 4),+ "model.layers.0.linear_attn.in_proj_z.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ }, 2, 4),+ "model.layers.0.linear_attn.in_proj_b.weight": mlx.FromValues([]float32{1, 0, 0, 0}, 1, 4),+ "model.layers.0.linear_attn.in_proj_a.weight": mlx.FromValues([]float32{0, 1, 0, 0}, 1, 4),+ "model.layers.0.linear_attn.out_proj.weight": mlx.FromValues([]float32{+ 1, 0,+ 0, 1,+ 1, 1,+ 0, 0,+ }, 4, 2),+ "model.layers.0.linear_attn.conv1d.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ 1, 1, 0, 0,+ 0, 1, 1, 0,+ }, 6, 4),+ "model.layers.0.linear_attn.norm.weight": mlx.FromValues([]float32{1, 1}, 2),+ "model.layers.0.linear_attn.dt_bias": mlx.FromValues([]float32{0}, 1),+ "model.layers.0.linear_attn.A_log": mlx.FromValues([]float32{0}, 1),+ "model.layers.0.mlp.gate_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ 1, 1, 0, 0,+ 0, 1, 1, 0,+ 0, 0, 1, 1,+ 1, 0, 0, 1,+ }, 8, 4),+ "model.layers.0.mlp.up_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ 1, 1, 0, 0,+ 0, 1, 1, 0,+ 0, 0, 1, 1,+ 1, 0, 0, 1,+ }, 8, 4),+ "model.layers.0.mlp.down_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0, 0, 0, 0, 0,+ 0, 1, 0, 0, 0, 0, 0, 0,+ 0, 0, 1, 0, 0, 0, 0, 0,+ 0, 0, 0, 1, 0, 0, 0, 0,+ }, 4, 8),+ "model.layers.1.input_layernorm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),+ "model.layers.1.post_attention_layernorm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),+ "model.layers.1.self_attn.q_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ 1, 1, 0, 0,+ 0, 1, 1, 0,+ 0, 0, 1, 1,+ 1, 0, 0, 1,+ }, 8, 4),+ "model.layers.1.self_attn.k_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ }, 4, 4),+ "model.layers.1.self_attn.v_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ }, 4, 4),+ "model.layers.1.self_attn.o_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ }, 4, 4),+ "model.layers.1.self_attn.q_norm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),+ "model.layers.1.self_attn.k_norm.weight": mlx.FromValues([]float32{1, 1, 1, 1}, 4),+ "model.layers.1.mlp.gate_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ 1, 1, 0, 0,+ 0, 1, 1, 0,+ 0, 0, 1, 1,+ 1, 0, 0, 1,+ }, 8, 4),+ "model.layers.1.mlp.up_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0,+ 0, 1, 0, 0,+ 0, 0, 1, 0,+ 0, 0, 0, 1,+ 1, 1, 0, 0,+ 0, 1, 1, 0,+ 0, 0, 1, 1,+ 1, 0, 0, 1,+ }, 8, 4),+ "model.layers.1.mlp.down_proj.weight": mlx.FromValues([]float32{+ 1, 0, 0, 0, 0, 0, 0, 0,+ 0, 1, 0, 0, 0, 0, 0, 0,+ 0, 0, 1, 0, 0, 0, 0, 0,+ 0, 0, 0, 1, 0, 0, 0, 0,+ }, 4, 8),+ }- if err := m.LoadWeights(tensors); err != nil {- t.Fatalf("LoadWeights failed: %v", err)- }+ if err := m.LoadWeights(tensors); err != nil {+ t.Fatalf("LoadWeights failed: %v", err)+ }- if got := m.Layers[0].InputNorm.Weight.DType(); got != f32 {- t.Fatalf("layer 0 input norm dtype = %v, want %v", got, f32)- }- if got := m.Layers[0].PostAttentionNorm.Weight.DType(); got != f32 {- t.Fatalf("layer 0 post-attn norm dtype = %v, want %v", got, f32)- }- if got := m.Layers[1].InputNorm.Weight.DType(); got != f32 {- t.Fatalf("layer 1 input norm dtype = %v, want %v", got, f32)- }- if got := m.Layers[1].PostAttentionNorm.Weight.DType(); got != f32 {- t.Fatalf("layer 1 post-attn norm dtype = %v, want %v", got, f32)- }+ if got := m.Layers[0].InputNorm.Weight.DType(); got != f32 {+ t.Fatalf("layer 0 input norm dtype = %v, want %v", got, f32)+ }+ if got := m.Layers[0].PostAttentionNorm.Weight.DType(); got != f32 {+ t.Fatalf("layer 0 post-attn norm dtype = %v, want %v", got, f32)+ }+ if got := m.Layers[1].InputNorm.Weight.DType(); got != f32 {+ t.Fatalf("layer 1 input norm dtype = %v, want %v", got, f32)+ }+ if got := m.Layers[1].PostAttentionNorm.Weight.DType(); got != f32 {+ t.Fatalf("layer 1 post-attn norm dtype = %v, want %v", got, f32)+ }- if got := m.Norm.Weight.DType(); got != f32 {- t.Fatalf("final norm dtype = %v, want %v", got, f32)- }- if got := m.Layers[0].Linear.NormWeight.DType(); got != f32 {- t.Fatalf("linear-attn norm dtype = %v, want %v", got, f32)- }- if got := m.Layers[1].FullAttn.QNorm.Weight.DType(); got != f32 {- t.Fatalf("q norm dtype = %v, want %v", got, f32)- }- if got := m.Layers[1].FullAttn.KNorm.Weight.DType(); got != f32 {- t.Fatalf("k norm dtype = %v, want %v", got, f32)- }+ if got := m.Norm.Weight.DType(); got != f32 {+ t.Fatalf("final norm dtype = %v, want %v", got, f32)+ }+ if got := m.Layers[0].Linear.NormWeight.DType(); got != f32 {+ t.Fatalf("linear-attn norm dtype = %v, want %v", got, f32)+ }+ if got := m.Layers[1].FullAttn.QNorm.Weight.DType(); got != f32 {+ t.Fatalf("q norm dtype = %v, want %v", got, f32)+ }+ if got := m.Layers[1].FullAttn.KNorm.Weight.DType(); got != f32 {+ t.Fatalf("k norm dtype = %v, want %v", got, f32)+ }+ })}diff --git a/x/models/qwen3_5/vision_test.go b/x/models/qwen3_5/vision_test.goindex e96307f6c..0099a84a7 100644--- a/x/models/qwen3_5/vision_test.go+++ b/x/models/qwen3_5/vision_test.go@@ -13,12 +13,13 @@ import ()func TestVisionAdapterWeightsAreCollectable(t *testing.T) {- mlxtest.Setup(t)- weight := mlx.FromValue(float32(1))- adapter := &VisionAdapter{Model: &Model{VisionTower: &VisionTower{PosEmbed: weight}}}- if got := mlx.Collect(adapter); len(got) != 1 || got[0] != weight {- t.Fatalf("mlx.Collect(adapter) = %v, want the tower weight", got)- }+ mlxtest.Run(t, func(t *mlxtest.T) {+ weight := mlx.FromValue(float32(1))+ adapter := &VisionAdapter{Model: &Model{VisionTower: &VisionTower{PosEmbed: weight}}}+ if got := mlx.Collect(adapter); len(got) != 1 || got[0] != weight {+ t.Fatalf("mlx.Collect(adapter) = %v, want the tower weight", got)+ }+ })}func TestSmartResize(t *testing.T) {diff --git a/x/models/qwen4_exp/engram_test.go b/x/models/qwen4_exp/engram_test.goindex 44a3f5f5b..f356e911c 100644--- a/x/models/qwen4_exp/engram_test.go+++ b/x/models/qwen4_exp/engram_test.go@@ -10,96 +10,98 @@ import ()func TestEngramHashes(t *testing.T) {- mlxtest.Setup(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ p := &PLE{+ LayerMultipliers: mlx.FromValues([]int64{3, 5, 7}, 3),+ HeadVocabSizes: mlx.FromValues([]int64{11, 13, 17, 19}, 4),+ HeadOffsets: mlx.FromValues([]int64{0, 11, 24, 41}, 4),+ }+ cfg := &Config{NGramSize: 3, HeadsPerNGram: 2, EOSTokenID: 9}+ b := &batch.Batch{InputIDs: mlx.FromValues([]int32{1, 9, 2, 3}, 1, 4)}+ history := mlx.FromValues([]int64{7, 8}, 1, 2)- p := &PLE{- LayerMultipliers: mlx.FromValues([]int64{3, 5, 7}, 3),- HeadVocabSizes: mlx.FromValues([]int64{11, 13, 17, 19}, 4),- HeadOffsets: mlx.FromValues([]int64{0, 11, 24, 41}, 4),- }- cfg := &Config{NGramSize: 3, HeadsPerNGram: 2, EOSTokenID: 9}- b := &batch.Batch{InputIDs: mlx.FromValues([]int32{1, 9, 2, 3}, 1, 4)}- history := mlx.FromValues([]int64{7, 8}, 1, 2)-- got, _ := p.hashes(b, history, cfg)- got = got.AsType(mlx.DTypeInt32)- mlx.Eval(got)- values := got.Ints()- want := []int32{10, 15, 33, 48, 8, 15, 28, 41, 10, 15, 27, 42, 3, 14, 33, 44}- if !slices.Equal(values, want) {- t.Fatalf("hashes = %v, want %v", values, want)- }+ got, _ := p.hashes(b, history, cfg)+ got = got.AsType(mlx.DTypeInt32)+ mlx.Eval(got)+ values := got.Ints()+ want := []int32{10, 15, 33, 48, 8, 15, 28, 41, 10, 15, 27, 42, 3, 14, 33, 44}+ if !slices.Equal(values, want) {+ t.Fatalf("hashes = %v, want %v", values, want)+ }+ })}func TestEngramCacheCarriesChunkHistory(t *testing.T) {- mlxtest.Setup(t)- c := newEngramCache(2, 3, 1, 9)- t.Cleanup(c.Free)- b := &batch.Batch{- InputIDs: mlx.FromValues([]int32{1, 2}, 1, 2),- SeqQueryLens: []int32{2},- }- c.put(b, mlx.FromValues([]int64{1, 2}, 1, 2), mlx.FromValues([]float32{10, 20}, 1, 2, 1))- b.InputIDs = mlx.FromValues([]int32{3, 4}, 1, 2)- c.put(b, mlx.FromValues([]int64{3, 4}, 1, 2), mlx.FromValues([]float32{30, 40}, 1, 2, 1))+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := newEngramCache(2, 3, 1, 9)+ t.Cleanup(c.Free)+ b := &batch.Batch{+ InputIDs: mlx.FromValues([]int32{1, 2}, 1, 2),+ SeqQueryLens: []int32{2},+ }+ c.put(b, mlx.FromValues([]int64{1, 2}, 1, 2), mlx.FromValues([]float32{10, 20}, 1, 2, 1))+ b.InputIDs = mlx.FromValues([]int32{3, 4}, 1, 2)+ c.put(b, mlx.FromValues([]int64{3, 4}, 1, 2), mlx.FromValues([]float32{30, 40}, 1, 2, 1))- history := c.history.AsType(mlx.DTypeInt32)- mlx.Eval(history, c.convHistory)- if got, want := history.Ints(), []int32{3, 4}; !slices.Equal(got, want) {- t.Fatalf("token history = %v, want %v", got, want)- }- if got, want := c.convHistory.Floats(), []float32{20, 30, 40}; !slices.Equal(got, want) {- t.Fatalf("conv history = %v, want %v", got, want)- }+ history := c.history.AsType(mlx.DTypeInt32)+ mlx.Eval(history, c.convHistory)+ if got, want := history.Ints(), []int32{3, 4}; !slices.Equal(got, want) {+ t.Fatalf("token history = %v, want %v", got, want)+ }+ if got, want := c.convHistory.Floats(), []float32{20, 30, 40}; !slices.Equal(got, want) {+ t.Fatalf("conv history = %v, want %v", got, want)+ }+ })}func TestEngramCacheRestoresScheduledSnapshot(t *testing.T) {- mlxtest.Setup(t)- c := newEngramCache(2, 3, 1, 9)- t.Cleanup(c.Free)- b := &batch.Batch{- InputIDs: mlx.FromValues([]int32{1, 2}, 1, 2),- SeqQueryLens: []int32{2},- }- c.put(b, mlx.FromValues([]int64{1, 2}, 1, 2), mlx.FromValues([]float32{10, 20}, 1, 2, 1))+ mlxtest.Run(t, func(t *mlxtest.T) {+ c := newEngramCache(2, 3, 1, 9)+ t.Cleanup(c.Free)+ b := &batch.Batch{+ InputIDs: mlx.FromValues([]int32{1, 2}, 1, 2),+ SeqQueryLens: []int32{2},+ }+ c.put(b, mlx.FromValues([]int64{1, 2}, 1, 2), mlx.FromValues([]float32{10, 20}, 1, 2, 1))- c.PrepareSnapshots([]int{3, 4})- b.InputIDs = mlx.FromValues([]int32{3, 4}, 1, 2)- inputIDs := mlx.FromValues([]int64{3, 4}, 1, 2)- convInput := mlx.FromValues([]float32{30, 40}, 1, 2, 1)- c.put(b, inputIDs, convInput)- snapshots := c.TakeSnapshots()- if len(snapshots) != 2 || snapshots[0] == nil || snapshots[1] == nil {- t.Fatalf("TakeSnapshots() = %v, want two captured snapshots", snapshots)- }- for _, snapshot := range snapshots {- t.Cleanup(snapshot.Close)- }- first := snapshots[0].(*engramSnapshot)- mlx.Eval(first.history, first.convHistory)- if got, want := first.convHistory.Floats(), []float32{10, 20, 30}; !slices.Equal(got, want) {- t.Fatalf("captured convolution history = %v, want %v", got, want)- }+ c.PrepareSnapshots([]int{3, 4})+ b.InputIDs = mlx.FromValues([]int32{3, 4}, 1, 2)+ inputIDs := mlx.FromValues([]int64{3, 4}, 1, 2)+ convInput := mlx.FromValues([]float32{30, 40}, 1, 2, 1)+ c.put(b, inputIDs, convInput)+ snapshots := c.TakeSnapshots()+ if len(snapshots) != 2 || snapshots[0] == nil || snapshots[1] == nil {+ t.Fatalf("TakeSnapshots() = %v, want two captured snapshots", snapshots)+ }+ for _, snapshot := range snapshots {+ t.Cleanup(snapshot.Close)+ }+ first := snapshots[0].(*engramSnapshot)+ mlx.Eval(first.history, first.convHistory)+ if got, want := first.convHistory.Floats(), []float32{10, 20, 30}; !slices.Equal(got, want) {+ t.Fatalf("captured convolution history = %v, want %v", got, want)+ }- b.InputIDs = mlx.FromValues([]int32{5, 6}, 1, 2)- c.put(b, mlx.FromValues([]int64{5, 6}, 1, 2), mlx.FromValues([]float32{50, 60}, 1, 2, 1))- if !c.Restore(snapshots[0], 3) {- t.Fatal("Restore(snapshot, 3) failed")- }- history := c.history.AsType(mlx.DTypeInt32)- mlx.Eval(history, c.convHistory)- if got, want := history.Ints(), []int32{2, 3}; !slices.Equal(got, want) {- t.Fatalf("restored token history = %v, want %v", got, want)- }- if got, want := c.convHistory.Floats(), []float32{10, 20, 30}; !slices.Equal(got, want) {- t.Fatalf("restored convolution history = %v, want %v", got, want)- }+ b.InputIDs = mlx.FromValues([]int32{5, 6}, 1, 2)+ c.put(b, mlx.FromValues([]int64{5, 6}, 1, 2), mlx.FromValues([]float32{50, 60}, 1, 2, 1))+ if !c.Restore(snapshots[0], 3) {+ t.Fatal("Restore(snapshot, 3) failed")+ }+ history := c.history.AsType(mlx.DTypeInt32)+ mlx.Eval(history, c.convHistory)+ if got, want := history.Ints(), []int32{2, 3}; !slices.Equal(got, want) {+ t.Fatalf("restored token history = %v, want %v", got, want)+ }+ if got, want := c.convHistory.Floats(), []float32{10, 20, 30}; !slices.Equal(got, want) {+ t.Fatalf("restored convolution history = %v, want %v", got, want)+ }- parent, child := c.Split(snapshots[1], 3)- if parent != nil || child != snapshots[1] {- t.Fatalf("Split(snapshot) = (%v, %v), want (nil, snapshot)", parent, child)- }- if merged := c.Merge(nil, child); merged != child {- t.Fatalf("Merge(nil, child) = %v, want child", merged)- }+ parent, child := c.Split(snapshots[1], 3)+ if parent != nil || child != snapshots[1] {+ t.Fatalf("Split(snapshot) = (%v, %v), want (nil, snapshot)", parent, child)+ }+ if merged := c.Merge(nil, child); merged != child {+ t.Fatalf("Merge(nil, child) = %v, want child", merged)+ }+ })}diff --git a/x/models/qwen4_exp/hyper_connection_test.go b/x/models/qwen4_exp/hyper_connection_test.goindex 58657b8a6..e6c93950b 100644--- a/x/models/qwen4_exp/hyper_connection_test.go+++ b/x/models/qwen4_exp/hyper_connection_test.go@@ -10,67 +10,67 @@ import ()func TestHyperConnectionMatchesReferenceFormula(t *testing.T) {- mlxtest.Setup(t)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{HCCount: 2, HiddenSize: 2, HCLowRank: 2, RMSNormEps: 1e-6}+ normWeight := []float32{1.1, 0.9, 1.2, 0.8}+ downWeight := [][]float32{{0.2, -0.3, 0.4, 0.1}, {-0.1, 0.5, 0.2, -0.4}}+ upWeight := [][]float32{{0.3, -0.2}, {0.1, 0.4}, {-0.5, 0.2}, {0.25, 0.15}}+ injectWeight := [][]float32{{0.2, -0.1, 0.3, 0.4}, {-0.3, 0.2, 0.1, 0.5}}+ input := []float32{1, 2, 3, 4}- cfg := &Config{HCCount: 2, HiddenSize: 2, HCLowRank: 2, RMSNormEps: 1e-6}- normWeight := []float32{1.1, 0.9, 1.2, 0.8}- downWeight := [][]float32{{0.2, -0.3, 0.4, 0.1}, {-0.1, 0.5, 0.2, -0.4}}- upWeight := [][]float32{{0.3, -0.2}, {0.1, 0.4}, {-0.5, 0.2}, {0.25, 0.15}}- injectWeight := [][]float32{{0.2, -0.1, 0.3, 0.4}, {-0.3, 0.2, 0.1, 0.5}}- input := []float32{1, 2, 3, 4}-- h := &hyperConnection{- Norm: &streamRMSNorm{Weight: mlx.FromValues(normWeight, 2, 2)},- InputMixDown: nn.NewLinear(matrix(downWeight), nil),- InputMixUp: nn.NewLinear(matrix(upWeight), nil),- BlockInject: nn.NewLinear(matrix(injectWeight), nil),- }- residual := mlx.FromValues(input, 1, 1, 4)- branch, state := h.Prepare(residual, cfg)- got := h.Inject(state, branch, cfg).AsType(mlx.DTypeFloat32)- reduced := h.Reduce(residual, cfg).AsType(mlx.DTypeFloat32)- mlx.Eval(got, reduced)-- normed := append([]float32(nil), input...)- for stream := range int(cfg.HCCount) {- start := stream * int(cfg.HiddenSize)- var square float64- for i := range int(cfg.HiddenSize) {- x := float64(input[start+i])- square += x * x+ h := &hyperConnection{+ Norm: &streamRMSNorm{Weight: mlx.FromValues(normWeight, 2, 2)},+ InputMixDown: nn.NewLinear(matrix(downWeight), nil),+ InputMixUp: nn.NewLinear(matrix(upWeight), nil),+ BlockInject: nn.NewLinear(matrix(injectWeight), nil),}- invRMS := 1 / math.Sqrt(square/float64(cfg.HiddenSize)+float64(cfg.RMSNormEps))- for i := range int(cfg.HiddenSize) {- normed[start+i] = float32(float64(input[start+i]) * invRMS * float64(normWeight[start+i]))+ residual := mlx.FromValues(input, 1, 1, 4)+ branch, state := h.Prepare(residual, cfg)+ got := h.Inject(state, branch, cfg).AsType(mlx.DTypeFloat32)+ reduced := h.Reduce(residual, cfg).AsType(mlx.DTypeFloat32)+ mlx.Eval(got, reduced)++ normed := append([]float32(nil), input...)+ for stream := range int(cfg.HCCount) {+ start := stream * int(cfg.HiddenSize)+ var square float64+ for i := range int(cfg.HiddenSize) {+ x := float64(input[start+i])+ square += x * x+ }+ invRMS := 1 / math.Sqrt(square/float64(cfg.HiddenSize)+float64(cfg.RMSNormEps))+ for i := range int(cfg.HiddenSize) {+ normed[start+i] = float32(float64(input[start+i]) * invRMS * float64(normWeight[start+i]))+ }}- }- down := matvec(downWeight, normed)- for i := range down {- down[i] /= float32(cfg.HCCount)- down[i] *= 1 / (1 + float32(math.Exp(float64(-down[i]))))- }- mix := matvec(upWeight, down)- wantBranch := make([]float32, cfg.HiddenSize)- for stream := range int(cfg.HCCount) {- for i := range int(cfg.HiddenSize) {- j := stream*int(cfg.HiddenSize) + i- gate := 1 / (1 + float32(math.Exp(float64(-mix[j]))))- wantBranch[i] += gate * normed[j] / float32(cfg.HCCount)+ down := matvec(downWeight, normed)+ for i := range down {+ down[i] /= float32(cfg.HCCount)+ down[i] *= 1 / (1 + float32(math.Exp(float64(-down[i]))))+ }+ mix := matvec(upWeight, down)+ wantBranch := make([]float32, cfg.HiddenSize)+ for stream := range int(cfg.HCCount) {+ for i := range int(cfg.HiddenSize) {+ j := stream*int(cfg.HiddenSize) + i+ gate := 1 / (1 + float32(math.Exp(float64(-mix[j]))))+ wantBranch[i] += gate * normed[j] / float32(cfg.HCCount)+ }}- }- injection := matvec(injectWeight, normed)- want := append([]float32(nil), input...)- for stream := range int(cfg.HCCount) {- weight := 2 / (1 + float32(math.Exp(float64(-injection[stream]/float32(cfg.HCCount)))))- for i := range int(cfg.HiddenSize) {- want[stream*int(cfg.HiddenSize)+i] += weight * wantBranch[i]+ injection := matvec(injectWeight, normed)+ want := append([]float32(nil), input...)+ for stream := range int(cfg.HCCount) {+ weight := 2 / (1 + float32(math.Exp(float64(-injection[stream]/float32(cfg.HCCount)))))+ for i := range int(cfg.HiddenSize) {+ want[stream*int(cfg.HiddenSize)+i] += weight * wantBranch[i]+ }}- }- assertClose(t, "mixed branch", reduced.Floats(), wantBranch)- assertClose(t, "injected streams", got.Floats(), want)+ assertClose(t, "mixed branch", reduced.Floats(), wantBranch)+ assertClose(t, "injected streams", got.Floats(), want)+ })}func matrix(rows [][]float32) *mlx.Array {@@ -91,7 +91,7 @@ func matvec(weight [][]float32, input []float32) []float32 {return output}-func assertClose(t *testing.T, name string, got, want []float32) {+func assertClose(t *mlxtest.T, name string, got, want []float32) {t.Helper()if len(got) != len(want) {t.Fatalf("%s length = %d, want %d", name, len(got), len(want))diff --git a/x/models/qwen4_exp/qsa_test.go b/x/models/qwen4_exp/qsa_test.goindex 9bf367c9a..f86dde497 100644--- a/x/models/qwen4_exp/qsa_test.go+++ b/x/models/qwen4_exp/qsa_test.go@@ -12,114 +12,119 @@ import ()func TestQSASelectsCompressedBlocksAndCausalTail(t *testing.T) {- mlxtest.Setup(t)- cfg := &Config{IndexerBudget: 8, IndexerCompressRatio: 4}- scores := mlx.FromValues([]float32{0.1, 4, 2, 3}, 1, 1, 4)- b := &batch.Batch{SeqOffsets: []int32{16}}+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{IndexerBudget: 8, IndexerCompressRatio: 4}+ scores := mlx.FromValues([]float32{0.1, 4, 2, 3}, 1, 1, 4)+ b := &batch.Batch{SeqOffsets: []int32{16}}- indices, valid := qsaLogicalIndices(scores, b, 16, cfg)- indices = indices.AsType(mlx.DTypeInt32)- valid = valid.AsType(mlx.DTypeInt32)- mlx.Eval(indices, valid)+ indices, valid := qsaLogicalIndices(scores, b, 16, cfg)+ indices = indices.AsType(mlx.DTypeInt32)+ valid = valid.AsType(mlx.DTypeInt32)+ mlx.Eval(indices, valid)- values, mask := indices.Ints(), valid.Ints()- selected := make([]int32, 0, len(values))- for i, value := range values {- if mask[i] != 0 {- selected = append(selected, value)+ values, mask := indices.Ints(), valid.Ints()+ selected := make([]int32, 0, len(values))+ for i, value := range values {+ if mask[i] != 0 {+ selected = append(selected, value)+ }}- }- slices.Sort(selected)- want := []int32{4, 5, 6, 7, 12, 13, 14, 15, 16}- if !slices.Equal(selected, want) {- t.Fatalf("selected indices = %v, want %v", selected, want)- }+ slices.Sort(selected)+ want := []int32{4, 5, 6, 7, 12, 13, 14, 15, 16}+ if !slices.Equal(selected, want) {+ t.Fatalf("selected indices = %v, want %v", selected, want)+ }+ })}func TestQSASelectionMasksFutureBlocks(t *testing.T) {- mlxtest.Setup(t)- cfg := &Config{IndexerBudget: 8, IndexerCompressRatio: 4}- // Only block 0 and token 4 are visible. Give every future block a much- // larger score so the test fails if selection sees cached-but-causal-junk.- scores := mlx.FromValues([]float32{0.1, 100, 90, 80, 70}, 1, 1, 5)- b := &batch.Batch{SeqOffsets: []int32{4}}+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{IndexerBudget: 8, IndexerCompressRatio: 4}+ // Only block 0 and token 4 are visible. Give every future block a much+ // larger score so the test fails if selection sees cached-but-causal-junk.+ scores := mlx.FromValues([]float32{0.1, 100, 90, 80, 70}, 1, 1, 5)+ b := &batch.Batch{SeqOffsets: []int32{4}}- indices, valid := qsaLogicalIndices(scores, b, 20, cfg)- indices = indices.AsType(mlx.DTypeInt32)- valid = valid.AsType(mlx.DTypeInt32)- mlx.Eval(indices, valid)+ indices, valid := qsaLogicalIndices(scores, b, 20, cfg)+ indices = indices.AsType(mlx.DTypeInt32)+ valid = valid.AsType(mlx.DTypeInt32)+ mlx.Eval(indices, valid)- var selected []int32- for i, value := range indices.Ints() {- if valid.Ints()[i] != 0 {- selected = append(selected, value)+ var selected []int32+ for i, value := range indices.Ints() {+ if valid.Ints()[i] != 0 {+ selected = append(selected, value)+ }+ }+ slices.Sort(selected)+ if want := []int32{0, 1, 2, 3, 4}; !slices.Equal(selected, want) {+ t.Fatalf("selected indices = %v, want %v", selected, want)}- }- slices.Sort(selected)- if want := []int32{0, 1, 2, 3, 4}; !slices.Equal(selected, want) {- t.Fatalf("selected indices = %v, want %v", selected, want)- }+ })}func TestQSASparseAttentionMatchesReference(t *testing.T) {- mlxtest.Setup(t)- cfg := &Config{NumKeyValueHeads: 1, Scale: 1}- q := mlx.FromValues([]float32{1, 0}, 1, 1, 1, 2)- k := mlx.FromValues([]float32{1, 0, 0, 1, 2, 0}, 1, 1, 3, 2)- v := mlx.FromValues([]float32{10, 1, 20, 2, 30, 3}, 1, 1, 3, 2)- indices := mlx.FromValues([]int32{2, 0}, 1, 1, 2)- valid := mlx.FromValues([]bool{true, true}, 1, 1, 2)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{NumKeyValueHeads: 1, Scale: 1}+ q := mlx.FromValues([]float32{1, 0}, 1, 1, 1, 2)+ k := mlx.FromValues([]float32{1, 0, 0, 1, 2, 0}, 1, 1, 3, 2)+ v := mlx.FromValues([]float32{10, 1, 20, 2, 30, 3}, 1, 1, 3, 2)+ indices := mlx.FromValues([]int32{2, 0}, 1, 1, 2)+ valid := mlx.FromValues([]bool{true, true}, 1, 1, 2)- out := qsaSparseAttention(q, nn.NewKVHistory(k, v, nil), indices, valid, cfg)- out = out.AsType(mlx.DTypeFloat32)- mlx.Eval(out)- got := out.Floats()- p2 := float32(math.Exp(2) / (math.Exp(2) + math.Exp(1)))- want := []float32{p2*30 + (1-p2)*10, p2*3 + (1-p2)*1}- for i := range want {- if math.Abs(float64(got[i]-want[i])) > 1e-4 {- t.Fatalf("sparse attention[%d] = %v, want %v", i, got[i], want[i])+ out := qsaSparseAttention(q, nn.NewKVHistory(k, v, nil), indices, valid, cfg)+ out = out.AsType(mlx.DTypeFloat32)+ mlx.Eval(out)+ got := out.Floats()+ p2 := float32(math.Exp(2) / (math.Exp(2) + math.Exp(1)))+ want := []float32{p2*30 + (1-p2)*10, p2*3 + (1-p2)*1}+ for i := range want {+ if math.Abs(float64(got[i]-want[i])) > 1e-4 {+ t.Fatalf("sparse attention[%d] = %v, want %v", i, got[i], want[i])+ }}- }+ })}func TestQSASparseAttentionIgnoresInvalidRows(t *testing.T) {- mlxtest.Setup(t)- cfg := &Config{NumKeyValueHeads: 1, Scale: 1}- q := mlx.FromValues([]float32{1, 0}, 1, 1, 1, 2)- // Row 0 is intentionally dominant junk. Only row 1 is logically valid.- k := mlx.FromValues([]float32{100, 0, 1, 0}, 1, 1, 2, 2)- v := mlx.FromValues([]float32{999, 999, 7, 3}, 1, 1, 2, 2)- indices := mlx.FromValues([]int32{0, 1}, 1, 1, 2)- valid := mlx.FromValues([]bool{false, true}, 1, 1, 2)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{NumKeyValueHeads: 1, Scale: 1}+ q := mlx.FromValues([]float32{1, 0}, 1, 1, 1, 2)+ // Row 0 is intentionally dominant junk. Only row 1 is logically valid.+ k := mlx.FromValues([]float32{100, 0, 1, 0}, 1, 1, 2, 2)+ v := mlx.FromValues([]float32{999, 999, 7, 3}, 1, 1, 2, 2)+ indices := mlx.FromValues([]int32{0, 1}, 1, 1, 2)+ valid := mlx.FromValues([]bool{false, true}, 1, 1, 2)- out := qsaSparseAttention(q, nn.NewKVHistory(k, v, nil), indices, valid, cfg)- out = out.AsType(mlx.DTypeFloat32)- mlx.Eval(out)- if got, want := out.Floats(), []float32{7, 3}; !slices.Equal(got, want) {- t.Fatalf("sparse attention = %v, want %v", got, want)- }+ out := qsaSparseAttention(q, nn.NewKVHistory(k, v, nil), indices, valid, cfg)+ out = out.AsType(mlx.DTypeFloat32)+ mlx.Eval(out)+ if got, want := out.Floats(), []float32{7, 3}; !slices.Equal(got, want) {+ t.Fatalf("sparse attention = %v, want %v", got, want)+ }+ })}func TestQSASparseAttentionKeepsBatchRowsIndependent(t *testing.T) {- mlxtest.Setup(t)- cfg := &Config{NumKeyValueHeads: 1, Scale: 1}- q := mlx.FromValues([]float32{1, 0, 1, 0}, 2, 1, 1, 2)- k := mlx.FromValues([]float32{- 1, 0, 0, 1,- 1, 0, 0, 1,- }, 2, 1, 2, 2)- v := mlx.FromValues([]float32{- 10, 1, 20, 2,- 30, 3, 40, 4,- }, 2, 1, 2, 2)- indices := mlx.FromValues([]int32{0, 1}, 2, 1, 1)- valid := mlx.FromValues([]bool{true, true}, 2, 1, 1)+ mlxtest.Run(t, func(t *mlxtest.T) {+ cfg := &Config{NumKeyValueHeads: 1, Scale: 1}+ q := mlx.FromValues([]float32{1, 0, 1, 0}, 2, 1, 1, 2)+ k := mlx.FromValues([]float32{+ 1, 0, 0, 1,+ 1, 0, 0, 1,+ }, 2, 1, 2, 2)+ v := mlx.FromValues([]float32{+ 10, 1, 20, 2,+ 30, 3, 40, 4,+ }, 2, 1, 2, 2)+ indices := mlx.FromValues([]int32{0, 1}, 2, 1, 1)+ valid := mlx.FromValues([]bool{true, true}, 2, 1, 1)- out := qsaSparseAttention(q, nn.NewKVHistory(k, v, nil), indices, valid, cfg)- out = out.AsType(mlx.DTypeFloat32)- mlx.Eval(out)- if got, want := out.Floats(), []float32{10, 1, 40, 4}; !slices.Equal(got, want) {- t.Fatalf("sparse attention = %v, want %v", got, want)- }+ out := qsaSparseAttention(q, nn.NewKVHistory(k, v, nil), indices, valid, cfg)+ out = out.AsType(mlx.DTypeFloat32)+ mlx.Eval(out)+ if got, want := out.Floats(), []float32{10, 1, 40, 4}; !slices.Equal(got, want) {+ t.Fatalf("sparse attention = %v, want %v", got, want)+ }+ })}diff --git a/x/tokenizer/tokenizer_benchmark_test.go b/x/tokenizer/tokenizer_benchmark_test.goindex dc9ccf579..e98a3b87b 100644--- a/x/tokenizer/tokenizer_benchmark_test.go+++ b/x/tokenizer/tokenizer_benchmark_test.go@@ -14,20 +14,6 @@ var (benchmarkSinkTok *Tokenizer)-const benchmarkWordPieceJSON = `{- "model": {- "type": "WordPiece",- "vocab": {- "[UNK]": 0,- "hello": 1,- "##world": 2,- "##ly": 3,- "##hello": 4- }- },- "added_tokens": []-}`-const benchmarkSentencePieceJSON = `{"model": {"type": "BPE",@@ -194,33 +180,6 @@ func BenchmarkTokenizerLoadFromBytes(b *testing.B) {})}-func BenchmarkTokenizerEncodeWordPiece(b *testing.B) {- tok := benchmarkLoadFromBytes(b, []byte(benchmarkWordPieceJSON))- text := strings.Repeat("helloworldly", 16)-- b.ReportAllocs()- b.SetBytes(int64(len(text)))- b.ResetTimer()-- for range b.N {- benchmarkSinkIDs = tok.Encode(text, false)- }-}--func BenchmarkTokenizerDecodeWordPiece(b *testing.B) {- tok := benchmarkLoadFromBytes(b, []byte(benchmarkWordPieceJSON))- text := strings.Repeat("helloworldly", 16)- ids := tok.Encode(text, false)-- b.ReportAllocs()- b.SetBytes(int64(len(text)))- b.ResetTimer()-- for range b.N {- benchmarkSinkStr = tok.Decode(ids)- }-}-func BenchmarkTokenizerEncodeSentencePiece(b *testing.B) {tok := benchmarkLoadFromBytes(b, []byte(benchmarkSentencePieceJSON))text := strings.Repeat("hello world\n", 64)diff --git a/x/transfer/transfer_test.go b/x/transfer/transfer_test.goindex f58d94dfd..12bee483b 100644--- a/x/transfer/transfer_test.go+++ b/x/transfer/transfer_test.go@@ -1680,6 +1680,10 @@ func BenchmarkUploadThroughput(b *testing.B) {case http.MethodPost:w.Header().Set("Location", fmt.Sprintf("%s/v2/library/_/blobs/uploads/1", serverURL))w.WriteHeader(http.StatusAccepted)+ case http.MethodPatch:+ io.Copy(io.Discard, r.Body)+ w.Header().Set("Location", r.URL.Path)+ w.WriteHeader(http.StatusAccepted)case http.MethodPut:io.Copy(io.Discard, r.Body)w.WriteHeader(http.StatusCreated)