Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

MLX, MLX-C: version bump (#18956)

Daniel Hiltgen374d97dd0e3e
diff --git a/MLX_C_VERSION b/MLX_C_VERSION
index aaf310acb..738443963 100644
--- a/MLX_C_VERSION
+++ b/MLX_C_VERSION
@@ -1 +1 @@
-ebc88f10caa1b625e6b581437a8dea6df8a70085
\ No newline at end of file
+a341b4925024b88b2c593468f16e12f5e17315da
\ No newline at end of file
diff --git a/MLX_VERSION b/MLX_VERSION
index 0bbe3060c..2794c128b 100644
--- a/MLX_VERSION
+++ b/MLX_VERSION
@@ -1 +1 @@
-a59cc23190f1e26fabcba8693607bb4c34b2ac2b
\ No newline at end of file
+97029597882f674a596958bc71fdec3213ccc192
\ No newline at end of file
diff --git a/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch b/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch
new file mode 100644
index 000000000..608a09852
--- /dev/null
+++ b/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch
@@ -0,0 +1,131 @@
+From 789be76e27a3f46fe4e73d78adad4cac2f5d0dba Mon Sep 17 00:00:00 2001
+From: Daniel Hiltgen <[email protected]>
+Date: Fri, 9 Oct 2026 00:13:03 -0700
+Subject: [PATCH] [Metal] Fix SDPA max threadgroup for M1/M2 generation (#4643)
+
+Co-authored-by: Cheng <[email protected]>
+---
+ .../scaled_dot_product_attention.metal | 15 +++++++-
+ mlx/backend/metal/kernels/sdpa_vector.h | 6 +---
+ .../metal/scaled_dot_product_attention.cpp | 12 +++++++
+ python/tests/test_fast_sdpa.py | 34 +++++++++++++++++++
+ 4 files changed, 61 insertions(+), 6 deletions(-)
+
+diff --git a/mlx/backend/metal/kernels/scaled_dot_product_attention.metal b/mlx/backend/metal/kernels/scaled_dot_product_attention.metal
+index efb93607d..d7862d256 100644
+--- a/mlx/backend/metal/kernels/scaled_dot_product_attention.metal
++++ b/mlx/backend/metal/kernels/scaled_dot_product_attention.metal
+@@ -79,4 +79,17 @@ instantiate_kernel(
+ instantiate_sdpa_vector_heads(float)
+ instantiate_sdpa_vector_heads(bfloat16_t)
+ instantiate_sdpa_vector_heads(float16_t)
+- // clang-format on
++
++#define instantiate_sdpa_vector_tk1(type, dim) \
++ instantiate_kernel( \
++ "sdpa_vector_2pass_1_" #type "_" #dim "_" #dim "_tk1", \
++ sdpa_vector_2pass_1, \
++ type, \
++ dim, \
++ dim, \
++ 1)
++
++instantiate_sdpa_vector_tk1(bfloat16_t, 256)
++instantiate_sdpa_vector_tk1(float, 192)
++instantiate_sdpa_vector_tk1(float, 256)
++// clang-format on
+diff --git a/mlx/backend/metal/kernels/sdpa_vector.h b/mlx/backend/metal/kernels/sdpa_vector.h
+index d94f21dff..1d312038c 100644
+--- a/mlx/backend/metal/kernels/sdpa_vector.h
++++ b/mlx/backend/metal/kernels/sdpa_vector.h
+@@ -177,7 +177,7 @@ template <typename T, int D, int V = D>
+ }
+ }
+
+-template <typename T, int D, int V = D>
++template <typename T, int D, int V = D, int TK = 4>
+ [[kernel]] void sdpa_vector_2pass_1(
+ const device T* queries [[buffer(0)]],
+ const device T* keys [[buffer(1)]],
+@@ -265,10 +265,6 @@ template <typename T, int D, int V = D>
+ const int stride_mask_tok =
+ (bool_mask || float_mask) ? blocks * mask_kv_seq_stride : 0;
+
+- // TK=4 is chosen to keep register pressure low.
+- // TODO: For some hardwares TK=1 would run faster for certain shapes.
+- constexpr int TK = 4;
+-
+ // For each key
+ int i = block_idx;
+ for (; i + (TK - 1) * blocks < N; i += blocks * TK) {
+diff --git a/mlx/backend/metal/scaled_dot_product_attention.cpp b/mlx/backend/metal/scaled_dot_product_attention.cpp
+index 9ea087b5d..5b157c6c5 100644
+--- a/mlx/backend/metal/scaled_dot_product_attention.cpp
++++ b/mlx/backend/metal/scaled_dot_product_attention.cpp
+@@ -859,6 +859,18 @@ void sdpa_vector_2pass(
+
+ // Get the kernel
+ auto kernel = d.get_kernel(kname, hash_name, func_consts);
++ // M1/M2 do not have enough threads per threadgroup for default TK=4 kernel
++ bool has_tk1_variant =
++ (q.dtype() == bfloat16 && q.shape(-1) == 256 && v.shape(-1) == 256) ||
++ (q.dtype() == float32 && q.shape(-1) == v.shape(-1) &&
++ (q.shape(-1) == 192 || q.shape(-1) == 256));
++ if (has_tk1_variant &&
++ group_dims.width * group_dims.height * group_dims.depth >
++ kernel->maxTotalThreadsPerThreadgroup()) {
++ kname += "_tk1";
++ hash_name += "_tk1";
++ kernel = d.get_kernel(kname, hash_name, func_consts);
++ }
+ check_kernel_threadgroup_size(kernel, group_dims, hash_name);
+
+ compute_encoder.set_compute_pipeline_state(kernel);
+diff --git a/python/tests/test_fast_sdpa.py b/python/tests/test_fast_sdpa.py
+index 527906580..b2d7e9121 100644
+--- a/python/tests/test_fast_sdpa.py
++++ b/python/tests/test_fast_sdpa.py
+@@ -892,6 +892,40 @@ class TestFastSDPA(mlx_tests.MLXTestCase):
+ ref = mlx_ref_attn(q, k, v, mask=mask)
+ self.assertTrue(mx.allclose(ref, out, atol=1e-4, rtol=1e-4))
+
++ @unittest.skipIf(not mx.metal.is_available(), "Metal kernel path only")
++ def test_sdpa_vector_2pass_threadgroup_limit(self):
++ for dtype, D, q_heads, kv_heads, q_len in (
++ (mx.float16, 192, 32, 8, 8),
++ (mx.float16, 256, 24, 4, 5),
++ (mx.float16, 256, 32, 4, 4),
++ (mx.bfloat16, 256, 24, 4, 5),
++ (mx.bfloat16, 256, 32, 4, 4),
++ (mx.float32, 192, 32, 8, 8),
++ (mx.float32, 256, 24, 4, 5),
++ (mx.float32, 256, 32, 4, 4),
++ ):
++ with self.subTest(dtype=dtype, D=D, q_len=q_len):
++ self.assertGreater(32 * q_heads // kv_heads * q_len, 896)
++ q = mx.random.normal(shape=(1, q_heads, q_len, D), dtype=dtype)
++ k = mx.random.normal(shape=(1, kv_heads, 1029, D), dtype=dtype)
++ v = mx.random.normal(shape=(1, kv_heads, 1029, D), dtype=dtype)
++ scale = D**-0.5
++
++ out = mx.fast.scaled_dot_product_attention(
++ q, k, v, scale=scale, mask="causal", force_fused=True
++ )
++ ref = mlx_ref_attn(
++ q.astype(mx.float32),
++ k.astype(mx.float32),
++ v.astype(mx.float32),
++ scale=scale,
++ mask="causal",
++ )
++ atol = 1e-4 if dtype == mx.float32 else 2e-3
++ if dtype == mx.bfloat16:
++ atol = 2e-2
++ self.assertTrue(mx.allclose(out.astype(mx.float32), ref, atol=atol))
++
+ @unittest.skipIf(not mx.is_available(mx.gpu), "GPU kernel path only")
+ def test_sdpa_blocks_env_override(self):
+ # MLX_SDPA_BLOCKS used to be applied as-is, and values that are not
+--
+2.50.1 (Apple Git-155)
+