Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

MLX: version bump (#18993)

Daniel Hiltgen292d7ce58c2f
diff --git a/MLX_VERSION b/MLX_VERSION
index 2794c128b..d97a502dd 100644
--- a/MLX_VERSION
+++ b/MLX_VERSION
@@ -1 +1 @@
-97029597882f674a596958bc71fdec3213ccc192
\ No newline at end of file
+bebd6422f8db88d56e6220520a35bcd146f823f8
\ No newline at end of file
diff --git a/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch b/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch
deleted file mode 100644
index 608a09852..000000000
--- a/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch
+++ /dev/null
@@ -1,131 +0,0 @@
-From 789be76e27a3f46fe4e73d78adad4cac2f5d0dba Mon Sep 17 00:00:00 2001
-From: Daniel Hiltgen <[email protected]>
-Date: Fri, 9 Oct 2026 00:13:03 -0700
-Subject: [PATCH] [Metal] Fix SDPA max threadgroup for M1/M2 generation (#4643)
-
-Co-authored-by: Cheng <[email protected]>
----
- .../scaled_dot_product_attention.metal | 15 +++++++-
- mlx/backend/metal/kernels/sdpa_vector.h | 6 +---
- .../metal/scaled_dot_product_attention.cpp | 12 +++++++
- python/tests/test_fast_sdpa.py | 34 +++++++++++++++++++
- 4 files changed, 61 insertions(+), 6 deletions(-)
-
-diff --git a/mlx/backend/metal/kernels/scaled_dot_product_attention.metal b/mlx/backend/metal/kernels/scaled_dot_product_attention.metal
-index efb93607d..d7862d256 100644
---- a/mlx/backend/metal/kernels/scaled_dot_product_attention.metal
-+++ b/mlx/backend/metal/kernels/scaled_dot_product_attention.metal
-@@ -79,4 +79,17 @@ instantiate_kernel(
- instantiate_sdpa_vector_heads(float)
- instantiate_sdpa_vector_heads(bfloat16_t)
- instantiate_sdpa_vector_heads(float16_t)
-- // clang-format on
-+
-+#define instantiate_sdpa_vector_tk1(type, dim) \
-+ instantiate_kernel( \
-+ "sdpa_vector_2pass_1_" #type "_" #dim "_" #dim "_tk1", \
-+ sdpa_vector_2pass_1, \
-+ type, \
-+ dim, \
-+ dim, \
-+ 1)
-+
-+instantiate_sdpa_vector_tk1(bfloat16_t, 256)
-+instantiate_sdpa_vector_tk1(float, 192)
-+instantiate_sdpa_vector_tk1(float, 256)
-+// clang-format on
-diff --git a/mlx/backend/metal/kernels/sdpa_vector.h b/mlx/backend/metal/kernels/sdpa_vector.h
-index d94f21dff..1d312038c 100644
---- a/mlx/backend/metal/kernels/sdpa_vector.h
-+++ b/mlx/backend/metal/kernels/sdpa_vector.h
-@@ -177,7 +177,7 @@ template <typename T, int D, int V = D>
- }
- }
-
--template <typename T, int D, int V = D>
-+template <typename T, int D, int V = D, int TK = 4>
- [[kernel]] void sdpa_vector_2pass_1(
- const device T* queries [[buffer(0)]],
- const device T* keys [[buffer(1)]],
-@@ -265,10 +265,6 @@ template <typename T, int D, int V = D>
- const int stride_mask_tok =
- (bool_mask || float_mask) ? blocks * mask_kv_seq_stride : 0;
-
-- // TK=4 is chosen to keep register pressure low.
-- // TODO: For some hardwares TK=1 would run faster for certain shapes.
-- constexpr int TK = 4;
--
- // For each key
- int i = block_idx;
- for (; i + (TK - 1) * blocks < N; i += blocks * TK) {
-diff --git a/mlx/backend/metal/scaled_dot_product_attention.cpp b/mlx/backend/metal/scaled_dot_product_attention.cpp
-index 9ea087b5d..5b157c6c5 100644
---- a/mlx/backend/metal/scaled_dot_product_attention.cpp
-+++ b/mlx/backend/metal/scaled_dot_product_attention.cpp
-@@ -859,6 +859,18 @@ void sdpa_vector_2pass(
-
- // Get the kernel
- auto kernel = d.get_kernel(kname, hash_name, func_consts);
-+ // M1/M2 do not have enough threads per threadgroup for default TK=4 kernel
-+ bool has_tk1_variant =
-+ (q.dtype() == bfloat16 && q.shape(-1) == 256 && v.shape(-1) == 256) ||
-+ (q.dtype() == float32 && q.shape(-1) == v.shape(-1) &&
-+ (q.shape(-1) == 192 || q.shape(-1) == 256));
-+ if (has_tk1_variant &&
-+ group_dims.width * group_dims.height * group_dims.depth >
-+ kernel->maxTotalThreadsPerThreadgroup()) {
-+ kname += "_tk1";
-+ hash_name += "_tk1";
-+ kernel = d.get_kernel(kname, hash_name, func_consts);
-+ }
- check_kernel_threadgroup_size(kernel, group_dims, hash_name);
-
- compute_encoder.set_compute_pipeline_state(kernel);
-diff --git a/python/tests/test_fast_sdpa.py b/python/tests/test_fast_sdpa.py
-index 527906580..b2d7e9121 100644
---- a/python/tests/test_fast_sdpa.py
-+++ b/python/tests/test_fast_sdpa.py
-@@ -892,6 +892,40 @@ class TestFastSDPA(mlx_tests.MLXTestCase):
- ref = mlx_ref_attn(q, k, v, mask=mask)
- self.assertTrue(mx.allclose(ref, out, atol=1e-4, rtol=1e-4))
-
-+ @unittest.skipIf(not mx.metal.is_available(), "Metal kernel path only")
-+ def test_sdpa_vector_2pass_threadgroup_limit(self):
-+ for dtype, D, q_heads, kv_heads, q_len in (
-+ (mx.float16, 192, 32, 8, 8),
-+ (mx.float16, 256, 24, 4, 5),
-+ (mx.float16, 256, 32, 4, 4),
-+ (mx.bfloat16, 256, 24, 4, 5),
-+ (mx.bfloat16, 256, 32, 4, 4),
-+ (mx.float32, 192, 32, 8, 8),
-+ (mx.float32, 256, 24, 4, 5),
-+ (mx.float32, 256, 32, 4, 4),
-+ ):
-+ with self.subTest(dtype=dtype, D=D, q_len=q_len):
-+ self.assertGreater(32 * q_heads // kv_heads * q_len, 896)
-+ q = mx.random.normal(shape=(1, q_heads, q_len, D), dtype=dtype)
-+ k = mx.random.normal(shape=(1, kv_heads, 1029, D), dtype=dtype)
-+ v = mx.random.normal(shape=(1, kv_heads, 1029, D), dtype=dtype)
-+ scale = D**-0.5
-+
-+ out = mx.fast.scaled_dot_product_attention(
-+ q, k, v, scale=scale, mask="causal", force_fused=True
-+ )
-+ ref = mlx_ref_attn(
-+ q.astype(mx.float32),
-+ k.astype(mx.float32),
-+ v.astype(mx.float32),
-+ scale=scale,
-+ mask="causal",
-+ )
-+ atol = 1e-4 if dtype == mx.float32 else 2e-3
-+ if dtype == mx.bfloat16:
-+ atol = 2e-2
-+ self.assertTrue(mx.allclose(out.astype(mx.float32), ref, atol=atol))
-+
- @unittest.skipIf(not mx.is_available(mx.gpu), "GPU kernel path only")
- def test_sdpa_blocks_env_override(self):
- # MLX_SDPA_BLOCKS used to be applied as-is, and values that are not
---
-2.50.1 (Apple Git-155)
-