Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
MLX: version bump (#18993)
Daniel Hiltgen292d7ce58c2f
diff --git a/MLX_VERSION b/MLX_VERSIONindex 2794c128b..d97a502dd 100644--- a/MLX_VERSION+++ b/MLX_VERSION@@ -1 +1 @@-97029597882f674a596958bc71fdec3213ccc192\ No newline at end of file+bebd6422f8db88d56e6220520a35bcd146f823f8\ No newline at end of filediff --git a/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch b/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patchdeleted file mode 100644index 608a09852..000000000--- a/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch+++ /dev/null@@ -1,131 +0,0 @@-From 789be76e27a3f46fe4e73d78adad4cac2f5d0dba Mon Sep 17 00:00:00 2001-From: Daniel Hiltgen <[email protected]>-Date: Fri, 9 Oct 2026 00:13:03 -0700-Subject: [PATCH] [Metal] Fix SDPA max threadgroup for M1/M2 generation (#4643)--Co-authored-by: Cheng <[email protected]>----- .../scaled_dot_product_attention.metal | 15 +++++++-- mlx/backend/metal/kernels/sdpa_vector.h | 6 +---- .../metal/scaled_dot_product_attention.cpp | 12 +++++++- python/tests/test_fast_sdpa.py | 34 +++++++++++++++++++- 4 files changed, 61 insertions(+), 6 deletions(-)--diff --git a/mlx/backend/metal/kernels/scaled_dot_product_attention.metal b/mlx/backend/metal/kernels/scaled_dot_product_attention.metal-index efb93607d..d7862d256 100644---- a/mlx/backend/metal/kernels/scaled_dot_product_attention.metal-+++ b/mlx/backend/metal/kernels/scaled_dot_product_attention.metal-@@ -79,4 +79,17 @@ instantiate_kernel(- instantiate_sdpa_vector_heads(float)- instantiate_sdpa_vector_heads(bfloat16_t)- instantiate_sdpa_vector_heads(float16_t)-- // clang-format on-+-+#define instantiate_sdpa_vector_tk1(type, dim) \-+ instantiate_kernel( \-+ "sdpa_vector_2pass_1_" #type "_" #dim "_" #dim "_tk1", \-+ sdpa_vector_2pass_1, \-+ type, \-+ dim, \-+ dim, \-+ 1)-+-+instantiate_sdpa_vector_tk1(bfloat16_t, 256)-+instantiate_sdpa_vector_tk1(float, 192)-+instantiate_sdpa_vector_tk1(float, 256)-+// clang-format on-diff --git a/mlx/backend/metal/kernels/sdpa_vector.h b/mlx/backend/metal/kernels/sdpa_vector.h-index d94f21dff..1d312038c 100644---- a/mlx/backend/metal/kernels/sdpa_vector.h-+++ b/mlx/backend/metal/kernels/sdpa_vector.h-@@ -177,7 +177,7 @@ template <typename T, int D, int V = D>- }- }---template <typename T, int D, int V = D>-+template <typename T, int D, int V = D, int TK = 4>- [[kernel]] void sdpa_vector_2pass_1(- const device T* queries [[buffer(0)]],- const device T* keys [[buffer(1)]],-@@ -265,10 +265,6 @@ template <typename T, int D, int V = D>- const int stride_mask_tok =- (bool_mask || float_mask) ? blocks * mask_kv_seq_stride : 0;--- // TK=4 is chosen to keep register pressure low.-- // TODO: For some hardwares TK=1 would run faster for certain shapes.-- constexpr int TK = 4;--- // For each key- int i = block_idx;- for (; i + (TK - 1) * blocks < N; i += blocks * TK) {-diff --git a/mlx/backend/metal/scaled_dot_product_attention.cpp b/mlx/backend/metal/scaled_dot_product_attention.cpp-index 9ea087b5d..5b157c6c5 100644---- a/mlx/backend/metal/scaled_dot_product_attention.cpp-+++ b/mlx/backend/metal/scaled_dot_product_attention.cpp-@@ -859,6 +859,18 @@ void sdpa_vector_2pass(-- // Get the kernel- auto kernel = d.get_kernel(kname, hash_name, func_consts);-+ // M1/M2 do not have enough threads per threadgroup for default TK=4 kernel-+ bool has_tk1_variant =-+ (q.dtype() == bfloat16 && q.shape(-1) == 256 && v.shape(-1) == 256) ||-+ (q.dtype() == float32 && q.shape(-1) == v.shape(-1) &&-+ (q.shape(-1) == 192 || q.shape(-1) == 256));-+ if (has_tk1_variant &&-+ group_dims.width * group_dims.height * group_dims.depth >-+ kernel->maxTotalThreadsPerThreadgroup()) {-+ kname += "_tk1";-+ hash_name += "_tk1";-+ kernel = d.get_kernel(kname, hash_name, func_consts);-+ }- check_kernel_threadgroup_size(kernel, group_dims, hash_name);-- compute_encoder.set_compute_pipeline_state(kernel);-diff --git a/python/tests/test_fast_sdpa.py b/python/tests/test_fast_sdpa.py-index 527906580..b2d7e9121 100644---- a/python/tests/test_fast_sdpa.py-+++ b/python/tests/test_fast_sdpa.py-@@ -892,6 +892,40 @@ class TestFastSDPA(mlx_tests.MLXTestCase):- ref = mlx_ref_attn(q, k, v, mask=mask)- self.assertTrue(mx.allclose(ref, out, atol=1e-4, rtol=1e-4))--+ @unittest.skipIf(not mx.metal.is_available(), "Metal kernel path only")-+ def test_sdpa_vector_2pass_threadgroup_limit(self):-+ for dtype, D, q_heads, kv_heads, q_len in (-+ (mx.float16, 192, 32, 8, 8),-+ (mx.float16, 256, 24, 4, 5),-+ (mx.float16, 256, 32, 4, 4),-+ (mx.bfloat16, 256, 24, 4, 5),-+ (mx.bfloat16, 256, 32, 4, 4),-+ (mx.float32, 192, 32, 8, 8),-+ (mx.float32, 256, 24, 4, 5),-+ (mx.float32, 256, 32, 4, 4),-+ ):-+ with self.subTest(dtype=dtype, D=D, q_len=q_len):-+ self.assertGreater(32 * q_heads // kv_heads * q_len, 896)-+ q = mx.random.normal(shape=(1, q_heads, q_len, D), dtype=dtype)-+ k = mx.random.normal(shape=(1, kv_heads, 1029, D), dtype=dtype)-+ v = mx.random.normal(shape=(1, kv_heads, 1029, D), dtype=dtype)-+ scale = D**-0.5-+-+ out = mx.fast.scaled_dot_product_attention(-+ q, k, v, scale=scale, mask="causal", force_fused=True-+ )-+ ref = mlx_ref_attn(-+ q.astype(mx.float32),-+ k.astype(mx.float32),-+ v.astype(mx.float32),-+ scale=scale,-+ mask="causal",-+ )-+ atol = 1e-4 if dtype == mx.float32 else 2e-3-+ if dtype == mx.bfloat16:-+ atol = 2e-2-+ self.assertTrue(mx.allclose(out.astype(mx.float32), ref, atol=atol))-+- @unittest.skipIf(not mx.is_available(mx.gpu), "GPU kernel path only")- def test_sdpa_blocks_env_override(self):- # MLX_SDPA_BLOCKS used to be applied as-is, and values that are not----2.50.1 (Apple Git-155)-