Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
MLX, MLX-C: version bump (#18956)
Daniel Hiltgen374d97dd0e3e
diff --git a/MLX_C_VERSION b/MLX_C_VERSIONindex aaf310acb..738443963 100644--- a/MLX_C_VERSION+++ b/MLX_C_VERSION@@ -1 +1 @@-ebc88f10caa1b625e6b581437a8dea6df8a70085\ No newline at end of file+a341b4925024b88b2c593468f16e12f5e17315da\ No newline at end of filediff --git a/MLX_VERSION b/MLX_VERSIONindex 0bbe3060c..2794c128b 100644--- a/MLX_VERSION+++ b/MLX_VERSION@@ -1 +1 @@-a59cc23190f1e26fabcba8693607bb4c34b2ac2b\ No newline at end of file+97029597882f674a596958bc71fdec3213ccc192\ No newline at end of filediff --git a/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch b/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patchnew file mode 100644index 000000000..608a09852--- /dev/null+++ b/mlx/compat/mlx/0001-metal-fix-sdpa-max-threadgroup-m1-m2.patch@@ -0,0 +1,131 @@+From 789be76e27a3f46fe4e73d78adad4cac2f5d0dba Mon Sep 17 00:00:00 2001+From: Daniel Hiltgen <[email protected]>+Date: Fri, 9 Oct 2026 00:13:03 -0700+Subject: [PATCH] [Metal] Fix SDPA max threadgroup for M1/M2 generation (#4643)++Co-authored-by: Cheng <[email protected]>+---+ .../scaled_dot_product_attention.metal | 15 +++++++-+ mlx/backend/metal/kernels/sdpa_vector.h | 6 +---+ .../metal/scaled_dot_product_attention.cpp | 12 ++++++++ python/tests/test_fast_sdpa.py | 34 ++++++++++++++++++++ 4 files changed, 61 insertions(+), 6 deletions(-)++diff --git a/mlx/backend/metal/kernels/scaled_dot_product_attention.metal b/mlx/backend/metal/kernels/scaled_dot_product_attention.metal+index efb93607d..d7862d256 100644+--- a/mlx/backend/metal/kernels/scaled_dot_product_attention.metal++++ b/mlx/backend/metal/kernels/scaled_dot_product_attention.metal+@@ -79,4 +79,17 @@ instantiate_kernel(+ instantiate_sdpa_vector_heads(float)+ instantiate_sdpa_vector_heads(bfloat16_t)+ instantiate_sdpa_vector_heads(float16_t)+- // clang-format on++++#define instantiate_sdpa_vector_tk1(type, dim) \++ instantiate_kernel( \++ "sdpa_vector_2pass_1_" #type "_" #dim "_" #dim "_tk1", \++ sdpa_vector_2pass_1, \++ type, \++ dim, \++ dim, \++ 1)++++instantiate_sdpa_vector_tk1(bfloat16_t, 256)++instantiate_sdpa_vector_tk1(float, 192)++instantiate_sdpa_vector_tk1(float, 256)++// clang-format on+diff --git a/mlx/backend/metal/kernels/sdpa_vector.h b/mlx/backend/metal/kernels/sdpa_vector.h+index d94f21dff..1d312038c 100644+--- a/mlx/backend/metal/kernels/sdpa_vector.h++++ b/mlx/backend/metal/kernels/sdpa_vector.h+@@ -177,7 +177,7 @@ template <typename T, int D, int V = D>+ }+ }++-template <typename T, int D, int V = D>++template <typename T, int D, int V = D, int TK = 4>+ [[kernel]] void sdpa_vector_2pass_1(+ const device T* queries [[buffer(0)]],+ const device T* keys [[buffer(1)]],+@@ -265,10 +265,6 @@ template <typename T, int D, int V = D>+ const int stride_mask_tok =+ (bool_mask || float_mask) ? blocks * mask_kv_seq_stride : 0;++- // TK=4 is chosen to keep register pressure low.+- // TODO: For some hardwares TK=1 would run faster for certain shapes.+- constexpr int TK = 4;+-+ // For each key+ int i = block_idx;+ for (; i + (TK - 1) * blocks < N; i += blocks * TK) {+diff --git a/mlx/backend/metal/scaled_dot_product_attention.cpp b/mlx/backend/metal/scaled_dot_product_attention.cpp+index 9ea087b5d..5b157c6c5 100644+--- a/mlx/backend/metal/scaled_dot_product_attention.cpp++++ b/mlx/backend/metal/scaled_dot_product_attention.cpp+@@ -859,6 +859,18 @@ void sdpa_vector_2pass(++ // Get the kernel+ auto kernel = d.get_kernel(kname, hash_name, func_consts);++ // M1/M2 do not have enough threads per threadgroup for default TK=4 kernel++ bool has_tk1_variant =++ (q.dtype() == bfloat16 && q.shape(-1) == 256 && v.shape(-1) == 256) ||++ (q.dtype() == float32 && q.shape(-1) == v.shape(-1) &&++ (q.shape(-1) == 192 || q.shape(-1) == 256));++ if (has_tk1_variant &&++ group_dims.width * group_dims.height * group_dims.depth >++ kernel->maxTotalThreadsPerThreadgroup()) {++ kname += "_tk1";++ hash_name += "_tk1";++ kernel = d.get_kernel(kname, hash_name, func_consts);++ }+ check_kernel_threadgroup_size(kernel, group_dims, hash_name);++ compute_encoder.set_compute_pipeline_state(kernel);+diff --git a/python/tests/test_fast_sdpa.py b/python/tests/test_fast_sdpa.py+index 527906580..b2d7e9121 100644+--- a/python/tests/test_fast_sdpa.py++++ b/python/tests/test_fast_sdpa.py+@@ -892,6 +892,40 @@ class TestFastSDPA(mlx_tests.MLXTestCase):+ ref = mlx_ref_attn(q, k, v, mask=mask)+ self.assertTrue(mx.allclose(ref, out, atol=1e-4, rtol=1e-4))+++ @unittest.skipIf(not mx.metal.is_available(), "Metal kernel path only")++ def test_sdpa_vector_2pass_threadgroup_limit(self):++ for dtype, D, q_heads, kv_heads, q_len in (++ (mx.float16, 192, 32, 8, 8),++ (mx.float16, 256, 24, 4, 5),++ (mx.float16, 256, 32, 4, 4),++ (mx.bfloat16, 256, 24, 4, 5),++ (mx.bfloat16, 256, 32, 4, 4),++ (mx.float32, 192, 32, 8, 8),++ (mx.float32, 256, 24, 4, 5),++ (mx.float32, 256, 32, 4, 4),++ ):++ with self.subTest(dtype=dtype, D=D, q_len=q_len):++ self.assertGreater(32 * q_heads // kv_heads * q_len, 896)++ q = mx.random.normal(shape=(1, q_heads, q_len, D), dtype=dtype)++ k = mx.random.normal(shape=(1, kv_heads, 1029, D), dtype=dtype)++ v = mx.random.normal(shape=(1, kv_heads, 1029, D), dtype=dtype)++ scale = D**-0.5++++ out = mx.fast.scaled_dot_product_attention(++ q, k, v, scale=scale, mask="causal", force_fused=True++ )++ ref = mlx_ref_attn(++ q.astype(mx.float32),++ k.astype(mx.float32),++ v.astype(mx.float32),++ scale=scale,++ mask="causal",++ )++ atol = 1e-4 if dtype == mx.float32 else 2e-3++ if dtype == mx.bfloat16:++ atol = 2e-2++ self.assertTrue(mx.allclose(out.astype(mx.float32), ref, atol=atol))+++ @unittest.skipIf(not mx.is_available(mx.gpu), "GPU kernel path only")+ def test_sdpa_blocks_env_override(self):+ # MLX_SDPA_BLOCKS used to be applied as-is, and values that are not+--+2.50.1 (Apple Git-155)+