A high-performance inference engine for AI models
fmt (#157)
uuuvne604781d48b5
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.h b/crates/uzu/src/backends/metal/kernel/activation/activation.hindex 491eb88c9..431183667 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activation.h+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.h@@ -17,7 +17,8 @@ inline T gelu_approx(T x) {constexpr float k0 = 0.044715f;constexpr float k1 = 0.7978845608f; // sqrt(2/pi)float xf = float(x);- float yf = 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));+ float yf =+ 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));return T(yf);}diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metalindex 480b71e48..8efd95c71 100644--- a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal@@ -390,9 +390,7 @@ template <for (short id = 0; id < TD; id++) {UZU_PRAGMA_UNROLLfor (short ik = 0; ik < TK; ik++) {- IF_CONSTEXPR(BD == 128) {- simdgroup_barrier(mem_flags::mem_none);- }+ IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }const short kk = ik * kFragSize;const short dd = id * kFragSize;@@ -401,9 +399,7 @@ template <&Vs[Vs_offset + kk * LDV_tgp + dd]);- IF_CONSTEXPR(BD == 128) {- simdgroup_barrier(mem_flags::mem_none);- }+ IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }MMAFrag_acc_t::mma(Otile.frag_at(0, id),diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metalindex 047eec575..88c28ef86 100644--- a/crates/uzu/src/backends/metal/kernel/definitions.metal+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal@@ -462,9 +462,9 @@ static T threadgroup_cooperative_reduce_min(// warning: constexpr if is a C++17 extension [-Wc++17-extensions]#if defined(__cpp_if_constexpr)- #define IF_CONSTEXPR(cond) if constexpr (cond)+#define IF_CONSTEXPR(cond) if constexpr (cond)#else- #define IF_CONSTEXPR(cond) if (cond)+#define IF_CONSTEXPR(cond) if (cond)#endif// MARK: - DSL Annotation Helpers@@ -478,7 +478,8 @@ static T threadgroup_cooperative_reduce_min(#define DSL_STR(X) #X#define DSL_XSTR(X) DSL_STR(X)-#define VARIANTS(TYPENAME, ...) DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)+#define VARIANTS(TYPENAME, ...) \+ DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME#define SPECIALIZE DSL_META("dsl.specialize")@@ -489,12 +490,26 @@ static T threadgroup_cooperative_reduce_min(// MARK: - Generate Template Kernels-#define generateKernel(max_threads, functionName, scalarType, outerArgs, innerArgs) \+#define generateKernel( \+ max_threads, \+ functionName, \+ scalarType, \+ outerArgs, \+ innerArgs \+) \[[max_total_threads_per_threadgroup(max_threads)]] \- kernel void functionName##_##scalarType outerArgs { functionName innerArgs; }+ kernel void functionName##_##scalarType outerArgs { \+ functionName innerArgs; \+ }#define generateKernels(max_threads, functionName) \- generateKernel(max_threads, functionName, float, outerArguments(float), innerArguments); \+ generateKernel( \+ max_threads, \+ functionName, \+ float, \+ outerArguments(float), \+ innerArguments \+ ); \generateKernel( \max_threads, \functionName, \@@ -502,6 +517,12 @@ static T threadgroup_cooperative_reduce_min(outerArguments(bfloat), \innerArguments \); \- generateKernel(max_threads, functionName, half, outerArguments(half), innerArguments);+ generateKernel( \+ max_threads, \+ functionName, \+ half, \+ outerArguments(half), \+ innerArguments \+ );#endif /* definitions_metal */diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex 6152a10e6..9e0a9225e 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -51,7 +51,8 @@ KERNEL(QuantizedEmbeddingLookup) (}} else if (quant_mode == QUANT_INT8) {const uint elem_idx = token_id * weights_stride + dim_idx;- const device int8_t* weights_i8 = reinterpret_cast<const device int8_t*>(weights);+ const device int8_t* weights_i8 =+ reinterpret_cast<const device int8_t*>(weights);quantized_value = int(weights_i8[elem_idx]);} else {const uint elem_idx = token_id * weights_stride + dim_idx;diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metalindex b35251ede..855d5e1d1 100644--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal@@ -35,7 +35,7 @@ void updateKVCache(#define outerArguments(T) \(device T * inPlaceKeys [[buffer(0)]], \device T * inPlaceValues [[buffer(1)]], \- const constant int2 * swaps [[buffer(2)]], \+ const constant int2* swaps [[buffer(2)]], \const constant int& swapCount [[buffer(3)]], \const constant int& numHeads [[buffer(4)]], \const constant int& maxSequenceLength [[buffer(5)]], \diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metalindex 2e84bebdc..a033243f4 100644--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal@@ -131,15 +131,15 @@ KERNEL(LayerNorm) () {const uint input_offset = batch_idx * model_dim;layer_norm_core<IN, SC, OUT, ACC>(- input + input_offset,- scales,- output + input_offset,- model_dim,- epsilon,- scale_offset,- shared_mean,- shared_variance,- thread_in_row,- full_layer+ input + input_offset,+ scales,+ output + input_offset,+ model_dim,+ epsilon,+ scale_offset,+ shared_mean,+ shared_variance,+ thread_in_row,+ full_layer);}diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.hindex b70b9fac5..3bf58387a 100644--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h@@ -96,9 +96,8 @@ struct BaseMMAFrag<T, 8, 8> {STEEL_PRAGMA_UNROLLfor (short j = 0; j < kElemCols; j++) {if ((off_x + i) < lim_x && (off_y + j) < lim_y) {- dst[i * kElemCols + j] = static_cast<T>(- src[(off_x + i) * str_x + (off_y + j) * str_y]- );+ dst[i * kElemCols + j] =+ static_cast<T>(src[(off_x + i) * str_x + (off_y + j) * str_y]);} else {dst[i * kElemCols + j] = T(0);}diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.hindex c1bf82170..8c0522fc9 100644--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h@@ -201,7 +201,8 @@ gemm_splitk_mlp_fused(///////////////////////////////////////////////////////////////////////////////template <typename AccT, typename OutT>-[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_mlp_fused_accum(+[[kernel, max_total_threads_per_threadgroup(256)]] void+gemm_splitk_mlp_fused_accum(const device AccT* C_split [[buffer(0)]],device OutT* D [[buffer(1)]],const constant int& k_partitions [[buffer(2)]],diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metalindex 653a9a796..743dddc44 100644--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal@@ -12,14 +12,16 @@ KERNEL(MoeCountsOffsetsFused)(device const int* topk_ids,device uint* offsets, // output: exclusive scan [E+1]device uint* sum_k_out, // output: total count [1]- device uint* partials, // output: partials [num_tiles * TILE_E] (for block_bases)+ device uint*+ partials, // output: partials [num_tiles * TILE_E] (for block_bases)constant uint& t_input,constant uint& e_input,constant uint& k_input,threadgroup _atomic<uint> tg_hist[TILE_E],threadgroup uint scan_shared[BLOCK_SIZE],threadgroup uint reduce_shared[BLOCK_SIZE],- threadgroup uint counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory+ threadgroup uint+ counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memorythreadgroup uint& carry,const uint lid THREADS(128)) {diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metalindex bf345f1ff..6c1abb418 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal@@ -61,14 +61,16 @@ KERNEL(MoeExpertsDecodeSinglePassA)(uint base_idx = i * 128 + simd_lid * 4;device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);- device const T* w_up_vec2 = reinterpret_cast<device const T*>(w_up_row + base_idx);+ device const T* w_up_vec2 =+ reinterpret_cast<device const T*>(w_up_row + base_idx);acc_up += float(x_vec2[0]) * float(w_up_vec2[0]);acc_up += float(x_vec2[1]) * float(w_up_vec2[1]);acc_up += float(x_vec2[2]) * float(w_up_vec2[2]);acc_up += float(x_vec2[3]) * float(w_up_vec2[3]);if (gating_sel > 1) {- device const T* w_gate_vec2 = reinterpret_cast<device const T*>(w_gate_row + base_idx);+ device const T* w_gate_vec2 =+ reinterpret_cast<device const T*>(w_gate_row + base_idx);acc_gate += float(x_vec2[0]) * float(w_gate_vec2[0]);acc_gate += float(x_vec2[1]) * float(w_gate_vec2[1]);acc_gate += float(x_vec2[2]) * float(w_gate_vec2[2]);@@ -116,7 +118,6 @@ KERNEL(MoeExpertsDecodeSinglePassA)(}}-// ============================================================================// Pass B (fused with finalize): hidden[k] @ W2[expert] → y (directly)// Computes: y[d] = Σ_k prob[k] * (hidden[k] @ W2[expert_k, d] + bias)@@ -165,8 +166,10 @@ KERNEL(MoeExpertsDecodeSinglePassB)(for (uint i = 0; i < vec_iters; ++i) {uint base_idx = i * 128 + simd_lid * 4;- float4 h_vec = *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);- device const T* w_vec2 = reinterpret_cast<device const T*>(w2_ptr + base_idx);+ float4 h_vec =+ *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);+ device const T* w_vec2 =+ reinterpret_cast<device const T*>(w2_ptr + base_idx);acc += h_vec.x * float(w_vec2[0]);acc += h_vec.y * float(w_vec2[1]);acc += h_vec.z * float(w_vec2[2]);diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metalindex 7f50ee8b6..f220d43aa 100644--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal@@ -72,14 +72,16 @@ KERNEL(MoeExpertsDecodePassA)(uint base_idx = i * 128 + simd_lid * 4;device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);- device const T* w_up_vec = reinterpret_cast<device const T*>(w_up_row + base_idx);+ device const T* w_up_vec =+ reinterpret_cast<device const T*>(w_up_row + base_idx);acc_up += float(x_vec[0]) * float(w_up_vec[0]);acc_up += float(x_vec[1]) * float(w_up_vec[1]);acc_up += float(x_vec[2]) * float(w_up_vec[2]);acc_up += float(x_vec[3]) * float(w_up_vec[3]);if (gating_sel > 1) {- device const T* w_gate_vec = reinterpret_cast<device const T*>(w_gate_row + base_idx);+ device const T* w_gate_vec =+ reinterpret_cast<device const T*>(w_gate_row + base_idx);acc_gate += float(x_vec[0]) * float(w_gate_vec[0]);acc_gate += float(x_vec[1]) * float(w_gate_vec[1]);acc_gate += float(x_vec[2]) * float(w_gate_vec[2]);@@ -110,11 +112,13 @@ KERNEL(MoeExpertsDecodePassA)(float activated;if (gating_sel <= 1) {- activated = (gating_sel == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);+ activated =+ (gating_sel == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);} else {float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);- float gate_act = (gating_sel == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);+ float gate_act = (gating_sel == 2) ? silu(gate_val, silu_alpha)+ : gelu_approx(gate_val);activated = gate_act * up_val;}@@ -184,14 +188,22 @@ KERNEL(MoeExpertsDecodeDownFused2D)(const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];// W2: lane-coalesced- const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);- const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);- const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);- const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);- const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);- const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);- const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);- const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);+ const AccumT w0 =+ AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);+ const AccumT w1 =+ AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);+ const AccumT w2 =+ AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);+ const AccumT w3 =+ AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);+ const AccumT w4 =+ AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);+ const AccumT w5 =+ AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);+ const AccumT w6 =+ AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);+ const AccumT w7 =+ AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);// dual trees for ILPacc0 = fma(h0, w0, acc0);diff --git a/crates/uzu/src/backends/metal/kernel/moe/moe_commons.h b/crates/uzu/src/backends/metal/kernel/moe/moe_commons.hindex cec3d51ab..8486e6cff 100644--- a/crates/uzu/src/backends/metal/kernel/moe/moe_commons.h+++ b/crates/uzu/src/backends/metal/kernel/moe/moe_commons.h@@ -13,7 +13,11 @@ static inline float gelu_approx(float x) {const float GELU_APPROX_K0 = 0.7978845608f; // sqrt(2/pi)const float GELU_APPROX_K1 = 0.044715f;- float t = metal::clamp(GELU_APPROX_K0 * (x + GELU_APPROX_K1 * x * x * x), -10.0f, 10.0f);+ float t = metal::clamp(+ GELU_APPROX_K0 * (x + GELU_APPROX_K1 * x * x * x),+ -10.0f,+ 10.0f+ );return 0.5f * x * (1.0f + metal::tanh(t));}diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metalindex 5474a6e52..5bc8cf5a3 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal@@ -95,9 +95,10 @@ KERNEL(MoeBuildTileMap)(// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1KERNEL(MoeWriteDispatchArgs)(- device const uint* total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows- device uint* dispatch_args, // [3] u32- constant uint& num_tiles_n, // x dimension+ device const uint*+ total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows+ device uint* dispatch_args, // [3] u32+ constant uint& num_tiles_n, // x dimensionuint gid AXIS(1, 1)) {const uint total_rows = total_tiles_buf[0];diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metalindex ec1522135..28b130777 100644--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal@@ -119,8 +119,9 @@ KERNEL(MoePassABuildTileMap)(// Write dispatch args for indirect dispatch (reusable from tiled version)KERNEL(MoePassAWriteDispatchArgs)(device const uint* total_tiles, // [1]- device uint* dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments- constant uint& num_tiles_y, // usually 1 for Pass A+ device uint*+ dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments+ constant uint& num_tiles_y, // usually 1 for Pass Auint tid AXIS(1, 1)) {dispatch_args[0] = total_tiles[0]; // x dimension = total tilesdiff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metalindex f4be25cb6..9e83f09e3 100644--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal@@ -4198,16 +4198,16 @@ qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(template <typename U>static inline U extract_zp(- const device uint8_t* zps,- int row_offset,- int zp_stride,- int bits,+ const device uint8_t* zps,+ int row_offset,+ int zp_stride,+ int bits,bool high_nibble) {uint8_t zp_byte = zps[row_offset * zp_stride];- return bits == 4- ? static_cast<U>(high_nibble ? (zp_byte >> 4) : (zp_byte & 0x0F))- : static_cast<U>(zp_byte);+ return bits == 4+ ? static_cast<U>(high_nibble ? (zp_byte >> 4) : (zp_byte & 0x0F))+ : static_cast<U>(zp_byte);};// MLP fused QMV kernel for decode path (M=1).diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metalindex 6ad763d0b..5cfda4c7e 100644--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal@@ -259,8 +259,14 @@ FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)// Strategy:// - One SIMD-group (32 threads) processes one head.// - A threadgroup processes multiple heads for a given token (batch).-// - No threadgroup-wide reductions/barriers: we only use simd_sum within a SIMD-group.-template <typename InputT, typename ScaleT, typename OutputT, typename AccumT, bool FULL_LAYER>+// - No threadgroup-wide reductions/barriers: we only use simd_sum within a+// SIMD-group.+template <+ typename InputT,+ typename ScaleT,+ typename OutputT,+ typename AccumT,+ bool FULL_LAYER>static inline void qk_norm_head_core(const device InputT* input_data,const device ScaleT* scales_data,@@ -281,8 +287,7 @@ static inline void qk_norm_head_core(AccumT vals[GRAIN_SIZE];for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;- vals[j] =- (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;}for (uint j = 0; j < GRAIN_SIZE; ++j) {partial_sum += vals[j] * vals[j];@@ -303,8 +308,7 @@ static inline void qk_norm_head_core(AccumT vals[GRAIN_SIZE];for (uint j = 0; j < GRAIN_SIZE; ++j) {uint i = base_i + j;- vals[j] =- (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;}for (uint j = 0; j < GRAIN_SIZE; ++j) {@@ -374,7 +378,7 @@ static inline void qk_norm_head_core(if (logical_head_idx >= total_heads_in_buffer) \return; \\- const ulong slice_offset = \+ const ulong slice_offset = \(ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim + \(ulong)logical_head_idx * (ulong)head_dim; \\diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metalindex 35739e2d5..0b318ed91 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal@@ -69,11 +69,12 @@ template <typename T>void conv1d_scan_kernel(device const T* padded [[buffer(0)]], // (prefix+suffix, channels)device const T* w [[buffer(1)]], // (channels, kernel)- device const T* b [[buffer(2), function_constant(has_bias)]], // optional (channels)- device T* x_out [[buffer(3)]], // (suffix, inner_dim)- device T* b_out [[buffer(4)]], // (suffix, proj_dim)- device T* c_out [[buffer(5)]], // (suffix, proj_dim)- device T* state_out [[buffer(6)]], // (channels, kernel-1)+ device const T* b+ [[buffer(2), function_constant(has_bias)]], // optional (channels)+ device T* x_out [[buffer(3)]], // (suffix, inner_dim)+ device T* b_out [[buffer(4)]], // (suffix, proj_dim)+ device T* c_out [[buffer(5)]], // (suffix, proj_dim)+ device T* state_out [[buffer(6)]], // (channels, kernel-1)constant const size_t& suffix_len [[buffer(7)]],constant const int& kernel_size [[buffer(8)]],constant const size_t& row_stride [[buffer(9)]],diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex a1966002e..bf1898172 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -183,8 +183,8 @@ KERNEL(SSDPrefill)(#pragma unrollfor (int chunk = 0; chunk < chunk_count; ++chunk) {const int idx = chunk * int(simd_width) + int(lane_idx);- const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride +- token * cb_token_stride;+ const uint cb_idx =+ cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;const float new_state =decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);lane_states[chunk] = new_state;@@ -234,7 +234,8 @@ KERNEL(SSDPrefillSequential)(size_t(dh_idx) * state_strides[1];for (uint token = 0; token < suffix_len; ++token) {- const uint x_idx = token * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];+ const uint x_idx =+ token * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];const uint dt_idx = token * dt_strides[0] + h_idx * dt_strides[1];const uint cb_base = token * cb_strides[0] + group_idx * cb_strides[1];diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metalindex 43eeed1ab..2765899bb 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal@@ -34,10 +34,14 @@ KERNEL(SSDUpdate)(const uint h_idx AXIS(h_size, 32),const uint dh_idx AXIS(dh_size, 1)) {- const uint cb_start_idx = b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];- const uint x_idx = b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];+ const uint cb_start_idx =+ b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];+ const uint x_idx =+ b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];const uint dt_idx = b_idx * dt_strides[0] + h_idx * dt_strides[1];- const uint state_start_idx = b_idx * state_strides[0] + h_idx * state_strides[1] + dh_idx * state_strides[2];+ const uint state_start_idx = b_idx * state_strides[0] ++ h_idx * state_strides[1] ++ dh_idx * state_strides[2];// load dataT this_x = x[x_idx];@@ -54,7 +58,8 @@ KERNEL(SSDUpdate)(for (uint i = 0; i < state_size; ++i) {uint cb_idx = cb_start_idx + i;uint state_idx = state_start_idx + i;- T this_new_state = state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;+ T this_new_state =+ state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;next_state[state_idx] = this_new_state;temp = temp + this_new_state * c[cb_idx];}diff --git a/crates/uzu/src/language_model/gpu_capture.rs b/crates/uzu/src/language_model/gpu_capture.rsindex d70da7b81..f4ff73708 100644--- a/crates/uzu/src/language_model/gpu_capture.rs+++ b/crates/uzu/src/language_model/gpu_capture.rs@@ -3,7 +3,7 @@ use std::{time::{SystemTime, UNIX_EPOCH},};-use objc2_foundation::{NSURL, NSString};+use objc2_foundation::{NSString, NSURL};use crate::{backends::metal::{@@ -82,9 +82,9 @@ impl GpuCaptureManager {// Set output URL using NSURLlet url_string = format!("file://{}", trace_path.display());unsafe {- if let Some(url) = NSURL::URLWithString(- &NSString::from_str(&url_string),- ) {+ if let Some(url) =+ NSURL::URLWithString(&NSString::from_str(&url_string))+ {capture_descriptor.set_output_url(Some(&url));}}diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rsindex f7a1b0bc2..c9b42fcb1 100644--- a/crates/uzu/src/language_model/language_model_generator.rs+++ b/crates/uzu/src/language_model/language_model_generator.rs@@ -436,15 +436,12 @@ impl LanguageModelGenerator {// Fill attention bias from KV cache state for the first decode position.// Windowed buffers get the correct mask; GPU patches maintain it for subsequent passes.// Full-attention buffers also get filled (masking beyond-prefix positions).- self.context- .cache_layers- .borrow()- .fill_attention_bias_scratch(- &self.context.scratch_buffers.attention_window_size_to_bias,- &[first_decode_position],- 1,- &self.context.mtl_context,- );+ self.context.cache_layers.borrow().fill_attention_bias_scratch(+ &self.context.scratch_buffers.attention_window_size_to_bias,+ &[first_decode_position],+ 1,+ &self.context.mtl_context,+ );self.context.async_buffersdiff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rsindex 9ef961bf9..5cefdc32e 100644--- a/crates/uzu/src/session/chat_session.rs+++ b/crates/uzu/src/session/chat_session.rs@@ -648,8 +648,12 @@ impl ChatSession {if let Some(ctx) = context {let mut llm_state =language_model_generator.context.cache_layers.borrow_mut();- for (_layer_idx, (ctx_layer, gen_layer)) in- ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()+ for (_layer_idx, (ctx_layer, gen_layer)) in ctx+ .cache_layers+ .data+ .iter()+ .zip(llm_state.data.iter_mut())+ .enumerate(){match (ctx_layer, gen_layer) {(