Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

fmt (#157)

uuuvne604781d48b5
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.h b/crates/uzu/src/backends/metal/kernel/activation/activation.h
index 491eb88c9..431183667 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activation.h
+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.h
@@ -17,7 +17,8 @@ inline T gelu_approx(T x) {
constexpr float k0 = 0.044715f;
constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
float xf = float(x);
- float yf = 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));
+ float yf =
+ 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));
return T(yf);
}
diff --git a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
index 480b71e48..8efd95c71 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
+++ b/crates/uzu/src/backends/metal/kernel/attention/gemm_attention.metal
@@ -390,9 +390,7 @@ template <
for (short id = 0; id < TD; id++) {
UZU_PRAGMA_UNROLL
for (short ik = 0; ik < TK; ik++) {
- IF_CONSTEXPR(BD == 128) {
- simdgroup_barrier(mem_flags::mem_none);
- }
+ IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }
const short kk = ik * kFragSize;
const short dd = id * kFragSize;
@@ -401,9 +399,7 @@ template <
&Vs[Vs_offset + kk * LDV_tgp + dd]
);
- IF_CONSTEXPR(BD == 128) {
- simdgroup_barrier(mem_flags::mem_none);
- }
+ IF_CONSTEXPR(BD == 128) { simdgroup_barrier(mem_flags::mem_none); }
MMAFrag_acc_t::mma(
Otile.frag_at(0, id),
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metal
index 047eec575..88c28ef86 100644
--- a/crates/uzu/src/backends/metal/kernel/definitions.metal
+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal
@@ -462,9 +462,9 @@ static T threadgroup_cooperative_reduce_min(
// warning: constexpr if is a C++17 extension [-Wc++17-extensions]
#if defined(__cpp_if_constexpr)
- #define IF_CONSTEXPR(cond) if constexpr (cond)
+#define IF_CONSTEXPR(cond) if constexpr (cond)
#else
- #define IF_CONSTEXPR(cond) if (cond)
+#define IF_CONSTEXPR(cond) if (cond)
#endif
// MARK: - DSL Annotation Helpers
@@ -478,7 +478,8 @@ static T threadgroup_cooperative_reduce_min(
#define DSL_STR(X) #X
#define DSL_XSTR(X) DSL_STR(X)
-#define VARIANTS(TYPENAME, ...) DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
+#define VARIANTS(TYPENAME, ...) \
+ DSL_META("dsl.variants", #TYPENAME, #__VA_ARGS__)
#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME
#define SPECIALIZE DSL_META("dsl.specialize")
@@ -489,12 +490,26 @@ static T threadgroup_cooperative_reduce_min(
// MARK: - Generate Template Kernels
-#define generateKernel(max_threads, functionName, scalarType, outerArgs, innerArgs) \
+#define generateKernel( \
+ max_threads, \
+ functionName, \
+ scalarType, \
+ outerArgs, \
+ innerArgs \
+) \
[[max_total_threads_per_threadgroup(max_threads)]] \
- kernel void functionName##_##scalarType outerArgs { functionName innerArgs; }
+ kernel void functionName##_##scalarType outerArgs { \
+ functionName innerArgs; \
+ }
#define generateKernels(max_threads, functionName) \
- generateKernel(max_threads, functionName, float, outerArguments(float), innerArguments); \
+ generateKernel( \
+ max_threads, \
+ functionName, \
+ float, \
+ outerArguments(float), \
+ innerArguments \
+ ); \
generateKernel( \
max_threads, \
functionName, \
@@ -502,6 +517,12 @@ static T threadgroup_cooperative_reduce_min(
outerArguments(bfloat), \
innerArguments \
); \
- generateKernel(max_threads, functionName, half, outerArguments(half), innerArguments);
+ generateKernel( \
+ max_threads, \
+ functionName, \
+ half, \
+ outerArguments(half), \
+ innerArguments \
+ );
#endif /* definitions_metal */
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index 6152a10e6..9e0a9225e 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -51,7 +51,8 @@ KERNEL(QuantizedEmbeddingLookup) (
}
} else if (quant_mode == QUANT_INT8) {
const uint elem_idx = token_id * weights_stride + dim_idx;
- const device int8_t* weights_i8 = reinterpret_cast<const device int8_t*>(weights);
+ const device int8_t* weights_i8 =
+ reinterpret_cast<const device int8_t*>(weights);
quantized_value = int(weights_i8[elem_idx]);
} else {
const uint elem_idx = token_id * weights_stride + dim_idx;
diff --git a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
index b35251ede..855d5e1d1 100644
--- a/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/kv_cache_update/kv_cache_update.metal
@@ -35,7 +35,7 @@ void updateKVCache(
#define outerArguments(T) \
(device T * inPlaceKeys [[buffer(0)]], \
device T * inPlaceValues [[buffer(1)]], \
- const constant int2 * swaps [[buffer(2)]], \
+ const constant int2* swaps [[buffer(2)]], \
const constant int& swapCount [[buffer(3)]], \
const constant int& numHeads [[buffer(4)]], \
const constant int& maxSequenceLength [[buffer(5)]], \
diff --git a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
index 2e84bebdc..a033243f4 100644
--- a/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/layer_norm/layer_norm.metal
@@ -131,15 +131,15 @@ KERNEL(LayerNorm) (
) {
const uint input_offset = batch_idx * model_dim;
layer_norm_core<IN, SC, OUT, ACC>(
- input + input_offset,
- scales,
- output + input_offset,
- model_dim,
- epsilon,
- scale_offset,
- shared_mean,
- shared_variance,
- thread_in_row,
- full_layer
+ input + input_offset,
+ scales,
+ output + input_offset,
+ model_dim,
+ epsilon,
+ scale_offset,
+ shared_mean,
+ shared_variance,
+ thread_in_row,
+ full_layer
);
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h
index b70b9fac5..3bf58387a 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/steel/gemm/mma.h
@@ -96,9 +96,8 @@ struct BaseMMAFrag<T, 8, 8> {
STEEL_PRAGMA_UNROLL
for (short j = 0; j < kElemCols; j++) {
if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
- dst[i * kElemCols + j] = static_cast<T>(
- src[(off_x + i) * str_x + (off_y + j) * str_y]
- );
+ dst[i * kElemCols + j] =
+ static_cast<T>(src[(off_x + i) * str_x + (off_y + j) * str_y]);
} else {
dst[i * kElemCols + j] = T(0);
}
diff --git a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
index c1bf82170..8c0522fc9 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
+++ b/crates/uzu/src/backends/metal/kernel/mlp_fused/split_k/steel_gemm_splitk_mlp_fused.h
@@ -201,7 +201,8 @@ gemm_splitk_mlp_fused(
///////////////////////////////////////////////////////////////////////////////
template <typename AccT, typename OutT>
-[[kernel, max_total_threads_per_threadgroup(256)]] void gemm_splitk_mlp_fused_accum(
+[[kernel, max_total_threads_per_threadgroup(256)]] void
+gemm_splitk_mlp_fused_accum(
const device AccT* C_split [[buffer(0)]],
device OutT* D [[buffer(1)]],
const constant int& k_partitions [[buffer(2)]],
diff --git a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
index 653a9a796..743dddc44 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/counts_offsets_fused.metal
@@ -12,14 +12,16 @@ KERNEL(MoeCountsOffsetsFused)(
device const int* topk_ids,
device uint* offsets, // output: exclusive scan [E+1]
device uint* sum_k_out, // output: total count [1]
- device uint* partials, // output: partials [num_tiles * TILE_E] (for block_bases)
+ device uint*
+ partials, // output: partials [num_tiles * TILE_E] (for block_bases)
constant uint& t_input,
constant uint& e_input,
constant uint& k_input,
threadgroup _atomic<uint> tg_hist[TILE_E],
threadgroup uint scan_shared[BLOCK_SIZE],
threadgroup uint reduce_shared[BLOCK_SIZE],
- threadgroup uint counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory
+ threadgroup uint
+ counts_shared[BLOCK_SIZE], // Cache counts in threadgroup memory
threadgroup uint& carry,
const uint lid THREADS(128)
) {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
index bf345f1ff..6c1abb418 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.metal
@@ -61,14 +61,16 @@ KERNEL(MoeExpertsDecodeSinglePassA)(
uint base_idx = i * 128 + simd_lid * 4;
device const T* x_vec2 = reinterpret_cast<device const T*>(x + base_idx);
- device const T* w_up_vec2 = reinterpret_cast<device const T*>(w_up_row + base_idx);
+ device const T* w_up_vec2 =
+ reinterpret_cast<device const T*>(w_up_row + base_idx);
acc_up += float(x_vec2[0]) * float(w_up_vec2[0]);
acc_up += float(x_vec2[1]) * float(w_up_vec2[1]);
acc_up += float(x_vec2[2]) * float(w_up_vec2[2]);
acc_up += float(x_vec2[3]) * float(w_up_vec2[3]);
if (gating_sel > 1) {
- device const T* w_gate_vec2 = reinterpret_cast<device const T*>(w_gate_row + base_idx);
+ device const T* w_gate_vec2 =
+ reinterpret_cast<device const T*>(w_gate_row + base_idx);
acc_gate += float(x_vec2[0]) * float(w_gate_vec2[0]);
acc_gate += float(x_vec2[1]) * float(w_gate_vec2[1]);
acc_gate += float(x_vec2[2]) * float(w_gate_vec2[2]);
@@ -116,7 +118,6 @@ KERNEL(MoeExpertsDecodeSinglePassA)(
}
}
-
// ============================================================================
// Pass B (fused with finalize): hidden[k] @ W2[expert] → y (directly)
// Computes: y[d] = Σ_k prob[k] * (hidden[k] @ W2[expert_k, d] + bias)
@@ -165,8 +166,10 @@ KERNEL(MoeExpertsDecodeSinglePassB)(
for (uint i = 0; i < vec_iters; ++i) {
uint base_idx = i * 128 + simd_lid * 4;
- float4 h_vec = *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
- device const T* w_vec2 = reinterpret_cast<device const T*>(w2_ptr + base_idx);
+ float4 h_vec =
+ *reinterpret_cast<device const float4*>(hidden_ptr + base_idx);
+ device const T* w_vec2 =
+ reinterpret_cast<device const T*>(w2_ptr + base_idx);
acc += h_vec.x * float(w_vec2[0]);
acc += h_vec.y * float(w_vec2[1]);
acc += h_vec.z * float(w_vec2[2]);
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
index 7f50ee8b6..f220d43aa 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_two_pass_decode.metal
@@ -72,14 +72,16 @@ KERNEL(MoeExpertsDecodePassA)(
uint base_idx = i * 128 + simd_lid * 4;
device const T* x_vec = reinterpret_cast<device const T*>(x_ptr + base_idx);
- device const T* w_up_vec = reinterpret_cast<device const T*>(w_up_row + base_idx);
+ device const T* w_up_vec =
+ reinterpret_cast<device const T*>(w_up_row + base_idx);
acc_up += float(x_vec[0]) * float(w_up_vec[0]);
acc_up += float(x_vec[1]) * float(w_up_vec[1]);
acc_up += float(x_vec[2]) * float(w_up_vec[2]);
acc_up += float(x_vec[3]) * float(w_up_vec[3]);
if (gating_sel > 1) {
- device const T* w_gate_vec = reinterpret_cast<device const T*>(w_gate_row + base_idx);
+ device const T* w_gate_vec =
+ reinterpret_cast<device const T*>(w_gate_row + base_idx);
acc_gate += float(x_vec[0]) * float(w_gate_vec[0]);
acc_gate += float(x_vec[1]) * float(w_gate_vec[1]);
acc_gate += float(x_vec[2]) * float(w_gate_vec[2]);
@@ -110,11 +112,13 @@ KERNEL(MoeExpertsDecodePassA)(
float activated;
if (gating_sel <= 1) {
- activated = (gating_sel == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);
+ activated =
+ (gating_sel == 0) ? gelu_approx(up_val) : silu(up_val, silu_alpha);
} else {
float gate_val = acc_gate + float(up_biases[bias_base + d_ff + h_idx]);
gate_val = clamp(gate_val, gate_clip_min, gate_clip_max);
- float gate_act = (gating_sel == 2) ? silu(gate_val, silu_alpha) : gelu_approx(gate_val);
+ float gate_act = (gating_sel == 2) ? silu(gate_val, silu_alpha)
+ : gelu_approx(gate_val);
activated = gate_act * up_val;
}
@@ -184,14 +188,22 @@ KERNEL(MoeExpertsDecodeDownFused2D)(
const AccumT h7 = hidden[hidden_base + k_base + 7 * THREADS_PER_SIMD];
// W2: lane-coalesced
- const AccumT w0 = AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
- const AccumT w1 = AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
- const AccumT w2 = AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
- const AccumT w3 = AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
- const AccumT w4 = AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
- const AccumT w5 = AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
- const AccumT w6 = AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
- const AccumT w7 = AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
+ const AccumT w0 =
+ AccumT(w2_all[w2_col_base + k_base + 0 * THREADS_PER_SIMD]);
+ const AccumT w1 =
+ AccumT(w2_all[w2_col_base + k_base + 1 * THREADS_PER_SIMD]);
+ const AccumT w2 =
+ AccumT(w2_all[w2_col_base + k_base + 2 * THREADS_PER_SIMD]);
+ const AccumT w3 =
+ AccumT(w2_all[w2_col_base + k_base + 3 * THREADS_PER_SIMD]);
+ const AccumT w4 =
+ AccumT(w2_all[w2_col_base + k_base + 4 * THREADS_PER_SIMD]);
+ const AccumT w5 =
+ AccumT(w2_all[w2_col_base + k_base + 5 * THREADS_PER_SIMD]);
+ const AccumT w6 =
+ AccumT(w2_all[w2_col_base + k_base + 6 * THREADS_PER_SIMD]);
+ const AccumT w7 =
+ AccumT(w2_all[w2_col_base + k_base + 7 * THREADS_PER_SIMD]);
// dual trees for ILP
acc0 = fma(h0, w0, acc0);
diff --git a/crates/uzu/src/backends/metal/kernel/moe/moe_commons.h b/crates/uzu/src/backends/metal/kernel/moe/moe_commons.h
index cec3d51ab..8486e6cff 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/moe_commons.h
+++ b/crates/uzu/src/backends/metal/kernel/moe/moe_commons.h
@@ -13,7 +13,11 @@ static inline float gelu_approx(float x) {
const float GELU_APPROX_K0 = 0.7978845608f; // sqrt(2/pi)
const float GELU_APPROX_K1 = 0.044715f;
- float t = metal::clamp(GELU_APPROX_K0 * (x + GELU_APPROX_K1 * x * x * x), -10.0f, 10.0f);
+ float t = metal::clamp(
+ GELU_APPROX_K0 * (x + GELU_APPROX_K1 * x * x * x),
+ -10.0f,
+ 10.0f
+ );
return 0.5f * x * (1.0f + metal::tanh(t));
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
index 5474a6e52..5bc8cf5a3 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_map.metal
@@ -95,9 +95,10 @@ KERNEL(MoeBuildTileMap)(
// Write MTLDispatchThreadgroupsIndirectArguments {x, y, z} where:
// x = num_tiles_n (computed on CPU and passed in), y = total_tiles, z = 1
KERNEL(MoeWriteDispatchArgs)(
- device const uint* total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows
- device uint* dispatch_args, // [3] u32
- constant uint& num_tiles_n, // x dimension
+ device const uint*
+ total_tiles_buf, // [>=1], total_tiles_buf[0] = total_rows
+ device uint* dispatch_args, // [3] u32
+ constant uint& num_tiles_n, // x dimension
uint gid AXIS(1, 1)
) {
const uint total_rows = total_tiles_buf[0];
diff --git a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
index ec1522135..28b130777 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
+++ b/crates/uzu/src/backends/metal/kernel/moe/tiles_pass_a.metal
@@ -119,8 +119,9 @@ KERNEL(MoePassABuildTileMap)(
// Write dispatch args for indirect dispatch (reusable from tiled version)
KERNEL(MoePassAWriteDispatchArgs)(
device const uint* total_tiles, // [1]
- device uint* dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments
- constant uint& num_tiles_y, // usually 1 for Pass A
+ device uint*
+ dispatch_args, // [3] - MTLDispatchThreadgroupsIndirectArguments
+ constant uint& num_tiles_y, // usually 1 for Pass A
uint tid AXIS(1, 1)
) {
dispatch_args[0] = total_tiles[0]; // x dimension = total tiles
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
index f4be25cb6..9e83f09e3 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/quant_matmul.metal
@@ -4198,16 +4198,16 @@ qmm_transposed<bfloat, 128, 4, true, 128, 32, 64>(
template <typename U>
static inline U extract_zp(
- const device uint8_t* zps,
- int row_offset,
- int zp_stride,
- int bits,
+ const device uint8_t* zps,
+ int row_offset,
+ int zp_stride,
+ int bits,
bool high_nibble
) {
uint8_t zp_byte = zps[row_offset * zp_stride];
- return bits == 4
- ? static_cast<U>(high_nibble ? (zp_byte >> 4) : (zp_byte & 0x0F))
- : static_cast<U>(zp_byte);
+ return bits == 4
+ ? static_cast<U>(high_nibble ? (zp_byte >> 4) : (zp_byte & 0x0F))
+ : static_cast<U>(zp_byte);
};
// MLP fused QMV kernel for decode path (M=1).
diff --git a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
index 6ad763d0b..5cfda4c7e 100644
--- a/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
+++ b/crates/uzu/src/backends/metal/kernel/rms_norm/rms_norm.metal
@@ -259,8 +259,14 @@ FOREACH_RMS_COMBO(DEFINE_RMS_KERNEL)
// Strategy:
// - One SIMD-group (32 threads) processes one head.
// - A threadgroup processes multiple heads for a given token (batch).
-// - No threadgroup-wide reductions/barriers: we only use simd_sum within a SIMD-group.
-template <typename InputT, typename ScaleT, typename OutputT, typename AccumT, bool FULL_LAYER>
+// - No threadgroup-wide reductions/barriers: we only use simd_sum within a
+// SIMD-group.
+template <
+ typename InputT,
+ typename ScaleT,
+ typename OutputT,
+ typename AccumT,
+ bool FULL_LAYER>
static inline void qk_norm_head_core(
const device InputT* input_data,
const device ScaleT* scales_data,
@@ -281,8 +287,7 @@ static inline void qk_norm_head_core(
AccumT vals[GRAIN_SIZE];
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
- vals[j] =
- (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
}
for (uint j = 0; j < GRAIN_SIZE; ++j) {
partial_sum += vals[j] * vals[j];
@@ -303,8 +308,7 @@ static inline void qk_norm_head_core(
AccumT vals[GRAIN_SIZE];
for (uint j = 0; j < GRAIN_SIZE; ++j) {
uint i = base_i + j;
- vals[j] =
- (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
+ vals[j] = (i < element_count) ? static_cast<AccumT>(input_data[i]) : 0.0f;
}
for (uint j = 0; j < GRAIN_SIZE; ++j) {
@@ -374,7 +378,7 @@ static inline void qk_norm_head_core(
if (logical_head_idx >= total_heads_in_buffer) \
return; \
\
- const ulong slice_offset = \
+ const ulong slice_offset = \
(ulong)batch_idx * (ulong)total_heads_in_buffer * (ulong)head_dim + \
(ulong)logical_head_idx * (ulong)head_dim; \
\
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
index 35739e2d5..0b318ed91 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.metal
@@ -69,11 +69,12 @@ template <typename T>
void conv1d_scan_kernel(
device const T* padded [[buffer(0)]], // (prefix+suffix, channels)
device const T* w [[buffer(1)]], // (channels, kernel)
- device const T* b [[buffer(2), function_constant(has_bias)]], // optional (channels)
- device T* x_out [[buffer(3)]], // (suffix, inner_dim)
- device T* b_out [[buffer(4)]], // (suffix, proj_dim)
- device T* c_out [[buffer(5)]], // (suffix, proj_dim)
- device T* state_out [[buffer(6)]], // (channels, kernel-1)
+ device const T* b
+ [[buffer(2), function_constant(has_bias)]], // optional (channels)
+ device T* x_out [[buffer(3)]], // (suffix, inner_dim)
+ device T* b_out [[buffer(4)]], // (suffix, proj_dim)
+ device T* c_out [[buffer(5)]], // (suffix, proj_dim)
+ device T* state_out [[buffer(6)]], // (channels, kernel-1)
constant const size_t& suffix_len [[buffer(7)]],
constant const int& kernel_size [[buffer(8)]],
constant const size_t& row_stride [[buffer(9)]],
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index a1966002e..bf1898172 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -183,8 +183,8 @@ KERNEL(SSDPrefill)(
#pragma unroll
for (int chunk = 0; chunk < chunk_count; ++chunk) {
const int idx = chunk * int(simd_width) + int(lane_idx);
- const uint cb_idx = cb_group_base + uint(idx) * cb_state_stride +
- token * cb_token_stride;
+ const uint cb_idx =
+ cb_group_base + uint(idx) * cb_state_stride + token * cb_token_stride;
const float new_state =
decay_val * lane_states[chunk] + dt_scaled_input * float(b[cb_idx]);
lane_states[chunk] = new_state;
@@ -234,7 +234,8 @@ KERNEL(SSDPrefillSequential)(
size_t(dh_idx) * state_strides[1];
for (uint token = 0; token < suffix_len; ++token) {
- const uint x_idx = token * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
+ const uint x_idx =
+ token * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
const uint dt_idx = token * dt_strides[0] + h_idx * dt_strides[1];
const uint cb_base = token * cb_strides[0] + group_idx * cb_strides[1];
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
index 43eeed1ab..2765899bb 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
@@ -34,10 +34,14 @@ KERNEL(SSDUpdate)(
const uint h_idx AXIS(h_size, 32),
const uint dh_idx AXIS(dh_size, 1)
) {
- const uint cb_start_idx = b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];
- const uint x_idx = b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
+ const uint cb_start_idx =
+ b_idx * cb_strides[0] + (h_idx / group_size) * cb_strides[1];
+ const uint x_idx =
+ b_idx * x_strides[0] + h_idx * x_strides[1] + dh_idx * x_strides[2];
const uint dt_idx = b_idx * dt_strides[0] + h_idx * dt_strides[1];
- const uint state_start_idx = b_idx * state_strides[0] + h_idx * state_strides[1] + dh_idx * state_strides[2];
+ const uint state_start_idx = b_idx * state_strides[0] +
+ h_idx * state_strides[1] +
+ dh_idx * state_strides[2];
// load data
T this_x = x[x_idx];
@@ -54,7 +58,8 @@ KERNEL(SSDUpdate)(
for (uint i = 0; i < state_size; ++i) {
uint cb_idx = cb_start_idx + i;
uint state_idx = state_start_idx + i;
- T this_new_state = state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;
+ T this_new_state =
+ state[state_idx] * this_decay + b[cb_idx] * dt_scaled_input;
next_state[state_idx] = this_new_state;
temp = temp + this_new_state * c[cb_idx];
}
diff --git a/crates/uzu/src/language_model/gpu_capture.rs b/crates/uzu/src/language_model/gpu_capture.rs
index d70da7b81..f4ff73708 100644
--- a/crates/uzu/src/language_model/gpu_capture.rs
+++ b/crates/uzu/src/language_model/gpu_capture.rs
@@ -3,7 +3,7 @@ use std::{
time::{SystemTime, UNIX_EPOCH},
};
-use objc2_foundation::{NSURL, NSString};
+use objc2_foundation::{NSString, NSURL};
use crate::{
backends::metal::{
@@ -82,9 +82,9 @@ impl GpuCaptureManager {
// Set output URL using NSURL
let url_string = format!("file://{}", trace_path.display());
unsafe {
- if let Some(url) = NSURL::URLWithString(
- &NSString::from_str(&url_string),
- ) {
+ if let Some(url) =
+ NSURL::URLWithString(&NSString::from_str(&url_string))
+ {
capture_descriptor.set_output_url(Some(&url));
}
}
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index f7a1b0bc2..c9b42fcb1 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -436,15 +436,12 @@ impl LanguageModelGenerator {
// Fill attention bias from KV cache state for the first decode position.
// Windowed buffers get the correct mask; GPU patches maintain it for subsequent passes.
// Full-attention buffers also get filled (masking beyond-prefix positions).
- self.context
- .cache_layers
- .borrow()
- .fill_attention_bias_scratch(
- &self.context.scratch_buffers.attention_window_size_to_bias,
- &[first_decode_position],
- 1,
- &self.context.mtl_context,
- );
+ self.context.cache_layers.borrow().fill_attention_bias_scratch(
+ &self.context.scratch_buffers.attention_window_size_to_bias,
+ &[first_decode_position],
+ 1,
+ &self.context.mtl_context,
+ );
self.context
.async_buffers
diff --git a/crates/uzu/src/session/chat_session.rs b/crates/uzu/src/session/chat_session.rs
index 9ef961bf9..5cefdc32e 100644
--- a/crates/uzu/src/session/chat_session.rs
+++ b/crates/uzu/src/session/chat_session.rs
@@ -648,8 +648,12 @@ impl ChatSession {
if let Some(ctx) = context {
let mut llm_state =
language_model_generator.context.cache_layers.borrow_mut();
- for (_layer_idx, (ctx_layer, gen_layer)) in
- ctx.cache_layers.data.iter().zip(llm_state.data.iter_mut()).enumerate()
+ for (_layer_idx, (ctx_layer, gen_layer)) in ctx
+ .cache_layers
+ .data
+ .iter()
+ .zip(llm_state.data.iter_mut())
+ .enumerate()
{
match (ctx_layer, gen_layer) {
(