A high-performance inference engine for AI models
add split-k gemm
eugene41b2e5434eb2
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex 55b8ad1cb..82202985f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -184,7 +184,11 @@ struct MxuMmaCore {threadgroup BT* b_shared,const thread ThreadContext& thread_context) {- const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);+ const uint partition = thread_context.threadgroup_position.z;+ const uint tile_y = thread_context.threadgroup_position.y;++ const uint2 tile =+ tile_id(uint2(thread_context.threadgroup_position.x, tile_y), params);const auto geometry =ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);@@ -195,9 +199,19 @@ struct MxuMmaCore {const size_t block_row = size_t(geometry.block_row_start);const size_t block_col = size_t(geometry.block_col_start);- const device AT* a_block = a + block_row * params->leading_dimension_a;+ const uint k_offset_per_block =+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision)+ ? uint(THREADGROUP_BLOCK_K_FP)+ : uint(QUANT_BK);+ const uint k_offset =+ partition * params->aligned_inner_iterations * k_offset_per_block;++ const device AT* a_block =+ a + block_row * params->leading_dimension_a + k_offset;const device BT* b_block_fp =- b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);+ b ++ (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) ++ (TRANSPOSE_B ? k_offset : k_offset * uint(params->leading_dimension_b));const ushort tile_row_offset =SIMDGROUP_BLOCK_M *@@ -207,7 +221,8 @@ struct MxuMmaCore {(thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);device DT* d_simdgroup =- d + block_row * params->leading_dimension_d + block_col ++ d + size_t(partition) * size_t(params->M) * size_t(params->N) ++ block_row * params->leading_dimension_d + block_col +tile_row_offset * params->leading_dimension_d + tile_col_offset;const short simdgroup_limit_m =@@ -271,7 +286,7 @@ struct MxuMmaCore {B_PROLOGUE == GemmBPrologueKind::FullPrecision) {const int aligned_k_iterations_fp =- int(params->K) / int(THREADGROUP_BLOCK_K_FP);+ int(params->aligned_inner_iterations);return uzu::matmul::gemm_loop<AT,BT,@@ -297,7 +312,7 @@ struct MxuMmaCore {);} else {const int aligned_k_iterations_q =- int(params->K) / int(QUANT_BK);+ int(params->aligned_inner_iterations);constexpr int pack_factor = get_pack_factor<BITS, 8>();constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();const int k_elements = int(params->K);@@ -305,18 +320,21 @@ struct MxuMmaCore {k_elements * bytes_per_pack / pack_factor;const int groups_per_row =(k_elements + GROUP_SIZE - 1) / GROUP_SIZE;+ const int k_offset_groups = int(k_offset) / GROUP_SIZE;const device uint8_t* weights_block =reinterpret_cast<const device uint8_t*>(b) +- block_col * weights_row_stride_bytes;+ block_col * weights_row_stride_bytes ++ int(k_offset) * bytes_per_pack / pack_factor;const device BT* scales_offset =- scales + block_col * groups_per_row;+ scales + block_col * groups_per_row + k_offset_groups;auto loader_b = [&]() {if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {const device BT* biases_offset =- biases + block_col * groups_per_row;+ biases + block_col * groups_per_row ++ k_offset_groups;return BLoaderScaleBias(weights_block,scales_offset,@@ -335,7 +353,9 @@ struct MxuMmaCore {: groups_per_row;const device uint8_t* zero_points_row_start =zero_points +- block_col * zero_point_stride_per_row;+ block_col * zero_point_stride_per_row ++ ((BITS == 4) ? (k_offset_groups / 2)+ : k_offset_groups);return BLoaderScaleZeroPoint(weights_block,scales_offset,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.hindex 0133c56a8..e892d0a2f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h@@ -80,28 +80,21 @@ inline void dequantize<bfloat, 8, 4>(bfloat bias,threadgroup bfloat* w_local) {- const device uint32_t* w_ptr = reinterpret_cast<const device uint32_t*>(w);- uint32_t packed = *w_ptr;-- bfloat4 v0, v1;-- v0.x = static_cast<bfloat>(packed & 0xF);- v0.y = static_cast<bfloat>((packed >> 4) & 0xF);- v0.z = static_cast<bfloat>((packed >> 8) & 0xF);- v0.w = static_cast<bfloat>((packed >> 12) & 0xF);-- v1.x = static_cast<bfloat>((packed >> 16) & 0xF);- v1.y = static_cast<bfloat>((packed >> 20) & 0xF);- v1.z = static_cast<bfloat>((packed >> 24) & 0xF);- v1.w = static_cast<bfloat>((packed >> 28) & 0xF);-- v0 = v0 * scale + bias;- v1 = v1 * scale + bias;-- threadgroup bfloat4* out_ptr =- reinterpret_cast<threadgroup bfloat4*>(w_local);- out_ptr[0] = v0;- out_ptr[1] = v1;+ const uint32_t packed = *reinterpret_cast<const device uint32_t*>(w);+ const bfloat4 lo =+ bfloat4(as_type<uchar4>(packed & 0x0f0f0f0fu)) * scale + bias;+ const bfloat4 hi = bfloat4(as_type<uchar4>(packed & 0xf0f0f0f0u)) *+ (scale * bfloat(0.0625f)) ++ bias;++ w_local[0] = lo.x;+ w_local[1] = hi.x;+ w_local[2] = lo.y;+ w_local[3] = hi.y;+ w_local[4] = lo.z;+ w_local[5] = hi.z;+ w_local[6] = lo.w;+ w_local[7] = hi.w;}} // namespace gemmdiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex e329b85a8..1ce69ed3b 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -239,7 +239,10 @@ struct SimdgroupMmaCore {threadgroup BT* b_shared,const thread ThreadContext& thread_context) {- const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);+ const uint partition = thread_context.threadgroup_position.z;+ const uint tile_row = thread_context.threadgroup_position.y;+ const uint2 tile =+ tile_id(uint2(thread_context.threadgroup_position.x, tile_row), params);const auto geometry =ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::compute(tile, params);@@ -247,13 +250,17 @@ struct SimdgroupMmaCore {return;}+ const uint k_offset =+ partition * params->aligned_inner_iterations * THREADGROUP_BLOCK_K;+threadgroup_barrier(mem_flags::mem_none);const size_t block_row = size_t(geometry.block_row_start);const size_t block_col = size_t(geometry.block_col_start);- a += block_row * params->leading_dimension_a;- d += block_row * params->leading_dimension_d + block_col;+ a += block_row * params->leading_dimension_a + k_offset;+ d += size_t(partition) * size_t(params->M) * size_t(params->N) ++ block_row * params->leading_dimension_d + block_col;thread ALoaderloader_a(a, params->leading_dimension_a, a_shared, thread_context);@@ -286,7 +293,9 @@ struct SimdgroupMmaCore {if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {const device BT* b_block_fp =b +- (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);+ (TRANSPOSE_B ? block_col * params->leading_dimension_b+ : block_col) ++ (TRANSPOSE_B ? k_offset : k_offset * params->leading_dimension_b);return BLoaderFp(b_block_fp,params->leading_dimension_b,@@ -300,13 +309,17 @@ struct SimdgroupMmaCore {const int weights_row_stride_bytes =k_elements * bytes_per_pack / pack_factor;const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;+ const int k_offset_groups = int(k_offset) / GROUP_SIZE;const device uint8_t* weights_block =reinterpret_cast<const device uint8_t*>(b) +- block_col * weights_row_stride_bytes;- const device BT* scales_offset = scales + block_col * groups_per_row;+ block_col * weights_row_stride_bytes ++ int(k_offset) * bytes_per_pack / pack_factor;+ const device BT* scales_offset =+ scales + block_col * groups_per_row + k_offset_groups;if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {- const device BT* biases_offset = biases + block_col * groups_per_row;+ const device BT* biases_offset =+ biases + block_col * groups_per_row + k_offset_groups;return BLoaderScaleBias(weights_block,scales_offset,@@ -322,7 +335,8 @@ struct SimdgroupMmaCore {const int zero_point_stride_per_row =(BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;const device uint8_t* zero_points_row_start =- zero_points + block_col * zero_point_stride_per_row;+ zero_points + block_col * zero_point_stride_per_row ++ ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);return BLoaderScaleZeroPoint(weights_block,scales_offset,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 70ac6cb24..62046f8b8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -96,12 +96,14 @@ KERNEL(Gemm)(const constant uzu::matmul::GemmParams* params,const constant uint& group_count_x,const constant uint& group_count_y,+ const constant uint& group_count_z,const GemmDTransform output_transform SPECIALIZE,const GemmAlignment alignment SPECIALIZE,threadgroup AT a_shared[GEMM_TGA_ELEMENTS],threadgroup BT b_shared[GEMM_TGB_ELEMENTS],const uint group_x GROUPS(group_count_x),const uint group_y GROUPS(group_count_y),+ const uint group_z GROUPS(group_count_z),const uint thread_x THREADS(METAL_SIMD_SIZE),const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),@@ -109,6 +111,7 @@ KERNEL(Gemm)() {(void)group_x;(void)group_y;+ (void)group_z;(void)thread_x;(void)thread_y;(void)thread_z;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metalnew file mode 100644index 000000000..cc579546d--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal@@ -0,0 +1,67 @@+#include "../../common/dsl.h"+#include "../../common/thread_context.h"+#include "../generated/gemm.h"++using namespace metal;+using namespace uzu::gemm;++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(GemmSplitKReduce)(+ const device T* partial_sums,+ device T* output,+ const device T* output_bias+ OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),+ const constant uint& element_count,+ const constant uint& partition_count,+ const constant uint& threadgroup_count,+ const constant uint& column_count,+ const constant float& output_scale+ OPTIONAL(output_transform.contains(GemmDTransform::SCALE)),+ const GemmDTransform output_transform SPECIALIZE,+ const uint threadgroup_index GROUPS(threadgroup_count),+ const uint thread_index_in_threadgroup THREADS(256),+ const ThreadContext thread_context+) {+ (void)threadgroup_index;+ (void)thread_index_in_threadgroup;++ const uint threads_per_threadgroup =+ thread_context.simdgroups_per_threadgroup * thread_context.simdgroup_size;+ const uint local_thread_index =+ thread_context.simdgroup_index * thread_context.simdgroup_size ++ thread_context.simd_lane_id;+ const uint vector_count = element_count / 4u;+ const uint vector_index =+ thread_context.threadgroup_position.x * threads_per_threadgroup ++ local_thread_index;+ if (vector_index >= vector_count) {+ return;+ }++ device vec<T, 4>* output_vectors =+ reinterpret_cast<device vec<T, 4>*>(output);+ const device vec<T, 4>* partial_sum_vectors =+ reinterpret_cast<const device vec<T, 4>*>(partial_sums);++ float4 accumulator = float4(0.0f);+ for (uint partition = 0u; partition < partition_count; ++partition) {+ accumulator +=+ float4(partial_sum_vectors[partition * vector_count + vector_index]);+ }++ if (output_transform.contains(GemmDTransform::SCALE)) {+ accumulator *= output_scale;+ }+ if (output_transform.contains(GemmDTransform::ACCUMULATE)) {+ accumulator += float4(output_vectors[vector_index]);+ }+ if (output_transform.contains(GemmDTransform::BIAS)) {+ const uint column = (vector_index * 4u) % column_count;+ accumulator += float4(+ *reinterpret_cast<const device vec<T, 4>*>(output_bias + column)+ );+ }++ output_vectors[vector_index] = vec<T, 4>(accumulator);+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 73737be31..d042df1c2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -7,7 +7,7 @@ use crate::{Allocation, AsBufferRangeRef, Backend, Buffer, Encoder,gpu_types::{GemmParams, HadamardTransformOrder,- gemm::{GemmAlignment, GemmDTransform, GemmTiling},+ gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},},kernel::{HadamardTransformKernel, Kernels, TensorAddBiasKernel,@@ -18,7 +18,7 @@ use crate::{Metal,context::MetalContext,error::MetalError,- kernel::{GemmMetalKernel, TensorAddBiasMetalKernel},+ kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel, TensorAddBiasMetalKernel},metal_extensions::DeviceExt,},},@@ -38,6 +38,7 @@ pub struct GemmKernel {kernels: HashMap<GemmSpecialization, GemmMetalKernel>,pub bias_add: TensorAddBiasMetalKernel,pub hadamard: <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel,+ split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,}impl GemmKernel {@@ -60,6 +61,7 @@ impl GemmKernel {kernels: HashMap::new(),bias_add,hadamard,+ split_k_reduce: HashMap::new(),};for specialization in GemmSpecialization::precompile_configs(weights_data_type) {kernel.get_or_create(context, specialization)?;@@ -105,6 +107,20 @@ impl GemmKernel {}}+ fn get_or_create_split_k_reduce(+ &mut self,+ context: &MetalContext,+ output_transform: GemmDTransform,+ ) -> Result<&GemmSplitKReduceMetalKernel, MetalError> {+ match self.split_k_reduce.entry(output_transform) {+ Entry::Occupied(entry) => Ok(entry.into_mut()),+ Entry::Vacant(entry) => {+ let kernel = GemmSplitKReduceMetalKernel::new(context, self.output_data_type, output_transform)?;+ Ok(entry.insert(kernel))+ },+ }+ }+pub fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(&mut self,arguments: MatmulArguments<'a, Metal, TB>,@@ -243,6 +259,38 @@ impl GemmKernel {let alignment =GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);+ if b_transpose && b_leading_dimension.is_none() && b_offset == 0 {+ let split_k = select_split_k(m, n, k, tiling, use_mxu, 0, true, false);+ if split_k > 1+ && split_k_output_supported(output_transform, n, self.weights_data_type, self.output_data_type)+ {+ return self.encode_split_k(+ a,+ a_offset,+ weights,+ b_offset,+ None,+ None,+ None,+ &mut *d,+ m,+ n,+ k,+ ab_scale,+ use_mxu,+ tiling,+ b_prologue,+ bits_per_b,+ group_size,+ split_k,+ output_transform,+ output_bias,+ rht_factors,+ encoder,+ );+ }+ }+let default_ldb = if b_transpose {k} else {@@ -287,6 +335,7 @@ impl GemmKernel {std::slice::from_ref(¶ms),group_count_x,group_count_y,+ 1,encoder,);},@@ -327,10 +376,41 @@ impl GemmKernel {};let alignment =GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);- let params = quant_params(m, n, k, tiling, ab_scale);+ let params = quant_params(m, n, k, tiling, use_mxu, group_size.unwrap_or(0), ab_scale);let group_count_x = n.div_ceil(tiling.block_n());let group_count_y = m.div_ceil(tiling.block_m());+ let zero_point_4bit = zero_points.is_some() && bits_per_b == Some(4);+ let split_k = select_split_k(m, n, k, tiling, use_mxu, group_size.unwrap_or(0), false, zero_point_4bit);+ if split_k > 1+ && split_k_output_supported(output_transform, n, self.weights_data_type, self.output_data_type)+ {+ return self.encode_split_k(+ a,+ a_offset,+ weights,+ b_offset,+ scales,+ biases,+ zero_points,+ &mut *d,+ m,+ n,+ k,+ ab_scale,+ use_mxu,+ tiling,+ b_prologue,+ bits_per_b,+ group_size,+ split_k,+ output_transform,+ output_bias,+ rht_factors,+ encoder,+ );+ }+let specialization = GemmSpecialization {weights_data_type: self.weights_data_type,tiling,@@ -356,6 +436,7 @@ impl GemmKernel {std::slice::from_ref(¶ms),group_count_x,group_count_y,+ 1,encoder,);},@@ -363,6 +444,114 @@ impl GemmKernel {Ok(())}++ #[allow(clippy::too_many_arguments)]+ fn encode_split_k<+ TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>,+ WB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>,+ >(+ &mut self,+ a: &TB,+ a_offset: usize,+ weights: &WB,+ b_offset: usize,+ scales: Option<&Allocation<Metal>>,+ biases: Option<&Allocation<Metal>>,+ zero_points: Option<&Allocation<Metal>>,+ d: &mut Allocation<Metal>,+ m: u32,+ n: u32,+ k: u32,+ ab_scale: f32,+ use_mxu: bool,+ tiling: GemmTiling,+ b_prologue: GemmBPrologueKind,+ bits_per_b: Option<u32>,+ group_size: Option<u32>,+ split_k: u32,+ output_transform: GemmDTransform,+ output_bias: Option<&Allocation<Metal>>,+ rht_factors: Option<&Allocation<Metal>>,+ encoder: &mut Encoder<Metal>,+ ) -> Result<(), MetalError> {+ let full_precision = matches!(b_prologue, GemmBPrologueKind::FullPrecision);+ let kp = k / split_k;+ let k_step = split_k_step(tiling, use_mxu, group_size.unwrap_or(0), full_precision).unwrap_or(1);+ let base_gx = n.div_ceil(tiling.block_n());+ let base_gy = m.div_ceil(tiling.block_m());+ let alignment =+ GemmAlignment::new(m.is_multiple_of(tiling.block_m()), n.is_multiple_of(tiling.block_n()), true);+ let part_spec = GemmSpecialization {+ weights_data_type: self.weights_data_type,+ tiling,+ use_mxu,+ output_transform: GemmDTransform::empty(),+ alignment,+ transpose_b: true,+ b_prologue,+ bits_per_b,+ group_size,+ };+ part_spec.validate()?;++ let elem = (m as usize) * (n as usize);+ let slice_bytes = elem * self.output_data_type.size_in_bytes();+ let mut temp = encoder.allocate_scratch(split_k as usize * slice_bytes)?;++ let params = GemmParams {+ M: m,+ N: n,+ K: k,+ leading_dimension_a: k,+ leading_dimension_b: k,+ leading_dimension_d: n,+ threadgroups_per_row: base_gx,+ threadgroups_per_column: base_gy,+ aligned_inner_iterations: kp / k_step,+ use_morton: false,+ ab_scale: 1.0,+ };+ let part_kernel = self.get_or_create(encoder.context(), part_spec)?;+ part_kernel.encode(+ (a, a_offset),+ (weights, b_offset),+ &mut temp,+ scales,+ biases,+ zero_points,+ None::<&Allocation<Metal>>,+ None::<&Allocation<Metal>>,+ std::slice::from_ref(¶ms),+ base_gx,+ base_gy,+ split_k,+ encoder,+ );++ debug_assert_eq!(elem % 4, 0, "split-K reduce requires M*N divisible by 4");+ let group_count = ((elem as u32) / 4).div_ceil(256);+ let reduce_transform =+ output_transform.intersection(GemmDTransform::SCALE | GemmDTransform::ACCUMULATE | GemmDTransform::BIAS);+ let bias_arg = if reduce_transform.contains(GemmDTransform::BIAS) {+ output_bias+ } else {+ None+ };+ let scale_arg = if reduce_transform.contains(GemmDTransform::SCALE) {+ Some(ab_scale)+ } else {+ None+ };+ let reduce = self.get_or_create_split_k_reduce(encoder.context(), reduce_transform)?;+ reduce.encode((&temp, 0usize), &mut *d, bias_arg, elem as u32, split_k, group_count, n, scale_arg, encoder);++ if output_transform.contains(GemmDTransform::RHT)+ && let Some(factors) = rht_factors+ {+ self.hadamard.encode(&mut *d, factors, n, m, encoder);+ }+ Ok(())+ }}fn quant_params(@@ -370,6 +559,8 @@ fn quant_params(n: u32,k: u32,tiling: GemmTiling,+ use_mxu: bool,+ group_size: u32,ab_scale: f32,) -> GemmParams {GemmParams {@@ -381,12 +572,75 @@ fn quant_params(leading_dimension_d: n,threadgroups_per_row: n.div_ceil(tiling.block_n()),threadgroups_per_column: m.div_ceil(tiling.block_m()),- aligned_inner_iterations: k / tiling.block_k(),+ aligned_inner_iterations: split_k_step(tiling, use_mxu, group_size, false).map_or(0, |step| k / step),use_morton: false,ab_scale,}}+fn split_k_step(+ tiling: GemmTiling,+ use_mxu: bool,+ group_size: u32,+ full_precision: bool,+) -> Option<u32> {+ let step = if use_mxu && !full_precision {+ group_size+ } else {+ tiling.block_k()+ };+ (step != 0).then_some(step)+}++fn split_k_output_supported(+ output_transform: GemmDTransform,+ n: u32,+ weights_data_type: DataType,+ output_data_type: DataType,+) -> bool {+ if !output_transform.contains(GemmDTransform::BIAS) {+ return true;+ }+ n.is_multiple_of(4) && weights_data_type == output_data_type+}++fn select_split_k(+ m: u32,+ n: u32,+ k: u32,+ tiling: GemmTiling,+ use_mxu: bool,+ group_size: u32,+ full_precision: bool,+ zero_point_4bit: bool,+) -> u32 {+ let base_tiles = n.div_ceil(tiling.block_n()) * m.div_ceil(tiling.block_m());+ if base_tiles == 0 {+ return 1;+ }+ if !((m as u64) * (n as u64)).is_multiple_of(4) {+ return 1;+ }+ let mut split_k = (512 / base_tiles).max(1);+ let step = match split_k_step(tiling, use_mxu, group_size, full_precision) {+ Some(s) => s,+ None => return 1,+ };+ let mut align = if use_mxu || full_precision {+ step+ } else {+ step.max(group_size)+ };+ if zero_point_4bit {+ align = align.max(2 * group_size);+ }+ split_k = split_k.min((k / align).max(1));+ while split_k > 1 && !k.is_multiple_of(split_k * align) {+ split_k -= 1;+ }+ split_k+}+fn select_simdgroup_tiling(m: u32,n: u32,@@ -440,7 +694,7 @@ fn select_quant_tiling(} else if m < 32 {GemmTiling::Tile8x32x32_Simdgroups1x1} else if m >= 64 && n <= 2048 {- GemmTiling::Tile64x32x32_Simdgroups2x2+ GemmTiling::Tile32x32x32_Simdgroups2x2} else if m >= 64 && n >= 6144 && n.is_multiple_of(64) {GemmTiling::Tile64x64x32_Simdgroups2x2} else {diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rsindex 87bca2467..4f91ca4ae 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs@@ -95,11 +95,7 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(#[case::gs32_4bit_mlx_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]#[case::gs64_4bit_mlx_prefill(64, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]#[case::gs128_4bit_mlx_prefill(64, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]-#[case::gs32_8bit_mlx_prefill(64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]-#[case::gs64_8bit_zp_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]-#[case::gs128_8bit_zp_prefill(64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]#[case::gs32_4bit_symmetric_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]-#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]#[case::gs32_4bit_mlx_decode(8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]#[case::gs16_4bit_zp_decode(8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]#[case::gs64_4bit_zp_decode(8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]@@ -115,6 +111,22 @@ fn parity_bf16(run_parity::<bf16>(m, k, n, gs, bits, method, 0.05, 0.4);}+#[rstest]+#[case::gs32_8bit_mlx_prefill(64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]+#[case::gs64_8bit_zp_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::gs128_8bit_zp_prefill(64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]+fn parity_bf16_8bit_splitk(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+ #[case] gs: u32,+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+) {+ run_parity::<bf16>(m, k, n, gs, bits, method, 0.05, 1.0);+}+fn run_parity_gemv<T: ArrayElement + Float + Debug + Display>(m: usize,k: usize,diff --git a/crates/download-manager/src/file_download_task_actor/download_task_actor.rs b/crates/download-manager/src/file_download_task_actor/download_task_actor.rsindex ef493a15a..d2c0f389a 100644--- a/crates/download-manager/src/file_download_task_actor/download_task_actor.rs+++ b/crates/download-manager/src/file_download_task_actor/download_task_actor.rs@@ -600,9 +600,20 @@ fn remove_resume_artifact(destination: &Path) {}async fn validate_completed_file(config: &DownloadConfig) -> Result<u64, String> {+ // After the backend reports completion the destination may not yet be fully+ // visible on disk: metadata can lag, and on the copy/move fallback path the+ // file briefly exists with fewer bytes than expected. Retry until it is+ // present with the expected size (or the budget runs out) so a transient+ // mismatch is not mistaken for a corrupt download. A genuinely truncated file+ // has a stable size and still fails once the retries are exhausted.let mut metadata = config.destination.metadata().ok();for _attempt in 0..10 {- if metadata.is_some() {+ let is_ready = match (&metadata, config.expected_bytes) {+ (Some(metadata), Some(expected_bytes)) => metadata.len() == expected_bytes,+ (Some(_), None) => true,+ (None, _) => false,+ };+ if is_ready {break;}tokio::time::sleep(std::time::Duration::from_millis(50)).await;diff --git a/crates/uzu/src/engine/downloader.rs b/crates/uzu/src/engine/downloader.rsindex 86a469890..feb6c2214 100644--- a/crates/uzu/src/engine/downloader.rs+++ b/crates/uzu/src/engine/downloader.rs@@ -118,7 +118,7 @@ impl Downloader {return Ok(DownloaderStream::empty(self.identifier.clone()));}let stream = self.storage.lock().await.subscribe();- Ok(DownloaderStream::new(self.identifier.clone(), stream))+ Ok(DownloaderStream::new(self.identifier.clone(), stream, self.storage.clone()))}}@@ -127,16 +127,19 @@ impl Downloader {pub struct DownloaderStream {identifier: String,stream: SharedAccess<Option<BroadcastStream<(String, DownloadState)>>>,+ storage: Option<SharedAccess<Storage>>,}impl DownloaderStream {pub(crate) fn new(identifier: String,stream: BroadcastStream<(String, DownloadState)>,+ storage: SharedAccess<Storage>,) -> Self {Self {identifier,stream: SharedAccess::new(Some(stream)),+ storage: Some(storage),}}@@ -144,7 +147,29 @@ impl DownloaderStream {Self {identifier,stream: SharedAccess::new(None),+ storage: None,+ }+ }++ /// A transient non-`Downloading` phase (for example a momentary `Paused`+ /// while a download is being resumed) is superseded by `Downloading` almost+ /// immediately, whereas a real pause/stop persists. Poll the canonical state+ /// briefly to tell them apart so a blip does not permanently end the stream.+ async fn download_has_settled(&self) -> bool {+ let Some(storage) = self.storage.as_ref() else {+ return true;+ };+ for _ in 0..10 {+ match storage.lock().await.state(&self.identifier).await {+ Some(state) if matches!(state.phase, DownloadPhase::Downloading {} | DownloadPhase::Downloaded {}) => {+ return false;+ },+ Some(_) => {},+ None => return true,+ }+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;}+ true}}@@ -162,7 +187,23 @@ impl DownloaderStream {bytes_total: state.total_bytes,bytes_downloaded: state.downloaded_bytes,};- if !Downloader::is_progress_streaming_phase(&state.phase) {+ let stream_ended = match state.phase {+ // Still actively downloading: keep streaming.+ DownloadPhase::Downloading {} => false,+ // Terminal: the download is over either way.+ DownloadPhase::Downloaded {}+ | DownloadPhase::Error {+ ..+ } => true,+ // Possibly a transient blip while (re)starting. End the+ // stream only if the model has genuinely stopped, so a+ // momentary state change does not cut off progress for a+ // download that is still on its way to completion.+ DownloadPhase::Paused {} | DownloadPhase::NotDownloaded {} | DownloadPhase::Locked {} => {+ self.download_has_settled().await+ },+ };+ if stream_ended {*stream_guard = None;}return Some(update);