Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add split-k gemm

eugene41b2e5434eb2
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index 55b8ad1cb..82202985f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -184,7 +184,11 @@ struct MxuMmaCore {
threadgroup BT* b_shared,
const thread ThreadContext& thread_context
) {
- const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);
+ const uint partition = thread_context.threadgroup_position.z;
+ const uint tile_y = thread_context.threadgroup_position.y;
+
+ const uint2 tile =
+ tile_id(uint2(thread_context.threadgroup_position.x, tile_y), params);
const auto geometry =
ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::
compute(tile, params);
@@ -195,9 +199,19 @@ struct MxuMmaCore {
const size_t block_row = size_t(geometry.block_row_start);
const size_t block_col = size_t(geometry.block_col_start);
- const device AT* a_block = a + block_row * params->leading_dimension_a;
+ const uint k_offset_per_block =
+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision)
+ ? uint(THREADGROUP_BLOCK_K_FP)
+ : uint(QUANT_BK);
+ const uint k_offset =
+ partition * params->aligned_inner_iterations * k_offset_per_block;
+
+ const device AT* a_block =
+ a + block_row * params->leading_dimension_a + k_offset;
const device BT* b_block_fp =
- b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);
+ b +
+ (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col) +
+ (TRANSPOSE_B ? k_offset : k_offset * uint(params->leading_dimension_b));
const ushort tile_row_offset =
SIMDGROUP_BLOCK_M *
@@ -207,7 +221,8 @@ struct MxuMmaCore {
(thread_context.simdgroup_index % SIMDGROUPS_PER_COLUMN);
device DT* d_simdgroup =
- d + block_row * params->leading_dimension_d + block_col +
+ d + size_t(partition) * size_t(params->M) * size_t(params->N) +
+ block_row * params->leading_dimension_d + block_col +
tile_row_offset * params->leading_dimension_d + tile_col_offset;
const short simdgroup_limit_m =
@@ -271,7 +286,7 @@ struct MxuMmaCore {
B_PROLOGUE == GemmBPrologueKind::FullPrecision
) {
const int aligned_k_iterations_fp =
- int(params->K) / int(THREADGROUP_BLOCK_K_FP);
+ int(params->aligned_inner_iterations);
return uzu::matmul::gemm_loop<
AT,
BT,
@@ -297,7 +312,7 @@ struct MxuMmaCore {
);
} else {
const int aligned_k_iterations_q =
- int(params->K) / int(QUANT_BK);
+ int(params->aligned_inner_iterations);
constexpr int pack_factor = get_pack_factor<BITS, 8>();
constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
const int k_elements = int(params->K);
@@ -305,18 +320,21 @@ struct MxuMmaCore {
k_elements * bytes_per_pack / pack_factor;
const int groups_per_row =
(k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
+ const int k_offset_groups = int(k_offset) / GROUP_SIZE;
const device uint8_t* weights_block =
reinterpret_cast<const device uint8_t*>(b) +
- block_col * weights_row_stride_bytes;
+ block_col * weights_row_stride_bytes +
+ int(k_offset) * bytes_per_pack / pack_factor;
const device BT* scales_offset =
- scales + block_col * groups_per_row;
+ scales + block_col * groups_per_row + k_offset_groups;
auto loader_b = [&]() {
if constexpr (
B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant
) {
const device BT* biases_offset =
- biases + block_col * groups_per_row;
+ biases + block_col * groups_per_row +
+ k_offset_groups;
return BLoaderScaleBias(
weights_block,
scales_offset,
@@ -335,7 +353,9 @@ struct MxuMmaCore {
: groups_per_row;
const device uint8_t* zero_points_row_start =
zero_points +
- block_col * zero_point_stride_per_row;
+ block_col * zero_point_stride_per_row +
+ ((BITS == 4) ? (k_offset_groups / 2)
+ : k_offset_groups);
return BLoaderScaleZeroPoint(
weights_block,
scales_offset,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h
index 0133c56a8..e892d0a2f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/quant_unpack.h
@@ -80,28 +80,21 @@ inline void dequantize<bfloat, 8, 4>(
bfloat bias,
threadgroup bfloat* w_local
) {
- const device uint32_t* w_ptr = reinterpret_cast<const device uint32_t*>(w);
- uint32_t packed = *w_ptr;
-
- bfloat4 v0, v1;
-
- v0.x = static_cast<bfloat>(packed & 0xF);
- v0.y = static_cast<bfloat>((packed >> 4) & 0xF);
- v0.z = static_cast<bfloat>((packed >> 8) & 0xF);
- v0.w = static_cast<bfloat>((packed >> 12) & 0xF);
-
- v1.x = static_cast<bfloat>((packed >> 16) & 0xF);
- v1.y = static_cast<bfloat>((packed >> 20) & 0xF);
- v1.z = static_cast<bfloat>((packed >> 24) & 0xF);
- v1.w = static_cast<bfloat>((packed >> 28) & 0xF);
-
- v0 = v0 * scale + bias;
- v1 = v1 * scale + bias;
-
- threadgroup bfloat4* out_ptr =
- reinterpret_cast<threadgroup bfloat4*>(w_local);
- out_ptr[0] = v0;
- out_ptr[1] = v1;
+ const uint32_t packed = *reinterpret_cast<const device uint32_t*>(w);
+ const bfloat4 lo =
+ bfloat4(as_type<uchar4>(packed & 0x0f0f0f0fu)) * scale + bias;
+ const bfloat4 hi = bfloat4(as_type<uchar4>(packed & 0xf0f0f0f0u)) *
+ (scale * bfloat(0.0625f)) +
+ bias;
+
+ w_local[0] = lo.x;
+ w_local[1] = hi.x;
+ w_local[2] = lo.y;
+ w_local[3] = hi.y;
+ w_local[4] = lo.z;
+ w_local[5] = hi.z;
+ w_local[6] = lo.w;
+ w_local[7] = hi.w;
}
} // namespace gemm
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index e329b85a8..1ce69ed3b 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -239,7 +239,10 @@ struct SimdgroupMmaCore {
threadgroup BT* b_shared,
const thread ThreadContext& thread_context
) {
- const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);
+ const uint partition = thread_context.threadgroup_position.z;
+ const uint tile_row = thread_context.threadgroup_position.y;
+ const uint2 tile =
+ tile_id(uint2(thread_context.threadgroup_position.x, tile_row), params);
const auto geometry =
ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::
compute(tile, params);
@@ -247,13 +250,17 @@ struct SimdgroupMmaCore {
return;
}
+ const uint k_offset =
+ partition * params->aligned_inner_iterations * THREADGROUP_BLOCK_K;
+
threadgroup_barrier(mem_flags::mem_none);
const size_t block_row = size_t(geometry.block_row_start);
const size_t block_col = size_t(geometry.block_col_start);
- a += block_row * params->leading_dimension_a;
- d += block_row * params->leading_dimension_d + block_col;
+ a += block_row * params->leading_dimension_a + k_offset;
+ d += size_t(partition) * size_t(params->M) * size_t(params->N) +
+ block_row * params->leading_dimension_d + block_col;
thread ALoader
loader_a(a, params->leading_dimension_a, a_shared, thread_context);
@@ -286,7 +293,9 @@ struct SimdgroupMmaCore {
if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
const device BT* b_block_fp =
b +
- (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);
+ (TRANSPOSE_B ? block_col * params->leading_dimension_b
+ : block_col) +
+ (TRANSPOSE_B ? k_offset : k_offset * params->leading_dimension_b);
return BLoaderFp(
b_block_fp,
params->leading_dimension_b,
@@ -300,13 +309,17 @@ struct SimdgroupMmaCore {
const int weights_row_stride_bytes =
k_elements * bytes_per_pack / pack_factor;
const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
+ const int k_offset_groups = int(k_offset) / GROUP_SIZE;
const device uint8_t* weights_block =
reinterpret_cast<const device uint8_t*>(b) +
- block_col * weights_row_stride_bytes;
- const device BT* scales_offset = scales + block_col * groups_per_row;
+ block_col * weights_row_stride_bytes +
+ int(k_offset) * bytes_per_pack / pack_factor;
+ const device BT* scales_offset =
+ scales + block_col * groups_per_row + k_offset_groups;
if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
- const device BT* biases_offset = biases + block_col * groups_per_row;
+ const device BT* biases_offset =
+ biases + block_col * groups_per_row + k_offset_groups;
return BLoaderScaleBias(
weights_block,
scales_offset,
@@ -322,7 +335,8 @@ struct SimdgroupMmaCore {
const int zero_point_stride_per_row =
(BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
const device uint8_t* zero_points_row_start =
- zero_points + block_col * zero_point_stride_per_row;
+ zero_points + block_col * zero_point_stride_per_row +
+ ((BITS == 4) ? (k_offset_groups / 2) : k_offset_groups);
return BLoaderScaleZeroPoint(
weights_block,
scales_offset,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 70ac6cb24..62046f8b8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -96,12 +96,14 @@ KERNEL(Gemm)(
const constant uzu::matmul::GemmParams* params,
const constant uint& group_count_x,
const constant uint& group_count_y,
+ const constant uint& group_count_z,
const GemmDTransform output_transform SPECIALIZE,
const GemmAlignment alignment SPECIALIZE,
threadgroup AT a_shared[GEMM_TGA_ELEMENTS],
threadgroup BT b_shared[GEMM_TGB_ELEMENTS],
const uint group_x GROUPS(group_count_x),
const uint group_y GROUPS(group_count_y),
+ const uint group_z GROUPS(group_count_z),
const uint thread_x THREADS(METAL_SIMD_SIZE),
const uint thread_y THREADS(gemm_tiling_simdgroups_per_column(GEMM_TILING)),
const uint thread_z THREADS(gemm_tiling_simdgroups_per_row(GEMM_TILING)),
@@ -109,6 +111,7 @@ KERNEL(Gemm)(
) {
(void)group_x;
(void)group_y;
+ (void)group_z;
(void)thread_x;
(void)thread_y;
(void)thread_z;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
new file mode 100644
index 000000000..cc579546d
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm_split_k_reduce.metal
@@ -0,0 +1,67 @@
+#include "../../common/dsl.h"
+#include "../../common/thread_context.h"
+#include "../generated/gemm.h"
+
+using namespace metal;
+using namespace uzu::gemm;
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(GemmSplitKReduce)(
+ const device T* partial_sums,
+ device T* output,
+ const device T* output_bias
+ OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),
+ const constant uint& element_count,
+ const constant uint& partition_count,
+ const constant uint& threadgroup_count,
+ const constant uint& column_count,
+ const constant float& output_scale
+ OPTIONAL(output_transform.contains(GemmDTransform::SCALE)),
+ const GemmDTransform output_transform SPECIALIZE,
+ const uint threadgroup_index GROUPS(threadgroup_count),
+ const uint thread_index_in_threadgroup THREADS(256),
+ const ThreadContext thread_context
+) {
+ (void)threadgroup_index;
+ (void)thread_index_in_threadgroup;
+
+ const uint threads_per_threadgroup =
+ thread_context.simdgroups_per_threadgroup * thread_context.simdgroup_size;
+ const uint local_thread_index =
+ thread_context.simdgroup_index * thread_context.simdgroup_size +
+ thread_context.simd_lane_id;
+ const uint vector_count = element_count / 4u;
+ const uint vector_index =
+ thread_context.threadgroup_position.x * threads_per_threadgroup +
+ local_thread_index;
+ if (vector_index >= vector_count) {
+ return;
+ }
+
+ device vec<T, 4>* output_vectors =
+ reinterpret_cast<device vec<T, 4>*>(output);
+ const device vec<T, 4>* partial_sum_vectors =
+ reinterpret_cast<const device vec<T, 4>*>(partial_sums);
+
+ float4 accumulator = float4(0.0f);
+ for (uint partition = 0u; partition < partition_count; ++partition) {
+ accumulator +=
+ float4(partial_sum_vectors[partition * vector_count + vector_index]);
+ }
+
+ if (output_transform.contains(GemmDTransform::SCALE)) {
+ accumulator *= output_scale;
+ }
+ if (output_transform.contains(GemmDTransform::ACCUMULATE)) {
+ accumulator += float4(output_vectors[vector_index]);
+ }
+ if (output_transform.contains(GemmDTransform::BIAS)) {
+ const uint column = (vector_index * 4u) % column_count;
+ accumulator += float4(
+ *reinterpret_cast<const device vec<T, 4>*>(output_bias + column)
+ );
+ }
+
+ output_vectors[vector_index] = vec<T, 4>(accumulator);
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 73737be31..d042df1c2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -7,7 +7,7 @@ use crate::{
Allocation, AsBufferRangeRef, Backend, Buffer, Encoder,
gpu_types::{
GemmParams, HadamardTransformOrder,
- gemm::{GemmAlignment, GemmDTransform, GemmTiling},
+ gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},
},
kernel::{
HadamardTransformKernel, Kernels, TensorAddBiasKernel,
@@ -18,7 +18,7 @@ use crate::{
Metal,
context::MetalContext,
error::MetalError,
- kernel::{GemmMetalKernel, TensorAddBiasMetalKernel},
+ kernel::{GemmMetalKernel, GemmSplitKReduceMetalKernel, TensorAddBiasMetalKernel},
metal_extensions::DeviceExt,
},
},
@@ -38,6 +38,7 @@ pub struct GemmKernel {
kernels: HashMap<GemmSpecialization, GemmMetalKernel>,
pub bias_add: TensorAddBiasMetalKernel,
pub hadamard: <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel,
+ split_k_reduce: HashMap<GemmDTransform, GemmSplitKReduceMetalKernel>,
}
impl GemmKernel {
@@ -60,6 +61,7 @@ impl GemmKernel {
kernels: HashMap::new(),
bias_add,
hadamard,
+ split_k_reduce: HashMap::new(),
};
for specialization in GemmSpecialization::precompile_configs(weights_data_type) {
kernel.get_or_create(context, specialization)?;
@@ -105,6 +107,20 @@ impl GemmKernel {
}
}
+ fn get_or_create_split_k_reduce(
+ &mut self,
+ context: &MetalContext,
+ output_transform: GemmDTransform,
+ ) -> Result<&GemmSplitKReduceMetalKernel, MetalError> {
+ match self.split_k_reduce.entry(output_transform) {
+ Entry::Occupied(entry) => Ok(entry.into_mut()),
+ Entry::Vacant(entry) => {
+ let kernel = GemmSplitKReduceMetalKernel::new(context, self.output_data_type, output_transform)?;
+ Ok(entry.insert(kernel))
+ },
+ }
+ }
+
pub fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
&mut self,
arguments: MatmulArguments<'a, Metal, TB>,
@@ -243,6 +259,38 @@ impl GemmKernel {
let alignment =
GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
+ if b_transpose && b_leading_dimension.is_none() && b_offset == 0 {
+ let split_k = select_split_k(m, n, k, tiling, use_mxu, 0, true, false);
+ if split_k > 1
+ && split_k_output_supported(output_transform, n, self.weights_data_type, self.output_data_type)
+ {
+ return self.encode_split_k(
+ a,
+ a_offset,
+ weights,
+ b_offset,
+ None,
+ None,
+ None,
+ &mut *d,
+ m,
+ n,
+ k,
+ ab_scale,
+ use_mxu,
+ tiling,
+ b_prologue,
+ bits_per_b,
+ group_size,
+ split_k,
+ output_transform,
+ output_bias,
+ rht_factors,
+ encoder,
+ );
+ }
+ }
+
let default_ldb = if b_transpose {
k
} else {
@@ -287,6 +335,7 @@ impl GemmKernel {
std::slice::from_ref(&params),
group_count_x,
group_count_y,
+ 1,
encoder,
);
},
@@ -327,10 +376,41 @@ impl GemmKernel {
};
let alignment =
GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
- let params = quant_params(m, n, k, tiling, ab_scale);
+ let params = quant_params(m, n, k, tiling, use_mxu, group_size.unwrap_or(0), ab_scale);
let group_count_x = n.div_ceil(tiling.block_n());
let group_count_y = m.div_ceil(tiling.block_m());
+ let zero_point_4bit = zero_points.is_some() && bits_per_b == Some(4);
+ let split_k = select_split_k(m, n, k, tiling, use_mxu, group_size.unwrap_or(0), false, zero_point_4bit);
+ if split_k > 1
+ && split_k_output_supported(output_transform, n, self.weights_data_type, self.output_data_type)
+ {
+ return self.encode_split_k(
+ a,
+ a_offset,
+ weights,
+ b_offset,
+ scales,
+ biases,
+ zero_points,
+ &mut *d,
+ m,
+ n,
+ k,
+ ab_scale,
+ use_mxu,
+ tiling,
+ b_prologue,
+ bits_per_b,
+ group_size,
+ split_k,
+ output_transform,
+ output_bias,
+ rht_factors,
+ encoder,
+ );
+ }
+
let specialization = GemmSpecialization {
weights_data_type: self.weights_data_type,
tiling,
@@ -356,6 +436,7 @@ impl GemmKernel {
std::slice::from_ref(&params),
group_count_x,
group_count_y,
+ 1,
encoder,
);
},
@@ -363,6 +444,114 @@ impl GemmKernel {
Ok(())
}
+
+ #[allow(clippy::too_many_arguments)]
+ fn encode_split_k<
+ TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>,
+ WB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>,
+ >(
+ &mut self,
+ a: &TB,
+ a_offset: usize,
+ weights: &WB,
+ b_offset: usize,
+ scales: Option<&Allocation<Metal>>,
+ biases: Option<&Allocation<Metal>>,
+ zero_points: Option<&Allocation<Metal>>,
+ d: &mut Allocation<Metal>,
+ m: u32,
+ n: u32,
+ k: u32,
+ ab_scale: f32,
+ use_mxu: bool,
+ tiling: GemmTiling,
+ b_prologue: GemmBPrologueKind,
+ bits_per_b: Option<u32>,
+ group_size: Option<u32>,
+ split_k: u32,
+ output_transform: GemmDTransform,
+ output_bias: Option<&Allocation<Metal>>,
+ rht_factors: Option<&Allocation<Metal>>,
+ encoder: &mut Encoder<Metal>,
+ ) -> Result<(), MetalError> {
+ let full_precision = matches!(b_prologue, GemmBPrologueKind::FullPrecision);
+ let kp = k / split_k;
+ let k_step = split_k_step(tiling, use_mxu, group_size.unwrap_or(0), full_precision).unwrap_or(1);
+ let base_gx = n.div_ceil(tiling.block_n());
+ let base_gy = m.div_ceil(tiling.block_m());
+ let alignment =
+ GemmAlignment::new(m.is_multiple_of(tiling.block_m()), n.is_multiple_of(tiling.block_n()), true);
+ let part_spec = GemmSpecialization {
+ weights_data_type: self.weights_data_type,
+ tiling,
+ use_mxu,
+ output_transform: GemmDTransform::empty(),
+ alignment,
+ transpose_b: true,
+ b_prologue,
+ bits_per_b,
+ group_size,
+ };
+ part_spec.validate()?;
+
+ let elem = (m as usize) * (n as usize);
+ let slice_bytes = elem * self.output_data_type.size_in_bytes();
+ let mut temp = encoder.allocate_scratch(split_k as usize * slice_bytes)?;
+
+ let params = GemmParams {
+ M: m,
+ N: n,
+ K: k,
+ leading_dimension_a: k,
+ leading_dimension_b: k,
+ leading_dimension_d: n,
+ threadgroups_per_row: base_gx,
+ threadgroups_per_column: base_gy,
+ aligned_inner_iterations: kp / k_step,
+ use_morton: false,
+ ab_scale: 1.0,
+ };
+ let part_kernel = self.get_or_create(encoder.context(), part_spec)?;
+ part_kernel.encode(
+ (a, a_offset),
+ (weights, b_offset),
+ &mut temp,
+ scales,
+ biases,
+ zero_points,
+ None::<&Allocation<Metal>>,
+ None::<&Allocation<Metal>>,
+ std::slice::from_ref(&params),
+ base_gx,
+ base_gy,
+ split_k,
+ encoder,
+ );
+
+ debug_assert_eq!(elem % 4, 0, "split-K reduce requires M*N divisible by 4");
+ let group_count = ((elem as u32) / 4).div_ceil(256);
+ let reduce_transform =
+ output_transform.intersection(GemmDTransform::SCALE | GemmDTransform::ACCUMULATE | GemmDTransform::BIAS);
+ let bias_arg = if reduce_transform.contains(GemmDTransform::BIAS) {
+ output_bias
+ } else {
+ None
+ };
+ let scale_arg = if reduce_transform.contains(GemmDTransform::SCALE) {
+ Some(ab_scale)
+ } else {
+ None
+ };
+ let reduce = self.get_or_create_split_k_reduce(encoder.context(), reduce_transform)?;
+ reduce.encode((&temp, 0usize), &mut *d, bias_arg, elem as u32, split_k, group_count, n, scale_arg, encoder);
+
+ if output_transform.contains(GemmDTransform::RHT)
+ && let Some(factors) = rht_factors
+ {
+ self.hadamard.encode(&mut *d, factors, n, m, encoder);
+ }
+ Ok(())
+ }
}
fn quant_params(
@@ -370,6 +559,8 @@ fn quant_params(
n: u32,
k: u32,
tiling: GemmTiling,
+ use_mxu: bool,
+ group_size: u32,
ab_scale: f32,
) -> GemmParams {
GemmParams {
@@ -381,12 +572,75 @@ fn quant_params(
leading_dimension_d: n,
threadgroups_per_row: n.div_ceil(tiling.block_n()),
threadgroups_per_column: m.div_ceil(tiling.block_m()),
- aligned_inner_iterations: k / tiling.block_k(),
+ aligned_inner_iterations: split_k_step(tiling, use_mxu, group_size, false).map_or(0, |step| k / step),
use_morton: false,
ab_scale,
}
}
+fn split_k_step(
+ tiling: GemmTiling,
+ use_mxu: bool,
+ group_size: u32,
+ full_precision: bool,
+) -> Option<u32> {
+ let step = if use_mxu && !full_precision {
+ group_size
+ } else {
+ tiling.block_k()
+ };
+ (step != 0).then_some(step)
+}
+
+fn split_k_output_supported(
+ output_transform: GemmDTransform,
+ n: u32,
+ weights_data_type: DataType,
+ output_data_type: DataType,
+) -> bool {
+ if !output_transform.contains(GemmDTransform::BIAS) {
+ return true;
+ }
+ n.is_multiple_of(4) && weights_data_type == output_data_type
+}
+
+fn select_split_k(
+ m: u32,
+ n: u32,
+ k: u32,
+ tiling: GemmTiling,
+ use_mxu: bool,
+ group_size: u32,
+ full_precision: bool,
+ zero_point_4bit: bool,
+) -> u32 {
+ let base_tiles = n.div_ceil(tiling.block_n()) * m.div_ceil(tiling.block_m());
+ if base_tiles == 0 {
+ return 1;
+ }
+ if !((m as u64) * (n as u64)).is_multiple_of(4) {
+ return 1;
+ }
+ let mut split_k = (512 / base_tiles).max(1);
+ let step = match split_k_step(tiling, use_mxu, group_size, full_precision) {
+ Some(s) => s,
+ None => return 1,
+ };
+ let mut align = if use_mxu || full_precision {
+ step
+ } else {
+ step.max(group_size)
+ };
+ if zero_point_4bit {
+ align = align.max(2 * group_size);
+ }
+ split_k = split_k.min((k / align).max(1));
+ while split_k > 1 && !k.is_multiple_of(split_k * align) {
+ split_k -= 1;
+ }
+ split_k
+}
+
fn select_simdgroup_tiling(
m: u32,
n: u32,
@@ -440,7 +694,7 @@ fn select_quant_tiling(
} else if m < 32 {
GemmTiling::Tile8x32x32_Simdgroups1x1
} else if m >= 64 && n <= 2048 {
- GemmTiling::Tile64x32x32_Simdgroups2x2
+ GemmTiling::Tile32x32x32_Simdgroups2x2
} else if m >= 64 && n >= 6144 && n.is_multiple_of(64) {
GemmTiling::Tile64x64x32_Simdgroups2x2
} else {
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
index 87bca2467..4f91ca4ae 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
@@ -95,11 +95,7 @@ fn run_parity<T: ArrayElement + Float + Debug + Display>(
#[case::gs32_4bit_mlx_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
#[case::gs64_4bit_mlx_prefill(64, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
#[case::gs128_4bit_mlx_prefill(64, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]
-#[case::gs32_8bit_mlx_prefill(64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]
-#[case::gs64_8bit_zp_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
-#[case::gs128_8bit_zp_prefill(64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
#[case::gs32_4bit_symmetric_prefill(64, 256, 64, 32, 4, QuantizationMethod::ScaleSymmetric)]
-#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
#[case::gs32_4bit_mlx_decode(8, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
#[case::gs16_4bit_zp_decode(8, 256, 64, 16, 4, QuantizationMethod::ScaleZeroPoint)]
#[case::gs64_4bit_zp_decode(8, 256, 64, 64, 4, QuantizationMethod::ScaleZeroPoint)]
@@ -115,6 +111,22 @@ fn parity_bf16(
run_parity::<bf16>(m, k, n, gs, bits, method, 0.05, 0.4);
}
+#[rstest]
+#[case::gs32_8bit_mlx_prefill(64, 256, 64, 32, 8, QuantizationMethod::ScaleBias)]
+#[case::gs64_8bit_zp_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs128_8bit_zp_prefill(64, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs64_8bit_symmetric_prefill(64, 256, 64, 64, 8, QuantizationMethod::ScaleSymmetric)]
+fn parity_bf16_8bit_splitk(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+ #[case] gs: u32,
+ #[case] bits: u32,
+ #[case] method: QuantizationMethod,
+) {
+ run_parity::<bf16>(m, k, n, gs, bits, method, 0.05, 1.0);
+}
+
fn run_parity_gemv<T: ArrayElement + Float + Debug + Display>(
m: usize,
k: usize,
diff --git a/crates/download-manager/src/file_download_task_actor/download_task_actor.rs b/crates/download-manager/src/file_download_task_actor/download_task_actor.rs
index ef493a15a..d2c0f389a 100644
--- a/crates/download-manager/src/file_download_task_actor/download_task_actor.rs
+++ b/crates/download-manager/src/file_download_task_actor/download_task_actor.rs
@@ -600,9 +600,20 @@ fn remove_resume_artifact(destination: &Path) {
}
async fn validate_completed_file(config: &DownloadConfig) -> Result<u64, String> {
+ // After the backend reports completion the destination may not yet be fully
+ // visible on disk: metadata can lag, and on the copy/move fallback path the
+ // file briefly exists with fewer bytes than expected. Retry until it is
+ // present with the expected size (or the budget runs out) so a transient
+ // mismatch is not mistaken for a corrupt download. A genuinely truncated file
+ // has a stable size and still fails once the retries are exhausted.
let mut metadata = config.destination.metadata().ok();
for _attempt in 0..10 {
- if metadata.is_some() {
+ let is_ready = match (&metadata, config.expected_bytes) {
+ (Some(metadata), Some(expected_bytes)) => metadata.len() == expected_bytes,
+ (Some(_), None) => true,
+ (None, _) => false,
+ };
+ if is_ready {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
diff --git a/crates/uzu/src/engine/downloader.rs b/crates/uzu/src/engine/downloader.rs
index 86a469890..feb6c2214 100644
--- a/crates/uzu/src/engine/downloader.rs
+++ b/crates/uzu/src/engine/downloader.rs
@@ -118,7 +118,7 @@ impl Downloader {
return Ok(DownloaderStream::empty(self.identifier.clone()));
}
let stream = self.storage.lock().await.subscribe();
- Ok(DownloaderStream::new(self.identifier.clone(), stream))
+ Ok(DownloaderStream::new(self.identifier.clone(), stream, self.storage.clone()))
}
}
@@ -127,16 +127,19 @@ impl Downloader {
pub struct DownloaderStream {
identifier: String,
stream: SharedAccess<Option<BroadcastStream<(String, DownloadState)>>>,
+ storage: Option<SharedAccess<Storage>>,
}
impl DownloaderStream {
pub(crate) fn new(
identifier: String,
stream: BroadcastStream<(String, DownloadState)>,
+ storage: SharedAccess<Storage>,
) -> Self {
Self {
identifier,
stream: SharedAccess::new(Some(stream)),
+ storage: Some(storage),
}
}
@@ -144,7 +147,29 @@ impl DownloaderStream {
Self {
identifier,
stream: SharedAccess::new(None),
+ storage: None,
+ }
+ }
+
+ /// A transient non-`Downloading` phase (for example a momentary `Paused`
+ /// while a download is being resumed) is superseded by `Downloading` almost
+ /// immediately, whereas a real pause/stop persists. Poll the canonical state
+ /// briefly to tell them apart so a blip does not permanently end the stream.
+ async fn download_has_settled(&self) -> bool {
+ let Some(storage) = self.storage.as_ref() else {
+ return true;
+ };
+ for _ in 0..10 {
+ match storage.lock().await.state(&self.identifier).await {
+ Some(state) if matches!(state.phase, DownloadPhase::Downloading {} | DownloadPhase::Downloaded {}) => {
+ return false;
+ },
+ Some(_) => {},
+ None => return true,
+ }
+ tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
+ true
}
}
@@ -162,7 +187,23 @@ impl DownloaderStream {
bytes_total: state.total_bytes,
bytes_downloaded: state.downloaded_bytes,
};
- if !Downloader::is_progress_streaming_phase(&state.phase) {
+ let stream_ended = match state.phase {
+ // Still actively downloading: keep streaming.
+ DownloadPhase::Downloading {} => false,
+ // Terminal: the download is over either way.
+ DownloadPhase::Downloaded {}
+ | DownloadPhase::Error {
+ ..
+ } => true,
+ // Possibly a transient blip while (re)starting. End the
+ // stream only if the model has genuinely stopped, so a
+ // momentary state change does not cut off progress for a
+ // download that is still on its way to completion.
+ DownloadPhase::Paused {} | DownloadPhase::NotDownloaded {} | DownloadPhase::Locked {} => {
+ self.download_has_settled().await
+ },
+ };
+ if stream_ended {
*stream_guard = None;
}
return Some(update);