Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dispatch descriptor

Eugene Bokhan230c85889b43
diff --git a/crates/matmul-benchmark/src/main.rs b/crates/matmul-benchmark/src/main.rs
index 41c86c578..716629e68 100644
--- a/crates/matmul-benchmark/src/main.rs
+++ b/crates/matmul-benchmark/src/main.rs
@@ -533,6 +533,7 @@ fn benchmark_uzu(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.expect("encode");
enc.end_encoding();
@@ -564,6 +565,7 @@ fn benchmark_uzu(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.expect("encode");
enc.end_encoding();
@@ -804,6 +806,7 @@ fn run_accuracy_test_f16(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.unwrap();
enc.end_encoding();
@@ -869,6 +872,7 @@ fn run_accuracy_test_f32(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.unwrap();
enc.end_encoding();
@@ -945,6 +949,7 @@ fn run_accuracy_test_bf16(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.unwrap();
enc.end_encoding();
diff --git a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
index 12833c077..835a07c4d 100644
--- a/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/embedding/full_precision_readout.rs
@@ -25,7 +25,7 @@ pub struct FullPrecisionEmbeddingReadout {
impl FullPrecisionEmbeddingReadout {
pub fn new(
- mtl_context: &MTLContext,
+ _mtl_context: &MTLContext,
data_type: DataType,
vocab_size: usize,
model_dim: usize,
@@ -69,10 +69,8 @@ impl FullPrecisionEmbeddingReadout {
let weights_buffer = unsafe { weights.mtl_buffer().to_owned() };
- // Weights are [vocab_size, model_dim], we compute input @ weights^T
- // MatmulKernel with transpose_b=true handles this
- let kernel = MatmulKernel::new(mtl_context, data_type, false, true)
- .map_err(EmbeddingError::MetalError)?;
+ let kernel =
+ MatmulKernel::new(data_type).map_err(EmbeddingError::MetalError)?;
Ok(Self {
kernel: RefCell::new(kernel),
@@ -130,6 +128,7 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {
b: &self.weights_buffer,
c: None,
d: output_buffer,
+ bias: None,
batch: batch_size as i32,
input_dim: self.model_dim as i32,
output_dim: self.vocab_size as i32,
@@ -139,6 +138,8 @@ impl EncodableBlock for FullPrecisionEmbeddingReadout {
batch_count: 1,
alpha: 1.0,
beta: 0.0,
+ transpose_a: false,
+ transpose_b: true,
};
self.kernel
diff --git a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
index 5d77b2ba3..857b2b592 100644
--- a/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/linear/full_precision.rs
@@ -16,7 +16,7 @@ use crate::{
pub struct FullPrecisionLinear {
kernel: RefCell<MatmulKernel>,
- biases_buffer: Option<MTLBuffer>,
+ bias_buffer: Option<MTLBuffer>,
weights_buffer: MTLBuffer,
input_dim: usize,
output_dim: usize,
@@ -26,7 +26,7 @@ pub struct FullPrecisionLinear {
impl FullPrecisionLinear {
pub fn new(
- mtl_context: &MTLContext,
+ _mtl_context: &MTLContext,
precision: DataType,
input_dim: usize,
output_dim: usize,
@@ -65,7 +65,7 @@ impl FullPrecisionLinear {
let weights_buffer: MTLBuffer =
unsafe { weights.mtl_buffer() }.to_owned();
- let biases_buffer = match parameter_tree.leaf("biases") {
+ let bias_buffer = match parameter_tree.leaf("biases") {
Ok(mut biases) => {
if biases.shape() != [output_dim] {
return Err(MTLError::Generic(format!(
@@ -81,18 +81,18 @@ impl FullPrecisionLinear {
precision
)));
}
- let biases_buffer: MTLBuffer =
+ let bias_buffer: MTLBuffer =
unsafe { biases.mtl_buffer() }.to_owned();
- Some(biases_buffer)
+ Some(bias_buffer)
},
Err(_) => None,
};
- let kernel = MatmulKernel::new(mtl_context, precision, false, true)?;
+ let kernel = MatmulKernel::new(precision)?;
Ok(Self {
kernel: RefCell::new(kernel),
- biases_buffer,
+ bias_buffer,
weights_buffer,
input_dim,
output_dim,
@@ -126,6 +126,7 @@ impl EncodableBlock for FullPrecisionLinear {
b: &self.weights_buffer,
c: None,
d: output_buffer,
+ bias: self.bias_buffer.as_ref(),
batch: batch_size as i32,
input_dim: self.input_dim as i32,
output_dim: self.output_dim as i32,
@@ -135,18 +136,14 @@ impl EncodableBlock for FullPrecisionLinear {
batch_count: 1,
alpha: 1.0,
beta: 0.0,
+ transpose_a: false,
+ transpose_b: true,
};
let mut kernel = self.kernel.borrow_mut();
- if let Some(bias_buf) = &self.biases_buffer {
- kernel
- .encode_with_bias(state.mtl_context(), encoder, args, bias_buf)
- .expect("Failed to encode matmul kernel");
- } else {
- kernel
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode matmul kernel");
- }
+ kernel
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode matmul kernel");
encoder.end_encoding();
@@ -181,6 +178,7 @@ impl EncodableBlock for FullPrecisionLinear {
b: &self.weights_buffer,
c: None,
d: output_buffer,
+ bias: self.bias_buffer.as_ref(),
batch: batch_size as i32,
input_dim: self.input_dim as i32,
output_dim: self.output_dim as i32,
@@ -190,17 +188,13 @@ impl EncodableBlock for FullPrecisionLinear {
batch_count: 1,
alpha: 1.0,
beta: 0.0,
+ transpose_a: false,
+ transpose_b: true,
};
let mut kernel = self.kernel.borrow_mut();
- if let Some(bias_buf) = &self.biases_buffer {
- kernel
- .encode_with_bias(state.mtl_context(), encoder, args, bias_buf)
- .expect("Failed to encode matmul kernel");
- } else {
- kernel
- .encode(state.mtl_context(), encoder, args)
- .expect("Failed to encode matmul kernel");
- }
+ kernel
+ .encode(state.mtl_context(), encoder, args)
+ .expect("Failed to encode matmul kernel");
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs
index 56dc4e2f6..dfb3cf769 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/common/matmul_arguments.rs
@@ -8,6 +8,7 @@ pub struct MatmulArguments<'a> {
pub b: &'a MTLBuffer,
pub c: Option<&'a MTLBuffer>,
pub d: &'a MTLBuffer,
+ pub bias: Option<&'a MTLBuffer>,
/// M dimension - batch/number of tokens (rows of A, rows of D)
pub batch: i32,
/// K dimension - input_dim/reduction dimension (cols of A, rows of B)
@@ -22,4 +23,6 @@ pub struct MatmulArguments<'a> {
/// Scaling factors for fused addmm (alpha * A @ B + beta * C)
pub alpha: f32,
pub beta: f32,
+ pub transpose_a: bool,
+ pub transpose_b: bool,
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
new file mode 100644
index 000000000..c4934b8ee
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/dispatch_descriptor.rs
@@ -0,0 +1,34 @@
+use super::{common::MatmulArguments, gemm, gemv, split_k};
+use crate::{
+ DataType,
+ backends::metal::{MTLContext, MTLError},
+};
+
+#[derive(Debug, Clone)]
+pub(crate) enum MatmulDispatchDescriptor {
+ Gemv(gemv::DispatchDescriptor),
+ SplitK(split_k::DispatchDescriptor),
+ Gemm(gemm::DispatchDescriptor),
+}
+
+pub(crate) fn choose_dispatch_descriptor(
+ context: &MTLContext,
+ data_type: DataType,
+ arguments: &MatmulArguments,
+) -> Result<MatmulDispatchDescriptor, MTLError> {
+ if let Some(descriptor) =
+ gemv::DispatchDescriptor::try_new(context, data_type, arguments)?
+ {
+ return Ok(MatmulDispatchDescriptor::Gemv(descriptor));
+ }
+
+ if let Some(descriptor) =
+ split_k::DispatchDescriptor::try_new(context, data_type, arguments)?
+ {
+ return Ok(MatmulDispatchDescriptor::SplitK(descriptor));
+ }
+
+ Ok(MatmulDispatchDescriptor::Gemm(
+ gemm::DispatchDescriptor::new(context, data_type, arguments)?,
+ ))
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
new file mode 100644
index 000000000..2861f4a3d
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/dispatch_descriptor.rs
@@ -0,0 +1,200 @@
+use metal::MTLSize;
+
+use super::{
+ pipeline_configuration::PipelineConfiguration,
+ tile_configuration::TileConfiguration,
+};
+use crate::{
+ DataType,
+ backends::metal::{
+ DeviceClass, MTLContext, MTLError,
+ kernel::matmul::common::{
+ GEMMAddMMParams, GEMMParams, MatmulArguments,
+ },
+ },
+};
+
+#[derive(Debug, Clone)]
+pub(crate) struct DispatchDescriptor {
+ pub(crate) pipeline_configuration: PipelineConfiguration,
+ pub(crate) params: GEMMParams,
+ pub(crate) addmm_params: Option<GEMMAddMMParams>,
+ pub(crate) threadgroups: MTLSize,
+ pub(crate) threads_per_threadgroup: MTLSize,
+}
+
+impl DispatchDescriptor {
+ pub(crate) fn new(
+ context: &MTLContext,
+ data_type: DataType,
+ arguments: &MatmulArguments,
+ ) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for GEMM: {data_type:?}"
+ )));
+ }
+
+ let tile = select_tile(context, data_type, arguments);
+
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ let k = arguments.input_dim;
+
+ let pipeline_configuration = PipelineConfiguration {
+ tile,
+ transpose_a: arguments.transpose_a,
+ transpose_b: arguments.transpose_b,
+ align_m: (m % tile.block_rows) == 0,
+ align_n: (n % tile.block_cols) == 0,
+ align_k: (k % tile.block_depth) == 0,
+ has_batch: arguments.batch_count > 1,
+ use_out_source: arguments.c.is_some(),
+ do_axpby: arguments.c.is_some()
+ && (arguments.alpha != 1.0 || arguments.beta != 0.0),
+ };
+
+ let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;
+ let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;
+ let swizzle_log = tile.swizzle_log2;
+
+ let tile_swizzle = 1 << swizzle_log;
+ let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
+ let tn_swizzled = tiles_n * tile_swizzle;
+
+ let elements_per_matrix_a =
+ (arguments.batch as i64) * (arguments.lda as i64);
+ let elements_per_matrix_b = if arguments.transpose_b {
+ (arguments.output_dim as i64) * (arguments.ldb as i64)
+ } else {
+ (arguments.input_dim as i64) * (arguments.ldb as i64)
+ };
+ let elements_per_matrix_d =
+ (arguments.batch as i64) * (arguments.ldd as i64);
+
+ let params = GEMMParams {
+ M: m,
+ N: n,
+ K: k,
+ lda: arguments.lda,
+ ldb: arguments.ldb,
+ ldd: arguments.ldd,
+ tiles_n,
+ tiles_m,
+ batch_stride_a: elements_per_matrix_a,
+ batch_stride_b: elements_per_matrix_b,
+ batch_stride_d: elements_per_matrix_d,
+ swizzle_log,
+ gemm_k_iterations_aligned: k / tile.block_depth,
+ batch_ndim: 1,
+ };
+
+ let addmm_params = if pipeline_configuration.use_out_source {
+ let batch_stride_c = if arguments.batch_count > 1 {
+ (arguments.ldd as i64) * (arguments.output_dim as i64)
+ } else {
+ 0
+ };
+ Some(GEMMAddMMParams {
+ ldc: arguments.ldd,
+ fdc: 1,
+ batch_stride_c,
+ alpha: arguments.alpha,
+ beta: arguments.beta,
+ })
+ } else {
+ None
+ };
+
+ let threads_per_threadgroup =
+ MTLSize::new(32, tile.warps_per_col, tile.warps_per_row);
+ let threadgroups = MTLSize::new(
+ tn_swizzled as u64,
+ tm_swizzled as u64,
+ arguments.batch_count as u64,
+ );
+
+ Ok(Self {
+ pipeline_configuration,
+ params,
+ addmm_params,
+ threadgroups,
+ threads_per_threadgroup,
+ })
+ }
+}
+
+fn select_tile(
+ context: &MTLContext,
+ data_type: DataType,
+ arguments: &MatmulArguments,
+) -> TileConfiguration {
+ let overall_work_elements = (arguments.batch_count as i64)
+ * (arguments.batch as i64)
+ * (arguments.output_dim as i64);
+ let is_float32 = matches!(data_type, DataType::F32);
+ let prefer_half_or_tf32 = !is_float32 || context.tf32_enabled();
+
+ if context.is_nax_available() && prefer_half_or_tf32 {
+ let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);
+ let tile_rows =
+ (arguments.batch + base_tile.block_rows - 1) / base_tile.block_rows;
+ let swizzle_log2 = if tile_rows <= 3 {
+ 0
+ } else {
+ 1
+ };
+ return TileConfiguration {
+ swizzle_log2,
+ ..base_tile
+ };
+ }
+
+ match context.architecture.device_class {
+ DeviceClass::Integrated
+ | DeviceClass::Phone
+ | DeviceClass::Unknown(_) => {
+ if prefer_half_or_tf32 {
+ if !arguments.transpose_a && arguments.transpose_b {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ } else {
+ TileConfiguration::new(64, 64, 16, 1, 2, 0)
+ }
+ } else if !arguments.transpose_a && arguments.transpose_b {
+ TileConfiguration::new(32, 64, 16, 1, 2, 0)
+ } else {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ }
+ },
+ DeviceClass::Desktop => {
+ if overall_work_elements >= (1_i64 << 20) {
+ if prefer_half_or_tf32 {
+ if 2 * std::cmp::max(arguments.batch, arguments.output_dim)
+ > arguments.input_dim
+ {
+ TileConfiguration::new(64, 64, 16, 2, 2, 0)
+ } else if !arguments.transpose_a && arguments.transpose_b {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ } else {
+ TileConfiguration::new(32, 64, 16, 1, 2, 0)
+ }
+ } else if !arguments.transpose_a && arguments.transpose_b {
+ TileConfiguration::new(32, 64, 16, 1, 2, 0)
+ } else {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ }
+ } else if prefer_half_or_tf32 {
+ if !arguments.transpose_a && arguments.transpose_b {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ } else {
+ TileConfiguration::new(64, 64, 16, 1, 2, 0)
+ }
+ } else if !arguments.transpose_a && arguments.transpose_b {
+ TileConfiguration::new(32, 64, 16, 1, 2, 0)
+ } else {
+ TileConfiguration::new(64, 32, 32, 2, 2, 0)
+ }
+ },
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 9742f81ae..cef66acbd 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -2,17 +2,15 @@ use std::collections::HashMap;
use metal::{
ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
- MTLSize,
};
use super::{
- pipeline_configuration::PipelineConfiguration,
- tile_configuration::TileConfiguration,
+ DispatchDescriptor, pipeline_configuration::PipelineConfiguration,
};
use crate::{
DataType,
backends::metal::{
- DeviceClass, MTLContext, MTLError,
+ MTLContext, MTLError,
kernel::matmul::common::{
GEMMAddMMParams, GEMMParams, MatmulArguments,
transpose_configuration,
@@ -22,17 +20,11 @@ use crate::{
pub struct Kernel {
data_type: DataType,
- transpose_a: bool,
- transpose_b: bool,
pipelines: HashMap<PipelineConfiguration, MTLComputePipelineState>,
}
impl Kernel {
- pub fn new(
- data_type: DataType,
- transpose_a: bool,
- transpose_b: bool,
- ) -> Result<Self, MTLError> {
+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {
if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
{
return Err(MTLError::Generic(format!(
@@ -41,8 +33,6 @@ impl Kernel {
}
Ok(Self {
data_type,
- transpose_a,
- transpose_b,
pipelines: HashMap::new(),
})
}
@@ -58,12 +48,14 @@ impl Kernel {
fn kernel_name(
&self,
- tile: &TileConfiguration,
+ configuration: &PipelineConfiguration,
) -> String {
let type_name = self.type_name();
- let transpose_suffix =
- transpose_configuration(self.transpose_a, self.transpose_b);
- let prefix = if tile.is_nax() {
+ let transpose_suffix = transpose_configuration(
+ configuration.transpose_a,
+ configuration.transpose_b,
+ );
+ let prefix = if configuration.tile.is_nax() {
"steel_gemm_nax"
} else {
"steel_gemm"
@@ -74,263 +66,114 @@ impl Kernel {
transpose_suffix.as_str(),
type_name,
type_name,
- tile.block_rows,
- tile.block_cols,
- tile.block_depth,
- tile.warps_per_row,
- tile.warps_per_col
+ configuration.tile.block_rows,
+ configuration.tile.block_cols,
+ configuration.tile.block_depth,
+ configuration.tile.warps_per_row,
+ configuration.tile.warps_per_col
)
}
- pub fn select_tile(
- &self,
- mtl: &MTLContext,
- args: &MatmulArguments,
- ) -> TileConfiguration {
- let overall_work_elements = (args.batch_count as i64)
- * (args.batch as i64)
- * (args.output_dim as i64);
- let is_float32 = matches!(self.data_type, DataType::F32);
- let prefer_half_or_tf32 = !is_float32 || mtl.tf32_enabled();
-
- if mtl.is_nax_available() && prefer_half_or_tf32 {
- let base_tile = TileConfiguration::new(128, 128, 512, 4, 4, 0);
- let tile_rows =
- (args.batch + base_tile.block_rows - 1) / base_tile.block_rows;
- let swizzle_log2 = if tile_rows <= 3 {
- 0
- } else {
- 1
- };
- return TileConfiguration {
- swizzle_log2,
- ..base_tile
- };
- }
-
- let device_class_code = match mtl.architecture.device_class {
- DeviceClass::Desktop => 'd',
- DeviceClass::Integrated => 'g',
- DeviceClass::Phone => 'p',
- DeviceClass::Unknown(_) => 'g',
- };
-
- match device_class_code {
- 'g' | 'p' => {
- if prefer_half_or_tf32 {
- if !self.transpose_a && self.transpose_b {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- } else {
- TileConfiguration::new(64, 64, 16, 1, 2, 0)
- }
- } else if !self.transpose_a && self.transpose_b {
- TileConfiguration::new(32, 64, 16, 1, 2, 0)
- } else {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- }
- },
- 'd' => {
- if overall_work_elements >= (1_i64 << 20) {
- if prefer_half_or_tf32 {
- if 2 * std::cmp::max(args.batch, args.output_dim)
- > args.input_dim
- {
- TileConfiguration::new(64, 64, 16, 2, 2, 0)
- } else if !self.transpose_a && self.transpose_b {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- } else {
- TileConfiguration::new(32, 64, 16, 1, 2, 0)
- }
- } else if !self.transpose_a && self.transpose_b {
- TileConfiguration::new(32, 64, 16, 1, 2, 0)
- } else {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- }
- } else if prefer_half_or_tf32 {
- if !self.transpose_a && self.transpose_b {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- } else {
- TileConfiguration::new(64, 64, 16, 1, 2, 0)
- }
- } else if !self.transpose_a && self.transpose_b {
- TileConfiguration::new(32, 64, 16, 1, 2, 0)
- } else {
- TileConfiguration::new(64, 32, 32, 2, 2, 0)
- }
- },
- _ => TileConfiguration::new(64, 64, 16, 2, 2, 0),
- }
- }
-
fn get_or_compile_pipeline(
&mut self,
- mtl: &MTLContext,
- config: &PipelineConfiguration,
+ context: &MTLContext,
+ configuration: &PipelineConfiguration,
) -> Result<&MTLComputePipelineState, MTLError> {
- if !self.pipelines.contains_key(config) {
- let name = self.kernel_name(&config.tile);
- let fcv = metal::FunctionConstantValues::new();
- fcv.set_constant_value_at_index(
- &config.has_batch as *const bool as *const _,
+ if !self.pipelines.contains_key(configuration) {
+ let kernel_name = self.kernel_name(configuration);
+ let function_constants = metal::FunctionConstantValues::new();
+ function_constants.set_constant_value_at_index(
+ &configuration.has_batch as *const bool as *const _,
metal::MTLDataType::Bool,
10,
);
- fcv.set_constant_value_at_index(
- &config.use_out_source as *const bool as *const _,
+ function_constants.set_constant_value_at_index(
+ &configuration.use_out_source as *const bool as *const _,
metal::MTLDataType::Bool,
100,
);
- fcv.set_constant_value_at_index(
- &config.do_axpby as *const bool as *const _,
+ function_constants.set_constant_value_at_index(
+ &configuration.do_axpby as *const bool as *const _,
metal::MTLDataType::Bool,
110,
);
- fcv.set_constant_value_at_index(
- &config.align_m as *const bool as *const _,
+ function_constants.set_constant_value_at_index(
+ &configuration.align_m as *const bool as *const _,
metal::MTLDataType::Bool,
200,
);
- fcv.set_constant_value_at_index(
- &config.align_n as *const bool as *const _,
+ function_constants.set_constant_value_at_index(
+ &configuration.align_n as *const bool as *const _,
metal::MTLDataType::Bool,
201,
);
- fcv.set_constant_value_at_index(
- &config.align_k as *const bool as *const _,
+ function_constants.set_constant_value_at_index(
+ &configuration.align_k as *const bool as *const _,
metal::MTLDataType::Bool,
202,
);
let cache_key = format!(
"{}_am{}_an{}_ak{}_hb{}_uo{}_ax{}",
- name,
- config.align_m as u8,
- config.align_n as u8,
- config.align_k as u8,
- config.has_batch as u8,
- config.use_out_source as u8,
- config.do_axpby as u8
+ kernel_name,
+ configuration.align_m as u8,
+ configuration.align_n as u8,
+ configuration.align_k as u8,
+ configuration.has_batch as u8,
+ configuration.use_out_source as u8,
+ configuration.do_axpby as u8
);
- let (ps, _) = mtl.compute_pipeline_state_with_reflection_cached(
- &cache_key,
- &name,
- Some(&fcv),
- )?;
- self.pipelines.insert(config.clone(), ps);
+ let (pipeline_state, _) = context
+ .compute_pipeline_state_with_reflection_cached(
+ &cache_key,
+ &kernel_name,
+ Some(&function_constants),
+ )?;
+ self.pipelines.insert(configuration.clone(), pipeline_state);
}
- Ok(self.pipelines.get(config).unwrap())
+ Ok(self.pipelines.get(configuration).unwrap())
}
- pub fn encode(
+ pub(crate) fn encode_descriptor(
&mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- ) -> Result<(), MTLError> {
- let tile = self.select_tile(mtl, args);
- self.encode_with_tile(mtl, enc, args, &tile)
- }
-
- pub fn encode_with_tile(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- tile: &TileConfiguration,
- ) -> Result<(), MTLError> {
- let m = args.batch;
- let n = args.output_dim;
- let k = args.input_dim;
-
- let config = PipelineConfiguration {
- tile: *tile,
- align_m: (m % tile.block_rows) == 0,
- align_n: (n % tile.block_cols) == 0,
- align_k: (k % tile.block_depth) == 0,
- has_batch: args.batch_count > 1,
- use_out_source: args.c.is_some(),
- do_axpby: args.c.is_some()
- && (args.alpha != 1.0 || args.beta != 0.0),
- };
-
- let ps = self.get_or_compile_pipeline(mtl, &config)?;
- enc.set_compute_pipeline_state(ps);
-
- enc.set_buffer(0, Some(args.a), args.a_offset);
- enc.set_buffer(1, Some(args.b), 0);
- if config.use_out_source {
- if let Some(c_buf) = args.c {
- enc.set_buffer(2, Some(c_buf), 0);
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ arguments: &MatmulArguments,
+ descriptor: &DispatchDescriptor,
+ ) -> Result<bool, MTLError> {
+ let pipeline_state = self.get_or_compile_pipeline(
+ context,
+ &descriptor.pipeline_configuration,
+ )?;
+ encoder.set_compute_pipeline_state(pipeline_state);
+
+ encoder.set_buffer(0, Some(arguments.a), arguments.a_offset);
+ encoder.set_buffer(1, Some(arguments.b), 0);
+ if descriptor.pipeline_configuration.use_out_source {
+ if let Some(c_buffer) = arguments.c {
+ encoder.set_buffer(2, Some(c_buffer), 0);
}
}
- enc.set_buffer(3, Some(args.d), 0);
-
- let tiles_n = (n + tile.block_cols - 1) / tile.block_cols;
- let tiles_m = (m + tile.block_rows - 1) / tile.block_rows;
- let swizzle_log = tile.swizzle_log2;
+ encoder.set_buffer(3, Some(arguments.d), 0);
- let tile_swizzle = 1 << swizzle_log;
- let tm_swizzled = (tiles_m + tile_swizzle - 1) / tile_swizzle;
- let tn_swizzled = tiles_n * tile_swizzle;
-
- let elements_per_matrix_a = (args.batch as i64) * (args.lda as i64);
- let elements_per_matrix_b = if self.transpose_b {
- (args.output_dim as i64) * (args.ldb as i64)
- } else {
- (args.input_dim as i64) * (args.ldb as i64)
- };
- let elements_per_matrix_d = (args.batch as i64) * (args.ldd as i64);
-
- let params = GEMMParams {
- M: m,
- N: n,
- K: k,
- lda: args.lda,
- ldb: args.ldb,
- ldd: args.ldd,
- tiles_n,
- tiles_m,
- batch_stride_a: elements_per_matrix_a,
- batch_stride_b: elements_per_matrix_b,
- batch_stride_d: elements_per_matrix_d,
- swizzle_log,
- gemm_k_iterations_aligned: k / tile.block_depth,
- batch_ndim: 1,
- };
- enc.set_bytes(
+ encoder.set_bytes(
4,
std::mem::size_of::<GEMMParams>() as u64,
- &params as *const GEMMParams as *const _,
+ &descriptor.params as *const _ as *const _,
);
- if config.use_out_source {
- let batch_stride_c = if args.batch_count > 1 {
- (args.ldd as i64) * (args.output_dim as i64)
- } else {
- 0
- };
- let addmm_params = GEMMAddMMParams {
- ldc: args.ldd,
- fdc: 1,
- batch_stride_c,
- alpha: args.alpha,
- beta: args.beta,
- };
- enc.set_bytes(
+ if let Some(addmm_params) = &descriptor.addmm_params {
+ encoder.set_bytes(
5,
std::mem::size_of::<GEMMAddMMParams>() as u64,
- &addmm_params as *const GEMMAddMMParams as *const _,
+ addmm_params as *const _ as *const _,
);
}
- let threads_per_tg =
- MTLSize::new(32, tile.warps_per_col, tile.warps_per_row);
- let tgs = MTLSize::new(
- tn_swizzled as u64,
- tm_swizzled as u64,
- args.batch_count as u64,
+ encoder.dispatch_thread_groups(
+ descriptor.threadgroups,
+ descriptor.threads_per_threadgroup,
);
- enc.dispatch_thread_groups(tgs, threads_per_tg);
- Ok(())
+ Ok(false)
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
index f78544f8c..b8eda5d5d 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -1,5 +1,7 @@
+mod dispatch_descriptor;
mod kernel;
mod pipeline_configuration;
mod tile_configuration;
pub use kernel::Kernel as GemmKernel;
+pub(crate) use dispatch_descriptor::DispatchDescriptor;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs
index e92065b03..252792c91 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemm/pipeline_configuration.rs
@@ -3,6 +3,8 @@ use super::tile_configuration::TileConfiguration;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct PipelineConfiguration {
pub tile: TileConfiguration,
+ pub transpose_a: bool,
+ pub transpose_b: bool,
pub align_m: bool,
pub align_n: bool,
pub align_k: bool,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs
new file mode 100644
index 000000000..94344f824
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/dispatch_descriptor.rs
@@ -0,0 +1,175 @@
+use metal::MTLSize;
+
+use super::pipeline_configuration::{
+ PipelineConfiguration, select_configuration,
+};
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLContext, MTLError, kernel::matmul::common::MatmulArguments,
+ },
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum AxpbySource {
+ None,
+ Bias,
+ C,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct DispatchDescriptor {
+ pub(crate) pipeline_configuration: PipelineConfiguration,
+ pub(crate) matrix_is_rhs: bool,
+ pub(crate) axpby_source: AxpbySource,
+ pub(crate) input_dimension: i32,
+ pub(crate) output_dimension: i32,
+ pub(crate) matrix_leading_dim: i32,
+ pub(crate) alpha: f32,
+ pub(crate) beta: f32,
+ pub(crate) batch_ndim: i32,
+ pub(crate) batch_shape: [i32; 1],
+ pub(crate) vector_batch_stride: [i64; 1],
+ pub(crate) matrix_batch_stride: [i64; 1],
+ pub(crate) bias_batch_stride: [i64; 1],
+ pub(crate) bias_stride: i32,
+ pub(crate) threadgroups: MTLSize,
+ pub(crate) threads_per_threadgroup: MTLSize,
+}
+
+impl DispatchDescriptor {
+ pub(crate) fn try_new(
+ _context: &MTLContext,
+ data_type: DataType,
+ arguments: &MatmulArguments,
+ ) -> Result<Option<Self>, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for GEMV: {data_type:?}"
+ )));
+ }
+
+ if arguments.transpose_a || !arguments.transpose_b {
+ return Ok(None);
+ }
+
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ if m != 1 && n != 1 {
+ return Ok(None);
+ }
+
+ let matrix_is_rhs = n != 1;
+ let transpose_matrix = if matrix_is_rhs {
+ !arguments.transpose_b
+ } else {
+ arguments.transpose_a
+ };
+
+ let axpby_source = if arguments.c.is_some() {
+ AxpbySource::C
+ } else if arguments.bias.is_some() {
+ AxpbySource::Bias
+ } else {
+ AxpbySource::None
+ };
+
+ let (do_axpby, alpha, beta, bias_stride) = match axpby_source {
+ AxpbySource::None => (false, 1.0f32, 0.0f32, 0),
+ AxpbySource::Bias => (true, 1.0f32, 1.0f32, 1),
+ AxpbySource::C => (true, arguments.alpha, arguments.beta, arguments.ldd),
+ };
+
+ let output_dimension = if matrix_is_rhs {
+ arguments.output_dim
+ } else {
+ arguments.batch
+ };
+ let pipeline_configuration = select_configuration(
+ arguments.transpose_a,
+ arguments.transpose_b,
+ transpose_matrix,
+ arguments.input_dim,
+ output_dimension,
+ false,
+ do_axpby,
+ );
+
+ let input_dimension = arguments.input_dim;
+ let matrix_leading_dim = if matrix_is_rhs {
+ arguments.ldb
+ } else {
+ arguments.lda
+ };
+
+ let batch_ndim = 1i32;
+ let batch_groups = arguments.batch_count.max(1);
+ let batch_shape = [if arguments.batch_count > 1 {
+ arguments.batch_count
+ } else {
+ 1
+ }];
+
+ let elements_per_matrix_a =
+ (arguments.batch as i64) * (arguments.lda as i64);
+ let elements_per_matrix_b = if arguments.transpose_b {
+ (arguments.output_dim as i64) * (arguments.ldb as i64)
+ } else {
+ (arguments.input_dim as i64) * (arguments.ldb as i64)
+ };
+
+ let vector_batch_stride = [if matrix_is_rhs {
+ elements_per_matrix_a
+ } else {
+ elements_per_matrix_b
+ }];
+
+ let matrix_batch_stride = [if matrix_is_rhs {
+ elements_per_matrix_b
+ } else {
+ elements_per_matrix_a
+ }];
+
+ let bias_batch_stride = [if arguments.batch_count > 1 {
+ (output_dimension as i64) * (arguments.ldd as i64)
+ } else {
+ 0
+ }];
+
+ let output_elements_per_threadgroup =
+ pipeline_configuration.output_elements_per_threadgroup();
+ let threadgroup_count_x =
+ ((output_dimension as u32 + output_elements_per_threadgroup - 1)
+ / output_elements_per_threadgroup) as u64;
+ let threadgroup_count_z = batch_groups.max(1) as u64;
+
+ let threadgroups =
+ MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
+ let threads_per_threadgroup =
+ pipeline_configuration.threads_per_threadgroup();
+
+ Ok(Some(Self {
+ pipeline_configuration,
+ matrix_is_rhs,
+ axpby_source,
+ input_dimension,
+ output_dimension,
+ matrix_leading_dim,
+ alpha,
+ beta,
+ batch_ndim,
+ batch_shape,
+ vector_batch_stride,
+ matrix_batch_stride,
+ bias_batch_stride,
+ bias_stride,
+ threadgroups,
+ threads_per_threadgroup,
+ }))
+ }
+
+ pub(crate) fn bias_is_fused(&self) -> bool {
+ matches!(self.axpby_source, AxpbySource::Bias)
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
index 53b50be63..bcf872da8 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/kernel.rs
@@ -1,11 +1,11 @@
use std::collections::HashMap;
-use metal::{
- Buffer as MTLBuffer, ComputeCommandEncoderRef, ComputePipelineState,
- MTLSize,
-};
+use metal::{ComputeCommandEncoderRef, ComputePipelineState};
-use super::configuration::{Configuration, select_configuration};
+use super::{
+ dispatch_descriptor::{AxpbySource, DispatchDescriptor},
+ pipeline_configuration::PipelineConfiguration,
+};
use crate::{
DataType,
backends::metal::{
@@ -15,7 +15,7 @@ use crate::{
fn gemv_kernel_name(
data_type: DataType,
- configuration: &Configuration,
+ config: &PipelineConfiguration,
) -> Result<String, MTLError> {
let dtype_name = match data_type {
DataType::F16 => "float16",
@@ -29,7 +29,7 @@ fn gemv_kernel_name(
},
};
- let prefix = if configuration.transpose_matrix {
+ let prefix = if config.transpose_matrix {
"gemv_t"
} else {
"gemv"
@@ -37,303 +37,174 @@ fn gemv_kernel_name(
Ok(format!(
"{prefix}_{dtype_name}_bm{}_bn{}_sm{}_sn{}_tm{}_tn{}_nc{}_axpby{}",
- configuration.threadgroup_rows,
- configuration.threadgroup_cols,
- configuration.threads_per_simdgroup_row,
- configuration.threads_per_simdgroup_col,
- configuration.elements_per_thread_row,
- configuration.elements_per_thread_col,
- configuration.non_contiguous_batch as u8,
- configuration.do_axpby as u8,
+ config.threadgroup_rows,
+ config.threadgroup_cols,
+ config.threads_per_simdgroup_row,
+ config.threads_per_simdgroup_col,
+ config.elements_per_thread_row,
+ config.elements_per_thread_col,
+ config.non_contiguous_batch as u8,
+ config.do_axpby as u8,
))
}
pub struct Kernel {
data_type: DataType,
- lhs_is_transposed: bool,
- rhs_is_transposed: bool,
- pipelines: HashMap<Configuration, ComputePipelineState>,
+ pipelines: HashMap<PipelineConfiguration, ComputePipelineState>,
}
impl Kernel {
- pub fn new(
- data_type: DataType,
- lhs_is_transposed: bool,
- rhs_is_transposed: bool,
- ) -> Self {
- Self {
+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported data type for GEMV: {:?}",
+ data_type
+ )));
+ }
+ Ok(Self {
data_type,
- lhs_is_transposed,
- rhs_is_transposed,
pipelines: HashMap::new(),
- }
+ })
}
fn get_pipeline(
&mut self,
context: &MTLContext,
- configuration: Configuration,
+ config: &PipelineConfiguration,
) -> Result<&ComputePipelineState, MTLError> {
- if !self.pipelines.contains_key(&configuration) {
- let kernel_name = gemv_kernel_name(self.data_type, &configuration)?;
+ if !self.pipelines.contains_key(config) {
+ let kernel_name = gemv_kernel_name(self.data_type, config)?;
let pipeline =
context.compute_pipeline_state(&kernel_name, None)?;
- self.pipelines.insert(configuration, pipeline);
+ self.pipelines.insert(*config, pipeline);
}
- Ok(self.pipelines.get(&configuration).unwrap())
+ Ok(self.pipelines.get(config).unwrap())
}
- #[allow(clippy::too_many_arguments)]
- fn encode_with_configuration(
+ pub(crate) fn encode_descriptor(
&mut self,
context: &MTLContext,
encoder: &ComputeCommandEncoderRef,
arguments: &MatmulArguments,
- configuration: Configuration,
- matrix_is_rhs: bool,
- bias_buffer: Option<&MTLBuffer>,
- alpha: f32,
- beta: f32,
- bias_stride: i32,
- ) -> Result<(), MTLError> {
- let pipeline = self.get_pipeline(context, configuration)?;
+ descriptor: &DispatchDescriptor,
+ ) -> Result<bool, MTLError> {
+ let pipeline =
+ self.get_pipeline(context, &descriptor.pipeline_configuration)?;
encoder.set_compute_pipeline_state(pipeline);
- let (buf0, off0) = if matrix_is_rhs {
+ let (buf0, off0) = if descriptor.matrix_is_rhs {
(arguments.b, 0)
} else {
(arguments.a, arguments.a_offset)
};
encoder.set_buffer(0, Some(buf0), off0);
- let (buf1, off1) = if matrix_is_rhs {
+ let (buf1, off1) = if descriptor.matrix_is_rhs {
(arguments.a, arguments.a_offset)
} else {
(arguments.b, 0)
};
encoder.set_buffer(1, Some(buf1), off1);
- if configuration.do_axpby {
- if let Some(bias) = bias_buffer {
- encoder.set_buffer(2, Some(bias), 0);
+ if descriptor.pipeline_configuration.do_axpby {
+ match descriptor.axpby_source {
+ AxpbySource::None => {
+ return Err(MTLError::Generic(
+ "GEMV descriptor mismatch: do_axpby=true but axpby_source=None"
+ .to_owned(),
+ ));
+ },
+ AxpbySource::Bias => {
+ let bias = arguments.bias.ok_or_else(|| {
+ MTLError::Generic(
+ "GEMV descriptor requires bias buffer".to_owned(),
+ )
+ })?;
+ encoder.set_buffer(2, Some(bias), 0);
+ },
+ AxpbySource::C => {
+ let c_buffer = arguments.c.ok_or_else(|| {
+ MTLError::Generic(
+ "GEMV descriptor requires C buffer".to_owned(),
+ )
+ })?;
+ encoder.set_buffer(2, Some(c_buffer), 0);
+ },
}
}
encoder.set_buffer(3, Some(arguments.d), 0);
- let input_dimension = arguments.input_dim;
- let output_dimension = if matrix_is_rhs {
- arguments.output_dim
- } else {
- arguments.batch
- };
- let matrix_leading_dim = if matrix_is_rhs {
- arguments.ldb
- } else {
- arguments.lda
- };
-
encoder.set_bytes(
4,
std::mem::size_of::<i32>() as u64,
- &input_dimension as *const i32 as *const std::ffi::c_void,
+ &descriptor.input_dimension as *const i32
+ as *const std::ffi::c_void,
);
encoder.set_bytes(
5,
std::mem::size_of::<i32>() as u64,
- &output_dimension as *const i32 as *const std::ffi::c_void,
+ &descriptor.output_dimension as *const i32
+ as *const std::ffi::c_void,
);
encoder.set_bytes(
6,
std::mem::size_of::<i32>() as u64,
- &matrix_leading_dim as *const i32 as *const std::ffi::c_void,
+ &descriptor.matrix_leading_dim as *const i32
+ as *const std::ffi::c_void,
);
encoder.set_bytes(
7,
std::mem::size_of::<f32>() as u64,
- &alpha as *const f32 as *const std::ffi::c_void,
+ &descriptor.alpha as *const f32 as *const std::ffi::c_void,
);
encoder.set_bytes(
8,
std::mem::size_of::<f32>() as u64,
- &beta as *const f32 as *const std::ffi::c_void,
+ &descriptor.beta as *const f32 as *const std::ffi::c_void,
);
- let batch_shape = if arguments.batch_count > 1 {
- vec![arguments.batch_count]
- } else {
- vec![1]
- };
- let batch_ndim = 1i32;
- let batch_groups = arguments.batch_count.max(1);
-
- let elements_per_matrix_a =
- (arguments.batch as i64) * (arguments.lda as i64);
- let elements_per_matrix_b = if self.rhs_is_transposed {
- (arguments.output_dim as i64) * (arguments.ldb as i64)
- } else {
- (arguments.input_dim as i64) * (arguments.ldb as i64)
- };
-
- let vector_batch_stride = if matrix_is_rhs {
- vec![elements_per_matrix_a]
- } else {
- vec![elements_per_matrix_b]
- };
-
- let matrix_batch_stride = if matrix_is_rhs {
- vec![elements_per_matrix_b]
- } else {
- vec![elements_per_matrix_a]
- };
-
- let bias_batch_stride = if arguments.batch_count > 1 {
- vec![(output_dimension as i64) * (arguments.ldd as i64)]
- } else {
- vec![0]
- };
-
encoder.set_bytes(
9,
std::mem::size_of::<i32>() as u64,
- &batch_ndim as *const i32 as *const std::ffi::c_void,
+ &descriptor.batch_ndim as *const i32 as *const std::ffi::c_void,
);
encoder.set_bytes(
10,
- (std::mem::size_of::<i32>() * batch_shape.len()) as u64,
- batch_shape.as_ptr() as *const std::ffi::c_void,
+ (std::mem::size_of::<i32>() * descriptor.batch_shape.len()) as u64,
+ descriptor.batch_shape.as_ptr() as *const std::ffi::c_void,
);
encoder.set_bytes(
11,
- (std::mem::size_of::<i64>() * vector_batch_stride.len()) as u64,
- vector_batch_stride.as_ptr() as *const std::ffi::c_void,
+ (std::mem::size_of::<i64>() * descriptor.vector_batch_stride.len())
+ as u64,
+ descriptor.vector_batch_stride.as_ptr() as *const std::ffi::c_void,
);
encoder.set_bytes(
12,
- (std::mem::size_of::<i64>() * matrix_batch_stride.len()) as u64,
- matrix_batch_stride.as_ptr() as *const std::ffi::c_void,
+ (std::mem::size_of::<i64>() * descriptor.matrix_batch_stride.len())
+ as u64,
+ descriptor.matrix_batch_stride.as_ptr() as *const std::ffi::c_void,
);
encoder.set_bytes(
13,
- (std::mem::size_of::<i64>() * bias_batch_stride.len()) as u64,
- bias_batch_stride.as_ptr() as *const std::ffi::c_void,
+ (std::mem::size_of::<i64>() * descriptor.bias_batch_stride.len())
+ as u64,
+ descriptor.bias_batch_stride.as_ptr() as *const std::ffi::c_void,
);
encoder.set_bytes(
14,
std::mem::size_of::<i32>() as u64,
- &bias_stride as *const i32 as *const std::ffi::c_void,
+ &descriptor.bias_stride as *const i32 as *const std::ffi::c_void,
);
- let output_elements_per_threadgroup =
- configuration.output_elements_per_threadgroup();
- let threadgroup_count_x =
- ((output_dimension as u32 + output_elements_per_threadgroup - 1)
- / output_elements_per_threadgroup) as u64;
- let threadgroup_count_z = batch_groups.max(1) as u64;
-
- let threadgroup_count =
- MTLSize::new(threadgroup_count_x, 1, threadgroup_count_z);
- let threads_per_threadgroup = configuration.threads_per_threadgroup();
-
- encoder
- .dispatch_thread_groups(threadgroup_count, threads_per_threadgroup);
- Ok(())
- }
-
- pub fn encode(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- arguments: &MatmulArguments,
- ) -> Result<(), MTLError> {
- self.encode_internal(context, encoder, arguments, None)
- }
-
- pub fn encode_with_bias(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- arguments: &MatmulArguments,
- bias: &MTLBuffer,
- ) -> Result<(), MTLError> {
- self.encode_internal(context, encoder, arguments, Some(bias))
- }
-
- fn encode_internal(
- &mut self,
- context: &MTLContext,
- encoder: &ComputeCommandEncoderRef,
- arguments: &MatmulArguments,
- bias: Option<&MTLBuffer>,
- ) -> Result<(), MTLError> {
- let m = arguments.batch;
- let n = arguments.output_dim;
- if m != 1 && n != 1 {
- return Ok(());
- }
-
- let matrix_is_rhs = n != 1;
- let transpose_matrix = if matrix_is_rhs {
- !self.rhs_is_transposed
- } else {
- self.lhs_is_transposed
- };
-
- let has_non_contiguous_batch = false;
-
- let (do_axpby, bias_buffer, alpha, beta, bias_stride) =
- if let Some(bias_buf) = bias {
- (true, Some(bias_buf), 1.0f32, 1.0f32, 1)
- } else if let Some(c_buf) = arguments.c {
- (
- true,
- Some(c_buf),
- arguments.alpha,
- arguments.beta,
- arguments.ldd,
- )
- } else {
- (false, None, 1.0f32, 0.0f32, 0)
- };
-
- let configuration = select_configuration(
- transpose_matrix,
- arguments.input_dim,
- if matrix_is_rhs {
- arguments.output_dim
- } else {
- arguments.batch
- },
- has_non_contiguous_batch,
- do_axpby,
+ encoder.dispatch_thread_groups(
+ descriptor.threadgroups,
+ descriptor.threads_per_threadgroup,
);
- if crate::utils::env_utils::debug_matmul_enabled() {
- let kernel_name = gemv_kernel_name(self.data_type, &configuration)
- .unwrap_or_default();
- eprintln!(
- "[matmul] GEMV m={} k={} n={} batch={} dtype={:?} transpose={} kernel={}",
- arguments.batch,
- arguments.input_dim,
- arguments.output_dim,
- arguments.batch_count,
- self.data_type,
- configuration.transpose_matrix,
- kernel_name
- );
- }
-
- self.encode_with_configuration(
- context,
- encoder,
- arguments,
- configuration,
- matrix_is_rhs,
- bias_buffer,
- alpha,
- beta,
- bias_stride,
- )
+ Ok(descriptor.bias_is_fused())
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
index bbd057f95..ca3ea07c9 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -1,4 +1,6 @@
-mod configuration;
+mod dispatch_descriptor;
mod kernel;
+mod pipeline_configuration;
pub use kernel::Kernel as GemvKernel;
+pub(crate) use dispatch_descriptor::DispatchDescriptor;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs
similarity index 91%
rename from crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs
rename to crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs
index 0b11354f0..b89852501 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/gemv/configuration.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/gemv/pipeline_configuration.rs
@@ -1,7 +1,9 @@
use metal::MTLSize;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct Configuration {
+pub struct PipelineConfiguration {
+ pub transpose_a: bool,
+ pub transpose_b: bool,
pub transpose_matrix: bool,
pub threadgroup_rows: u32,
pub threadgroup_cols: u32,
@@ -13,7 +15,7 @@ pub struct Configuration {
pub do_axpby: bool,
}
-impl Configuration {
+impl PipelineConfiguration {
pub fn output_elements_per_threadgroup(&self) -> u32 {
if self.transpose_matrix {
self.threadgroup_cols
@@ -36,12 +38,14 @@ impl Configuration {
}
pub fn select_configuration(
+ transpose_a: bool,
+ transpose_b: bool,
transpose_matrix: bool,
input_dimension: i32,
output_dimension: i32,
non_contiguous_batch: bool,
do_axpby: bool,
-) -> Configuration {
+) -> PipelineConfiguration {
let (threadgroup_rows, threadgroup_cols);
let (threads_per_simdgroup_row, threads_per_simdgroup_col);
let (elements_per_thread_row, elements_per_thread_col);
@@ -107,7 +111,9 @@ pub fn select_configuration(
elements_per_thread_col = 4;
}
- Configuration {
+ PipelineConfiguration {
+ transpose_a,
+ transpose_b,
transpose_matrix,
threadgroup_rows,
threadgroup_cols,
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
index fedf3cfe2..4d84aedd4 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/kernel.rs
@@ -1,8 +1,11 @@
-use metal::{Buffer as MTLBuffer, ComputeCommandEncoderRef};
+use metal::ComputeCommandEncoderRef;
use super::{
super::{KernelDataType, TensorAddBias},
common::MatmulArguments,
+ dispatch_descriptor::{
+ MatmulDispatchDescriptor, choose_dispatch_descriptor,
+ },
gemm,
gemv::GemvKernel,
split_k::SplitKGemm,
@@ -10,13 +13,10 @@ use super::{
use crate::{
DataType,
backends::metal::{MTLContext, MTLError},
- utils::env_utils::debug_matmul_enabled,
};
pub struct MatmulKernel {
data_type: DataType,
- lhs_is_transposed: bool,
- rhs_is_transposed: bool,
gemm: Option<gemm::GemmKernel>,
gemv: Option<GemvKernel>,
splitk: Option<SplitKGemm>,
@@ -24,12 +24,7 @@ pub struct MatmulKernel {
}
impl MatmulKernel {
- pub fn new(
- _mtl: &MTLContext,
- data_type: DataType,
- lhs_is_transposed: bool,
- rhs_is_transposed: bool,
- ) -> Result<Self, MTLError> {
+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {
if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
{
return Err(MTLError::Generic(format!(
@@ -39,8 +34,6 @@ impl MatmulKernel {
Ok(Self {
data_type,
- lhs_is_transposed,
- rhs_is_transposed,
gemm: None,
gemv: None,
splitk: None,
@@ -48,177 +41,90 @@ impl MatmulKernel {
})
}
- fn maybe_use_gemv_impl(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- bias: Option<&MTLBuffer>,
- ) -> Result<bool, MTLError> {
- let m = args.batch;
- let n = args.output_dim;
- if self.lhs_is_transposed || !self.rhs_is_transposed {
- return Ok(false);
- }
-
- if m != 1 && n != 1 {
- return Ok(false);
- }
- let gemv = self.gemv.get_or_insert_with(|| {
- GemvKernel::new(
- self.data_type,
- self.lhs_is_transposed,
- self.rhs_is_transposed,
- )
- });
- if let Some(bias) = bias {
- gemv.encode_with_bias(mtl, enc, args, bias)?;
- } else {
- gemv.encode(mtl, enc, args)?;
- }
- Ok(true)
- }
-
- fn maybe_use_gemv(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- ) -> Result<bool, MTLError> {
- self.maybe_use_gemv_impl(mtl, enc, args, None)
- }
-
- fn maybe_use_gemv_with_bias(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- bias: &MTLBuffer,
- ) -> Result<bool, MTLError> {
- self.maybe_use_gemv_impl(mtl, enc, args, Some(bias))
- }
-
- fn maybe_use_splitk(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- ) -> Result<bool, MTLError> {
- let m = args.batch;
- let n = args.output_dim;
- let k = args.input_dim;
- let batch_count = args.batch_count;
-
- if !SplitKGemm::should_use_splitk(m, n, k, batch_count) {
- return Ok(false);
- }
-
- let splitk = self.splitk.get_or_insert_with(|| {
- SplitKGemm::new(
- self.data_type,
- self.lhs_is_transposed,
- self.rhs_is_transposed,
- )
- });
- splitk.encode(mtl, enc, args)?;
- Ok(true)
- }
-
fn get_or_create_gemm(
&mut self
) -> Result<&mut gemm::GemmKernel, MTLError> {
if self.gemm.is_none() {
- self.gemm = Some(gemm::GemmKernel::new(
- self.data_type,
- self.lhs_is_transposed,
- self.rhs_is_transposed,
- )?);
+ self.gemm = Some(gemm::GemmKernel::new(self.data_type)?);
}
Ok(self.gemm.as_mut().unwrap())
}
- fn encode_gemm(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- ) -> Result<(), MTLError> {
- let gemm = self.get_or_create_gemm()?;
- gemm.encode(mtl, enc, args)
- }
-
- pub fn encode(
- &mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- mut args: MatmulArguments,
- ) -> Result<(), MTLError> {
- self.apply_batch_collapse(&mut args);
-
- if self.maybe_use_gemv(mtl, enc, &args)? {
- if debug_matmul_enabled() {
- self.log_gemv(&args);
- }
- return Ok(());
+ fn get_or_create_gemv(&mut self) -> Result<&mut GemvKernel, MTLError> {
+ if self.gemv.is_none() {
+ self.gemv = Some(GemvKernel::new(self.data_type)?);
}
+ Ok(self.gemv.as_mut().unwrap())
+ }
- if self.maybe_use_splitk(mtl, enc, &args)? {
- if debug_matmul_enabled() {
- self.log_splitk(&args);
- }
- return Ok(());
+ fn get_or_create_splitk(&mut self) -> Result<&mut SplitKGemm, MTLError> {
+ if self.splitk.is_none() {
+ self.splitk = Some(SplitKGemm::new(self.data_type)?);
}
+ Ok(self.splitk.as_mut().unwrap())
+ }
- if debug_matmul_enabled() {
- self.log_gemm(&args, mtl);
+ fn encode_dispatch_descriptor(
+ &mut self,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ arguments: &MatmulArguments,
+ descriptor: &MatmulDispatchDescriptor,
+ ) -> Result<bool, MTLError> {
+ match descriptor {
+ MatmulDispatchDescriptor::Gemv(descriptor) => {
+ let gemv = self.get_or_create_gemv()?;
+ gemv.encode_descriptor(context, encoder, arguments, descriptor)
+ },
+ MatmulDispatchDescriptor::SplitK(descriptor) => {
+ let splitk = self.get_or_create_splitk()?;
+ splitk
+ .encode_descriptor(context, encoder, arguments, descriptor)
+ },
+ MatmulDispatchDescriptor::Gemm(descriptor) => {
+ let gemm = self.get_or_create_gemm()?;
+ gemm.encode_descriptor(context, encoder, arguments, descriptor)
+ },
}
-
- self.encode_gemm(mtl, enc, &args)
}
- pub fn encode_with_bias(
+ pub fn encode(
&mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- mut args: MatmulArguments,
- bias: &MTLBuffer,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ mut arguments: MatmulArguments,
) -> Result<(), MTLError> {
- self.apply_batch_collapse(&mut args);
+ Self::apply_batch_collapse(&mut arguments);
- if self.maybe_use_gemv_with_bias(mtl, enc, &args, bias)? {
- if debug_matmul_enabled() {
- self.log_gemv(&args);
- }
- return Ok(());
- }
+ let descriptor =
+ choose_dispatch_descriptor(context, self.data_type, &arguments)?;
- if self.maybe_use_splitk(mtl, enc, &args)? {
- if debug_matmul_enabled() {
- self.log_splitk(&args);
- }
- self.apply_bias_add(mtl, enc, &args, bias)?;
- return Ok(());
- }
+ let bias_fused = self.encode_dispatch_descriptor(
+ context,
+ encoder,
+ &arguments,
+ &descriptor,
+ )?;
- if debug_matmul_enabled() {
- self.log_gemm(&args, mtl);
+ if let Some(bias) = arguments.bias {
+ if !bias_fused {
+ self.apply_bias_add(context, encoder, &arguments, bias)?;
+ }
}
- self.encode_gemm(mtl, enc, &args)?;
- self.apply_bias_add(mtl, enc, &args, bias)?;
Ok(())
}
fn apply_bias_add(
&mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- bias: &MTLBuffer,
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ arguments: &MatmulArguments,
+ bias: &metal::Buffer,
) -> Result<(), MTLError> {
- let m = args.batch as usize;
- let n = args.output_dim as usize;
- let batch_count = args.batch_count as usize;
+ let m = arguments.batch as usize;
+ let n = arguments.output_dim as usize;
+ let batch_count = arguments.batch_count as usize;
let total_len = m * n * batch_count;
if total_len == 0 {
return Ok(());
@@ -226,80 +132,33 @@ impl MatmulKernel {
if self.bias_add.is_none() {
self.bias_add = Some(TensorAddBias::new(
- mtl,
+ context,
KernelDataType::from(self.data_type),
)?);
}
let bias_add = self.bias_add.as_ref().unwrap();
- bias_add
- .encode_with_encoder(args.d, bias, args.d, n, total_len, enc, None);
+ bias_add.encode_with_encoder(
+ arguments.d,
+ bias,
+ arguments.d,
+ n,
+ total_len,
+ encoder,
+ None,
+ );
Ok(())
}
- fn apply_batch_collapse(
- &self,
- args: &mut MatmulArguments,
- ) {
- if self.lhs_is_transposed {
+ fn apply_batch_collapse(arguments: &mut MatmulArguments) {
+ if arguments.transpose_a {
return;
}
- if args.batch_count <= 1 {
+ if arguments.batch_count <= 1 {
return;
}
- if args.lda == args.input_dim && self.rhs_is_transposed {
- args.batch *= args.batch_count;
- args.batch_count = 1;
- }
- }
-
- fn log_gemv(
- &self,
- args: &MatmulArguments,
- ) {
- eprintln!(
- "[matmul] GEMV m={} k={} n={} batch={} dtype={:?}",
- args.batch,
- args.input_dim,
- args.output_dim,
- args.batch_count,
- self.data_type
- );
- }
-
- fn log_splitk(
- &self,
- args: &MatmulArguments,
- ) {
- eprintln!(
- "[matmul] SplitK m={} k={} n={} batch={} dtype={:?}",
- args.batch,
- args.input_dim,
- args.output_dim,
- args.batch_count,
- self.data_type
- );
- }
-
- fn log_gemm(
- &self,
- args: &MatmulArguments,
- mtl: &MTLContext,
- ) {
- let gemm_ref = self.gemm.as_ref();
- if let Some(gemm) = gemm_ref {
- let tile = gemm.select_tile(mtl, args);
- eprintln!(
- "[matmul] GEMM m={} k={} n={} batch={} dtype={:?} tile={}x{}x{} nax={}",
- args.batch,
- args.input_dim,
- args.output_dim,
- args.batch_count,
- self.data_type,
- tile.block_rows,
- tile.block_cols,
- tile.block_depth,
- tile.is_nax()
- );
+ if arguments.lda == arguments.input_dim && arguments.transpose_b {
+ arguments.batch *= arguments.batch_count;
+ arguments.batch_count = 1;
}
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
index 2ef5b3948..581cc089d 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -1,4 +1,5 @@
pub mod common;
+mod dispatch_descriptor;
mod gemm;
mod gemv;
mod kernel;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs
new file mode 100644
index 000000000..a052e8d66
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/dispatch_descriptor.rs
@@ -0,0 +1,152 @@
+use metal::MTLSize;
+
+use super::{
+ pipeline_configuration::PipelineConfiguration,
+ tile_configuration::select_tile_configuration,
+};
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLContext, MTLError,
+ kernel::matmul::common::{
+ GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments,
+ },
+ },
+};
+
+#[derive(Debug, Clone)]
+pub(crate) struct DispatchDescriptor {
+ pub(crate) pipeline_configuration: PipelineConfiguration,
+ pub(crate) params: SplitKGEMMParams,
+ pub(crate) partition_count: i32,
+ pub(crate) output_elements_per_partition: i32,
+ pub(crate) accumulator_bytes: usize,
+ pub(crate) partial_threadgroups: MTLSize,
+ pub(crate) partial_threads_per_threadgroup: MTLSize,
+ pub(crate) accum_total_threads: MTLSize,
+ pub(crate) accum_threads_per_threadgroup: MTLSize,
+}
+
+impl DispatchDescriptor {
+ pub(crate) fn try_new(
+ _context: &MTLContext,
+ data_type: DataType,
+ arguments: &MatmulArguments,
+ ) -> Result<Option<Self>, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for Split-K: {:?}",
+ data_type
+ )));
+ }
+
+ if arguments.c.is_some() {
+ return Ok(None);
+ }
+
+ let m = arguments.batch;
+ let n = arguments.output_dim;
+ let k = arguments.input_dim;
+ let batch_count = arguments.batch_count;
+
+ if !should_use_splitk(m, n, k, batch_count) {
+ return Ok(None);
+ }
+
+ let tile = select_tile_configuration(m, n);
+ let partition_count = compute_partition_count(k);
+ let mn_aligned =
+ (m % tile.tile_rows) == 0 && (n % tile.tile_cols) == 0;
+ let k_aligned = (k % tile.tile_depth) == 0;
+
+ let pipeline_configuration = PipelineConfiguration {
+ tile,
+ transpose_a: arguments.transpose_a,
+ transpose_b: arguments.transpose_b,
+ mn_aligned,
+ k_aligned,
+ };
+
+ let tile_count_m = (m + tile.tile_rows - 1) / tile.tile_rows;
+ let tile_count_n = (n + tile.tile_cols - 1) / tile.tile_cols;
+
+ let gemm_k_iterations = (k / tile.tile_depth) / partition_count;
+ let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;
+ let output_elements_per_partition = m * n;
+
+ let accumulator_element_count = partition_count
+ * output_elements_per_partition
+ * arguments.batch_count;
+ let accumulator_bytes =
+ (accumulator_element_count as usize) * std::mem::size_of::<f32>();
+
+ let params = SplitKGEMMParams {
+ M: m,
+ N: n,
+ K: k,
+ lda: arguments.lda,
+ ldb: arguments.ldb,
+ ldc: n,
+ tiles_n: tile_count_n,
+ tiles_m: tile_count_m,
+ split_k_partitions: partition_count,
+ split_k_partition_stride: output_elements_per_partition,
+ split_k_partition_size: k_elements_per_partition,
+ gemm_k_iterations_aligned: gemm_k_iterations,
+ };
+
+ let partial_threads_per_threadgroup = MTLSize::new(
+ 32,
+ tile.warps_per_col as u64,
+ tile.warps_per_row as u64,
+ );
+ let partial_threadgroups = MTLSize::new(
+ tile_count_n as u64,
+ tile_count_m as u64,
+ partition_count as u64,
+ );
+
+ let accum_total_threads = MTLSize::new(n as u64, m as u64, 1);
+ let accum_threads_per_threadgroup =
+ MTLSize::new(16.min(n as u64), 16.min(m as u64), 1);
+
+ Ok(Some(Self {
+ pipeline_configuration,
+ params,
+ partition_count,
+ output_elements_per_partition,
+ accumulator_bytes,
+ partial_threadgroups,
+ partial_threads_per_threadgroup,
+ accum_total_threads,
+ accum_threads_per_threadgroup,
+ }))
+ }
+}
+
+fn compute_partition_count(k: i32) -> i32 {
+ let k_tiles = k / 16;
+ if k_tiles < 16 {
+ 2
+ } else if k_tiles < 32 {
+ 4
+ } else if k_tiles < 64 {
+ 8
+ } else {
+ 16
+ }
+}
+
+fn should_use_splitk(m: i32, n: i32, k: i32, batch_count: i32) -> bool {
+ if batch_count != 1 {
+ return false;
+ }
+ if m <= 0 || n <= 0 || k <= 0 {
+ return false;
+ }
+ let m_tiles = m / 16;
+ let n_tiles = n / 16;
+ let k_tiles = k / 16;
+ (m_tiles * n_tiles) <= 32 && k_tiles >= 8
+}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
index 531755fa0..5809da8b6 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/kernel.rs
@@ -1,43 +1,48 @@
use std::collections::HashMap;
use metal::{
- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
- MTLSize,
+ Buffer as MTLBuffer, ComputeCommandEncoderRef,
+ ComputePipelineState as MTLComputePipelineState,
};
use super::{
- pipeline_configuration::PipelineConfiguration,
- tile_configuration::{TileConfiguration, select_tile_configuration},
+ DispatchDescriptor, pipeline_configuration::PipelineConfiguration,
};
use crate::{
DataType,
backends::metal::{
MTLContext, MTLError,
- kernel::matmul::common::{GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments},
+ kernel::matmul::common::{
+ GEMMSpiltKParams as SplitKGEMMParams, MatmulArguments,
+ transpose_configuration,
+ },
},
};
pub struct Kernel {
data_type: DataType,
- transpose_a: bool,
- transpose_b: bool,
partial_pipelines: HashMap<PipelineConfiguration, MTLComputePipelineState>,
accum_pipeline: Option<MTLComputePipelineState>,
+ accumulator_buffer: Option<MTLBuffer>,
+ accumulator_buffer_bytes: usize,
}
impl Kernel {
- pub fn new(
- data_type: DataType,
- transpose_a: bool,
- transpose_b: bool,
- ) -> Self {
- Self {
+ pub fn new(data_type: DataType) -> Result<Self, MTLError> {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32)
+ {
+ return Err(MTLError::Generic(format!(
+ "Unsupported dtype for Split-K: {:?}",
+ data_type
+ )));
+ }
+ Ok(Self {
data_type,
- transpose_a,
- transpose_b,
partial_pipelines: HashMap::new(),
accum_pipeline: None,
- }
+ accumulator_buffer: None,
+ accumulator_buffer_bytes: 0,
+ })
}
fn steel_type_name(&self) -> Result<&'static str, MTLError> {
@@ -56,36 +61,35 @@ impl Kernel {
"float32"
}
- fn transpose_suffix(&self) -> &'static str {
- match (self.transpose_a, self.transpose_b) {
- (false, false) => "nn",
- (false, true) => "nt",
- (true, false) => "tn",
- (true, true) => "tt",
- }
- }
-
fn partial_kernel_name(
&self,
- tile: &TileConfiguration,
- mn_aligned: bool,
- k_aligned: bool,
+ config: &PipelineConfiguration,
) -> Result<String, MTLError> {
let in_name = self.steel_type_name()?;
let out_name = self.splitk_partial_out_name();
- let transpose_suffix = self.transpose_suffix();
- let mn_tag = if mn_aligned { "taligned" } else { "naligned" };
- let k_tag = if k_aligned { "taligned" } else { "naligned" };
+ let transpose_suffix =
+ transpose_configuration(config.transpose_a, config.transpose_b)
+ .as_str();
+ let mn_tag = if config.mn_aligned {
+ "taligned"
+ } else {
+ "naligned"
+ };
+ let k_tag = if config.k_aligned {
+ "taligned"
+ } else {
+ "naligned"
+ };
Ok(format!(
"steel_gemm_splitk_{}_{}_{}_bm{}_bn{}_bk{}_wm{}_wn{}_MN_{}_K_{}",
transpose_suffix,
in_name,
out_name,
- tile.tile_rows,
- tile.tile_cols,
- tile.tile_depth,
- tile.warps_per_row,
- tile.warps_per_col,
+ config.tile.tile_rows,
+ config.tile.tile_cols,
+ config.tile.tile_depth,
+ config.tile.warps_per_row,
+ config.tile.warps_per_col,
mn_tag,
k_tag,
))
@@ -106,8 +110,7 @@ impl Kernel {
config: &PipelineConfiguration,
) -> Result<&MTLComputePipelineState, MTLError> {
if !self.partial_pipelines.contains_key(config) {
- let name =
- self.partial_kernel_name(&config.tile, config.mn_aligned, config.k_aligned)?;
+ let name = self.partial_kernel_name(config)?;
let ps = mtl.compute_pipeline_state(&name, None)?;
self.partial_pipelines.insert(config.clone(), ps);
}
@@ -126,19 +129,6 @@ impl Kernel {
Ok(self.accum_pipeline.as_ref().unwrap())
}
- fn compute_partition_count(k: i32) -> i32 {
- let k_tiles = k / 16;
- if k_tiles < 16 {
- 2
- } else if k_tiles < 32 {
- 4
- } else if k_tiles < 64 {
- 8
- } else {
- 16
- }
- }
-
pub fn should_use_splitk(
m: i32,
n: i32,
@@ -157,108 +147,86 @@ impl Kernel {
(m_tiles * n_tiles) <= 32 && k_tiles >= 8
}
- pub fn encode(
+ pub(crate) fn encode_descriptor(
&mut self,
- mtl: &MTLContext,
- enc: &ComputeCommandEncoderRef,
- args: &MatmulArguments,
- ) -> Result<(), MTLError> {
- let m = args.batch;
- let n = args.output_dim;
- let k = args.input_dim;
-
- let tile = select_tile_configuration(m, n);
- let partition_count = Self::compute_partition_count(k);
- let mn_aligned =
- (m % tile.tile_rows) == 0 && (n % tile.tile_cols) == 0;
- let k_aligned = (k % tile.tile_depth) == 0;
-
- let config = PipelineConfiguration {
- tile,
- mn_aligned,
- k_aligned,
- };
-
- let tile_count_m = (m + tile.tile_rows - 1) / tile.tile_rows;
- let tile_count_n = (n + tile.tile_cols - 1) / tile.tile_cols;
-
- let gemm_k_iterations = (k / tile.tile_depth) / partition_count;
- let k_elements_per_partition = gemm_k_iterations * tile.tile_depth;
- let output_elements_per_partition = m * n;
-
- let accumulator_element_count =
- partition_count * output_elements_per_partition * args.batch_count;
- let accumulator_bytes =
- (accumulator_element_count as usize) * std::mem::size_of::<f32>();
- let accumulator_buffer = mtl.device.new_buffer(
- accumulator_bytes as u64,
- metal::MTLResourceOptions::StorageModePrivate,
- );
-
- let params = SplitKGEMMParams {
- M: m,
- N: n,
- K: k,
- lda: args.lda,
- ldb: args.ldb,
- ldc: n,
- tiles_n: tile_count_n,
- tiles_m: tile_count_m,
- split_k_partitions: partition_count,
- split_k_partition_stride: output_elements_per_partition,
- split_k_partition_size: k_elements_per_partition,
- gemm_k_iterations_aligned: gemm_k_iterations,
- };
-
- let partial_ps = self.get_partial_pipeline(mtl, &config)?;
-
- enc.set_compute_pipeline_state(partial_ps);
- enc.set_buffer(0, Some(args.a), args.a_offset);
- enc.set_buffer(1, Some(args.b), 0);
- enc.set_buffer(2, Some(&accumulator_buffer), 0);
- enc.set_bytes(
+ context: &MTLContext,
+ encoder: &ComputeCommandEncoderRef,
+ arguments: &MatmulArguments,
+ descriptor: &DispatchDescriptor,
+ ) -> Result<bool, MTLError> {
+ self.ensure_accumulator_buffer(context, descriptor.accumulator_bytes);
+ let accumulator_buffer = self
+ .accumulator_buffer
+ .as_ref()
+ .cloned()
+ .expect("Accumulator buffer must be initialized");
+
+ let partial_pipeline_state = self.get_partial_pipeline(
+ context,
+ &descriptor.pipeline_configuration,
+ )?;
+
+ encoder.set_compute_pipeline_state(partial_pipeline_state);
+ encoder.set_buffer(0, Some(arguments.a), arguments.a_offset);
+ encoder.set_buffer(1, Some(arguments.b), 0);
+ encoder.set_buffer(2, Some(&accumulator_buffer), 0);
+ encoder.set_bytes(
3,
std::mem::size_of::<SplitKGEMMParams>() as u64,
- &params as *const SplitKGEMMParams as *const _,
+ &descriptor.params as *const _ as *const _,
);
-
- let threads_per_threadgroup = MTLSize::new(
- 32,
- tile.warps_per_col as u64,
- tile.warps_per_row as u64,
+ encoder.dispatch_thread_groups(
+ descriptor.partial_threadgroups,
+ descriptor.partial_threads_per_threadgroup,
);
- let threadgroups = MTLSize::new(
- tile_count_n as u64,
- tile_count_m as u64,
- partition_count as u64,
- );
- enc.dispatch_thread_groups(threadgroups, threads_per_threadgroup);
- let accum_ps = self.get_accum_pipeline(mtl)?;
- enc.set_compute_pipeline_state(accum_ps);
- enc.set_buffer(0, Some(&accumulator_buffer), 0);
- enc.set_buffer(1, Some(args.d), 0);
- enc.set_bytes(
+ let accum_pipeline_state = self.get_accum_pipeline(context)?;
+ encoder.set_compute_pipeline_state(accum_pipeline_state);
+ encoder.set_buffer(0, Some(&accumulator_buffer), 0);
+ encoder.set_buffer(1, Some(arguments.d), 0);
+
+ let partition_count = descriptor.partition_count;
+ let output_elements_per_partition =
+ descriptor.output_elements_per_partition;
+ encoder.set_bytes(
2,
- 4,
+ std::mem::size_of::<i32>() as u64,
&partition_count as *const i32 as *const std::ffi::c_void,
);
- enc.set_bytes(
+ encoder.set_bytes(
3,
- 4,
- &output_elements_per_partition as *const i32 as *const std::ffi::c_void,
+ std::mem::size_of::<i32>() as u64,
+ &output_elements_per_partition as *const i32
+ as *const std::ffi::c_void,
);
- enc.set_bytes(
- 4,
+ encoder.set_bytes(
4,
- &(args.ldd) as *const i32 as *const std::ffi::c_void,
+ std::mem::size_of::<i32>() as u64,
+ &(arguments.ldd) as *const i32 as *const std::ffi::c_void,
+ );
+
+ encoder.dispatch_threads(
+ descriptor.accum_total_threads,
+ descriptor.accum_threads_per_threadgroup,
);
- let accum_total_threads = MTLSize::new(n as u64, m as u64, 1);
- let accum_threads_per_threadgroup =
- MTLSize::new(16.min(n as u64), 16.min(m as u64), 1);
- enc.dispatch_threads(accum_total_threads, accum_threads_per_threadgroup);
+ Ok(false)
+ }
- Ok(())
+ fn ensure_accumulator_buffer(
+ &mut self,
+ mtl: &MTLContext,
+ required_bytes: usize,
+ ) {
+ if required_bytes <= self.accumulator_buffer_bytes
+ && self.accumulator_buffer.is_some()
+ {
+ return;
+ }
+ self.accumulator_buffer = Some(mtl.device.new_buffer(
+ required_bytes as u64,
+ metal::MTLResourceOptions::StorageModePrivate,
+ ));
+ self.accumulator_buffer_bytes = required_bytes;
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
index 54a1730c2..22383d1f8 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/mod.rs
@@ -1,5 +1,7 @@
+mod dispatch_descriptor;
mod kernel;
mod pipeline_configuration;
mod tile_configuration;
pub use kernel::Kernel as SplitKGemm;
+pub(crate) use dispatch_descriptor::DispatchDescriptor;
diff --git a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs b/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs
index b745b347a..1eab77254 100644
--- a/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs
+++ b/crates/uzu/src/backends/metal/kernel/matmul/split_k/pipeline_configuration.rs
@@ -3,6 +3,8 @@ use super::tile_configuration::TileConfiguration;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct PipelineConfiguration {
pub tile: TileConfiguration,
+ pub transpose_a: bool,
+ pub transpose_b: bool,
pub mn_aligned: bool,
pub k_aligned: bool,
}
diff --git a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
index 1698da06d..75d71378d 100644
--- a/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/quant_matmul/mod.rs
@@ -199,20 +199,6 @@ impl QuantizedMatmulKernel {
}
})?;
- if crate::utils::env_utils::debug_matmul_enabled() {
- eprintln!(
- "[qmm] {:?} m={} k={} n={} quant={:?} block={}x{} transposed={}",
- variant,
- args.batch,
- args.input_dim,
- args.output_dim,
- args.quantization_type,
- bm,
- bn,
- self.weights_transposed
- );
- }
-
encoder.set_compute_pipeline_state(pipeline);
// Set buffers
diff --git a/crates/uzu/src/utils/env_utils.rs b/crates/uzu/src/utils/env_utils.rs
index f87bf1211..27593aef8 100644
--- a/crates/uzu/src/utils/env_utils.rs
+++ b/crates/uzu/src/utils/env_utils.rs
@@ -3,7 +3,6 @@ pub enum MetalEnvVar {
DeviceWrapperType,
CaptureFirstDecode,
CaptureFirstPrefill,
- DebugMatmul,
}
impl MetalEnvVar {
@@ -12,7 +11,6 @@ impl MetalEnvVar {
MetalEnvVar::DeviceWrapperType => "METAL_DEVICE_WRAPPER_TYPE",
MetalEnvVar::CaptureFirstDecode => "UZU_CAPTURE_FIRST_DECODE",
MetalEnvVar::CaptureFirstPrefill => "UZU_CAPTURE_FIRST_PREFILL",
- MetalEnvVar::DebugMatmul => "UZU_DEBUG_MATMUL",
}
}
@@ -25,11 +23,3 @@ impl MetalEnvVar {
matches!(upper.as_str(), "1" | "YES" | "TRUE")
}
}
-
-use std::sync::OnceLock;
-
-static DEBUG_MATMUL_ENABLED: OnceLock<bool> = OnceLock::new();
-
-pub fn debug_matmul_enabled() -> bool {
- *DEBUG_MATMUL_ENABLED.get_or_init(|| MetalEnvVar::DebugMatmul.is_enabled())
-}
diff --git a/crates/uzu/tests/matmul_correctness_test.rs b/crates/uzu/tests/matmul_correctness_test.rs
index 678504e49..98ff7029d 100644
--- a/crates/uzu/tests/matmul_correctness_test.rs
+++ b/crates/uzu/tests/matmul_correctness_test.rs
@@ -72,6 +72,7 @@ fn run_metal_matmul(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.expect("encode");
enc.end_encoding();
diff --git a/crates/uzu/tests/mlp_fused_performance_test.rs b/crates/uzu/tests/mlp_fused_performance_test.rs
index 99cc81695..317fa812e 100644
--- a/crates/uzu/tests/mlp_fused_performance_test.rs
+++ b/crates/uzu/tests/mlp_fused_performance_test.rs
@@ -151,6 +151,7 @@ fn benchmark_gemv_fused(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.unwrap();
encoder.end_encoding();
@@ -184,6 +185,7 @@ fn benchmark_gemv_fused(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.unwrap();
encoder.end_encoding();
@@ -300,6 +302,7 @@ fn benchmark_gemm_fused(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.unwrap();
encoder.end_encoding();
@@ -332,6 +335,7 @@ fn benchmark_gemm_fused(
alpha: 1.0,
beta: 0.0,
},
+ None,
)
.unwrap();
encoder.end_encoding();