Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

update bench

Eugene Bokhane66a79c53efd
diff --git a/crates/benchmarks/src/mlp_fused_bench.rs b/crates/benchmarks/src/mlp_fused_bench.rs
index d96ca425f..5759c592f 100644
--- a/crates/benchmarks/src/mlp_fused_bench.rs
+++ b/crates/benchmarks/src/mlp_fused_bench.rs
@@ -26,313 +26,323 @@ fn create_test_context() -> Option<MTLContext> {
MTLContext::new(device, command_queue).ok()
}
-struct BenchResult {
- fused_time_us: f64,
- unfused_time_us: f64,
+struct BenchmarkResult {
+ fused_time_microseconds: f64,
+ unfused_time_microseconds: f64,
speedup: f64,
}
-impl BenchResult {
+impl BenchmarkResult {
fn new(
- fused: Duration,
- unfused: Duration,
+ fused_duration: Duration,
+ unfused_duration: Duration,
) -> Self {
- let fused_us = fused.as_secs_f64() * 1_000_000.0;
- let unfused_us = unfused.as_secs_f64() * 1_000_000.0;
+ let fused_microseconds = fused_duration.as_secs_f64() * 1_000_000.0;
+ let unfused_microseconds = unfused_duration.as_secs_f64() * 1_000_000.0;
Self {
- fused_time_us: fused_us,
- unfused_time_us: unfused_us,
- speedup: unfused_us / fused_us,
+ fused_time_microseconds: fused_microseconds,
+ unfused_time_microseconds: unfused_microseconds,
+ speedup: unfused_microseconds / fused_microseconds,
}
}
}
-fn bench_gemv_fused(
- ctx: &MTLContext,
- k: usize,
- hidden_dim: usize,
- iterations: usize,
- warmup: usize,
-) -> BenchResult {
+fn benchmark_gemv_fused(
+ context: &MTLContext,
+ input_dimension: usize,
+ hidden_dimension: usize,
+ iteration_count: usize,
+ warmup_count: usize,
+) -> BenchmarkResult {
// Allocate buffers
- let input_buf = ctx.device.new_buffer(
- (k * std::mem::size_of::<f16>()) as u64,
+ let input_buffer = context.device.new_buffer(
+ (input_dimension * std::mem::size_of::<f16>()) as u64,
MTLResourceOptions::StorageModeShared,
);
- let weights_buf = ctx.device.new_buffer(
- (2 * hidden_dim * k * std::mem::size_of::<f16>()) as u64,
+ let weights_buffer = context.device.new_buffer(
+ (2 * hidden_dimension * input_dimension * std::mem::size_of::<f16>())
+ as u64,
MTLResourceOptions::StorageModeShared,
);
- let output_buf = ctx.device.new_buffer(
- (hidden_dim * std::mem::size_of::<f16>()) as u64,
+ let output_buffer = context.device.new_buffer(
+ (hidden_dimension * std::mem::size_of::<f16>()) as u64,
MTLResourceOptions::StorageModeShared,
);
- let fused_up_buf = ctx.device.new_buffer(
- (2 * hidden_dim * std::mem::size_of::<f16>()) as u64,
+ let fused_up_buffer = context.device.new_buffer(
+ (2 * hidden_dimension * std::mem::size_of::<f16>()) as u64,
MTLResourceOptions::StorageModeShared,
);
let mut fused_kernel =
MlpFusedGemvKernel::new(DataType::F16).expect("fused kernel");
let mut matmul_kernel =
- MatmulKernel::new(ctx, DataType::F16, false, true).expect("matmul");
+ MatmulKernel::new(context, DataType::F16, false, true).expect("matmul");
// Warmup fused
- for _ in 0..warmup {
- let cb = ctx.command_queue.new_command_buffer().to_owned();
- let enc = cb.new_compute_command_encoder();
- let args = MlpFusedGemvArguments {
- weights: &weights_buf,
- input: &input_buf,
+ for _ in 0..warmup_count {
+ let command_buffer =
+ context.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
+ let arguments = MlpFusedGemvArguments {
+ weights: &weights_buffer,
+ input: &input_buffer,
input_offset: 0,
- output: &output_buf,
- input_dim: k as i32,
- hidden_dim: hidden_dim as i32,
- weights_ld: k as i32,
+ output: &output_buffer,
+ input_dim: input_dimension as i32,
+ hidden_dim: hidden_dimension as i32,
+ weights_ld: input_dimension as i32,
batch_count: 1,
activation: MlpActivationType::SiLU,
};
- fused_kernel.encode(ctx, &enc, &args).unwrap();
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ fused_kernel.encode(context, &encoder, &arguments).unwrap();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
// Benchmark fused
let fused_start = Instant::now();
- for _ in 0..iterations {
- let cb = ctx.command_queue.new_command_buffer().to_owned();
- let enc = cb.new_compute_command_encoder();
- let args = MlpFusedGemvArguments {
- weights: &weights_buf,
- input: &input_buf,
+ for _ in 0..iteration_count {
+ let command_buffer =
+ context.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
+ let arguments = MlpFusedGemvArguments {
+ weights: &weights_buffer,
+ input: &input_buffer,
input_offset: 0,
- output: &output_buf,
- input_dim: k as i32,
- hidden_dim: hidden_dim as i32,
- weights_ld: k as i32,
+ output: &output_buffer,
+ input_dim: input_dimension as i32,
+ hidden_dim: hidden_dimension as i32,
+ weights_ld: input_dimension as i32,
batch_count: 1,
activation: MlpActivationType::SiLU,
};
- fused_kernel.encode(ctx, &enc, &args).unwrap();
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ fused_kernel.encode(context, &encoder, &arguments).unwrap();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
- let fused_time = fused_start.elapsed() / iterations as u32;
+ let fused_duration = fused_start.elapsed() / iteration_count as u32;
// Warmup unfused (matmul only - simplified comparison)
- for _ in 0..warmup {
- let cb = ctx.command_queue.new_command_buffer().to_owned();
- let enc = cb.new_compute_command_encoder();
+ for _ in 0..warmup_count {
+ let command_buffer =
+ context.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
matmul_kernel
.encode(
- ctx,
- &enc,
+ context,
+ &encoder,
MatmulArguments {
- a: &input_buf,
+ a: &input_buffer,
a_offset: 0,
- b: &weights_buf,
+ b: &weights_buffer,
c: None,
- d: &fused_up_buf,
+ d: &fused_up_buffer,
batch: 1,
- input_dim: k as i32,
- output_dim: (2 * hidden_dim) as i32,
- lda: k as i32,
- ldb: k as i32,
- ldd: (2 * hidden_dim) as i32,
+ input_dim: input_dimension as i32,
+ output_dim: (2 * hidden_dimension) as i32,
+ lda: input_dimension as i32,
+ ldb: input_dimension as i32,
+ ldd: (2 * hidden_dimension) as i32,
batch_count: 1,
alpha: 1.0,
beta: 0.0,
},
)
.unwrap();
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
// Benchmark unfused (matmul + separate activation would add more time)
let unfused_start = Instant::now();
- for _ in 0..iterations {
- let cb = ctx.command_queue.new_command_buffer().to_owned();
- let enc = cb.new_compute_command_encoder();
+ for _ in 0..iteration_count {
+ let command_buffer =
+ context.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
matmul_kernel
.encode(
- ctx,
- &enc,
+ context,
+ &encoder,
MatmulArguments {
- a: &input_buf,
+ a: &input_buffer,
a_offset: 0,
- b: &weights_buf,
+ b: &weights_buffer,
c: None,
- d: &fused_up_buf,
+ d: &fused_up_buffer,
batch: 1,
- input_dim: k as i32,
- output_dim: (2 * hidden_dim) as i32,
- lda: k as i32,
- ldb: k as i32,
- ldd: (2 * hidden_dim) as i32,
+ input_dim: input_dimension as i32,
+ output_dim: (2 * hidden_dimension) as i32,
+ lda: input_dimension as i32,
+ ldb: input_dimension as i32,
+ ldd: (2 * hidden_dimension) as i32,
batch_count: 1,
alpha: 1.0,
beta: 0.0,
},
)
.unwrap();
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
- let unfused_time = unfused_start.elapsed() / iterations as u32;
+ let unfused_duration = unfused_start.elapsed() / iteration_count as u32;
- BenchResult::new(fused_time, unfused_time)
+ BenchmarkResult::new(fused_duration, unfused_duration)
}
-fn bench_gemm_fused(
- ctx: &MTLContext,
- m: usize,
- k: usize,
- hidden_dim: usize,
- iterations: usize,
- warmup: usize,
-) -> BenchResult {
- let input_buf = ctx.device.new_buffer(
- (m * k * std::mem::size_of::<f16>()) as u64,
+fn benchmark_gemm_fused(
+ context: &MTLContext,
+ batch_size: usize,
+ input_dimension: usize,
+ hidden_dimension: usize,
+ iteration_count: usize,
+ warmup_count: usize,
+) -> BenchmarkResult {
+ let input_buffer = context.device.new_buffer(
+ (batch_size * input_dimension * std::mem::size_of::<f16>()) as u64,
MTLResourceOptions::StorageModeShared,
);
- let weights_buf = ctx.device.new_buffer(
- (2 * hidden_dim * k * std::mem::size_of::<f16>()) as u64,
+ let weights_buffer = context.device.new_buffer(
+ (2 * hidden_dimension * input_dimension * std::mem::size_of::<f16>())
+ as u64,
MTLResourceOptions::StorageModeShared,
);
- let output_buf = ctx.device.new_buffer(
- (m * hidden_dim * std::mem::size_of::<f16>()) as u64,
+ let output_buffer = context.device.new_buffer(
+ (batch_size * hidden_dimension * std::mem::size_of::<f16>()) as u64,
MTLResourceOptions::StorageModeShared,
);
- let fused_up_buf = ctx.device.new_buffer(
- (m * 2 * hidden_dim * std::mem::size_of::<f16>()) as u64,
+ let fused_up_buffer = context.device.new_buffer(
+ (batch_size * 2 * hidden_dimension * std::mem::size_of::<f16>()) as u64,
MTLResourceOptions::StorageModeShared,
);
let mut fused_kernel =
MlpFusedGemmKernel::new(DataType::F16, true).expect("fused kernel");
let mut matmul_kernel =
- MatmulKernel::new(ctx, DataType::F16, false, true).expect("matmul");
+ MatmulKernel::new(context, DataType::F16, false, true).expect("matmul");
// Warmup fused
- for _ in 0..warmup {
- let cb = ctx.command_queue.new_command_buffer().to_owned();
- let enc = cb.new_compute_command_encoder();
- let args = MlpFusedGemmArguments {
- input: &input_buf,
+ for _ in 0..warmup_count {
+ let command_buffer =
+ context.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
+ let arguments = MlpFusedGemmArguments {
+ input: &input_buffer,
input_offset: 0,
- weights: &weights_buf,
- output: &output_buf,
- batch: m as i32,
- input_dim: k as i32,
- hidden_dim: hidden_dim as i32,
- lda: k as i32,
- ldb: k as i32,
- ldd: hidden_dim as i32,
+ weights: &weights_buffer,
+ output: &output_buffer,
+ batch: batch_size as i32,
+ input_dim: input_dimension as i32,
+ hidden_dim: hidden_dimension as i32,
+ lda: input_dimension as i32,
+ ldb: input_dimension as i32,
+ ldd: hidden_dimension as i32,
activation: MlpActivationType::SiLU,
};
- fused_kernel.encode(ctx, &enc, &args).unwrap();
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ fused_kernel.encode(context, &encoder, &arguments).unwrap();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
let fused_start = Instant::now();
- for _ in 0..iterations {
- let cb = ctx.command_queue.new_command_buffer().to_owned();
- let enc = cb.new_compute_command_encoder();
- let args = MlpFusedGemmArguments {
- input: &input_buf,
+ for _ in 0..iteration_count {
+ let command_buffer =
+ context.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
+ let arguments = MlpFusedGemmArguments {
+ input: &input_buffer,
input_offset: 0,
- weights: &weights_buf,
- output: &output_buf,
- batch: m as i32,
- input_dim: k as i32,
- hidden_dim: hidden_dim as i32,
- lda: k as i32,
- ldb: k as i32,
- ldd: hidden_dim as i32,
+ weights: &weights_buffer,
+ output: &output_buffer,
+ batch: batch_size as i32,
+ input_dim: input_dimension as i32,
+ hidden_dim: hidden_dimension as i32,
+ lda: input_dimension as i32,
+ ldb: input_dimension as i32,
+ ldd: hidden_dimension as i32,
activation: MlpActivationType::SiLU,
};
- fused_kernel.encode(ctx, &enc, &args).unwrap();
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ fused_kernel.encode(context, &encoder, &arguments).unwrap();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
- let fused_time = fused_start.elapsed() / iterations as u32;
+ let fused_duration = fused_start.elapsed() / iteration_count as u32;
// Warmup unfused
- for _ in 0..warmup {
- let cb = ctx.command_queue.new_command_buffer().to_owned();
- let enc = cb.new_compute_command_encoder();
+ for _ in 0..warmup_count {
+ let command_buffer =
+ context.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
matmul_kernel
.encode(
- ctx,
- &enc,
+ context,
+ &encoder,
MatmulArguments {
- a: &input_buf,
+ a: &input_buffer,
a_offset: 0,
- b: &weights_buf,
+ b: &weights_buffer,
c: None,
- d: &fused_up_buf,
- batch: m as i32,
- input_dim: k as i32,
- output_dim: (2 * hidden_dim) as i32,
- lda: k as i32,
- ldb: k as i32,
- ldd: (2 * hidden_dim) as i32,
+ d: &fused_up_buffer,
+ batch: batch_size as i32,
+ input_dim: input_dimension as i32,
+ output_dim: (2 * hidden_dimension) as i32,
+ lda: input_dimension as i32,
+ ldb: input_dimension as i32,
+ ldd: (2 * hidden_dimension) as i32,
batch_count: 1,
alpha: 1.0,
beta: 0.0,
},
)
.unwrap();
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
let unfused_start = Instant::now();
- for _ in 0..iterations {
- let cb = ctx.command_queue.new_command_buffer().to_owned();
- let enc = cb.new_compute_command_encoder();
+ for _ in 0..iteration_count {
+ let command_buffer =
+ context.command_queue.new_command_buffer().to_owned();
+ let encoder = command_buffer.new_compute_command_encoder();
matmul_kernel
.encode(
- ctx,
- &enc,
+ context,
+ &encoder,
MatmulArguments {
- a: &input_buf,
+ a: &input_buffer,
a_offset: 0,
- b: &weights_buf,
+ b: &weights_buffer,
c: None,
- d: &fused_up_buf,
- batch: m as i32,
- input_dim: k as i32,
- output_dim: (2 * hidden_dim) as i32,
- lda: k as i32,
- ldb: k as i32,
- ldd: (2 * hidden_dim) as i32,
+ d: &fused_up_buffer,
+ batch: batch_size as i32,
+ input_dim: input_dimension as i32,
+ output_dim: (2 * hidden_dimension) as i32,
+ lda: input_dimension as i32,
+ ldb: input_dimension as i32,
+ ldd: (2 * hidden_dimension) as i32,
batch_count: 1,
alpha: 1.0,
beta: 0.0,
},
)
.unwrap();
- enc.end_encoding();
- cb.commit();
- cb.wait_until_completed();
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
}
- let unfused_time = unfused_start.elapsed() / iterations as u32;
+ let unfused_duration = unfused_start.elapsed() / iteration_count as u32;
- BenchResult::new(fused_time, unfused_time)
+ BenchmarkResult::new(fused_duration, unfused_duration)
}
/// MLP shapes from real model configs
-/// Format: (K=model_dim, H=hidden_dim)
+/// Format: (input_dimension, hidden_dimension)
const MLP_SHAPES: &[(usize, usize)] = &[
(896, 4864), // Qwen2.5-Coder-0.5B
(1024, 3072), // Qwen3-0.6B
@@ -358,79 +368,65 @@ const MLP_SHAPES: &[(usize, usize)] = &[
(6144, 16384), // Codestral-22B
];
+/// Batch sizes to test: decode (1) and prefill sizes
+const BATCH_SIZES: &[usize] = &[1, 8, 16, 32, 64, 128, 256, 512];
+
pub fn run_mlp_fused_benchmark() {
autoreleasepool(|_| {
- let ctx = match create_test_context() {
- Some(c) => c,
+ let context = match create_test_context() {
+ Some(context) => context,
None => {
eprintln!("No Metal device available");
return;
},
};
- println!("\n=== MLP Fused Kernel Benchmark (Real Model Shapes) ===\n");
-
- let iterations = 100;
- let warmup = 10;
+ let iteration_count = 100;
+ let warmup_count = 10;
- // GEMV benchmark (decode path, M=1)
- println!("=== GEMV (Decode, M=1) ===");
println!(
- "{:<25} {:>12} {:>12} {:>10}",
- "Shape (K, H)", "Fused (µs)", "Unfused (µs)", "Speedup"
+ "{:>12} {:>12} {:>12} {:>12} {:>12} {:>10}",
+ "batch_size",
+ "input_dim",
+ "hidden_dim",
+ "Fused (µs)",
+ "Unfused (µs)",
+ "Speedup"
);
- println!("{}", "-".repeat(61));
-
- for &(k, hidden_dim) in MLP_SHAPES {
- let result =
- bench_gemv_fused(&ctx, k, hidden_dim, iterations, warmup);
- let speedup_str = if result.speedup >= 1.0 {
- format!("{:.2}x", result.speedup)
- } else {
- format!("{:.2}x", result.speedup)
- };
- println!(
- "({:>4}, {:>5}) {:>12.1} {:>12.1} {:>10}",
- k,
- hidden_dim,
- result.fused_time_us,
- result.unfused_time_us,
- speedup_str
- );
- }
-
- println!();
-
- // GEMM benchmark (prefill path, M=32)
- println!("=== GEMM (Prefill, M=32) ===");
- println!(
- "{:<25} {:>12} {:>12} {:>10}",
- "Shape (K, H)", "Fused (µs)", "Unfused (µs)", "Speedup"
- );
- println!("{}", "-".repeat(61));
-
- for &(k, hidden_dim) in MLP_SHAPES {
- let result =
- bench_gemm_fused(&ctx, 32, k, hidden_dim, iterations, warmup);
- let speedup_str = if result.speedup >= 1.0 {
- format!("{:.2}x", result.speedup)
- } else {
- format!("{:.2}x", result.speedup)
- };
- println!(
- "({:>4}, {:>5}) {:>12.1} {:>12.1} {:>10}",
- k,
- hidden_dim,
- result.fused_time_us,
- result.unfused_time_us,
- speedup_str
- );
+ println!("{}", "-".repeat(74));
+
+ for &batch_size in BATCH_SIZES {
+ for &(input_dimension, hidden_dimension) in MLP_SHAPES {
+ let result = if batch_size == 1 {
+ benchmark_gemv_fused(
+ &context,
+ input_dimension,
+ hidden_dimension,
+ iteration_count,
+ warmup_count,
+ )
+ } else {
+ benchmark_gemm_fused(
+ &context,
+ batch_size,
+ input_dimension,
+ hidden_dimension,
+ iteration_count,
+ warmup_count,
+ )
+ };
+ let speedup_string = format!("{:.2}x", result.speedup);
+ println!(
+ "{:>12} {:>12} {:>12} {:>12.1} {:>12.1} {:>10}",
+ batch_size,
+ input_dimension,
+ hidden_dimension,
+ result.fused_time_microseconds,
+ result.unfused_time_microseconds,
+ speedup_string
+ );
+ }
}
-
- println!("\nNote: Unfused time excludes activation kernel overhead.");
- println!(
- " Actual unfused path would be slower due to activation dispatch.\n"
- );
});
}
@@ -442,8 +438,8 @@ mod tests {
fn test_benchmark_runs() {
// Just verify the benchmark doesn't crash
autoreleasepool(|_| {
- if let Some(ctx) = create_test_context() {
- let _ = bench_gemv_fused(&ctx, 512, 256, 5, 2);
+ if let Some(context) = create_test_context() {
+ let _ = benchmark_gemv_fused(&context, 512, 256, 5, 2);
}
});
}
diff --git a/crates/uzu/src/backends/metal/classifier_context.rs b/crates/uzu/src/backends/metal/classifier_context.rs
index 567507df0..d3909de1d 100644
--- a/crates/uzu/src/backends/metal/classifier_context.rs
+++ b/crates/uzu/src/backends/metal/classifier_context.rs
@@ -198,7 +198,7 @@ impl ClassifierContext {
})?;
Ok(ClassifierLayer::new(
- &mtl_context,
+ mtl_context.clone(),
layer_config,
compilation_config.clone(),
layer_index,
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
index 959ed3dcc..260cd9f9a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
@@ -38,7 +38,7 @@ pub struct ClassifierLayer {
impl ClassifierLayer {
pub fn new(
- mtl_context: &MTLContext,
+ mtl_context: Rc<MTLContext>,
layer_config: &TransformerLayerConfig,
_compilation_config: Rc<CompilationConfig>,
layer_index: usize,
@@ -52,6 +52,7 @@ impl ClassifierLayer {
rope: Rc<Box<dyn EncodableBlock>>,
) -> Self {
autoreleasepool(|_| {
+ let ctx = &*mtl_context; // Reference for functions expecting &MTLContext
let attention_config = layer_config
.mixer_config
.as_attention()
@@ -65,7 +66,7 @@ impl ClassifierLayer {
let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock> = Box::new(
TensorCopy::new(
- mtl_context,
+ ctx,
kernel_data_type,
vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
)
@@ -79,7 +80,7 @@ impl ClassifierLayer {
if layer_loader.subtree("pre_mixer_norm").is_ok() {
Some(Box::new(
Normalization::new(
- mtl_context,
+ ctx,
intermediate_data_type,
norm_config.clone(),
ArrayId::Main,
@@ -108,7 +109,7 @@ impl ClassifierLayer {
num_groups * head_dim,
num_groups * head_dim,
],
- mtl_context,
+ ctx,
&layer_loader.subtree("mixer.qkv_projection").unwrap(),
ArrayId::Main,
ArrayId::QKV,
@@ -121,7 +122,7 @@ impl ClassifierLayer {
|| attention_config.key_norm_config.is_some()
{
match QKNorm::new(
- mtl_context,
+ ctx,
intermediate_data_type,
attention_config.query_norm_config.clone(),
attention_config.key_norm_config.clone(),
@@ -146,7 +147,7 @@ impl ClassifierLayer {
attention_config.has_out_biases,
num_heads * head_dim,
[model_dim],
- mtl_context,
+ ctx,
&layer_loader.subtree("mixer.out_projection").unwrap(),
ArrayId::AttentionOutput,
ArrayId::Main,
@@ -159,7 +160,7 @@ impl ClassifierLayer {
{
Some(Box::new(
Normalization::new(
- mtl_context,
+ ctx,
intermediate_data_type,
norm_config.clone(),
ArrayId::Main,
@@ -174,7 +175,7 @@ impl ClassifierLayer {
let mixer_residual_add: Box<dyn EncodableBlock> = Box::new(
TensorAddSwap::new(
- mtl_context,
+ ctx,
kernel_data_type,
vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
)
@@ -183,7 +184,7 @@ impl ClassifierLayer {
let copy_main_to_shortcut_mlp: Box<dyn EncodableBlock> = Box::new(
TensorCopy::new(
- mtl_context,
+ ctx,
kernel_data_type,
vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
)
@@ -192,7 +193,7 @@ impl ClassifierLayer {
let pre_mlp_norm: Box<dyn EncodableBlock> = Box::new(
Normalization::new(
- mtl_context,
+ ctx,
intermediate_data_type,
layer_config.pre_mlp_norm_config.clone(),
ArrayId::Main,
@@ -206,7 +207,7 @@ impl ClassifierLayer {
&layer_config.mlp_config,
model_dim,
hidden_dim,
- mtl_context,
+ ctx,
&layer_loader.subtree("mlp").unwrap(),
)
.expect("Failed to create mlp block");
@@ -215,7 +216,7 @@ impl ClassifierLayer {
if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(Box::new(
Normalization::new(
- mtl_context,
+ ctx,
intermediate_data_type,
norm_config.clone(),
ArrayId::Main,
@@ -230,7 +231,7 @@ impl ClassifierLayer {
let attention: Box<dyn EncodableBlock> = Box::new(
Attention::new(
- mtl_context,
+ ctx,
kernel_data_type,
layer_index,
attention_scale,
@@ -243,7 +244,7 @@ impl ClassifierLayer {
let mlp_residual_add: Box<dyn EncodableBlock> = Box::new(
TensorAddSwap::new(
- mtl_context,
+ ctx,
kernel_data_type,
vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
)
diff --git a/crates/uzu/src/backends/metal/encodable_block/decoder.rs b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
index 349f0713b..655ba2fb1 100644
--- a/crates/uzu/src/backends/metal/encodable_block/decoder.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/decoder.rs
@@ -129,7 +129,7 @@ impl Decoder {
.unwrap();
LayerExecutables::new(
- &mtl_context,
+ mtl_context.clone(),
layer_config,
layer_type,
compilation_config.clone(),
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
index 26b58bfe4..19fc3477f 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -40,7 +40,7 @@ pub struct LayerExecutables {
impl LayerExecutables {
#[allow(clippy::too_many_arguments)]
pub fn new(
- mtl_context: &MTLContext,
+ mtl_context: Rc<MTLContext>,
layer_config: &DecoderLayerConfig,
layer_type: &DecoderLayerType,
compilation_config: Rc<CompilationConfig>,
@@ -55,6 +55,7 @@ impl LayerExecutables {
rope: Option<Rc<Box<dyn EncodableBlock>>>,
) -> Self {
autoreleasepool(|_| {
+ let ctx = &*mtl_context; // Reference for functions expecting &MTLContext
let intermediate_data_type: DataType =
match &layer_config.mixer_config {
MixerConfig::Attention(attention) => attention
@@ -74,7 +75,7 @@ impl LayerExecutables {
let copy_main_to_shortcut: Box<dyn EncodableBlock> = Box::new(
TensorCopy::new(
- mtl_context,
+ ctx,
kernel_data_type,
vec![ArrayId::Main, ArrayId::Shortcut].into_boxed_slice(),
)
@@ -83,7 +84,7 @@ impl LayerExecutables {
let pre_attention_norm: Box<dyn EncodableBlock> = Box::new(
RMSNorm::new(
- mtl_context,
+ ctx,
intermediate_data_type,
layer_config.pre_attention_norm_config.clone(),
ArrayId::Main,
@@ -108,7 +109,7 @@ impl LayerExecutables {
num_groups * head_dim,
num_groups * head_dim,
],
- mtl_context,
+ ctx,
&decoder_layer_loader
.subtree("mixer.qkv_projection")
.unwrap(),
@@ -122,7 +123,7 @@ impl LayerExecutables {
|| attention_config.key_norm_config.is_some()
{
match QKNorm::new(
- mtl_context,
+ ctx,
intermediate_data_type,
attention_config.query_norm_config.clone(),
attention_config.key_norm_config.clone(),
@@ -148,7 +149,7 @@ impl LayerExecutables {
attention_config.has_out_biases,
num_heads * head_dim,
[model_dim],
- mtl_context,
+ ctx,
&decoder_layer_loader
.subtree("mixer.out_projection")
.unwrap(),
@@ -159,7 +160,7 @@ impl LayerExecutables {
let attention = Box::new(
Attention::new(
- mtl_context,
+ ctx,
kernel_data_type,
layer_index,
attention_scale,
@@ -181,7 +182,7 @@ impl LayerExecutables {
MixerConfig::Mamba(mamba_config) => {
let mixer: Box<dyn EncodableBlock> =
Box::new(MambaMixer::new(
- mtl_context,
+ ctx,
layer_type.clone(),
mamba_config.clone(),
compilation_config.clone(),
@@ -199,7 +200,7 @@ impl LayerExecutables {
MixerConfig::ShortConv(short_conv_config) => {
let mixer: Box<dyn EncodableBlock> =
Box::new(ShortConvMixer::new(
- mtl_context,
+ ctx,
layer_type.clone(),
short_conv_config.clone(),
compilation_config.clone(),
@@ -219,7 +220,7 @@ impl LayerExecutables {
{
Some(Box::new(
RMSNorm::new(
- mtl_context,
+ ctx,
intermediate_data_type,
norm_config.clone(),
ArrayId::Main,
@@ -236,7 +237,7 @@ impl LayerExecutables {
let main_shortcut_add_swap: Box<dyn EncodableBlock> = Box::new(
TensorAddSwap::new(
- mtl_context,
+ ctx,
kernel_data_type,
vec![ArrayId::Shortcut, ArrayId::Main].into_boxed_slice(),
)
@@ -245,7 +246,7 @@ impl LayerExecutables {
let pre_mlp_norm: Box<dyn EncodableBlock> = Box::new(
RMSNorm::new(
- mtl_context,
+ ctx,
intermediate_data_type,
layer_config.pre_mlp_norm_config.clone(),
ArrayId::Main,
@@ -259,7 +260,7 @@ impl LayerExecutables {
&layer_config.mlp_config,
model_dim,
hidden_dim,
- mtl_context,
+ ctx,
&decoder_layer_loader.subtree("mlp").unwrap(),
)
.expect("Failed to create mlp block");
@@ -268,7 +269,7 @@ impl LayerExecutables {
if let Some(norm_config) = &layer_config.post_mlp_norm_config {
Some(Box::new(
RMSNorm::new(
- mtl_context,
+ ctx,
intermediate_data_type,
norm_config.clone(),
ArrayId::Main,
diff --git a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
index ea855f24a..1da60d21a 100644
--- a/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/transformer_layer.rs
@@ -2,14 +2,16 @@ use std::rc::Rc;
use super::{
EncodableBlock, FullPrecisionEmbeddingLookup,
- FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock, MoeBlock,
- QuantizedEmbeddingLookup, QuantizedEmbeddingReadout, QuantizedLinear,
+ FullPrecisionEmbeddingReadout, FullPrecisionLinear, MlpBlock,
+ MlpFusedBlock, MoeBlock, QuantizedEmbeddingLookup,
+ QuantizedEmbeddingReadout, QuantizedLinear,
};
use crate::{
DataType,
backends::metal::{
- MTLContext, MTLError, forward_pass::ArrayId,
- kernel::mlp::MlpGateActMulEncodable,
+ MTLContext, MTLError,
+ forward_pass::ArrayId,
+ kernel::{mlp::MlpGateActMulEncodable, quant_matmul::QuantizationType},
},
config::{DecoderConfig, EmbeddingConfig, LinearConfig, MLPConfig},
parameters::ParameterTree,
@@ -18,22 +20,22 @@ use crate::{
pub fn linear_block<const N: usize>(
config: &LinearConfig,
_has_biases: bool,
- input_dim: usize,
- output_dims: [usize; N],
+ input_dimension: usize,
+ output_dimensions: [usize; N],
context: &MTLContext,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
input_array_id: ArrayId,
output_array_id: ArrayId,
) -> Result<Box<dyn EncodableBlock>, MTLError> {
- let out_sum: usize = output_dims.iter().sum();
+ let output_dimension_sum: usize = output_dimensions.iter().sum();
match config {
- LinearConfig::Quantized(quant_config)
- | LinearConfig::MLXQuantized(quant_config) => {
+ LinearConfig::Quantized(quantization_config)
+ | LinearConfig::MLXQuantized(quantization_config) => {
let block = QuantizedLinear::new(
context,
- quant_config,
- input_dim,
- out_sum,
+ quantization_config,
+ input_dimension,
+ output_dimension_sum,
parameter_tree,
input_array_id,
output_array_id,
@@ -46,8 +48,8 @@ pub fn linear_block<const N: usize>(
let block = FullPrecisionLinear::new(
context,
(*precision).into(),
- input_dim,
- out_sum,
+ input_dimension,
+ output_dimension_sum,
parameter_tree,
input_array_id,
output_array_id,
@@ -62,92 +64,220 @@ pub fn linear_block<const N: usize>(
}
}
+/// Creates an MLP block using the unfused implementation (separate up, activation, down)
pub fn mlp_block(
config: &MLPConfig,
- model_dim: usize,
- hidden_dim: usize,
+ model_dimension: usize,
+ hidden_dimension: usize,
context: &MTLContext,
parameter_tree: &ParameterTree<Rc<MTLContext>>,
) -> Result<Box<dyn EncodableBlock>, MTLError> {
- if let crate::config::MLPConfig::Dense(dense) = config {
- match &dense.linear_config {
- LinearConfig::Quantized(quant_config)
- | LinearConfig::MLXQuantized(quant_config) => {
- let dtype: DataType =
- dense.linear_config.activation_precision().into();
-
- let up = QuantizedLinear::new(
- context,
- quant_config,
- model_dim,
- 2 * hidden_dim,
- &parameter_tree
- .subtree("up_projection")
- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,
- ArrayId::Main,
- ArrayId::MlpFusedUp,
- )?;
+ if let crate::config::MLPConfig::Dense(dense_config) = config {
+ let data_type: DataType =
+ dense_config.linear_config.activation_precision().into();
+
+ // Up projection (outputs 2*hidden_dimension for gate and up)
+ let up_projection = linear_block(
+ &dense_config.linear_config,
+ false,
+ model_dimension,
+ [2 * hidden_dimension],
+ context,
+ &parameter_tree
+ .subtree("up_projection")
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
+ ArrayId::Main,
+ ArrayId::MlpFusedUp,
+ )?;
+
+ // Gate activation + multiply
+ let gate_activation = MlpGateActMulEncodable::new(
+ context,
+ data_type,
+ dense_config.activation.clone(),
+ hidden_dimension,
+ )?;
+
+ // Down projection
+ let down_projection = linear_block(
+ &dense_config.linear_config,
+ false,
+ hidden_dimension,
+ [model_dimension],
+ context,
+ &parameter_tree
+ .subtree("down_projection")
+ .map_err(|error| MTLError::Generic(format!("{:?}", error)))?,
+ ArrayId::MlpHidden,
+ ArrayId::Main,
+ )?;
+
+ return Ok(Box::new(MlpBlock::new(
+ up_projection,
+ gate_activation,
+ down_projection,
+ )));
+ }
- let gate_op = MlpGateActMulEncodable::new(
- context,
- dtype,
- dense.activation,
- hidden_dim,
- )?;
+ if let crate::config::MLPConfig::MixtureOfExperts(
+ mixture_of_experts_config,
+ ) = config
+ {
+ let mixture_of_experts_block = MoeBlock::new(
+ context,
+ mixture_of_experts_config,
+ model_dimension,
+ hidden_dimension,
+ parameter_tree,
+ )?;
+ return Ok(Box::new(mixture_of_experts_block));
+ }
- let down = QuantizedLinear::new(
- context,
- quant_config,
- hidden_dim,
- model_dim,
- &parameter_tree
- .subtree("down_projection")
- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,
+ unreachable!("Unknown MLP config")
+}
+
+/// Creates an MLP block using the fused implementation
+/// (up projection + activation fused into single kernel)
+pub fn mlp_fused_block(
+ config: &MLPConfig,
+ model_dimension: usize,
+ hidden_dimension: usize,
+ context: Rc<MTLContext>,
+ parameter_tree: &ParameterTree<Rc<MTLContext>>,
+) -> Result<Box<dyn EncodableBlock>, MTLError> {
+ if let crate::config::MLPConfig::Dense(dense_config) = config {
+ match &dense_config.linear_config {
+ LinearConfig::Quantized(quantization_config)
+ | LinearConfig::MLXQuantized(quantization_config) => {
+ let data_type: DataType =
+ dense_config.linear_config.activation_precision().into();
+
+ // Load up_projection weights directly for fused kernel
+ let up_projection_tree =
+ parameter_tree.subtree("up_projection").map_err(
+ |error| MTLError::Generic(format!("{:?}", error)),
+ )?;
+
+ let mut up_projection_weights =
+ up_projection_tree.leaf("weights").map_err(|error| {
+ MTLError::Generic(format!(
+ "Failed to load up weights: {:?}",
+ error
+ ))
+ })?;
+ let up_projection_weights_buffer =
+ unsafe { up_projection_weights.mtl_buffer() }.to_owned();
+
+ let mut up_projection_scales =
+ up_projection_tree.leaf("scales").map_err(|error| {
+ MTLError::Generic(format!(
+ "Failed to load up scales: {:?}",
+ error
+ ))
+ })?;
+ let up_projection_scales_buffer =
+ unsafe { up_projection_scales.mtl_buffer() }.to_owned();
+
+ // Load zero_points or biases depending on quantization type
+ let (
+ up_projection_zero_points_or_biases_buffer,
+ quantization_type,
+ ) = if let Ok(mut biases) = up_projection_tree.leaf("biases") {
+ (
+ unsafe { biases.mtl_buffer() }.to_owned(),
+ QuantizationType::Mlx,
+ )
+ } else if let Ok(mut zero_points) =
+ up_projection_tree.leaf("zero_points")
+ {
+ (
+ unsafe { zero_points.mtl_buffer() }.to_owned(),
+ QuantizationType::ZeroPoint,
+ )
+ } else {
+ return Err(MTLError::Generic(
+ "Missing zero_points or biases for quantized up_projection"
+ .to_string(),
+ ));
+ };
+
+ // Create down projection as separate linear
+ let down_projection = QuantizedLinear::new(
+ &context,
+ quantization_config,
+ hidden_dimension,
+ model_dimension,
+ &parameter_tree.subtree("down_projection").map_err(
+ |error| MTLError::Generic(format!("{:?}", error)),
+ )?,
ArrayId::MlpHidden,
ArrayId::Main,
)?;
- let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));
- return Ok(Box::new(enc));
+ let fused_block = MlpFusedBlock::new_quantized(
+ context,
+ data_type,
+ up_projection_weights_buffer,
+ up_projection_scales_buffer,
+ up_projection_zero_points_or_biases_buffer,
+ model_dimension,
+ hidden_dimension,
+ quantization_config.group_size,
+ quantization_config.weight_quantization_mode,
+ quantization_type,
+ &dense_config.activation,
+ Box::new(down_projection),
+ ArrayId::Main,
+ ArrayId::MlpHidden,
+ )?;
+ return Ok(Box::new(fused_block));
},
LinearConfig::FullPrecision {
precision,
} => {
- let dtype: DataType = (*precision).into();
-
- let up = FullPrecisionLinear::new(
- context,
- dtype,
- model_dim,
- 2 * hidden_dim,
- &parameter_tree
- .subtree("up_projection")
- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,
+ let data_type: DataType = (*precision).into();
+
+ // Load up_projection weights directly for fused kernel
+ let up_projection_tree =
+ parameter_tree.subtree("up_projection").map_err(
+ |error| MTLError::Generic(format!("{:?}", error)),
+ )?;
+
+ let mut up_projection_weights =
+ up_projection_tree.leaf("weights").map_err(|error| {
+ MTLError::Generic(format!(
+ "Failed to load up weights: {:?}",
+ error
+ ))
+ })?;
+ let up_projection_weights_buffer =
+ unsafe { up_projection_weights.mtl_buffer() }.to_owned();
+
+ // Create down projection as separate linear
+ let down_projection = FullPrecisionLinear::new(
+ &context,
+ data_type,
+ hidden_dimension,
+ model_dimension,
+ &parameter_tree.subtree("down_projection").map_err(
+ |error| MTLError::Generic(format!("{:?}", error)),
+ )?,
+ ArrayId::MlpHidden,
ArrayId::Main,
- ArrayId::MlpFusedUp,
)?;
- let gate_op = MlpGateActMulEncodable::new(
+ let fused_block = MlpFusedBlock::new_full_precision(
context,
- dtype,
- dense.activation,
- hidden_dim,
- )?;
-
- let down = FullPrecisionLinear::new(
- context,
- dtype,
- hidden_dim,
- model_dim,
- &parameter_tree
- .subtree("down_projection")
- .map_err(|e| MTLError::Generic(format!("{:?}", e)))?,
- ArrayId::MlpHidden,
+ data_type,
+ up_projection_weights_buffer,
+ model_dimension,
+ hidden_dimension,
+ &dense_config.activation,
+ Box::new(down_projection),
ArrayId::Main,
+ ArrayId::MlpHidden,
)?;
-
- let enc = MlpBlock::new(Box::new(up), gate_op, Box::new(down));
- return Ok(Box::new(enc));
+ return Ok(Box::new(fused_block));
},
LinearConfig::QLoRA {
..
@@ -159,10 +289,18 @@ pub fn mlp_block(
}
}
- if let crate::config::MLPConfig::MixtureOfExperts(moe) = config {
- let moe_block =
- MoeBlock::new(context, moe, model_dim, hidden_dim, parameter_tree)?;
- return Ok(Box::new(moe_block));
+ if let crate::config::MLPConfig::MixtureOfExperts(
+ mixture_of_experts_config,
+ ) = config
+ {
+ let mixture_of_experts_block = MoeBlock::new(
+ &context,
+ mixture_of_experts_config,
+ model_dimension,
+ hidden_dimension,
+ parameter_tree,
+ )?;
+ return Ok(Box::new(mixture_of_experts_block));
}
unreachable!("Unknown MLP config")