Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

update test

Eugene Bokhane7fa09eebed4
diff --git a/crates/benchmarks/src/mlp_fused_bench.rs b/crates/benchmarks/src/mlp_fused_bench.rs
index d70dc18f5..d96ca425f 100644
--- a/crates/benchmarks/src/mlp_fused_bench.rs
+++ b/crates/benchmarks/src/mlp_fused_bench.rs
@@ -331,6 +331,33 @@ fn bench_gemm_fused(
BenchResult::new(fused_time, unfused_time)
}
+/// MLP shapes from real model configs
+/// Format: (K=model_dim, H=hidden_dim)
+const MLP_SHAPES: &[(usize, usize)] = &[
+ (896, 4864), // Qwen2.5-Coder-0.5B
+ (1024, 3072), // Qwen3-0.6B
+ (1024, 4608), // LFM2-350M
+ (1152, 6912), // gemma-3-1b
+ (1536, 6912), // LFM2-700M
+ (1536, 8960), // Qwen2.5-Coder-1.5B, DeepSeek-R1-Distill-Qwen-1.5B
+ (2048, 6144), // Qwen3-1.7B
+ (2048, 8192), // Llamba-1B, SmolLM2-1.7B, Llama-3.2-1B, LFM2-1.2B
+ (2048, 10752), // LFM2-2.6B
+ (2048, 11008), // Qwen2.5-Coder-3B
+ (2560, 9728), // Qwen3-4B
+ (2560, 10240), // gemma-3-4b
+ (3072, 8192), // Llama-3.2-3B, Llamba-3B
+ (3584, 18944), // Qwen2.5-Coder-7B
+ (4096, 12288), // Qwen3-8B
+ (4096, 14336), // Llamba-8B, Llama-3.1-8B
+ (4096, 16384), // rnj-1-instruct
+ (5120, 13824), // Qwen2.5-Coder-14B
+ (5120, 17408), // Qwen3-14B
+ (5120, 25600), // Qwen3-32B
+ (5376, 21504), // gemma-3-27b
+ (6144, 16384), // Codestral-22B
+];
+
pub fn run_mlp_fused_benchmark() {
autoreleasepool(|_| {
let ctx = match create_test_context() {
@@ -341,44 +368,62 @@ pub fn run_mlp_fused_benchmark() {
},
};
- println!("\n=== MLP Fused Kernel Benchmark ===\n");
- println!(
- "{:<40} {:>12} {:>12} {:>10}",
- "Configuration", "Fused (µs)", "Unfused (µs)", "Speedup"
- );
- println!("{}", "-".repeat(76));
+ println!("\n=== MLP Fused Kernel Benchmark (Real Model Shapes) ===\n");
let iterations = 100;
let warmup = 10;
- // Decode configurations (M=1)
- for (k, hidden_dim) in [(2048, 4096), (4096, 11008), (8192, 22016)] {
+ // GEMV benchmark (decode path, M=1)
+ println!("=== GEMV (Decode, M=1) ===");
+ println!(
+ "{:<25} {:>12} {:>12} {:>10}",
+ "Shape (K, H)", "Fused (µs)", "Unfused (µs)", "Speedup"
+ );
+ println!("{}", "-".repeat(61));
+
+ for &(k, hidden_dim) in MLP_SHAPES {
let result =
bench_gemv_fused(&ctx, k, hidden_dim, iterations, warmup);
- println!("GEMV decode K={} H={}", k, hidden_dim);
+ let speedup_str = if result.speedup >= 1.0 {
+ format!("{:.2}x", result.speedup)
+ } else {
+ format!("{:.2}x", result.speedup)
+ };
println!(
- "{:<40} {:>12.2} {:>12.2} {:>9.2}x",
- format!(" M=1, K={}, H={}", k, hidden_dim),
+ "({:>4}, {:>5}) {:>12.1} {:>12.1} {:>10}",
+ k,
+ hidden_dim,
result.fused_time_us,
result.unfused_time_us,
- result.speedup
+ speedup_str
);
}
println!();
- // Prefill configurations (M>1)
- for (m, k, hidden_dim) in
- [(32, 2048, 4096), (128, 2048, 4096), (512, 4096, 11008)]
- {
+ // GEMM benchmark (prefill path, M=32)
+ println!("=== GEMM (Prefill, M=32) ===");
+ println!(
+ "{:<25} {:>12} {:>12} {:>10}",
+ "Shape (K, H)", "Fused (µs)", "Unfused (µs)", "Speedup"
+ );
+ println!("{}", "-".repeat(61));
+
+ for &(k, hidden_dim) in MLP_SHAPES {
let result =
- bench_gemm_fused(&ctx, m, k, hidden_dim, iterations, warmup);
+ bench_gemm_fused(&ctx, 32, k, hidden_dim, iterations, warmup);
+ let speedup_str = if result.speedup >= 1.0 {
+ format!("{:.2}x", result.speedup)
+ } else {
+ format!("{:.2}x", result.speedup)
+ };
println!(
- "{:<40} {:>12.2} {:>12.2} {:>9.2}x",
- format!("GEMM M={}, K={}, H={}", m, k, hidden_dim),
+ "({:>4}, {:>5}) {:>12.1} {:>12.1} {:>10}",
+ k,
+ hidden_dim,
result.fused_time_us,
result.unfused_time_us,
- result.speedup
+ speedup_str
);
}