A high-performance inference engine for AI models
Support gated deltanet tree `conv_scan` kernels (#605)
CCYeh0430f214c1eb
diff --git a/Cargo.lock b/Cargo.lockindex 7c9ef7865..e82d20b00 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -6366,9 +6366,9 @@ dependencies = [[[package]]name = "spin"-version = "0.9.8"+version = "0.9.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"+checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"[[package]]name = "spm_precompiled"diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rsnew file mode 100644index 000000000..84d6b5513--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs@@ -0,0 +1,72 @@+use half::bf16;+use num_traits::Float;+use proc_macros::kernel;++use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};++#[kernel(ConvTreeScan)]+#[variants(T, f32, bf16)]+pub fn conv_tree_scan<T: ArrayElement + Float>(+ in_proj: *const T,+ conv_weight: *const f32,+ #[optional(has_bias)] bias: Option<*const f32>,+ base_state: *const f32,+ parents: *const i32,+ out_proj: *mut T,+ suffix_state: *mut f32,+ suffix_len: u32,+ #[specialize] kernel_size: u32,+ total_proj_dim: u32,+ conv_dim: u32,+ #[specialize] has_bias: bool,+) {+ let suffix_len = suffix_len as usize;+ let kernel_size = kernel_size as usize;+ let total_proj_dim = total_proj_dim as usize;+ let conv_dim = conv_dim as usize;+ let state_stride = kernel_size - 1;++ for node_idx in 0..suffix_len {+ for channel_idx in 0..total_proj_dim {+ let proj_idx = node_idx * total_proj_dim + channel_idx;+ if channel_idx >= conv_dim {+ unsafe { *out_proj.add(proj_idx) = *in_proj.add(proj_idx) };+ continue;+ }++ let mut acc = if has_bias {+ unsafe { *bias.unwrap().add(channel_idx) }+ } else {+ 0.0+ };+ let weight_offset = channel_idx * kernel_size;+ let base_state_offset = channel_idx * state_stride;+ let mut source_row = node_idx as i32;+ for history_offset in 0..kernel_size {+ let sample = if source_row >= 0 {+ let source_proj_idx = source_row as usize * total_proj_dim + channel_idx;+ unsafe { (*in_proj.add(source_proj_idx)).to_f32().unwrap() }+ } else {+ let base_state_tap = state_stride - source_row.unsigned_abs() as usize;+ unsafe { *base_state.add(base_state_offset + base_state_tap) }+ };+ let weight_tap = kernel_size - 1 - history_offset;+ acc += sample * unsafe { *conv_weight.add(weight_offset + weight_tap) };++ if history_offset < state_stride {+ let state_tap = state_stride - 1 - history_offset;+ let state_idx = (node_idx * conv_dim + channel_idx) * state_stride + state_tap;+ unsafe { *suffix_state.add(state_idx) = sample };+ }++ source_row = if source_row >= 0 {+ unsafe { *parents.add(source_row as usize) }+ } else {+ source_row - 1+ };+ }++ unsafe { *out_proj.add(proj_idx) = T::from(ActivationType::SILU.activate(acc)).unwrap() };+ }+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rsindex 029372d1e..c752d4cbd 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs@@ -1,3 +1,4 @@+pub mod conv_scan;pub mod out;pub mod prefix_beta;pub mod tree_gram;diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/conv_scan.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/conv_scan.metalnew file mode 100644index 000000000..028df1dbf--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/conv_scan.metal@@ -0,0 +1,68 @@+#include <metal_stdlib>+#include "../../activation/activations.h"+#include "../../common/dsl.h"++using namespace metal;++// Applies short convolution along each proposal-tree path and records every+// node's pending convolution state. Other projection channels pass through.+template <typename T>+VARIANTS(T, float, bfloat)+PUBLIC KERNEL(ConvTreeScan)(+ // [suffix_len, total_proj_dim]+ device const T* in_proj,+ // [conv_dim, kernel_size]+ device const float* conv_weight,+ // [conv_dim]+ device const float* bias OPTIONAL(has_bias),+ // [conv_dim, kernel_size - 1], read-only+ device const float* base_state,+ // [suffix_len]+ device const int* parents,+ // [suffix_len, total_proj_dim]+ device T* out_proj,+ // [suffix_len, conv_dim, kernel_size - 1]+ device float* suffix_state,+ constant const uint& suffix_len,+ const uint kernel_size SPECIALIZE,+ constant const uint& total_proj_dim,+ constant const uint& conv_dim,+ const bool has_bias SPECIALIZE,+ const uint node_idx AXIS(suffix_len, 1),+ const uint channel_idx AXIS(total_proj_dim, 256)+) {+ const uint proj_idx = node_idx * total_proj_dim + channel_idx;+ if (channel_idx >= conv_dim) {+ out_proj[proj_idx] = in_proj[proj_idx];+ return;+ }++ const uint state_stride = kernel_size - 1;+ float acc = has_bias ? bias[channel_idx] : 0.0f;+ const uint weight_offset = channel_idx * kernel_size;+ const uint base_state_offset = channel_idx * state_stride;+ int source_row = int(node_idx);+ for (uint history_offset = 0; history_offset < kernel_size; ++history_offset) {+ float sample;+ if (source_row >= 0) {+ const uint source_proj_idx = uint(source_row) * total_proj_dim + channel_idx;+ sample = float(in_proj[source_proj_idx]);+ } else {+ const uint base_state_tap = state_stride - uint(-source_row);+ sample = base_state[base_state_offset + base_state_tap];+ }++ const uint weight_tap = kernel_size - 1 - history_offset;+ acc += sample * conv_weight[weight_offset + weight_tap];++ if (history_offset < state_stride) {+ const uint state_tap = state_stride - 1 - history_offset;+ const uint state_idx = (node_idx * conv_dim + channel_idx) * state_stride + state_tap;+ suffix_state[state_idx] = sample;+ }++ source_row = source_row >= 0 ? parents[source_row] : source_row - 1;+ }++ out_proj[proj_idx] = static_cast<T>(activate_silu(acc));+}diff --git a/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/mod.rsnew file mode 100644index 000000000..adcd93b69--- /dev/null+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/mod.rs@@ -0,0 +1,2 @@+mod delta_net_test;+mod tree_verify;diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/conv_scan_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/conv_scan_test.rsnew file mode 100644index 000000000..ea4d0ecea--- /dev/null+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/conv_scan_test.rs@@ -0,0 +1,187 @@+#[cfg(metal_backend)]+use std::{mem::size_of, time::Duration};++#[cfg(metal_backend)]+use criterion::Criterion;+#[cfg(metal_backend)]+use half::bf16;+#[cfg(metal_backend)]+use proc_macros::uzu_bench;+use proc_macros::uzu_test;+use test_runner::for_each_non_cpu_backend;++#[cfg(metal_backend)]+use crate::{+ backends::{common::Allocation, metal::Metal},+ tests::{cold_pool::ColdPool, matmul::iter_encode_loop_named},+};+use crate::{+ backends::{+ common::{Backend, Context, Encoder, Kernels, kernel::ConvTreeScanKernel},+ cpu::Cpu,+ },+ data_type::DataType,+ tests::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},+ },+};++const CONV_DIM: usize = 37;+const TOTAL_PROJ_DIM: usize = 48;+const KERNEL_SIZE: usize = 4;+const STATE_STRIDE: usize = KERNEL_SIZE - 1;++fn parents(+ tree_size: usize,+ shape: &str,+) -> Vec<i32> {+ (0..tree_size)+ .map(|node| {+ if node == 0 {+ -1+ } else {+ match shape {+ "chain" => node as i32 - 1,+ "star" => 0,+ "binary" => ((node - 1) / 2) as i32,+ "random" => (((node ^ (node >> 1)) * 17 + 3) % node) as i32,+ _ => unreachable!(),+ }+ }+ })+ .collect()+}++fn run<B: Backend>(+ tree_size: usize,+ shape: &str,+ perturb: bool,+) -> (Vec<f32>, Vec<f32>) {+ let context = B::Context::new().expect("context");+ let kernel = <<B as Backend>::Kernels as Kernels>::ConvTreeScanKernel::new(+ &context,+ DataType::F32,+ KERNEL_SIZE as u32,+ true,+ )+ .expect("kernel");+ let mut input = (0..tree_size * TOTAL_PROJ_DIM).map(|i| (i % 37) as f32 * 0.01 - 0.2).collect::<Vec<_>>();+ if perturb {+ input[TOTAL_PROJ_DIM..TOTAL_PROJ_DIM + CONV_DIM].iter_mut().for_each(|value| *value += 0.5);+ }+ let weights = (0..CONV_DIM * KERNEL_SIZE).map(|i| (i % 11) as f32 * 0.02 - 0.1).collect::<Vec<_>>();+ let bias = (0..CONV_DIM).map(|i| i as f32 * 0.003 - 0.04).collect::<Vec<_>>();+ let base_state = (0..CONV_DIM * STATE_STRIDE).map(|i| (i % 13) as f32 * 0.01 - 0.05).collect::<Vec<_>>();++ let input = alloc_allocation_with_data::<B, f32>(&context, &input);+ let weights = alloc_allocation_with_data::<B, f32>(&context, &weights);+ let bias = alloc_allocation_with_data::<B, f32>(&context, &bias);+ let base_state_buffer = alloc_allocation_with_data::<B, f32>(&context, &base_state);+ let parents = alloc_allocation_with_data::<B, i32>(&context, &parents(tree_size, shape));+ let mut output = alloc_allocation::<B, f32>(&context, tree_size * TOTAL_PROJ_DIM);+ let mut suffix_state = alloc_allocation::<B, f32>(&context, tree_size * CONV_DIM * STATE_STRIDE);++ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");+ kernel.encode(+ &input,+ &weights,+ Some(&bias),+ &base_state_buffer,+ &parents,+ &mut output,+ &mut suffix_state,+ tree_size as u32,+ TOTAL_PROJ_DIM as u32,+ CONV_DIM as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ assert_eq_float(+ &base_state,+ &allocation_to_vec(&base_state_buffer),+ 0.0,+ &format!("base state {shape} T={tree_size}"),+ );+ (allocation_to_vec(&output), allocation_to_vec(&suffix_state))+}++#[uzu_test]+fn test_conv_tree_scan() {+ for tree_size in [49, 64, 128] {+ for shape in ["chain", "star", "binary", "random"] {+ let expected = run::<Cpu>(tree_size, shape, false);+ let label = format!("{shape} T={tree_size}");+ for_each_non_cpu_backend!(|B| {+ let actual = run::<B>(tree_size, shape, false);+ assert_eq_float(&expected.0, &actual.0, 1e-5, &format!("output {label}"));+ assert_eq_float(&expected.1, &actual.1, 1e-6, &format!("suffix state {label}"));+ });+ }+ }++ let baseline = run::<Cpu>(49, "binary", false);+ let perturbed = run::<Cpu>(49, "binary", true);+ let output = 2 * TOTAL_PROJ_DIM..3 * TOTAL_PROJ_DIM;+ let state = 2 * CONV_DIM * STATE_STRIDE..3 * CONV_DIM * STATE_STRIDE;+ assert_eq_float(&baseline.0[output.clone()], &perturbed.0[output], 0.0, "sibling output");+ assert_eq_float(&baseline.1[state.clone()], &perturbed.1[state], 0.0, "sibling state");+}++#[cfg(metal_backend)]+struct Buffers {+ input: Allocation<Metal>,+ output: Allocation<Metal>,+ suffix_state: Allocation<Metal>,+}++#[cfg(metal_backend)]+#[uzu_bench]+fn bench_conv_tree_scan(c: &mut Criterion) {+ const BENCH_CONV_DIM: usize = 10_240;+ const BENCH_TOTAL_PROJ_DIM: usize = 16_480;+ const BENCHMARK: &str = "Metal/Kernel/GDNTreeVerify/ConvTreeScan";++ let context = crate::tests::util::shared_metal_context();+ let kernel = <<Metal as Backend>::Kernels as Kernels>::ConvTreeScanKernel::new(+ &context,+ DataType::BF16,+ KERNEL_SIZE as u32,+ true,+ )+ .expect("kernel");+ let weights = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.01; BENCH_CONV_DIM * KERNEL_SIZE]);+ let bias = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.0; BENCH_CONV_DIM]);+ let base_state = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.0; BENCH_CONV_DIM * STATE_STRIDE]);+ let mut group = c.benchmark_group(BENCHMARK);+ group.sample_size(30).warm_up_time(Duration::from_millis(300)).measurement_time(Duration::from_secs(1));++ for tree_size in [49usize, 64, 128] {+ let parents = alloc_allocation_with_data::<Metal, i32>(&context, &parents(tree_size, "binary"));+ let input_len = tree_size * BENCH_TOTAL_PROJ_DIM;+ let state_len = tree_size * BENCH_CONV_DIM * STATE_STRIDE;+ let mut buffers = ColdPool::new(2 * input_len * size_of::<bf16>() + state_len * size_of::<f32>(), || Buffers {+ input: alloc_allocation_with_data::<Metal, bf16>(&context, &vec![bf16::from_f32(0.1); input_len]),+ output: alloc_allocation::<Metal, bf16>(&context, input_len),+ suffix_state: alloc_allocation::<Metal, f32>(&context, state_len),+ });+ group.bench_function(format!("T{tree_size}"), |bencher| {+ iter_encode_loop_named::<Metal, _>(&context, bencher, &format!("{BENCHMARK}/T{tree_size}"), |encoder| {+ let buffers = buffers.next_mut();+ kernel.encode(+ &buffers.input,+ &weights,+ Some(&bias),+ &base_state,+ &parents,+ &mut buffers.output,+ &mut buffers.suffix_state,+ tree_size as u32,+ BENCH_TOTAL_PROJ_DIM as u32,+ BENCH_CONV_DIM as u32,+ encoder,+ );+ });+ });+ }+}diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rsnew file mode 100644index 000000000..3bab06e07--- /dev/null+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs@@ -0,0 +1,9 @@+mod conv_scan_test;+mod out_bench;+mod out_test;+mod prefix_beta_bench;+mod prefix_beta_test;+mod tree_gram_bench;+mod tree_gram_test;+mod tree_update_solve_bench;+mod tree_update_solve_test;diff --git a/crates/backend-uzu/unit/backends/common/kernel/out_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/out_bench.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/out_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/out_test.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/prefix_beta_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/prefix_beta_bench.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/prefix_beta_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/prefix_beta_test.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/tree_gram_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/tree_gram_bench.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/tree_gram_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/tree_gram_test.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/tree_update_solve_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/tree_update_solve_bench.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/tree_update_solve_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rssimilarity index 100%rename from crates/backend-uzu/unit/backends/common/kernel/tree_update_solve_test.rsrename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rsdiff --git a/crates/backend-uzu/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/mod.rsindex e3af1ea6f..442f83e34 100644--- a/crates/backend-uzu/unit/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/mod.rs@@ -1,18 +1,14 @@mod activation_test;mod attention;-mod delta_net_test;mod embedding;mod gated_act_mul_test;+mod gdn;mod hadamard_transform_test;mod kv_cache_update_test;mod logit_soft_cap_test;mod matmul;mod moe;-mod out_bench;-mod out_test;mod pooling;-mod prefix_beta_bench;-mod prefix_beta_test;mod qkv_norm_test;mod short_conv;mod ssm;@@ -21,7 +17,3 @@ mod tensor_add_scale_test;mod tensor_add_swap_test;mod tensor_copy_test;mod token_copy_test;-mod tree_gram_bench;-mod tree_gram_test;-mod tree_update_solve_bench;-mod tree_update_solve_test;