Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Support gated deltanet tree `conv_scan` kernels (#605)

CCYeh0430f214c1eb
diff --git a/Cargo.lock b/Cargo.lock
index 7c9ef7865..e82d20b00 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6366,9 +6366,9 @@ dependencies = [
[[package]]
name = "spin"
-version = "0.9.8"
+version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
[[package]]
name = "spm_precompiled"
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs
new file mode 100644
index 000000000..84d6b5513
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/conv_scan.rs
@@ -0,0 +1,72 @@
+use half::bf16;
+use num_traits::Float;
+use proc_macros::kernel;
+
+use crate::{array::ArrayElement, backends::common::gpu_types::ActivationType};
+
+#[kernel(ConvTreeScan)]
+#[variants(T, f32, bf16)]
+pub fn conv_tree_scan<T: ArrayElement + Float>(
+ in_proj: *const T,
+ conv_weight: *const f32,
+ #[optional(has_bias)] bias: Option<*const f32>,
+ base_state: *const f32,
+ parents: *const i32,
+ out_proj: *mut T,
+ suffix_state: *mut f32,
+ suffix_len: u32,
+ #[specialize] kernel_size: u32,
+ total_proj_dim: u32,
+ conv_dim: u32,
+ #[specialize] has_bias: bool,
+) {
+ let suffix_len = suffix_len as usize;
+ let kernel_size = kernel_size as usize;
+ let total_proj_dim = total_proj_dim as usize;
+ let conv_dim = conv_dim as usize;
+ let state_stride = kernel_size - 1;
+
+ for node_idx in 0..suffix_len {
+ for channel_idx in 0..total_proj_dim {
+ let proj_idx = node_idx * total_proj_dim + channel_idx;
+ if channel_idx >= conv_dim {
+ unsafe { *out_proj.add(proj_idx) = *in_proj.add(proj_idx) };
+ continue;
+ }
+
+ let mut acc = if has_bias {
+ unsafe { *bias.unwrap().add(channel_idx) }
+ } else {
+ 0.0
+ };
+ let weight_offset = channel_idx * kernel_size;
+ let base_state_offset = channel_idx * state_stride;
+ let mut source_row = node_idx as i32;
+ for history_offset in 0..kernel_size {
+ let sample = if source_row >= 0 {
+ let source_proj_idx = source_row as usize * total_proj_dim + channel_idx;
+ unsafe { (*in_proj.add(source_proj_idx)).to_f32().unwrap() }
+ } else {
+ let base_state_tap = state_stride - source_row.unsigned_abs() as usize;
+ unsafe { *base_state.add(base_state_offset + base_state_tap) }
+ };
+ let weight_tap = kernel_size - 1 - history_offset;
+ acc += sample * unsafe { *conv_weight.add(weight_offset + weight_tap) };
+
+ if history_offset < state_stride {
+ let state_tap = state_stride - 1 - history_offset;
+ let state_idx = (node_idx * conv_dim + channel_idx) * state_stride + state_tap;
+ unsafe { *suffix_state.add(state_idx) = sample };
+ }
+
+ source_row = if source_row >= 0 {
+ unsafe { *parents.add(source_row as usize) }
+ } else {
+ source_row - 1
+ };
+ }
+
+ unsafe { *out_proj.add(proj_idx) = T::from(ActivationType::SILU.activate(acc)).unwrap() };
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
index 029372d1e..c752d4cbd 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/gdn/tree_verify/mod.rs
@@ -1,3 +1,4 @@
+pub mod conv_scan;
pub mod out;
pub mod prefix_beta;
pub mod tree_gram;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/conv_scan.metal b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/conv_scan.metal
new file mode 100644
index 000000000..028df1dbf
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/gdn/tree_verify/conv_scan.metal
@@ -0,0 +1,68 @@
+#include <metal_stdlib>
+#include "../../activation/activations.h"
+#include "../../common/dsl.h"
+
+using namespace metal;
+
+// Applies short convolution along each proposal-tree path and records every
+// node's pending convolution state. Other projection channels pass through.
+template <typename T>
+VARIANTS(T, float, bfloat)
+PUBLIC KERNEL(ConvTreeScan)(
+ // [suffix_len, total_proj_dim]
+ device const T* in_proj,
+ // [conv_dim, kernel_size]
+ device const float* conv_weight,
+ // [conv_dim]
+ device const float* bias OPTIONAL(has_bias),
+ // [conv_dim, kernel_size - 1], read-only
+ device const float* base_state,
+ // [suffix_len]
+ device const int* parents,
+ // [suffix_len, total_proj_dim]
+ device T* out_proj,
+ // [suffix_len, conv_dim, kernel_size - 1]
+ device float* suffix_state,
+ constant const uint& suffix_len,
+ const uint kernel_size SPECIALIZE,
+ constant const uint& total_proj_dim,
+ constant const uint& conv_dim,
+ const bool has_bias SPECIALIZE,
+ const uint node_idx AXIS(suffix_len, 1),
+ const uint channel_idx AXIS(total_proj_dim, 256)
+) {
+ const uint proj_idx = node_idx * total_proj_dim + channel_idx;
+ if (channel_idx >= conv_dim) {
+ out_proj[proj_idx] = in_proj[proj_idx];
+ return;
+ }
+
+ const uint state_stride = kernel_size - 1;
+ float acc = has_bias ? bias[channel_idx] : 0.0f;
+ const uint weight_offset = channel_idx * kernel_size;
+ const uint base_state_offset = channel_idx * state_stride;
+ int source_row = int(node_idx);
+ for (uint history_offset = 0; history_offset < kernel_size; ++history_offset) {
+ float sample;
+ if (source_row >= 0) {
+ const uint source_proj_idx = uint(source_row) * total_proj_dim + channel_idx;
+ sample = float(in_proj[source_proj_idx]);
+ } else {
+ const uint base_state_tap = state_stride - uint(-source_row);
+ sample = base_state[base_state_offset + base_state_tap];
+ }
+
+ const uint weight_tap = kernel_size - 1 - history_offset;
+ acc += sample * conv_weight[weight_offset + weight_tap];
+
+ if (history_offset < state_stride) {
+ const uint state_tap = state_stride - 1 - history_offset;
+ const uint state_idx = (node_idx * conv_dim + channel_idx) * state_stride + state_tap;
+ suffix_state[state_idx] = sample;
+ }
+
+ source_row = source_row >= 0 ? parents[source_row] : source_row - 1;
+ }
+
+ out_proj[proj_idx] = static_cast<T>(activate_silu(acc));
+}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/delta_net_test.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/delta_net_test.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/mod.rs
new file mode 100644
index 000000000..adcd93b69
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/mod.rs
@@ -0,0 +1,2 @@
+mod delta_net_test;
+mod tree_verify;
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/conv_scan_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/conv_scan_test.rs
new file mode 100644
index 000000000..ea4d0ecea
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/conv_scan_test.rs
@@ -0,0 +1,187 @@
+#[cfg(metal_backend)]
+use std::{mem::size_of, time::Duration};
+
+#[cfg(metal_backend)]
+use criterion::Criterion;
+#[cfg(metal_backend)]
+use half::bf16;
+#[cfg(metal_backend)]
+use proc_macros::uzu_bench;
+use proc_macros::uzu_test;
+use test_runner::for_each_non_cpu_backend;
+
+#[cfg(metal_backend)]
+use crate::{
+ backends::{common::Allocation, metal::Metal},
+ tests::{cold_pool::ColdPool, matmul::iter_encode_loop_named},
+};
+use crate::{
+ backends::{
+ common::{Backend, Context, Encoder, Kernels, kernel::ConvTreeScanKernel},
+ cpu::Cpu,
+ },
+ data_type::DataType,
+ tests::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
+ },
+};
+
+const CONV_DIM: usize = 37;
+const TOTAL_PROJ_DIM: usize = 48;
+const KERNEL_SIZE: usize = 4;
+const STATE_STRIDE: usize = KERNEL_SIZE - 1;
+
+fn parents(
+ tree_size: usize,
+ shape: &str,
+) -> Vec<i32> {
+ (0..tree_size)
+ .map(|node| {
+ if node == 0 {
+ -1
+ } else {
+ match shape {
+ "chain" => node as i32 - 1,
+ "star" => 0,
+ "binary" => ((node - 1) / 2) as i32,
+ "random" => (((node ^ (node >> 1)) * 17 + 3) % node) as i32,
+ _ => unreachable!(),
+ }
+ }
+ })
+ .collect()
+}
+
+fn run<B: Backend>(
+ tree_size: usize,
+ shape: &str,
+ perturb: bool,
+) -> (Vec<f32>, Vec<f32>) {
+ let context = B::Context::new().expect("context");
+ let kernel = <<B as Backend>::Kernels as Kernels>::ConvTreeScanKernel::new(
+ &context,
+ DataType::F32,
+ KERNEL_SIZE as u32,
+ true,
+ )
+ .expect("kernel");
+ let mut input = (0..tree_size * TOTAL_PROJ_DIM).map(|i| (i % 37) as f32 * 0.01 - 0.2).collect::<Vec<_>>();
+ if perturb {
+ input[TOTAL_PROJ_DIM..TOTAL_PROJ_DIM + CONV_DIM].iter_mut().for_each(|value| *value += 0.5);
+ }
+ let weights = (0..CONV_DIM * KERNEL_SIZE).map(|i| (i % 11) as f32 * 0.02 - 0.1).collect::<Vec<_>>();
+ let bias = (0..CONV_DIM).map(|i| i as f32 * 0.003 - 0.04).collect::<Vec<_>>();
+ let base_state = (0..CONV_DIM * STATE_STRIDE).map(|i| (i % 13) as f32 * 0.01 - 0.05).collect::<Vec<_>>();
+
+ let input = alloc_allocation_with_data::<B, f32>(&context, &input);
+ let weights = alloc_allocation_with_data::<B, f32>(&context, &weights);
+ let bias = alloc_allocation_with_data::<B, f32>(&context, &bias);
+ let base_state_buffer = alloc_allocation_with_data::<B, f32>(&context, &base_state);
+ let parents = alloc_allocation_with_data::<B, i32>(&context, &parents(tree_size, shape));
+ let mut output = alloc_allocation::<B, f32>(&context, tree_size * TOTAL_PROJ_DIM);
+ let mut suffix_state = alloc_allocation::<B, f32>(&context, tree_size * CONV_DIM * STATE_STRIDE);
+
+ let mut encoder = Encoder::new(context.as_ref()).expect("encoder");
+ kernel.encode(
+ &input,
+ &weights,
+ Some(&bias),
+ &base_state_buffer,
+ &parents,
+ &mut output,
+ &mut suffix_state,
+ tree_size as u32,
+ TOTAL_PROJ_DIM as u32,
+ CONV_DIM as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ assert_eq_float(
+ &base_state,
+ &allocation_to_vec(&base_state_buffer),
+ 0.0,
+ &format!("base state {shape} T={tree_size}"),
+ );
+ (allocation_to_vec(&output), allocation_to_vec(&suffix_state))
+}
+
+#[uzu_test]
+fn test_conv_tree_scan() {
+ for tree_size in [49, 64, 128] {
+ for shape in ["chain", "star", "binary", "random"] {
+ let expected = run::<Cpu>(tree_size, shape, false);
+ let label = format!("{shape} T={tree_size}");
+ for_each_non_cpu_backend!(|B| {
+ let actual = run::<B>(tree_size, shape, false);
+ assert_eq_float(&expected.0, &actual.0, 1e-5, &format!("output {label}"));
+ assert_eq_float(&expected.1, &actual.1, 1e-6, &format!("suffix state {label}"));
+ });
+ }
+ }
+
+ let baseline = run::<Cpu>(49, "binary", false);
+ let perturbed = run::<Cpu>(49, "binary", true);
+ let output = 2 * TOTAL_PROJ_DIM..3 * TOTAL_PROJ_DIM;
+ let state = 2 * CONV_DIM * STATE_STRIDE..3 * CONV_DIM * STATE_STRIDE;
+ assert_eq_float(&baseline.0[output.clone()], &perturbed.0[output], 0.0, "sibling output");
+ assert_eq_float(&baseline.1[state.clone()], &perturbed.1[state], 0.0, "sibling state");
+}
+
+#[cfg(metal_backend)]
+struct Buffers {
+ input: Allocation<Metal>,
+ output: Allocation<Metal>,
+ suffix_state: Allocation<Metal>,
+}
+
+#[cfg(metal_backend)]
+#[uzu_bench]
+fn bench_conv_tree_scan(c: &mut Criterion) {
+ const BENCH_CONV_DIM: usize = 10_240;
+ const BENCH_TOTAL_PROJ_DIM: usize = 16_480;
+ const BENCHMARK: &str = "Metal/Kernel/GDNTreeVerify/ConvTreeScan";
+
+ let context = crate::tests::util::shared_metal_context();
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::ConvTreeScanKernel::new(
+ &context,
+ DataType::BF16,
+ KERNEL_SIZE as u32,
+ true,
+ )
+ .expect("kernel");
+ let weights = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.01; BENCH_CONV_DIM * KERNEL_SIZE]);
+ let bias = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.0; BENCH_CONV_DIM]);
+ let base_state = alloc_allocation_with_data::<Metal, f32>(&context, &vec![0.0; BENCH_CONV_DIM * STATE_STRIDE]);
+ let mut group = c.benchmark_group(BENCHMARK);
+ group.sample_size(30).warm_up_time(Duration::from_millis(300)).measurement_time(Duration::from_secs(1));
+
+ for tree_size in [49usize, 64, 128] {
+ let parents = alloc_allocation_with_data::<Metal, i32>(&context, &parents(tree_size, "binary"));
+ let input_len = tree_size * BENCH_TOTAL_PROJ_DIM;
+ let state_len = tree_size * BENCH_CONV_DIM * STATE_STRIDE;
+ let mut buffers = ColdPool::new(2 * input_len * size_of::<bf16>() + state_len * size_of::<f32>(), || Buffers {
+ input: alloc_allocation_with_data::<Metal, bf16>(&context, &vec![bf16::from_f32(0.1); input_len]),
+ output: alloc_allocation::<Metal, bf16>(&context, input_len),
+ suffix_state: alloc_allocation::<Metal, f32>(&context, state_len),
+ });
+ group.bench_function(format!("T{tree_size}"), |bencher| {
+ iter_encode_loop_named::<Metal, _>(&context, bencher, &format!("{BENCHMARK}/T{tree_size}"), |encoder| {
+ let buffers = buffers.next_mut();
+ kernel.encode(
+ &buffers.input,
+ &weights,
+ Some(&bias),
+ &base_state,
+ &parents,
+ &mut buffers.output,
+ &mut buffers.suffix_state,
+ tree_size as u32,
+ BENCH_TOTAL_PROJ_DIM as u32,
+ BENCH_CONV_DIM as u32,
+ encoder,
+ );
+ });
+ });
+ }
+}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs
new file mode 100644
index 000000000..3bab06e07
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/mod.rs
@@ -0,0 +1,9 @@
+mod conv_scan_test;
+mod out_bench;
+mod out_test;
+mod prefix_beta_bench;
+mod prefix_beta_test;
+mod tree_gram_bench;
+mod tree_gram_test;
+mod tree_update_solve_bench;
+mod tree_update_solve_test;
diff --git a/crates/backend-uzu/unit/backends/common/kernel/out_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/out_bench.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_bench.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/out_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/out_test.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/out_test.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/prefix_beta_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/prefix_beta_bench.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_bench.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/prefix_beta_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/prefix_beta_test.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/prefix_beta_test.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/tree_gram_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/tree_gram_bench.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_bench.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/tree_gram_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/tree_gram_test.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_gram_test.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/tree_update_solve_bench.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/tree_update_solve_bench.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_bench.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/tree_update_solve_test.rs b/crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
similarity index 100%
rename from crates/backend-uzu/unit/backends/common/kernel/tree_update_solve_test.rs
rename to crates/backend-uzu/unit/backends/common/kernel/gdn/tree_verify/tree_update_solve_test.rs
diff --git a/crates/backend-uzu/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/mod.rs
index e3af1ea6f..442f83e34 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/mod.rs
@@ -1,18 +1,14 @@
mod activation_test;
mod attention;
-mod delta_net_test;
mod embedding;
mod gated_act_mul_test;
+mod gdn;
mod hadamard_transform_test;
mod kv_cache_update_test;
mod logit_soft_cap_test;
mod matmul;
mod moe;
-mod out_bench;
-mod out_test;
mod pooling;
-mod prefix_beta_bench;
-mod prefix_beta_test;
mod qkv_norm_test;
mod short_conv;
mod ssm;
@@ -21,7 +17,3 @@ mod tensor_add_scale_test;
mod tensor_add_swap_test;
mod tensor_copy_test;
mod token_copy_test;
-mod tree_gram_bench;
-mod tree_gram_test;
-mod tree_update_solve_bench;
-mod tree_update_solve_test;