A high-performance inference engine for AI models
AttentionLastQuery for Weaver (#618)
CCYeh5ebe4e9c9336
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rsnew file mode 100644index 000000000..d8611830c--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs@@ -0,0 +1,81 @@+use half::bf16;+use proc_macros::kernel;++use super::attention_single_pass::attention_single_pass;++#[kernel(AttentionLastQuery)]+#[variants(HEAD_DIM, 128)]+fn attention_last_query<const HEAD_DIM: u32>(+ prefix_qkv: *const bf16,+ node_qkv: *mut bf16,+ current_qkv: *const bf16,+ ancestor_indices: *const u32,+ ancestor_counts: *const u32,+ node_indices: *const u32,+ output: *mut bf16,+ rows: u32,+ prefix_length: u32,+ ancestor_stride: u32,+ scale: f32,+ #[specialize] num_heads: u32,+) {+ const QKV_COMPONENTS: usize = 3;+ const KV_COMPONENTS: usize = 2;+ const KEY_COMPONENT: usize = 1;+ const VALUE_COMPONENT: usize = 2;+ let head_dim = HEAD_DIM as usize;+ let num_heads = num_heads as usize;+ let qkv_width = QKV_COMPONENTS * num_heads * head_dim;+ let kv_offset = KEY_COMPONENT * num_heads * head_dim;+ let kv_width = KV_COMPONENTS * num_heads * head_dim;++ for row in 0..rows as usize {+ unsafe {+ let current_row = current_qkv.add(row * qkv_width);+ let dest_node = node_qkv.add(*node_indices.add(row) as usize * qkv_width);++ let ancestor_count = *ancestor_counts.add(row) as usize;+ let length = prefix_length as usize + ancestor_count + 1;+ let mut sequence = vec![bf16::ZERO; length * qkv_width];+ std::ptr::copy_nonoverlapping(prefix_qkv, sequence.as_mut_ptr(), prefix_length as usize * qkv_width);+ for offset in 0..ancestor_count {+ let ancestor = *ancestor_indices.add(row * ancestor_stride as usize + offset) as usize;+ std::ptr::copy_nonoverlapping(+ node_qkv.add(ancestor * qkv_width + kv_offset),+ sequence.as_mut_ptr().add((prefix_length as usize + offset) * qkv_width + kv_offset),+ kv_width,+ );+ }+ std::ptr::copy_nonoverlapping(+ current_row.add(kv_offset),+ sequence.as_mut_ptr().add((length - 1) * qkv_width + kv_offset),+ kv_width,+ );+ attention_single_pass::<bf16, HEAD_DIM>(+ current_row,+ sequence.as_ptr().add(kv_offset),+ sequence.as_ptr().add(VALUE_COMPONENT * num_heads * head_dim),+ output.add(row * num_heads * head_dim),+ 1,+ length as u32,+ HEAD_DIM,+ qkv_width as u32,+ HEAD_DIM,+ qkv_width as u32,+ None,+ scale,+ None,+ None,+ None,+ num_heads as u32,+ 1,+ false,+ false,+ false,+ false,+ false,+ );+ std::ptr::copy_nonoverlapping(current_row.add(kv_offset), dest_node.add(kv_offset), kv_width);+ }+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rsindex c60c4c8e1..813c08dce 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs@@ -1,4 +1,5 @@pub mod attention_fallback;+pub mod attention_last_query;pub mod attention_prepare;pub mod attention_single_pass;pub mod attention_two_pass;diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metalnew file mode 100644index 000000000..1fead0584--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal@@ -0,0 +1,127 @@+#include <metal_simdgroup>+#include <metal_stdlib>+#include "../common/defines.h"+#include "../common/integral_constant.h"+#include "../common/dsl.h"+#include "../common/thread_context.h"++using namespace metal;++#define SIMD_GROUPS_PER_THREADGROUP 4+#define THREADS_PER_THREADGROUP (SIMD_GROUPS_PER_THREADGROUP * METAL_SIMD_SIZE)++constant uint VALUES_PER_VECTOR = 4;+constant uint QKV_COMPONENTS = 3;+constant uint KEY_COMPONENT = 1;+constant uint VALUE_COMPONENT = 2;++template <ushort UNROLL_COUNT, typename GetPtrBaseFn>+METAL_FUNC void attend_qkv(+ float4 query,+ uint key_offset,+ uint value_offset,+ thread float4& values,+ thread float& max_score,+ thread float& sum,+ GetPtrBaseFn get_ptr_base+) {+ float4 keys[UNROLL_COUNT];+ float4 position_values[UNROLL_COUNT];+ uzu::const_for_loop<0, UNROLL_COUNT, 1>([&](auto position) {+ const device bfloat4* vectors = get_ptr_base(position);+ keys[position] = float4(vectors[key_offset]);+ position_values[position] = float4(vectors[value_offset]);+ });+ float scores[UNROLL_COUNT];+ uzu::const_for_loop<0, UNROLL_COUNT, 1>([&](auto position) {+ scores[position] = simd_sum(dot(query, keys[position]));+ });+ float new_max = max_score;+ uzu::const_for_loop<0, UNROLL_COUNT, 1>([&](auto position) { new_max = max(new_max, scores[position]); });+ const float old_factor = fast::exp(max_score - new_max);+ sum *= old_factor;+ values *= old_factor;+ uzu::const_for_loop<0, UNROLL_COUNT, 1>([&](auto position) {+ const float factor = fast::exp(scores[position] - new_max);+ sum += factor;+ values += factor * position_values[position];+ });+ max_score = new_max;+}++template <uint HEAD_DIM>+VARIANTS(HEAD_DIM, 128)+PUBLIC KERNEL(AttentionLastQuery)(+ const device bfloat* prefix_qkv,+ device bfloat* node_qkv,+ const device bfloat* current_qkv,+ const device uint* ancestor_indices,+ const device uint* ancestor_counts,+ const device uint* node_indices,+ device bfloat* output,+ constant uint& rows,+ constant uint& prefix_length,+ constant uint& ancestor_stride,+ constant float& scale,+ const uint num_heads SPECIALIZE,+ const ThreadContext thread_context,+ const uint group GROUPS((rows * num_heads).div_ceil(SIMD_GROUPS_PER_THREADGROUP)),+ const uint lid THREADS(THREADS_PER_THREADGROUP)+) {+ const uint row_head = group * SIMD_GROUPS_PER_THREADGROUP + thread_context.simdgroup_index;+ if (row_head >= rows * num_heads) {+ return;+ }++ constexpr uint vectors_per_head = HEAD_DIM / VALUES_PER_VECTOR;+ constexpr ushort unroll_count = 4;+ const uint row = row_head / num_heads;+ const uint head = row_head % num_heads;+ const uint lane = thread_context.simd_lane_id;+ const uint qkv_vectors = QKV_COMPONENTS * num_heads * vectors_per_head;+ const uint key_offset = (KEY_COMPONENT * num_heads + head) * vectors_per_head + lane;+ const uint value_offset = (VALUE_COMPONENT * num_heads + head) * vectors_per_head + lane;+ const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_qkv;+ const device bfloat4* node_vectors = (const device bfloat4*)node_qkv;+ const device bfloat4* current_row = (const device bfloat4*)current_qkv + row * qkv_vectors;+ device bfloat4* dest_node = (device bfloat4*)node_qkv + node_indices[row] * qkv_vectors;++ const float4 query = float4(current_row[head * vectors_per_head + lane]) * scale;++ // prefix_length >= 1 (the u0 token): seed the online softmax from position 0.+ float max_score = simd_sum(dot(query, float4(prefix_vectors[key_offset])));+ float4 values = float4(prefix_vectors[value_offset]);+ float sum = 1.0f;++ uint position = 1;+ for (; position + unroll_count - 1 < prefix_length; position += unroll_count) {+ attend_qkv<unroll_count>(query, key_offset, value_offset, values, max_score, sum, [&](int step) {+ return prefix_vectors + (position + step) * qkv_vectors;+ });+ }+ for (; position < prefix_length; ++position) {+ attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) {+ return prefix_vectors + position * qkv_vectors;+ });+ }++ const uint ancestor_count = ancestor_counts[row];+ const device uint* row_ancestors = ancestor_indices + row * ancestor_stride;+ uint offset = 0;+ for (; offset + unroll_count - 1 < ancestor_count; offset += unroll_count) {+ attend_qkv<unroll_count>(query, key_offset, value_offset, values, max_score, sum, [&](int step) {+ return node_vectors + row_ancestors[offset + step] * qkv_vectors;+ });+ }+ for (; offset < ancestor_count; ++offset) {+ attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) {+ return node_vectors + row_ancestors[offset] * qkv_vectors;+ });+ }++ attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) { return current_row; });++ dest_node[key_offset] = current_row[key_offset];+ dest_node[value_offset] = current_row[value_offset];+ ((device bfloat4*)output)[(row * num_heads + head) * vectors_per_head + lane] = bfloat4(values / sum);+}diff --git a/crates/backend-uzu/unit/backends/common/kernel/attention/attention_last_query_test.rs b/crates/backend-uzu/unit/backends/common/kernel/attention/attention_last_query_test.rsnew file mode 100644index 000000000..3f31f6aae--- /dev/null+++ b/crates/backend-uzu/unit/backends/common/kernel/attention/attention_last_query_test.rs@@ -0,0 +1,163 @@+#![cfg(metal_backend)]++use std::{sync::Arc, time::Duration};++use half::bf16;+use proc_macros::uzu_test;++use crate::{+ backends::{+ common::{Allocation, Backend, Encoder, Kernels, kernel::AttentionLastQueryKernel},+ cpu::Cpu,+ metal::Metal,+ },+ tests::{+ assert::assert_eq_float,+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},+ },+};++const HEAD_DIM: usize = 128;+const NUM_HEADS: u32 = 16;++struct Runner<B: Backend> {+ context: Arc<B::Context>,+ kernel: <B::Kernels as Kernels>::AttentionLastQueryKernel,+ prefix_qkv: Allocation<B>,+ node_qkv: Allocation<B>,+ current_qkv: Allocation<B>,+ ancestor_indices: Allocation<B>,+ ancestor_counts: Allocation<B>,+ node_indices: Allocation<B>,+ output: Allocation<B>,+ rows: u32,+ prefix_length: u32,+ ancestor_stride: u32,+}++impl<B: Backend> Runner<B> {+ fn new(+ rows: usize,+ prefix_length: usize,+ ancestor_stride: usize,+ nodes: usize,+ ) -> Self {+ assert!(rows < nodes && ancestor_stride > 0);+ let packed_width = 3 * NUM_HEADS as usize * HEAD_DIM;+ let values = |length, offset| {+ (0..length)+ .map(|index| bf16::from_f32((((index + offset) * 17 % 251) as f32 - 125.0) / 128.0))+ .collect::<Vec<_>>()+ };+ let first_output_node = nodes - rows;+ let ancestor_counts = (0..rows).map(|row| (row % (ancestor_stride + 1)) as u32).collect::<Vec<_>>();+ let mut ancestor_indices = vec![0; rows * ancestor_stride];+ for row in 0..rows {+ for offset in 0..ancestor_counts[row] as usize {+ ancestor_indices[row * ancestor_stride + offset] =+ ((row * ancestor_stride + offset) % first_output_node) as u32;+ }+ }++ let context = create_context::<B>();+ let kernel =+ <B::Kernels as Kernels>::AttentionLastQueryKernel::new(context.as_ref(), HEAD_DIM as u32, NUM_HEADS)+ .unwrap();+ Self {+ prefix_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(prefix_length * packed_width, 0)),+ node_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(nodes * packed_width, 11)),+ current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * packed_width, 29)),+ ancestor_indices: alloc_allocation_with_data::<B, u32>(&context, &ancestor_indices),+ ancestor_counts: alloc_allocation_with_data::<B, u32>(&context, &ancestor_counts),+ node_indices: alloc_allocation_with_data::<B, u32>(+ &context,+ &(first_output_node as u32..nodes as u32).collect::<Vec<_>>(),+ ),+ output: alloc_allocation::<B, bf16>(&context, rows * NUM_HEADS as usize * HEAD_DIM),+ context,+ kernel,+ rows: rows as u32,+ prefix_length: prefix_length as u32,+ ancestor_stride: ancestor_stride as u32,+ }+ }++ fn encode(+ &mut self,+ repetitions: u32,+ ) -> Duration {+ let mut encoder = Encoder::new(self.context.as_ref()).unwrap();+ for _ in 0..repetitions {+ self.kernel.encode(+ &self.prefix_qkv,+ &mut self.node_qkv,+ &self.current_qkv,+ &self.ancestor_indices,+ &self.ancestor_counts,+ &self.node_indices,+ &mut self.output,+ self.rows,+ self.prefix_length,+ self.ancestor_stride,+ 1.0 / (HEAD_DIM as f32).sqrt(),+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()+ }+}++#[uzu_test]+fn attention_last_query_matches_cpu() {+ let mut cpu = Runner::<Cpu>::new(4, 5, 3, 16);+ let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);+ let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);+ let node_indices = allocation_to_vec::<Cpu, u32>(&cpu.node_indices);+ cpu.encode(1);+ let expected_output = allocation_to_vec::<Cpu, bf16>(&cpu.output);+ let expected_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);++ let packed_width = 3 * NUM_HEADS as usize * HEAD_DIM;+ let q_width = NUM_HEADS as usize * HEAD_DIM;+ for node in 0..expected_nodes.len() / packed_width {+ let range = node * packed_width..(node + 1) * packed_width;+ if let Some(row) = node_indices.iter().position(|&index| index as usize == node) {+ let current = row * packed_width;+ assert_eq!(+ expected_nodes[range.start..range.start + q_width],+ initial_nodes[range.start..range.start + q_width]+ );+ assert_eq!(+ expected_nodes[range.start + q_width..range.end],+ current_qkv[current + q_width..current + packed_width],+ );+ } else {+ assert_eq!(expected_nodes[range.clone()], initial_nodes[range]);+ }+ }++ let mut metal = Runner::<Metal>::new(4, 5, 3, 16);+ metal.encode(1);+ let actual_output = allocation_to_vec::<Metal, bf16>(&metal.output);+ let actual_nodes = allocation_to_vec::<Metal, bf16>(&metal.node_qkv);++ assert_eq_float(&expected_output, &actual_output, 0.02, "AttentionLastQuery output");+ assert_eq!(expected_nodes, actual_nodes);+}++#[uzu_test]+#[ignore = "benchmark"]+fn benchmark_attention_last_query() {+ const BATCH: u32 = 32;+ const SAMPLES: usize = 50;++ let mut runner = Runner::<Metal>::new(8, 16, 8, 65);+ let mut run = || runner.encode(BATCH).div_f64(BATCH as f64);+ let warmup = std::time::Instant::now();+ while warmup.elapsed() < std::time::Duration::from_millis(500) {+ run();+ }+ let mut samples = (0..SAMPLES).map(|_| run()).collect::<Vec<_>>();+ samples.sort_unstable();+ eprintln!("attention_last_query gpu={:?}", samples[SAMPLES / 2]);+}diff --git a/crates/backend-uzu/unit/backends/common/kernel/attention/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/attention/mod.rsindex d843ffc6d..0b723476d 100644--- a/crates/backend-uzu/unit/backends/common/kernel/attention/mod.rs+++ b/crates/backend-uzu/unit/backends/common/kernel/attention/mod.rs@@ -1,4 +1,5 @@mod attention_fallback_test;+mod attention_last_query_test;mod attention_single_pass_test;mod attention_two_pass_test;mod sigmoid_gate_test;