Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

AttentionLastQuery for Weaver (#618)

CCYeh5ebe4e9c9336
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs
new file mode 100644
index 000000000..d8611830c
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs
@@ -0,0 +1,81 @@
+use half::bf16;
+use proc_macros::kernel;
+
+use super::attention_single_pass::attention_single_pass;
+
+#[kernel(AttentionLastQuery)]
+#[variants(HEAD_DIM, 128)]
+fn attention_last_query<const HEAD_DIM: u32>(
+ prefix_qkv: *const bf16,
+ node_qkv: *mut bf16,
+ current_qkv: *const bf16,
+ ancestor_indices: *const u32,
+ ancestor_counts: *const u32,
+ node_indices: *const u32,
+ output: *mut bf16,
+ rows: u32,
+ prefix_length: u32,
+ ancestor_stride: u32,
+ scale: f32,
+ #[specialize] num_heads: u32,
+) {
+ const QKV_COMPONENTS: usize = 3;
+ const KV_COMPONENTS: usize = 2;
+ const KEY_COMPONENT: usize = 1;
+ const VALUE_COMPONENT: usize = 2;
+ let head_dim = HEAD_DIM as usize;
+ let num_heads = num_heads as usize;
+ let qkv_width = QKV_COMPONENTS * num_heads * head_dim;
+ let kv_offset = KEY_COMPONENT * num_heads * head_dim;
+ let kv_width = KV_COMPONENTS * num_heads * head_dim;
+
+ for row in 0..rows as usize {
+ unsafe {
+ let current_row = current_qkv.add(row * qkv_width);
+ let dest_node = node_qkv.add(*node_indices.add(row) as usize * qkv_width);
+
+ let ancestor_count = *ancestor_counts.add(row) as usize;
+ let length = prefix_length as usize + ancestor_count + 1;
+ let mut sequence = vec![bf16::ZERO; length * qkv_width];
+ std::ptr::copy_nonoverlapping(prefix_qkv, sequence.as_mut_ptr(), prefix_length as usize * qkv_width);
+ for offset in 0..ancestor_count {
+ let ancestor = *ancestor_indices.add(row * ancestor_stride as usize + offset) as usize;
+ std::ptr::copy_nonoverlapping(
+ node_qkv.add(ancestor * qkv_width + kv_offset),
+ sequence.as_mut_ptr().add((prefix_length as usize + offset) * qkv_width + kv_offset),
+ kv_width,
+ );
+ }
+ std::ptr::copy_nonoverlapping(
+ current_row.add(kv_offset),
+ sequence.as_mut_ptr().add((length - 1) * qkv_width + kv_offset),
+ kv_width,
+ );
+ attention_single_pass::<bf16, HEAD_DIM>(
+ current_row,
+ sequence.as_ptr().add(kv_offset),
+ sequence.as_ptr().add(VALUE_COMPONENT * num_heads * head_dim),
+ output.add(row * num_heads * head_dim),
+ 1,
+ length as u32,
+ HEAD_DIM,
+ qkv_width as u32,
+ HEAD_DIM,
+ qkv_width as u32,
+ None,
+ scale,
+ None,
+ None,
+ None,
+ num_heads as u32,
+ 1,
+ false,
+ false,
+ false,
+ false,
+ false,
+ );
+ std::ptr::copy_nonoverlapping(current_row.add(kv_offset), dest_node.add(kv_offset), kv_width);
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
index c60c4c8e1..813c08dce 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/mod.rs
@@ -1,4 +1,5 @@
pub mod attention_fallback;
+pub mod attention_last_query;
pub mod attention_prepare;
pub mod attention_single_pass;
pub mod attention_two_pass;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal
new file mode 100644
index 000000000..1fead0584
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal
@@ -0,0 +1,127 @@
+#include <metal_simdgroup>
+#include <metal_stdlib>
+#include "../common/defines.h"
+#include "../common/integral_constant.h"
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+
+using namespace metal;
+
+#define SIMD_GROUPS_PER_THREADGROUP 4
+#define THREADS_PER_THREADGROUP (SIMD_GROUPS_PER_THREADGROUP * METAL_SIMD_SIZE)
+
+constant uint VALUES_PER_VECTOR = 4;
+constant uint QKV_COMPONENTS = 3;
+constant uint KEY_COMPONENT = 1;
+constant uint VALUE_COMPONENT = 2;
+
+template <ushort UNROLL_COUNT, typename GetPtrBaseFn>
+METAL_FUNC void attend_qkv(
+ float4 query,
+ uint key_offset,
+ uint value_offset,
+ thread float4& values,
+ thread float& max_score,
+ thread float& sum,
+ GetPtrBaseFn get_ptr_base
+) {
+ float4 keys[UNROLL_COUNT];
+ float4 position_values[UNROLL_COUNT];
+ uzu::const_for_loop<0, UNROLL_COUNT, 1>([&](auto position) {
+ const device bfloat4* vectors = get_ptr_base(position);
+ keys[position] = float4(vectors[key_offset]);
+ position_values[position] = float4(vectors[value_offset]);
+ });
+ float scores[UNROLL_COUNT];
+ uzu::const_for_loop<0, UNROLL_COUNT, 1>([&](auto position) {
+ scores[position] = simd_sum(dot(query, keys[position]));
+ });
+ float new_max = max_score;
+ uzu::const_for_loop<0, UNROLL_COUNT, 1>([&](auto position) { new_max = max(new_max, scores[position]); });
+ const float old_factor = fast::exp(max_score - new_max);
+ sum *= old_factor;
+ values *= old_factor;
+ uzu::const_for_loop<0, UNROLL_COUNT, 1>([&](auto position) {
+ const float factor = fast::exp(scores[position] - new_max);
+ sum += factor;
+ values += factor * position_values[position];
+ });
+ max_score = new_max;
+}
+
+template <uint HEAD_DIM>
+VARIANTS(HEAD_DIM, 128)
+PUBLIC KERNEL(AttentionLastQuery)(
+ const device bfloat* prefix_qkv,
+ device bfloat* node_qkv,
+ const device bfloat* current_qkv,
+ const device uint* ancestor_indices,
+ const device uint* ancestor_counts,
+ const device uint* node_indices,
+ device bfloat* output,
+ constant uint& rows,
+ constant uint& prefix_length,
+ constant uint& ancestor_stride,
+ constant float& scale,
+ const uint num_heads SPECIALIZE,
+ const ThreadContext thread_context,
+ const uint group GROUPS((rows * num_heads).div_ceil(SIMD_GROUPS_PER_THREADGROUP)),
+ const uint lid THREADS(THREADS_PER_THREADGROUP)
+) {
+ const uint row_head = group * SIMD_GROUPS_PER_THREADGROUP + thread_context.simdgroup_index;
+ if (row_head >= rows * num_heads) {
+ return;
+ }
+
+ constexpr uint vectors_per_head = HEAD_DIM / VALUES_PER_VECTOR;
+ constexpr ushort unroll_count = 4;
+ const uint row = row_head / num_heads;
+ const uint head = row_head % num_heads;
+ const uint lane = thread_context.simd_lane_id;
+ const uint qkv_vectors = QKV_COMPONENTS * num_heads * vectors_per_head;
+ const uint key_offset = (KEY_COMPONENT * num_heads + head) * vectors_per_head + lane;
+ const uint value_offset = (VALUE_COMPONENT * num_heads + head) * vectors_per_head + lane;
+ const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_qkv;
+ const device bfloat4* node_vectors = (const device bfloat4*)node_qkv;
+ const device bfloat4* current_row = (const device bfloat4*)current_qkv + row * qkv_vectors;
+ device bfloat4* dest_node = (device bfloat4*)node_qkv + node_indices[row] * qkv_vectors;
+
+ const float4 query = float4(current_row[head * vectors_per_head + lane]) * scale;
+
+ // prefix_length >= 1 (the u0 token): seed the online softmax from position 0.
+ float max_score = simd_sum(dot(query, float4(prefix_vectors[key_offset])));
+ float4 values = float4(prefix_vectors[value_offset]);
+ float sum = 1.0f;
+
+ uint position = 1;
+ for (; position + unroll_count - 1 < prefix_length; position += unroll_count) {
+ attend_qkv<unroll_count>(query, key_offset, value_offset, values, max_score, sum, [&](int step) {
+ return prefix_vectors + (position + step) * qkv_vectors;
+ });
+ }
+ for (; position < prefix_length; ++position) {
+ attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) {
+ return prefix_vectors + position * qkv_vectors;
+ });
+ }
+
+ const uint ancestor_count = ancestor_counts[row];
+ const device uint* row_ancestors = ancestor_indices + row * ancestor_stride;
+ uint offset = 0;
+ for (; offset + unroll_count - 1 < ancestor_count; offset += unroll_count) {
+ attend_qkv<unroll_count>(query, key_offset, value_offset, values, max_score, sum, [&](int step) {
+ return node_vectors + row_ancestors[offset + step] * qkv_vectors;
+ });
+ }
+ for (; offset < ancestor_count; ++offset) {
+ attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) {
+ return node_vectors + row_ancestors[offset] * qkv_vectors;
+ });
+ }
+
+ attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) { return current_row; });
+
+ dest_node[key_offset] = current_row[key_offset];
+ dest_node[value_offset] = current_row[value_offset];
+ ((device bfloat4*)output)[(row * num_heads + head) * vectors_per_head + lane] = bfloat4(values / sum);
+}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/attention/attention_last_query_test.rs b/crates/backend-uzu/unit/backends/common/kernel/attention/attention_last_query_test.rs
new file mode 100644
index 000000000..3f31f6aae
--- /dev/null
+++ b/crates/backend-uzu/unit/backends/common/kernel/attention/attention_last_query_test.rs
@@ -0,0 +1,163 @@
+#![cfg(metal_backend)]
+
+use std::{sync::Arc, time::Duration};
+
+use half::bf16;
+use proc_macros::uzu_test;
+
+use crate::{
+ backends::{
+ common::{Allocation, Backend, Encoder, Kernels, kernel::AttentionLastQueryKernel},
+ cpu::Cpu,
+ metal::Metal,
+ },
+ tests::{
+ assert::assert_eq_float,
+ helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},
+ },
+};
+
+const HEAD_DIM: usize = 128;
+const NUM_HEADS: u32 = 16;
+
+struct Runner<B: Backend> {
+ context: Arc<B::Context>,
+ kernel: <B::Kernels as Kernels>::AttentionLastQueryKernel,
+ prefix_qkv: Allocation<B>,
+ node_qkv: Allocation<B>,
+ current_qkv: Allocation<B>,
+ ancestor_indices: Allocation<B>,
+ ancestor_counts: Allocation<B>,
+ node_indices: Allocation<B>,
+ output: Allocation<B>,
+ rows: u32,
+ prefix_length: u32,
+ ancestor_stride: u32,
+}
+
+impl<B: Backend> Runner<B> {
+ fn new(
+ rows: usize,
+ prefix_length: usize,
+ ancestor_stride: usize,
+ nodes: usize,
+ ) -> Self {
+ assert!(rows < nodes && ancestor_stride > 0);
+ let packed_width = 3 * NUM_HEADS as usize * HEAD_DIM;
+ let values = |length, offset| {
+ (0..length)
+ .map(|index| bf16::from_f32((((index + offset) * 17 % 251) as f32 - 125.0) / 128.0))
+ .collect::<Vec<_>>()
+ };
+ let first_output_node = nodes - rows;
+ let ancestor_counts = (0..rows).map(|row| (row % (ancestor_stride + 1)) as u32).collect::<Vec<_>>();
+ let mut ancestor_indices = vec![0; rows * ancestor_stride];
+ for row in 0..rows {
+ for offset in 0..ancestor_counts[row] as usize {
+ ancestor_indices[row * ancestor_stride + offset] =
+ ((row * ancestor_stride + offset) % first_output_node) as u32;
+ }
+ }
+
+ let context = create_context::<B>();
+ let kernel =
+ <B::Kernels as Kernels>::AttentionLastQueryKernel::new(context.as_ref(), HEAD_DIM as u32, NUM_HEADS)
+ .unwrap();
+ Self {
+ prefix_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(prefix_length * packed_width, 0)),
+ node_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(nodes * packed_width, 11)),
+ current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * packed_width, 29)),
+ ancestor_indices: alloc_allocation_with_data::<B, u32>(&context, &ancestor_indices),
+ ancestor_counts: alloc_allocation_with_data::<B, u32>(&context, &ancestor_counts),
+ node_indices: alloc_allocation_with_data::<B, u32>(
+ &context,
+ &(first_output_node as u32..nodes as u32).collect::<Vec<_>>(),
+ ),
+ output: alloc_allocation::<B, bf16>(&context, rows * NUM_HEADS as usize * HEAD_DIM),
+ context,
+ kernel,
+ rows: rows as u32,
+ prefix_length: prefix_length as u32,
+ ancestor_stride: ancestor_stride as u32,
+ }
+ }
+
+ fn encode(
+ &mut self,
+ repetitions: u32,
+ ) -> Duration {
+ let mut encoder = Encoder::new(self.context.as_ref()).unwrap();
+ for _ in 0..repetitions {
+ self.kernel.encode(
+ &self.prefix_qkv,
+ &mut self.node_qkv,
+ &self.current_qkv,
+ &self.ancestor_indices,
+ &self.ancestor_counts,
+ &self.node_indices,
+ &mut self.output,
+ self.rows,
+ self.prefix_length,
+ self.ancestor_stride,
+ 1.0 / (HEAD_DIM as f32).sqrt(),
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
+ }
+}
+
+#[uzu_test]
+fn attention_last_query_matches_cpu() {
+ let mut cpu = Runner::<Cpu>::new(4, 5, 3, 16);
+ let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);
+ let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);
+ let node_indices = allocation_to_vec::<Cpu, u32>(&cpu.node_indices);
+ cpu.encode(1);
+ let expected_output = allocation_to_vec::<Cpu, bf16>(&cpu.output);
+ let expected_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);
+
+ let packed_width = 3 * NUM_HEADS as usize * HEAD_DIM;
+ let q_width = NUM_HEADS as usize * HEAD_DIM;
+ for node in 0..expected_nodes.len() / packed_width {
+ let range = node * packed_width..(node + 1) * packed_width;
+ if let Some(row) = node_indices.iter().position(|&index| index as usize == node) {
+ let current = row * packed_width;
+ assert_eq!(
+ expected_nodes[range.start..range.start + q_width],
+ initial_nodes[range.start..range.start + q_width]
+ );
+ assert_eq!(
+ expected_nodes[range.start + q_width..range.end],
+ current_qkv[current + q_width..current + packed_width],
+ );
+ } else {
+ assert_eq!(expected_nodes[range.clone()], initial_nodes[range]);
+ }
+ }
+
+ let mut metal = Runner::<Metal>::new(4, 5, 3, 16);
+ metal.encode(1);
+ let actual_output = allocation_to_vec::<Metal, bf16>(&metal.output);
+ let actual_nodes = allocation_to_vec::<Metal, bf16>(&metal.node_qkv);
+
+ assert_eq_float(&expected_output, &actual_output, 0.02, "AttentionLastQuery output");
+ assert_eq!(expected_nodes, actual_nodes);
+}
+
+#[uzu_test]
+#[ignore = "benchmark"]
+fn benchmark_attention_last_query() {
+ const BATCH: u32 = 32;
+ const SAMPLES: usize = 50;
+
+ let mut runner = Runner::<Metal>::new(8, 16, 8, 65);
+ let mut run = || runner.encode(BATCH).div_f64(BATCH as f64);
+ let warmup = std::time::Instant::now();
+ while warmup.elapsed() < std::time::Duration::from_millis(500) {
+ run();
+ }
+ let mut samples = (0..SAMPLES).map(|_| run()).collect::<Vec<_>>();
+ samples.sort_unstable();
+ eprintln!("attention_last_query gpu={:?}", samples[SAMPLES / 2]);
+}
diff --git a/crates/backend-uzu/unit/backends/common/kernel/attention/mod.rs b/crates/backend-uzu/unit/backends/common/kernel/attention/mod.rs
index d843ffc6d..0b723476d 100644
--- a/crates/backend-uzu/unit/backends/common/kernel/attention/mod.rs
+++ b/crates/backend-uzu/unit/backends/common/kernel/attention/mod.rs
@@ -1,4 +1,5 @@
mod attention_fallback_test;
+mod attention_last_query_test;
mod attention_single_pass_test;
mod attention_two_pass_test;
mod sigmoid_gate_test;