Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Weaver forward (#624)

CCYeh0d7768e51725
diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rs
index 41bd7ee75..99bb99c76 100644
--- a/crates/backend-uzu/src/backends/common/encoder.rs
+++ b/crates/backend-uzu/src/backends/common/encoder.rs
@@ -1,9 +1,12 @@
use std::{
+ mem::size_of_val,
ops::{Bound, Range, RangeBounds},
sync::Arc,
time::Duration,
};
+use bytemuck::{AnyBitPattern, NoUninit};
+
use crate::backends::common::{
AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer,
BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,
@@ -72,6 +75,15 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
)
}
+ pub fn allocate_constant_from_slice<T: NoUninit + AnyBitPattern>(
+ &mut self,
+ data: &[T],
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut allocation = self.allocate_constant(size_of_val(data))?;
+ allocation.copyin(data);
+ Ok(allocation)
+ }
+
// This is valid on gpu timeline only
pub fn allocate_scratch(
&mut self,
diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rs
index b96024331..d23a882ec 100644
--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs
@@ -5,6 +5,7 @@ pub mod delta_net_chunked_prefill;
pub mod delta_net_tree_verify;
pub mod matmul;
pub mod radix_top_k_small;
+pub(crate) mod weaver;
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
diff --git a/crates/backend-uzu/src/backends/common/kernel/weaver/mod.rs b/crates/backend-uzu/src/backends/common/kernel/weaver/mod.rs
new file mode 100644
index 000000000..81b3d20d4
--- /dev/null
+++ b/crates/backend-uzu/src/backends/common/kernel/weaver/mod.rs
@@ -0,0 +1 @@
+pub(crate) const MAX_CANDIDATES: usize = 512;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs
index d8611830c..1ba4acb36 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs
@@ -7,11 +7,10 @@ use super::attention_single_pass::attention_single_pass;
#[variants(HEAD_DIM, 128)]
fn attention_last_query<const HEAD_DIM: u32>(
prefix_qkv: *const bf16,
- node_qkv: *mut bf16,
+ node_qkv: *const bf16,
current_qkv: *const bf16,
ancestor_indices: *const u32,
ancestor_counts: *const u32,
- node_indices: *const u32,
output: *mut bf16,
rows: u32,
prefix_length: u32,
@@ -32,7 +31,6 @@ fn attention_last_query<const HEAD_DIM: u32>(
for row in 0..rows as usize {
unsafe {
let current_row = current_qkv.add(row * qkv_width);
- let dest_node = node_qkv.add(*node_indices.add(row) as usize * qkv_width);
let ancestor_count = *ancestor_counts.add(row) as usize;
let length = prefix_length as usize + ancestor_count + 1;
@@ -75,7 +73,6 @@ fn attention_last_query<const HEAD_DIM: u32>(
false,
false,
);
- std::ptr::copy_nonoverlapping(current_row.add(kv_offset), dest_node.add(kv_offset), kv_width);
}
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
index 9675a849a..046f70af0 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs
@@ -22,6 +22,7 @@ mod tensor_add_bias;
mod tensor_add_scale;
mod tensor_add_swap;
mod tensor_copy;
+mod weaver;
include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
index b07a5e976..86da05d80 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs
@@ -12,15 +12,24 @@ pub fn tensor_add_bias<T: ArrayElement + Float, BiasT: ArrayElement + Float>(
#[optional(!in_place)]
input: Option<*const T>,
#[allow(unused)] bias: *const BiasT,
+ #[allow(unused)]
+ #[optional(indexed)]
+ bias_row_indices: Option<*const u32>,
#[allow(unused)] output: *mut T,
#[allow(unused)] num_cols: u32,
#[allow(unused)] length: u32,
#[specialize] in_place: bool,
+ #[specialize] indexed: bool,
) {
assert_eq!(input.is_none(), in_place);
+ assert_eq!(bias_row_indices.is_some(), indexed);
for i in 0usize..(length as usize) {
- let bias_position = i % num_cols as usize;
+ let column = i % num_cols as usize;
+ let bias_position = match bias_row_indices {
+ Some(indices) => unsafe { *indices.add(i / num_cols as usize) as usize * num_cols as usize + column },
+ None => column,
+ };
unsafe {
let value: f32 = if let Some(in_data) = input {
num_traits::cast(*in_data.add(i)).unwrap()
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs
new file mode 100644
index 000000000..c70352ab3
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs
@@ -0,0 +1,2 @@
+pub mod weaver_node_cache_write;
+pub mod weaver_top_children;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs
new file mode 100644
index 000000000..94d831d26
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs
@@ -0,0 +1,26 @@
+use half::bf16;
+use num_traits::Float;
+use proc_macros::kernel;
+
+use crate::array::ArrayElement;
+
+#[kernel(WeaverNodeCacheWrite)]
+#[variants(T, f32, bf16)]
+pub fn weaver_node_cache_write<T: ArrayElement + Float>(
+ current_qkv: *const T,
+ node_qkv: *mut T,
+ node_indices: *const u32,
+ model_dim: u32,
+ total: u32,
+) {
+ let model_dim = model_dim as usize;
+ let qkv_width = 3 * model_dim;
+ unsafe {
+ for position in 0..total as usize {
+ let row = position / (2 * model_dim);
+ let offset = position - row * 2 * model_dim;
+ *node_qkv.add(*node_indices.add(row) as usize * qkv_width + model_dim + offset) =
+ *current_qkv.add(row * qkv_width + model_dim + offset);
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
new file mode 100644
index 000000000..c6027f6ed
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs
@@ -0,0 +1,43 @@
+use half::bf16;
+use proc_macros::kernel;
+
+use crate::backends::common::kernel::weaver::MAX_CANDIDATES;
+
+#[kernel(WeaverTopChildren)]
+pub fn weaver_top_children(
+ residual_logits: *const bf16,
+ candidate_scores: *const f32,
+ candidate_ids: *const u32,
+ output_ids: *mut u32,
+ output_logprobs: *mut f32,
+ rows: u32,
+ candidates: u32,
+ children: u32,
+) {
+ let rows = rows as usize;
+ let candidates = candidates as usize;
+ let children = children as usize;
+ if candidates == 0 || candidates > MAX_CANDIDATES || children == 0 || children > candidates {
+ return;
+ }
+ for row in 0..rows {
+ let base = row * candidates;
+ let logits = (0..candidates)
+ .map(|index| unsafe { *candidate_scores.add(base + index) + (*residual_logits.add(base + index)).to_f32() })
+ .collect::<Vec<_>>();
+ let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
+ let log_sum = logits.iter().map(|value| (value - max).exp()).sum::<f32>().ln() + max;
+ let mut indices = (0..candidates).collect::<Vec<_>>();
+ indices.sort_by(|&left, &right| {
+ logits[right]
+ .total_cmp(&logits[left])
+ .then_with(|| unsafe { (*candidate_ids.add(base + left)).cmp(&*candidate_ids.add(base + right)) })
+ });
+ for (rank, index) in indices.into_iter().take(children).enumerate() {
+ unsafe {
+ *output_ids.add(row * children + rank) = *candidate_ids.add(base + index);
+ *output_logprobs.add(row * children + rank) = logits[index] - log_sum;
+ }
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal
index 1fead0584..f4b749570 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal
@@ -53,11 +53,10 @@ template <uint HEAD_DIM>
VARIANTS(HEAD_DIM, 128)
PUBLIC KERNEL(AttentionLastQuery)(
const device bfloat* prefix_qkv,
- device bfloat* node_qkv,
+ const device bfloat* node_qkv,
const device bfloat* current_qkv,
const device uint* ancestor_indices,
const device uint* ancestor_counts,
- const device uint* node_indices,
device bfloat* output,
constant uint& rows,
constant uint& prefix_length,
@@ -84,7 +83,6 @@ PUBLIC KERNEL(AttentionLastQuery)(
const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_qkv;
const device bfloat4* node_vectors = (const device bfloat4*)node_qkv;
const device bfloat4* current_row = (const device bfloat4*)current_qkv + row * qkv_vectors;
- device bfloat4* dest_node = (device bfloat4*)node_qkv + node_indices[row] * qkv_vectors;
const float4 query = float4(current_row[head * vectors_per_head + lane]) * scale;
@@ -121,7 +119,5 @@ PUBLIC KERNEL(AttentionLastQuery)(
attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) { return current_row; });
- dest_node[key_offset] = current_row[key_offset];
- dest_node[value_offset] = current_row[value_offset];
((device bfloat4*)output)[(row * num_heads + head) * vectors_per_head + lane] = bfloat4(values / sum);
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index b99f152c4..ab58a3be9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -48,7 +48,7 @@ impl GemmKernel {
input_data_type: DataType,
output_data_type: DataType,
) -> Result<Self, MetalError> {
- let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true)?;
+ let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true, false)?;
let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(
context,
output_data_type,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
index 8c2541c5d..76913bae1 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal
@@ -7,16 +7,20 @@ VARIANTS(BiasT, float, half, bfloat)
PUBLIC KERNEL(TensorAddBias)(
const device T* input OPTIONAL(!in_place),
const device BiasT* bias,
+ const device uint* bias_row_indices OPTIONAL(indexed),
device T* output,
constant uint& num_cols,
constant uint& length,
const uint position AXIS(length, 32),
- const bool in_place SPECIALIZE
+ const bool in_place SPECIALIZE,
+ const bool indexed SPECIALIZE
) {
if (in_place) {
input = output;
}
- uint col = position % num_cols;
- output[position] = static_cast<T>(static_cast<float>(input[position]) + static_cast<float>(bias[col]));
+ const uint row = position / num_cols;
+ const uint column = position % num_cols;
+ const ulong bias_position = indexed ? ulong(bias_row_indices[row]) * ulong(num_cols) + ulong(column) : column;
+ output[position] = static_cast<T>(static_cast<float>(input[position]) + static_cast<float>(bias[bias_position]));
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal
new file mode 100644
index 000000000..c0a56d3eb
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal
@@ -0,0 +1,21 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+// Scatters the key/value halves of the current step's packed QKV rows into their
+// node-cache slots. Kept out of AttentionLastQuery so that the node arena stays
+// read-only for the whole attention dispatch.
+template <typename T>
+VARIANTS(T, float, bfloat)
+PUBLIC KERNEL(WeaverNodeCacheWrite)(
+ const device T* current_qkv,
+ device T* node_qkv,
+ const device uint* node_indices,
+ constant uint& model_dim,
+ constant uint& total,
+ const uint position AXIS(total, 256)
+) {
+ const uint row = position / (2 * model_dim);
+ const uint offset = position - row * 2 * model_dim;
+ const uint qkv_width = 3 * model_dim;
+ node_qkv[node_indices[row] * qkv_width + model_dim + offset] = current_qkv[row * qkv_width + model_dim + offset];
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal
new file mode 100644
index 000000000..ef7360872
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal
@@ -0,0 +1,147 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/top_k.h"
+
+using namespace metal;
+
+constant uint WEAVER_CANDIDATES_MAX = 512;
+constant uint WEAVER_TOP_CHILDREN_THREADS = 256;
+constant uint WEAVER_TOP_CHILDREN_SIMDGROUPS = 8;
+
+METAL_FUNC bool weaver_better(uint score, uint token, uint index, uint best_score, uint best_token, uint best_index) {
+ return score > best_score ||
+ (score == best_score && (token < best_token || (token == best_token && index < best_index)));
+}
+
+PUBLIC KERNEL(WeaverTopChildren)(
+ const device bfloat* residual_logits,
+ const device float* candidate_scores,
+ const device uint* candidate_ids,
+ device uint* output_ids,
+ device float* output_logprobs,
+ constant uint& rows,
+ constant uint& candidates,
+ constant uint& children,
+ threadgroup float reduce_float[WEAVER_TOP_CHILDREN_SIMDGROUPS],
+ threadgroup uint reduce_score[WEAVER_TOP_CHILDREN_SIMDGROUPS],
+ threadgroup uint reduce_token[WEAVER_TOP_CHILDREN_SIMDGROUPS],
+ threadgroup uint reduce_index[WEAVER_TOP_CHILDREN_SIMDGROUPS],
+ threadgroup float& logit_max,
+ threadgroup float& log_sum,
+ threadgroup uint& winner_score,
+ threadgroup uint& winner_token,
+ threadgroup uint& winner_index,
+ const ThreadContext thread_context,
+ const uint row GROUPS(rows),
+ const uint lid THREADS(256)
+) {
+ if (candidates == 0 || candidates > WEAVER_CANDIDATES_MAX || children == 0 || children > candidates) {
+ return;
+ }
+
+ const uint base = row * candidates;
+ const uint first_index = lid;
+ const uint second_index = lid + WEAVER_TOP_CHILDREN_THREADS;
+ const bool first_valid = first_index < candidates;
+ const bool second_valid = second_index < candidates;
+ const float first_logit =
+ first_valid ? candidate_scores[base + first_index] + float(residual_logits[base + first_index]) : -INFINITY;
+ const float second_logit =
+ second_valid ? candidate_scores[base + second_index] + float(residual_logits[base + second_index]) : -INFINITY;
+ const uint first_token = first_valid ? uint(candidate_ids[base + first_index]) : 0xffffffffu;
+ const uint second_token = second_valid ? uint(candidate_ids[base + second_index]) : 0xffffffffu;
+ const uint first_score = first_valid ? top_k_score_key(first_logit) : 0u;
+ const uint second_score = second_valid ? top_k_score_key(second_logit) : 0u;
+ bool first_active = first_valid;
+ bool second_active = second_valid;
+
+ const float local_max = fmax(first_logit, second_logit);
+ const float simd_maximum = simd_max(local_max);
+ if (thread_context.simd_lane_id == 0) {
+ reduce_float[thread_context.simdgroup_index] = simd_maximum;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (thread_context.simdgroup_index == 0) {
+ const float group_value = thread_context.simd_lane_id < WEAVER_TOP_CHILDREN_SIMDGROUPS
+ ? reduce_float[thread_context.simd_lane_id]
+ : -INFINITY;
+ const float maximum = simd_max(group_value);
+ if (thread_context.simd_lane_id == 0) {
+ logit_max = maximum;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const float local_sum =
+ (first_valid ? exp(first_logit - logit_max) : 0.0f) + (second_valid ? exp(second_logit - logit_max) : 0.0f);
+ const float simd_total = simd_sum(local_sum);
+ if (thread_context.simd_lane_id == 0) {
+ reduce_float[thread_context.simdgroup_index] = simd_total;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (thread_context.simdgroup_index == 0) {
+ const float group_value =
+ thread_context.simd_lane_id < WEAVER_TOP_CHILDREN_SIMDGROUPS ? reduce_float[thread_context.simd_lane_id] : 0.0f;
+ const float total = simd_sum(group_value);
+ if (thread_context.simd_lane_id == 0) {
+ log_sum = log(total) + logit_max;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ for (uint child = 0; child < children; ++child) {
+ uint local_score = 0u;
+ uint local_token = 0xffffffffu;
+ uint local_index = 0xffffffffu;
+ if (first_active) {
+ local_score = first_score;
+ local_token = first_token;
+ local_index = first_index;
+ }
+ if (second_active &&
+ weaver_better(second_score, second_token, second_index, local_score, local_token, local_index)) {
+ local_score = second_score;
+ local_token = second_token;
+ local_index = second_index;
+ }
+
+ const uint simd_score = simd_max(local_score);
+ const uint simd_token = simd_min(local_score == simd_score ? local_token : 0xffffffffu);
+ const uint simd_index =
+ simd_min(local_score == simd_score && local_token == simd_token ? local_index : 0xffffffffu);
+ if (thread_context.simd_lane_id == 0) {
+ reduce_score[thread_context.simdgroup_index] = simd_score;
+ reduce_token[thread_context.simdgroup_index] = simd_token;
+ reduce_index[thread_context.simdgroup_index] = simd_index;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (thread_context.simdgroup_index == 0) {
+ const bool lane_valid = thread_context.simd_lane_id < WEAVER_TOP_CHILDREN_SIMDGROUPS;
+ const uint group_score = lane_valid ? reduce_score[thread_context.simd_lane_id] : 0u;
+ const uint selected_score = simd_max(group_score);
+ const uint group_token =
+ lane_valid && group_score == selected_score ? reduce_token[thread_context.simd_lane_id] : 0xffffffffu;
+ const uint selected_token = simd_min(group_token);
+ const uint group_index = lane_valid && group_score == selected_score && group_token == selected_token
+ ? reduce_index[thread_context.simd_lane_id]
+ : 0xffffffffu;
+ const uint selected_index = simd_min(group_index);
+ if (thread_context.simd_lane_id == 0) {
+ winner_score = selected_score;
+ winner_token = selected_token;
+ winner_index = selected_index;
+ }
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ if (lid == 0) {
+ const float winner_logit = top_k_score_from_key(winner_score);
+ output_ids[row * children + child] = winner_token;
+ output_logprobs[row * children + child] = winner_logit - log_sum;
+ }
+ first_active = first_active && first_index != winner_index;
+ second_active = second_active && second_index != winner_index;
+ }
+}
diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
index a4356849b..9c0c60cd8 100644
--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs
@@ -29,7 +29,7 @@ mod mode;
mod qkv_norm;
mod state;
-pub(crate) use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState};
+pub(crate) use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, AttentionStateType};
pub mod rope;
diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
index 030c5e769..705a5958f 100644
--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs
@@ -211,8 +211,9 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
let gate_act_mul = <B::Kernels as Kernels>::GatedActMulKernel::new(context, data_type, false, false)
.map_err(PerLayerEmbeddingError::BackendError)?;
- let residual_finalize = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, data_type, true)
- .map_err(PerLayerEmbeddingError::BackendError)?;
+ let residual_finalize =
+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, data_type, true, false)
+ .map_err(PerLayerEmbeddingError::BackendError)?;
let residual_combine = <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, data_type, true)
.map_err(PerLayerEmbeddingError::BackendError)?;
@@ -246,6 +247,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
self.residual_finalize.encode(
None::<&Allocation<B>>,
hidden,
+ None::<&Allocation<B>>,
&mut *outputs,
length as u32,
length as u32,
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index c48b1a31e..0b5385895 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -1,15 +1,58 @@
+#![allow(dead_code)]
+
use thiserror::Error;
use crate::{
- backends::common::{Allocation, Backend},
+ array::size_for_shape,
+ backends::common::{
+ Allocation, AllocationType, Backend, Context, Encoder, Kernels,
+ kernel::{
+ ActivationKernel, AttentionLastQueryKernel, AttentionPrepareKernel, TensorAddBiasKernel,
+ TensorAddSwapKernel, WeaverNodeCacheWriteKernel, WeaverTopChildrenKernel, weaver::MAX_CANDIDATES,
+ },
+ },
config::{normalization::NormalizationConfig, weaver::WeaverConfig},
data_type::DataType,
- encodable_block::linear::{Linear, LinearBlockError},
+ encodable_block::{
+ embedding::{Embedding, EmbeddingError},
+ linear::{Linear, LinearBlockError},
+ mixer::attention::{
+ AttentionStateType,
+ core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},
+ },
+ normalization::{Normalization, NormalizationNewError, PostLayerScalar},
+ },
parameters::{ParameterLoaderError, ParameterTree},
};
-#[allow(dead_code)]
-pub struct Weaver<B: Backend> {
+pub(crate) struct WeaverPrefix<B: Backend> {
+ layer_qkv: Box<[Allocation<B>]>,
+ pub(crate) length: usize,
+}
+
+pub(crate) struct WeaverBatchStepInput<'a> {
+ pub node_index: usize,
+ pub parent_token: u32,
+ pub candidates: &'a [u32],
+ pub candidate_scores: &'a [f32],
+ pub ancestors: &'a [usize],
+ pub depth: usize,
+}
+
+pub(crate) struct WeaverNodeState<B: Backend> {
+ layer_qkv: Box<[Allocation<B>]>,
+ capacity: usize,
+}
+
+struct WeaverStepInputs<B: Backend> {
+ candidate_ids: Allocation<B>,
+ candidate_scores: Allocation<B>,
+ ancestor_indices: Allocation<B>,
+ metadata: Allocation<B>,
+ ancestor_stride: usize,
+}
+
+pub(crate) struct Weaver<B: Backend> {
embedding_norm: WeaverNorm<B>,
hidden_state_norm: WeaverNorm<B>,
embedding_projection: Box<dyn Linear<B>>,
@@ -18,22 +61,41 @@ pub struct Weaver<B: Backend> {
hidden_state_projection: Box<dyn Linear<B>>,
query_projection: Box<dyn Linear<B>>,
position_embeddings: Allocation<B>,
+ position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
+ indexed_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,
+ top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,
+ model_dim: usize,
+ target_model_dim: usize,
+ max_depth: usize,
+ data_type: DataType,
}
-#[allow(dead_code)]
struct WeaverNorm<B: Backend> {
- scales: Allocation<B>,
+ normalization: Normalization<B>,
biases: Option<Allocation<B>>,
+ bias_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,
+ dimension: usize,
}
-#[allow(dead_code)]
struct WeaverBlock<B: Backend> {
- pre_attention_norm: WeaverNorm<B>,
qkv_projection: Box<dyn Linear<B>>,
out_projection: Box<dyn Linear<B>>,
+ prefix_attention: AttentionCores<B>,
+ attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,
+ pre_attention_norm: WeaverNorm<B>,
pre_mlp_norm: WeaverNorm<B>,
up_projection: Box<dyn Linear<B>>,
down_projection: Box<dyn Linear<B>>,
+ activation: <B::Kernels as Kernels>::ActivationKernel,
+ residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,
+ last_query_attention: <B::Kernels as Kernels>::AttentionLastQueryKernel,
+ node_cache_write: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,
+ attention_scale: f32,
+ model_dim: usize,
+ hidden_dim: usize,
+ num_heads: usize,
+ head_dim: usize,
+ data_type: DataType,
}
#[derive(Debug, Error)]
@@ -42,48 +104,139 @@ pub enum WeaverNewError<B: Backend> {
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("linear error: {0}")]
Linear(#[from] LinearBlockError<B>),
+ #[error("normalization error: {0}")]
+ Normalization(#[from] NormalizationNewError<B>),
+ #[error("backend error: {0}")]
+ Backend(#[source] B::Error),
#[error("model_dim must be divisible by num_heads")]
InvalidHeadConfig,
+ #[error("candidate_pool_size must be in 1..={MAX_CANDIDATES}, got {0}")]
+ InvalidCandidatePoolSize(usize),
+}
+
+#[derive(Debug, Error)]
+pub enum WeaverEncodeError<B: Backend> {
+ #[error("backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("embedding error: {0}")]
+ Embedding(#[from] EmbeddingError<B>),
}
-fn plain_norm<B: Backend>(
- dim: usize,
- config: &NormalizationConfig,
+fn linear<B: Backend>(
+ context: &B::Context,
parameter_tree: &ParameterTree<B>,
-) -> Result<WeaverNorm<B>, ParameterLoaderError<B>> {
- let scales = parameter_tree.leaf("scales")?.validate(&[dim], DataType::F32)?.read_allocation()?;
- let biases = config
- .has_biases
- .then(|| parameter_tree.leaf("biases")?.validate(&[dim], DataType::F32)?.read_allocation())
- .transpose()?;
- Ok(WeaverNorm {
- scales,
- biases,
- })
+ name: &str,
+ input_dim: usize,
+ output_dim: usize,
+ has_biases: bool,
+ data_type: DataType,
+) -> Result<Box<dyn Linear<B>>, WeaverNewError<B>> {
+ Ok(<dyn Linear<B>>::new(input_dim, [output_dim], has_biases, context, data_type, &parameter_tree.subtree(name)?)?)
+}
+
+impl<B: Backend> WeaverNorm<B> {
+ fn new(
+ context: &B::Context,
+ dim: usize,
+ config: &NormalizationConfig,
+ parameter_tree: &ParameterTree<B>,
+ data_type: DataType,
+ ) -> Result<Self, WeaverNewError<B>> {
+ let mut normalization_config = config.clone();
+ normalization_config.has_biases = false;
+ let normalization = Normalization::new(
+ dim,
+ None,
+ false,
+ false,
+ PostLayerScalar::None,
+ data_type,
+ &normalization_config,
+ parameter_tree,
+ context,
+ )?;
+ let biases = config
+ .has_biases
+ .then(|| parameter_tree.leaf("biases")?.validate(&[dim], DataType::F32)?.read_allocation())
+ .transpose()?;
+ let bias_kernel = biases
+ .as_ref()
+ .map(|_| <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, false))
+ .transpose()
+ .map_err(WeaverNewError::Backend)?;
+ Ok(Self {
+ normalization,
+ biases,
+ bias_kernel,
+ dimension: dim,
+ })
+ }
+
+ fn encode(
+ &self,
+ input: &Allocation<B>,
+ row_count: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let mut output = self.normalization.encode(input, 0, row_count, None, encoder)?;
+ if let (Some(biases), Some(kernel)) = (&self.biases, &self.bias_kernel) {
+ kernel.encode(
+ None::<&Allocation<B>>,
+ biases,
+ None::<&Allocation<B>>,
+ &mut output,
+ self.dimension as u32,
+ (row_count * self.dimension) as u32,
+ encoder,
+ );
+ }
+ Ok(output)
+ }
}
impl<B: Backend> Weaver<B> {
+ pub(crate) fn create_node_state(
+ &self,
+ capacity: usize,
+ context: &B::Context,
+ ) -> Result<WeaverNodeState<B>, WeaverEncodeError<B>> {
+ assert!(capacity > 0);
+ let qkv_size = size_for_shape(&[capacity, 3, self.model_dim], self.data_type);
+ let layer_qkv = (0..self.blocks.len())
+ .map(|_| context.create_allocation(qkv_size, AllocationType::Global).map_err(WeaverEncodeError::Backend))
+ .collect::<Result<Box<[_]>, _>>()?;
+ Ok(WeaverNodeState {
+ layer_qkv,
+ capacity,
+ })
+ }
+
pub(crate) fn new(
context: &B::Context,
config: &WeaverConfig,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
) -> Result<Self, WeaverNewError<B>> {
+ assert_eq!(data_type, DataType::BF16, "Weaver only supports BF16");
if config.num_heads == 0 || !config.model_dim.is_multiple_of(config.num_heads) {
return Err(WeaverNewError::InvalidHeadConfig);
}
-
- let embedding_norm =
- plain_norm(config.target_embedding_dim, &config.norm_config, &parameter_tree.subtree("embedding_norm")?)?;
- let hidden_state_norm =
- plain_norm(config.target_model_dim, &config.norm_config, &parameter_tree.subtree("hidden_state_norm")?)?;
- let embedding_projection = <dyn Linear<B>>::new(
+ if config.candidate_pool_size == 0 || config.candidate_pool_size > MAX_CANDIDATES {
+ return Err(WeaverNewError::InvalidCandidatePoolSize(config.candidate_pool_size));
+ }
+ let norm = |dim, name: &str| -> Result<_, WeaverNewError<B>> {
+ WeaverNorm::new(context, dim, &config.norm_config, &parameter_tree.subtree(name)?, data_type)
+ };
+ let embedding_norm = norm(config.target_embedding_dim, "embedding_norm")?;
+ let hidden_state_norm = norm(config.target_model_dim, "hidden_state_norm")?;
+ let embedding_projection = linear(
+ context,
+ parameter_tree,
+ "embedding_projection",
config.target_embedding_dim,
- [config.model_dim],
+ config.model_dim,
true,
- context,
data_type,
- &parameter_tree.subtree("embedding_projection")?,
)?;
let blocks_tree = parameter_tree.subtree("blocks")?;
let blocks = (0..config.num_layers)
@@ -92,34 +245,44 @@ impl<B: Backend> Weaver<B> {
context,
config.model_dim,
config.hidden_dim,
+ config.num_heads,
&config.norm_config,
&blocks_tree.subtree(&index.to_string())?,
data_type,
)
})
.collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;
- let output_norm = plain_norm(config.model_dim, &config.norm_config, &parameter_tree.subtree("output_norm")?)?;
- let hidden_state_projection = <dyn Linear<B>>::new(
+ let output_norm = norm(config.model_dim, "output_norm")?;
+ let hidden_state_projection = linear(
+ context,
+ parameter_tree,
+ "hidden_state_projection",
config.target_model_dim,
- [config.model_dim],
+ config.model_dim,
true,
- context,
data_type,
- &parameter_tree.subtree("hidden_state_projection")?,
)?;
- let query_projection = <dyn Linear<B>>::new(
+ let query_projection = linear(
+ context,
+ parameter_tree,
+ "query_projection",
config.model_dim,
- [config.target_model_dim],
+ config.target_model_dim,
false,
- context,
data_type,
- &parameter_tree.subtree("query_projection")?,
)?;
let position_embeddings = parameter_tree
.leaf("position_embeddings")?
.validate(&[config.max_depth, config.model_dim], DataType::F32)?
.read_allocation()?;
-
+ let position_add =
+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, false)
+ .map_err(WeaverNewError::Backend)?;
+ let indexed_position_add =
+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, true)
+ .map_err(WeaverNewError::Backend)?;
+ let top_children =
+ <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;
Ok(Self {
embedding_norm,
hidden_state_norm,
@@ -129,8 +292,257 @@ impl<B: Backend> Weaver<B> {
hidden_state_projection,
query_projection,
position_embeddings,
+ position_add,
+ indexed_position_add,
+ top_children,
+ model_dim: config.model_dim,
+ target_model_dim: config.target_model_dim,
+ max_depth: config.max_depth,
+ data_type,
+ })
+ }
+
+ pub(crate) fn build_prefix(
+ &self,
+ target_hidden: &Allocation<B>,
+ lookaheads: &Allocation<B>,
+ lookahead_offset: usize,
+ lookahead_count: usize,
+ context: &B::Context,
+ ) -> Result<WeaverPrefix<B>, WeaverEncodeError<B>> {
+ let length = lookahead_count + 1;
+ assert!(lookahead_count <= self.max_depth);
+ let row_bytes = self.target_model_dim * self.data_type.size_in_bytes();
+ assert!(target_hidden.size() >= row_bytes);
+ assert!(lookaheads.size() >= (lookahead_offset + lookahead_count) * row_bytes);
+
+ let mut encoder = Encoder::new(context).map_err(WeaverEncodeError::Backend)?;
+ let mut input = encoder
+ .allocate_scratch(size_for_shape(&[length, self.target_model_dim], self.data_type))
+ .map_err(WeaverEncodeError::Backend)?;
+ encoder.encode_copy(target_hidden, 0..row_bytes, &mut input, 0..row_bytes);
+ if lookahead_count > 0 {
+ encoder.encode_copy(
+ lookaheads,
+ lookahead_offset * row_bytes..(lookahead_offset + lookahead_count) * row_bytes,
+ &mut input,
+ row_bytes..length * row_bytes,
+ );
+ }
+ let normalized =
+ self.hidden_state_norm.encode(&input, length, &mut encoder).map_err(WeaverEncodeError::Backend)?;
+ let mut hidden = self
+ .hidden_state_projection
+ .encode(normalized, length, &mut encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let position_elements = lookahead_count * self.model_dim;
+ self.position_add.encode(
+ None::<&Allocation<B>>,
+ &self.position_embeddings,
+ None::<&Allocation<B>>,
+ (&mut hidden, self.model_dim * self.data_type.size_in_bytes()),
+ position_elements as u32,
+ position_elements as u32,
+ &mut encoder,
+ );
+
+ let mut layer_qkv = Vec::with_capacity(self.blocks.len());
+ for block in &self.blocks {
+ let (next_hidden, qkv) =
+ block.encode_prefix(hidden, length, &mut encoder).map_err(WeaverEncodeError::Backend)?;
+ let mut cached_qkv =
+ context.create_allocation(qkv.size(), AllocationType::Global).map_err(WeaverEncodeError::Backend)?;
+ encoder.encode_copy(&qkv, .., &mut cached_qkv, ..);
+ layer_qkv.push(cached_qkv);
+ hidden = next_hidden;
+ }
+ let completed = encoder.end_encoding().submit().wait_until_completed().map_err(WeaverEncodeError::Backend)?;
+ drop(input);
+ drop(hidden);
+ drop(completed);
+ Ok(WeaverPrefix {
+ layer_qkv: layer_qkv.into_boxed_slice(),
+ length,
})
}
+
+ pub(crate) fn step_batch(
+ &self,
+ prefix: &WeaverPrefix<B>,
+ inputs: &[WeaverBatchStepInput<'_>],
+ state: &mut WeaverNodeState<B>,
+ children: usize,
+ target_embedding: &Embedding<B>,
+ context: &B::Context,
+ ) -> Result<Vec<Vec<(u32, f32)>>, WeaverEncodeError<B>> {
+ assert!(!inputs.is_empty());
+ let mut encoder = Encoder::new(context).map_err(WeaverEncodeError::Backend)?;
+ let (child_ids, child_logprobs) =
+ self.encode_step_batch(prefix, inputs, state, children, target_embedding, &mut encoder)?;
+ let completed = encoder.end_encoding().submit().wait_until_completed().map_err(WeaverEncodeError::Backend)?;
+ let child_ids_output = child_ids.copyout::<u32>();
+ let child_logprobs_output = child_logprobs.copyout::<f32>();
+ let outputs = (0..inputs.len())
+ .map(|batch_index| {
+ (0..children)
+ .map(|child| {
+ let index = batch_index * children + child;
+ (child_ids_output[index], child_logprobs_output[index])
+ })
+ .collect()
+ })
+ .collect();
+ drop(child_ids);
+ drop(child_logprobs);
+ drop(completed);
+ Ok(outputs)
+ }
+
+ fn encode_step_batch(
+ &self,
+ prefix: &WeaverPrefix<B>,
+ inputs: &[WeaverBatchStepInput<'_>],
+ state: &mut WeaverNodeState<B>,
+ children: usize,
+ target_embedding: &Embedding<B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(Allocation<B>, Allocation<B>), WeaverEncodeError<B>> {
+ let rows = inputs.len();
+ let candidates = inputs[0].candidates.len();
+ assert!(candidates > 0 && candidates <= MAX_CANDIDATES);
+ assert!(children > 0 && children <= candidates);
+ // Indices reach the GPU unchecked, so they must stay inside the arena.
+ // Batch nodes are expected to be mutually independent: a child only
+ // enters the frontier once its parent's step has been read back, so a
+ // parent and child can never share a batch.
+ assert!(
+ inputs.iter().all(|input| {
+ input.candidates.len() == candidates
+ && input.candidate_scores.len() == candidates
+ && input.depth < self.max_depth
+ && input.node_index < state.capacity
+ && input.ancestors.iter().all(|&ancestor| ancestor < state.capacity)
+ }),
+ "inputs must reference node indices within the node-state capacity"
+ );
+
+ let (mut current, step) = self.encode_step_inputs(inputs, target_embedding, encoder)?;
+
+ for (layer_index, block) in self.blocks.iter().enumerate() {
+ current = block
+ .encode_step(
+ current,
+ &prefix.layer_qkv[layer_index],
+ &mut state.layer_qkv[layer_index],
+ &step,
+ prefix.length,
+ rows,
+ encoder,
+ )
+ .map_err(WeaverEncodeError::Backend)?;
+ }
+
+ self.encode_step_output(&current, &step, rows, candidates, children, target_embedding, encoder)
+ }
+
+ fn encode_step_inputs(
+ &self,
+ inputs: &[WeaverBatchStepInput<'_>],
+ target_embedding: &Embedding<B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(Allocation<B>, WeaverStepInputs<B>), WeaverEncodeError<B>> {
+ let rows = inputs.len();
+ let token_ids = encoder
+ .allocate_constant_from_slice(&inputs.iter().map(|input| input.parent_token).collect::<Vec<_>>())
+ .map_err(WeaverEncodeError::Backend)?;
+ let candidate_ids = encoder
+ .allocate_constant_from_slice(
+ &inputs.iter().flat_map(|input| input.candidates.iter().copied()).collect::<Vec<_>>(),
+ )
+ .map_err(WeaverEncodeError::Backend)?;
+ let candidate_scores = encoder
+ .allocate_constant_from_slice(
+ &inputs.iter().flat_map(|input| input.candidate_scores.iter().copied()).collect::<Vec<_>>(),
+ )
+ .map_err(WeaverEncodeError::Backend)?;
+ let token_embedding = target_embedding.encode_lookup(&token_ids, rows, encoder)?;
+ let embedding_normalized =
+ self.embedding_norm.encode(&token_embedding, rows, encoder).map_err(WeaverEncodeError::Backend)?;
+ let mut current = self
+ .embedding_projection
+ .encode(embedding_normalized, rows, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ // [depths, ancestor counts, node indices]
+ let mut metadata_values = Vec::with_capacity(3 * rows);
+ metadata_values.extend(inputs.iter().map(|input| input.depth as u32));
+ metadata_values.extend(inputs.iter().map(|input| input.ancestors.len() as u32));
+ metadata_values.extend(inputs.iter().map(|input| input.node_index as u32));
+ let metadata = encoder.allocate_constant_from_slice(&metadata_values).map_err(WeaverEncodeError::Backend)?;
+ self.indexed_position_add.encode(
+ None::<&Allocation<B>>,
+ &self.position_embeddings,
+ Some(&metadata),
+ &mut current,
+ self.model_dim as u32,
+ (rows * self.model_dim) as u32,
+ encoder,
+ );
+
+ let ancestor_stride = inputs.iter().map(|input| input.ancestors.len()).max().unwrap().max(1);
+ let mut ancestors = vec![0u32; rows * ancestor_stride];
+ for (row, input) in inputs.iter().enumerate() {
+ for (offset, &ancestor) in input.ancestors.iter().enumerate() {
+ ancestors[row * ancestor_stride + offset] = ancestor as u32;
+ }
+ }
+ let ancestor_indices = encoder.allocate_constant_from_slice(&ancestors).map_err(WeaverEncodeError::Backend)?;
+ Ok((
+ current,
+ WeaverStepInputs {
+ candidate_ids,
+ candidate_scores,
+ ancestor_indices,
+ metadata,
+ ancestor_stride,
+ },
+ ))
+ }
+
+ /// Outputs must outlive the batch.
+ fn encode_step_output(
+ &self,
+ current: &Allocation<B>,
+ step: &WeaverStepInputs<B>,
+ rows: usize,
+ candidates: usize,
+ children: usize,
+ target_embedding: &Embedding<B>,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(Allocation<B>, Allocation<B>), WeaverEncodeError<B>> {
+ let output_normalized = self.output_norm.encode(current, rows, encoder).map_err(WeaverEncodeError::Backend)?;
+ let query =
+ self.query_projection.encode(output_normalized, rows, encoder).map_err(WeaverEncodeError::Backend)?;
+ let candidate_logits =
+ target_embedding.encode_readout_sparse(&query, &step.candidate_ids, rows, candidates, encoder)?;
+ let mut child_ids = encoder
+ .allocate_scratch(size_for_shape(&[rows, children], DataType::U32))
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut child_logprobs = encoder
+ .allocate_scratch(size_for_shape(&[rows, children], DataType::F32))
+ .map_err(WeaverEncodeError::Backend)?;
+ self.top_children.encode(
+ &candidate_logits,
+ &step.candidate_scores,
+ &step.candidate_ids,
+ &mut child_ids,
+ &mut child_logprobs,
+ rows as u32,
+ candidates as u32,
+ children as u32,
+ encoder,
+ );
+ Ok((child_ids, child_logprobs))
+ }
}
impl<B: Backend> WeaverBlock<B> {
@@ -138,52 +550,186 @@ impl<B: Backend> WeaverBlock<B> {
context: &B::Context,
model_dim: usize,
hidden_dim: usize,
+ num_heads: usize,
norm_config: &NormalizationConfig,
parameter_tree: &ParameterTree<B>,
data_type: DataType,
) -> Result<Self, WeaverNewError<B>> {
- let pre_attention_norm = plain_norm(model_dim, norm_config, &parameter_tree.subtree("pre_attention_norm")?)?;
- let qkv_projection = <dyn Linear<B>>::new(
- model_dim,
- [model_dim, model_dim, model_dim],
- false,
- context,
- data_type,
- &parameter_tree.subtree("qkv_projection")?,
- )?;
- let out_projection = <dyn Linear<B>>::new(
- model_dim,
- [model_dim],
- false,
- context,
- data_type,
- &parameter_tree.subtree("out_projection")?,
- )?;
- let pre_mlp_norm = plain_norm(model_dim, norm_config, &parameter_tree.subtree("pre_mlp_norm")?)?;
- let up_projection = <dyn Linear<B>>::new(
- model_dim,
- [hidden_dim],
- true,
- context,
- data_type,
- &parameter_tree.subtree("up_projection")?,
- )?;
- let down_projection = <dyn Linear<B>>::new(
- hidden_dim,
- [model_dim],
- true,
+ let head_dim = model_dim / num_heads;
+ let attention_scale = 1.0 / (head_dim as f32).sqrt();
+ let qkv_projection =
+ linear(context, parameter_tree, "qkv_projection", model_dim, 3 * model_dim, false, data_type)?;
+ let out_projection = linear(context, parameter_tree, "out_projection", model_dim, model_dim, false, data_type)?;
+ let prefix_attention = AttentionCores::new(
+ AttentionCoreNewArguments {
+ head_dim,
+ num_groups: num_heads,
+ num_q_heads: num_heads,
+ has_sinks: false,
+ is_kv_cache_ring: false,
+ is_causal: true,
+ is_trie: false,
+ sliding_window_size: None,
+ scale: Some(attention_scale),
+ data_type,
+ },
context,
- data_type,
- &parameter_tree.subtree("down_projection")?,
- )?;
-
+ )
+ .map_err(WeaverNewError::Backend)?;
+ let attention_prepare =
+ <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, data_type, DataType::F32, true, false)
+ .map_err(WeaverNewError::Backend)?;
+ let norm = |name: &str| -> Result<_, WeaverNewError<B>> {
+ WeaverNorm::new(context, model_dim, norm_config, &parameter_tree.subtree(name)?, data_type)
+ };
+ let pre_attention_norm = norm("pre_attention_norm")?;
+ let pre_mlp_norm = norm("pre_mlp_norm")?;
+ let up_projection = linear(context, parameter_tree, "up_projection", model_dim, hidden_dim, true, data_type)?;
+ let down_projection =
+ linear(context, parameter_tree, "down_projection", hidden_dim, model_dim, true, data_type)?;
+ let activation = <B::Kernels as Kernels>::ActivationKernel::new(context, data_type, true)
+ .map_err(WeaverNewError::Backend)?;
+ let residual_add =
+ <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type).map_err(WeaverNewError::Backend)?;
+ let last_query_attention =
+ <B::Kernels as Kernels>::AttentionLastQueryKernel::new(context, head_dim as u32, num_heads as u32)
+ .map_err(WeaverNewError::Backend)?;
+ let node_cache_write = <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context, data_type)
+ .map_err(WeaverNewError::Backend)?;
Ok(Self {
- pre_attention_norm,
qkv_projection,
out_projection,
+ prefix_attention,
+ attention_prepare,
+ pre_attention_norm,
pre_mlp_norm,
up_projection,
down_projection,
+ activation,
+ residual_add,
+ last_query_attention,
+ node_cache_write,
+ attention_scale,
+ model_dim,
+ hidden_dim,
+ num_heads,
+ head_dim,
+ data_type,
})
}
+
+ fn encode_step(
+ &self,
+ current: Allocation<B>,
+ prefix_qkv: &Allocation<B>,
+ state_qkv: &mut Allocation<B>,
+ step: &WeaverStepInputs<B>,
+ prefix_length: usize,
+ rows: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<Allocation<B>, B::Error> {
+ let normalized = self.pre_attention_norm.encode(&current, rows, encoder)?;
+ let current_qkv = self.qkv_projection.encode(normalized, rows, encoder)?;
+ let metadata_row_bytes = rows * DataType::U32.size_in_bytes();
+ let ancestor_counts = (&step.metadata, metadata_row_bytes);
+ let node_indices = (&step.metadata, 2 * metadata_row_bytes);
+ let mut attention = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;
+ self.last_query_attention.encode(
+ prefix_qkv,
+ &*state_qkv,
+ &current_qkv,
+ &step.ancestor_indices,
+ ancestor_counts,
+ &mut attention,
+ rows as u32,
+ prefix_length as u32,
+ step.ancestor_stride as u32,
+ self.attention_scale,
+ encoder,
+ );
+ // Separate dispatch so the node arena is read-only above; the encoder
+ // serializes it after the attention that reads the ancestor slots.
+ self.node_cache_write.encode(
+ &current_qkv,
+ state_qkv,
+ node_indices,
+ self.model_dim as u32,
+ (rows * 2 * self.model_dim) as u32,
+ encoder,
+ );
+ attention = self.out_projection.encode(attention, rows, encoder)?;
+ let mut residual = current;
+ self.residual_add.encode(&mut residual, &mut attention, (rows * self.model_dim) as u32, encoder);
+ let normalized = self.pre_mlp_norm.encode(&residual, rows, encoder)?;
+ let mut mlp = self.up_projection.encode(normalized, rows, encoder)?;
+ self.activation.encode(
+ None::<&Allocation<B>>,
+ &mut mlp,
+ (rows * self.hidden_dim) as u32,
+ crate::backends::common::gpu_types::ActivationType::GELUExact,
+ encoder,
+ );
+ let mut output = self.down_projection.encode(mlp, rows, encoder)?;
+ self.residual_add.encode(&mut output, &mut residual, (rows * self.model_dim) as u32, encoder);
+ Ok(residual)
+ }
+
+ fn encode_prefix(
+ &self,
+ hidden: Allocation<B>,
+ rows: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {
+ let normalized = self.pre_attention_norm.encode(&hidden, rows, encoder)?;
+ let qkv = self.qkv_projection.encode(normalized, rows, encoder)?;
+ let mut queries =
+ encoder.allocate_scratch(size_for_shape(&[self.num_heads, rows, self.head_dim], self.data_type))?;
+ let mut keys = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;
+ let mut values = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;
+ self.attention_prepare.encode(
+ &qkv,
+ &mut queries,
+ Some(&mut keys),
+ Some(&mut values),
+ None::<&Allocation<B>>,
+ None::<&Allocation<B>>,
+ self.num_heads as u32,
+ Some(self.num_heads as u32),
+ self.head_dim as u32,
+ None,
+ Some(0),
+ rows as u32,
+ encoder,
+ );
+ let state_type = AttentionStateType::Full {
+ length: 0,
+ };
+ let attention = self.prefix_attention.encode(
+ AttentionCoreEncodeArguments {
+ queries: &queries,
+ keys: &keys,
+ values: &values,
+ suffix_length: rows,
+ trie: None,
+ sinks: None,
+ state_type: &state_type,
+ },
+ encoder,
+ )?;
+ let mut attention = self.out_projection.encode(attention, rows, encoder)?;
+ let mut residual = hidden;
+ self.residual_add.encode(&mut residual, &mut attention, (rows * self.model_dim) as u32, encoder);
+ let normalized = self.pre_mlp_norm.encode(&residual, rows, encoder)?;
+ let mut mlp = self.up_projection.encode(normalized, rows, encoder)?;
+ self.activation.encode(
+ None::<&Allocation<B>>,
+ &mut mlp,
+ (rows * self.hidden_dim) as u32,
+ crate::backends::common::gpu_types::ActivationType::GELUExact,
+ encoder,
+ );
+ let mut output = self.down_projection.encode(mlp, rows, encoder)?;
+ self.residual_add.encode(&mut output, &mut residual, (rows * self.model_dim) as u32, encoder);
+ Ok((residual, qkv))
+ }
}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs
index 3f31f6aae..0e36f1cf4 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs
@@ -28,7 +28,6 @@ struct Runner<B: Backend> {
current_qkv: Allocation<B>,
ancestor_indices: Allocation<B>,
ancestor_counts: Allocation<B>,
- node_indices: Allocation<B>,
output: Allocation<B>,
rows: u32,
prefix_length: u32,
@@ -69,10 +68,6 @@ impl<B: Backend> Runner<B> {
current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * packed_width, 29)),
ancestor_indices: alloc_allocation_with_data::<B, u32>(&context, &ancestor_indices),
ancestor_counts: alloc_allocation_with_data::<B, u32>(&context, &ancestor_counts),
- node_indices: alloc_allocation_with_data::<B, u32>(
- &context,
- &(first_output_node as u32..nodes as u32).collect::<Vec<_>>(),
- ),
output: alloc_allocation::<B, bf16>(&context, rows * NUM_HEADS as usize * HEAD_DIM),
context,
kernel,
@@ -90,11 +85,10 @@ impl<B: Backend> Runner<B> {
for _ in 0..repetitions {
self.kernel.encode(
&self.prefix_qkv,
- &mut self.node_qkv,
+ &self.node_qkv,
&self.current_qkv,
&self.ancestor_indices,
&self.ancestor_counts,
- &self.node_indices,
&mut self.output,
self.rows,
self.prefix_length,
@@ -111,38 +105,17 @@ impl<B: Backend> Runner<B> {
fn attention_last_query_matches_cpu() {
let mut cpu = Runner::<Cpu>::new(4, 5, 3, 16);
let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);
- let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);
- let node_indices = allocation_to_vec::<Cpu, u32>(&cpu.node_indices);
cpu.encode(1);
let expected_output = allocation_to_vec::<Cpu, bf16>(&cpu.output);
- let expected_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);
- let packed_width = 3 * NUM_HEADS as usize * HEAD_DIM;
- let q_width = NUM_HEADS as usize * HEAD_DIM;
- for node in 0..expected_nodes.len() / packed_width {
- let range = node * packed_width..(node + 1) * packed_width;
- if let Some(row) = node_indices.iter().position(|&index| index as usize == node) {
- let current = row * packed_width;
- assert_eq!(
- expected_nodes[range.start..range.start + q_width],
- initial_nodes[range.start..range.start + q_width]
- );
- assert_eq!(
- expected_nodes[range.start + q_width..range.end],
- current_qkv[current + q_width..current + packed_width],
- );
- } else {
- assert_eq!(expected_nodes[range.clone()], initial_nodes[range]);
- }
- }
+ assert_eq!(allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv), initial_nodes);
let mut metal = Runner::<Metal>::new(4, 5, 3, 16);
metal.encode(1);
let actual_output = allocation_to_vec::<Metal, bf16>(&metal.output);
- let actual_nodes = allocation_to_vec::<Metal, bf16>(&metal.node_qkv);
assert_eq_float(&expected_output, &actual_output, 0.02, "AttentionLastQuery output");
- assert_eq!(expected_nodes, actual_nodes);
+ assert_eq!(allocation_to_vec::<Metal, bf16>(&metal.node_qkv), initial_nodes);
}
#[uzu_test]
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs
index 89b8930d1..b4cfbf441 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs
@@ -17,3 +17,4 @@ mod tensor_add_bias_test;
mod tensor_add_scale_test;
mod tensor_add_swap_test;
mod tensor_copy_test;
+mod weaver;
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/tensor_add_bias_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/tensor_add_bias_test.rs
index 5ceee6461..0646ea5fc 100644
--- a/crates/backend-uzu/tests/unit/backends/common/kernel/tensor_add_bias_test.rs
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/tensor_add_bias_test.rs
@@ -7,7 +7,7 @@ use test_runner::for_each_backend;
use crate::{
array::ArrayElement,
- backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},
+ backends::common::{Allocation, Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},
tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
};
@@ -64,6 +64,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
T::data_type(),
T::data_type(),
in_place,
+ false,
)
.expect("Failed to create TensorAddBiasKernel");
@@ -79,6 +80,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(
kernel.encode(
input_allocation.as_ref(),
&bias_allocation,
+ None::<&Allocation<B>>,
&mut output_allocation,
input.num_cols,
input.length,
@@ -126,3 +128,30 @@ fn test_bf16() {
fn test_bf16_in_place() {
test::<bf16>(true);
}
+
+#[uzu_test]
+fn test_bf16_indexed_rows() {
+ let values = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0].map(bf16::from_f32);
+ let bias = [10.0f32, 20.0, 30.0, 40.0, 50.0, 60.0];
+ let bias_row_indices = [1u32, 0];
+ let expected = [41.0, 52.0, 63.0, 14.0, 25.0, 36.0].map(bf16::from_f32);
+
+ for_each_backend!(|B| {
+ let context = <B as Backend>::Context::new().unwrap();
+ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(
+ &context,
+ bf16::data_type(),
+ f32::data_type(),
+ true,
+ true,
+ )
+ .unwrap();
+ let mut output = alloc_allocation_with_data::<B, bf16>(&context, &values);
+ let bias = alloc_allocation_with_data::<B, f32>(&context, &bias);
+ let bias_row_indices = alloc_allocation_with_data::<B, u32>(&context, &bias_row_indices);
+ let mut encoder = Encoder::new(context.as_ref()).unwrap();
+ kernel.encode(None::<&Allocation<B>>, &bias, Some(&bias_row_indices), &mut output, 3, 6, &mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ assert_eq!(allocation_to_vec::<B, bf16>(&output), expected);
+ });
+}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs
new file mode 100644
index 000000000..34c4d8777
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs
@@ -0,0 +1,2 @@
+mod weaver_node_cache_write_test;
+mod weaver_top_children_test;
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs
new file mode 100644
index 000000000..eefb0f025
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs
@@ -0,0 +1,96 @@
+#![cfg(metal_backend)]
+
+use half::bf16;
+use proc_macros::uzu_test;
+
+use crate::{
+ backends::{
+ common::{Allocation, Backend, Encoder, Kernels, kernel::WeaverNodeCacheWriteKernel},
+ cpu::Cpu,
+ metal::Metal,
+ },
+ data_type::DataType,
+ tests::helpers::{alloc_allocation_with_data, allocation_to_vec, create_context},
+};
+
+const MODEL_DIM: usize = 64;
+const ROWS: usize = 4;
+const NODES: usize = 9;
+
+struct Runner<B: Backend> {
+ context: std::sync::Arc<B::Context>,
+ kernel: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,
+ current_qkv: Allocation<B>,
+ node_qkv: Allocation<B>,
+ node_indices: Allocation<B>,
+}
+
+fn values(
+ length: usize,
+ offset: usize,
+) -> Vec<bf16> {
+ (0..length).map(|index| bf16::from_f32((((index + offset) * 17 % 251) as f32 - 125.0) / 128.0)).collect()
+}
+
+/// Slots are deliberately non-contiguous and out of order so the scatter is
+/// exercised rather than a coincidentally-contiguous copy.
+const NODE_INDICES: [u32; ROWS] = [7, 1, 8, 4];
+
+impl<B: Backend> Runner<B> {
+ fn new() -> Self {
+ let packed_width = 3 * MODEL_DIM;
+ let context = create_context::<B>();
+ let kernel =
+ <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context.as_ref(), DataType::BF16).unwrap();
+ Self {
+ current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(ROWS * packed_width, 29)),
+ node_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(NODES * packed_width, 11)),
+ node_indices: alloc_allocation_with_data::<B, u32>(&context, &NODE_INDICES),
+ context,
+ kernel,
+ }
+ }
+
+ fn run(&mut self) -> Vec<bf16> {
+ let mut encoder = Encoder::new(self.context.as_ref()).unwrap();
+ self.kernel.encode(
+ &self.current_qkv,
+ &mut self.node_qkv,
+ &self.node_indices,
+ MODEL_DIM as u32,
+ (ROWS * 2 * MODEL_DIM) as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ allocation_to_vec::<B, bf16>(&self.node_qkv)
+ }
+}
+
+#[uzu_test]
+fn weaver_node_cache_write_matches_cpu() {
+ let packed_width = 3 * MODEL_DIM;
+
+ let mut cpu = Runner::<Cpu>::new();
+ let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);
+ let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);
+ let expected = cpu.run();
+
+ // Every written slot takes the K/V halves of its row; the query third and
+ // every untouched slot keep their prior contents.
+ for node in 0..NODES {
+ let base = node * packed_width;
+ match NODE_INDICES.iter().position(|&index| index as usize == node) {
+ Some(row) => {
+ assert_eq!(expected[base..base + MODEL_DIM], initial_nodes[base..base + MODEL_DIM]);
+ assert_eq!(
+ expected[base + MODEL_DIM..base + packed_width],
+ current_qkv[row * packed_width + MODEL_DIM..(row + 1) * packed_width],
+ );
+ },
+ None => assert_eq!(expected[base..base + packed_width], initial_nodes[base..base + packed_width]),
+ }
+ }
+
+ let mut metal = Runner::<Metal>::new();
+ assert_eq!(metal.run(), expected);
+}
diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
new file mode 100644
index 000000000..047ddceba
--- /dev/null
+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs
@@ -0,0 +1,68 @@
+use half::bf16;
+use proc_macros::uzu_test;
+use test_runner::for_each_non_cpu_backend;
+
+use crate::{
+ backends::{
+ common::{Backend, Encoder, Kernels, kernel::WeaverTopChildrenKernel},
+ cpu::Cpu,
+ },
+ tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},
+};
+
+const CANDIDATES: usize = 512;
+const CHILDREN: usize = 8;
+
+fn top_children<B: Backend>(
+ residual: &[bf16],
+ scores: &[f32],
+ ids: &[u32],
+) -> (Vec<u32>, Vec<f32>) {
+ let rows = residual.len() / CANDIDATES;
+ assert_eq!(residual.len(), rows * CANDIDATES);
+ assert_eq!(scores.len(), rows * CANDIDATES);
+ assert_eq!(ids.len(), rows * CANDIDATES);
+ let context = create_context::<B>();
+ let residual = alloc_allocation_with_data::<B, bf16>(&context, residual);
+ let scores = alloc_allocation_with_data::<B, f32>(&context, scores);
+ let ids = alloc_allocation_with_data::<B, u32>(&context, ids);
+ let mut output_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);
+ let mut output_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);
+ let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context).unwrap();
+ let mut encoder = Encoder::new(context.as_ref()).unwrap();
+ kernel.encode(
+ &residual,
+ &scores,
+ &ids,
+ &mut output_ids,
+ &mut output_logprobs,
+ rows as u32,
+ CANDIDATES as u32,
+ CHILDREN as u32,
+ &mut encoder,
+ );
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ (allocation_to_vec(&output_ids), allocation_to_vec(&output_logprobs))
+}
+
+#[uzu_test]
+fn weaver_top_children_matches_cpu() {
+ const ROWS: usize = 3;
+ let residual = (0..ROWS * CANDIDATES)
+ .map(|index| bf16::from_f32(((index as f32 * 0.017).cos() * 4.0).round() * 0.125))
+ .collect::<Vec<_>>();
+ let scores =
+ (0..ROWS * CANDIDATES).map(|index| ((index as f32 * 0.011).sin() * 3.0).round() * 0.125).collect::<Vec<_>>();
+ let ids = (0..ROWS)
+ .flat_map(|row| (0..CANDIDATES).rev().map(move |index| 70_000 + (row * CANDIDATES + index) as u32))
+ .collect::<Vec<_>>();
+
+ for_each_non_cpu_backend!(|B| {
+ let expected = top_children::<Cpu>(&residual, &scores, &ids);
+ let actual = top_children::<B>(&residual, &scores, &ids);
+ assert_eq!(actual.0, expected.0);
+ for (actual, expected) in actual.1.iter().zip(expected.1) {
+ assert!((actual - expected).abs() < 1e-5);
+ }
+ });
+}