A high-performance inference engine for AI models
Weaver forward (#624)
CCYeh0d7768e51725
diff --git a/crates/backend-uzu/src/backends/common/encoder.rs b/crates/backend-uzu/src/backends/common/encoder.rsindex 41bd7ee75..99bb99c76 100644--- a/crates/backend-uzu/src/backends/common/encoder.rs+++ b/crates/backend-uzu/src/backends/common/encoder.rs@@ -1,9 +1,12 @@use std::{+ mem::size_of_val,ops::{Bound, Range, RangeBounds},sync::Arc,time::Duration,};+use bytemuck::{AnyBitPattern, NoUninit};+use crate::backends::common::{AccessFlags, Allocation, AllocationPool, AllocationType, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer,BufferGpuAddressRangeExt, CommandBuffer, CommandBufferCompleted, CommandBufferEncoding, CommandBufferExecutable,@@ -72,6 +75,15 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {)}+ pub fn allocate_constant_from_slice<T: NoUninit + AnyBitPattern>(+ &mut self,+ data: &[T],+ ) -> Result<Allocation<B>, B::Error> {+ let mut allocation = self.allocate_constant(size_of_val(data))?;+ allocation.copyin(data);+ Ok(allocation)+ }+// This is valid on gpu timeline onlypub fn allocate_scratch(&mut self,diff --git a/crates/backend-uzu/src/backends/common/kernel/mod.rs b/crates/backend-uzu/src/backends/common/kernel/mod.rsindex b96024331..d23a882ec 100644--- a/crates/backend-uzu/src/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/common/kernel/mod.rs@@ -5,6 +5,7 @@ pub mod delta_net_chunked_prefill;pub mod delta_net_tree_verify;pub mod matmul;pub mod radix_top_k_small;+pub(crate) mod weaver;include!(concat!(env!("OUT_DIR"), "/traits.rs"));diff --git a/crates/backend-uzu/src/backends/common/kernel/weaver/mod.rs b/crates/backend-uzu/src/backends/common/kernel/weaver/mod.rsnew file mode 100644index 000000000..81b3d20d4--- /dev/null+++ b/crates/backend-uzu/src/backends/common/kernel/weaver/mod.rs@@ -0,0 +1 @@+pub(crate) const MAX_CANDIDATES: usize = 512;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rsindex d8611830c..1ba4acb36 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/attention/attention_last_query.rs@@ -7,11 +7,10 @@ use super::attention_single_pass::attention_single_pass;#[variants(HEAD_DIM, 128)]fn attention_last_query<const HEAD_DIM: u32>(prefix_qkv: *const bf16,- node_qkv: *mut bf16,+ node_qkv: *const bf16,current_qkv: *const bf16,ancestor_indices: *const u32,ancestor_counts: *const u32,- node_indices: *const u32,output: *mut bf16,rows: u32,prefix_length: u32,@@ -32,7 +31,6 @@ fn attention_last_query<const HEAD_DIM: u32>(for row in 0..rows as usize {unsafe {let current_row = current_qkv.add(row * qkv_width);- let dest_node = node_qkv.add(*node_indices.add(row) as usize * qkv_width);let ancestor_count = *ancestor_counts.add(row) as usize;let length = prefix_length as usize + ancestor_count + 1;@@ -75,7 +73,6 @@ fn attention_last_query<const HEAD_DIM: u32>(false,false,);- std::ptr::copy_nonoverlapping(current_row.add(kv_offset), dest_node.add(kv_offset), kv_width);}}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/mod.rsindex 9675a849a..046f70af0 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/mod.rs@@ -22,6 +22,7 @@ mod tensor_add_bias;mod tensor_add_scale;mod tensor_add_swap;mod tensor_copy;+mod weaver;include!(concat!(env!("OUT_DIR"), "/cpu/dsl.rs"));diff --git a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rsindex b07a5e976..86da05d80 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/tensor_add_bias/tensor_add_bias.rs@@ -12,15 +12,24 @@ pub fn tensor_add_bias<T: ArrayElement + Float, BiasT: ArrayElement + Float>(#[optional(!in_place)]input: Option<*const T>,#[allow(unused)] bias: *const BiasT,+ #[allow(unused)]+ #[optional(indexed)]+ bias_row_indices: Option<*const u32>,#[allow(unused)] output: *mut T,#[allow(unused)] num_cols: u32,#[allow(unused)] length: u32,#[specialize] in_place: bool,+ #[specialize] indexed: bool,) {assert_eq!(input.is_none(), in_place);+ assert_eq!(bias_row_indices.is_some(), indexed);for i in 0usize..(length as usize) {- let bias_position = i % num_cols as usize;+ let column = i % num_cols as usize;+ let bias_position = match bias_row_indices {+ Some(indices) => unsafe { *indices.add(i / num_cols as usize) as usize * num_cols as usize + column },+ None => column,+ };unsafe {let value: f32 = if let Some(in_data) = input {num_traits::cast(*in_data.add(i)).unwrap()diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rsnew file mode 100644index 000000000..c70352ab3--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/mod.rs@@ -0,0 +1,2 @@+pub mod weaver_node_cache_write;+pub mod weaver_top_children;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rsnew file mode 100644index 000000000..94d831d26--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_node_cache_write.rs@@ -0,0 +1,26 @@+use half::bf16;+use num_traits::Float;+use proc_macros::kernel;++use crate::array::ArrayElement;++#[kernel(WeaverNodeCacheWrite)]+#[variants(T, f32, bf16)]+pub fn weaver_node_cache_write<T: ArrayElement + Float>(+ current_qkv: *const T,+ node_qkv: *mut T,+ node_indices: *const u32,+ model_dim: u32,+ total: u32,+) {+ let model_dim = model_dim as usize;+ let qkv_width = 3 * model_dim;+ unsafe {+ for position in 0..total as usize {+ let row = position / (2 * model_dim);+ let offset = position - row * 2 * model_dim;+ *node_qkv.add(*node_indices.add(row) as usize * qkv_width + model_dim + offset) =+ *current_qkv.add(row * qkv_width + model_dim + offset);+ }+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rsnew file mode 100644index 000000000..c6027f6ed--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_top_children.rs@@ -0,0 +1,43 @@+use half::bf16;+use proc_macros::kernel;++use crate::backends::common::kernel::weaver::MAX_CANDIDATES;++#[kernel(WeaverTopChildren)]+pub fn weaver_top_children(+ residual_logits: *const bf16,+ candidate_scores: *const f32,+ candidate_ids: *const u32,+ output_ids: *mut u32,+ output_logprobs: *mut f32,+ rows: u32,+ candidates: u32,+ children: u32,+) {+ let rows = rows as usize;+ let candidates = candidates as usize;+ let children = children as usize;+ if candidates == 0 || candidates > MAX_CANDIDATES || children == 0 || children > candidates {+ return;+ }+ for row in 0..rows {+ let base = row * candidates;+ let logits = (0..candidates)+ .map(|index| unsafe { *candidate_scores.add(base + index) + (*residual_logits.add(base + index)).to_f32() })+ .collect::<Vec<_>>();+ let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);+ let log_sum = logits.iter().map(|value| (value - max).exp()).sum::<f32>().ln() + max;+ let mut indices = (0..candidates).collect::<Vec<_>>();+ indices.sort_by(|&left, &right| {+ logits[right]+ .total_cmp(&logits[left])+ .then_with(|| unsafe { (*candidate_ids.add(base + left)).cmp(&*candidate_ids.add(base + right)) })+ });+ for (rank, index) in indices.into_iter().take(children).enumerate() {+ unsafe {+ *output_ids.add(row * children + rank) = *candidate_ids.add(base + index);+ *output_logprobs.add(row * children + rank) = logits[index] - log_sum;+ }+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metalindex 1fead0584..f4b749570 100644--- a/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/attention/attention_last_query.metal@@ -53,11 +53,10 @@ template <uint HEAD_DIM>VARIANTS(HEAD_DIM, 128)PUBLIC KERNEL(AttentionLastQuery)(const device bfloat* prefix_qkv,- device bfloat* node_qkv,+ const device bfloat* node_qkv,const device bfloat* current_qkv,const device uint* ancestor_indices,const device uint* ancestor_counts,- const device uint* node_indices,device bfloat* output,constant uint& rows,constant uint& prefix_length,@@ -84,7 +83,6 @@ PUBLIC KERNEL(AttentionLastQuery)(const device bfloat4* prefix_vectors = (const device bfloat4*)prefix_qkv;const device bfloat4* node_vectors = (const device bfloat4*)node_qkv;const device bfloat4* current_row = (const device bfloat4*)current_qkv + row * qkv_vectors;- device bfloat4* dest_node = (device bfloat4*)node_qkv + node_indices[row] * qkv_vectors;const float4 query = float4(current_row[head * vectors_per_head + lane]) * scale;@@ -121,7 +119,5 @@ PUBLIC KERNEL(AttentionLastQuery)(attend_qkv<1>(query, key_offset, value_offset, values, max_score, sum, [&](int) { return current_row; });- dest_node[key_offset] = current_row[key_offset];- dest_node[value_offset] = current_row[value_offset];((device bfloat4*)output)[(row * num_heads + head) * vectors_per_head + lane] = bfloat4(values / sum);}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex b99f152c4..ab58a3be9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -48,7 +48,7 @@ impl GemmKernel {input_data_type: DataType,output_data_type: DataType,) -> Result<Self, MetalError> {- let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true)?;+ let bias_add = TensorAddBiasMetalKernel::new(context, output_data_type, weights_data_type, true, false)?;let hadamard = <<Metal as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(context,output_data_type,diff --git a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metalindex 8c2541c5d..76913bae1 100644--- a/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/tensor_add_bias/tensor_add_bias.metal@@ -7,16 +7,20 @@ VARIANTS(BiasT, float, half, bfloat)PUBLIC KERNEL(TensorAddBias)(const device T* input OPTIONAL(!in_place),const device BiasT* bias,+ const device uint* bias_row_indices OPTIONAL(indexed),device T* output,constant uint& num_cols,constant uint& length,const uint position AXIS(length, 32),- const bool in_place SPECIALIZE+ const bool in_place SPECIALIZE,+ const bool indexed SPECIALIZE) {if (in_place) {input = output;}- uint col = position % num_cols;- output[position] = static_cast<T>(static_cast<float>(input[position]) + static_cast<float>(bias[col]));+ const uint row = position / num_cols;+ const uint column = position % num_cols;+ const ulong bias_position = indexed ? ulong(bias_row_indices[row]) * ulong(num_cols) + ulong(column) : column;+ output[position] = static_cast<T>(static_cast<float>(input[position]) + static_cast<float>(bias[bias_position]));}diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metalnew file mode 100644index 000000000..c0a56d3eb--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_node_cache_write.metal@@ -0,0 +1,21 @@+#include <metal_stdlib>+#include "../common/dsl.h"++// Scatters the key/value halves of the current step's packed QKV rows into their+// node-cache slots. Kept out of AttentionLastQuery so that the node arena stays+// read-only for the whole attention dispatch.+template <typename T>+VARIANTS(T, float, bfloat)+PUBLIC KERNEL(WeaverNodeCacheWrite)(+ const device T* current_qkv,+ device T* node_qkv,+ const device uint* node_indices,+ constant uint& model_dim,+ constant uint& total,+ const uint position AXIS(total, 256)+) {+ const uint row = position / (2 * model_dim);+ const uint offset = position - row * 2 * model_dim;+ const uint qkv_width = 3 * model_dim;+ node_qkv[node_indices[row] * qkv_width + model_dim + offset] = current_qkv[row * qkv_width + model_dim + offset];+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metalnew file mode 100644index 000000000..ef7360872--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_top_children.metal@@ -0,0 +1,147 @@+#include <metal_stdlib>+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/top_k.h"++using namespace metal;++constant uint WEAVER_CANDIDATES_MAX = 512;+constant uint WEAVER_TOP_CHILDREN_THREADS = 256;+constant uint WEAVER_TOP_CHILDREN_SIMDGROUPS = 8;++METAL_FUNC bool weaver_better(uint score, uint token, uint index, uint best_score, uint best_token, uint best_index) {+ return score > best_score ||+ (score == best_score && (token < best_token || (token == best_token && index < best_index)));+}++PUBLIC KERNEL(WeaverTopChildren)(+ const device bfloat* residual_logits,+ const device float* candidate_scores,+ const device uint* candidate_ids,+ device uint* output_ids,+ device float* output_logprobs,+ constant uint& rows,+ constant uint& candidates,+ constant uint& children,+ threadgroup float reduce_float[WEAVER_TOP_CHILDREN_SIMDGROUPS],+ threadgroup uint reduce_score[WEAVER_TOP_CHILDREN_SIMDGROUPS],+ threadgroup uint reduce_token[WEAVER_TOP_CHILDREN_SIMDGROUPS],+ threadgroup uint reduce_index[WEAVER_TOP_CHILDREN_SIMDGROUPS],+ threadgroup float& logit_max,+ threadgroup float& log_sum,+ threadgroup uint& winner_score,+ threadgroup uint& winner_token,+ threadgroup uint& winner_index,+ const ThreadContext thread_context,+ const uint row GROUPS(rows),+ const uint lid THREADS(256)+) {+ if (candidates == 0 || candidates > WEAVER_CANDIDATES_MAX || children == 0 || children > candidates) {+ return;+ }++ const uint base = row * candidates;+ const uint first_index = lid;+ const uint second_index = lid + WEAVER_TOP_CHILDREN_THREADS;+ const bool first_valid = first_index < candidates;+ const bool second_valid = second_index < candidates;+ const float first_logit =+ first_valid ? candidate_scores[base + first_index] + float(residual_logits[base + first_index]) : -INFINITY;+ const float second_logit =+ second_valid ? candidate_scores[base + second_index] + float(residual_logits[base + second_index]) : -INFINITY;+ const uint first_token = first_valid ? uint(candidate_ids[base + first_index]) : 0xffffffffu;+ const uint second_token = second_valid ? uint(candidate_ids[base + second_index]) : 0xffffffffu;+ const uint first_score = first_valid ? top_k_score_key(first_logit) : 0u;+ const uint second_score = second_valid ? top_k_score_key(second_logit) : 0u;+ bool first_active = first_valid;+ bool second_active = second_valid;++ const float local_max = fmax(first_logit, second_logit);+ const float simd_maximum = simd_max(local_max);+ if (thread_context.simd_lane_id == 0) {+ reduce_float[thread_context.simdgroup_index] = simd_maximum;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (thread_context.simdgroup_index == 0) {+ const float group_value = thread_context.simd_lane_id < WEAVER_TOP_CHILDREN_SIMDGROUPS+ ? reduce_float[thread_context.simd_lane_id]+ : -INFINITY;+ const float maximum = simd_max(group_value);+ if (thread_context.simd_lane_id == 0) {+ logit_max = maximum;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ const float local_sum =+ (first_valid ? exp(first_logit - logit_max) : 0.0f) + (second_valid ? exp(second_logit - logit_max) : 0.0f);+ const float simd_total = simd_sum(local_sum);+ if (thread_context.simd_lane_id == 0) {+ reduce_float[thread_context.simdgroup_index] = simd_total;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (thread_context.simdgroup_index == 0) {+ const float group_value =+ thread_context.simd_lane_id < WEAVER_TOP_CHILDREN_SIMDGROUPS ? reduce_float[thread_context.simd_lane_id] : 0.0f;+ const float total = simd_sum(group_value);+ if (thread_context.simd_lane_id == 0) {+ log_sum = log(total) + logit_max;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ for (uint child = 0; child < children; ++child) {+ uint local_score = 0u;+ uint local_token = 0xffffffffu;+ uint local_index = 0xffffffffu;+ if (first_active) {+ local_score = first_score;+ local_token = first_token;+ local_index = first_index;+ }+ if (second_active &&+ weaver_better(second_score, second_token, second_index, local_score, local_token, local_index)) {+ local_score = second_score;+ local_token = second_token;+ local_index = second_index;+ }++ const uint simd_score = simd_max(local_score);+ const uint simd_token = simd_min(local_score == simd_score ? local_token : 0xffffffffu);+ const uint simd_index =+ simd_min(local_score == simd_score && local_token == simd_token ? local_index : 0xffffffffu);+ if (thread_context.simd_lane_id == 0) {+ reduce_score[thread_context.simdgroup_index] = simd_score;+ reduce_token[thread_context.simdgroup_index] = simd_token;+ reduce_index[thread_context.simdgroup_index] = simd_index;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (thread_context.simdgroup_index == 0) {+ const bool lane_valid = thread_context.simd_lane_id < WEAVER_TOP_CHILDREN_SIMDGROUPS;+ const uint group_score = lane_valid ? reduce_score[thread_context.simd_lane_id] : 0u;+ const uint selected_score = simd_max(group_score);+ const uint group_token =+ lane_valid && group_score == selected_score ? reduce_token[thread_context.simd_lane_id] : 0xffffffffu;+ const uint selected_token = simd_min(group_token);+ const uint group_index = lane_valid && group_score == selected_score && group_token == selected_token+ ? reduce_index[thread_context.simd_lane_id]+ : 0xffffffffu;+ const uint selected_index = simd_min(group_index);+ if (thread_context.simd_lane_id == 0) {+ winner_score = selected_score;+ winner_token = selected_token;+ winner_index = selected_index;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (lid == 0) {+ const float winner_logit = top_k_score_from_key(winner_score);+ output_ids[row * children + child] = winner_token;+ output_logprobs[row * children + child] = winner_logit - log_sum;+ }+ first_active = first_active && first_index != winner_index;+ second_active = second_active && second_index != winner_index;+ }+}diff --git a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rsindex a4356849b..9c0c60cd8 100644--- a/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs+++ b/crates/backend-uzu/src/encodable_block/mixer/attention/mod.rs@@ -29,7 +29,7 @@ mod mode;mod qkv_norm;mod state;-pub(crate) use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState};+pub(crate) use state::{ATTENTION_SUFFIX_CAPACITY, AttentionState, AttentionStateType};pub mod rope;diff --git a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rsindex 030c5e769..705a5958f 100644--- a/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs+++ b/crates/backend-uzu/src/encodable_block/per_layer_embedding.rs@@ -211,8 +211,9 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {let gate_act_mul = <B::Kernels as Kernels>::GatedActMulKernel::new(context, data_type, false, false).map_err(PerLayerEmbeddingError::BackendError)?;- let residual_finalize = <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, data_type, true)- .map_err(PerLayerEmbeddingError::BackendError)?;+ let residual_finalize =+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, data_type, true, false)+ .map_err(PerLayerEmbeddingError::BackendError)?;let residual_combine = <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, data_type, true).map_err(PerLayerEmbeddingError::BackendError)?;@@ -246,6 +247,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {self.residual_finalize.encode(None::<&Allocation<B>>,hidden,+ None::<&Allocation<B>>,&mut *outputs,length as u32,length as u32,diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsindex c48b1a31e..0b5385895 100644--- a/crates/backend-uzu/src/encodable_block/weaver.rs+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -1,15 +1,58 @@+#![allow(dead_code)]+use thiserror::Error;use crate::{- backends::common::{Allocation, Backend},+ array::size_for_shape,+ backends::common::{+ Allocation, AllocationType, Backend, Context, Encoder, Kernels,+ kernel::{+ ActivationKernel, AttentionLastQueryKernel, AttentionPrepareKernel, TensorAddBiasKernel,+ TensorAddSwapKernel, WeaverNodeCacheWriteKernel, WeaverTopChildrenKernel, weaver::MAX_CANDIDATES,+ },+ },config::{normalization::NormalizationConfig, weaver::WeaverConfig},data_type::DataType,- encodable_block::linear::{Linear, LinearBlockError},+ encodable_block::{+ embedding::{Embedding, EmbeddingError},+ linear::{Linear, LinearBlockError},+ mixer::attention::{+ AttentionStateType,+ core::{AttentionCoreEncodeArguments, AttentionCoreNewArguments, AttentionCores},+ },+ normalization::{Normalization, NormalizationNewError, PostLayerScalar},+ },parameters::{ParameterLoaderError, ParameterTree},};-#[allow(dead_code)]-pub struct Weaver<B: Backend> {+pub(crate) struct WeaverPrefix<B: Backend> {+ layer_qkv: Box<[Allocation<B>]>,+ pub(crate) length: usize,+}++pub(crate) struct WeaverBatchStepInput<'a> {+ pub node_index: usize,+ pub parent_token: u32,+ pub candidates: &'a [u32],+ pub candidate_scores: &'a [f32],+ pub ancestors: &'a [usize],+ pub depth: usize,+}++pub(crate) struct WeaverNodeState<B: Backend> {+ layer_qkv: Box<[Allocation<B>]>,+ capacity: usize,+}++struct WeaverStepInputs<B: Backend> {+ candidate_ids: Allocation<B>,+ candidate_scores: Allocation<B>,+ ancestor_indices: Allocation<B>,+ metadata: Allocation<B>,+ ancestor_stride: usize,+}++pub(crate) struct Weaver<B: Backend> {embedding_norm: WeaverNorm<B>,hidden_state_norm: WeaverNorm<B>,embedding_projection: Box<dyn Linear<B>>,@@ -18,22 +61,41 @@ pub struct Weaver<B: Backend> {hidden_state_projection: Box<dyn Linear<B>>,query_projection: Box<dyn Linear<B>>,position_embeddings: Allocation<B>,+ position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,+ indexed_position_add: <B::Kernels as Kernels>::TensorAddBiasKernel,+ top_children: <B::Kernels as Kernels>::WeaverTopChildrenKernel,+ model_dim: usize,+ target_model_dim: usize,+ max_depth: usize,+ data_type: DataType,}-#[allow(dead_code)]struct WeaverNorm<B: Backend> {- scales: Allocation<B>,+ normalization: Normalization<B>,biases: Option<Allocation<B>>,+ bias_kernel: Option<<B::Kernels as Kernels>::TensorAddBiasKernel>,+ dimension: usize,}-#[allow(dead_code)]struct WeaverBlock<B: Backend> {- pre_attention_norm: WeaverNorm<B>,qkv_projection: Box<dyn Linear<B>>,out_projection: Box<dyn Linear<B>>,+ prefix_attention: AttentionCores<B>,+ attention_prepare: <B::Kernels as Kernels>::AttentionPrepareKernel,+ pre_attention_norm: WeaverNorm<B>,pre_mlp_norm: WeaverNorm<B>,up_projection: Box<dyn Linear<B>>,down_projection: Box<dyn Linear<B>>,+ activation: <B::Kernels as Kernels>::ActivationKernel,+ residual_add: <B::Kernels as Kernels>::TensorAddSwapKernel,+ last_query_attention: <B::Kernels as Kernels>::AttentionLastQueryKernel,+ node_cache_write: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,+ attention_scale: f32,+ model_dim: usize,+ hidden_dim: usize,+ num_heads: usize,+ head_dim: usize,+ data_type: DataType,}#[derive(Debug, Error)]@@ -42,48 +104,139 @@ pub enum WeaverNewError<B: Backend> {ParameterLoader(#[from] ParameterLoaderError<B>),#[error("linear error: {0}")]Linear(#[from] LinearBlockError<B>),+ #[error("normalization error: {0}")]+ Normalization(#[from] NormalizationNewError<B>),+ #[error("backend error: {0}")]+ Backend(#[source] B::Error),#[error("model_dim must be divisible by num_heads")]InvalidHeadConfig,+ #[error("candidate_pool_size must be in 1..={MAX_CANDIDATES}, got {0}")]+ InvalidCandidatePoolSize(usize),+}++#[derive(Debug, Error)]+pub enum WeaverEncodeError<B: Backend> {+ #[error("backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("embedding error: {0}")]+ Embedding(#[from] EmbeddingError<B>),}-fn plain_norm<B: Backend>(- dim: usize,- config: &NormalizationConfig,+fn linear<B: Backend>(+ context: &B::Context,parameter_tree: &ParameterTree<B>,-) -> Result<WeaverNorm<B>, ParameterLoaderError<B>> {- let scales = parameter_tree.leaf("scales")?.validate(&[dim], DataType::F32)?.read_allocation()?;- let biases = config- .has_biases- .then(|| parameter_tree.leaf("biases")?.validate(&[dim], DataType::F32)?.read_allocation())- .transpose()?;- Ok(WeaverNorm {- scales,- biases,- })+ name: &str,+ input_dim: usize,+ output_dim: usize,+ has_biases: bool,+ data_type: DataType,+) -> Result<Box<dyn Linear<B>>, WeaverNewError<B>> {+ Ok(<dyn Linear<B>>::new(input_dim, [output_dim], has_biases, context, data_type, ¶meter_tree.subtree(name)?)?)+}++impl<B: Backend> WeaverNorm<B> {+ fn new(+ context: &B::Context,+ dim: usize,+ config: &NormalizationConfig,+ parameter_tree: &ParameterTree<B>,+ data_type: DataType,+ ) -> Result<Self, WeaverNewError<B>> {+ let mut normalization_config = config.clone();+ normalization_config.has_biases = false;+ let normalization = Normalization::new(+ dim,+ None,+ false,+ false,+ PostLayerScalar::None,+ data_type,+ &normalization_config,+ parameter_tree,+ context,+ )?;+ let biases = config+ .has_biases+ .then(|| parameter_tree.leaf("biases")?.validate(&[dim], DataType::F32)?.read_allocation())+ .transpose()?;+ let bias_kernel = biases+ .as_ref()+ .map(|_| <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, false))+ .transpose()+ .map_err(WeaverNewError::Backend)?;+ Ok(Self {+ normalization,+ biases,+ bias_kernel,+ dimension: dim,+ })+ }++ fn encode(+ &self,+ input: &Allocation<B>,+ row_count: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let mut output = self.normalization.encode(input, 0, row_count, None, encoder)?;+ if let (Some(biases), Some(kernel)) = (&self.biases, &self.bias_kernel) {+ kernel.encode(+ None::<&Allocation<B>>,+ biases,+ None::<&Allocation<B>>,+ &mut output,+ self.dimension as u32,+ (row_count * self.dimension) as u32,+ encoder,+ );+ }+ Ok(output)+ }}impl<B: Backend> Weaver<B> {+ pub(crate) fn create_node_state(+ &self,+ capacity: usize,+ context: &B::Context,+ ) -> Result<WeaverNodeState<B>, WeaverEncodeError<B>> {+ assert!(capacity > 0);+ let qkv_size = size_for_shape(&[capacity, 3, self.model_dim], self.data_type);+ let layer_qkv = (0..self.blocks.len())+ .map(|_| context.create_allocation(qkv_size, AllocationType::Global).map_err(WeaverEncodeError::Backend))+ .collect::<Result<Box<[_]>, _>>()?;+ Ok(WeaverNodeState {+ layer_qkv,+ capacity,+ })+ }+pub(crate) fn new(context: &B::Context,config: &WeaverConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, WeaverNewError<B>> {+ assert_eq!(data_type, DataType::BF16, "Weaver only supports BF16");if config.num_heads == 0 || !config.model_dim.is_multiple_of(config.num_heads) {return Err(WeaverNewError::InvalidHeadConfig);}-- let embedding_norm =- plain_norm(config.target_embedding_dim, &config.norm_config, ¶meter_tree.subtree("embedding_norm")?)?;- let hidden_state_norm =- plain_norm(config.target_model_dim, &config.norm_config, ¶meter_tree.subtree("hidden_state_norm")?)?;- let embedding_projection = <dyn Linear<B>>::new(+ if config.candidate_pool_size == 0 || config.candidate_pool_size > MAX_CANDIDATES {+ return Err(WeaverNewError::InvalidCandidatePoolSize(config.candidate_pool_size));+ }+ let norm = |dim, name: &str| -> Result<_, WeaverNewError<B>> {+ WeaverNorm::new(context, dim, &config.norm_config, ¶meter_tree.subtree(name)?, data_type)+ };+ let embedding_norm = norm(config.target_embedding_dim, "embedding_norm")?;+ let hidden_state_norm = norm(config.target_model_dim, "hidden_state_norm")?;+ let embedding_projection = linear(+ context,+ parameter_tree,+ "embedding_projection",config.target_embedding_dim,- [config.model_dim],+ config.model_dim,true,- context,data_type,- ¶meter_tree.subtree("embedding_projection")?,)?;let blocks_tree = parameter_tree.subtree("blocks")?;let blocks = (0..config.num_layers)@@ -92,34 +245,44 @@ impl<B: Backend> Weaver<B> {context,config.model_dim,config.hidden_dim,+ config.num_heads,&config.norm_config,&blocks_tree.subtree(&index.to_string())?,data_type,)}).collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;- let output_norm = plain_norm(config.model_dim, &config.norm_config, ¶meter_tree.subtree("output_norm")?)?;- let hidden_state_projection = <dyn Linear<B>>::new(+ let output_norm = norm(config.model_dim, "output_norm")?;+ let hidden_state_projection = linear(+ context,+ parameter_tree,+ "hidden_state_projection",config.target_model_dim,- [config.model_dim],+ config.model_dim,true,- context,data_type,- ¶meter_tree.subtree("hidden_state_projection")?,)?;- let query_projection = <dyn Linear<B>>::new(+ let query_projection = linear(+ context,+ parameter_tree,+ "query_projection",config.model_dim,- [config.target_model_dim],+ config.target_model_dim,false,- context,data_type,- ¶meter_tree.subtree("query_projection")?,)?;let position_embeddings = parameter_tree.leaf("position_embeddings")?.validate(&[config.max_depth, config.model_dim], DataType::F32)?.read_allocation()?;-+ let position_add =+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, false)+ .map_err(WeaverNewError::Backend)?;+ let indexed_position_add =+ <B::Kernels as Kernels>::TensorAddBiasKernel::new(context, data_type, DataType::F32, true, true)+ .map_err(WeaverNewError::Backend)?;+ let top_children =+ <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(context).map_err(WeaverNewError::Backend)?;Ok(Self {embedding_norm,hidden_state_norm,@@ -129,8 +292,257 @@ impl<B: Backend> Weaver<B> {hidden_state_projection,query_projection,position_embeddings,+ position_add,+ indexed_position_add,+ top_children,+ model_dim: config.model_dim,+ target_model_dim: config.target_model_dim,+ max_depth: config.max_depth,+ data_type,+ })+ }++ pub(crate) fn build_prefix(+ &self,+ target_hidden: &Allocation<B>,+ lookaheads: &Allocation<B>,+ lookahead_offset: usize,+ lookahead_count: usize,+ context: &B::Context,+ ) -> Result<WeaverPrefix<B>, WeaverEncodeError<B>> {+ let length = lookahead_count + 1;+ assert!(lookahead_count <= self.max_depth);+ let row_bytes = self.target_model_dim * self.data_type.size_in_bytes();+ assert!(target_hidden.size() >= row_bytes);+ assert!(lookaheads.size() >= (lookahead_offset + lookahead_count) * row_bytes);++ let mut encoder = Encoder::new(context).map_err(WeaverEncodeError::Backend)?;+ let mut input = encoder+ .allocate_scratch(size_for_shape(&[length, self.target_model_dim], self.data_type))+ .map_err(WeaverEncodeError::Backend)?;+ encoder.encode_copy(target_hidden, 0..row_bytes, &mut input, 0..row_bytes);+ if lookahead_count > 0 {+ encoder.encode_copy(+ lookaheads,+ lookahead_offset * row_bytes..(lookahead_offset + lookahead_count) * row_bytes,+ &mut input,+ row_bytes..length * row_bytes,+ );+ }+ let normalized =+ self.hidden_state_norm.encode(&input, length, &mut encoder).map_err(WeaverEncodeError::Backend)?;+ let mut hidden = self+ .hidden_state_projection+ .encode(normalized, length, &mut encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let position_elements = lookahead_count * self.model_dim;+ self.position_add.encode(+ None::<&Allocation<B>>,+ &self.position_embeddings,+ None::<&Allocation<B>>,+ (&mut hidden, self.model_dim * self.data_type.size_in_bytes()),+ position_elements as u32,+ position_elements as u32,+ &mut encoder,+ );++ let mut layer_qkv = Vec::with_capacity(self.blocks.len());+ for block in &self.blocks {+ let (next_hidden, qkv) =+ block.encode_prefix(hidden, length, &mut encoder).map_err(WeaverEncodeError::Backend)?;+ let mut cached_qkv =+ context.create_allocation(qkv.size(), AllocationType::Global).map_err(WeaverEncodeError::Backend)?;+ encoder.encode_copy(&qkv, .., &mut cached_qkv, ..);+ layer_qkv.push(cached_qkv);+ hidden = next_hidden;+ }+ let completed = encoder.end_encoding().submit().wait_until_completed().map_err(WeaverEncodeError::Backend)?;+ drop(input);+ drop(hidden);+ drop(completed);+ Ok(WeaverPrefix {+ layer_qkv: layer_qkv.into_boxed_slice(),+ length,})}++ pub(crate) fn step_batch(+ &self,+ prefix: &WeaverPrefix<B>,+ inputs: &[WeaverBatchStepInput<'_>],+ state: &mut WeaverNodeState<B>,+ children: usize,+ target_embedding: &Embedding<B>,+ context: &B::Context,+ ) -> Result<Vec<Vec<(u32, f32)>>, WeaverEncodeError<B>> {+ assert!(!inputs.is_empty());+ let mut encoder = Encoder::new(context).map_err(WeaverEncodeError::Backend)?;+ let (child_ids, child_logprobs) =+ self.encode_step_batch(prefix, inputs, state, children, target_embedding, &mut encoder)?;+ let completed = encoder.end_encoding().submit().wait_until_completed().map_err(WeaverEncodeError::Backend)?;+ let child_ids_output = child_ids.copyout::<u32>();+ let child_logprobs_output = child_logprobs.copyout::<f32>();+ let outputs = (0..inputs.len())+ .map(|batch_index| {+ (0..children)+ .map(|child| {+ let index = batch_index * children + child;+ (child_ids_output[index], child_logprobs_output[index])+ })+ .collect()+ })+ .collect();+ drop(child_ids);+ drop(child_logprobs);+ drop(completed);+ Ok(outputs)+ }++ fn encode_step_batch(+ &self,+ prefix: &WeaverPrefix<B>,+ inputs: &[WeaverBatchStepInput<'_>],+ state: &mut WeaverNodeState<B>,+ children: usize,+ target_embedding: &Embedding<B>,+ encoder: &mut Encoder<B>,+ ) -> Result<(Allocation<B>, Allocation<B>), WeaverEncodeError<B>> {+ let rows = inputs.len();+ let candidates = inputs[0].candidates.len();+ assert!(candidates > 0 && candidates <= MAX_CANDIDATES);+ assert!(children > 0 && children <= candidates);+ // Indices reach the GPU unchecked, so they must stay inside the arena.+ // Batch nodes are expected to be mutually independent: a child only+ // enters the frontier once its parent's step has been read back, so a+ // parent and child can never share a batch.+ assert!(+ inputs.iter().all(|input| {+ input.candidates.len() == candidates+ && input.candidate_scores.len() == candidates+ && input.depth < self.max_depth+ && input.node_index < state.capacity+ && input.ancestors.iter().all(|&ancestor| ancestor < state.capacity)+ }),+ "inputs must reference node indices within the node-state capacity"+ );++ let (mut current, step) = self.encode_step_inputs(inputs, target_embedding, encoder)?;++ for (layer_index, block) in self.blocks.iter().enumerate() {+ current = block+ .encode_step(+ current,+ &prefix.layer_qkv[layer_index],+ &mut state.layer_qkv[layer_index],+ &step,+ prefix.length,+ rows,+ encoder,+ )+ .map_err(WeaverEncodeError::Backend)?;+ }++ self.encode_step_output(¤t, &step, rows, candidates, children, target_embedding, encoder)+ }++ fn encode_step_inputs(+ &self,+ inputs: &[WeaverBatchStepInput<'_>],+ target_embedding: &Embedding<B>,+ encoder: &mut Encoder<B>,+ ) -> Result<(Allocation<B>, WeaverStepInputs<B>), WeaverEncodeError<B>> {+ let rows = inputs.len();+ let token_ids = encoder+ .allocate_constant_from_slice(&inputs.iter().map(|input| input.parent_token).collect::<Vec<_>>())+ .map_err(WeaverEncodeError::Backend)?;+ let candidate_ids = encoder+ .allocate_constant_from_slice(+ &inputs.iter().flat_map(|input| input.candidates.iter().copied()).collect::<Vec<_>>(),+ )+ .map_err(WeaverEncodeError::Backend)?;+ let candidate_scores = encoder+ .allocate_constant_from_slice(+ &inputs.iter().flat_map(|input| input.candidate_scores.iter().copied()).collect::<Vec<_>>(),+ )+ .map_err(WeaverEncodeError::Backend)?;+ let token_embedding = target_embedding.encode_lookup(&token_ids, rows, encoder)?;+ let embedding_normalized =+ self.embedding_norm.encode(&token_embedding, rows, encoder).map_err(WeaverEncodeError::Backend)?;+ let mut current = self+ .embedding_projection+ .encode(embedding_normalized, rows, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ // [depths, ancestor counts, node indices]+ let mut metadata_values = Vec::with_capacity(3 * rows);+ metadata_values.extend(inputs.iter().map(|input| input.depth as u32));+ metadata_values.extend(inputs.iter().map(|input| input.ancestors.len() as u32));+ metadata_values.extend(inputs.iter().map(|input| input.node_index as u32));+ let metadata = encoder.allocate_constant_from_slice(&metadata_values).map_err(WeaverEncodeError::Backend)?;+ self.indexed_position_add.encode(+ None::<&Allocation<B>>,+ &self.position_embeddings,+ Some(&metadata),+ &mut current,+ self.model_dim as u32,+ (rows * self.model_dim) as u32,+ encoder,+ );++ let ancestor_stride = inputs.iter().map(|input| input.ancestors.len()).max().unwrap().max(1);+ let mut ancestors = vec![0u32; rows * ancestor_stride];+ for (row, input) in inputs.iter().enumerate() {+ for (offset, &ancestor) in input.ancestors.iter().enumerate() {+ ancestors[row * ancestor_stride + offset] = ancestor as u32;+ }+ }+ let ancestor_indices = encoder.allocate_constant_from_slice(&ancestors).map_err(WeaverEncodeError::Backend)?;+ Ok((+ current,+ WeaverStepInputs {+ candidate_ids,+ candidate_scores,+ ancestor_indices,+ metadata,+ ancestor_stride,+ },+ ))+ }++ /// Outputs must outlive the batch.+ fn encode_step_output(+ &self,+ current: &Allocation<B>,+ step: &WeaverStepInputs<B>,+ rows: usize,+ candidates: usize,+ children: usize,+ target_embedding: &Embedding<B>,+ encoder: &mut Encoder<B>,+ ) -> Result<(Allocation<B>, Allocation<B>), WeaverEncodeError<B>> {+ let output_normalized = self.output_norm.encode(current, rows, encoder).map_err(WeaverEncodeError::Backend)?;+ let query =+ self.query_projection.encode(output_normalized, rows, encoder).map_err(WeaverEncodeError::Backend)?;+ let candidate_logits =+ target_embedding.encode_readout_sparse(&query, &step.candidate_ids, rows, candidates, encoder)?;+ let mut child_ids = encoder+ .allocate_scratch(size_for_shape(&[rows, children], DataType::U32))+ .map_err(WeaverEncodeError::Backend)?;+ let mut child_logprobs = encoder+ .allocate_scratch(size_for_shape(&[rows, children], DataType::F32))+ .map_err(WeaverEncodeError::Backend)?;+ self.top_children.encode(+ &candidate_logits,+ &step.candidate_scores,+ &step.candidate_ids,+ &mut child_ids,+ &mut child_logprobs,+ rows as u32,+ candidates as u32,+ children as u32,+ encoder,+ );+ Ok((child_ids, child_logprobs))+ }}impl<B: Backend> WeaverBlock<B> {@@ -138,52 +550,186 @@ impl<B: Backend> WeaverBlock<B> {context: &B::Context,model_dim: usize,hidden_dim: usize,+ num_heads: usize,norm_config: &NormalizationConfig,parameter_tree: &ParameterTree<B>,data_type: DataType,) -> Result<Self, WeaverNewError<B>> {- let pre_attention_norm = plain_norm(model_dim, norm_config, ¶meter_tree.subtree("pre_attention_norm")?)?;- let qkv_projection = <dyn Linear<B>>::new(- model_dim,- [model_dim, model_dim, model_dim],- false,- context,- data_type,- ¶meter_tree.subtree("qkv_projection")?,- )?;- let out_projection = <dyn Linear<B>>::new(- model_dim,- [model_dim],- false,- context,- data_type,- ¶meter_tree.subtree("out_projection")?,- )?;- let pre_mlp_norm = plain_norm(model_dim, norm_config, ¶meter_tree.subtree("pre_mlp_norm")?)?;- let up_projection = <dyn Linear<B>>::new(- model_dim,- [hidden_dim],- true,- context,- data_type,- ¶meter_tree.subtree("up_projection")?,- )?;- let down_projection = <dyn Linear<B>>::new(- hidden_dim,- [model_dim],- true,+ let head_dim = model_dim / num_heads;+ let attention_scale = 1.0 / (head_dim as f32).sqrt();+ let qkv_projection =+ linear(context, parameter_tree, "qkv_projection", model_dim, 3 * model_dim, false, data_type)?;+ let out_projection = linear(context, parameter_tree, "out_projection", model_dim, model_dim, false, data_type)?;+ let prefix_attention = AttentionCores::new(+ AttentionCoreNewArguments {+ head_dim,+ num_groups: num_heads,+ num_q_heads: num_heads,+ has_sinks: false,+ is_kv_cache_ring: false,+ is_causal: true,+ is_trie: false,+ sliding_window_size: None,+ scale: Some(attention_scale),+ data_type,+ },context,- data_type,- ¶meter_tree.subtree("down_projection")?,- )?;-+ )+ .map_err(WeaverNewError::Backend)?;+ let attention_prepare =+ <B::Kernels as Kernels>::AttentionPrepareKernel::new(context, data_type, DataType::F32, true, false)+ .map_err(WeaverNewError::Backend)?;+ let norm = |name: &str| -> Result<_, WeaverNewError<B>> {+ WeaverNorm::new(context, model_dim, norm_config, ¶meter_tree.subtree(name)?, data_type)+ };+ let pre_attention_norm = norm("pre_attention_norm")?;+ let pre_mlp_norm = norm("pre_mlp_norm")?;+ let up_projection = linear(context, parameter_tree, "up_projection", model_dim, hidden_dim, true, data_type)?;+ let down_projection =+ linear(context, parameter_tree, "down_projection", hidden_dim, model_dim, true, data_type)?;+ let activation = <B::Kernels as Kernels>::ActivationKernel::new(context, data_type, true)+ .map_err(WeaverNewError::Backend)?;+ let residual_add =+ <B::Kernels as Kernels>::TensorAddSwapKernel::new(context, data_type).map_err(WeaverNewError::Backend)?;+ let last_query_attention =+ <B::Kernels as Kernels>::AttentionLastQueryKernel::new(context, head_dim as u32, num_heads as u32)+ .map_err(WeaverNewError::Backend)?;+ let node_cache_write = <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context, data_type)+ .map_err(WeaverNewError::Backend)?;Ok(Self {- pre_attention_norm,qkv_projection,out_projection,+ prefix_attention,+ attention_prepare,+ pre_attention_norm,pre_mlp_norm,up_projection,down_projection,+ activation,+ residual_add,+ last_query_attention,+ node_cache_write,+ attention_scale,+ model_dim,+ hidden_dim,+ num_heads,+ head_dim,+ data_type,})}++ fn encode_step(+ &self,+ current: Allocation<B>,+ prefix_qkv: &Allocation<B>,+ state_qkv: &mut Allocation<B>,+ step: &WeaverStepInputs<B>,+ prefix_length: usize,+ rows: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<Allocation<B>, B::Error> {+ let normalized = self.pre_attention_norm.encode(¤t, rows, encoder)?;+ let current_qkv = self.qkv_projection.encode(normalized, rows, encoder)?;+ let metadata_row_bytes = rows * DataType::U32.size_in_bytes();+ let ancestor_counts = (&step.metadata, metadata_row_bytes);+ let node_indices = (&step.metadata, 2 * metadata_row_bytes);+ let mut attention = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;+ self.last_query_attention.encode(+ prefix_qkv,+ &*state_qkv,+ ¤t_qkv,+ &step.ancestor_indices,+ ancestor_counts,+ &mut attention,+ rows as u32,+ prefix_length as u32,+ step.ancestor_stride as u32,+ self.attention_scale,+ encoder,+ );+ // Separate dispatch so the node arena is read-only above; the encoder+ // serializes it after the attention that reads the ancestor slots.+ self.node_cache_write.encode(+ ¤t_qkv,+ state_qkv,+ node_indices,+ self.model_dim as u32,+ (rows * 2 * self.model_dim) as u32,+ encoder,+ );+ attention = self.out_projection.encode(attention, rows, encoder)?;+ let mut residual = current;+ self.residual_add.encode(&mut residual, &mut attention, (rows * self.model_dim) as u32, encoder);+ let normalized = self.pre_mlp_norm.encode(&residual, rows, encoder)?;+ let mut mlp = self.up_projection.encode(normalized, rows, encoder)?;+ self.activation.encode(+ None::<&Allocation<B>>,+ &mut mlp,+ (rows * self.hidden_dim) as u32,+ crate::backends::common::gpu_types::ActivationType::GELUExact,+ encoder,+ );+ let mut output = self.down_projection.encode(mlp, rows, encoder)?;+ self.residual_add.encode(&mut output, &mut residual, (rows * self.model_dim) as u32, encoder);+ Ok(residual)+ }++ fn encode_prefix(+ &self,+ hidden: Allocation<B>,+ rows: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<(Allocation<B>, Allocation<B>), B::Error> {+ let normalized = self.pre_attention_norm.encode(&hidden, rows, encoder)?;+ let qkv = self.qkv_projection.encode(normalized, rows, encoder)?;+ let mut queries =+ encoder.allocate_scratch(size_for_shape(&[self.num_heads, rows, self.head_dim], self.data_type))?;+ let mut keys = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;+ let mut values = encoder.allocate_scratch(size_for_shape(&[rows, self.model_dim], self.data_type))?;+ self.attention_prepare.encode(+ &qkv,+ &mut queries,+ Some(&mut keys),+ Some(&mut values),+ None::<&Allocation<B>>,+ None::<&Allocation<B>>,+ self.num_heads as u32,+ Some(self.num_heads as u32),+ self.head_dim as u32,+ None,+ Some(0),+ rows as u32,+ encoder,+ );+ let state_type = AttentionStateType::Full {+ length: 0,+ };+ let attention = self.prefix_attention.encode(+ AttentionCoreEncodeArguments {+ queries: &queries,+ keys: &keys,+ values: &values,+ suffix_length: rows,+ trie: None,+ sinks: None,+ state_type: &state_type,+ },+ encoder,+ )?;+ let mut attention = self.out_projection.encode(attention, rows, encoder)?;+ let mut residual = hidden;+ self.residual_add.encode(&mut residual, &mut attention, (rows * self.model_dim) as u32, encoder);+ let normalized = self.pre_mlp_norm.encode(&residual, rows, encoder)?;+ let mut mlp = self.up_projection.encode(normalized, rows, encoder)?;+ self.activation.encode(+ None::<&Allocation<B>>,+ &mut mlp,+ (rows * self.hidden_dim) as u32,+ crate::backends::common::gpu_types::ActivationType::GELUExact,+ encoder,+ );+ let mut output = self.down_projection.encode(mlp, rows, encoder)?;+ self.residual_add.encode(&mut output, &mut residual, (rows * self.model_dim) as u32, encoder);+ Ok((residual, qkv))+ }}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rsindex 3f31f6aae..0e36f1cf4 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/attention/attention_last_query_test.rs@@ -28,7 +28,6 @@ struct Runner<B: Backend> {current_qkv: Allocation<B>,ancestor_indices: Allocation<B>,ancestor_counts: Allocation<B>,- node_indices: Allocation<B>,output: Allocation<B>,rows: u32,prefix_length: u32,@@ -69,10 +68,6 @@ impl<B: Backend> Runner<B> {current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(rows * packed_width, 29)),ancestor_indices: alloc_allocation_with_data::<B, u32>(&context, &ancestor_indices),ancestor_counts: alloc_allocation_with_data::<B, u32>(&context, &ancestor_counts),- node_indices: alloc_allocation_with_data::<B, u32>(- &context,- &(first_output_node as u32..nodes as u32).collect::<Vec<_>>(),- ),output: alloc_allocation::<B, bf16>(&context, rows * NUM_HEADS as usize * HEAD_DIM),context,kernel,@@ -90,11 +85,10 @@ impl<B: Backend> Runner<B> {for _ in 0..repetitions {self.kernel.encode(&self.prefix_qkv,- &mut self.node_qkv,+ &self.node_qkv,&self.current_qkv,&self.ancestor_indices,&self.ancestor_counts,- &self.node_indices,&mut self.output,self.rows,self.prefix_length,@@ -111,38 +105,17 @@ impl<B: Backend> Runner<B> {fn attention_last_query_matches_cpu() {let mut cpu = Runner::<Cpu>::new(4, 5, 3, 16);let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);- let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);- let node_indices = allocation_to_vec::<Cpu, u32>(&cpu.node_indices);cpu.encode(1);let expected_output = allocation_to_vec::<Cpu, bf16>(&cpu.output);- let expected_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);- let packed_width = 3 * NUM_HEADS as usize * HEAD_DIM;- let q_width = NUM_HEADS as usize * HEAD_DIM;- for node in 0..expected_nodes.len() / packed_width {- let range = node * packed_width..(node + 1) * packed_width;- if let Some(row) = node_indices.iter().position(|&index| index as usize == node) {- let current = row * packed_width;- assert_eq!(- expected_nodes[range.start..range.start + q_width],- initial_nodes[range.start..range.start + q_width]- );- assert_eq!(- expected_nodes[range.start + q_width..range.end],- current_qkv[current + q_width..current + packed_width],- );- } else {- assert_eq!(expected_nodes[range.clone()], initial_nodes[range]);- }- }+ assert_eq!(allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv), initial_nodes);let mut metal = Runner::<Metal>::new(4, 5, 3, 16);metal.encode(1);let actual_output = allocation_to_vec::<Metal, bf16>(&metal.output);- let actual_nodes = allocation_to_vec::<Metal, bf16>(&metal.node_qkv);assert_eq_float(&expected_output, &actual_output, 0.02, "AttentionLastQuery output");- assert_eq!(expected_nodes, actual_nodes);+ assert_eq!(allocation_to_vec::<Metal, bf16>(&metal.node_qkv), initial_nodes);}#[uzu_test]diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rsindex 89b8930d1..b4cfbf441 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/mod.rs@@ -17,3 +17,4 @@ mod tensor_add_bias_test;mod tensor_add_scale_test;mod tensor_add_swap_test;mod tensor_copy_test;+mod weaver;diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/tensor_add_bias_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/tensor_add_bias_test.rsindex 5ceee6461..0646ea5fc 100644--- a/crates/backend-uzu/tests/unit/backends/common/kernel/tensor_add_bias_test.rs+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/tensor_add_bias_test.rs@@ -7,7 +7,7 @@ use test_runner::for_each_backend;use crate::{array::ArrayElement,- backends::common::{Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},+ backends::common::{Allocation, Backend, Context, Encoder, Kernels, kernel::TensorAddBiasKernel},tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},};@@ -64,6 +64,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(T::data_type(),T::data_type(),in_place,+ false,).expect("Failed to create TensorAddBiasKernel");@@ -79,6 +80,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(kernel.encode(input_allocation.as_ref(),&bias_allocation,+ None::<&Allocation<B>>,&mut output_allocation,input.num_cols,input.length,@@ -126,3 +128,30 @@ fn test_bf16() {fn test_bf16_in_place() {test::<bf16>(true);}++#[uzu_test]+fn test_bf16_indexed_rows() {+ let values = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0].map(bf16::from_f32);+ let bias = [10.0f32, 20.0, 30.0, 40.0, 50.0, 60.0];+ let bias_row_indices = [1u32, 0];+ let expected = [41.0, 52.0, 63.0, 14.0, 25.0, 36.0].map(bf16::from_f32);++ for_each_backend!(|B| {+ let context = <B as Backend>::Context::new().unwrap();+ let kernel = <<B as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(+ &context,+ bf16::data_type(),+ f32::data_type(),+ true,+ true,+ )+ .unwrap();+ let mut output = alloc_allocation_with_data::<B, bf16>(&context, &values);+ let bias = alloc_allocation_with_data::<B, f32>(&context, &bias);+ let bias_row_indices = alloc_allocation_with_data::<B, u32>(&context, &bias_row_indices);+ let mut encoder = Encoder::new(context.as_ref()).unwrap();+ kernel.encode(None::<&Allocation<B>>, &bias, Some(&bias_row_indices), &mut output, 3, 6, &mut encoder);+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ assert_eq!(allocation_to_vec::<B, bf16>(&output), expected);+ });+}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rsnew file mode 100644index 000000000..34c4d8777--- /dev/null+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/mod.rs@@ -0,0 +1,2 @@+mod weaver_node_cache_write_test;+mod weaver_top_children_test;diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rsnew file mode 100644index 000000000..eefb0f025--- /dev/null+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_node_cache_write_test.rs@@ -0,0 +1,96 @@+#![cfg(metal_backend)]++use half::bf16;+use proc_macros::uzu_test;++use crate::{+ backends::{+ common::{Allocation, Backend, Encoder, Kernels, kernel::WeaverNodeCacheWriteKernel},+ cpu::Cpu,+ metal::Metal,+ },+ data_type::DataType,+ tests::helpers::{alloc_allocation_with_data, allocation_to_vec, create_context},+};++const MODEL_DIM: usize = 64;+const ROWS: usize = 4;+const NODES: usize = 9;++struct Runner<B: Backend> {+ context: std::sync::Arc<B::Context>,+ kernel: <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel,+ current_qkv: Allocation<B>,+ node_qkv: Allocation<B>,+ node_indices: Allocation<B>,+}++fn values(+ length: usize,+ offset: usize,+) -> Vec<bf16> {+ (0..length).map(|index| bf16::from_f32((((index + offset) * 17 % 251) as f32 - 125.0) / 128.0)).collect()+}++/// Slots are deliberately non-contiguous and out of order so the scatter is+/// exercised rather than a coincidentally-contiguous copy.+const NODE_INDICES: [u32; ROWS] = [7, 1, 8, 4];++impl<B: Backend> Runner<B> {+ fn new() -> Self {+ let packed_width = 3 * MODEL_DIM;+ let context = create_context::<B>();+ let kernel =+ <B::Kernels as Kernels>::WeaverNodeCacheWriteKernel::new(context.as_ref(), DataType::BF16).unwrap();+ Self {+ current_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(ROWS * packed_width, 29)),+ node_qkv: alloc_allocation_with_data::<B, bf16>(&context, &values(NODES * packed_width, 11)),+ node_indices: alloc_allocation_with_data::<B, u32>(&context, &NODE_INDICES),+ context,+ kernel,+ }+ }++ fn run(&mut self) -> Vec<bf16> {+ let mut encoder = Encoder::new(self.context.as_ref()).unwrap();+ self.kernel.encode(+ &self.current_qkv,+ &mut self.node_qkv,+ &self.node_indices,+ MODEL_DIM as u32,+ (ROWS * 2 * MODEL_DIM) as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ allocation_to_vec::<B, bf16>(&self.node_qkv)+ }+}++#[uzu_test]+fn weaver_node_cache_write_matches_cpu() {+ let packed_width = 3 * MODEL_DIM;++ let mut cpu = Runner::<Cpu>::new();+ let initial_nodes = allocation_to_vec::<Cpu, bf16>(&cpu.node_qkv);+ let current_qkv = allocation_to_vec::<Cpu, bf16>(&cpu.current_qkv);+ let expected = cpu.run();++ // Every written slot takes the K/V halves of its row; the query third and+ // every untouched slot keep their prior contents.+ for node in 0..NODES {+ let base = node * packed_width;+ match NODE_INDICES.iter().position(|&index| index as usize == node) {+ Some(row) => {+ assert_eq!(expected[base..base + MODEL_DIM], initial_nodes[base..base + MODEL_DIM]);+ assert_eq!(+ expected[base + MODEL_DIM..base + packed_width],+ current_qkv[row * packed_width + MODEL_DIM..(row + 1) * packed_width],+ );+ },+ None => assert_eq!(expected[base..base + packed_width], initial_nodes[base..base + packed_width]),+ }+ }++ let mut metal = Runner::<Metal>::new();+ assert_eq!(metal.run(), expected);+}diff --git a/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rsnew file mode 100644index 000000000..047ddceba--- /dev/null+++ b/crates/backend-uzu/tests/unit/backends/common/kernel/weaver/weaver_top_children_test.rs@@ -0,0 +1,68 @@+use half::bf16;+use proc_macros::uzu_test;+use test_runner::for_each_non_cpu_backend;++use crate::{+ backends::{+ common::{Backend, Encoder, Kernels, kernel::WeaverTopChildrenKernel},+ cpu::Cpu,+ },+ tests::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec, create_context},+};++const CANDIDATES: usize = 512;+const CHILDREN: usize = 8;++fn top_children<B: Backend>(+ residual: &[bf16],+ scores: &[f32],+ ids: &[u32],+) -> (Vec<u32>, Vec<f32>) {+ let rows = residual.len() / CANDIDATES;+ assert_eq!(residual.len(), rows * CANDIDATES);+ assert_eq!(scores.len(), rows * CANDIDATES);+ assert_eq!(ids.len(), rows * CANDIDATES);+ let context = create_context::<B>();+ let residual = alloc_allocation_with_data::<B, bf16>(&context, residual);+ let scores = alloc_allocation_with_data::<B, f32>(&context, scores);+ let ids = alloc_allocation_with_data::<B, u32>(&context, ids);+ let mut output_ids = alloc_allocation::<B, u32>(&context, rows * CHILDREN);+ let mut output_logprobs = alloc_allocation::<B, f32>(&context, rows * CHILDREN);+ let kernel = <B::Kernels as Kernels>::WeaverTopChildrenKernel::new(&context).unwrap();+ let mut encoder = Encoder::new(context.as_ref()).unwrap();+ kernel.encode(+ &residual,+ &scores,+ &ids,+ &mut output_ids,+ &mut output_logprobs,+ rows as u32,+ CANDIDATES as u32,+ CHILDREN as u32,+ &mut encoder,+ );+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ (allocation_to_vec(&output_ids), allocation_to_vec(&output_logprobs))+}++#[uzu_test]+fn weaver_top_children_matches_cpu() {+ const ROWS: usize = 3;+ let residual = (0..ROWS * CANDIDATES)+ .map(|index| bf16::from_f32(((index as f32 * 0.017).cos() * 4.0).round() * 0.125))+ .collect::<Vec<_>>();+ let scores =+ (0..ROWS * CANDIDATES).map(|index| ((index as f32 * 0.011).sin() * 3.0).round() * 0.125).collect::<Vec<_>>();+ let ids = (0..ROWS)+ .flat_map(|row| (0..CANDIDATES).rev().map(move |index| 70_000 + (row * CANDIDATES + index) as u32))+ .collect::<Vec<_>>();++ for_each_non_cpu_backend!(|B| {+ let expected = top_children::<Cpu>(&residual, &scores, &ids);+ let actual = top_children::<B>(&residual, &scores, &ids);+ assert_eq!(actual.0, expected.0);+ for (actual, expected) in actual.1.iter().zip(expected.1) {+ assert!((actual - expected).abs() < 1e-5);+ }+ });+}