A high-performance inference engine for AI models
Weaver cleanup (#655)
CCYehc2eb724669a1
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rsindex 627b7602d..c8f94c7f3 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs@@ -15,52 +15,51 @@ fn top_k_score_key(score: f32) -> u32 {#[kernel(WeaverFrontierScatter)]pub fn weaver_frontier_scatter(- tree: *const u32,- round_metadata: *const u32,- round_valid: *const u32,+ packed_tree: *const u32,+ node_metadata: *const u32,+ node_valid: *const u32,child_ids: *const u32,child_logprobs: *const f32,frontier: *mut u32,- capacity: u32,- tree_slots: u32,- rows: u32,+ frontier_capacity: u32,+ tree_slot_count: u32,+ node_count: u32,children_per_node: u32,) {- if capacity == 0 || tree_slots == 0 || children_per_node == 0 {+ if frontier_capacity == 0 || tree_slot_count == 0 || children_per_node == 0 {return;}- let (capacity, tree_slots, rows, children_per_node) =- (capacity as usize, tree_slots as usize, rows as usize, children_per_node as usize);- let tree = unsafe { std::slice::from_raw_parts(tree, TreeIdx::COUNT * tree_slots) };- let parent_indices =- unsafe { std::slice::from_raw_parts(round_metadata.add(MetadataIdx::TreeSlot as usize * rows), rows) };- let round_valid = unsafe { std::slice::from_raw_parts(round_valid, rows) };- let child_ids = unsafe { std::slice::from_raw_parts(child_ids, rows * children_per_node) };- let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, rows * children_per_node) };- let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * capacity) };+ let (frontier_capacity, tree_slot_count, node_count, children_per_node) =+ (frontier_capacity as usize, tree_slot_count as usize, node_count as usize, children_per_node as usize);+ let packed_tree = unsafe { std::slice::from_raw_parts(packed_tree, TreeIdx::COUNT * tree_slot_count) };+ let parent_indices = unsafe {+ std::slice::from_raw_parts(node_metadata.add(MetadataIdx::TreeSlot as usize * node_count), node_count)+ };+ let node_valid = unsafe { std::slice::from_raw_parts(node_valid, node_count) };+ let child_ids = unsafe { std::slice::from_raw_parts(child_ids, node_count * children_per_node) };+ let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, node_count * children_per_node) };+ let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * frontier_capacity) };- for index in 0..rows * children_per_node {+ for index in 0..node_count * children_per_node {let row = index / children_per_node;- if round_valid[row] == 0 {+ if node_valid[row] == 0 {continue;}let parent = parent_indices[row] as usize;let slot = parent * children_per_node + index % children_per_node;- if parent >= tree_slots || slot >= capacity {+ if parent >= tree_slot_count || slot >= frontier_capacity {continue;}let logprob = child_logprobs[index];let cumulative_logprob =- f32::from_bits(tree[TreeIdx::PathLogprobBits as usize * tree_slots + parent]) + logprob;- let mut set = |field: FrontierIdx, value| {- frontier[field as usize * capacity + slot] = value;- };- set(FrontierIdx::TokenId, child_ids[index]);- set(FrontierIdx::ParentSlot, parent as u32);- set(FrontierIdx::Depth, tree[TreeIdx::Depth as usize * tree_slots + parent] + 1);- set(FrontierIdx::PathLogprobBits, cumulative_logprob.to_bits());- set(FrontierIdx::EdgeLogprobBits, logprob.to_bits());- set(FrontierIdx::PathScoreKey, top_k_score_key(cumulative_logprob));- set(FrontierIdx::Active, 1);+ f32::from_bits(packed_tree[TreeIdx::PathLogprobBits as usize * tree_slot_count + parent]) + logprob;+ frontier[FrontierIdx::TokenId as usize * frontier_capacity + slot] = child_ids[index];+ frontier[FrontierIdx::ParentSlot as usize * frontier_capacity + slot] = parent as u32;+ frontier[FrontierIdx::Depth as usize * frontier_capacity + slot] =+ packed_tree[TreeIdx::Depth as usize * tree_slot_count + parent] + 1;+ frontier[FrontierIdx::PathLogprobBits as usize * frontier_capacity + slot] = cumulative_logprob.to_bits();+ frontier[FrontierIdx::EdgeLogprobBits as usize * frontier_capacity + slot] = logprob.to_bits();+ frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot] = top_k_score_key(cumulative_logprob);+ frontier[FrontierIdx::Active as usize * frontier_capacity + slot] = 1;}}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rsindex c90db4461..04db364a3 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs@@ -7,68 +7,69 @@ use crate::backends::common::gpu_types::weaver::{#[kernel(WeaverFrontierSelect)]pub fn weaver_frontier_select(frontier: *mut u32,- tree: *mut u32,+ packed_tree: *mut u32,slot_ancestors: *mut u32,- round_token_ids: *mut u32,- round_metadata: *mut u32,- round_ancestors: *mut u32,- round_valid: *mut u32,+ node_token_ids: *mut u32,+ node_metadata: *mut u32,+ node_ancestor_indices: *mut u32,+ node_valid: *mut u32,candidate_pool_ids: *const u32,- candidate_pool_scores: *const f32,- round_candidate_ids: *mut u32,- round_candidate_scores: *mut f32,- capacity: u32,- tree_slots: u32,- width: u32,- slot_start: u32,+ candidate_pool_logits: *const f32,+ node_candidate_ids: *mut u32,+ node_candidate_logits: *mut f32,+ frontier_capacity: u32,+ tree_slot_count: u32,+ node_count: u32,+ batch_start_slot: u32,ancestor_stride: u32,max_depth: u32,lookahead_count: u32,- candidate_pool_rows: u32,- candidate_pool_size: u32,+ candidate_depth_count: u32,+ candidates_per_depth: u32,) {- if capacity == 0- || capacity as usize > FRONTIER_MAX_SLOTS- || width == 0- || width as usize > FRONTIER_MAX_WIDTH+ if frontier_capacity == 0+ || frontier_capacity as usize > FRONTIER_MAX_SLOTS+ || node_count == 0+ || node_count as usize > FRONTIER_MAX_WIDTH|| ancestor_stride == 0|| max_depth == 0- || tree_slots == 0- || slot_start + width > tree_slots- || candidate_pool_rows == 0- || candidate_pool_size == 0+ || tree_slot_count == 0+ || batch_start_slot + node_count > tree_slot_count+ || candidate_depth_count == 0+ || candidates_per_depth == 0{return;}- let (capacity, slots, width, stride) =- (capacity as usize, tree_slots as usize, width as usize, ancestor_stride as usize);- let (candidate_pool_rows, candidate_pool_size) = (candidate_pool_rows as usize, candidate_pool_size as usize);- let pool_len = candidate_pool_rows * candidate_pool_size;- let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * capacity) };- let tree = unsafe { std::slice::from_raw_parts_mut(tree, TreeIdx::COUNT * slots) };- let slot_ancestors = unsafe { std::slice::from_raw_parts_mut(slot_ancestors, slots * stride) };- let round_token_ids = unsafe { std::slice::from_raw_parts_mut(round_token_ids, width) };- let round_metadata = unsafe { std::slice::from_raw_parts_mut(round_metadata, MetadataIdx::COUNT * width) };- let round_ancestors = unsafe { std::slice::from_raw_parts_mut(round_ancestors, width * stride) };- let round_valid = unsafe { std::slice::from_raw_parts_mut(round_valid, width) };+ let (frontier_capacity, tree_slot_count, node_count, ancestor_stride) =+ (frontier_capacity as usize, tree_slot_count as usize, node_count as usize, ancestor_stride as usize);+ let (candidate_depth_count, candidates_per_depth) = (candidate_depth_count as usize, candidates_per_depth as usize);+ let pool_len = candidate_depth_count * candidates_per_depth;+ let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * frontier_capacity) };+ let packed_tree = unsafe { std::slice::from_raw_parts_mut(packed_tree, TreeIdx::COUNT * tree_slot_count) };+ let slot_ancestors = unsafe { std::slice::from_raw_parts_mut(slot_ancestors, tree_slot_count * ancestor_stride) };+ let node_token_ids = unsafe { std::slice::from_raw_parts_mut(node_token_ids, node_count) };+ let node_metadata = unsafe { std::slice::from_raw_parts_mut(node_metadata, MetadataIdx::COUNT * node_count) };+ let node_ancestor_indices =+ unsafe { std::slice::from_raw_parts_mut(node_ancestor_indices, node_count * ancestor_stride) };+ let node_valid = unsafe { std::slice::from_raw_parts_mut(node_valid, node_count) };let candidate_pool_ids = unsafe { std::slice::from_raw_parts(candidate_pool_ids, pool_len) };- let candidate_pool_scores = unsafe { std::slice::from_raw_parts(candidate_pool_scores, pool_len) };- let round_candidate_ids =- unsafe { std::slice::from_raw_parts_mut(round_candidate_ids, width * candidate_pool_size) };- let round_candidate_scores =- unsafe { std::slice::from_raw_parts_mut(round_candidate_scores, width * candidate_pool_size) };+ let candidate_pool_logits = unsafe { std::slice::from_raw_parts(candidate_pool_logits, pool_len) };+ let node_candidate_ids =+ unsafe { std::slice::from_raw_parts_mut(node_candidate_ids, node_count * candidates_per_depth) };+ let node_candidate_logits =+ unsafe { std::slice::from_raw_parts_mut(node_candidate_logits, node_count * candidates_per_depth) };- for row in 0..width {+ for node in 0..node_count {let (mut key, mut parent, mut token, mut winner) =(0, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);- for slot in 0..capacity {- if frontier[FrontierIdx::Active as usize * capacity + slot] == 0 {+ for slot in 0..frontier_capacity {+ if frontier[FrontierIdx::Active as usize * frontier_capacity + slot] == 0 {continue;}let next = (- frontier[FrontierIdx::PathScoreKey as usize * capacity + slot],- frontier[FrontierIdx::ParentSlot as usize * capacity + slot],- frontier[FrontierIdx::TokenId as usize * capacity + slot],+ frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot],+ frontier[FrontierIdx::ParentSlot as usize * frontier_capacity + slot],+ frontier[FrontierIdx::TokenId as usize * frontier_capacity + slot],);if next.0 > key || (next.0 == key && (next.1 < parent || (next.1 == parent && next.2 < token))) {(key, parent, token, winner) = (next.0, next.1, next.2, slot as u32);@@ -80,57 +81,56 @@ pub fn weaver_frontier_select(} else {0};- let tree_slot = slot_start as usize + row;- let field_value = |field: FrontierIdx| u32::from(real) * frontier[field as usize * capacity + winner];+ let tree_slot = batch_start_slot as usize + node;+ let field_value = |field: FrontierIdx| u32::from(real) * frontier[field as usize * frontier_capacity + winner];let token = field_value(FrontierIdx::TokenId);let depth = field_value(FrontierIdx::Depth);let cumulative_logprob = field_value(FrontierIdx::PathLogprobBits);let logprob = field_value(FrontierIdx::EdgeLogprobBits);- tree[TreeIdx::TokenId as usize * slots + tree_slot] = token;- tree[TreeIdx::ParentSlot as usize * slots + tree_slot] = if real {+ packed_tree[TreeIdx::TokenId as usize * tree_slot_count + tree_slot] = token;+ packed_tree[TreeIdx::ParentSlot as usize * tree_slot_count + tree_slot] = if real {parent} else {FRONTIER_NO_WINNER};- tree[TreeIdx::Depth as usize * slots + tree_slot] = depth;- tree[TreeIdx::PathLogprobBits as usize * slots + tree_slot] = cumulative_logprob;- tree[TreeIdx::EdgeLogprobBits as usize * slots + tree_slot] = logprob;- tree[TreeIdx::Valid as usize * slots + tree_slot] = u32::from(real);+ packed_tree[TreeIdx::Depth as usize * tree_slot_count + tree_slot] = depth;+ packed_tree[TreeIdx::PathLogprobBits as usize * tree_slot_count + tree_slot] = cumulative_logprob;+ packed_tree[TreeIdx::EdgeLogprobBits as usize * tree_slot_count + tree_slot] = logprob;+ packed_tree[TreeIdx::Valid as usize * tree_slot_count + tree_slot] = u32::from(real);if real {- frontier[FrontierIdx::Active as usize * capacity + winner] = 0;+ frontier[FrontierIdx::Active as usize * frontier_capacity + winner] = 0;}- let parent_slot = if real && (parent as usize) < slots {+ let parent_slot = if real && (parent as usize) < tree_slot_count {parent as usize} else {0};- for index in 0..stride {+ for index in 0..ancestor_stride {let ancestor = if real && index + 1 < depth as usize {- slot_ancestors[parent_slot * stride + index]+ slot_ancestors[parent_slot * ancestor_stride + index]} else if real && index + 1 == depth as usize {parent_slot as u32} else {0};- slot_ancestors[tree_slot * stride + index] = ancestor;- round_ancestors[row * stride + index] = ancestor;+ slot_ancestors[tree_slot * ancestor_stride + index] = ancestor;+ node_ancestor_indices[node * ancestor_stride + index] = ancestor;}- round_token_ids[row] = token;- round_metadata[MetadataIdx::Depth as usize * width + row] = depth.min(max_depth - 1);- round_metadata[MetadataIdx::AncestorCount as usize * width + row] = depth.min(stride as u32);- round_metadata[MetadataIdx::TreeSlot as usize * width + row] = tree_slot as u32;- round_valid[row] = u32::from(real && depth < lookahead_count && depth < max_depth);+ node_token_ids[node] = token;+ node_metadata[MetadataIdx::Depth as usize * node_count + node] = depth.min(max_depth - 1);+ node_metadata[MetadataIdx::AncestorCount as usize * node_count + node] = depth.min(ancestor_stride as u32);+ node_metadata[MetadataIdx::TreeSlot as usize * node_count + node] = tree_slot as u32;+ node_valid[node] = u32::from(real && depth < lookahead_count && depth < max_depth);- // Every row of a round expands the pool for its own depth, so the winner's pool row is- // copied into the contiguous per-round buffer the Weaver step reads.- let candidate_pool_row = (depth as usize).min(candidate_pool_rows - 1);- let source = candidate_pool_row * candidate_pool_size..(candidate_pool_row + 1) * candidate_pool_size;- let destination = row * candidate_pool_size..(row + 1) * candidate_pool_size;- round_candidate_ids[destination.clone()].copy_from_slice(&candidate_pool_ids[source.clone()]);- round_candidate_scores[destination].copy_from_slice(&candidate_pool_scores[source]);+ // Every node expands the candidate pool for its own depth.+ let candidate_depth = (depth as usize).min(candidate_depth_count - 1);+ let source = candidate_depth * candidates_per_depth..(candidate_depth + 1) * candidates_per_depth;+ let destination = node * candidates_per_depth..(node + 1) * candidates_per_depth;+ node_candidate_ids[destination.clone()].copy_from_slice(&candidate_pool_ids[source.clone()]);+ node_candidate_logits[destination].copy_from_slice(&candidate_pool_logits[source]);}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metalindex d5b1ad4da..ee2f95c33 100644--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal@@ -6,44 +6,46 @@using namespace metal;PUBLIC KERNEL(WeaverFrontierScatter)(- const device uint* tree,- const device uint* round_metadata,- const device uint* round_valid,+ const device uint* packed_tree,+ const device uint* node_metadata,+ const device uint* node_valid,const device uint* child_ids,const device float* child_logprobs,device uint* frontier,- constant uint& capacity,- constant uint& tree_slots,- constant uint& rows,+ constant uint& frontier_capacity,+ constant uint& tree_slot_count,+ constant uint& node_count,constant uint& children_per_node,- const uint position AXIS(rows* children_per_node, 64)+ const uint position AXIS(node_count* children_per_node, 64)) {- if (capacity == 0 || tree_slots == 0 || children_per_node == 0) {+ if (frontier_capacity == 0 || tree_slot_count == 0 || children_per_node == 0) {return;}const uint row = position / children_per_node, child = position % children_per_node;- if (round_valid[row] == 0u) {+ if (node_valid[row] == 0u) {return;}- const uint parent = round_metadata[uint(MetadataIdx::TreeSlot) * rows + row];- if (parent >= tree_slots) {+ const uint parent = node_metadata[uint(MetadataIdx::TreeSlot) * node_count + row];+ if (parent >= tree_slot_count) {return;}const uint slot = parent * children_per_node + child;- if (slot >= capacity) {+ if (slot >= frontier_capacity) {return;}const float logprob = child_logprobs[row * children_per_node + child];- const float cumulative_logprob = as_type<float>(tree[uint(TreeIdx::PathLogprobBits) * tree_slots + parent]) + logprob;+ const float cumulative_logprob =+ as_type<float>(packed_tree[uint(TreeIdx::PathLogprobBits) * tree_slot_count + parent]) + logprob;- frontier[uint(FrontierIdx::TokenId) * capacity + slot] = child_ids[row * children_per_node + child];- frontier[uint(FrontierIdx::ParentSlot) * capacity + slot] = parent;- frontier[uint(FrontierIdx::Depth) * capacity + slot] = tree[uint(TreeIdx::Depth) * tree_slots + parent] + 1u;- frontier[uint(FrontierIdx::PathLogprobBits) * capacity + slot] = as_type<uint>(cumulative_logprob);- frontier[uint(FrontierIdx::EdgeLogprobBits) * capacity + slot] = as_type<uint>(logprob);- frontier[uint(FrontierIdx::PathScoreKey) * capacity + slot] = top_k_score_key(cumulative_logprob);- frontier[uint(FrontierIdx::Active) * capacity + slot] = 1u;+ frontier[uint(FrontierIdx::TokenId) * frontier_capacity + slot] = child_ids[row * children_per_node + child];+ frontier[uint(FrontierIdx::ParentSlot) * frontier_capacity + slot] = parent;+ frontier[uint(FrontierIdx::Depth) * frontier_capacity + slot] =+ packed_tree[uint(TreeIdx::Depth) * tree_slot_count + parent] + 1u;+ frontier[uint(FrontierIdx::PathLogprobBits) * frontier_capacity + slot] = as_type<uint>(cumulative_logprob);+ frontier[uint(FrontierIdx::EdgeLogprobBits) * frontier_capacity + slot] = as_type<uint>(logprob);+ frontier[uint(FrontierIdx::PathScoreKey) * frontier_capacity + slot] = top_k_score_key(cumulative_logprob);+ frontier[uint(FrontierIdx::Active) * frontier_capacity + slot] = 1u;}diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metalindex 0ced94673..7f57fbfee 100644--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal@@ -8,53 +8,54 @@ using namespace metal;PUBLIC KERNEL(WeaverFrontierSelect)(device uint* frontier,- device uint* tree,+ device uint* packed_tree,device uint* slot_ancestors,- device uint* round_token_ids,- device uint* round_metadata,- device uint* round_ancestors,- device uint* round_valid,+ device uint* node_token_ids,+ device uint* node_metadata,+ device uint* node_ancestor_indices,+ device uint* node_valid,const device uint* candidate_pool_ids,- const device float* candidate_pool_scores,- device uint* round_candidate_ids,- device float* round_candidate_scores,- constant uint& capacity,- constant uint& tree_slots,- constant uint& width,- constant uint& slot_start,+ const device float* candidate_pool_logits,+ device uint* node_candidate_ids,+ device float* node_candidate_logits,+ constant uint& frontier_capacity,+ constant uint& tree_slot_count,+ constant uint& node_count,+ constant uint& batch_start_slot,constant uint& ancestor_stride,constant uint& max_depth,constant uint& lookahead_count,- constant uint& candidate_pool_rows,- constant uint& candidate_pool_size,+ constant uint& candidate_depth_count,+ constant uint& candidates_per_depth,threadgroup uint4 reduce[FRONTIER_SELECT_SIMDGROUPS],threadgroup uint winner_slot[FRONTIER_MAX_WIDTH],- threadgroup uint row_candidate_pool_row[FRONTIER_MAX_WIDTH],+ threadgroup uint node_candidate_depth[FRONTIER_MAX_WIDTH],const ThreadContext thread_context,const uint group_index GROUPS(1),const uint lid THREADS(FRONTIER_SELECT_THREADS)) {(void)group_index;- if (capacity == 0 || capacity > FRONTIER_MAX_SLOTS || width == 0 || width > FRONTIER_MAX_WIDTH ||- ancestor_stride == 0 || max_depth == 0 || tree_slots == 0 || slot_start + width > tree_slots ||- candidate_pool_rows == 0 || candidate_pool_size == 0) {+ if (frontier_capacity == 0 || frontier_capacity > FRONTIER_MAX_SLOTS || node_count == 0 ||+ node_count > FRONTIER_MAX_WIDTH || ancestor_stride == 0 || max_depth == 0 || tree_slot_count == 0 ||+ batch_start_slot + node_count > tree_slot_count || candidate_depth_count == 0 || candidates_per_depth == 0) {return;}bool entry_active[FRONTIER_ENTRIES_PER_THREAD];for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {const uint slot = lid + entry * FRONTIER_SELECT_THREADS;- entry_active[entry] = slot < capacity && frontier[uint(FrontierIdx::Active) * capacity + slot] != 0u;+ entry_active[entry] =+ slot < frontier_capacity && frontier[uint(FrontierIdx::Active) * frontier_capacity + slot] != 0u;}- for (uint child = 0; child < width; ++child) {+ for (uint child = 0; child < node_count; ++child) {uint4 local = uint4(0u, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {const uint slot = lid + entry * FRONTIER_SELECT_THREADS;if (entry_active[entry]) {- const uint key = frontier[uint(FrontierIdx::PathScoreKey) * capacity + slot];- const uint parent = frontier[uint(FrontierIdx::ParentSlot) * capacity + slot];- const uint token = frontier[uint(FrontierIdx::TokenId) * capacity + slot];+ const uint key = frontier[uint(FrontierIdx::PathScoreKey) * frontier_capacity + slot];+ const uint parent = frontier[uint(FrontierIdx::ParentSlot) * frontier_capacity + slot];+ const uint token = frontier[uint(FrontierIdx::TokenId) * frontier_capacity + slot];if (key > local.x || (key == local.x && (parent < local.y || (parent == local.y && token < local.z)))) {local = uint4(key, parent, token, slot);}@@ -92,56 +93,56 @@ PUBLIC KERNEL(WeaverFrontierSelect)(}}- if (lid < width) {- const uint row = lid;- const uint slot = winner_slot[row];+ if (lid < node_count) {+ const uint node = lid;+ const uint slot = winner_slot[node];const bool real = slot != FRONTIER_NO_WINNER;- const uint tree_slot = slot_start + row;+ const uint tree_slot = batch_start_slot + node;- const uint token = real ? frontier[uint(FrontierIdx::TokenId) * capacity + slot] : 0u;- const uint parent = real ? frontier[uint(FrontierIdx::ParentSlot) * capacity + slot] : FRONTIER_NO_WINNER;- const uint depth = real ? frontier[uint(FrontierIdx::Depth) * capacity + slot] : 0u;- const uint cumulative_logprob = real ? frontier[uint(FrontierIdx::PathLogprobBits) * capacity + slot] : 0u;- const uint logprob = real ? frontier[uint(FrontierIdx::EdgeLogprobBits) * capacity + slot] : 0u;+ const uint token = real ? frontier[uint(FrontierIdx::TokenId) * frontier_capacity + slot] : 0u;+ const uint parent = real ? frontier[uint(FrontierIdx::ParentSlot) * frontier_capacity + slot] : FRONTIER_NO_WINNER;+ const uint depth = real ? frontier[uint(FrontierIdx::Depth) * frontier_capacity + slot] : 0u;+ const uint cumulative_logprob = real ? frontier[uint(FrontierIdx::PathLogprobBits) * frontier_capacity + slot] : 0u;+ const uint logprob = real ? frontier[uint(FrontierIdx::EdgeLogprobBits) * frontier_capacity + slot] : 0u;- tree[uint(TreeIdx::TokenId) * tree_slots + tree_slot] = token;- tree[uint(TreeIdx::ParentSlot) * tree_slots + tree_slot] = parent;- tree[uint(TreeIdx::Depth) * tree_slots + tree_slot] = depth;- tree[uint(TreeIdx::PathLogprobBits) * tree_slots + tree_slot] = cumulative_logprob;- tree[uint(TreeIdx::EdgeLogprobBits) * tree_slots + tree_slot] = logprob;- tree[uint(TreeIdx::Valid) * tree_slots + tree_slot] = real ? 1u : 0u;+ packed_tree[uint(TreeIdx::TokenId) * tree_slot_count + tree_slot] = token;+ packed_tree[uint(TreeIdx::ParentSlot) * tree_slot_count + tree_slot] = parent;+ packed_tree[uint(TreeIdx::Depth) * tree_slot_count + tree_slot] = depth;+ packed_tree[uint(TreeIdx::PathLogprobBits) * tree_slot_count + tree_slot] = cumulative_logprob;+ packed_tree[uint(TreeIdx::EdgeLogprobBits) * tree_slot_count + tree_slot] = logprob;+ packed_tree[uint(TreeIdx::Valid) * tree_slot_count + tree_slot] = real ? 1u : 0u;if (real) {- frontier[uint(FrontierIdx::Active) * capacity + slot] = 0u;+ frontier[uint(FrontierIdx::Active) * frontier_capacity + slot] = 0u;}- const uint parent_slot = real && parent < tree_slots ? parent : 0u;+ const uint parent_slot = real && parent < tree_slot_count ? parent : 0u;for (uint index = 0; index < ancestor_stride; ++index) {const uint ancestor =real && index + 1u <= depth? (index + 1u == depth ? parent_slot : slot_ancestors[parent_slot * ancestor_stride + index]): 0u;slot_ancestors[tree_slot * ancestor_stride + index] = ancestor;- round_ancestors[row * ancestor_stride + index] = ancestor;+ node_ancestor_indices[node * ancestor_stride + index] = ancestor;}- round_token_ids[row] = token;- round_metadata[uint(MetadataIdx::Depth) * width + row] = min(depth, max_depth - 1u);- round_metadata[uint(MetadataIdx::AncestorCount) * width + row] = min(depth, ancestor_stride);- round_metadata[uint(MetadataIdx::TreeSlot) * width + row] = tree_slot;- round_valid[row] = real && depth < lookahead_count && depth < max_depth ? 1u : 0u;+ node_token_ids[node] = token;+ node_metadata[uint(MetadataIdx::Depth) * node_count + node] = min(depth, max_depth - 1u);+ node_metadata[uint(MetadataIdx::AncestorCount) * node_count + node] = min(depth, ancestor_stride);+ node_metadata[uint(MetadataIdx::TreeSlot) * node_count + node] = tree_slot;+ node_valid[node] = real && depth < lookahead_count && depth < max_depth ? 1u : 0u;- row_candidate_pool_row[row] = min(depth, candidate_pool_rows - 1u);+ node_candidate_depth[node] = min(depth, candidate_depth_count - 1u);}threadgroup_barrier(mem_flags::mem_threadgroup);- for (uint row = 0; row < width; ++row) {- const uint source = row_candidate_pool_row[row] * candidate_pool_size;- const uint destination = row * candidate_pool_size;- for (uint candidate = lid; candidate < candidate_pool_size; candidate += FRONTIER_SELECT_THREADS) {- round_candidate_ids[destination + candidate] = candidate_pool_ids[source + candidate];- round_candidate_scores[destination + candidate] = candidate_pool_scores[source + candidate];+ for (uint node = 0; node < node_count; ++node) {+ const uint source = node_candidate_depth[node] * candidates_per_depth;+ const uint destination = node * candidates_per_depth;+ for (uint candidate = lid; candidate < candidates_per_depth; candidate += FRONTIER_SELECT_THREADS) {+ node_candidate_ids[destination + candidate] = candidate_pool_ids[source + candidate];+ node_candidate_logits[destination + candidate] = candidate_pool_logits[source + candidate];}}}diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rsindex 344a1838c..4ee117304 100644--- a/crates/backend-uzu/src/encodable_block/weaver.rs+++ b/crates/backend-uzu/src/encodable_block/weaver.rs@@ -34,75 +34,85 @@ struct TopKChildren<B: Backend> {logprobs: Allocation<B>,}-struct PrefixOutput<B: Backend> {- hidden: Allocation<B>,+struct PrefixLayerOutput<B: Backend> {+ mlp_delta: Allocation<B>,kv_cache: Allocation<B>,}-struct PrefixAttentionInputs<B: Backend> {+struct PreparedPrefixAttention<B: Backend> {queries: Allocation<B>,kv_cache: Allocation<B>,}-struct WeaverPrefixKvCache<B: Backend> {- layer_kv: Box<[Allocation<B>]>,- token_count: usize,+struct PrefixKvCache<B: Backend> {+ layers: Box<[Allocation<B>]>,+ length: usize,}-struct WeaverNodeKvCache<B: Backend> {- layer_kv: Box<[Allocation<B>]>,- node_capacity: u32,+struct NodeExpansionKvCache<B: Backend> {+ layers: Box<[Allocation<B>]>,+ capacity: u32,}-struct WeaverStepBatch<'a, B: Backend> {- node_count: usize,- candidates_per_node: usize,- ancestor_stride: usize,- node_token_ids: &'a Allocation<B>,- candidate_ids: &'a Allocation<B>,- candidate_logits: &'a Allocation<B>,+struct NodeBatch<'a, B: Backend> {+ count: usize,+ token_ids: &'a Allocation<B>,+ metadata: &'a Allocation<B>,ancestor_indices: &'a Allocation<B>,- node_metadata: &'a Allocation<B>,+ ancestor_stride: usize,+}++struct CandidateBatch<'a, B: Backend> {+ ids: &'a Allocation<B>,+ logits: &'a Allocation<B>,+ candidates_per_node: usize,depth_seeds: &'a Allocation<B>,}-pub(crate) struct WeaverTreeInput<'a, B: Backend> {- pub(crate) target_hidden: &'a Allocation<B>,- pub(crate) draft_hidden: &'a Allocation<B>,- pub(crate) target_embedding: &'a Embedding<B>,- pub(crate) candidate_ids: &'a Allocation<B>,- pub(crate) candidate_scores: &'a Allocation<B>,- pub(crate) candidate_rows: usize,- pub(crate) candidates_per_row: usize,- pub(crate) depth_seeds: &'a [u64],- pub(crate) root_token_id: u32,- pub(crate) tree_budget: usize,- pub(crate) frontier_width: usize,- pub(crate) children_per_node: usize,+pub struct CandidatePool<'a, B: Backend> {+ pub ids: &'a Allocation<B>,+ pub logits: &'a Allocation<B>,+ pub depth_count: usize,+ pub candidates_per_depth: usize,}-pub(crate) struct EncodedWeaverTree<B: Backend> {- slots: Allocation<B>,+pub struct WeaverInputs<'a, B: Backend> {+ pub target_hidden: &'a Allocation<B>,+ pub draft_hidden: &'a Allocation<B>,+ pub target_embedding: &'a Embedding<B>,+ pub candidates: CandidatePool<'a, B>,+ pub depth_seeds: &'a [u64],+ pub root_token_id: u32,}-pub(crate) struct ProposalNode {- pub(crate) token: u32,- pub(crate) depth: usize,- pub(crate) children: Vec<usize>,+pub struct TreeShape {+ pub budget: usize,+ pub frontier_width: usize,+ pub children_per_node: usize,+}++pub struct EncodedWeaverTree<B: Backend> {+ packed_tree: Allocation<B>,+}++pub struct ProposalNode {+ pub token_id: u32,+ pub depth: usize,+ pub child_indices: Vec<usize>,}impl<B: Backend> EncodedWeaverTree<B> {- pub(crate) fn decode(self) -> Vec<ProposalNode> {- decode_tree_slots(&self.slots.copyout::<u32>())+ pub fn read_nodes(self) -> Vec<ProposalNode> {+ nodes_from_packed_tree(&self.packed_tree.copyout::<u32>())}}-pub(crate) struct Weaver<B: Backend> {+pub struct Weaver<B: Backend> {token_embedding_norm: Normalization<B>,token_embedding_projection: Box<dyn Linear<B>>,hidden_state_norm: Normalization<B>,hidden_state_projection: Box<dyn Linear<B>>,- blocks: Box<[WeaverBlock<B>]>,+ layers: Box<[WeaverLayer<B>]>,readout_norm: Normalization<B>,readout_query_projection: Box<dyn Linear<B>>,position_embeddings: Allocation<B>,@@ -116,7 +126,7 @@ pub(crate) struct Weaver<B: Backend> {max_depth: usize,}-struct WeaverBlock<B: Backend> {+struct WeaverLayer<B: Backend> {// Attentionpre_attention_norm: Normalization<B>,qkv_projection: Box<dyn Linear<B>>,@@ -169,7 +179,7 @@ pub enum WeaverEncodeError<B: Backend> {}impl<B: Backend> Weaver<B> {- pub(crate) fn new(+ pub fn new(context: &B::Context,config: &WeaverConfig,parameter_tree: &ParameterTree<B>,@@ -211,17 +221,17 @@ impl<B: Backend> Weaver<B> {DATA_TYPE,¶meter_tree.subtree("embedding_projection")?,)?;- let blocks_tree = parameter_tree.subtree("blocks")?;- let blocks = (0..config.num_layers)+ let layer_parameters = parameter_tree.subtree("blocks")?;+ let layers = (0..config.num_layers).map(|index| {- WeaverBlock::new(+ WeaverLayer::new(context,config.model_dim,config.hidden_dim,config.num_heads,index > 0,&config.norm_config,- &blocks_tree.subtree(&index.to_string())?,+ &layer_parameters.subtree(&index.to_string())?,)}).collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;@@ -272,7 +282,7 @@ impl<B: Backend> Weaver<B> {token_embedding_projection,hidden_state_norm,hidden_state_projection,- blocks,+ layers,readout_norm,readout_query_projection,position_embeddings,@@ -287,311 +297,317 @@ impl<B: Backend> Weaver<B> {})}- pub(crate) fn encode_tree(+ pub fn encode_tree(&self,- input: WeaverTreeInput<'_, B>,+ inputs: WeaverInputs<'_, B>,+ shape: TreeShape,context: &B::Context,encoder: &mut Encoder<B>,) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {- let slot_count = input.tree_budget + 1;- let frontier_capacity = slot_count * input.children_per_node;- let lookahead_count = self.max_depth.min(input.candidate_rows);- if input.tree_budget == 0- || input.frontier_width == 0- || input.children_per_node == 0- || input.children_per_node > input.candidates_per_row+ let tree_slot_count = shape.budget + 1;+ let frontier_capacity = tree_slot_count * shape.children_per_node;+ let lookahead_count = self.max_depth.min(inputs.candidates.depth_count);+ let ancestor_stride = self.max_depth;+ if shape.budget == 0+ || shape.frontier_width == 0+ || shape.children_per_node == 0+ || shape.children_per_node > inputs.candidates.candidates_per_depth|| frontier_capacity > FRONTIER_MAX_SLOTS|| lookahead_count == 0- || input.depth_seeds.len() != self.max_depth+ || inputs.depth_seeds.len() != self.max_depth{return Err(WeaverEncodeError::InvalidTreeInput);}- let prefix = self.build_prefix(input.target_hidden, input.draft_hidden, lookahead_count, encoder)?;- let mut node_cache = self.create_node_kv_cache(slot_count, context)?;+ let prefix_cache = self+ .encode_prefix(inputs.target_hidden, inputs.draft_hidden, lookahead_count, encoder)+ .map_err(WeaverEncodeError::Backend)?;+ let mut node_cache =+ self.create_node_expansion_kv_cache(tree_slot_count, context).map_err(WeaverEncodeError::Backend)?;- let mut tree_values = vec![0u32; TreeIdx::COUNT * slot_count];- for slot in 0..slot_count {- tree_values[TreeIdx::ParentSlot as usize * slot_count + slot] = FRONTIER_NO_WINNER;+ let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slot_count];+ for slot in 0..tree_slot_count {+ tree_init[TreeIdx::ParentSlot as usize * tree_slot_count + slot] = FRONTIER_NO_WINNER;}- tree_values[TreeIdx::TokenId as usize * slot_count] = input.root_token_id;- tree_values[TreeIdx::Valid as usize * slot_count] = 1;+ tree_init[TreeIdx::TokenId as usize * tree_slot_count] = inputs.root_token_id;+ tree_init[TreeIdx::Valid as usize * tree_slot_count] = 1;- let mut tree = encoder.allocate_constant_from_slice(&tree_values).map_err(WeaverEncodeError::Backend)?;+ let mut packed_tree = encoder.allocate_constant_from_slice(&tree_init).map_err(WeaverEncodeError::Backend)?;let mut frontier = encoder.allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * frontier_capacity]).map_err(WeaverEncodeError::Backend)?;let mut slot_ancestors = encoder- .allocate_constant_from_slice(&vec![0u32; slot_count * self.max_depth])+ .allocate_constant_from_slice(&vec![0u32; tree_slot_count * ancestor_stride]).map_err(WeaverEncodeError::Backend)?;- let mut round_token_id_values = vec![0u32; input.frontier_width];- round_token_id_values[0] = input.root_token_id;- let mut round_valid_values = vec![0u32; input.frontier_width];- round_valid_values[0] = 1;- let mut round_token_ids =- encoder.allocate_constant_from_slice(&round_token_id_values).map_err(WeaverEncodeError::Backend)?;- let mut round_metadata = encoder- .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * input.frontier_width])+ let mut initial_node_token_ids = vec![0u32; shape.frontier_width];+ initial_node_token_ids[0] = inputs.root_token_id;+ let mut initial_node_valid = vec![0u32; shape.frontier_width];+ initial_node_valid[0] = 1;+ let mut node_token_ids =+ encoder.allocate_constant_from_slice(&initial_node_token_ids).map_err(WeaverEncodeError::Backend)?;+ let mut node_metadata = encoder+ .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * shape.frontier_width]).map_err(WeaverEncodeError::Backend)?;- let mut round_ancestors = encoder- .allocate_constant_from_slice(&vec![0u32; input.frontier_width * self.max_depth])+ let mut node_ancestor_indices = encoder+ .allocate_constant_from_slice(&vec![0u32; shape.frontier_width * ancestor_stride]).map_err(WeaverEncodeError::Backend)?;- let mut round_valid =- encoder.allocate_constant_from_slice(&round_valid_values).map_err(WeaverEncodeError::Backend)?;- let mut round_candidate_ids = encoder- .allocate_constant_from_slice(&vec![0u32; input.frontier_width * input.candidates_per_row])+ let mut node_valid =+ encoder.allocate_constant_from_slice(&initial_node_valid).map_err(WeaverEncodeError::Backend)?;+ let mut node_candidate_ids = encoder+ .allocate_constant_from_slice(&vec![0u32; shape.frontier_width * inputs.candidates.candidates_per_depth]).map_err(WeaverEncodeError::Backend)?;- let mut round_candidate_scores = encoder- .allocate_constant_from_slice(&vec![0.0f32; input.frontier_width * input.candidates_per_row])+ let mut node_candidate_logits = encoder+ .allocate_constant_from_slice(&vec![0.0f32; shape.frontier_width * inputs.candidates.candidates_per_depth]).map_err(WeaverEncodeError::Backend)?;let depth_seeds =- encoder.allocate_constant_from_slice(input.depth_seeds).map_err(WeaverEncodeError::Backend)?;+ encoder.allocate_constant_from_slice(inputs.depth_seeds).map_err(WeaverEncodeError::Backend)?;- let mut slot_start = 0;- while slot_start < slot_count {- let node_count = if slot_start == 0 {+ let mut batch_start_slot = 0;+ while batch_start_slot < tree_slot_count {+ let batch_node_count = if batch_start_slot == 0 {1} else {- input.frontier_width.min(slot_count - slot_start)+ shape.frontier_width.min(tree_slot_count - batch_start_slot)};- if slot_start > 0 {+ if batch_start_slot > 0 {self.frontier_select.encode(&mut frontier,- &mut tree,+ &mut packed_tree,&mut slot_ancestors,- &mut round_token_ids,- &mut round_metadata,- &mut round_ancestors,- &mut round_valid,- input.candidate_ids,- input.candidate_scores,- &mut round_candidate_ids,- &mut round_candidate_scores,+ &mut node_token_ids,+ &mut node_metadata,+ &mut node_ancestor_indices,+ &mut node_valid,+ inputs.candidates.ids,+ inputs.candidates.logits,+ &mut node_candidate_ids,+ &mut node_candidate_logits,frontier_capacity as u32,- slot_count as u32,- node_count as u32,- slot_start as u32,- self.max_depth as u32,+ tree_slot_count as u32,+ batch_node_count as u32,+ batch_start_slot as u32,+ ancestor_stride as u32,self.max_depth as u32,lookahead_count as u32,- input.candidate_rows as u32,- input.candidates_per_row as u32,+ inputs.candidates.depth_count as u32,+ inputs.candidates.candidates_per_depth as u32,encoder,);}- let (candidate_ids, candidate_scores) = if slot_start == 0 {- (input.candidate_ids, input.candidate_scores)+ let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {+ (inputs.candidates.ids, inputs.candidates.logits)} else {- (&round_candidate_ids, &round_candidate_scores)+ (&node_candidate_ids, &node_candidate_logits)};- let batch = WeaverStepBatch {- node_count,- candidates_per_node: input.candidates_per_row,- ancestor_stride: self.max_depth,- node_token_ids: &round_token_ids,- candidate_ids,- candidate_logits: candidate_scores,- ancestor_indices: &round_ancestors,- node_metadata: &round_metadata,+ let nodes = NodeBatch {+ count: batch_node_count,+ token_ids: &node_token_ids,+ metadata: &node_metadata,+ ancestor_indices: &node_ancestor_indices,+ ancestor_stride,+ };+ let candidates = CandidateBatch {+ ids: batch_candidate_ids,+ logits: batch_candidate_logits,+ candidates_per_node: inputs.candidates.candidates_per_depth,depth_seeds: &depth_seeds,};- let children = self.encode_step_batch(- &prefix,- &batch,+ let selected_children = self.encode_node_expansion(+ &prefix_cache,+ &nodes,+ &candidates,&mut node_cache,- input.children_per_node,- input.target_embedding,+ shape.children_per_node,+ inputs.target_embedding,encoder,)?;self.frontier_scatter.encode(- &tree,- &round_metadata,- &round_valid,- &children.token_ids,- &children.logprobs,+ &packed_tree,+ &node_metadata,+ &node_valid,+ &selected_children.token_ids,+ &selected_children.logprobs,&mut frontier,frontier_capacity as u32,- slot_count as u32,- node_count as u32,- input.children_per_node as u32,+ tree_slot_count as u32,+ batch_node_count as u32,+ shape.children_per_node as u32,encoder,);- slot_start += node_count;+ batch_start_slot += batch_node_count;}Ok(EncodedWeaverTree {- slots: tree,+ packed_tree,})}- fn build_prefix(+ fn encode_prefix(&self,target_hidden: &Allocation<B>,draft_hidden: &Allocation<B>,lookahead_count: usize,encoder: &mut Encoder<B>,- ) -> Result<WeaverPrefixKvCache<B>, WeaverEncodeError<B>> {+ ) -> Result<PrefixKvCache<B>, B::Error> {assert!((1..=self.max_depth).contains(&lookahead_count));let token_count = lookahead_count + 1;let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();- let mut prefix_input = encoder- .allocate_scratch(size_for_shape(&[token_count, self.target_model_dim], DATA_TYPE))- .map_err(WeaverEncodeError::Backend)?;- encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_input, 0..hidden_row_bytes);+ let mut prefix_hidden =+ encoder.allocate_scratch(size_for_shape(&[token_count, self.target_model_dim], DATA_TYPE))?;+ encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);encoder.encode_copy(draft_hidden,hidden_row_bytes..token_count * hidden_row_bytes,- &mut prefix_input,+ &mut prefix_hidden,hidden_row_bytes..token_count * hidden_row_bytes,);- let normalized_prefix = self- .hidden_state_norm- .encode(&prefix_input, 0, token_count, None, encoder)- .map_err(WeaverEncodeError::Backend)?;- let mut hidden = self- .hidden_state_projection- .encode(normalized_prefix, token_count, encoder)- .map_err(WeaverEncodeError::Backend)?;+ let normalized_prefix = self.hidden_state_norm.encode(&prefix_hidden, 0, token_count, None, encoder)?;+ let mut residual_input = self.hidden_state_projection.encode(normalized_prefix, token_count, encoder)?;let position_elements = lookahead_count * self.model_dim;self.prefix_position_add.encode(None::<&Allocation<B>>,&self.position_embeddings,None::<&Allocation<B>>,- (&mut hidden, self.model_dim * DATA_TYPE.size_in_bytes()),+ (&mut residual_input, self.model_dim * DATA_TYPE.size_in_bytes()),position_elements as u32,position_elements as u32,encoder,);- let (last_block, preceding_blocks) = self.blocks.split_last().expect("Weaver must have at least one block");- let mut residual = encoder.allocate_scratch(hidden.size()).map_err(WeaverEncodeError::Backend)?;- let mut layer_kv = Vec::with_capacity(self.blocks.len());- for block in preceding_blocks {- let PrefixOutput {- hidden: next_hidden,+ let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");+ let mut residual_state = encoder.allocate_scratch(residual_input.size())?;+ let mut prefix_layers = Vec::with_capacity(self.layers.len());+ for layer in preceding_layers {+ let PrefixLayerOutput {+ mlp_delta,kv_cache,- } = block.encode_prefix(hidden, &mut residual, token_count, encoder).map_err(WeaverEncodeError::Backend)?;- layer_kv.push(kv_cache);- hidden = next_hidden;+ } = layer.encode_prefix_tokens(residual_input, &mut residual_state, token_count, encoder)?;+ prefix_layers.push(kv_cache);+ residual_input = mlp_delta;}- layer_kv.push(- last_block- .prepare_prefix_attention(&hidden, &mut residual, token_count, encoder)- .map_err(WeaverEncodeError::Backend)?- .kv_cache,+ prefix_layers.push(+ last_layer.encode_prefix_attention(&residual_input, &mut residual_state, token_count, encoder)?.kv_cache,);- Ok(WeaverPrefixKvCache {- layer_kv: layer_kv.into_boxed_slice(),- token_count,+ Ok(PrefixKvCache {+ layers: prefix_layers.into_boxed_slice(),+ length: token_count,})}- fn create_node_kv_cache(+ fn create_node_expansion_kv_cache(&self,capacity: usize,context: &B::Context,- ) -> Result<WeaverNodeKvCache<B>, WeaverEncodeError<B>> {+ ) -> Result<NodeExpansionKvCache<B>, B::Error> {assert!(capacity > 0, "Weaver node capacity must be positive");- let node_capacity = u32::try_from(capacity).expect("Weaver node capacity exceeds the kernel limit");let kv_size = size_for_shape(&[2, capacity, self.model_dim], DATA_TYPE);- let layer_kv = (0..self.blocks.len())- .map(|_| context.create_allocation(kv_size, AllocationType::Global).map_err(WeaverEncodeError::Backend))+ let layers = (0..self.layers.len())+ .map(|_| context.create_allocation(kv_size, AllocationType::Global)).collect::<Result<Box<[_]>, _>>()?;- Ok(WeaverNodeKvCache {- layer_kv,- node_capacity,+ Ok(NodeExpansionKvCache {+ layers,+ capacity: capacity as u32,})}- fn encode_step_batch(+ fn encode_node_expansion(&self,- prefix: &WeaverPrefixKvCache<B>,- batch: &WeaverStepBatch<'_, B>,- node_cache: &mut WeaverNodeKvCache<B>,+ prefix_cache: &PrefixKvCache<B>,+ nodes: &NodeBatch<'_, B>,+ candidates: &CandidateBatch<'_, B>,+ node_cache: &mut NodeExpansionKvCache<B>,children_per_node: usize,target_embedding: &Embedding<B>,encoder: &mut Encoder<B>,) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {- let token_embedding = target_embedding.encode_lookup(batch.node_token_ids, batch.node_count, encoder)?;+ let token_embedding = target_embedding.encode_lookup(nodes.token_ids, nodes.count, encoder)?;let normalized_embedding = self.token_embedding_norm- .encode(&token_embedding, 0, batch.node_count, None, encoder)+ .encode(&token_embedding, 0, nodes.count, None, encoder).map_err(WeaverEncodeError::Backend)?;- let mut hidden = self+ let mut residual_input = self.token_embedding_projection- .encode(normalized_embedding, batch.node_count, encoder)+ .encode(normalized_embedding, nodes.count, encoder).map_err(WeaverEncodeError::Backend)?;self.node_position_add.encode(None::<&Allocation<B>>,&self.position_embeddings,- Some(batch.node_metadata),- &mut hidden,+ Some(nodes.metadata),+ &mut residual_input,self.model_dim as u32,- (batch.node_count * self.model_dim) as u32,+ (nodes.count * self.model_dim) as u32,encoder,);- let mut residual = encoder.allocate_scratch(hidden.size()).map_err(WeaverEncodeError::Backend)?;- let node_capacity = node_cache.node_capacity;- for (layer_index, block) in self.blocks.iter().enumerate() {- hidden = block- .encode_step(- hidden,- &mut residual,- &prefix.layer_kv[layer_index],- &mut node_cache.layer_kv[layer_index],+ let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;+ let node_capacity = node_cache.capacity;+ for (layer_index, layer) in self.layers.iter().enumerate() {+ residual_input = layer+ .encode_node_batch(+ residual_input,+ &mut residual_state,+ &prefix_cache.layers[layer_index],+ &mut node_cache.layers[layer_index],node_capacity,- batch,- prefix.token_count,+ nodes,+ prefix_cache.length,encoder,).map_err(WeaverEncodeError::Backend)?;}- self.encode_step_output(&hidden, &mut residual, batch, children_per_node, target_embedding, encoder)+ self.encode_child_selection(+ &residual_input,+ &mut residual_state,+ nodes,+ candidates,+ children_per_node,+ target_embedding,+ encoder,+ )}- /// Outputs must outlive the batch.- fn encode_step_output(+ fn encode_child_selection(&self,- hidden: &Allocation<B>,- residual: &mut Allocation<B>,- batch: &WeaverStepBatch<'_, B>,+ residual_output: &Allocation<B>,+ residual_state: &mut Allocation<B>,+ nodes: &NodeBatch<'_, B>,+ candidates: &CandidateBatch<'_, B>,children_per_node: usize,target_embedding: &Embedding<B>,encoder: &mut Encoder<B>,) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {let normalized_output = self.readout_norm- .encode(hidden, 0, batch.node_count, Some(residual), encoder)+ .encode(residual_output, 0, nodes.count, Some(residual_state), encoder).map_err(WeaverEncodeError::Backend)?;let query = self.readout_query_projection- .encode(normalized_output, batch.node_count, encoder)+ .encode(normalized_output, nodes.count, encoder).map_err(WeaverEncodeError::Backend)?;- let residual_logits = target_embedding.encode_readout_sparse(+ let logit_residuals = target_embedding.encode_readout_sparse(&query,- batch.candidate_ids,- batch.node_count,- batch.candidates_per_node,+ candidates.ids,+ nodes.count,+ candidates.candidates_per_node,encoder,)?;let mut token_ids = encoder- .allocate_scratch(size_for_shape(&[batch.node_count, children_per_node], DataType::U32))+ .allocate_scratch(size_for_shape(&[nodes.count, children_per_node], DataType::U32)).map_err(WeaverEncodeError::Backend)?;let mut logprobs = encoder- .allocate_scratch(size_for_shape(&[batch.node_count, children_per_node], DataType::F32))+ .allocate_scratch(size_for_shape(&[nodes.count, children_per_node], DataType::F32)).map_err(WeaverEncodeError::Backend)?;self.top_children.encode(- &residual_logits,- batch.candidate_logits,- batch.candidate_ids,- batch.depth_seeds,- batch.node_metadata,+ &logit_residuals,+ candidates.logits,+ candidates.ids,+ candidates.depth_seeds,+ nodes.metadata,&mut token_ids,&mut logprobs,- batch.node_count as u32,- batch.candidates_per_node as u32,+ nodes.count as u32,+ candidates.candidates_per_node as u32,children_per_node as u32,target_embedding.vocab_size() as u32,encoder,@@ -603,7 +619,7 @@ impl<B: Backend> Weaver<B> {}}-impl<B: Backend> WeaverBlock<B> {+impl<B: Backend> WeaverLayer<B> {fn new(context: &B::Context,model_dim: usize,@@ -717,22 +733,22 @@ impl<B: Backend> WeaverBlock<B> {})}- fn encode_prefix(+ fn encode_prefix_tokens(&self,- input: Allocation<B>,- residual: &mut Allocation<B>,+ residual_input: Allocation<B>,+ residual_state: &mut Allocation<B>,token_count: usize,encoder: &mut Encoder<B>,- ) -> Result<PrefixOutput<B>, B::Error> {- let PrefixAttentionInputs {+ ) -> Result<PrefixLayerOutput<B>, B::Error> {+ let PreparedPrefixAttention {queries,kv_cache,- } = self.prepare_prefix_attention(&input, residual, token_count, encoder)?;+ } = self.encode_prefix_attention(&residual_input, residual_state, token_count, encoder)?;let state_type = AttentionStateType::Full {length: 0,};let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);- let attention = self.prefix_attention.encode(+ let attention_output = self.prefix_attention.encode(AttentionCoreEncodeArguments {queries: &queries,keys: &kv_cache,@@ -744,29 +760,30 @@ impl<B: Backend> WeaverBlock<B> {},encoder,)?;- let hidden = self.encode_post_attention(attention, residual, token_count, encoder)?;- Ok(PrefixOutput {- hidden,+ let mlp_delta = self.encode_post_attention(attention_output, residual_state, token_count, encoder)?;+ Ok(PrefixLayerOutput {+ mlp_delta,kv_cache,})}- fn prepare_prefix_attention(+ fn encode_prefix_attention(&self,- input: &Allocation<B>,- residual: &mut Allocation<B>,+ residual_input: &Allocation<B>,+ residual_state: &mut Allocation<B>,token_count: usize,encoder: &mut Encoder<B>,- ) -> Result<PrefixAttentionInputs<B>, B::Error> {- let attention_input = self.pre_attention_norm.encode(input, 0, token_count, Some(residual), encoder)?;+ ) -> Result<PreparedPrefixAttention<B>, B::Error> {+ let attention_input =+ self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), encoder)?;let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;let mut queries =encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);- let mut kv = encoder+ let mut kv_cache = encoder.context().create_allocation(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE), AllocationType::Global)?;- let (keys, values) = kv.as_buffer_range_mut().split_at(kv_plane_bytes);+ let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);self.attention_prepare.encode(&qkv,&mut queries,@@ -782,39 +799,41 @@ impl<B: Backend> WeaverBlock<B> {token_count as u32,encoder,);- Ok(PrefixAttentionInputs {+ Ok(PreparedPrefixAttention {queries,- kv_cache: kv,+ kv_cache,})}- fn encode_step(+ fn encode_node_batch(&self,- input: Allocation<B>,- residual: &mut Allocation<B>,+ residual_input: Allocation<B>,+ residual_state: &mut Allocation<B>,prefix_kv: &Allocation<B>,node_kv_cache: &mut Allocation<B>,node_capacity: u32,- batch: &WeaverStepBatch<'_, B>,+ nodes: &NodeBatch<'_, B>,prefix_length: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let attention_input = self.pre_attention_norm.encode(&input, 0, batch.node_count, Some(residual), encoder)?;- let current_qkv = self.qkv_projection.encode(attention_input, batch.node_count, encoder)?;- let metadata_lane_bytes = batch.node_count * DataType::U32.size_in_bytes();- let ancestor_counts = (batch.node_metadata, MetadataIdx::AncestorCount as usize * metadata_lane_bytes);- let tree_slots = (batch.node_metadata, MetadataIdx::TreeSlot as usize * metadata_lane_bytes);- let mut attention = encoder.allocate_scratch(size_for_shape(&[batch.node_count, self.model_dim], DATA_TYPE))?;+ let attention_input =+ self.pre_attention_norm.encode(&residual_input, 0, nodes.count, Some(residual_state), encoder)?;+ let current_qkv = self.qkv_projection.encode(attention_input, nodes.count, encoder)?;+ let metadata_field_bytes = nodes.count * DataType::U32.size_in_bytes();+ let ancestor_counts = (nodes.metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes);+ let tree_slot_indices = (nodes.metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes);+ let mut attention_output =+ encoder.allocate_scratch(size_for_shape(&[nodes.count, self.model_dim], DATA_TYPE))?;self.ancestor_attention.encode(prefix_kv,&*node_kv_cache,¤t_qkv,- batch.ancestor_indices,+ nodes.ancestor_indices,ancestor_counts,- &mut attention,- batch.node_count as u32,+ &mut attention_output,+ nodes.count as u32,prefix_length as u32,- batch.ancestor_stride as u32,+ nodes.ancestor_stride as u32,node_capacity,self.attention_scale,encoder,@@ -824,61 +843,61 @@ impl<B: Backend> WeaverBlock<B> {self.node_cache_write.encode(¤t_qkv,node_kv_cache,- tree_slots,+ tree_slot_indices,self.model_dim as u32,node_capacity,- (batch.node_count * 2 * self.model_dim) as u32,+ (nodes.count * 2 * self.model_dim) as u32,encoder,);- self.encode_post_attention(attention, residual, batch.node_count, encoder)+ self.encode_post_attention(attention_output, residual_state, nodes.count, encoder)}fn encode_post_attention(&self,- attention: Allocation<B>,- residual: &mut Allocation<B>,- row_count: usize,+ attention_output: Allocation<B>,+ residual_state: &mut Allocation<B>,+ token_count: usize,encoder: &mut Encoder<B>,) -> Result<Allocation<B>, B::Error> {- let projected_attention = self.out_projection.encode(attention, row_count, encoder)?;- let mlp_input = self.pre_mlp_norm.encode(&projected_attention, 0, row_count, Some(residual), encoder)?;- let mut mlp_hidden = self.up_projection.encode(mlp_input, row_count, encoder)?;+ let projected_attention = self.out_projection.encode(attention_output, token_count, encoder)?;+ let mlp_input =+ self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), encoder)?;+ let mut mlp_hidden = self.up_projection.encode(mlp_input, token_count, encoder)?;self.activation.encode(None::<&Allocation<B>>,&mut mlp_hidden,- (row_count * self.hidden_dim) as u32,+ (token_count * self.hidden_dim) as u32,crate::backends::common::gpu_types::ActivationType::GELUExact,encoder,);- self.down_projection.encode(mlp_hidden, row_count, encoder)+ self.down_projection.encode(mlp_hidden, token_count, encoder)}}-fn decode_tree_slots(tree: &[u32]) -> Vec<ProposalNode> {- assert!(tree.len().is_multiple_of(TreeIdx::COUNT), "tree array must contain {} equal-length lanes", TreeIdx::COUNT);- let slot_count = tree.len() / TreeIdx::COUNT;- let field = |field: TreeIdx, slot: usize| tree[field as usize * slot_count + slot];- let mut slot_to_node = vec![usize::MAX; slot_count];- let mut nodes: Vec<ProposalNode> = Vec::with_capacity(slot_count);- for slot in 0..slot_count {+fn nodes_from_packed_tree(packed_tree: &[u32]) -> Vec<ProposalNode> {+ let tree_slot_count = packed_tree.len() / TreeIdx::COUNT;+ let field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];+ let mut slot_to_node_index = vec![usize::MAX; tree_slot_count];+ let mut nodes: Vec<ProposalNode> = Vec::with_capacity(tree_slot_count);+ for slot in 0..tree_slot_count {if field(TreeIdx::Valid, slot) == 0 {continue;}let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;let parent = (parent_slot >= 0).then(|| {- let parent = slot_to_node[parent_slot as usize];+ let parent = slot_to_node_index[parent_slot as usize];assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");parent});let index = nodes.len();- slot_to_node[slot] = index;+ slot_to_node_index[slot] = index;if let Some(parent) = parent {- nodes[parent].children.push(index);+ nodes[parent].child_indices.push(index);}nodes.push(ProposalNode {- token: field(TreeIdx::TokenId, slot),+ token_id: field(TreeIdx::TokenId, slot),depth: field(TreeIdx::Depth, slot) as usize,- children: Vec::new(),+ child_indices: Vec::new(),});}nodesdiff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rsindex bbee7a1f1..3f9a9d8a3 100644--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs@@ -18,7 +18,7 @@ use crate::{dflash::{DFlashDraft, DFlashDraftEncodeError, DFlashDraftNewError, DFlashDraftOutput},embedding::Embedding,sampling::PRng,- weaver::{ProposalNode, Weaver, WeaverEncodeError, WeaverNewError, WeaverTreeInput},+ weaver::{CandidatePool, ProposalNode, TreeShape, Weaver, WeaverEncodeError, WeaverInputs, WeaverNewError},},parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},trie::TrieNode,@@ -168,17 +168,21 @@ impl<B: Backend> DFlashSpeculator<B> {let depth_seeds =(0..max_depth).map(|depth| prng.derive((root_position + depth) as u64)).collect::<Box<[u64]>>();let tree = weaver.encode_tree(- WeaverTreeInput {+ WeaverInputs {target_hidden: target_output_norm,draft_hidden: &draft_hidden,target_embedding,- candidate_ids: &candidates.ids,- candidate_scores: &candidates.scores,- candidate_rows: candidates.rows,- candidates_per_row: candidates.candidates_per_row,+ candidates: CandidatePool {+ ids: &candidates.ids,+ logits: &candidates.scores,+ depth_count: candidates.rows,+ candidates_per_depth: candidates.candidates_per_row,+ },depth_seeds: &depth_seeds,root_token_id: target_output_token,- tree_budget: options.budget,+ },+ TreeShape {+ budget: options.budget,frontier_width: options.frontier_width,children_per_node: options.children_per_node,},@@ -186,7 +190,7 @@ impl<B: Backend> DFlashSpeculator<B> {&mut encoder,)?;let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;- let nodes = tree.decode();+ let nodes = tree.read_nodes();drop(candidates);drop(draft_hidden);drop(completed);@@ -208,20 +212,20 @@ impl<B: Backend> DFlashSpeculator<B> {drop(completed);let mut nodes = vec![ProposalNode {- token: target_output_token,+ token_id: target_output_token,depth: 0,- children: Vec::new(),+ child_indices: Vec::new(),}];for depth in 0..options.budget.min(candidate_rows) {let token = candidate_id_values[depth * candidates_per_row];let parent = nodes.len() - 1;let index = nodes.len();nodes.push(ProposalNode {- token,+ token_id: token,depth: depth + 1,- children: Vec::new(),+ child_indices: Vec::new(),});- nodes[parent].children.push(index);+ nodes[parent].child_indices.push(index);}Ok(Self::finish_tree(nodes,@@ -246,11 +250,11 @@ impl<B: Backend> DFlashSpeculator<B> {#[cfg(grammar)] grammar: &mut Option<&mut Grammar>,) -> TrieNode {let node = &nodes[index];- let mut trie_node = TrieNode::new(node.token as u64, prng.derive((root_position + node.depth) as u64));- for &child_index in &node.children {+ let mut trie_node = TrieNode::new(node.token_id as u64, prng.derive((root_position + node.depth) as u64));+ for &child_index in &node.child_indices {#[cfg(grammar)]if let Some(grammar) = grammar.as_mut()- && grammar.accept_token(nodes[child_index].token as u64).is_err()+ && grammar.accept_token(nodes[child_index].token_id as u64).is_err(){continue;}