Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Weaver cleanup (#655)

CCYehc2eb724669a1
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs
index 627b7602d..c8f94c7f3 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_scatter.rs
@@ -15,52 +15,51 @@ fn top_k_score_key(score: f32) -> u32 {
#[kernel(WeaverFrontierScatter)]
pub fn weaver_frontier_scatter(
- tree: *const u32,
- round_metadata: *const u32,
- round_valid: *const u32,
+ packed_tree: *const u32,
+ node_metadata: *const u32,
+ node_valid: *const u32,
child_ids: *const u32,
child_logprobs: *const f32,
frontier: *mut u32,
- capacity: u32,
- tree_slots: u32,
- rows: u32,
+ frontier_capacity: u32,
+ tree_slot_count: u32,
+ node_count: u32,
children_per_node: u32,
) {
- if capacity == 0 || tree_slots == 0 || children_per_node == 0 {
+ if frontier_capacity == 0 || tree_slot_count == 0 || children_per_node == 0 {
return;
}
- let (capacity, tree_slots, rows, children_per_node) =
- (capacity as usize, tree_slots as usize, rows as usize, children_per_node as usize);
- let tree = unsafe { std::slice::from_raw_parts(tree, TreeIdx::COUNT * tree_slots) };
- let parent_indices =
- unsafe { std::slice::from_raw_parts(round_metadata.add(MetadataIdx::TreeSlot as usize * rows), rows) };
- let round_valid = unsafe { std::slice::from_raw_parts(round_valid, rows) };
- let child_ids = unsafe { std::slice::from_raw_parts(child_ids, rows * children_per_node) };
- let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, rows * children_per_node) };
- let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * capacity) };
+ let (frontier_capacity, tree_slot_count, node_count, children_per_node) =
+ (frontier_capacity as usize, tree_slot_count as usize, node_count as usize, children_per_node as usize);
+ let packed_tree = unsafe { std::slice::from_raw_parts(packed_tree, TreeIdx::COUNT * tree_slot_count) };
+ let parent_indices = unsafe {
+ std::slice::from_raw_parts(node_metadata.add(MetadataIdx::TreeSlot as usize * node_count), node_count)
+ };
+ let node_valid = unsafe { std::slice::from_raw_parts(node_valid, node_count) };
+ let child_ids = unsafe { std::slice::from_raw_parts(child_ids, node_count * children_per_node) };
+ let child_logprobs = unsafe { std::slice::from_raw_parts(child_logprobs, node_count * children_per_node) };
+ let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * frontier_capacity) };
- for index in 0..rows * children_per_node {
+ for index in 0..node_count * children_per_node {
let row = index / children_per_node;
- if round_valid[row] == 0 {
+ if node_valid[row] == 0 {
continue;
}
let parent = parent_indices[row] as usize;
let slot = parent * children_per_node + index % children_per_node;
- if parent >= tree_slots || slot >= capacity {
+ if parent >= tree_slot_count || slot >= frontier_capacity {
continue;
}
let logprob = child_logprobs[index];
let cumulative_logprob =
- f32::from_bits(tree[TreeIdx::PathLogprobBits as usize * tree_slots + parent]) + logprob;
- let mut set = |field: FrontierIdx, value| {
- frontier[field as usize * capacity + slot] = value;
- };
- set(FrontierIdx::TokenId, child_ids[index]);
- set(FrontierIdx::ParentSlot, parent as u32);
- set(FrontierIdx::Depth, tree[TreeIdx::Depth as usize * tree_slots + parent] + 1);
- set(FrontierIdx::PathLogprobBits, cumulative_logprob.to_bits());
- set(FrontierIdx::EdgeLogprobBits, logprob.to_bits());
- set(FrontierIdx::PathScoreKey, top_k_score_key(cumulative_logprob));
- set(FrontierIdx::Active, 1);
+ f32::from_bits(packed_tree[TreeIdx::PathLogprobBits as usize * tree_slot_count + parent]) + logprob;
+ frontier[FrontierIdx::TokenId as usize * frontier_capacity + slot] = child_ids[index];
+ frontier[FrontierIdx::ParentSlot as usize * frontier_capacity + slot] = parent as u32;
+ frontier[FrontierIdx::Depth as usize * frontier_capacity + slot] =
+ packed_tree[TreeIdx::Depth as usize * tree_slot_count + parent] + 1;
+ frontier[FrontierIdx::PathLogprobBits as usize * frontier_capacity + slot] = cumulative_logprob.to_bits();
+ frontier[FrontierIdx::EdgeLogprobBits as usize * frontier_capacity + slot] = logprob.to_bits();
+ frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot] = top_k_score_key(cumulative_logprob);
+ frontier[FrontierIdx::Active as usize * frontier_capacity + slot] = 1;
}
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
index c90db4461..04db364a3 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/weaver/weaver_frontier_select.rs
@@ -7,68 +7,69 @@ use crate::backends::common::gpu_types::weaver::{
#[kernel(WeaverFrontierSelect)]
pub fn weaver_frontier_select(
frontier: *mut u32,
- tree: *mut u32,
+ packed_tree: *mut u32,
slot_ancestors: *mut u32,
- round_token_ids: *mut u32,
- round_metadata: *mut u32,
- round_ancestors: *mut u32,
- round_valid: *mut u32,
+ node_token_ids: *mut u32,
+ node_metadata: *mut u32,
+ node_ancestor_indices: *mut u32,
+ node_valid: *mut u32,
candidate_pool_ids: *const u32,
- candidate_pool_scores: *const f32,
- round_candidate_ids: *mut u32,
- round_candidate_scores: *mut f32,
- capacity: u32,
- tree_slots: u32,
- width: u32,
- slot_start: u32,
+ candidate_pool_logits: *const f32,
+ node_candidate_ids: *mut u32,
+ node_candidate_logits: *mut f32,
+ frontier_capacity: u32,
+ tree_slot_count: u32,
+ node_count: u32,
+ batch_start_slot: u32,
ancestor_stride: u32,
max_depth: u32,
lookahead_count: u32,
- candidate_pool_rows: u32,
- candidate_pool_size: u32,
+ candidate_depth_count: u32,
+ candidates_per_depth: u32,
) {
- if capacity == 0
- || capacity as usize > FRONTIER_MAX_SLOTS
- || width == 0
- || width as usize > FRONTIER_MAX_WIDTH
+ if frontier_capacity == 0
+ || frontier_capacity as usize > FRONTIER_MAX_SLOTS
+ || node_count == 0
+ || node_count as usize > FRONTIER_MAX_WIDTH
|| ancestor_stride == 0
|| max_depth == 0
- || tree_slots == 0
- || slot_start + width > tree_slots
- || candidate_pool_rows == 0
- || candidate_pool_size == 0
+ || tree_slot_count == 0
+ || batch_start_slot + node_count > tree_slot_count
+ || candidate_depth_count == 0
+ || candidates_per_depth == 0
{
return;
}
- let (capacity, slots, width, stride) =
- (capacity as usize, tree_slots as usize, width as usize, ancestor_stride as usize);
- let (candidate_pool_rows, candidate_pool_size) = (candidate_pool_rows as usize, candidate_pool_size as usize);
- let pool_len = candidate_pool_rows * candidate_pool_size;
- let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * capacity) };
- let tree = unsafe { std::slice::from_raw_parts_mut(tree, TreeIdx::COUNT * slots) };
- let slot_ancestors = unsafe { std::slice::from_raw_parts_mut(slot_ancestors, slots * stride) };
- let round_token_ids = unsafe { std::slice::from_raw_parts_mut(round_token_ids, width) };
- let round_metadata = unsafe { std::slice::from_raw_parts_mut(round_metadata, MetadataIdx::COUNT * width) };
- let round_ancestors = unsafe { std::slice::from_raw_parts_mut(round_ancestors, width * stride) };
- let round_valid = unsafe { std::slice::from_raw_parts_mut(round_valid, width) };
+ let (frontier_capacity, tree_slot_count, node_count, ancestor_stride) =
+ (frontier_capacity as usize, tree_slot_count as usize, node_count as usize, ancestor_stride as usize);
+ let (candidate_depth_count, candidates_per_depth) = (candidate_depth_count as usize, candidates_per_depth as usize);
+ let pool_len = candidate_depth_count * candidates_per_depth;
+ let frontier = unsafe { std::slice::from_raw_parts_mut(frontier, FrontierIdx::COUNT * frontier_capacity) };
+ let packed_tree = unsafe { std::slice::from_raw_parts_mut(packed_tree, TreeIdx::COUNT * tree_slot_count) };
+ let slot_ancestors = unsafe { std::slice::from_raw_parts_mut(slot_ancestors, tree_slot_count * ancestor_stride) };
+ let node_token_ids = unsafe { std::slice::from_raw_parts_mut(node_token_ids, node_count) };
+ let node_metadata = unsafe { std::slice::from_raw_parts_mut(node_metadata, MetadataIdx::COUNT * node_count) };
+ let node_ancestor_indices =
+ unsafe { std::slice::from_raw_parts_mut(node_ancestor_indices, node_count * ancestor_stride) };
+ let node_valid = unsafe { std::slice::from_raw_parts_mut(node_valid, node_count) };
let candidate_pool_ids = unsafe { std::slice::from_raw_parts(candidate_pool_ids, pool_len) };
- let candidate_pool_scores = unsafe { std::slice::from_raw_parts(candidate_pool_scores, pool_len) };
- let round_candidate_ids =
- unsafe { std::slice::from_raw_parts_mut(round_candidate_ids, width * candidate_pool_size) };
- let round_candidate_scores =
- unsafe { std::slice::from_raw_parts_mut(round_candidate_scores, width * candidate_pool_size) };
+ let candidate_pool_logits = unsafe { std::slice::from_raw_parts(candidate_pool_logits, pool_len) };
+ let node_candidate_ids =
+ unsafe { std::slice::from_raw_parts_mut(node_candidate_ids, node_count * candidates_per_depth) };
+ let node_candidate_logits =
+ unsafe { std::slice::from_raw_parts_mut(node_candidate_logits, node_count * candidates_per_depth) };
- for row in 0..width {
+ for node in 0..node_count {
let (mut key, mut parent, mut token, mut winner) =
(0, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);
- for slot in 0..capacity {
- if frontier[FrontierIdx::Active as usize * capacity + slot] == 0 {
+ for slot in 0..frontier_capacity {
+ if frontier[FrontierIdx::Active as usize * frontier_capacity + slot] == 0 {
continue;
}
let next = (
- frontier[FrontierIdx::PathScoreKey as usize * capacity + slot],
- frontier[FrontierIdx::ParentSlot as usize * capacity + slot],
- frontier[FrontierIdx::TokenId as usize * capacity + slot],
+ frontier[FrontierIdx::PathScoreKey as usize * frontier_capacity + slot],
+ frontier[FrontierIdx::ParentSlot as usize * frontier_capacity + slot],
+ frontier[FrontierIdx::TokenId as usize * frontier_capacity + slot],
);
if next.0 > key || (next.0 == key && (next.1 < parent || (next.1 == parent && next.2 < token))) {
(key, parent, token, winner) = (next.0, next.1, next.2, slot as u32);
@@ -80,57 +81,56 @@ pub fn weaver_frontier_select(
} else {
0
};
- let tree_slot = slot_start as usize + row;
- let field_value = |field: FrontierIdx| u32::from(real) * frontier[field as usize * capacity + winner];
+ let tree_slot = batch_start_slot as usize + node;
+ let field_value = |field: FrontierIdx| u32::from(real) * frontier[field as usize * frontier_capacity + winner];
let token = field_value(FrontierIdx::TokenId);
let depth = field_value(FrontierIdx::Depth);
let cumulative_logprob = field_value(FrontierIdx::PathLogprobBits);
let logprob = field_value(FrontierIdx::EdgeLogprobBits);
- tree[TreeIdx::TokenId as usize * slots + tree_slot] = token;
- tree[TreeIdx::ParentSlot as usize * slots + tree_slot] = if real {
+ packed_tree[TreeIdx::TokenId as usize * tree_slot_count + tree_slot] = token;
+ packed_tree[TreeIdx::ParentSlot as usize * tree_slot_count + tree_slot] = if real {
parent
} else {
FRONTIER_NO_WINNER
};
- tree[TreeIdx::Depth as usize * slots + tree_slot] = depth;
- tree[TreeIdx::PathLogprobBits as usize * slots + tree_slot] = cumulative_logprob;
- tree[TreeIdx::EdgeLogprobBits as usize * slots + tree_slot] = logprob;
- tree[TreeIdx::Valid as usize * slots + tree_slot] = u32::from(real);
+ packed_tree[TreeIdx::Depth as usize * tree_slot_count + tree_slot] = depth;
+ packed_tree[TreeIdx::PathLogprobBits as usize * tree_slot_count + tree_slot] = cumulative_logprob;
+ packed_tree[TreeIdx::EdgeLogprobBits as usize * tree_slot_count + tree_slot] = logprob;
+ packed_tree[TreeIdx::Valid as usize * tree_slot_count + tree_slot] = u32::from(real);
if real {
- frontier[FrontierIdx::Active as usize * capacity + winner] = 0;
+ frontier[FrontierIdx::Active as usize * frontier_capacity + winner] = 0;
}
- let parent_slot = if real && (parent as usize) < slots {
+ let parent_slot = if real && (parent as usize) < tree_slot_count {
parent as usize
} else {
0
};
- for index in 0..stride {
+ for index in 0..ancestor_stride {
let ancestor = if real && index + 1 < depth as usize {
- slot_ancestors[parent_slot * stride + index]
+ slot_ancestors[parent_slot * ancestor_stride + index]
} else if real && index + 1 == depth as usize {
parent_slot as u32
} else {
0
};
- slot_ancestors[tree_slot * stride + index] = ancestor;
- round_ancestors[row * stride + index] = ancestor;
+ slot_ancestors[tree_slot * ancestor_stride + index] = ancestor;
+ node_ancestor_indices[node * ancestor_stride + index] = ancestor;
}
- round_token_ids[row] = token;
- round_metadata[MetadataIdx::Depth as usize * width + row] = depth.min(max_depth - 1);
- round_metadata[MetadataIdx::AncestorCount as usize * width + row] = depth.min(stride as u32);
- round_metadata[MetadataIdx::TreeSlot as usize * width + row] = tree_slot as u32;
- round_valid[row] = u32::from(real && depth < lookahead_count && depth < max_depth);
+ node_token_ids[node] = token;
+ node_metadata[MetadataIdx::Depth as usize * node_count + node] = depth.min(max_depth - 1);
+ node_metadata[MetadataIdx::AncestorCount as usize * node_count + node] = depth.min(ancestor_stride as u32);
+ node_metadata[MetadataIdx::TreeSlot as usize * node_count + node] = tree_slot as u32;
+ node_valid[node] = u32::from(real && depth < lookahead_count && depth < max_depth);
- // Every row of a round expands the pool for its own depth, so the winner's pool row is
- // copied into the contiguous per-round buffer the Weaver step reads.
- let candidate_pool_row = (depth as usize).min(candidate_pool_rows - 1);
- let source = candidate_pool_row * candidate_pool_size..(candidate_pool_row + 1) * candidate_pool_size;
- let destination = row * candidate_pool_size..(row + 1) * candidate_pool_size;
- round_candidate_ids[destination.clone()].copy_from_slice(&candidate_pool_ids[source.clone()]);
- round_candidate_scores[destination].copy_from_slice(&candidate_pool_scores[source]);
+ // Every node expands the candidate pool for its own depth.
+ let candidate_depth = (depth as usize).min(candidate_depth_count - 1);
+ let source = candidate_depth * candidates_per_depth..(candidate_depth + 1) * candidates_per_depth;
+ let destination = node * candidates_per_depth..(node + 1) * candidates_per_depth;
+ node_candidate_ids[destination.clone()].copy_from_slice(&candidate_pool_ids[source.clone()]);
+ node_candidate_logits[destination].copy_from_slice(&candidate_pool_logits[source]);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal
index d5b1ad4da..ee2f95c33 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_scatter.metal
@@ -6,44 +6,46 @@
using namespace metal;
PUBLIC KERNEL(WeaverFrontierScatter)(
- const device uint* tree,
- const device uint* round_metadata,
- const device uint* round_valid,
+ const device uint* packed_tree,
+ const device uint* node_metadata,
+ const device uint* node_valid,
const device uint* child_ids,
const device float* child_logprobs,
device uint* frontier,
- constant uint& capacity,
- constant uint& tree_slots,
- constant uint& rows,
+ constant uint& frontier_capacity,
+ constant uint& tree_slot_count,
+ constant uint& node_count,
constant uint& children_per_node,
- const uint position AXIS(rows* children_per_node, 64)
+ const uint position AXIS(node_count* children_per_node, 64)
) {
- if (capacity == 0 || tree_slots == 0 || children_per_node == 0) {
+ if (frontier_capacity == 0 || tree_slot_count == 0 || children_per_node == 0) {
return;
}
const uint row = position / children_per_node, child = position % children_per_node;
- if (round_valid[row] == 0u) {
+ if (node_valid[row] == 0u) {
return;
}
- const uint parent = round_metadata[uint(MetadataIdx::TreeSlot) * rows + row];
- if (parent >= tree_slots) {
+ const uint parent = node_metadata[uint(MetadataIdx::TreeSlot) * node_count + row];
+ if (parent >= tree_slot_count) {
return;
}
const uint slot = parent * children_per_node + child;
- if (slot >= capacity) {
+ if (slot >= frontier_capacity) {
return;
}
const float logprob = child_logprobs[row * children_per_node + child];
- const float cumulative_logprob = as_type<float>(tree[uint(TreeIdx::PathLogprobBits) * tree_slots + parent]) + logprob;
+ const float cumulative_logprob =
+ as_type<float>(packed_tree[uint(TreeIdx::PathLogprobBits) * tree_slot_count + parent]) + logprob;
- frontier[uint(FrontierIdx::TokenId) * capacity + slot] = child_ids[row * children_per_node + child];
- frontier[uint(FrontierIdx::ParentSlot) * capacity + slot] = parent;
- frontier[uint(FrontierIdx::Depth) * capacity + slot] = tree[uint(TreeIdx::Depth) * tree_slots + parent] + 1u;
- frontier[uint(FrontierIdx::PathLogprobBits) * capacity + slot] = as_type<uint>(cumulative_logprob);
- frontier[uint(FrontierIdx::EdgeLogprobBits) * capacity + slot] = as_type<uint>(logprob);
- frontier[uint(FrontierIdx::PathScoreKey) * capacity + slot] = top_k_score_key(cumulative_logprob);
- frontier[uint(FrontierIdx::Active) * capacity + slot] = 1u;
+ frontier[uint(FrontierIdx::TokenId) * frontier_capacity + slot] = child_ids[row * children_per_node + child];
+ frontier[uint(FrontierIdx::ParentSlot) * frontier_capacity + slot] = parent;
+ frontier[uint(FrontierIdx::Depth) * frontier_capacity + slot] =
+ packed_tree[uint(TreeIdx::Depth) * tree_slot_count + parent] + 1u;
+ frontier[uint(FrontierIdx::PathLogprobBits) * frontier_capacity + slot] = as_type<uint>(cumulative_logprob);
+ frontier[uint(FrontierIdx::EdgeLogprobBits) * frontier_capacity + slot] = as_type<uint>(logprob);
+ frontier[uint(FrontierIdx::PathScoreKey) * frontier_capacity + slot] = top_k_score_key(cumulative_logprob);
+ frontier[uint(FrontierIdx::Active) * frontier_capacity + slot] = 1u;
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
index 0ced94673..7f57fbfee 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/weaver/weaver_frontier_select.metal
@@ -8,53 +8,54 @@ using namespace metal;
PUBLIC KERNEL(WeaverFrontierSelect)(
device uint* frontier,
- device uint* tree,
+ device uint* packed_tree,
device uint* slot_ancestors,
- device uint* round_token_ids,
- device uint* round_metadata,
- device uint* round_ancestors,
- device uint* round_valid,
+ device uint* node_token_ids,
+ device uint* node_metadata,
+ device uint* node_ancestor_indices,
+ device uint* node_valid,
const device uint* candidate_pool_ids,
- const device float* candidate_pool_scores,
- device uint* round_candidate_ids,
- device float* round_candidate_scores,
- constant uint& capacity,
- constant uint& tree_slots,
- constant uint& width,
- constant uint& slot_start,
+ const device float* candidate_pool_logits,
+ device uint* node_candidate_ids,
+ device float* node_candidate_logits,
+ constant uint& frontier_capacity,
+ constant uint& tree_slot_count,
+ constant uint& node_count,
+ constant uint& batch_start_slot,
constant uint& ancestor_stride,
constant uint& max_depth,
constant uint& lookahead_count,
- constant uint& candidate_pool_rows,
- constant uint& candidate_pool_size,
+ constant uint& candidate_depth_count,
+ constant uint& candidates_per_depth,
threadgroup uint4 reduce[FRONTIER_SELECT_SIMDGROUPS],
threadgroup uint winner_slot[FRONTIER_MAX_WIDTH],
- threadgroup uint row_candidate_pool_row[FRONTIER_MAX_WIDTH],
+ threadgroup uint node_candidate_depth[FRONTIER_MAX_WIDTH],
const ThreadContext thread_context,
const uint group_index GROUPS(1),
const uint lid THREADS(FRONTIER_SELECT_THREADS)
) {
(void)group_index;
- if (capacity == 0 || capacity > FRONTIER_MAX_SLOTS || width == 0 || width > FRONTIER_MAX_WIDTH ||
- ancestor_stride == 0 || max_depth == 0 || tree_slots == 0 || slot_start + width > tree_slots ||
- candidate_pool_rows == 0 || candidate_pool_size == 0) {
+ if (frontier_capacity == 0 || frontier_capacity > FRONTIER_MAX_SLOTS || node_count == 0 ||
+ node_count > FRONTIER_MAX_WIDTH || ancestor_stride == 0 || max_depth == 0 || tree_slot_count == 0 ||
+ batch_start_slot + node_count > tree_slot_count || candidate_depth_count == 0 || candidates_per_depth == 0) {
return;
}
bool entry_active[FRONTIER_ENTRIES_PER_THREAD];
for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {
const uint slot = lid + entry * FRONTIER_SELECT_THREADS;
- entry_active[entry] = slot < capacity && frontier[uint(FrontierIdx::Active) * capacity + slot] != 0u;
+ entry_active[entry] =
+ slot < frontier_capacity && frontier[uint(FrontierIdx::Active) * frontier_capacity + slot] != 0u;
}
- for (uint child = 0; child < width; ++child) {
+ for (uint child = 0; child < node_count; ++child) {
uint4 local = uint4(0u, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER, FRONTIER_NO_WINNER);
for (uint entry = 0; entry < FRONTIER_ENTRIES_PER_THREAD; ++entry) {
const uint slot = lid + entry * FRONTIER_SELECT_THREADS;
if (entry_active[entry]) {
- const uint key = frontier[uint(FrontierIdx::PathScoreKey) * capacity + slot];
- const uint parent = frontier[uint(FrontierIdx::ParentSlot) * capacity + slot];
- const uint token = frontier[uint(FrontierIdx::TokenId) * capacity + slot];
+ const uint key = frontier[uint(FrontierIdx::PathScoreKey) * frontier_capacity + slot];
+ const uint parent = frontier[uint(FrontierIdx::ParentSlot) * frontier_capacity + slot];
+ const uint token = frontier[uint(FrontierIdx::TokenId) * frontier_capacity + slot];
if (key > local.x || (key == local.x && (parent < local.y || (parent == local.y && token < local.z)))) {
local = uint4(key, parent, token, slot);
}
@@ -92,56 +93,56 @@ PUBLIC KERNEL(WeaverFrontierSelect)(
}
}
- if (lid < width) {
- const uint row = lid;
- const uint slot = winner_slot[row];
+ if (lid < node_count) {
+ const uint node = lid;
+ const uint slot = winner_slot[node];
const bool real = slot != FRONTIER_NO_WINNER;
- const uint tree_slot = slot_start + row;
+ const uint tree_slot = batch_start_slot + node;
- const uint token = real ? frontier[uint(FrontierIdx::TokenId) * capacity + slot] : 0u;
- const uint parent = real ? frontier[uint(FrontierIdx::ParentSlot) * capacity + slot] : FRONTIER_NO_WINNER;
- const uint depth = real ? frontier[uint(FrontierIdx::Depth) * capacity + slot] : 0u;
- const uint cumulative_logprob = real ? frontier[uint(FrontierIdx::PathLogprobBits) * capacity + slot] : 0u;
- const uint logprob = real ? frontier[uint(FrontierIdx::EdgeLogprobBits) * capacity + slot] : 0u;
+ const uint token = real ? frontier[uint(FrontierIdx::TokenId) * frontier_capacity + slot] : 0u;
+ const uint parent = real ? frontier[uint(FrontierIdx::ParentSlot) * frontier_capacity + slot] : FRONTIER_NO_WINNER;
+ const uint depth = real ? frontier[uint(FrontierIdx::Depth) * frontier_capacity + slot] : 0u;
+ const uint cumulative_logprob = real ? frontier[uint(FrontierIdx::PathLogprobBits) * frontier_capacity + slot] : 0u;
+ const uint logprob = real ? frontier[uint(FrontierIdx::EdgeLogprobBits) * frontier_capacity + slot] : 0u;
- tree[uint(TreeIdx::TokenId) * tree_slots + tree_slot] = token;
- tree[uint(TreeIdx::ParentSlot) * tree_slots + tree_slot] = parent;
- tree[uint(TreeIdx::Depth) * tree_slots + tree_slot] = depth;
- tree[uint(TreeIdx::PathLogprobBits) * tree_slots + tree_slot] = cumulative_logprob;
- tree[uint(TreeIdx::EdgeLogprobBits) * tree_slots + tree_slot] = logprob;
- tree[uint(TreeIdx::Valid) * tree_slots + tree_slot] = real ? 1u : 0u;
+ packed_tree[uint(TreeIdx::TokenId) * tree_slot_count + tree_slot] = token;
+ packed_tree[uint(TreeIdx::ParentSlot) * tree_slot_count + tree_slot] = parent;
+ packed_tree[uint(TreeIdx::Depth) * tree_slot_count + tree_slot] = depth;
+ packed_tree[uint(TreeIdx::PathLogprobBits) * tree_slot_count + tree_slot] = cumulative_logprob;
+ packed_tree[uint(TreeIdx::EdgeLogprobBits) * tree_slot_count + tree_slot] = logprob;
+ packed_tree[uint(TreeIdx::Valid) * tree_slot_count + tree_slot] = real ? 1u : 0u;
if (real) {
- frontier[uint(FrontierIdx::Active) * capacity + slot] = 0u;
+ frontier[uint(FrontierIdx::Active) * frontier_capacity + slot] = 0u;
}
- const uint parent_slot = real && parent < tree_slots ? parent : 0u;
+ const uint parent_slot = real && parent < tree_slot_count ? parent : 0u;
for (uint index = 0; index < ancestor_stride; ++index) {
const uint ancestor =
real && index + 1u <= depth
? (index + 1u == depth ? parent_slot : slot_ancestors[parent_slot * ancestor_stride + index])
: 0u;
slot_ancestors[tree_slot * ancestor_stride + index] = ancestor;
- round_ancestors[row * ancestor_stride + index] = ancestor;
+ node_ancestor_indices[node * ancestor_stride + index] = ancestor;
}
- round_token_ids[row] = token;
- round_metadata[uint(MetadataIdx::Depth) * width + row] = min(depth, max_depth - 1u);
- round_metadata[uint(MetadataIdx::AncestorCount) * width + row] = min(depth, ancestor_stride);
- round_metadata[uint(MetadataIdx::TreeSlot) * width + row] = tree_slot;
- round_valid[row] = real && depth < lookahead_count && depth < max_depth ? 1u : 0u;
+ node_token_ids[node] = token;
+ node_metadata[uint(MetadataIdx::Depth) * node_count + node] = min(depth, max_depth - 1u);
+ node_metadata[uint(MetadataIdx::AncestorCount) * node_count + node] = min(depth, ancestor_stride);
+ node_metadata[uint(MetadataIdx::TreeSlot) * node_count + node] = tree_slot;
+ node_valid[node] = real && depth < lookahead_count && depth < max_depth ? 1u : 0u;
- row_candidate_pool_row[row] = min(depth, candidate_pool_rows - 1u);
+ node_candidate_depth[node] = min(depth, candidate_depth_count - 1u);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
- for (uint row = 0; row < width; ++row) {
- const uint source = row_candidate_pool_row[row] * candidate_pool_size;
- const uint destination = row * candidate_pool_size;
- for (uint candidate = lid; candidate < candidate_pool_size; candidate += FRONTIER_SELECT_THREADS) {
- round_candidate_ids[destination + candidate] = candidate_pool_ids[source + candidate];
- round_candidate_scores[destination + candidate] = candidate_pool_scores[source + candidate];
+ for (uint node = 0; node < node_count; ++node) {
+ const uint source = node_candidate_depth[node] * candidates_per_depth;
+ const uint destination = node * candidates_per_depth;
+ for (uint candidate = lid; candidate < candidates_per_depth; candidate += FRONTIER_SELECT_THREADS) {
+ node_candidate_ids[destination + candidate] = candidate_pool_ids[source + candidate];
+ node_candidate_logits[destination + candidate] = candidate_pool_logits[source + candidate];
}
}
}
diff --git a/crates/backend-uzu/src/encodable_block/weaver.rs b/crates/backend-uzu/src/encodable_block/weaver.rs
index 344a1838c..4ee117304 100644
--- a/crates/backend-uzu/src/encodable_block/weaver.rs
+++ b/crates/backend-uzu/src/encodable_block/weaver.rs
@@ -34,75 +34,85 @@ struct TopKChildren<B: Backend> {
logprobs: Allocation<B>,
}
-struct PrefixOutput<B: Backend> {
- hidden: Allocation<B>,
+struct PrefixLayerOutput<B: Backend> {
+ mlp_delta: Allocation<B>,
kv_cache: Allocation<B>,
}
-struct PrefixAttentionInputs<B: Backend> {
+struct PreparedPrefixAttention<B: Backend> {
queries: Allocation<B>,
kv_cache: Allocation<B>,
}
-struct WeaverPrefixKvCache<B: Backend> {
- layer_kv: Box<[Allocation<B>]>,
- token_count: usize,
+struct PrefixKvCache<B: Backend> {
+ layers: Box<[Allocation<B>]>,
+ length: usize,
}
-struct WeaverNodeKvCache<B: Backend> {
- layer_kv: Box<[Allocation<B>]>,
- node_capacity: u32,
+struct NodeExpansionKvCache<B: Backend> {
+ layers: Box<[Allocation<B>]>,
+ capacity: u32,
}
-struct WeaverStepBatch<'a, B: Backend> {
- node_count: usize,
- candidates_per_node: usize,
- ancestor_stride: usize,
- node_token_ids: &'a Allocation<B>,
- candidate_ids: &'a Allocation<B>,
- candidate_logits: &'a Allocation<B>,
+struct NodeBatch<'a, B: Backend> {
+ count: usize,
+ token_ids: &'a Allocation<B>,
+ metadata: &'a Allocation<B>,
ancestor_indices: &'a Allocation<B>,
- node_metadata: &'a Allocation<B>,
+ ancestor_stride: usize,
+}
+
+struct CandidateBatch<'a, B: Backend> {
+ ids: &'a Allocation<B>,
+ logits: &'a Allocation<B>,
+ candidates_per_node: usize,
depth_seeds: &'a Allocation<B>,
}
-pub(crate) struct WeaverTreeInput<'a, B: Backend> {
- pub(crate) target_hidden: &'a Allocation<B>,
- pub(crate) draft_hidden: &'a Allocation<B>,
- pub(crate) target_embedding: &'a Embedding<B>,
- pub(crate) candidate_ids: &'a Allocation<B>,
- pub(crate) candidate_scores: &'a Allocation<B>,
- pub(crate) candidate_rows: usize,
- pub(crate) candidates_per_row: usize,
- pub(crate) depth_seeds: &'a [u64],
- pub(crate) root_token_id: u32,
- pub(crate) tree_budget: usize,
- pub(crate) frontier_width: usize,
- pub(crate) children_per_node: usize,
+pub struct CandidatePool<'a, B: Backend> {
+ pub ids: &'a Allocation<B>,
+ pub logits: &'a Allocation<B>,
+ pub depth_count: usize,
+ pub candidates_per_depth: usize,
}
-pub(crate) struct EncodedWeaverTree<B: Backend> {
- slots: Allocation<B>,
+pub struct WeaverInputs<'a, B: Backend> {
+ pub target_hidden: &'a Allocation<B>,
+ pub draft_hidden: &'a Allocation<B>,
+ pub target_embedding: &'a Embedding<B>,
+ pub candidates: CandidatePool<'a, B>,
+ pub depth_seeds: &'a [u64],
+ pub root_token_id: u32,
}
-pub(crate) struct ProposalNode {
- pub(crate) token: u32,
- pub(crate) depth: usize,
- pub(crate) children: Vec<usize>,
+pub struct TreeShape {
+ pub budget: usize,
+ pub frontier_width: usize,
+ pub children_per_node: usize,
+}
+
+pub struct EncodedWeaverTree<B: Backend> {
+ packed_tree: Allocation<B>,
+}
+
+pub struct ProposalNode {
+ pub token_id: u32,
+ pub depth: usize,
+ pub child_indices: Vec<usize>,
}
impl<B: Backend> EncodedWeaverTree<B> {
- pub(crate) fn decode(self) -> Vec<ProposalNode> {
- decode_tree_slots(&self.slots.copyout::<u32>())
+ pub fn read_nodes(self) -> Vec<ProposalNode> {
+ nodes_from_packed_tree(&self.packed_tree.copyout::<u32>())
}
}
-pub(crate) struct Weaver<B: Backend> {
+pub struct Weaver<B: Backend> {
token_embedding_norm: Normalization<B>,
token_embedding_projection: Box<dyn Linear<B>>,
hidden_state_norm: Normalization<B>,
hidden_state_projection: Box<dyn Linear<B>>,
- blocks: Box<[WeaverBlock<B>]>,
+ layers: Box<[WeaverLayer<B>]>,
readout_norm: Normalization<B>,
readout_query_projection: Box<dyn Linear<B>>,
position_embeddings: Allocation<B>,
@@ -116,7 +126,7 @@ pub(crate) struct Weaver<B: Backend> {
max_depth: usize,
}
-struct WeaverBlock<B: Backend> {
+struct WeaverLayer<B: Backend> {
// Attention
pre_attention_norm: Normalization<B>,
qkv_projection: Box<dyn Linear<B>>,
@@ -169,7 +179,7 @@ pub enum WeaverEncodeError<B: Backend> {
}
impl<B: Backend> Weaver<B> {
- pub(crate) fn new(
+ pub fn new(
context: &B::Context,
config: &WeaverConfig,
parameter_tree: &ParameterTree<B>,
@@ -211,17 +221,17 @@ impl<B: Backend> Weaver<B> {
DATA_TYPE,
&parameter_tree.subtree("embedding_projection")?,
)?;
- let blocks_tree = parameter_tree.subtree("blocks")?;
- let blocks = (0..config.num_layers)
+ let layer_parameters = parameter_tree.subtree("blocks")?;
+ let layers = (0..config.num_layers)
.map(|index| {
- WeaverBlock::new(
+ WeaverLayer::new(
context,
config.model_dim,
config.hidden_dim,
config.num_heads,
index > 0,
&config.norm_config,
- &blocks_tree.subtree(&index.to_string())?,
+ &layer_parameters.subtree(&index.to_string())?,
)
})
.collect::<Result<Box<[_]>, WeaverNewError<B>>>()?;
@@ -272,7 +282,7 @@ impl<B: Backend> Weaver<B> {
token_embedding_projection,
hidden_state_norm,
hidden_state_projection,
- blocks,
+ layers,
readout_norm,
readout_query_projection,
position_embeddings,
@@ -287,311 +297,317 @@ impl<B: Backend> Weaver<B> {
})
}
- pub(crate) fn encode_tree(
+ pub fn encode_tree(
&self,
- input: WeaverTreeInput<'_, B>,
+ inputs: WeaverInputs<'_, B>,
+ shape: TreeShape,
context: &B::Context,
encoder: &mut Encoder<B>,
) -> Result<EncodedWeaverTree<B>, WeaverEncodeError<B>> {
- let slot_count = input.tree_budget + 1;
- let frontier_capacity = slot_count * input.children_per_node;
- let lookahead_count = self.max_depth.min(input.candidate_rows);
- if input.tree_budget == 0
- || input.frontier_width == 0
- || input.children_per_node == 0
- || input.children_per_node > input.candidates_per_row
+ let tree_slot_count = shape.budget + 1;
+ let frontier_capacity = tree_slot_count * shape.children_per_node;
+ let lookahead_count = self.max_depth.min(inputs.candidates.depth_count);
+ let ancestor_stride = self.max_depth;
+ if shape.budget == 0
+ || shape.frontier_width == 0
+ || shape.children_per_node == 0
+ || shape.children_per_node > inputs.candidates.candidates_per_depth
|| frontier_capacity > FRONTIER_MAX_SLOTS
|| lookahead_count == 0
- || input.depth_seeds.len() != self.max_depth
+ || inputs.depth_seeds.len() != self.max_depth
{
return Err(WeaverEncodeError::InvalidTreeInput);
}
- let prefix = self.build_prefix(input.target_hidden, input.draft_hidden, lookahead_count, encoder)?;
- let mut node_cache = self.create_node_kv_cache(slot_count, context)?;
+ let prefix_cache = self
+ .encode_prefix(inputs.target_hidden, inputs.draft_hidden, lookahead_count, encoder)
+ .map_err(WeaverEncodeError::Backend)?;
+ let mut node_cache =
+ self.create_node_expansion_kv_cache(tree_slot_count, context).map_err(WeaverEncodeError::Backend)?;
- let mut tree_values = vec![0u32; TreeIdx::COUNT * slot_count];
- for slot in 0..slot_count {
- tree_values[TreeIdx::ParentSlot as usize * slot_count + slot] = FRONTIER_NO_WINNER;
+ let mut tree_init = vec![0u32; TreeIdx::COUNT * tree_slot_count];
+ for slot in 0..tree_slot_count {
+ tree_init[TreeIdx::ParentSlot as usize * tree_slot_count + slot] = FRONTIER_NO_WINNER;
}
- tree_values[TreeIdx::TokenId as usize * slot_count] = input.root_token_id;
- tree_values[TreeIdx::Valid as usize * slot_count] = 1;
+ tree_init[TreeIdx::TokenId as usize * tree_slot_count] = inputs.root_token_id;
+ tree_init[TreeIdx::Valid as usize * tree_slot_count] = 1;
- let mut tree = encoder.allocate_constant_from_slice(&tree_values).map_err(WeaverEncodeError::Backend)?;
+ let mut packed_tree = encoder.allocate_constant_from_slice(&tree_init).map_err(WeaverEncodeError::Backend)?;
let mut frontier = encoder
.allocate_constant_from_slice(&vec![0u32; FrontierIdx::COUNT * frontier_capacity])
.map_err(WeaverEncodeError::Backend)?;
let mut slot_ancestors = encoder
- .allocate_constant_from_slice(&vec![0u32; slot_count * self.max_depth])
+ .allocate_constant_from_slice(&vec![0u32; tree_slot_count * ancestor_stride])
.map_err(WeaverEncodeError::Backend)?;
- let mut round_token_id_values = vec![0u32; input.frontier_width];
- round_token_id_values[0] = input.root_token_id;
- let mut round_valid_values = vec![0u32; input.frontier_width];
- round_valid_values[0] = 1;
- let mut round_token_ids =
- encoder.allocate_constant_from_slice(&round_token_id_values).map_err(WeaverEncodeError::Backend)?;
- let mut round_metadata = encoder
- .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * input.frontier_width])
+ let mut initial_node_token_ids = vec![0u32; shape.frontier_width];
+ initial_node_token_ids[0] = inputs.root_token_id;
+ let mut initial_node_valid = vec![0u32; shape.frontier_width];
+ initial_node_valid[0] = 1;
+ let mut node_token_ids =
+ encoder.allocate_constant_from_slice(&initial_node_token_ids).map_err(WeaverEncodeError::Backend)?;
+ let mut node_metadata = encoder
+ .allocate_constant_from_slice(&vec![0u32; MetadataIdx::COUNT * shape.frontier_width])
.map_err(WeaverEncodeError::Backend)?;
- let mut round_ancestors = encoder
- .allocate_constant_from_slice(&vec![0u32; input.frontier_width * self.max_depth])
+ let mut node_ancestor_indices = encoder
+ .allocate_constant_from_slice(&vec![0u32; shape.frontier_width * ancestor_stride])
.map_err(WeaverEncodeError::Backend)?;
- let mut round_valid =
- encoder.allocate_constant_from_slice(&round_valid_values).map_err(WeaverEncodeError::Backend)?;
- let mut round_candidate_ids = encoder
- .allocate_constant_from_slice(&vec![0u32; input.frontier_width * input.candidates_per_row])
+ let mut node_valid =
+ encoder.allocate_constant_from_slice(&initial_node_valid).map_err(WeaverEncodeError::Backend)?;
+ let mut node_candidate_ids = encoder
+ .allocate_constant_from_slice(&vec![0u32; shape.frontier_width * inputs.candidates.candidates_per_depth])
.map_err(WeaverEncodeError::Backend)?;
- let mut round_candidate_scores = encoder
- .allocate_constant_from_slice(&vec![0.0f32; input.frontier_width * input.candidates_per_row])
+ let mut node_candidate_logits = encoder
+ .allocate_constant_from_slice(&vec![0.0f32; shape.frontier_width * inputs.candidates.candidates_per_depth])
.map_err(WeaverEncodeError::Backend)?;
let depth_seeds =
- encoder.allocate_constant_from_slice(input.depth_seeds).map_err(WeaverEncodeError::Backend)?;
+ encoder.allocate_constant_from_slice(inputs.depth_seeds).map_err(WeaverEncodeError::Backend)?;
- let mut slot_start = 0;
- while slot_start < slot_count {
- let node_count = if slot_start == 0 {
+ let mut batch_start_slot = 0;
+ while batch_start_slot < tree_slot_count {
+ let batch_node_count = if batch_start_slot == 0 {
1
} else {
- input.frontier_width.min(slot_count - slot_start)
+ shape.frontier_width.min(tree_slot_count - batch_start_slot)
};
- if slot_start > 0 {
+ if batch_start_slot > 0 {
self.frontier_select.encode(
&mut frontier,
- &mut tree,
+ &mut packed_tree,
&mut slot_ancestors,
- &mut round_token_ids,
- &mut round_metadata,
- &mut round_ancestors,
- &mut round_valid,
- input.candidate_ids,
- input.candidate_scores,
- &mut round_candidate_ids,
- &mut round_candidate_scores,
+ &mut node_token_ids,
+ &mut node_metadata,
+ &mut node_ancestor_indices,
+ &mut node_valid,
+ inputs.candidates.ids,
+ inputs.candidates.logits,
+ &mut node_candidate_ids,
+ &mut node_candidate_logits,
frontier_capacity as u32,
- slot_count as u32,
- node_count as u32,
- slot_start as u32,
- self.max_depth as u32,
+ tree_slot_count as u32,
+ batch_node_count as u32,
+ batch_start_slot as u32,
+ ancestor_stride as u32,
self.max_depth as u32,
lookahead_count as u32,
- input.candidate_rows as u32,
- input.candidates_per_row as u32,
+ inputs.candidates.depth_count as u32,
+ inputs.candidates.candidates_per_depth as u32,
encoder,
);
}
- let (candidate_ids, candidate_scores) = if slot_start == 0 {
- (input.candidate_ids, input.candidate_scores)
+ let (batch_candidate_ids, batch_candidate_logits) = if batch_start_slot == 0 {
+ (inputs.candidates.ids, inputs.candidates.logits)
} else {
- (&round_candidate_ids, &round_candidate_scores)
+ (&node_candidate_ids, &node_candidate_logits)
};
- let batch = WeaverStepBatch {
- node_count,
- candidates_per_node: input.candidates_per_row,
- ancestor_stride: self.max_depth,
- node_token_ids: &round_token_ids,
- candidate_ids,
- candidate_logits: candidate_scores,
- ancestor_indices: &round_ancestors,
- node_metadata: &round_metadata,
+ let nodes = NodeBatch {
+ count: batch_node_count,
+ token_ids: &node_token_ids,
+ metadata: &node_metadata,
+ ancestor_indices: &node_ancestor_indices,
+ ancestor_stride,
+ };
+ let candidates = CandidateBatch {
+ ids: batch_candidate_ids,
+ logits: batch_candidate_logits,
+ candidates_per_node: inputs.candidates.candidates_per_depth,
depth_seeds: &depth_seeds,
};
- let children = self.encode_step_batch(
- &prefix,
- &batch,
+ let selected_children = self.encode_node_expansion(
+ &prefix_cache,
+ &nodes,
+ &candidates,
&mut node_cache,
- input.children_per_node,
- input.target_embedding,
+ shape.children_per_node,
+ inputs.target_embedding,
encoder,
)?;
self.frontier_scatter.encode(
- &tree,
- &round_metadata,
- &round_valid,
- &children.token_ids,
- &children.logprobs,
+ &packed_tree,
+ &node_metadata,
+ &node_valid,
+ &selected_children.token_ids,
+ &selected_children.logprobs,
&mut frontier,
frontier_capacity as u32,
- slot_count as u32,
- node_count as u32,
- input.children_per_node as u32,
+ tree_slot_count as u32,
+ batch_node_count as u32,
+ shape.children_per_node as u32,
encoder,
);
- slot_start += node_count;
+ batch_start_slot += batch_node_count;
}
Ok(EncodedWeaverTree {
- slots: tree,
+ packed_tree,
})
}
- fn build_prefix(
+ fn encode_prefix(
&self,
target_hidden: &Allocation<B>,
draft_hidden: &Allocation<B>,
lookahead_count: usize,
encoder: &mut Encoder<B>,
- ) -> Result<WeaverPrefixKvCache<B>, WeaverEncodeError<B>> {
+ ) -> Result<PrefixKvCache<B>, B::Error> {
assert!((1..=self.max_depth).contains(&lookahead_count));
let token_count = lookahead_count + 1;
let hidden_row_bytes = self.target_model_dim * DATA_TYPE.size_in_bytes();
- let mut prefix_input = encoder
- .allocate_scratch(size_for_shape(&[token_count, self.target_model_dim], DATA_TYPE))
- .map_err(WeaverEncodeError::Backend)?;
- encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_input, 0..hidden_row_bytes);
+ let mut prefix_hidden =
+ encoder.allocate_scratch(size_for_shape(&[token_count, self.target_model_dim], DATA_TYPE))?;
+ encoder.encode_copy(target_hidden, 0..hidden_row_bytes, &mut prefix_hidden, 0..hidden_row_bytes);
encoder.encode_copy(
draft_hidden,
hidden_row_bytes..token_count * hidden_row_bytes,
- &mut prefix_input,
+ &mut prefix_hidden,
hidden_row_bytes..token_count * hidden_row_bytes,
);
- let normalized_prefix = self
- .hidden_state_norm
- .encode(&prefix_input, 0, token_count, None, encoder)
- .map_err(WeaverEncodeError::Backend)?;
- let mut hidden = self
- .hidden_state_projection
- .encode(normalized_prefix, token_count, encoder)
- .map_err(WeaverEncodeError::Backend)?;
+ let normalized_prefix = self.hidden_state_norm.encode(&prefix_hidden, 0, token_count, None, encoder)?;
+ let mut residual_input = self.hidden_state_projection.encode(normalized_prefix, token_count, encoder)?;
let position_elements = lookahead_count * self.model_dim;
self.prefix_position_add.encode(
None::<&Allocation<B>>,
&self.position_embeddings,
None::<&Allocation<B>>,
- (&mut hidden, self.model_dim * DATA_TYPE.size_in_bytes()),
+ (&mut residual_input, self.model_dim * DATA_TYPE.size_in_bytes()),
position_elements as u32,
position_elements as u32,
encoder,
);
- let (last_block, preceding_blocks) = self.blocks.split_last().expect("Weaver must have at least one block");
- let mut residual = encoder.allocate_scratch(hidden.size()).map_err(WeaverEncodeError::Backend)?;
- let mut layer_kv = Vec::with_capacity(self.blocks.len());
- for block in preceding_blocks {
- let PrefixOutput {
- hidden: next_hidden,
+ let (last_layer, preceding_layers) = self.layers.split_last().expect("Weaver must have at least one layer");
+ let mut residual_state = encoder.allocate_scratch(residual_input.size())?;
+ let mut prefix_layers = Vec::with_capacity(self.layers.len());
+ for layer in preceding_layers {
+ let PrefixLayerOutput {
+ mlp_delta,
kv_cache,
- } = block.encode_prefix(hidden, &mut residual, token_count, encoder).map_err(WeaverEncodeError::Backend)?;
- layer_kv.push(kv_cache);
- hidden = next_hidden;
+ } = layer.encode_prefix_tokens(residual_input, &mut residual_state, token_count, encoder)?;
+ prefix_layers.push(kv_cache);
+ residual_input = mlp_delta;
}
- layer_kv.push(
- last_block
- .prepare_prefix_attention(&hidden, &mut residual, token_count, encoder)
- .map_err(WeaverEncodeError::Backend)?
- .kv_cache,
+ prefix_layers.push(
+ last_layer.encode_prefix_attention(&residual_input, &mut residual_state, token_count, encoder)?.kv_cache,
);
- Ok(WeaverPrefixKvCache {
- layer_kv: layer_kv.into_boxed_slice(),
- token_count,
+ Ok(PrefixKvCache {
+ layers: prefix_layers.into_boxed_slice(),
+ length: token_count,
})
}
- fn create_node_kv_cache(
+ fn create_node_expansion_kv_cache(
&self,
capacity: usize,
context: &B::Context,
- ) -> Result<WeaverNodeKvCache<B>, WeaverEncodeError<B>> {
+ ) -> Result<NodeExpansionKvCache<B>, B::Error> {
assert!(capacity > 0, "Weaver node capacity must be positive");
- let node_capacity = u32::try_from(capacity).expect("Weaver node capacity exceeds the kernel limit");
let kv_size = size_for_shape(&[2, capacity, self.model_dim], DATA_TYPE);
- let layer_kv = (0..self.blocks.len())
- .map(|_| context.create_allocation(kv_size, AllocationType::Global).map_err(WeaverEncodeError::Backend))
+ let layers = (0..self.layers.len())
+ .map(|_| context.create_allocation(kv_size, AllocationType::Global))
.collect::<Result<Box<[_]>, _>>()?;
- Ok(WeaverNodeKvCache {
- layer_kv,
- node_capacity,
+ Ok(NodeExpansionKvCache {
+ layers,
+ capacity: capacity as u32,
})
}
- fn encode_step_batch(
+ fn encode_node_expansion(
&self,
- prefix: &WeaverPrefixKvCache<B>,
- batch: &WeaverStepBatch<'_, B>,
- node_cache: &mut WeaverNodeKvCache<B>,
+ prefix_cache: &PrefixKvCache<B>,
+ nodes: &NodeBatch<'_, B>,
+ candidates: &CandidateBatch<'_, B>,
+ node_cache: &mut NodeExpansionKvCache<B>,
children_per_node: usize,
target_embedding: &Embedding<B>,
encoder: &mut Encoder<B>,
) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {
- let token_embedding = target_embedding.encode_lookup(batch.node_token_ids, batch.node_count, encoder)?;
+ let token_embedding = target_embedding.encode_lookup(nodes.token_ids, nodes.count, encoder)?;
let normalized_embedding = self
.token_embedding_norm
- .encode(&token_embedding, 0, batch.node_count, None, encoder)
+ .encode(&token_embedding, 0, nodes.count, None, encoder)
.map_err(WeaverEncodeError::Backend)?;
- let mut hidden = self
+ let mut residual_input = self
.token_embedding_projection
- .encode(normalized_embedding, batch.node_count, encoder)
+ .encode(normalized_embedding, nodes.count, encoder)
.map_err(WeaverEncodeError::Backend)?;
self.node_position_add.encode(
None::<&Allocation<B>>,
&self.position_embeddings,
- Some(batch.node_metadata),
- &mut hidden,
+ Some(nodes.metadata),
+ &mut residual_input,
self.model_dim as u32,
- (batch.node_count * self.model_dim) as u32,
+ (nodes.count * self.model_dim) as u32,
encoder,
);
- let mut residual = encoder.allocate_scratch(hidden.size()).map_err(WeaverEncodeError::Backend)?;
- let node_capacity = node_cache.node_capacity;
- for (layer_index, block) in self.blocks.iter().enumerate() {
- hidden = block
- .encode_step(
- hidden,
- &mut residual,
- &prefix.layer_kv[layer_index],
- &mut node_cache.layer_kv[layer_index],
+ let mut residual_state = encoder.allocate_scratch(residual_input.size()).map_err(WeaverEncodeError::Backend)?;
+ let node_capacity = node_cache.capacity;
+ for (layer_index, layer) in self.layers.iter().enumerate() {
+ residual_input = layer
+ .encode_node_batch(
+ residual_input,
+ &mut residual_state,
+ &prefix_cache.layers[layer_index],
+ &mut node_cache.layers[layer_index],
node_capacity,
- batch,
- prefix.token_count,
+ nodes,
+ prefix_cache.length,
encoder,
)
.map_err(WeaverEncodeError::Backend)?;
}
- self.encode_step_output(&hidden, &mut residual, batch, children_per_node, target_embedding, encoder)
+ self.encode_child_selection(
+ &residual_input,
+ &mut residual_state,
+ nodes,
+ candidates,
+ children_per_node,
+ target_embedding,
+ encoder,
+ )
}
- /// Outputs must outlive the batch.
- fn encode_step_output(
+ fn encode_child_selection(
&self,
- hidden: &Allocation<B>,
- residual: &mut Allocation<B>,
- batch: &WeaverStepBatch<'_, B>,
+ residual_output: &Allocation<B>,
+ residual_state: &mut Allocation<B>,
+ nodes: &NodeBatch<'_, B>,
+ candidates: &CandidateBatch<'_, B>,
children_per_node: usize,
target_embedding: &Embedding<B>,
encoder: &mut Encoder<B>,
) -> Result<TopKChildren<B>, WeaverEncodeError<B>> {
let normalized_output = self
.readout_norm
- .encode(hidden, 0, batch.node_count, Some(residual), encoder)
+ .encode(residual_output, 0, nodes.count, Some(residual_state), encoder)
.map_err(WeaverEncodeError::Backend)?;
let query = self
.readout_query_projection
- .encode(normalized_output, batch.node_count, encoder)
+ .encode(normalized_output, nodes.count, encoder)
.map_err(WeaverEncodeError::Backend)?;
- let residual_logits = target_embedding.encode_readout_sparse(
+ let logit_residuals = target_embedding.encode_readout_sparse(
&query,
- batch.candidate_ids,
- batch.node_count,
- batch.candidates_per_node,
+ candidates.ids,
+ nodes.count,
+ candidates.candidates_per_node,
encoder,
)?;
let mut token_ids = encoder
- .allocate_scratch(size_for_shape(&[batch.node_count, children_per_node], DataType::U32))
+ .allocate_scratch(size_for_shape(&[nodes.count, children_per_node], DataType::U32))
.map_err(WeaverEncodeError::Backend)?;
let mut logprobs = encoder
- .allocate_scratch(size_for_shape(&[batch.node_count, children_per_node], DataType::F32))
+ .allocate_scratch(size_for_shape(&[nodes.count, children_per_node], DataType::F32))
.map_err(WeaverEncodeError::Backend)?;
self.top_children.encode(
- &residual_logits,
- batch.candidate_logits,
- batch.candidate_ids,
- batch.depth_seeds,
- batch.node_metadata,
+ &logit_residuals,
+ candidates.logits,
+ candidates.ids,
+ candidates.depth_seeds,
+ nodes.metadata,
&mut token_ids,
&mut logprobs,
- batch.node_count as u32,
- batch.candidates_per_node as u32,
+ nodes.count as u32,
+ candidates.candidates_per_node as u32,
children_per_node as u32,
target_embedding.vocab_size() as u32,
encoder,
@@ -603,7 +619,7 @@ impl<B: Backend> Weaver<B> {
}
}
-impl<B: Backend> WeaverBlock<B> {
+impl<B: Backend> WeaverLayer<B> {
fn new(
context: &B::Context,
model_dim: usize,
@@ -717,22 +733,22 @@ impl<B: Backend> WeaverBlock<B> {
})
}
- fn encode_prefix(
+ fn encode_prefix_tokens(
&self,
- input: Allocation<B>,
- residual: &mut Allocation<B>,
+ residual_input: Allocation<B>,
+ residual_state: &mut Allocation<B>,
token_count: usize,
encoder: &mut Encoder<B>,
- ) -> Result<PrefixOutput<B>, B::Error> {
- let PrefixAttentionInputs {
+ ) -> Result<PrefixLayerOutput<B>, B::Error> {
+ let PreparedPrefixAttention {
queries,
kv_cache,
- } = self.prepare_prefix_attention(&input, residual, token_count, encoder)?;
+ } = self.encode_prefix_attention(&residual_input, residual_state, token_count, encoder)?;
let state_type = AttentionStateType::Full {
length: 0,
};
let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
- let attention = self.prefix_attention.encode(
+ let attention_output = self.prefix_attention.encode(
AttentionCoreEncodeArguments {
queries: &queries,
keys: &kv_cache,
@@ -744,29 +760,30 @@ impl<B: Backend> WeaverBlock<B> {
},
encoder,
)?;
- let hidden = self.encode_post_attention(attention, residual, token_count, encoder)?;
- Ok(PrefixOutput {
- hidden,
+ let mlp_delta = self.encode_post_attention(attention_output, residual_state, token_count, encoder)?;
+ Ok(PrefixLayerOutput {
+ mlp_delta,
kv_cache,
})
}
- fn prepare_prefix_attention(
+ fn encode_prefix_attention(
&self,
- input: &Allocation<B>,
- residual: &mut Allocation<B>,
+ residual_input: &Allocation<B>,
+ residual_state: &mut Allocation<B>,
token_count: usize,
encoder: &mut Encoder<B>,
- ) -> Result<PrefixAttentionInputs<B>, B::Error> {
- let attention_input = self.pre_attention_norm.encode(input, 0, token_count, Some(residual), encoder)?;
+ ) -> Result<PreparedPrefixAttention<B>, B::Error> {
+ let attention_input =
+ self.pre_attention_norm.encode(residual_input, 0, token_count, Some(residual_state), encoder)?;
let qkv = self.qkv_projection.encode(attention_input, token_count, encoder)?;
let mut queries =
encoder.allocate_scratch(size_for_shape(&[self.num_heads, token_count, self.head_dim], DATA_TYPE))?;
let kv_plane_bytes = size_for_shape(&[token_count, self.model_dim], DATA_TYPE);
- let mut kv = encoder
+ let mut kv_cache = encoder
.context()
.create_allocation(size_for_shape(&[2, token_count, self.model_dim], DATA_TYPE), AllocationType::Global)?;
- let (keys, values) = kv.as_buffer_range_mut().split_at(kv_plane_bytes);
+ let (keys, values) = kv_cache.as_buffer_range_mut().split_at(kv_plane_bytes);
self.attention_prepare.encode(
&qkv,
&mut queries,
@@ -782,39 +799,41 @@ impl<B: Backend> WeaverBlock<B> {
token_count as u32,
encoder,
);
- Ok(PrefixAttentionInputs {
+ Ok(PreparedPrefixAttention {
queries,
- kv_cache: kv,
+ kv_cache,
})
}
- fn encode_step(
+ fn encode_node_batch(
&self,
- input: Allocation<B>,
- residual: &mut Allocation<B>,
+ residual_input: Allocation<B>,
+ residual_state: &mut Allocation<B>,
prefix_kv: &Allocation<B>,
node_kv_cache: &mut Allocation<B>,
node_capacity: u32,
- batch: &WeaverStepBatch<'_, B>,
+ nodes: &NodeBatch<'_, B>,
prefix_length: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let attention_input = self.pre_attention_norm.encode(&input, 0, batch.node_count, Some(residual), encoder)?;
- let current_qkv = self.qkv_projection.encode(attention_input, batch.node_count, encoder)?;
- let metadata_lane_bytes = batch.node_count * DataType::U32.size_in_bytes();
- let ancestor_counts = (batch.node_metadata, MetadataIdx::AncestorCount as usize * metadata_lane_bytes);
- let tree_slots = (batch.node_metadata, MetadataIdx::TreeSlot as usize * metadata_lane_bytes);
- let mut attention = encoder.allocate_scratch(size_for_shape(&[batch.node_count, self.model_dim], DATA_TYPE))?;
+ let attention_input =
+ self.pre_attention_norm.encode(&residual_input, 0, nodes.count, Some(residual_state), encoder)?;
+ let current_qkv = self.qkv_projection.encode(attention_input, nodes.count, encoder)?;
+ let metadata_field_bytes = nodes.count * DataType::U32.size_in_bytes();
+ let ancestor_counts = (nodes.metadata, MetadataIdx::AncestorCount as usize * metadata_field_bytes);
+ let tree_slot_indices = (nodes.metadata, MetadataIdx::TreeSlot as usize * metadata_field_bytes);
+ let mut attention_output =
+ encoder.allocate_scratch(size_for_shape(&[nodes.count, self.model_dim], DATA_TYPE))?;
self.ancestor_attention.encode(
prefix_kv,
&*node_kv_cache,
&current_qkv,
- batch.ancestor_indices,
+ nodes.ancestor_indices,
ancestor_counts,
- &mut attention,
- batch.node_count as u32,
+ &mut attention_output,
+ nodes.count as u32,
prefix_length as u32,
- batch.ancestor_stride as u32,
+ nodes.ancestor_stride as u32,
node_capacity,
self.attention_scale,
encoder,
@@ -824,61 +843,61 @@ impl<B: Backend> WeaverBlock<B> {
self.node_cache_write.encode(
&current_qkv,
node_kv_cache,
- tree_slots,
+ tree_slot_indices,
self.model_dim as u32,
node_capacity,
- (batch.node_count * 2 * self.model_dim) as u32,
+ (nodes.count * 2 * self.model_dim) as u32,
encoder,
);
- self.encode_post_attention(attention, residual, batch.node_count, encoder)
+ self.encode_post_attention(attention_output, residual_state, nodes.count, encoder)
}
fn encode_post_attention(
&self,
- attention: Allocation<B>,
- residual: &mut Allocation<B>,
- row_count: usize,
+ attention_output: Allocation<B>,
+ residual_state: &mut Allocation<B>,
+ token_count: usize,
encoder: &mut Encoder<B>,
) -> Result<Allocation<B>, B::Error> {
- let projected_attention = self.out_projection.encode(attention, row_count, encoder)?;
- let mlp_input = self.pre_mlp_norm.encode(&projected_attention, 0, row_count, Some(residual), encoder)?;
- let mut mlp_hidden = self.up_projection.encode(mlp_input, row_count, encoder)?;
+ let projected_attention = self.out_projection.encode(attention_output, token_count, encoder)?;
+ let mlp_input =
+ self.pre_mlp_norm.encode(&projected_attention, 0, token_count, Some(residual_state), encoder)?;
+ let mut mlp_hidden = self.up_projection.encode(mlp_input, token_count, encoder)?;
self.activation.encode(
None::<&Allocation<B>>,
&mut mlp_hidden,
- (row_count * self.hidden_dim) as u32,
+ (token_count * self.hidden_dim) as u32,
crate::backends::common::gpu_types::ActivationType::GELUExact,
encoder,
);
- self.down_projection.encode(mlp_hidden, row_count, encoder)
+ self.down_projection.encode(mlp_hidden, token_count, encoder)
}
}
-fn decode_tree_slots(tree: &[u32]) -> Vec<ProposalNode> {
- assert!(tree.len().is_multiple_of(TreeIdx::COUNT), "tree array must contain {} equal-length lanes", TreeIdx::COUNT);
- let slot_count = tree.len() / TreeIdx::COUNT;
- let field = |field: TreeIdx, slot: usize| tree[field as usize * slot_count + slot];
- let mut slot_to_node = vec![usize::MAX; slot_count];
- let mut nodes: Vec<ProposalNode> = Vec::with_capacity(slot_count);
- for slot in 0..slot_count {
+fn nodes_from_packed_tree(packed_tree: &[u32]) -> Vec<ProposalNode> {
+ let tree_slot_count = packed_tree.len() / TreeIdx::COUNT;
+ let field = |field: TreeIdx, slot: usize| packed_tree[field as usize * tree_slot_count + slot];
+ let mut slot_to_node_index = vec![usize::MAX; tree_slot_count];
+ let mut nodes: Vec<ProposalNode> = Vec::with_capacity(tree_slot_count);
+ for slot in 0..tree_slot_count {
if field(TreeIdx::Valid, slot) == 0 {
continue;
}
let parent_slot = field(TreeIdx::ParentSlot, slot) as i32;
let parent = (parent_slot >= 0).then(|| {
- let parent = slot_to_node[parent_slot as usize];
+ let parent = slot_to_node_index[parent_slot as usize];
assert_ne!(parent, usize::MAX, "tree slot {slot} names padding slot {parent_slot} as its parent");
parent
});
let index = nodes.len();
- slot_to_node[slot] = index;
+ slot_to_node_index[slot] = index;
if let Some(parent) = parent {
- nodes[parent].children.push(index);
+ nodes[parent].child_indices.push(index);
}
nodes.push(ProposalNode {
- token: field(TreeIdx::TokenId, slot),
+ token_id: field(TreeIdx::TokenId, slot),
depth: field(TreeIdx::Depth, slot) as usize,
- children: Vec::new(),
+ child_indices: Vec::new(),
});
}
nodes
diff --git a/crates/backend-uzu/src/speculators/dflash_speculator.rs b/crates/backend-uzu/src/speculators/dflash_speculator.rs
index bbee7a1f1..3f9a9d8a3 100644
--- a/crates/backend-uzu/src/speculators/dflash_speculator.rs
+++ b/crates/backend-uzu/src/speculators/dflash_speculator.rs
@@ -18,7 +18,7 @@ use crate::{
dflash::{DFlashDraft, DFlashDraftEncodeError, DFlashDraftNewError, DFlashDraftOutput},
embedding::Embedding,
sampling::PRng,
- weaver::{ProposalNode, Weaver, WeaverEncodeError, WeaverNewError, WeaverTreeInput},
+ weaver::{CandidatePool, ProposalNode, TreeShape, Weaver, WeaverEncodeError, WeaverInputs, WeaverNewError},
},
parameters::{HeaderLoadingError, ParameterLoader, ParameterLoaderError},
trie::TrieNode,
@@ -168,17 +168,21 @@ impl<B: Backend> DFlashSpeculator<B> {
let depth_seeds =
(0..max_depth).map(|depth| prng.derive((root_position + depth) as u64)).collect::<Box<[u64]>>();
let tree = weaver.encode_tree(
- WeaverTreeInput {
+ WeaverInputs {
target_hidden: target_output_norm,
draft_hidden: &draft_hidden,
target_embedding,
- candidate_ids: &candidates.ids,
- candidate_scores: &candidates.scores,
- candidate_rows: candidates.rows,
- candidates_per_row: candidates.candidates_per_row,
+ candidates: CandidatePool {
+ ids: &candidates.ids,
+ logits: &candidates.scores,
+ depth_count: candidates.rows,
+ candidates_per_depth: candidates.candidates_per_row,
+ },
depth_seeds: &depth_seeds,
root_token_id: target_output_token,
- tree_budget: options.budget,
+ },
+ TreeShape {
+ budget: options.budget,
frontier_width: options.frontier_width,
children_per_node: options.children_per_node,
},
@@ -186,7 +190,7 @@ impl<B: Backend> DFlashSpeculator<B> {
&mut encoder,
)?;
let completed = encoder.end_encoding().submit().wait_until_completed().map_err(DFlashTreeError::Backend)?;
- let nodes = tree.decode();
+ let nodes = tree.read_nodes();
drop(candidates);
drop(draft_hidden);
drop(completed);
@@ -208,20 +212,20 @@ impl<B: Backend> DFlashSpeculator<B> {
drop(completed);
let mut nodes = vec![ProposalNode {
- token: target_output_token,
+ token_id: target_output_token,
depth: 0,
- children: Vec::new(),
+ child_indices: Vec::new(),
}];
for depth in 0..options.budget.min(candidate_rows) {
let token = candidate_id_values[depth * candidates_per_row];
let parent = nodes.len() - 1;
let index = nodes.len();
nodes.push(ProposalNode {
- token,
+ token_id: token,
depth: depth + 1,
- children: Vec::new(),
+ child_indices: Vec::new(),
});
- nodes[parent].children.push(index);
+ nodes[parent].child_indices.push(index);
}
Ok(Self::finish_tree(
nodes,
@@ -246,11 +250,11 @@ impl<B: Backend> DFlashSpeculator<B> {
#[cfg(grammar)] grammar: &mut Option<&mut Grammar>,
) -> TrieNode {
let node = &nodes[index];
- let mut trie_node = TrieNode::new(node.token as u64, prng.derive((root_position + node.depth) as u64));
- for &child_index in &node.children {
+ let mut trie_node = TrieNode::new(node.token_id as u64, prng.derive((root_position + node.depth) as u64));
+ for &child_index in &node.child_indices {
#[cfg(grammar)]
if let Some(grammar) = grammar.as_mut()
- && grammar.accept_token(nodes[child_index].token as u64).is_err()
+ && grammar.accept_token(nodes[child_index].token_id as u64).is_err()
{
continue;
}