Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add DFlash chain proposal acceptance semantics (#532)

CCYehd769c95c65fb
diff --git a/crates/backend-uzu/src/speculators/chain_acceptance.rs b/crates/backend-uzu/src/speculators/chain_acceptance.rs
new file mode 100644
index 000000000..8a5ba8db4
--- /dev/null
+++ b/crates/backend-uzu/src/speculators/chain_acceptance.rs
@@ -0,0 +1,88 @@
+pub struct ChainProposal {
+ pub token_ids: Box<[u64]>,
+ pub token_positions: Box<[usize]>,
+}
+
+pub struct AcceptedProposal {
+ pub committed_token_ids: Vec<u64>,
+ pub committed_token_positions: Vec<usize>,
+ pub num_verified_nodes: usize,
+}
+
+impl ChainProposal {
+ /// `sampled_token_ids[i]` is the target's next token after node `i`.
+ pub fn accept(
+ &self,
+ sampled_token_ids: &[u64],
+ remaining_length: usize,
+ eos_token_ids: &[u64],
+ ) -> AcceptedProposal {
+ let num_nodes = self.token_ids.len();
+ debug_assert!(num_nodes > 0, "chain proposal must contain at least the last committed token");
+ debug_assert_eq!(self.token_positions.len(), num_nodes, "token_positions length must match token_ids");
+ debug_assert_eq!(sampled_token_ids.len(), num_nodes, "expected one sampled token per proposal node");
+
+ let num_verified_drafts =
+ (0..num_nodes - 1).take_while(|&i| self.token_ids[i + 1] == sampled_token_ids[i]).count();
+ let num_verified_nodes = num_verified_drafts + 1;
+
+ let mut committed_token_ids = Vec::with_capacity(num_verified_nodes);
+ let mut committed_token_positions = Vec::with_capacity(num_verified_nodes);
+ let mut has_draft_eos = false;
+ for draft in 0..num_verified_drafts {
+ let token_id = self.token_ids[draft + 1];
+ committed_token_ids.push(token_id);
+ committed_token_positions.push(self.token_positions[draft + 1]);
+ if eos_token_ids.contains(&token_id) {
+ has_draft_eos = true;
+ break;
+ }
+ }
+
+ // A draft EOS suppresses the bonus token; a bonus EOS is kept.
+ if !has_draft_eos {
+ committed_token_ids.push(sampled_token_ids[num_verified_drafts]);
+ committed_token_positions.push(self.token_positions[num_verified_drafts] + 1);
+ }
+
+ committed_token_ids.truncate(remaining_length);
+ committed_token_positions.truncate(remaining_length);
+
+ AcceptedProposal {
+ committed_token_ids,
+ committed_token_positions,
+ num_verified_nodes,
+ }
+ }
+}
+
+impl AcceptedProposal {
+ pub fn next_context_nodes(&self) -> std::ops::Range<usize> {
+ 0..self.num_verified_nodes
+ }
+
+ pub fn last_token_id(&self) -> Option<u64> {
+ self.committed_token_ids.last().copied()
+ }
+
+ /// One behind the last committed token (the bonus, not yet forward-passed).
+ pub fn last_covered_position(&self) -> Option<usize> {
+ if let Some(position) = self.committed_token_positions.last() {
+ debug_assert!(*position >= 1, "committed token positions are 1-based");
+ Some(position - 1)
+ } else {
+ None
+ }
+ }
+
+ pub fn has_eos(
+ &self,
+ eos_token_ids: &[u64],
+ ) -> bool {
+ self.committed_token_ids.iter().any(|token| eos_token_ids.contains(token))
+ }
+}
+
+#[cfg(test)]
+#[path = "../../unit/speculators/chain_acceptance.rs"]
+mod tests;
diff --git a/crates/backend-uzu/src/speculators/mod.rs b/crates/backend-uzu/src/speculators/mod.rs
index e61f3e1b9..06f1fa8e6 100644
--- a/crates/backend-uzu/src/speculators/mod.rs
+++ b/crates/backend-uzu/src/speculators/mod.rs
@@ -1,3 +1,4 @@
+pub mod chain_acceptance;
pub mod empty_speculator;
pub mod fixed_token_speculator;
pub mod ngram_speculator;
diff --git a/crates/backend-uzu/unit/speculators/chain_acceptance.rs b/crates/backend-uzu/unit/speculators/chain_acceptance.rs
new file mode 100644
index 000000000..0c7a08ba8
--- /dev/null
+++ b/crates/backend-uzu/unit/speculators/chain_acceptance.rs
@@ -0,0 +1,122 @@
+use proc_macros::uzu_test;
+
+use crate::speculators::chain_acceptance::{AcceptedProposal, ChainProposal};
+
+const NO_EOS: &[u64] = &[99999];
+
+fn make_proposal() -> ChainProposal {
+ ChainProposal {
+ token_ids: Box::new([10, 11, 12, 13]),
+ token_positions: Box::new([5, 6, 7, 8]),
+ }
+}
+
+fn accept(
+ sampled_token_ids: &[u64],
+ eos_token_ids: &[u64],
+) -> AcceptedProposal {
+ make_proposal().accept(sampled_token_ids, 16, eos_token_ids)
+}
+
+#[uzu_test]
+fn test_full_chain_accepted_with_bonus() {
+ let accepted = accept(&[11, 12, 13, 14], NO_EOS);
+
+ assert_eq!(accepted.committed_token_ids, vec![11, 12, 13, 14]);
+ assert_eq!(accepted.committed_token_positions, vec![6, 7, 8, 9]);
+ assert_eq!(accepted.num_verified_nodes, 4);
+ assert_eq!(accepted.next_context_nodes(), 0..4);
+}
+
+#[uzu_test]
+fn test_partial_chain_accepted() {
+ let accepted = accept(&[11, 99, 13, 14], NO_EOS);
+
+ assert_eq!(accepted.committed_token_ids, vec![11, 99]);
+ assert_eq!(accepted.committed_token_positions, vec![6, 7]);
+ assert_eq!(accepted.num_verified_nodes, 2);
+}
+
+#[uzu_test]
+fn test_all_drafts_rejected_emits_bonus_only() {
+ let accepted = accept(&[99, 98, 97, 96], NO_EOS);
+
+ assert_eq!(accepted.committed_token_ids, vec![99]);
+ assert_eq!(accepted.committed_token_positions, vec![6]);
+ assert_eq!(accepted.num_verified_nodes, 1);
+}
+
+#[uzu_test]
+fn test_eos_in_drafts_truncates_and_suppresses_bonus() {
+ let accepted = accept(&[11, 12, 13, 14], &[12]);
+
+ assert_eq!(accepted.committed_token_ids, vec![11, 12]);
+ assert!(accepted.has_eos(&[12]));
+ // EOS truncates the committed output only; all four nodes stay verified.
+ assert_eq!(accepted.num_verified_nodes, 4);
+}
+
+#[uzu_test]
+fn test_eos_at_first_accepted_draft() {
+ let accepted = accept(&[11, 12, 13, 14], &[11]);
+
+ assert_eq!(accepted.committed_token_ids, vec![11]);
+ assert!(accepted.has_eos(&[11]));
+ assert_eq!(accepted.num_verified_nodes, 4);
+}
+
+#[uzu_test]
+fn test_eos_as_bonus_token() {
+ let accepted = accept(&[11, 99, 13, 14], &[99]);
+
+ assert_eq!(accepted.committed_token_ids, vec![11, 99]);
+ assert!(accepted.has_eos(&[99]));
+ assert_eq!(accepted.num_verified_nodes, 2);
+}
+
+#[uzu_test]
+fn test_remaining_length_caps_emission() {
+ let accepted = make_proposal().accept(&[11, 12, 13, 14], 2, NO_EOS);
+
+ assert_eq!(accepted.committed_token_ids, vec![11, 12]);
+ // The output budget caps committed tokens only, not verified nodes.
+ assert_eq!(accepted.num_verified_nodes, 4);
+}
+
+#[uzu_test]
+fn test_zero_remaining_length_commits_nothing() {
+ let accepted = make_proposal().accept(&[11, 99, 13, 14], 0, NO_EOS);
+
+ assert!(accepted.committed_token_ids.is_empty());
+ assert_eq!(accepted.num_verified_nodes, 2);
+ assert_eq!(accepted.last_token_id(), None);
+ assert_eq!(accepted.last_covered_position(), None);
+}
+
+#[uzu_test]
+fn test_root_only_proposal_emits_bonus() {
+ let proposal = ChainProposal {
+ token_ids: Box::new([10]),
+ token_positions: Box::new([5]),
+ };
+ let accepted = proposal.accept(&[42], 16, NO_EOS);
+
+ assert_eq!(accepted.committed_token_ids, vec![42]);
+ assert_eq!(accepted.committed_token_positions, vec![6]);
+ assert_eq!(accepted.num_verified_nodes, 1);
+}
+
+#[uzu_test]
+fn test_last_token_id_and_index() {
+ let accepted = accept(&[11, 99, 13, 14], NO_EOS);
+
+ assert_eq!(accepted.last_token_id(), Some(99));
+ assert_eq!(accepted.last_covered_position(), Some(6));
+}
+
+#[uzu_test]
+fn test_has_eos_negative() {
+ let accepted = accept(&[11, 12, 13, 14], NO_EOS);
+
+ assert!(!accepted.has_eos(&[12345]));
+}