A high-performance inference engine for AI models
ngram speculator (#243)
uuuvndc4b5d12f3f3
diff --git a/Cargo.lock b/Cargo.lockindex 5eca14321..9937dc855 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2235,6 +2235,15 @@ version = "2.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"+[[package]]+name = "memmap2"+version = "0.9.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"+dependencies = [+ "libc",+]+[[package]]name = "metal"version = "0.32.0"@@ -4355,6 +4364,7 @@ dependencies = ["is_close","itertools 0.14.0","libc",+ "memmap2","minijinja","minijinja-contrib","mpsgraph",@@ -4380,6 +4390,8 @@ dependencies = ["tokio","walkdir","xgrammar-rs",+ "xxhash-rust",+ "zerocopy",][[package]]@@ -5089,6 +5101,12 @@ dependencies = ["walkdir",]+[[package]]+name = "xxhash-rust"+version = "0.8.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"+[[package]]name = "yansi"version = "1.0.1"diff --git a/Cargo.toml b/Cargo.tomlindex da4244059..ae3d2dab6 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -27,6 +27,9 @@ objc2-foundation = "0.3"metal = { package = "mtl-rs", version = "0.1.6" }block2 = "0.6"bytemuck = "1.23"+zerocopy = "0.8.27"+xxhash-rust = "0.8.15"+memmap2 = "0.9.8"num-traits = "0.2"regex = "1.12"libc = "0.2"diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rsindex feee0c1d5..491117fc5 100644--- a/crates/cli/src/handlers/run.rs+++ b/crates/cli/src/handlers/run.rs@@ -34,9 +34,10 @@ pub fn handle_run(tokens_limit: usize,prefill_step_size: Option<usize>,seed: Option<u64>,+ speculator: Option<String>,mut message: Option<String>,) {- let mut session = load_session(model_path, prefill_step_size, seed);+ let mut session = load_session(model_path, prefill_step_size, seed, speculator);let is_model_running = Arc::new(AtomicBool::new(false));let is_model_running_for_ctrlc = is_model_running.clone();diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rsindex a995223d7..8fc6ac6c6 100644--- a/crates/cli/src/main.rs+++ b/crates/cli/src/main.rs@@ -18,6 +18,9 @@ enum Commands {// Seed#[arg(long)]seed: Option<u64>,+ // Speculator+ #[arg(long)]+ speculator: Option<String>,/// Non-interactive mode: run a single message and exit#[arg(long, short)]message: Option<String>,@@ -48,9 +51,10 @@ fn main() {model_path,prefill_step_size,seed,+ speculator,message,}) => {- handle_run(model_path, 2048, prefill_step_size, seed, message);+ handle_run(model_path, 2048, prefill_step_size, seed, speculator, message);},Some(Commands::Serve {model_path,diff --git a/crates/cli/src/server/main.rs b/crates/cli/src/server/main.rsindex 96ab9f75a..9037511aa 100644--- a/crates/cli/src/server/main.rs+++ b/crates/cli/src/server/main.rs@@ -43,7 +43,7 @@ pub async fn run_server(println!("π Server will be available at: http://localhost:{}", config.port);println!("π Endpoints:\n POST /chat/completions - Chat completions API\n");- let session = load_session(model_path, prefill_step_size, None);+ let session = load_session(model_path, prefill_step_size, None, None);let state = SessionState {model_name,session_wrapper: std::sync::Arc::new(SessionWrapper::new(session)),diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rsindex e40a9e440..cc7b3d939 100644--- a/crates/cli/src/server/state.rs+++ b/crates/cli/src/server/state.rs@@ -6,7 +6,7 @@ use std::{use console::Style;use indicatif::{ProgressBar, ProgressStyle};use uzu::{- prelude::SamplingSeed,+ prelude::{SamplingSeed, SpeculatorConfig},session::{Session,config::{DecodingConfig, RunConfig},@@ -59,6 +59,7 @@ pub fn load_session(model_path: String,prefill_step_size: Option<usize>,seed: Option<u64>,+ speculator: Option<String>,) -> Session {let style_bold = Style::new().bold();@@ -77,10 +78,27 @@ pub fn load_session(prefill_step_size_config = PrefillStepSize::Default;}- let decoding_config =- DecodingConfig::default().with_prefill_step_size(prefill_step_size_config).with_sampling_seed(match seed {+ let decoding_config = DecodingConfig::default()+ .with_prefill_step_size(prefill_step_size_config)+ .with_sampling_seed(match seed {Some(seed) => SamplingSeed::Custom(seed),None => SamplingSeed::Default,+ })+ .with_speculator_config(match speculator {+ Some(speculator) => {+ let (speculator, number_of_speculated_tokens) =+ speculator.split_once(':').unwrap_or((&speculator, "1"));++ let number_of_speculated_tokens = number_of_speculated_tokens.parse().unwrap();++ let speculator = Arc::new(uzu::speculators::ngram_speculator::NGramSpeculator::load(speculator));++ SpeculatorConfig {+ number_of_speculated_tokens,+ speculator,+ }+ },+ None => SpeculatorConfig::default(),});let session = Session::new(model_path_buf, decoding_config).expect("Failed to create session");diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 8860c9265..de8de29ef 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -15,6 +15,9 @@ thiserror.workspace = truetokenizers.workspace = truempsgraph.workspace = truebytemuck.workspace = true+zerocopy = { workspace = true, features = ["derive"] }+xxhash-rust = { workspace = true, features = ["xxh3"] }+memmap2.workspace = truenum-traits.workspace = trueregex.workspace = truelibc.workspace = truediff --git a/crates/uzu/src/speculators/mod.rs b/crates/uzu/src/speculators/mod.rsindex 2503b207b..65e563276 100644--- a/crates/uzu/src/speculators/mod.rs+++ b/crates/uzu/src/speculators/mod.rs@@ -1,2 +1,3 @@pub mod empty_speculator;+pub mod ngram_speculator;pub mod speculator;diff --git a/crates/uzu/src/speculators/ngram_speculator.rs b/crates/uzu/src/speculators/ngram_speculator.rsnew file mode 100644index 000000000..5504f2547--- /dev/null+++ b/crates/uzu/src/speculators/ngram_speculator.rs@@ -0,0 +1,134 @@+use std::{+ collections::HashMap,+ iter::repeat_n,+ ops::{Deref, Range},+};++use bytemuck::cast_slice;+use xxhash_rust::xxh3::xxh3_64;+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};++use crate::speculators::speculator::Speculator;++#[cfg(not(target_endian = "little"))]+compile_error!("Only little endian is supported");++fn seqhash(+ seq: &[u32],+ size: u32,+) -> u32 {+ xxh3_64(cast_slice(seq)) as u32 % size+}++#[repr(C)]+#[derive(FromBytes, IntoBytes, KnownLayout, Immutable)]+struct NGramSpeculatorHeader {+ hashtable_size: u32,+ top_k: u32,+ ngram_n: u32,+ ngram_pad: u32,+}++pub struct NGramSpeculator<B: Deref<Target = [u8]> + Send + Sync> {+ bytes: B,+ ngram_keys_range: Range<usize>,+ ngram_values_range: Range<usize>,+ ngram_counts_range: Range<usize>,+}++impl<B: Deref<Target = [u8]> + Send + Sync> NGramSpeculator<B> {+ const HEADER_SIZE: usize = size_of::<NGramSpeculatorHeader>();++ pub fn new(bytes: B) -> Self {+ let mut off = 0;++ let header = NGramSpeculatorHeader::ref_from_bytes(&bytes[..Self::HEADER_SIZE]).unwrap();+ off += Self::HEADER_SIZE;++ let ngram_kv_size = 4 * header.top_k as usize * header.hashtable_size as usize;+ let ngram_c_size = 4 * header.hashtable_size as usize;++ let ngram_keys_range = off..(off + ngram_kv_size);+ off += ngram_kv_size;++ let ngram_values_range = off..(off + ngram_kv_size);+ off += ngram_kv_size;++ let ngram_counts_range = off..(off + ngram_c_size);+ off += ngram_c_size;++ assert_eq!(off, bytes.len());++ Self {+ bytes,+ ngram_keys_range,+ ngram_values_range,+ ngram_counts_range,+ }+ }++ #[inline]+ fn header(&self) -> &NGramSpeculatorHeader {+ NGramSpeculatorHeader::ref_from_bytes(&self.bytes[..Self::HEADER_SIZE]).unwrap()+ }++ #[inline]+ fn ngram_keys(&self) -> &[u32] {+ cast_slice(&self.bytes[self.ngram_keys_range.clone()])+ }++ #[inline]+ fn ngram_values(&self) -> &[f32] {+ cast_slice(&self.bytes[self.ngram_values_range.clone()])+ }++ #[inline]+ fn ngram_counts(&self) -> &[u32] {+ cast_slice(&self.bytes[self.ngram_counts_range.clone()])+ }++ fn _seq_slice<'a>(+ &'a self,+ seq: &[u64],+ ) -> (&'a [u32], &'a [f32], &'a u32) {+ let ngram_ctx = (self.header().ngram_n - 1) as usize;++ let padded_seq: Vec<u32> = if seq.len() >= ngram_ctx {+ seq[(seq.len() - ngram_ctx)..].iter().map(|&x| x as u32).collect()+ } else {+ repeat_n(self.header().ngram_pad, ngram_ctx - seq.len()).chain(seq.iter().map(|&x| x as u32)).collect()+ };++ assert!(padded_seq.len() == ngram_ctx);++ let seq_hash = seqhash(&padded_seq, self.header().hashtable_size);+ let idx_s = (seq_hash * self.header().top_k) as usize;+ let idx_e = (seq_hash * self.header().top_k + self.header().top_k) as usize;++ return (+ &self.ngram_keys()[idx_s..idx_e],+ &self.ngram_values()[idx_s..idx_e],+ &self.ngram_counts()[seq_hash as usize],+ );+ }+}++impl NGramSpeculator<memmap2::Mmap> {+ pub fn load(path: &str) -> Self {+ let file = std::fs::File::open(path).unwrap();+ let mmap = unsafe { memmap2::MmapOptions::default().map(&file).unwrap() };++ Self::new(mmap)+ }+}++impl<B: Deref<Target = [u8]> + Send + Sync> Speculator for NGramSpeculator<B> {+ fn speculate(+ &self,+ prefix: &[u64],+ ) -> HashMap<u64, f32> {+ let (ngram_keys, ngram_values, _ngram_counts) = self._seq_slice(prefix);++ ngram_keys.iter().copied().map(u64::from).zip(ngram_values.iter().copied()).collect()+ }+}