Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

ngram speculator (#243)

uuuvndc4b5d12f3f3
diff --git a/Cargo.lock b/Cargo.lock
index 5eca14321..9937dc855 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2235,6 +2235,15 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+[[package]]
+name = "memmap2"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "metal"
version = "0.32.0"
@@ -4355,6 +4364,7 @@ dependencies = [
"is_close",
"itertools 0.14.0",
"libc",
+ "memmap2",
"minijinja",
"minijinja-contrib",
"mpsgraph",
@@ -4380,6 +4390,8 @@ dependencies = [
"tokio",
"walkdir",
"xgrammar-rs",
+ "xxhash-rust",
+ "zerocopy",
]
[[package]]
@@ -5089,6 +5101,12 @@ dependencies = [
"walkdir",
]
+[[package]]
+name = "xxhash-rust"
+version = "0.8.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
+
[[package]]
name = "yansi"
version = "1.0.1"
diff --git a/Cargo.toml b/Cargo.toml
index da4244059..ae3d2dab6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,6 +27,9 @@ objc2-foundation = "0.3"
metal = { package = "mtl-rs", version = "0.1.6" }
block2 = "0.6"
bytemuck = "1.23"
+zerocopy = "0.8.27"
+xxhash-rust = "0.8.15"
+memmap2 = "0.9.8"
num-traits = "0.2"
regex = "1.12"
libc = "0.2"
diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rs
index feee0c1d5..491117fc5 100644
--- a/crates/cli/src/handlers/run.rs
+++ b/crates/cli/src/handlers/run.rs
@@ -34,9 +34,10 @@ pub fn handle_run(
tokens_limit: usize,
prefill_step_size: Option<usize>,
seed: Option<u64>,
+ speculator: Option<String>,
mut message: Option<String>,
) {
- let mut session = load_session(model_path, prefill_step_size, seed);
+ let mut session = load_session(model_path, prefill_step_size, seed, speculator);
let is_model_running = Arc::new(AtomicBool::new(false));
let is_model_running_for_ctrlc = is_model_running.clone();
diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs
index a995223d7..8fc6ac6c6 100644
--- a/crates/cli/src/main.rs
+++ b/crates/cli/src/main.rs
@@ -18,6 +18,9 @@ enum Commands {
// Seed
#[arg(long)]
seed: Option<u64>,
+ // Speculator
+ #[arg(long)]
+ speculator: Option<String>,
/// Non-interactive mode: run a single message and exit
#[arg(long, short)]
message: Option<String>,
@@ -48,9 +51,10 @@ fn main() {
model_path,
prefill_step_size,
seed,
+ speculator,
message,
}) => {
- handle_run(model_path, 2048, prefill_step_size, seed, message);
+ handle_run(model_path, 2048, prefill_step_size, seed, speculator, message);
},
Some(Commands::Serve {
model_path,
diff --git a/crates/cli/src/server/main.rs b/crates/cli/src/server/main.rs
index 96ab9f75a..9037511aa 100644
--- a/crates/cli/src/server/main.rs
+++ b/crates/cli/src/server/main.rs
@@ -43,7 +43,7 @@ pub async fn run_server(
println!("🌐 Server will be available at: http://localhost:{}", config.port);
println!("πŸ“ Endpoints:\n POST /chat/completions - Chat completions API\n");
- let session = load_session(model_path, prefill_step_size, None);
+ let session = load_session(model_path, prefill_step_size, None, None);
let state = SessionState {
model_name,
session_wrapper: std::sync::Arc::new(SessionWrapper::new(session)),
diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rs
index e40a9e440..cc7b3d939 100644
--- a/crates/cli/src/server/state.rs
+++ b/crates/cli/src/server/state.rs
@@ -6,7 +6,7 @@ use std::{
use console::Style;
use indicatif::{ProgressBar, ProgressStyle};
use uzu::{
- prelude::SamplingSeed,
+ prelude::{SamplingSeed, SpeculatorConfig},
session::{
Session,
config::{DecodingConfig, RunConfig},
@@ -59,6 +59,7 @@ pub fn load_session(
model_path: String,
prefill_step_size: Option<usize>,
seed: Option<u64>,
+ speculator: Option<String>,
) -> Session {
let style_bold = Style::new().bold();
@@ -77,10 +78,27 @@ pub fn load_session(
prefill_step_size_config = PrefillStepSize::Default;
}
- let decoding_config =
- DecodingConfig::default().with_prefill_step_size(prefill_step_size_config).with_sampling_seed(match seed {
+ let decoding_config = DecodingConfig::default()
+ .with_prefill_step_size(prefill_step_size_config)
+ .with_sampling_seed(match seed {
Some(seed) => SamplingSeed::Custom(seed),
None => SamplingSeed::Default,
+ })
+ .with_speculator_config(match speculator {
+ Some(speculator) => {
+ let (speculator, number_of_speculated_tokens) =
+ speculator.split_once(':').unwrap_or((&speculator, "1"));
+
+ let number_of_speculated_tokens = number_of_speculated_tokens.parse().unwrap();
+
+ let speculator = Arc::new(uzu::speculators::ngram_speculator::NGramSpeculator::load(speculator));
+
+ SpeculatorConfig {
+ number_of_speculated_tokens,
+ speculator,
+ }
+ },
+ None => SpeculatorConfig::default(),
});
let session = Session::new(model_path_buf, decoding_config).expect("Failed to create session");
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 8860c9265..de8de29ef 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -15,6 +15,9 @@ thiserror.workspace = true
tokenizers.workspace = true
mpsgraph.workspace = true
bytemuck.workspace = true
+zerocopy = { workspace = true, features = ["derive"] }
+xxhash-rust = { workspace = true, features = ["xxh3"] }
+memmap2.workspace = true
num-traits.workspace = true
regex.workspace = true
libc.workspace = true
diff --git a/crates/uzu/src/speculators/mod.rs b/crates/uzu/src/speculators/mod.rs
index 2503b207b..65e563276 100644
--- a/crates/uzu/src/speculators/mod.rs
+++ b/crates/uzu/src/speculators/mod.rs
@@ -1,2 +1,3 @@
pub mod empty_speculator;
+pub mod ngram_speculator;
pub mod speculator;
diff --git a/crates/uzu/src/speculators/ngram_speculator.rs b/crates/uzu/src/speculators/ngram_speculator.rs
new file mode 100644
index 000000000..5504f2547
--- /dev/null
+++ b/crates/uzu/src/speculators/ngram_speculator.rs
@@ -0,0 +1,134 @@
+use std::{
+ collections::HashMap,
+ iter::repeat_n,
+ ops::{Deref, Range},
+};
+
+use bytemuck::cast_slice;
+use xxhash_rust::xxh3::xxh3_64;
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
+
+use crate::speculators::speculator::Speculator;
+
+#[cfg(not(target_endian = "little"))]
+compile_error!("Only little endian is supported");
+
+fn seqhash(
+ seq: &[u32],
+ size: u32,
+) -> u32 {
+ xxh3_64(cast_slice(seq)) as u32 % size
+}
+
+#[repr(C)]
+#[derive(FromBytes, IntoBytes, KnownLayout, Immutable)]
+struct NGramSpeculatorHeader {
+ hashtable_size: u32,
+ top_k: u32,
+ ngram_n: u32,
+ ngram_pad: u32,
+}
+
+pub struct NGramSpeculator<B: Deref<Target = [u8]> + Send + Sync> {
+ bytes: B,
+ ngram_keys_range: Range<usize>,
+ ngram_values_range: Range<usize>,
+ ngram_counts_range: Range<usize>,
+}
+
+impl<B: Deref<Target = [u8]> + Send + Sync> NGramSpeculator<B> {
+ const HEADER_SIZE: usize = size_of::<NGramSpeculatorHeader>();
+
+ pub fn new(bytes: B) -> Self {
+ let mut off = 0;
+
+ let header = NGramSpeculatorHeader::ref_from_bytes(&bytes[..Self::HEADER_SIZE]).unwrap();
+ off += Self::HEADER_SIZE;
+
+ let ngram_kv_size = 4 * header.top_k as usize * header.hashtable_size as usize;
+ let ngram_c_size = 4 * header.hashtable_size as usize;
+
+ let ngram_keys_range = off..(off + ngram_kv_size);
+ off += ngram_kv_size;
+
+ let ngram_values_range = off..(off + ngram_kv_size);
+ off += ngram_kv_size;
+
+ let ngram_counts_range = off..(off + ngram_c_size);
+ off += ngram_c_size;
+
+ assert_eq!(off, bytes.len());
+
+ Self {
+ bytes,
+ ngram_keys_range,
+ ngram_values_range,
+ ngram_counts_range,
+ }
+ }
+
+ #[inline]
+ fn header(&self) -> &NGramSpeculatorHeader {
+ NGramSpeculatorHeader::ref_from_bytes(&self.bytes[..Self::HEADER_SIZE]).unwrap()
+ }
+
+ #[inline]
+ fn ngram_keys(&self) -> &[u32] {
+ cast_slice(&self.bytes[self.ngram_keys_range.clone()])
+ }
+
+ #[inline]
+ fn ngram_values(&self) -> &[f32] {
+ cast_slice(&self.bytes[self.ngram_values_range.clone()])
+ }
+
+ #[inline]
+ fn ngram_counts(&self) -> &[u32] {
+ cast_slice(&self.bytes[self.ngram_counts_range.clone()])
+ }
+
+ fn _seq_slice<'a>(
+ &'a self,
+ seq: &[u64],
+ ) -> (&'a [u32], &'a [f32], &'a u32) {
+ let ngram_ctx = (self.header().ngram_n - 1) as usize;
+
+ let padded_seq: Vec<u32> = if seq.len() >= ngram_ctx {
+ seq[(seq.len() - ngram_ctx)..].iter().map(|&x| x as u32).collect()
+ } else {
+ repeat_n(self.header().ngram_pad, ngram_ctx - seq.len()).chain(seq.iter().map(|&x| x as u32)).collect()
+ };
+
+ assert!(padded_seq.len() == ngram_ctx);
+
+ let seq_hash = seqhash(&padded_seq, self.header().hashtable_size);
+ let idx_s = (seq_hash * self.header().top_k) as usize;
+ let idx_e = (seq_hash * self.header().top_k + self.header().top_k) as usize;
+
+ return (
+ &self.ngram_keys()[idx_s..idx_e],
+ &self.ngram_values()[idx_s..idx_e],
+ &self.ngram_counts()[seq_hash as usize],
+ );
+ }
+}
+
+impl NGramSpeculator<memmap2::Mmap> {
+ pub fn load(path: &str) -> Self {
+ let file = std::fs::File::open(path).unwrap();
+ let mmap = unsafe { memmap2::MmapOptions::default().map(&file).unwrap() };
+
+ Self::new(mmap)
+ }
+}
+
+impl<B: Deref<Target = [u8]> + Send + Sync> Speculator for NGramSpeculator<B> {
+ fn speculate(
+ &self,
+ prefix: &[u64],
+ ) -> HashMap<u64, f32> {
+ let (ngram_keys, ngram_values, _ngram_counts) = self._seq_slice(prefix);
+
+ ngram_keys.iter().copied().map(u64::from).zip(ngram_values.iter().copied()).collect()
+ }
+}