A high-performance inference engine for AI models
update xgrammar
eugene39531bddbad3
diff --git a/Cargo.lock b/Cargo.lockindex 9a3057ff6..e8daa0a0c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -986,17 +986,6 @@ dependencies = ["cc",]-[[package]]-name = "codespan-reporting"-version = "0.13.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"-dependencies = [- "serde",- "termcolor",- "unicode-width 0.2.2",-]-[[package]]name = "color_quant"version = "1.1.0"@@ -1374,72 +1363,6 @@ version = "1.0.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fb22e947478ccf9dc44d8922042c677a63fbb88f2cb468521d1145816e5087cb"-[[package]]-name = "cxx"-version = "1.0.194"-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"-dependencies = [- "cxx-build",- "cxx-cc",- "cxxbridge-cmd",- "cxxbridge-macro",- "foldhash",- "link-cplusplus",-]--[[package]]-name = "cxx-build"-version = "1.0.194"-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"-dependencies = [- "cc",- "codespan-reporting",- "cxx-cc",- "indexmap 2.14.0",- "proc-macro2",- "quote",- "scratch",- "syn 2.0.118",-]--[[package]]-name = "cxx-cc"-version = "1.0.194"-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"-dependencies = [- "cc",- "cxxbridge-flags",-]--[[package]]-name = "cxxbridge-cmd"-version = "1.0.194"-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"-dependencies = [- "clap",- "codespan-reporting",- "indexmap 2.14.0",- "proc-macro2",- "quote",- "syn 2.0.118",-]--[[package]]-name = "cxxbridge-flags"-version = "1.0.194"-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"--[[package]]-name = "cxxbridge-macro"-version = "1.0.194"-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"-dependencies = [- "indexmap 2.14.0",- "proc-macro2",- "quote",- "syn 2.0.118",-]-[[package]]name = "darling"version = "0.20.11"@@ -3337,15 +3260,6 @@ dependencies = ["bitflags 2.13.0",]-[[package]]-name = "link-cplusplus"-version = "1.0.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82"-dependencies = [- "cc",-]-[[package]]name = "linux-raw-sys"version = "0.12.1"@@ -5865,12 +5779,6 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"-[[package]]-name = "scratch"-version = "1.0.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2"-[[package]]name = "scroll"version = "0.12.0"@@ -6222,6 +6130,9 @@ name = "smallvec"version = "1.15.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"+dependencies = [+ "serde",+][[package]]name = "smartstring"@@ -6539,15 +6450,6 @@ dependencies = ["windows-sys 0.61.2",]-[[package]]-name = "termcolor"-version = "1.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"-dependencies = [- "winapi-util",-]-[[package]]name = "termina"version = "0.3.3"@@ -8386,16 +8288,18 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"[[package]]name = "xgrammar-rs"-version = "0.2.3"-source = "git+https://github.com/trymirai/xgrammar-rs.git?branch=main#04995a307d58f3b2cd015cd5bfd4d7bbeab98f3e"+version = "0.2.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1bc58f5791add99e3a2038ca5ca0f7ea6938521c58c678cf326a71340a10668"dependencies = [- "cc",- "cmake",- "cxx",- "cxx-build",+ "ahash",+ "getrandom 0.3.4",+ "rayon",+ "serde","serde_json",+ "smallvec",+ "thiserror 2.0.18","tokenizers",- "walkdir",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex a20e4328f..fa412fd6f 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -61,7 +61,7 @@ keisoku = { path = "crates/legacy/keisoku", default-features = false }kanka = { path = "crates/legacy/kanka", default-features = false }kiban = { path = "crates/legacy/kiban", default-features = false }uzu = { path = "crates/legacy/uzu", default-features = false }-xgrammar = { package = "xgrammar-rs", git = "https://github.com/trymirai/xgrammar-rs.git", branch = "main", features = ["tokenizers"] }+xgrammar = { package = "xgrammar-rs", version = "0.2.8", features = ["tokenizers"] }half = { version = "2.7", features = ["bytemuck", "num-traits"] }ndarray = "0.17"serde = { version = "1.0", features = ["derive"] }diff --git a/crates/uzu-engine/src/engine/language_model/grammar/data_type.rs b/crates/uzu-engine/src/engine/language_model/grammar/data_type.rsdeleted file mode 100644index 51c08c50b..000000000--- a/crates/uzu-engine/src/engine/language_model/grammar/data_type.rs+++ /dev/null@@ -1,28 +0,0 @@-use xgrammar::{DLDataType, DLDataTypeCode};--use crate::data_type::DataType;--trait DLDataTypeCodeProvider {- fn dl_data_type_code(self) -> DLDataTypeCode;-}--impl DLDataTypeCodeProvider for DataType {- fn dl_data_type_code(self) -> DLDataTypeCode {- match self {- DataType::BF16 => DLDataTypeCode::kDLBfloat,- DataType::F16 | DataType::F32 | DataType::F64 => DLDataTypeCode::kDLFloat,- DataType::I4 | DataType::I8 | DataType::I16 | DataType::I32 | DataType::I64 => DLDataTypeCode::kDLInt,- DataType::U4 | DataType::U8 | DataType::U16 | DataType::U32 | DataType::U64 => DLDataTypeCode::kDLUInt,- }- }-}--impl From<DataType> for DLDataType {- fn from(data_type: DataType) -> Self {- Self {- code: data_type.dl_data_type_code() as u8,- bits: data_type.size_in_bits() as u8,- lanes: 1,- }- }-}diff --git a/crates/uzu-engine/src/engine/language_model/grammar/error.rs b/crates/uzu-engine/src/engine/language_model/grammar/error.rsnew file mode 100644index 000000000..3dfbc5b56--- /dev/null+++ b/crates/uzu-engine/src/engine/language_model/grammar/error.rs@@ -0,0 +1,11 @@+use thiserror::Error;++#[derive(Debug, Error)]+pub enum GrammarError {+ #[error("Grammar rejected the token")]+ GrammarReject,+ #[error("XGrammar error: {0}")]+ XGrammar(String),+ #[error("XGrammar compilation error: {0}")]+ Compile(#[from] xgrammar::CompileError),+}diff --git a/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs b/crates/uzu-engine/src/engine/language_model/grammar/grammar.rsnew file mode 100644index 000000000..07fa38a9d--- /dev/null+++ b/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs@@ -0,0 +1,127 @@+use tokenizers::Tokenizer;+use xgrammar::{GrammarCompiler, GrammarMatcher, TokenizerInfo, get_bitmask_size};++use super::{GrammarConfig, GrammarError, engagement::GrammarEngagementState};++pub struct Grammar {+ bitmask_size: usize,+ matcher: GrammarMatcher,+ engagement_state: GrammarEngagementState,+}++impl Grammar {+ pub fn new(+ config: &GrammarConfig,+ tokenizer: &Tokenizer,+ trigger_token_sequence: Option<Vec<u64>>,+ stop_token_ids: Option<&[i32]>,+ ) -> Result<Self, GrammarError> {+ let tokenizer_info =+ TokenizerInfo::from_huggingface(tokenizer, None, stop_token_ids).map_err(GrammarError::XGrammar)?;++ let bitmask_size = get_bitmask_size(tokenizer_info.vocab_size()) as usize;+ let compiler = GrammarCompiler::with_defaults(tokenizer_info);++ let compiled = match config {+ GrammarConfig::JsonSchema {+ schema,+ any_whitespace,+ indent,+ separators,+ strict_mode,+ } => {+ let separators_ref = separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));+ compiler.compile_json_schema(schema, *any_whitespace, *indent, separators_ref, *strict_mode, None)?+ },+ GrammarConfig::Regex {+ pattern,+ print_converted_ebnf,+ } => {+ let compiled = compiler.compile_regex(pattern)?;+ if *print_converted_ebnf {+ println!("{}", compiled.grammar());+ }+ compiled+ },+ GrammarConfig::BuiltinJson => compiler.compile_builtin_json_grammar(),+ };+ let matcher = GrammarMatcher::from_compiled_grammar(&compiled, true);++ let engagement_state = match trigger_token_sequence.filter(|sequence| !sequence.is_empty()) {+ Some(trigger_sequence) => GrammarEngagementState::Triggered {+ trigger_sequence,+ match_history: Vec::new(),+ },+ None => GrammarEngagementState::Always,+ };++ Ok(Self {+ bitmask_size,+ matcher,+ engagement_state,+ })+ }+}++impl Grammar {+ pub fn next_bitmask(+ &mut self,+ bitmask: &mut [u32],+ ) -> bool {+ assert!(bitmask.len() >= self.bitmask_size); // NOTE: tokenizer vocab can be smaller than model vocab++ if self.engagement_state.is_engaged() {+ bitmask[self.bitmask_size..].fill(0);+ self.matcher+ .fill_next_token_bitmask(+ bytemuck::cast_slice_mut(&mut bitmask[..self.bitmask_size]),+ &[self.bitmask_size as i64],+ 0,+ )+ .expect("bitmask shape matches the tokenizer vocabulary")+ } else {+ bitmask.fill(u32::MAX);++ false+ }+ }++ pub fn accept_token(+ &mut self,+ token_id: u64,+ ) -> Result<(), GrammarError> {+ // A terminated matcher cannot advance and its bitmask only allows stop+ // tokens, which close generation without being part of the grammar, so+ // tokens sampled after termination must not be rejected.+ if self.engagement_state.is_engaged()+ && !self.matcher.is_terminated()+ && !self.matcher.accept_token(token_id as i32)+ {+ return Err(GrammarError::GrammarReject);+ }++ self.engagement_state.accept_token(token_id);+ Ok(())+ }++ pub fn rollback(+ &mut self,+ num_tokens: usize,+ ) {+ let num_grammar_tokens = self.engagement_state.rollback(num_tokens);++ if num_grammar_tokens > 0 {+ self.matcher+ .rollback(num_grammar_tokens as i32)+ .expect("cannot roll back more tokens than the grammar accepted");+ }+ }++ pub fn is_terminated(&self) -> bool {+ self.matcher.is_terminated()+ }+}++#[cfg(test)]+#[path = "../../../../unit/engine/language_model/grammar/grammar_test.rs"]+mod tests;diff --git a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs b/crates/uzu-engine/src/engine/language_model/grammar/mod.rsindex 59bcd5628..9eb75ec8a 100644--- a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs+++ b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs@@ -1,161 +1,8 @@-use thiserror::Error;-use tokenizers::Tokenizer;-use xgrammar::{- DLDataType, DLDevice, DLDeviceType, DLTensor, Grammar as XGrammarGrammar, GrammarCompiler, GrammarMatcher,- TokenizerInfo, c_void,-};--use crate::{data_type::DataType, engine::language_model::grammar::engagement::GrammarEngagementState};-mod config;-mod data_type;mod engagement;+mod error;+mod grammar;pub use config::GrammarConfig;--// TODO: jumpforward?--pub struct Grammar {- vocab_size: usize,- matcher: GrammarMatcher,- engagement_state: GrammarEngagementState,-}--#[derive(Debug, Error)]-pub enum GrammarError {- #[error("Grammar rejected the token")]- GrammarReject,- #[error("XGrammar error: {0}")]- XGrammar(String),-}--impl Grammar {- pub fn new(- config: &GrammarConfig,- tokenizer: &Tokenizer,- trigger_token_sequence: Option<Vec<u64>>,- stop_token_ids: Option<&[i32]>,- ) -> Result<Self, GrammarError> {- let tokenizer_info =- TokenizerInfo::from_huggingface(tokenizer, None, stop_token_ids).map_err(GrammarError::XGrammar)?;-- let vocab_size = tokenizer_info.vocab_size();-- let grammar = match config {- GrammarConfig::JsonSchema {- schema,- any_whitespace,- indent,- separators,- strict_mode,- } => {- let separators_ref = separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));- XGrammarGrammar::from_json_schema(- schema,- *any_whitespace,- *indent,- separators_ref,- *strict_mode,- None,- false,- false,- )- .map_err(GrammarError::XGrammar)?- },- GrammarConfig::Regex {- pattern,- print_converted_ebnf,- } => XGrammarGrammar::from_regex(pattern, *print_converted_ebnf).map_err(GrammarError::XGrammar)?,- GrammarConfig::BuiltinJson => XGrammarGrammar::builtin_json_grammar(),- };- let mut compiler = GrammarCompiler::new(&tokenizer_info, 8, true, -1).map_err(GrammarError::XGrammar)?;- let compiled = compiler.compile_grammar(&grammar).map_err(GrammarError::XGrammar)?;- let matcher = GrammarMatcher::new(&compiled, None, true, -1).map_err(GrammarError::XGrammar)?;-- let engagement_state = match trigger_token_sequence.filter(|sequence| !sequence.is_empty()) {- Some(trigger_sequence) => GrammarEngagementState::Triggered {- trigger_sequence,- match_history: Vec::new(),- },- None => GrammarEngagementState::Always,- };-- Ok(Self {- vocab_size,- matcher,- engagement_state,- })- }-}--impl Grammar {- pub fn next_bitmask(- &mut self,- bitmask: &mut [u32],- ) -> bool {- let vocab_size_in_u32s = self.vocab_size.div_ceil(DataType::U32.size_in_bits());- assert!(bitmask.len() >= vocab_size_in_u32s); // NOTE: tokenizer vocab can be smaller than model vocab-- if self.engagement_state.is_engaged() {- let mut shape_i64 = [vocab_size_in_u32s as i64];- let mut bitmask_tensor = unsafe {- DLTensor::new(- bitmask.as_mut_ptr() as *mut c_void,- DLDevice {- device_type: DLDeviceType::kDLCPU,- device_id: 0,- },- 1,- DLDataType {- code: 0,- bits: 32,- lanes: 1,- },- shape_i64.as_mut_ptr(),- core::ptr::null_mut(),- 0,- )- };-- bitmask[vocab_size_in_u32s..].fill(0);- self.matcher.fill_next_token_bitmask(&mut bitmask_tensor, 0, false)- } else {- bitmask.fill(u32::MAX);-- false- }- }-- pub fn accept_token(- &mut self,- token_id: u64,- ) -> Result<(), GrammarError> {- // A terminated matcher cannot advance and its bitmask only allows stop- // tokens, which close generation without being part of the grammar, so- // tokens sampled after termination must not be rejected.- if self.engagement_state.is_engaged()- && !self.matcher.is_terminated()- && !self.matcher.accept_token(token_id as i32)- {- return Err(GrammarError::GrammarReject);- }-- self.engagement_state.accept_token(token_id);- Ok(())- }-- pub fn rollback(- &mut self,- num_tokens: usize,- ) {- let num_grammar_tokens = self.engagement_state.rollback(num_tokens);-- if num_grammar_tokens > 0 {- self.matcher.rollback(num_grammar_tokens as i32);- }- }-- pub fn is_terminated(&self) -> bool {- self.matcher.is_terminated()- }-}+pub use error::GrammarError;+pub use grammar::Grammar;diff --git a/crates/uzu-engine/unit/engine/language_model/grammar/grammar_test.rs b/crates/uzu-engine/unit/engine/language_model/grammar/grammar_test.rsnew file mode 100644index 000000000..50ebac9fa--- /dev/null+++ b/crates/uzu-engine/unit/engine/language_model/grammar/grammar_test.rs@@ -0,0 +1,97 @@+use tokenizers::{Tokenizer, models::wordlevel::WordLevel};+use uzu_engine_macros::uzu_test;++use super::{Grammar, GrammarConfig, GrammarError};++fn grammar(+ config: &GrammarConfig,+ trigger: Option<Vec<u64>>,+) -> Grammar {+ let vocab = ["a", "b", "ab", "{", "}", "[", "]", "<eos>", "<unk>"]+ .into_iter()+ .enumerate()+ .map(|(id, token)| (token.to_string(), id as u32))+ .collect();+ let tokenizer = Tokenizer::new(WordLevel::builder().vocab(vocab).unk_token("<unk>".into()).build().unwrap());+ Grammar::new(config, &tokenizer, trigger, Some(&[7])).unwrap()+}++#[uzu_test]+fn regex_masks_reject_invalid_tokens_and_allow_stop_after_completion() {+ let mut grammar = grammar(&GrammarConfig::regex("ab".into(), false), None);+ let mut mask = [u32::MAX; 3];+ assert!(grammar.next_bitmask(&mut mask));+ assert_eq!(mask, [0b101, 0, 0]);+ assert!(matches!(grammar.accept_token(1), Err(GrammarError::GrammarReject)));+ grammar.accept_token(0).unwrap();+ assert!(grammar.next_bitmask(&mut mask));+ assert_eq!(mask, [0b10, 0, 0]);+ grammar.accept_token(1).unwrap();+ assert!(grammar.is_terminated());+ assert!(grammar.next_bitmask(&mut mask));+ assert_eq!(mask, [1 << 7, 0, 0]);+ grammar.accept_token(7).unwrap();+}++#[uzu_test]+fn rollback_restores_masks_and_termination() {+ let mut grammar = grammar(&GrammarConfig::regex("ab".into(), false), None);+ grammar.accept_token(0).unwrap();+ grammar.accept_token(1).unwrap();+ grammar.rollback(1);+ assert!(!grammar.is_terminated());+ let mut mask = [0];+ grammar.next_bitmask(&mut mask);+ assert_eq!(mask, [0b10]);+ grammar.rollback(1);+ grammar.next_bitmask(&mut mask);+ assert_eq!(mask, [0b101]);+}++#[uzu_test]+fn trigger_rollback_restores_unconstrained_sampling() {+ let mut grammar = grammar(&GrammarConfig::regex("ab".into(), false), Some(vec![5, 6]));+ let mut mask = [0; 2];+ assert!(!grammar.next_bitmask(&mut mask));+ assert_eq!(mask, [u32::MAX; 2]);+ grammar.accept_token(5).unwrap();+ grammar.accept_token(6).unwrap();+ grammar.accept_token(2).unwrap();+ assert!(grammar.is_terminated());+ grammar.rollback(2);+ assert!(!grammar.is_terminated());+ assert!(!grammar.next_bitmask(&mut mask));+ assert_eq!(mask, [u32::MAX; 2]);+ grammar.accept_token(6).unwrap();+ assert!(grammar.next_bitmask(&mut mask));+ assert_eq!(mask, [0b101, 0]);+}++#[uzu_test]+fn json_and_json_schema_accept_complete_objects() {+ for config in [+ GrammarConfig::builtin_json(),+ GrammarConfig::json_schema_simple(r#"{"type":"object","additionalProperties":false}"#.into()),+ ] {+ let mut grammar = grammar(&config, None);+ grammar.accept_token(3).unwrap();+ grammar.accept_token(4).unwrap();+ assert!(grammar.is_terminated());+ let mut mask = [u32::MAX; 2];+ assert!(grammar.next_bitmask(&mut mask));+ assert_eq!(mask, [1 << 7, 0]);+ }+}++#[uzu_test]+fn invalid_grammars_preserve_compilation_errors() {+ let tokenizer = Tokenizer::new(WordLevel::default());+ for config in [GrammarConfig::regex("[".into(), false), GrammarConfig::json_schema_simple("{".into())] {+ let error = match Grammar::new(&config, &tokenizer, None, None) {+ Err(error) => error,+ Ok(_) => panic!("invalid grammar compiled"),+ };+ assert!(matches!(error, GrammarError::Compile(_)));+ assert!(std::error::Error::source(&error).is_some());+ }+}diff --git a/deny.toml b/deny.tomlindex 812eaf805..23f2bfb7f 100644--- a/deny.toml+++ b/deny.toml@@ -26,6 +26,3 @@ allow = [[bans]multiple-versions = "allow"--[sources.allow-org]-github = ["trymirai"]