Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

update xgrammar

eugene39531bddbad3
diff --git a/Cargo.lock b/Cargo.lock
index 9a3057ff6..e8daa0a0c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -986,17 +986,6 @@ dependencies = [
"cc",
]
-[[package]]
-name = "codespan-reporting"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"
-dependencies = [
- "serde",
- "termcolor",
- "unicode-width 0.2.2",
-]
-
[[package]]
name = "color_quant"
version = "1.1.0"
@@ -1374,72 +1363,6 @@ version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb22e947478ccf9dc44d8922042c677a63fbb88f2cb468521d1145816e5087cb"
-[[package]]
-name = "cxx"
-version = "1.0.194"
-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"
-dependencies = [
- "cxx-build",
- "cxx-cc",
- "cxxbridge-cmd",
- "cxxbridge-macro",
- "foldhash",
- "link-cplusplus",
-]
-
-[[package]]
-name = "cxx-build"
-version = "1.0.194"
-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"
-dependencies = [
- "cc",
- "codespan-reporting",
- "cxx-cc",
- "indexmap 2.14.0",
- "proc-macro2",
- "quote",
- "scratch",
- "syn 2.0.118",
-]
-
-[[package]]
-name = "cxx-cc"
-version = "1.0.194"
-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"
-dependencies = [
- "cc",
- "cxxbridge-flags",
-]
-
-[[package]]
-name = "cxxbridge-cmd"
-version = "1.0.194"
-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"
-dependencies = [
- "clap",
- "codespan-reporting",
- "indexmap 2.14.0",
- "proc-macro2",
- "quote",
- "syn 2.0.118",
-]
-
-[[package]]
-name = "cxxbridge-flags"
-version = "1.0.194"
-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"
-
-[[package]]
-name = "cxxbridge-macro"
-version = "1.0.194"
-source = "git+https://github.com/trymirai/cxx.git?branch=master#aefe85bdd845e391c68e5400e19bdb437502859d"
-dependencies = [
- "indexmap 2.14.0",
- "proc-macro2",
- "quote",
- "syn 2.0.118",
-]
-
[[package]]
name = "darling"
version = "0.20.11"
@@ -3337,15 +3260,6 @@ dependencies = [
"bitflags 2.13.0",
]
-[[package]]
-name = "link-cplusplus"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82"
-dependencies = [
- "cc",
-]
-
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@@ -5865,12 +5779,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-[[package]]
-name = "scratch"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2"
-
[[package]]
name = "scroll"
version = "0.12.0"
@@ -6222,6 +6130,9 @@ name = "smallvec"
version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
+dependencies = [
+ "serde",
+]
[[package]]
name = "smartstring"
@@ -6539,15 +6450,6 @@ dependencies = [
"windows-sys 0.61.2",
]
-[[package]]
-name = "termcolor"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
-dependencies = [
- "winapi-util",
-]
-
[[package]]
name = "termina"
version = "0.3.3"
@@ -8386,16 +8288,18 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
[[package]]
name = "xgrammar-rs"
-version = "0.2.3"
-source = "git+https://github.com/trymirai/xgrammar-rs.git?branch=main#04995a307d58f3b2cd015cd5bfd4d7bbeab98f3e"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1bc58f5791add99e3a2038ca5ca0f7ea6938521c58c678cf326a71340a10668"
dependencies = [
- "cc",
- "cmake",
- "cxx",
- "cxx-build",
+ "ahash",
+ "getrandom 0.3.4",
+ "rayon",
+ "serde",
"serde_json",
+ "smallvec",
+ "thiserror 2.0.18",
"tokenizers",
- "walkdir",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index a20e4328f..fa412fd6f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -61,7 +61,7 @@ keisoku = { path = "crates/legacy/keisoku", default-features = false }
kanka = { path = "crates/legacy/kanka", default-features = false }
kiban = { path = "crates/legacy/kiban", default-features = false }
uzu = { path = "crates/legacy/uzu", default-features = false }
-xgrammar = { package = "xgrammar-rs", git = "https://github.com/trymirai/xgrammar-rs.git", branch = "main", features = ["tokenizers"] }
+xgrammar = { package = "xgrammar-rs", version = "0.2.8", features = ["tokenizers"] }
half = { version = "2.7", features = ["bytemuck", "num-traits"] }
ndarray = "0.17"
serde = { version = "1.0", features = ["derive"] }
diff --git a/crates/uzu-engine/src/engine/language_model/grammar/data_type.rs b/crates/uzu-engine/src/engine/language_model/grammar/data_type.rs
deleted file mode 100644
index 51c08c50b..000000000
--- a/crates/uzu-engine/src/engine/language_model/grammar/data_type.rs
+++ /dev/null
@@ -1,28 +0,0 @@
-use xgrammar::{DLDataType, DLDataTypeCode};
-
-use crate::data_type::DataType;
-
-trait DLDataTypeCodeProvider {
- fn dl_data_type_code(self) -> DLDataTypeCode;
-}
-
-impl DLDataTypeCodeProvider for DataType {
- fn dl_data_type_code(self) -> DLDataTypeCode {
- match self {
- DataType::BF16 => DLDataTypeCode::kDLBfloat,
- DataType::F16 | DataType::F32 | DataType::F64 => DLDataTypeCode::kDLFloat,
- DataType::I4 | DataType::I8 | DataType::I16 | DataType::I32 | DataType::I64 => DLDataTypeCode::kDLInt,
- DataType::U4 | DataType::U8 | DataType::U16 | DataType::U32 | DataType::U64 => DLDataTypeCode::kDLUInt,
- }
- }
-}
-
-impl From<DataType> for DLDataType {
- fn from(data_type: DataType) -> Self {
- Self {
- code: data_type.dl_data_type_code() as u8,
- bits: data_type.size_in_bits() as u8,
- lanes: 1,
- }
- }
-}
diff --git a/crates/uzu-engine/src/engine/language_model/grammar/error.rs b/crates/uzu-engine/src/engine/language_model/grammar/error.rs
new file mode 100644
index 000000000..3dfbc5b56
--- /dev/null
+++ b/crates/uzu-engine/src/engine/language_model/grammar/error.rs
@@ -0,0 +1,11 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum GrammarError {
+ #[error("Grammar rejected the token")]
+ GrammarReject,
+ #[error("XGrammar error: {0}")]
+ XGrammar(String),
+ #[error("XGrammar compilation error: {0}")]
+ Compile(#[from] xgrammar::CompileError),
+}
diff --git a/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs b/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs
new file mode 100644
index 000000000..07fa38a9d
--- /dev/null
+++ b/crates/uzu-engine/src/engine/language_model/grammar/grammar.rs
@@ -0,0 +1,127 @@
+use tokenizers::Tokenizer;
+use xgrammar::{GrammarCompiler, GrammarMatcher, TokenizerInfo, get_bitmask_size};
+
+use super::{GrammarConfig, GrammarError, engagement::GrammarEngagementState};
+
+pub struct Grammar {
+ bitmask_size: usize,
+ matcher: GrammarMatcher,
+ engagement_state: GrammarEngagementState,
+}
+
+impl Grammar {
+ pub fn new(
+ config: &GrammarConfig,
+ tokenizer: &Tokenizer,
+ trigger_token_sequence: Option<Vec<u64>>,
+ stop_token_ids: Option<&[i32]>,
+ ) -> Result<Self, GrammarError> {
+ let tokenizer_info =
+ TokenizerInfo::from_huggingface(tokenizer, None, stop_token_ids).map_err(GrammarError::XGrammar)?;
+
+ let bitmask_size = get_bitmask_size(tokenizer_info.vocab_size()) as usize;
+ let compiler = GrammarCompiler::with_defaults(tokenizer_info);
+
+ let compiled = match config {
+ GrammarConfig::JsonSchema {
+ schema,
+ any_whitespace,
+ indent,
+ separators,
+ strict_mode,
+ } => {
+ let separators_ref = separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));
+ compiler.compile_json_schema(schema, *any_whitespace, *indent, separators_ref, *strict_mode, None)?
+ },
+ GrammarConfig::Regex {
+ pattern,
+ print_converted_ebnf,
+ } => {
+ let compiled = compiler.compile_regex(pattern)?;
+ if *print_converted_ebnf {
+ println!("{}", compiled.grammar());
+ }
+ compiled
+ },
+ GrammarConfig::BuiltinJson => compiler.compile_builtin_json_grammar(),
+ };
+ let matcher = GrammarMatcher::from_compiled_grammar(&compiled, true);
+
+ let engagement_state = match trigger_token_sequence.filter(|sequence| !sequence.is_empty()) {
+ Some(trigger_sequence) => GrammarEngagementState::Triggered {
+ trigger_sequence,
+ match_history: Vec::new(),
+ },
+ None => GrammarEngagementState::Always,
+ };
+
+ Ok(Self {
+ bitmask_size,
+ matcher,
+ engagement_state,
+ })
+ }
+}
+
+impl Grammar {
+ pub fn next_bitmask(
+ &mut self,
+ bitmask: &mut [u32],
+ ) -> bool {
+ assert!(bitmask.len() >= self.bitmask_size); // NOTE: tokenizer vocab can be smaller than model vocab
+
+ if self.engagement_state.is_engaged() {
+ bitmask[self.bitmask_size..].fill(0);
+ self.matcher
+ .fill_next_token_bitmask(
+ bytemuck::cast_slice_mut(&mut bitmask[..self.bitmask_size]),
+ &[self.bitmask_size as i64],
+ 0,
+ )
+ .expect("bitmask shape matches the tokenizer vocabulary")
+ } else {
+ bitmask.fill(u32::MAX);
+
+ false
+ }
+ }
+
+ pub fn accept_token(
+ &mut self,
+ token_id: u64,
+ ) -> Result<(), GrammarError> {
+ // A terminated matcher cannot advance and its bitmask only allows stop
+ // tokens, which close generation without being part of the grammar, so
+ // tokens sampled after termination must not be rejected.
+ if self.engagement_state.is_engaged()
+ && !self.matcher.is_terminated()
+ && !self.matcher.accept_token(token_id as i32)
+ {
+ return Err(GrammarError::GrammarReject);
+ }
+
+ self.engagement_state.accept_token(token_id);
+ Ok(())
+ }
+
+ pub fn rollback(
+ &mut self,
+ num_tokens: usize,
+ ) {
+ let num_grammar_tokens = self.engagement_state.rollback(num_tokens);
+
+ if num_grammar_tokens > 0 {
+ self.matcher
+ .rollback(num_grammar_tokens as i32)
+ .expect("cannot roll back more tokens than the grammar accepted");
+ }
+ }
+
+ pub fn is_terminated(&self) -> bool {
+ self.matcher.is_terminated()
+ }
+}
+
+#[cfg(test)]
+#[path = "../../../../unit/engine/language_model/grammar/grammar_test.rs"]
+mod tests;
diff --git a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
index 59bcd5628..9eb75ec8a 100644
--- a/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
+++ b/crates/uzu-engine/src/engine/language_model/grammar/mod.rs
@@ -1,161 +1,8 @@
-use thiserror::Error;
-use tokenizers::Tokenizer;
-use xgrammar::{
- DLDataType, DLDevice, DLDeviceType, DLTensor, Grammar as XGrammarGrammar, GrammarCompiler, GrammarMatcher,
- TokenizerInfo, c_void,
-};
-
-use crate::{data_type::DataType, engine::language_model::grammar::engagement::GrammarEngagementState};
-
mod config;
-mod data_type;
mod engagement;
+mod error;
+mod grammar;
pub use config::GrammarConfig;
-
-// TODO: jumpforward?
-
-pub struct Grammar {
- vocab_size: usize,
- matcher: GrammarMatcher,
- engagement_state: GrammarEngagementState,
-}
-
-#[derive(Debug, Error)]
-pub enum GrammarError {
- #[error("Grammar rejected the token")]
- GrammarReject,
- #[error("XGrammar error: {0}")]
- XGrammar(String),
-}
-
-impl Grammar {
- pub fn new(
- config: &GrammarConfig,
- tokenizer: &Tokenizer,
- trigger_token_sequence: Option<Vec<u64>>,
- stop_token_ids: Option<&[i32]>,
- ) -> Result<Self, GrammarError> {
- let tokenizer_info =
- TokenizerInfo::from_huggingface(tokenizer, None, stop_token_ids).map_err(GrammarError::XGrammar)?;
-
- let vocab_size = tokenizer_info.vocab_size();
-
- let grammar = match config {
- GrammarConfig::JsonSchema {
- schema,
- any_whitespace,
- indent,
- separators,
- strict_mode,
- } => {
- let separators_ref = separators.as_ref().map(|(a, b)| (a.as_str(), b.as_str()));
- XGrammarGrammar::from_json_schema(
- schema,
- *any_whitespace,
- *indent,
- separators_ref,
- *strict_mode,
- None,
- false,
- false,
- )
- .map_err(GrammarError::XGrammar)?
- },
- GrammarConfig::Regex {
- pattern,
- print_converted_ebnf,
- } => XGrammarGrammar::from_regex(pattern, *print_converted_ebnf).map_err(GrammarError::XGrammar)?,
- GrammarConfig::BuiltinJson => XGrammarGrammar::builtin_json_grammar(),
- };
- let mut compiler = GrammarCompiler::new(&tokenizer_info, 8, true, -1).map_err(GrammarError::XGrammar)?;
- let compiled = compiler.compile_grammar(&grammar).map_err(GrammarError::XGrammar)?;
- let matcher = GrammarMatcher::new(&compiled, None, true, -1).map_err(GrammarError::XGrammar)?;
-
- let engagement_state = match trigger_token_sequence.filter(|sequence| !sequence.is_empty()) {
- Some(trigger_sequence) => GrammarEngagementState::Triggered {
- trigger_sequence,
- match_history: Vec::new(),
- },
- None => GrammarEngagementState::Always,
- };
-
- Ok(Self {
- vocab_size,
- matcher,
- engagement_state,
- })
- }
-}
-
-impl Grammar {
- pub fn next_bitmask(
- &mut self,
- bitmask: &mut [u32],
- ) -> bool {
- let vocab_size_in_u32s = self.vocab_size.div_ceil(DataType::U32.size_in_bits());
- assert!(bitmask.len() >= vocab_size_in_u32s); // NOTE: tokenizer vocab can be smaller than model vocab
-
- if self.engagement_state.is_engaged() {
- let mut shape_i64 = [vocab_size_in_u32s as i64];
- let mut bitmask_tensor = unsafe {
- DLTensor::new(
- bitmask.as_mut_ptr() as *mut c_void,
- DLDevice {
- device_type: DLDeviceType::kDLCPU,
- device_id: 0,
- },
- 1,
- DLDataType {
- code: 0,
- bits: 32,
- lanes: 1,
- },
- shape_i64.as_mut_ptr(),
- core::ptr::null_mut(),
- 0,
- )
- };
-
- bitmask[vocab_size_in_u32s..].fill(0);
- self.matcher.fill_next_token_bitmask(&mut bitmask_tensor, 0, false)
- } else {
- bitmask.fill(u32::MAX);
-
- false
- }
- }
-
- pub fn accept_token(
- &mut self,
- token_id: u64,
- ) -> Result<(), GrammarError> {
- // A terminated matcher cannot advance and its bitmask only allows stop
- // tokens, which close generation without being part of the grammar, so
- // tokens sampled after termination must not be rejected.
- if self.engagement_state.is_engaged()
- && !self.matcher.is_terminated()
- && !self.matcher.accept_token(token_id as i32)
- {
- return Err(GrammarError::GrammarReject);
- }
-
- self.engagement_state.accept_token(token_id);
- Ok(())
- }
-
- pub fn rollback(
- &mut self,
- num_tokens: usize,
- ) {
- let num_grammar_tokens = self.engagement_state.rollback(num_tokens);
-
- if num_grammar_tokens > 0 {
- self.matcher.rollback(num_grammar_tokens as i32);
- }
- }
-
- pub fn is_terminated(&self) -> bool {
- self.matcher.is_terminated()
- }
-}
+pub use error::GrammarError;
+pub use grammar::Grammar;
diff --git a/crates/uzu-engine/unit/engine/language_model/grammar/grammar_test.rs b/crates/uzu-engine/unit/engine/language_model/grammar/grammar_test.rs
new file mode 100644
index 000000000..50ebac9fa
--- /dev/null
+++ b/crates/uzu-engine/unit/engine/language_model/grammar/grammar_test.rs
@@ -0,0 +1,97 @@
+use tokenizers::{Tokenizer, models::wordlevel::WordLevel};
+use uzu_engine_macros::uzu_test;
+
+use super::{Grammar, GrammarConfig, GrammarError};
+
+fn grammar(
+ config: &GrammarConfig,
+ trigger: Option<Vec<u64>>,
+) -> Grammar {
+ let vocab = ["a", "b", "ab", "{", "}", "[", "]", "<eos>", "<unk>"]
+ .into_iter()
+ .enumerate()
+ .map(|(id, token)| (token.to_string(), id as u32))
+ .collect();
+ let tokenizer = Tokenizer::new(WordLevel::builder().vocab(vocab).unk_token("<unk>".into()).build().unwrap());
+ Grammar::new(config, &tokenizer, trigger, Some(&[7])).unwrap()
+}
+
+#[uzu_test]
+fn regex_masks_reject_invalid_tokens_and_allow_stop_after_completion() {
+ let mut grammar = grammar(&GrammarConfig::regex("ab".into(), false), None);
+ let mut mask = [u32::MAX; 3];
+ assert!(grammar.next_bitmask(&mut mask));
+ assert_eq!(mask, [0b101, 0, 0]);
+ assert!(matches!(grammar.accept_token(1), Err(GrammarError::GrammarReject)));
+ grammar.accept_token(0).unwrap();
+ assert!(grammar.next_bitmask(&mut mask));
+ assert_eq!(mask, [0b10, 0, 0]);
+ grammar.accept_token(1).unwrap();
+ assert!(grammar.is_terminated());
+ assert!(grammar.next_bitmask(&mut mask));
+ assert_eq!(mask, [1 << 7, 0, 0]);
+ grammar.accept_token(7).unwrap();
+}
+
+#[uzu_test]
+fn rollback_restores_masks_and_termination() {
+ let mut grammar = grammar(&GrammarConfig::regex("ab".into(), false), None);
+ grammar.accept_token(0).unwrap();
+ grammar.accept_token(1).unwrap();
+ grammar.rollback(1);
+ assert!(!grammar.is_terminated());
+ let mut mask = [0];
+ grammar.next_bitmask(&mut mask);
+ assert_eq!(mask, [0b10]);
+ grammar.rollback(1);
+ grammar.next_bitmask(&mut mask);
+ assert_eq!(mask, [0b101]);
+}
+
+#[uzu_test]
+fn trigger_rollback_restores_unconstrained_sampling() {
+ let mut grammar = grammar(&GrammarConfig::regex("ab".into(), false), Some(vec![5, 6]));
+ let mut mask = [0; 2];
+ assert!(!grammar.next_bitmask(&mut mask));
+ assert_eq!(mask, [u32::MAX; 2]);
+ grammar.accept_token(5).unwrap();
+ grammar.accept_token(6).unwrap();
+ grammar.accept_token(2).unwrap();
+ assert!(grammar.is_terminated());
+ grammar.rollback(2);
+ assert!(!grammar.is_terminated());
+ assert!(!grammar.next_bitmask(&mut mask));
+ assert_eq!(mask, [u32::MAX; 2]);
+ grammar.accept_token(6).unwrap();
+ assert!(grammar.next_bitmask(&mut mask));
+ assert_eq!(mask, [0b101, 0]);
+}
+
+#[uzu_test]
+fn json_and_json_schema_accept_complete_objects() {
+ for config in [
+ GrammarConfig::builtin_json(),
+ GrammarConfig::json_schema_simple(r#"{"type":"object","additionalProperties":false}"#.into()),
+ ] {
+ let mut grammar = grammar(&config, None);
+ grammar.accept_token(3).unwrap();
+ grammar.accept_token(4).unwrap();
+ assert!(grammar.is_terminated());
+ let mut mask = [u32::MAX; 2];
+ assert!(grammar.next_bitmask(&mut mask));
+ assert_eq!(mask, [1 << 7, 0]);
+ }
+}
+
+#[uzu_test]
+fn invalid_grammars_preserve_compilation_errors() {
+ let tokenizer = Tokenizer::new(WordLevel::default());
+ for config in [GrammarConfig::regex("[".into(), false), GrammarConfig::json_schema_simple("{".into())] {
+ let error = match Grammar::new(&config, &tokenizer, None, None) {
+ Err(error) => error,
+ Ok(_) => panic!("invalid grammar compiled"),
+ };
+ assert!(matches!(error, GrammarError::Compile(_)));
+ assert!(std::error::Error::source(&error).is_some());
+ }
+}
diff --git a/deny.toml b/deny.toml
index 812eaf805..23f2bfb7f 100644
--- a/deny.toml
+++ b/deny.toml
@@ -26,6 +26,3 @@ allow = [
[bans]
multiple-versions = "allow"
-
-[sources.allow-org]
-github = ["trymirai"]