Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(rust): add HuggingFace tokenizer binding

kevin-gatimuf8dc8113d195
diff --git a/laya-rust/src/tokenization/hf_tokenizer.rs b/laya-rust/src/tokenization/hf_tokenizer.rs
new file mode 100644
index 00000000..9f91fce1
--- /dev/null
+++ b/laya-rust/src/tokenization/hf_tokenizer.rs
@@ -0,0 +1,485 @@
+//! [`LayaTokenizer`] backed by the `tokenizers` crate — the same HuggingFace Rust `tokenizers`
+//! library the Python SDK uses.
+//!
+//! ## No `tokenizer.nopost.json` strip-and-cache step needed
+//!
+//! `tokenizer.json` ships with a `TemplateProcessing` post-processor that wraps every encode in
+//! special tokens (`<bos> A <eos>` for multilingual, `[CLS] A [SEP]` for the ModernBERT
+//! checkpoints). Python bypasses it by calling the tokenizer with `add_special_tokens=False`. The
+//! `tokenizers` Rust crate used here exposes `Tokenizer::encode(text, add_special_tokens: false)`
+//! directly — exactly what Python does — so there is nothing to strip, cache, or fingerprint.
+
+use crate::error::{LayaError, Result};
+use crate::tokenization::{LayaTokenizer, SpecialTokens};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::path::Path;
+use tokenizers::Tokenizer as InnerTokenizer;
+
+/// [`LayaTokenizer`] backed by `tokenizer/tokenizer.json`. See the module docs for what this
+/// ports.
+pub struct HfTokenizer {
+ inner: InnerTokenizer,
+ special: SpecialTokens,
+}
+
+impl HfTokenizer {
+ /// Loads the tokenizer from `tokenizer_json_path` and resolves its special-token ids from the
+ /// sibling `tokenizer_config.json` (derived automatically: same directory, file name
+ /// `tokenizer_config.json`) plus `tokenizer.json`'s own `added_tokens` / post-processor data —
+ /// never hard-coded, since the ids differ by checkpoint.
+ ///
+ /// This is a convenience wrapper for the fused layout, which always ships a sibling
+ /// `tokenizer_config.json`. The split layout (laya-ts's exporter output) does not, and needs
+ /// [`Self::from_file_with_config`] with an explicit, possibly-absent config path instead —
+ /// [`crate::model_artifacts::ModelArtifacts`] already resolves the right one to pass in.
+ ///
+ /// Unlike [`Self::from_file_with_config`], a missing sibling `tokenizer_config.json` here
+ /// surfaces as [`LayaError::Io`] (a plain "file not found") rather than being wrapped in a
+ /// custom message. Callers that want the added_tokens-only fallback even for a sibling that
+ /// simply happens to be missing should call `from_file_with_config` directly instead.
+ pub fn from_file(tokenizer_json_path: impl AsRef<Path>) -> Result<Self> {
+ let tokenizer_json_path = tokenizer_json_path.as_ref();
+ let dir = tokenizer_json_path
+ .parent()
+ .unwrap_or_else(|| Path::new("."));
+ let config_path = dir.join("tokenizer_config.json");
+
+ let inner = InnerTokenizer::from_file(tokenizer_json_path).map_err(|e| {
+ LayaError::Tokenizer(format!(
+ "failed to load tokenizer '{}': {e}",
+ tokenizer_json_path.display()
+ ))
+ })?;
+ let special = read_special_tokens(tokenizer_json_path, Some(&config_path))?;
+ Ok(Self { inner, special })
+ }
+
+ /// Loads the tokenizer from `tokenizer_json_path`, resolving special-token ids from
+ /// `tokenizer_config_path` when given, or — when `None`, as for the split layout, which never
+ /// ships a `tokenizer_config.json` — falling back to `tokenizer.json` alone: CLS/SEP from its
+ /// `post_processor` template, the other roles from its `added_tokens` by matching well-known
+ /// content aliases (`[PAD]`/`<pad>`, `[MASK]`/`<mask>`, `[UNK]`/`<unk>`); see
+ /// [`resolve_special_tokens_from_added_tokens`].
+ pub fn from_file_with_config(
+ tokenizer_json_path: impl AsRef<Path>,
+ tokenizer_config_path: Option<&Path>,
+ ) -> Result<Self> {
+ let tokenizer_json_path = tokenizer_json_path.as_ref();
+ let inner = InnerTokenizer::from_file(tokenizer_json_path).map_err(|e| {
+ LayaError::Tokenizer(format!(
+ "failed to load tokenizer '{}': {e}",
+ tokenizer_json_path.display()
+ ))
+ })?;
+ let special = read_special_tokens(tokenizer_json_path, tokenizer_config_path)?;
+ Ok(Self { inner, special })
+ }
+}
+
+impl LayaTokenizer for HfTokenizer {
+ fn encode(&self, text: &str) -> Result<Vec<u32>> {
+ // add_special_tokens = false: the post-processor is never invoked, so no [CLS]/[SEP]/
+ // <bos>/<eos> gets injected. SequenceBuilder inserts every special token itself.
+ let encoding = self
+ .inner
+ .encode(text, false)
+ .map_err(|e| LayaError::Tokenizer(format!("failed to encode text: {e}")))?;
+ Ok(encoding.get_ids().to_vec())
+ }
+
+ fn special(&self) -> &SpecialTokens {
+ &self.special
+ }
+}
+
+// ── special-token resolution ─────────────────────────────────────────────────
+
+/// Reads the special-token ids for the checkpoint whose tokenizer sits at `tokenizer_json_path`.
+///
+/// When `tokenizer_config_path` is `Some`, this ports `HfTokenizer.ReadSpecialTokenIds` exactly:
+/// the config supplies the token *strings* (role -> string), and `tokenizer.json`'s
+/// `added_tokens` (falling back to `post_processor.special_tokens`) supplies the string -> id
+/// mapping.
+///
+/// When it is `None` — the split layout never ships a `tokenizer_config.json` — each role's
+/// string is instead resolved by matching well-known content aliases directly against
+/// `tokenizer.json`'s own `added_tokens` (see [`resolve_special_tokens_from_added_tokens`]),
+/// mirroring what a checkpoint's own vocabulary calls its special tokens (mmBERT: `<bos>`/`<eos>`/
+/// `<pad>`/`<mask>`/`<unk>`; ModernBERT: `[CLS]`/`[SEP]`/`[PAD]`/`[MASK]`/`[UNK]`) without needing
+/// the config file to spell out which string plays which role. CLS and SEP are taken from the
+/// `post_processor`'s single-sequence template first, when it has one.
+fn read_special_tokens(
+ tokenizer_json_path: &Path,
+ tokenizer_config_path: Option<&Path>,
+) -> Result<SpecialTokens> {
+ // Build a content -> id map. Primary: added_tokens enumerates every extended-vocabulary
+ // token, including the special ones for both mmBERT (ids 0-4) and ModernBERT (ids
+ // 50280-50284). Fallback: post_processor.special_tokens, for a tokenizer that omits a special
+ // token from added_tokens.
+ let tok_text = std::fs::read_to_string(tokenizer_json_path)?;
+ let tok_json: Value = serde_json::from_str(&tok_text)?;
+
+ let mut id_by_content: HashMap<String, u32> = HashMap::new();
+ if let Some(added) = tok_json.get("added_tokens").and_then(Value::as_array) {
+ for entry in added {
+ if let (Some(content), Some(id)) = (
+ entry.get("content").and_then(Value::as_str),
+ entry.get("id").and_then(Value::as_u64),
+ ) {
+ id_by_content.insert(content.to_string(), id as u32);
+ }
+ }
+ }
+ if let Some(special_tokens) = tok_json
+ .get("post_processor")
+ .and_then(Value::as_object)
+ .and_then(|pp| pp.get("special_tokens"))
+ .and_then(Value::as_object)
+ {
+ for (content, entry) in special_tokens {
+ if id_by_content.contains_key(content) {
+ continue;
+ }
+ if let Some(first_id) = entry
+ .get("ids")
+ .and_then(Value::as_array)
+ .and_then(|ids| ids.first())
+ .and_then(Value::as_u64)
+ {
+ id_by_content.insert(content.clone(), first_id as u32);
+ }
+ }
+ }
+
+ match tokenizer_config_path {
+ Some(config_path) => {
+ // Which string plays each role? A missing file propagates as a plain LayaError::Io.
+ let config_text = std::fs::read_to_string(config_path)?;
+ let config: Value = serde_json::from_str(&config_text)?;
+ let cls_str = require_token_string(&config, "cls_token", config_path)?;
+ let sep_str = require_token_string(&config, "sep_token", config_path)?;
+ let pad_str = require_token_string(&config, "pad_token", config_path)?;
+ let unk_str = require_token_string(&config, "unk_token", config_path)?;
+ let mask_str = require_token_string(&config, "mask_token", config_path)?;
+
+ // Resolve each role. Any string missing here means the two files are from different
+ // checkpoints, which would silently produce wrong positions if left unchecked.
+ Ok(SpecialTokens {
+ pad_id: resolve(&pad_str, "pad_token", &id_by_content, config_path)?,
+ cls_id: resolve(&cls_str, "cls_token", &id_by_content, config_path)?,
+ sep_id: resolve(&sep_str, "sep_token", &id_by_content, config_path)?,
+ mask_id: resolve(&mask_str, "mask_token", &id_by_content, config_path)?,
+ unk_id: resolve(&unk_str, "unk_token", &id_by_content, config_path)?,
+ mask_token: mask_str,
+ })
+ }
+ None => resolve_special_tokens_from_added_tokens(
+ &id_by_content,
+ template_cls_sep(&tok_json),
+ tokenizer_json_path,
+ ),
+ }
+}
+
+/// Reads the CLS and SEP strings from the `post_processor`'s single-sequence template
+/// (`[CLS] $A [SEP]`, or `<bos> $A <eos>` for mmBERT): the first and last `SpecialToken` entries.
+/// Returns `None` when the tokenizer has no such template.
+fn template_cls_sep(tok_json: &Value) -> Option<(String, String)> {
+ let specials: Vec<&str> = tok_json
+ .get("post_processor")?
+ .get("single")?
+ .as_array()?
+ .iter()
+ .filter_map(|piece| piece.get("SpecialToken")?.get("id")?.as_str())
+ .collect();
+ match specials.as_slice() {
+ [first, .., last] => Some((first.to_string(), last.to_string())),
+ _ => None,
+ }
+}
+
+// Alias lists: the content strings a checkpoint's vocabulary is known to use for each special
+// role. mmBERT uses `<bos>`/`<eos>`/`<pad>`/`<mask>`/`<unk>`; ModernBERT/BERT-style tokenizers use
+// `[CLS]`/`[SEP]`/`[PAD]`/`[MASK]`/`[UNK]`; RoBERTa-style ones use `<s>`/`</s>`. Tried in the order
+// listed; the first alias present in `added_tokens` wins. The order matters: mmBERT's vocabulary
+// also holds `<s>`/`</s>`, but they are not its CLS/SEP, so `<bos>`/`<eos>` come first. CLS/SEP
+// only fall back to these lists when the tokenizer has no post_processor template.
+const CLS_ALIASES: &[&str] = &["[CLS]", "<bos>", "<s>"];
+const SEP_ALIASES: &[&str] = &["[SEP]", "<eos>", "</s>"];
+const PAD_ALIASES: &[&str] = &["[PAD]", "<pad>"];
+const MASK_ALIASES: &[&str] = &["[MASK]", "<mask>"];
+const UNK_ALIASES: &[&str] = &["[UNK]", "<unk>"];
+
+/// Resolves every special-token role directly from `tokenizer.json`'s own `added_tokens` (already
+/// flattened into `id_by_content` by the caller), for a checkpoint with no `tokenizer_config.json`
+/// to name which string plays which role. Each role tries its aliases in order and uses the first
+/// one present; a role with none of its aliases present is reported by name against
+/// `tokenizer_json_path`, so the message points at the actual file that turned out to be missing
+/// the token, and — unlike [`resolve`], which is about a stale mismatch between two files — this
+/// implies the vocabulary itself is missing an expected special token altogether.
+fn resolve_special_tokens_from_added_tokens(
+ id_by_content: &HashMap<String, u32>,
+ template_cls_sep: Option<(String, String)>,
+ tokenizer_json_path: &Path,
+) -> Result<SpecialTokens> {
+ let (cls_id, sep_id) = match template_cls_sep {
+ Some((cls, sep)) => (
+ resolve(&cls, "cls_token", id_by_content, tokenizer_json_path)?,
+ resolve(&sep, "sep_token", id_by_content, tokenizer_json_path)?,
+ ),
+ None => (
+ resolve_alias(id_by_content, CLS_ALIASES, "cls_token", tokenizer_json_path)?.1,
+ resolve_alias(id_by_content, SEP_ALIASES, "sep_token", tokenizer_json_path)?.1,
+ ),
+ };
+ let (mask_token, mask_id) = resolve_alias(
+ id_by_content,
+ MASK_ALIASES,
+ "mask_token",
+ tokenizer_json_path,
+ )?;
+ Ok(SpecialTokens {
+ pad_id: resolve_alias(id_by_content, PAD_ALIASES, "pad_token", tokenizer_json_path)?.1,
+ cls_id,
+ sep_id,
+ mask_id,
+ unk_id: resolve_alias(id_by_content, UNK_ALIASES, "unk_token", tokenizer_json_path)?.1,
+ mask_token,
+ })
+}
+
+/// Tries each of `aliases` in order against `id_by_content`, returning the first alias string
+/// (owned) and id that is present.
+fn resolve_alias(
+ id_by_content: &HashMap<String, u32>,
+ aliases: &[&str],
+ role: &str,
+ tokenizer_json_path: &Path,
+) -> Result<(String, u32)> {
+ for alias in aliases {
+ if let Some(&id) = id_by_content.get(*alias) {
+ return Ok((alias.to_string(), id));
+ }
+ }
+ Err(LayaError::Tokenizer(format!(
+ "'{}' has no tokenizer_config.json alongside it, and none of the usual {role} spellings \
+ ({}) were found in its added_tokens. Cannot determine the {role} id.",
+ tokenizer_json_path.display(),
+ aliases.join(", ")
+ )))
+}
+
+/// Extracts the token string for `key` from a `tokenizer_config.json` root. HuggingFace stores a
+/// special token either as a plain string or as a `{"content": "...", ...}` dict; both are
+/// handled, matching `HfTokenizer.RequireTokenString`.
+fn require_token_string(root: &Value, key: &str, config_path: &Path) -> Result<String> {
+ let val = root.get(key).ok_or_else(|| {
+ LayaError::Tokenizer(format!(
+ "'{}' is missing the '{key}' key — cannot determine the special-token ids.",
+ config_path.display()
+ ))
+ })?;
+
+ match val {
+ Value::Object(_) => val
+ .get("content")
+ .and_then(Value::as_str)
+ .map(str::to_string)
+ .ok_or_else(|| {
+ LayaError::Tokenizer(format!(
+ "'{key}' in '{}' is a dict with no (non-null) 'content' field.",
+ config_path.display()
+ ))
+ }),
+ Value::String(s) => Ok(s.clone()),
+ _ => Err(LayaError::Tokenizer(format!(
+ "'{key}' is null (or an unexpected type) in '{}'.",
+ config_path.display()
+ ))),
+ }
+}
+
+/// Looks up `token` in `id_by_content`, with a clear error naming the missing token when it is
+/// absent — typically meaning the two tokenizer files are from different checkpoints. Ports
+/// `HfTokenizer.Resolve`.
+fn resolve(
+ token: &str,
+ role: &str,
+ id_by_content: &HashMap<String, u32>,
+ config_path: &Path,
+) -> Result<u32> {
+ id_by_content.get(token).copied().ok_or_else(|| {
+ LayaError::Tokenizer(format!(
+ "Special token '{token}' (role: {role}, declared in '{}') was not found in \
+ tokenizer.json's added_tokens or post_processor.special_tokens. Check that \
+ tokenizer.json and tokenizer_config.json belong to the same checkpoint.",
+ config_path.display()
+ ))
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// A tiny mmBERT-style `tokenizer.json` fixture with the real checkpoint's ids: just enough
+ /// `added_tokens` to resolve every special-token role, without a full vocabulary/model section
+ /// (unneeded by `read_special_tokens`). Like the real vocabulary it also holds `<s>`/`</s>`,
+ /// which are not its CLS/SEP.
+ fn mmbert_style_tokenizer_json() -> &'static str {
+ r#"{
+ "added_tokens": [
+ {"id": 0, "content": "<pad>"},
+ {"id": 1, "content": "<eos>"},
+ {"id": 2, "content": "<bos>"},
+ {"id": 3, "content": "<unk>"},
+ {"id": 4, "content": "<mask>"},
+ {"id": 204, "content": "<s>"},
+ {"id": 213, "content": "</s>"}
+ ]
+ }"#
+ }
+
+ fn modernbert_style_tokenizer_json() -> &'static str {
+ r#"{
+ "added_tokens": [
+ {"id": 50281, "content": "[CLS]"},
+ {"id": 50282, "content": "[SEP]"},
+ {"id": 50283, "content": "[PAD]"},
+ {"id": 50284, "content": "[MASK]"},
+ {"id": 50280, "content": "[UNK]"}
+ ]
+ }"#
+ }
+
+ #[test]
+ fn added_tokens_fallback_resolves_mmbert_style_aliases() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let tok_path = dir.path().join("tokenizer.json");
+ std::fs::write(&tok_path, mmbert_style_tokenizer_json()).unwrap();
+
+ let special = read_special_tokens(&tok_path, None).expect("should resolve via aliases");
+ assert_eq!(special.pad_id, 0);
+ assert_eq!(special.sep_id, 1);
+ assert_eq!(special.cls_id, 2);
+ assert_eq!(special.unk_id, 3);
+ assert_eq!(special.mask_id, 4);
+ assert_eq!(special.mask_token, "<mask>");
+ }
+
+ #[test]
+ fn added_tokens_fallback_takes_cls_and_sep_from_the_post_processor_template() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let tok_path = dir.path().join("tokenizer.json");
+ // The template names "<s> $A </s>", so it must win over the "<bos>"/"<eos>" aliases.
+ std::fs::write(
+ &tok_path,
+ r#"{
+ "added_tokens": [
+ {"id": 0, "content": "<pad>"},
+ {"id": 1, "content": "<eos>"},
+ {"id": 2, "content": "<bos>"},
+ {"id": 3, "content": "<unk>"},
+ {"id": 4, "content": "<mask>"},
+ {"id": 204, "content": "<s>"},
+ {"id": 213, "content": "</s>"}
+ ],
+ "post_processor": {
+ "type": "TemplateProcessing",
+ "single": [
+ {"SpecialToken": {"id": "<s>", "type_id": 0}},
+ {"Sequence": {"id": "A", "type_id": 0}},
+ {"SpecialToken": {"id": "</s>", "type_id": 0}}
+ ]
+ }
+ }"#,
+ )
+ .unwrap();
+
+ let special = read_special_tokens(&tok_path, None).expect("should resolve via template");
+ assert_eq!(special.cls_id, 204);
+ assert_eq!(special.sep_id, 213);
+ assert_eq!(special.pad_id, 0);
+ }
+
+ #[test]
+ fn added_tokens_fallback_resolves_modernbert_style_aliases() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let tok_path = dir.path().join("tokenizer.json");
+ std::fs::write(&tok_path, modernbert_style_tokenizer_json()).unwrap();
+
+ let special = read_special_tokens(&tok_path, None).expect("should resolve via aliases");
+ assert_eq!(special.cls_id, 50281);
+ assert_eq!(special.sep_id, 50282);
+ assert_eq!(special.pad_id, 50283);
+ assert_eq!(special.mask_id, 50284);
+ assert_eq!(special.unk_id, 50280);
+ assert_eq!(special.mask_token, "[MASK]");
+ }
+
+ #[test]
+ fn added_tokens_fallback_errors_clearly_when_a_role_has_no_alias_present() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let tok_path = dir.path().join("tokenizer.json");
+ // Missing a mask token entirely.
+ std::fs::write(
+ &tok_path,
+ r#"{
+ "added_tokens": [
+ {"id": 0, "content": "<pad>"},
+ {"id": 1, "content": "<eos>"},
+ {"id": 2, "content": "<bos>"},
+ {"id": 3, "content": "<unk>"}
+ ]
+ }"#,
+ )
+ .unwrap();
+
+ let err = read_special_tokens(&tok_path, None).expect_err("must fail: no mask alias");
+ let message = err.to_string();
+ assert!(
+ message.contains("mask_token") && message.contains("tokenizer_config.json"),
+ "message was: {message}"
+ );
+ }
+
+ #[test]
+ fn config_based_lookup_still_takes_precedence_when_a_config_path_is_given() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let tok_path = dir.path().join("tokenizer.json");
+ std::fs::write(&tok_path, mmbert_style_tokenizer_json()).unwrap();
+
+ let config_path = dir.path().join("tokenizer_config.json");
+ std::fs::write(
+ &config_path,
+ r#"{
+ "cls_token": "<s>",
+ "sep_token": "</s>",
+ "pad_token": "<pad>",
+ "unk_token": "<unk>",
+ "mask_token": "<mask>"
+ }"#,
+ )
+ .unwrap();
+
+ let special =
+ read_special_tokens(&tok_path, Some(&config_path)).expect("should resolve via config");
+ // The config names "<s>"/"</s>", not the alias-preferred "<bos>"/"<eos>", so it must win.
+ assert_eq!(special.cls_id, 204);
+ assert_eq!(special.sep_id, 213);
+ assert_eq!(special.mask_token, "<mask>");
+ }
+
+ #[test]
+ fn config_based_lookup_errors_when_the_config_file_is_missing() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let tok_path = dir.path().join("tokenizer.json");
+ std::fs::write(&tok_path, mmbert_style_tokenizer_json()).unwrap();
+ let missing_config = dir.path().join("tokenizer_config.json");
+
+ let err = read_special_tokens(&tok_path, Some(&missing_config))
+ .expect_err("must fail: config file does not exist");
+ assert!(matches!(err, LayaError::Io(_)));
+ }
+}
diff --git a/laya-rust/src/tokenization/laya_tokenizer.rs b/laya-rust/src/tokenization/laya_tokenizer.rs
new file mode 100644
index 00000000..17b739a1
--- /dev/null
+++ b/laya-rust/src/tokenization/laya_tokenizer.rs
@@ -0,0 +1,39 @@
+//! The tokenizer trait the engine depends on, decoupling it from any one backend.
+//! `hf_tokenizer.rs` (a real HuggingFace-backed implementation) is the one the engine uses.
+
+use crate::error::Result;
+
+/// The special token ids a checkpoint's tokenizer resolves to, plus the literal mask token string
+/// used to scrub forged markers out of user input (see `sequence-construction.md`'s "Mask
+/// injection defense").
+///
+/// Ids and the mask token string differ by checkpoint (mmBERT uses ids 0-4, ModernBERT uses ids
+/// 50280-50284) and must always be read from the checkpoint's own config files rather than
+/// hard-coded.
+#[derive(Debug, Clone)]
+pub struct SpecialTokens {
+ /// Token id of the padding token, used to right-pad sequences to a uniform length.
+ pub pad_id: u32,
+ /// Token id of the CLS token, prepended to every sequence before the head fragment.
+ pub cls_id: u32,
+ /// Token id of the SEP token, inserted between the head, the option fragments, and the state.
+ pub sep_id: u32,
+ /// Token id of the mask token. Each option fragment starts with one; its position in the
+ /// sequence is recorded as a marker and later read by the scoring head.
+ pub mask_id: u32,
+ /// Token id of the unknown-token.
+ pub unk_id: u32,
+ /// The literal mask token string (e.g. `"<mask>"` for mmBERT or `"[MASK]"` for ModernBERT).
+ pub mask_token: String,
+}
+
+/// Tokenizes text for the Laya ONNX model, matching the behaviour of the Python `tokenizers`
+/// library called with `add_special_tokens=False`. Ports `ILayaTokenizer`.
+pub trait LayaTokenizer: Send + Sync {
+ /// Encodes `text` to token ids with **no** special tokens prepended or appended. Equivalent to
+ /// Python's `tok(text, add_special_tokens=False).input_ids`.
+ fn encode(&self, text: &str) -> Result<Vec<u32>>;
+
+ /// The special token ids and mask token string this tokenizer resolved at load time.
+ fn special(&self) -> &SpecialTokens;
+}
diff --git a/laya-rust/src/tokenization/mod.rs b/laya-rust/src/tokenization/mod.rs
new file mode 100644
index 00000000..961bd4d9
--- /dev/null
+++ b/laya-rust/src/tokenization/mod.rs
@@ -0,0 +1,9 @@
+//! The tokenizer abstraction [`crate::sequence_builder::SequenceBuilder::build`] is written
+//! against, plus [`hf_tokenizer::HfTokenizer`], the real HuggingFace-backed implementation
+//! (porting `HfTokenizer.cs`) used by [`crate::laya_engine::LayaEngine`].
+
+pub mod hf_tokenizer;
+pub mod laya_tokenizer;
+
+pub use hf_tokenizer::HfTokenizer;
+pub use laya_tokenizer::{LayaTokenizer, SpecialTokens};
diff --git a/laya-rust/tests/common/stub_tokenizer.rs b/laya-rust/tests/common/stub_tokenizer.rs
new file mode 100644
index 00000000..5cfb8d46
--- /dev/null
+++ b/laya-rust/tests/common/stub_tokenizer.rs
@@ -0,0 +1,80 @@
+//! A deterministic tokenizer with no model behind it, for testing sequence structure.
+//!
+//! It emits one id per whitespace-separated word, so a sequence's shape — marker positions, the
+//! order of head, options and state, where truncation lands — can be asserted by counting words.
+//! It says nothing about real token ids; that is what the tokenizer-parity tests are for.
+
+use laya::error::Result;
+use laya::tokenization::{LayaTokenizer, SpecialTokens};
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
+use std::sync::Mutex;
+
+/// Ids start here, above every special id, so a word is never mistaken for one.
+pub const FIRST_WORD_ID: u32 = 100;
+
+/// See the module docs.
+pub struct StubTokenizer {
+ special: SpecialTokens,
+ encoded: Mutex<Vec<String>>,
+}
+
+impl StubTokenizer {
+ /// A stub tokenizer with the mmBERT-style special token ids (0-4) `SequenceBuilder` tests are
+ /// written against.
+ pub fn new() -> Self {
+ Self {
+ special: SpecialTokens {
+ pad_id: 0,
+ sep_id: 1,
+ cls_id: 2,
+ unk_id: 3,
+ mask_id: 4,
+ mask_token: "<mask>".to_string(),
+ },
+ encoded: Mutex::new(Vec::new()),
+ }
+ }
+
+ /// Every text passed to [`LayaTokenizer::encode`], in call order.
+ pub fn encoded(&self) -> Vec<String> {
+ self.encoded
+ .lock()
+ .expect("encoded log mutex poisoned")
+ .clone()
+ }
+
+ /// How many times [`LayaTokenizer::encode`] has been called.
+ pub fn encode_calls(&self) -> usize {
+ self.encoded().len()
+ }
+}
+
+impl Default for StubTokenizer {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl LayaTokenizer for StubTokenizer {
+ fn encode(&self, text: &str) -> Result<Vec<u32>> {
+ self.encoded
+ .lock()
+ .expect("encoded log mutex poisoned")
+ .push(text.to_string());
+ // A stable, collision-tolerant id per word: the tests only need determinism, and hashing
+ // keeps identical words mapping to identical ids across calls.
+ Ok(text
+ .split_whitespace()
+ .map(|word| {
+ let mut hasher = DefaultHasher::new();
+ word.hash(&mut hasher);
+ FIRST_WORD_ID + (hasher.finish() % 10_000) as u32
+ })
+ .collect())
+ }
+
+ fn special(&self) -> &SpecialTokens {
+ &self.special
+ }
+}