Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(dotnet): add HuggingFace tokenizer binding
kevin-gatimu12cd932af9c2
diff --git a/laya-dotnet/src/Laya/Tokenization/HfTokenizer.cs b/laya-dotnet/src/Laya/Tokenization/HfTokenizer.csnew file mode 100644index 00000000..b97ba71a--- /dev/null+++ b/laya-dotnet/src/Laya/Tokenization/HfTokenizer.cs@@ -0,0 +1,382 @@+using System.Security.Cryptography;+using System.Text;+using System.Text.Json;+using System.Text.Json.Nodes;+using Tokenizers.DotNet;++namespace Laya.Tokenization;++/// <summary>+/// <see cref="ILayaTokenizer"/> backed by the <c>Tokenizers.DotNet</c> native wrapper of the+/// HuggingFace Rust tokenizers library.+/// </summary>+/// <remarks>+/// <para>+/// The <c>tokenizer.json</c> shipped with Laya contains a <c>post_processor</c> of type+/// <c>TemplateProcessing</c> that injects special tokens on every encode:+/// <c><bos> A <eos></c> for the multilingual checkpoint and+/// <c>[CLS] A [SEP]</c> for the ModernBERT checkpoints.+/// The Python code calls the tokenizer with <c>add_special_tokens=False</c> on many fragments+/// separately and assembles them itself, so those injected ids would corrupt every downstream+/// position. <c>Tokenizers.DotNet</c> offers no equivalent flag, so this class rewrites the+/// JSON with <c>"post_processor": null</c> before loading it, and caches the result.+/// </para>+/// <para>+/// The special-token ids are read from <c>tokenizer_config.json</c> (which maps role names+/// such as <c>cls_token</c> to token strings) and from <c>tokenizer.json</c>'s+/// <c>added_tokens</c> array; the <c>post_processor.special_tokens</c> section serves as a+/// fallback for tokenizers where not every special token appears in <c>added_tokens</c>.+/// This config-driven approach works for both the mmBERT-style tokens+/// (<c><pad></c>, <c><bos></c>, …) and the ModernBERT-style tokens+/// (<c>[PAD]</c>, <c>[CLS]</c>, …) without any hard-coding.+/// </para>+/// <para>+/// The split ONNX layout (the laya-ts exporter's output) ships <c>tokenizer.json</c> with no+/// <c>tokenizer_config.json</c> beside it. When that file is missing, each role's token string+/// is instead guessed from conventional special-token spellings (the ModernBERT bracketed form,+/// the SentencePiece/BPE form, and the BOS/EOS convention some tokenizers use for CLS/SEP) and+/// confirmed against <c>tokenizer.json</c>'s own token map before being accepted — see+/// <see cref="FirstKnownToken"/>.+/// </para>+/// </remarks>+public sealed class HfTokenizer : ILayaTokenizer, IDisposable+{+ private readonly Tokenizer _inner;+ private readonly string _maskToken;++ /// <inheritdoc/>+ public int PadId { get; }++ /// <inheritdoc/>+ public int ClsId { get; }++ /// <inheritdoc/>+ public int SepId { get; }++ /// <inheritdoc/>+ public int MaskId { get; }++ /// <inheritdoc/>+ public int UnkId { get; }++ /// <inheritdoc/>+ public string MaskToken => _maskToken;++ /// <summary>+ /// Loads the tokenizer from <paramref name="tokenizerJsonPath"/>, stripping the+ /// <c>post_processor</c> and caching the modified file next to the original (or in a+ /// temp directory when the source directory is read-only).+ /// </summary>+ /// <param name="tokenizerJsonPath">+ /// Absolute or relative path to <c>tokenizer.json</c>. A sibling <c>tokenizer_config.json</c>+ /// is used when present (the fused layout); when absent (the split layout), special-token+ /// strings are instead inferred from convention — see <see cref="FirstKnownToken"/>.+ /// </param>+ /// <exception cref="InvalidOperationException">+ /// When <c>tokenizer_config.json</c> is present but missing a required special-token key, or+ /// a token string is absent from the id maps in <c>tokenizer.json</c> — which typically means+ /// the two files belong to different checkpoints — or, when it is absent, when none of the+ /// conventional spellings for a role appear in <c>tokenizer.json</c> either.+ /// </exception>+ public HfTokenizer(string tokenizerJsonPath)+ {+ ArgumentException.ThrowIfNullOrEmpty(tokenizerJsonPath);++ var strippedPath = EnsureStrippedTokenizer(tokenizerJsonPath);+ _inner = new Tokenizer(vocabPath: strippedPath);++ var tokens = ReadSpecialTokenIds(tokenizerJsonPath);+ PadId = tokens.Pad;+ ClsId = tokens.Cls;+ SepId = tokens.Sep;+ MaskId = tokens.Mask;+ UnkId = tokens.Unk;+ _maskToken = tokens.MaskToken;+ }++ /// <inheritdoc/>+ public int[] Encode(string text)+ {+ var tokens = _inner.Encode(text);+ return Array.ConvertAll(tokens, t => (int)t);+ }++ /// <inheritdoc/>+ public void Dispose() => _inner.Dispose();++ // ── post-processor stripping ───────────────────────────────────────────────++ /// <summary>+ /// Returns the path to a cached copy of the tokenizer with <c>post_processor</c> set+ /// to <see langword="null"/>. Regenerates the cache when the source file has changed.+ /// </summary>+ private static string EnsureStrippedTokenizer(string sourcePath)+ {+ sourcePath = Path.GetFullPath(sourcePath);+ var srcInfo = new FileInfo(sourcePath);++ // Fingerprint: source file length and last-write UTC ticks, separated by |.+ // Using ticks avoids locale-dependent date formatting.+ var fingerprint = $"{srcInfo.Length}|{srcInfo.LastWriteTimeUtc.Ticks}";++ // Primary cache: alongside the original.+ var primaryDir = Path.GetDirectoryName(sourcePath)!;+ var primaryPath = Path.Combine(primaryDir, "tokenizer.nopost.json");++ if (IsCacheValid(primaryPath, fingerprint))+ return primaryPath;++ if (TryWriteStripped(sourcePath, primaryPath, fingerprint))+ return primaryPath;++ // Fallback: a per-source temp directory when the artifact directory is read-only.+ var fallbackDir = Path.Combine(+ Path.GetTempPath(), "laya", FingerprintToSafeName(sourcePath));+ System.IO.Directory.CreateDirectory(fallbackDir);++ var fallbackPath = Path.Combine(fallbackDir, "tokenizer.nopost.json");++ if (IsCacheValid(fallbackPath, fingerprint))+ return fallbackPath;++ if (TryWriteStripped(sourcePath, fallbackPath, fingerprint))+ return fallbackPath;++ throw new IOException(+ $"Cannot write stripped tokenizer cache to either:\n" ++ $" {primaryPath}\n {fallbackPath}\n" ++ "Check that at least one of those directories is writable.");+ }++ private static bool IsCacheValid(string cachedPath, string fingerprint)+ {+ var metaPath = cachedPath + ".meta";+ if (!File.Exists(cachedPath) || !File.Exists(metaPath))+ return false;+ try { return File.ReadAllText(metaPath).Trim() == fingerprint; }+ catch (IOException) { return false; }+ }++ private static bool TryWriteStripped(string sourcePath, string destPath, string fingerprint)+ {+ try+ {+ WriteStrippedTokenizer(sourcePath, destPath);+ File.WriteAllText(destPath + ".meta", fingerprint);+ return true;+ }+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)+ {+ return false;+ }+ }++ /// <summary>+ /// Parses <paramref name="sourcePath"/> with <c>JsonNode</c>, sets+ /// <c>"post_processor"</c> to <c>null</c>, and writes the result to+ /// <paramref name="destPath"/> via a <c>.part</c> temp file so an interrupted write+ /// never leaves a partial cache.+ /// </summary>+ private static void WriteStrippedTokenizer(string sourcePath, string destPath)+ {+ JsonObject root;+ using (var stream = File.OpenRead(sourcePath))+ root = JsonNode.Parse(stream)!.AsObject();++ // Setting to null writes "post_processor": null in the output, which the+ // underlying Rust tokenizers library treats identically to the key being absent.+ root["post_processor"] = null;++ var partPath = destPath + ".part";+ using (var outStream = new FileStream(+ partPath, FileMode.Create, FileAccess.Write, FileShare.None))+ {+ using var writer = new Utf8JsonWriter(outStream);+ root.WriteTo(writer);+ }++ if (File.Exists(destPath)) File.Delete(destPath);+ File.Move(partPath, destPath);+ }++ /// <summary>+ /// Produces a short ASCII directory-name fragment that is stable for a given source+ /// path, used to keep per-source fallback cache directories separated in %TEMP%.+ /// </summary>+ private static string FingerprintToSafeName(string sourcePath)+ {+ // SHA-256 of the UTF-8 path gives a deterministic hash that survives process restarts.+ // string.GetHashCode() is randomised per process in .NET (ASLR seed), so the fallback+ // cache would never hit across runs — exactly the scenario it exists for.+ var hash = SHA256.HashData(Encoding.UTF8.GetBytes(sourcePath));+ // First 8 hex chars (4 bytes) are enough for collision resistance within one machine's paths.+ return Convert.ToHexString(hash, 0, 4).ToLowerInvariant();+ }++ // ── special-token ids ──────────────────────────────────────────────────────++ /// <summary>+ /// Reads the special-token ids for the checkpoint whose files are at+ /// <paramref name="tokenizerJsonPath"/>. <c>tokenizer.json</c>'s <c>added_tokens</c> and, as+ /// a fallback, its <c>post_processor.special_tokens</c> supply the id mapping. The token+ /// *strings* normally come from a sibling <c>tokenizer_config.json</c> (the fused layout);+ /// when that file is absent (the split layout, which the laya-ts exporter writes without a+ /// <c>tokenizer_config.json</c>), the strings are instead guessed from conventional special-+ /// token spellings and confirmed against <c>added_tokens</c> — see <see cref="FirstKnownToken"/>.+ /// </summary>+ private static (int Pad, int Cls, int Sep, int Mask, int Unk, string MaskToken)+ ReadSpecialTokenIds(string tokenizerJsonPath)+ {+ tokenizerJsonPath = Path.GetFullPath(tokenizerJsonPath);+ var configPath = Path.Combine(+ Path.GetDirectoryName(tokenizerJsonPath)!, "tokenizer_config.json");++ // 1. Build a content→id map. Primary: added_tokens enumerates all extended-vocabulary+ // tokens including special ones (both the small mmBERT special-token block at the+ // front of the vocab, and the ModernBERT block appended at the end). Fallback:+ // post_processor.special_tokens is also populated by ModernBERT tokenizers and+ // is useful for any tokenizer that omits a special token from added_tokens. This map+ // is built before looking at tokenizer_config.json, since the split layout has no+ // such file and needs added_tokens alone to resolve every role.+ var idByContent = new Dictionary<string, int>(StringComparer.Ordinal);+ string? templateCls = null, templateSep = null;+ using (var f = File.OpenRead(tokenizerJsonPath))+ using (var tok = JsonDocument.Parse(f))+ {+ var r = tok.RootElement;+ foreach (var entry in r.GetProperty("added_tokens").EnumerateArray())+ idByContent[entry.GetProperty("content").GetString()!] =+ entry.GetProperty("id").GetInt32();++ if (r.TryGetProperty("post_processor", out var pp)+ && pp.ValueKind == JsonValueKind.Object+ && pp.TryGetProperty("special_tokens", out var st)+ && st.ValueKind == JsonValueKind.Object)+ {+ foreach (var kvp in st.EnumerateObject())+ if (!idByContent.ContainsKey(kvp.Name)+ && kvp.Value.TryGetProperty("ids", out var ids)+ && ids.GetArrayLength() > 0)+ idByContent[kvp.Name] = ids[0].GetInt32();+ }++ // The single-sequence template ("[CLS] $A [SEP]", or "<bos> $A <eos>" for mmBERT)+ // names the CLS and SEP strings outright. Only the split layout needs it.+ if (pp.ValueKind == JsonValueKind.Object+ && pp.TryGetProperty("single", out var single)+ && single.ValueKind == JsonValueKind.Array)+ {+ var specials = single.EnumerateArray()+ .Where(p => p.TryGetProperty("SpecialToken", out _))+ .Select(p => p.GetProperty("SpecialToken").GetProperty("id").GetString()!)+ .ToList();+ if (specials.Count >= 2)+ (templateCls, templateSep) = (specials[0], specials[^1]);+ }+ }++ string clsStr, sepStr, padStr, unkStr, maskStr;+ if (File.Exists(configPath))+ {+ // 2a. Fused layout: tokenizer_config.json says "cls_token": "[CLS]" (for ModernBERT)+ // or "cls_token": "<bos>" (for mmBERT). HuggingFace sometimes stores these as+ // plain strings and sometimes as {"content": "...", ...} dicts.+ using var f = File.OpenRead(configPath);+ using var cfg = JsonDocument.Parse(f);+ var r = cfg.RootElement;+ clsStr = RequireTokenString(r, "cls_token", configPath);+ sepStr = RequireTokenString(r, "sep_token", configPath);+ padStr = RequireTokenString(r, "pad_token", configPath);+ unkStr = RequireTokenString(r, "unk_token", configPath);+ maskStr = RequireTokenString(r, "mask_token", configPath);+ }+ else+ {+ // 2b. Split layout: no tokenizer_config.json. CLS and SEP come from the+ // post_processor template when it has one; a spelling guess is not enough there,+ // because mmBERT's vocab holds both "<s>" and "<bos>" but only "<bos>" is its CLS.+ // The other roles (and CLS/SEP without a template) are guessed from conventional+ // special-token names — the ModernBERT bracketed form and the SentencePiece/BPE+ // form — and accepted only if the spelling appears in tokenizer.json's token map.+ clsStr = templateCls ?? FirstKnownToken(idByContent, "cls_token", configPath, "[CLS]", "<bos>", "<s>");+ sepStr = templateSep ?? FirstKnownToken(idByContent, "sep_token", configPath, "[SEP]", "<eos>", "</s>");+ padStr = FirstKnownToken(idByContent, "pad_token", configPath, "[PAD]", "<pad>");+ maskStr = FirstKnownToken(idByContent, "mask_token", configPath, "[MASK]", "<mask>");+ unkStr = FirstKnownToken(idByContent, "unk_token", configPath, "[UNK]", "<unk>");+ }++ // 3. Resolve each role to its id. Any missing token here means the two files are+ // from different checkpoints, which would silently produce wrong positions.+ return (+ Resolve(padStr, "pad_token", idByContent, configPath),+ Resolve(clsStr, "cls_token", idByContent, configPath),+ Resolve(sepStr, "sep_token", idByContent, configPath),+ Resolve(maskStr, "mask_token", idByContent, configPath),+ Resolve(unkStr, "unk_token", idByContent, configPath),+ maskStr+ );+ }++ /// <summary>+ /// Used only when <c>tokenizer_config.json</c> is absent (the split layout): returns the+ /// first of <paramref name="candidates"/> that is a known token in <paramref name="idByContent"/>,+ /// i.e. actually appears in <c>tokenizer.json</c>.+ /// </summary>+ /// <exception cref="InvalidOperationException">+ /// When none of the candidate spellings is known — the loaded tokenizer.json uses a+ /// convention this method does not yet recognise for <paramref name="role"/>.+ /// </exception>+ private static string FirstKnownToken(+ Dictionary<string, int> idByContent, string role, string configPath, params string[] candidates)+ {+ foreach (var candidate in candidates)+ if (idByContent.ContainsKey(candidate))+ return candidate;++ throw new InvalidOperationException(+ $"Could not determine the '{role}' token: no 'tokenizer_config.json' was found "+ + $"next to the tokenizer (expected at '{configPath}'), and none of the conventional "+ + $"spellings ({string.Join(", ", candidates)}) appear in tokenizer.json's "+ + "added_tokens or post_processor.special_tokens.");+ }++ /// <summary>+ /// Extracts the token string for <paramref name="key"/> from a tokenizer_config root.+ /// HuggingFace can store a special token either as a plain string or as a+ /// <c>{"content": "...", ...}</c> dict; both are handled.+ /// </summary>+ private static string RequireTokenString(JsonElement root, string key, string configPath)+ {+ if (!root.TryGetProperty(key, out var val))+ throw new InvalidOperationException(+ $"'{configPath}' is missing the '{key}' key — cannot determine the special-token ids.");++ if (val.ValueKind == JsonValueKind.Object)+ {+ if (val.TryGetProperty("content", out var content))+ return content.GetString()+ ?? throw new InvalidOperationException(+ $"'{key}'.content is null in '{configPath}'.");+ throw new InvalidOperationException(+ $"'{key}' in '{configPath}' is a dict with no 'content' field.");+ }++ return val.GetString()+ ?? throw new InvalidOperationException($"'{key}' is null in '{configPath}'.");+ }++ /// <summary>+ /// Looks up <paramref name="token"/> in <paramref name="idByContent"/>, throwing a+ /// clear error when it is absent — which typically means the two tokenizer files are+ /// from different checkpoints and would silently produce wrong model outputs.+ /// </summary>+ private static int Resolve(string token, string role,+ Dictionary<string, int> idByContent, string configPath)+ {+ if (idByContent.TryGetValue(token, out var id)) return id;+ throw new InvalidOperationException(+ $"Special token '{token}' (role: {role}, declared in '{configPath}') was not found "+ + "in tokenizer.json's added_tokens or post_processor.special_tokens. "+ + "Check that tokenizer.json and tokenizer_config.json belong to the same checkpoint.");+ }+}diff --git a/laya-dotnet/src/Laya/Tokenization/ILayaTokenizer.cs b/laya-dotnet/src/Laya/Tokenization/ILayaTokenizer.csnew file mode 100644index 00000000..1755f6fb--- /dev/null+++ b/laya-dotnet/src/Laya/Tokenization/ILayaTokenizer.cs@@ -0,0 +1,50 @@+namespace Laya.Tokenization;++/// <summary>+/// Tokenizes text for the Laya ONNX model, matching the behaviour of the Python+/// <c>tokenizers</c> library called with <c>add_special_tokens=False</c>.+/// </summary>+public interface ILayaTokenizer+{+ /// <summary>+ /// Encodes <paramref name="text"/> to token ids with <b>no</b> special tokens+ /// prepended or appended. Equivalent to Python's+ /// <c>tok(text, add_special_tokens=False).input_ids</c>.+ /// </summary>+ int[] Encode(string text);++ /// <summary>Token id of the padding token, used to right-pad sequences to a uniform length.</summary>+ int PadId { get; }++ /// <summary>+ /// Token id of the CLS token. Python prepends this to every sequence before the+ /// head fragment. Named <c>ClsId</c> to match the HuggingFace convention; the+ /// underlying token string varies by checkpoint (<c><bos></c> for mmBERT,+ /// <c>[CLS]</c> for ModernBERT).+ /// </summary>+ int ClsId { get; }++ /// <summary>+ /// Token id of the SEP token. Python inserts this between the head, the option+ /// fragments, and the state. Named <c>SepId</c> to match the HuggingFace convention;+ /// the underlying token string varies by checkpoint (<c><eos></c> for mmBERT,+ /// <c>[SEP]</c> for ModernBERT).+ /// </summary>+ int SepId { get; }++ /// <summary>+ /// Token id of the mask token. Each option fragment starts with one — its position+ /// in the sequence is recorded as a marker and later read by the scoring head.+ /// </summary>+ int MaskId { get; }++ /// <summary>Token id of the unknown-token.</summary>+ int UnkId { get; }++ /// <summary>+ /// The literal mask token string (e.g. <c>"<mask>"</c> for mmBERT or+ /// <c>"[MASK]"</c> for ModernBERT). Used to scrub any accidental occurrence of the+ /// mask token from user-supplied text before encoding.+ /// </summary>+ string MaskToken { get; }+}diff --git a/laya-dotnet/tests/Laya.Tests/HfTokenizerFallbackTests.cs b/laya-dotnet/tests/Laya.Tests/HfTokenizerFallbackTests.csnew file mode 100644index 00000000..9a5873c3--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/HfTokenizerFallbackTests.cs@@ -0,0 +1,184 @@+using Laya.Tokenization;++namespace Laya.Tests;++/// <summary>+/// Tier 1: <see cref="HfTokenizer"/>'s special-token resolution against a minimal, synthetic+/// <c>tokenizer.json</c> in a temp directory, covering the split layout's missing+/// <c>tokenizer_config.json</c> without needing a real checkpoint.+/// </summary>+/// <remarks>+/// The fixture below is not a usable tokenizer (its "vocabulary" only has enough entries to prove+/// which ids got resolved), but it is enough to exercise <c>HfTokenizer</c>'s constructor end to+/// end: the native <c>Tokenizers.DotNet</c> wrapper does load it, since a WordPiece model only+/// needs a <c>vocab</c> map and an <c>unk_token</c> to be valid.+/// </remarks>+public sealed class HfTokenizerFallbackTests : IDisposable+{+ private readonly string _dir;++ public HfTokenizerFallbackTests()+ {+ _dir = Path.Combine(Path.GetTempPath(), "laya-tokenizer-tests-" + Guid.NewGuid().ToString("N"));+ Directory.CreateDirectory(_dir);+ }++ public void Dispose()+ {+ try { Directory.Delete(_dir, recursive: true); }+ catch (IOException) { /* best effort */ }+ }++ private static string AddedToken(string content, int id) =>+ $$"""{"id": {{id}}, "content": "{{content}}", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}""";++ /// <summary>A minimal but loadable WordPiece tokenizer.json using ModernBERT-style bracketed tokens.</summary>+ private const string BracketedVocab =+ """{"[PAD]":0,"[UNK]":1,"[CLS]":2,"[SEP]":3,"[MASK]":4,"hello":5}""";++ private void WriteTokenizerJson(string addedTokensJson, string vocabJson)+ {+ var json = $$"""+ {+ "version": "1.0",+ "truncation": null,+ "padding": null,+ "added_tokens": [{{addedTokensJson}}],+ "normalizer": null,+ "pre_tokenizer": {"type": "Whitespace"},+ "post_processor": null,+ "decoder": null,+ "model": {+ "type": "WordPiece",+ "unk_token": "[UNK]",+ "continuing_subword_prefix": "##",+ "max_input_chars_per_word": 100,+ "vocab": {{vocabJson}}+ }+ }+ """;+ File.WriteAllText(Path.Combine(_dir, "tokenizer.json"), json);+ }++ [Fact]+ public void NoTokenizerConfig_ResolvesBracketedSpecialTokensFromAddedTokens()+ {+ // Split layout: no tokenizer_config.json beside tokenizer.json. Every role must be+ // resolved purely from added_tokens' conventional spellings.+ var added = string.Join(",", [+ AddedToken("[PAD]", 0), AddedToken("[UNK]", 1), AddedToken("[CLS]", 2),+ AddedToken("[SEP]", 3), AddedToken("[MASK]", 4),+ ]);+ WriteTokenizerJson(added, BracketedVocab);++ using var tok = new HfTokenizer(Path.Combine(_dir, "tokenizer.json"));++ Assert.Equal(0, tok.PadId);+ Assert.Equal(1, tok.UnkId);+ Assert.Equal(2, tok.ClsId);+ Assert.Equal(3, tok.SepId);+ Assert.Equal(4, tok.MaskId);+ Assert.Equal("[MASK]", tok.MaskToken);+ }++ [Fact]+ public void NoTokenizerConfig_ResolvesBosEosSpellingForClsAndSep()+ {+ // Some split exports use the BOS/EOS convention for CLS/SEP instead of the bracketed+ // ModernBERT form (e.g. mmBERT-style vocabularies). FirstKnownToken must fall through to+ // that spelling when the bracketed one is absent.+ var vocab = """{"<pad>":0,"<unk>":1,"<bos>":2,"<eos>":3,"<mask>":4,"hello":5}""";+ var added = string.Join(",", [+ AddedToken("<pad>", 0), AddedToken("<unk>", 1), AddedToken("<bos>", 2),+ AddedToken("<eos>", 3), AddedToken("<mask>", 4),+ ]);+ WriteTokenizerJson(added, vocab);++ using var tok = new HfTokenizer(Path.Combine(_dir, "tokenizer.json"));++ Assert.Equal(0, tok.PadId);+ Assert.Equal(1, tok.UnkId);+ Assert.Equal(2, tok.ClsId);+ Assert.Equal(3, tok.SepId);+ Assert.Equal(4, tok.MaskId);+ Assert.Equal("<mask>", tok.MaskToken);+ }++ [Fact]+ public void NoTokenizerConfig_TakesClsAndSepFromThePostProcessorTemplate()+ {+ // The real multilingual (mmBERT) vocabulary holds "<s>"/"</s>" as ordinary tokens as well+ // as its real CLS/SEP "<bos>"/"<eos>" (ids 2 and 1). The post_processor template names the+ // right pair, so it must win over any spelling guess.+ var vocab = """{"<pad>":0,"<eos>":1,"<bos>":2,"<unk>":3,"<mask>":4,"<s>":5,"</s>":6,"hello":7}""";+ var added = string.Join(",", [+ AddedToken("<pad>", 0), AddedToken("<eos>", 1), AddedToken("<bos>", 2),+ AddedToken("<unk>", 3), AddedToken("<mask>", 4), AddedToken("<s>", 5), AddedToken("</s>", 6),+ ]);+ WriteTokenizerJson(added, vocab);+ var path = Path.Combine(_dir, "tokenizer.json");+ File.WriteAllText(path, File.ReadAllText(path).Replace("\"post_processor\": null", """+ "post_processor": {"type": "TemplateProcessing", "single": [+ {"SpecialToken": {"id": "<bos>", "type_id": 0}},+ {"Sequence": {"id": "A", "type_id": 0}},+ {"SpecialToken": {"id": "<eos>", "type_id": 0}}]}+ """));++ using var tok = new HfTokenizer(path);++ Assert.Equal(2, tok.ClsId);+ Assert.Equal(1, tok.SepId);+ Assert.Equal(0, tok.PadId);+ Assert.Equal(3, tok.UnkId);+ Assert.Equal(4, tok.MaskId);+ }++ [Fact]+ public void NoTokenizerConfig_NoRecognisedSpelling_ThrowsNamingTheRole()+ {+ // A vocabulary that uses none of the conventional spellings for cls_token must fail+ // loudly and name which role could not be resolved, rather than silently picking an+ // unrelated id or crashing somewhere downstream with no context.+ var vocab = """{"[PAD]":0,"[UNK]":1,"[SEP]":3,"[MASK]":4,"hello":5}""";+ var added = string.Join(",", [+ AddedToken("[PAD]", 0), AddedToken("[UNK]", 1),+ AddedToken("[SEP]", 3), AddedToken("[MASK]", 4),+ ]);+ WriteTokenizerJson(added, vocab);++ var ex = Assert.Throws<InvalidOperationException>(+ () => new HfTokenizer(Path.Combine(_dir, "tokenizer.json")));++ Assert.Contains("cls_token", ex.Message);+ }++ [Fact]+ public void TokenizerConfigPresent_UsesItInsteadOfGuessing()+ {+ // Fused layout: tokenizer_config.json is present, so the config-driven path (not the+ // fallback) must be the one that runs, even though the fallback's conventional spellings+ // would also happen to resolve on this vocabulary.+ var added = string.Join(",", [+ AddedToken("[PAD]", 0), AddedToken("[UNK]", 1), AddedToken("[CLS]", 2),+ AddedToken("[SEP]", 3), AddedToken("[MASK]", 4),+ ]);+ WriteTokenizerJson(added, BracketedVocab);+ File.WriteAllText(Path.Combine(_dir, "tokenizer_config.json"), """+ {+ "cls_token": "[CLS]",+ "sep_token": "[SEP]",+ "pad_token": "[PAD]",+ "unk_token": "[UNK]",+ "mask_token": "[MASK]"+ }+ """);++ using var tok = new HfTokenizer(Path.Combine(_dir, "tokenizer.json"));++ Assert.Equal(0, tok.PadId);+ Assert.Equal(2, tok.ClsId);+ Assert.Equal(3, tok.SepId);+ Assert.Equal(4, tok.MaskId);+ Assert.Equal(1, tok.UnkId);+ }+}diff --git a/laya-dotnet/tests/Laya.Tests/StubTokenizer.cs b/laya-dotnet/tests/Laya.Tests/StubTokenizer.csnew file mode 100644index 00000000..9e3112da--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/StubTokenizer.cs@@ -0,0 +1,53 @@+using Laya.Tokenization;++namespace Laya.Tests;++/// <summary>+/// A deterministic tokenizer with no model behind it, for testing sequence structure.+/// </summary>+/// <remarks>+/// It emits one id per whitespace-separated word, so a sequence's shape — marker positions, the+/// order of head, options and state, where truncation lands — can be asserted by counting words.+/// It says nothing about real token ids; that is what the golden vectors in+/// <see cref="TokenizerParityTests"/> are for.+/// </remarks>+public sealed class StubTokenizer : ILayaTokenizer+{+ /// <summary>Ids start here, above every special id, so a word is never mistaken for one.</summary>+ public const int FirstWordId = 100;++ /// <inheritdoc/>+ public int PadId => 0;++ /// <inheritdoc/>+ public int SepId => 1;++ /// <inheritdoc/>+ public int ClsId => 2;++ /// <inheritdoc/>+ public int UnkId => 3;++ /// <inheritdoc/>+ public int MaskId => 4;++ /// <inheritdoc/>+ public string MaskToken => "<mask>";++ /// <summary>How many times <see cref="Encode"/> has been called.</summary>+ public int EncodeCalls { get; private set; }++ /// <summary>Every text passed to <see cref="Encode"/>, in order.</summary>+ public List<string> Encoded { get; } = [];++ /// <inheritdoc/>+ public int[] Encode(string text)+ {+ EncodeCalls++;+ Encoded.Add(text);+ var words = text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);+ // A stable, collision-tolerant id per word: the test only needs determinism, and using the+ // hash keeps identical words mapping to identical ids across calls.+ return [.. words.Select(w => FirstWordId + (Math.Abs(w.GetHashCode(StringComparison.Ordinal)) % 10_000))];+ }+}