Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #527 from Divit-aggarwal/fix/ts-tokenizer-added-tokens
Nandakishor7a75c291144f
diff --cc laya-ts/src/tokenizer.tsindex f54c2575,203bd2fd..3e460d0d--- a/laya-ts/src/tokenizer.ts+++ b/laya-ts/src/tokenizer.ts@@@ -27,10 -27,12 +27,14 @@@ export interface TokenizerDatamaskToken: string;/** Normalizer Replace rules (pattern -> content) applied in order before pre-tokenizing. */replaces: Array<[string, string]>;+ /** A character the vocab lacks becomes its `<0xNN>` byte tokens instead of unk (HF byte_fallback). */+ byteFallback?: boolean;+ /** Tokens cut out of the text before it is tokenized (HF added_tokens). */+ added?: AddedToken[];}+ export interface AddedToken { content: string; id: number; normalized: boolean; lstrip: boolean; rstrip: boolean }+/** Metaspace word-boundary marker (HF SentencePiece-style replacement for ' '). */export const METASPACE_REPLACEMENT = "▁";@@@ -295,10 -285,9 +299,9 @@@ export function metaspaceEncodereturn out;}- /** Dispatch to the Metaspace or GPT-2/ByteLevel encoder based on the parsed pre-tokenizer. */- export function encodeWithData(data: TokenizerData, text: string): number[] {+ function encodeSegment(data: TokenizerData, text: string): number[] {return data.kind === "metaspace"- ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces)+ ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces, data.byteFallback): bpeEncode(data.vocab, data.merges, text);}@@@ -337,10 -401,10 +415,10 @@@ function collectReplaces(node: unknownexport function parseTokenizerJson(raw: unknown): TokenizerData | null {try {const r = raw as {- model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };+ model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]>; byte_fallback?: boolean };normalizer?: unknown;pre_tokenizer?: unknown;- added_tokens?: Array<{ id?: number; content?: string }>;+ added_tokens?: Array<{ id?: number; content?: string; normalized?: boolean; lstrip?: boolean; rstrip?: boolean }>;};const vocabObj = r?.model?.vocab;if (!vocabObj || typeof vocabObj !== "object") return null;@@@ -376,7 -447,7 +461,8 @@@kind,maskToken: mask.token,replaces,+ byteFallback: r.model?.byte_fallback === true,+ added: addedTokens,};} catch {return null;diff --cc laya-ts/tests/tokenizer-parity.test.tsindex 6f632b3a,90c8d977..0191f33a--- a/laya-ts/tests/tokenizer-parity.test.ts+++ b/laya-ts/tests/tokenizer-parity.test.ts@@@ -65,49 -65,67 +65,110 @@@ describe("parseTokenizerJson specials a});});+// Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).+// With byte_fallback a character the vocab lacks becomes its UTF-8 `<0xNN>` tokens, or <unk> if any of+// those tokens is missing too. The Gemma checkpoint sets it: a Hangul syllable or CJK ideograph outside+// the 256k vocab reached the model as <unk> here.+describe("byte_fallback (HF parity)", () => {+ const make = (byteFallback: boolean) => parseTokenizerJson({+ normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },+ pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },+ model: {+ byte_fallback: byteFallback,+ vocab: {+ "▁": 0, a: 1, b: 2, "▁a": 3, "<unk>": 4,+ "<0xC7>": 5, "<0x85>": 6, "<0xF0>": 7, "<0x9F>": 8, "<0x98>": 9, "<0x80>": 10, "<0xE4>": 11,+ },+ merges: ["▁ a"],+ },+ })!;++ it.each<[string, number[]]>([+ ["Dž", [0, 5, 6]],+ ["aDžb", [3, 5, 6, 2]],+ ["DžDž", [0, 5, 6, 5, 6]],+ ["a Dž", [3, 0, 5, 6]],+ ["😀", [0, 7, 8, 9, 10]],+ ["a😀b", [3, 7, 8, 9, 10, 2]],+ ["é", [0, 4]],+ ["aéb", [3, 4, 2]],+ ["中", [0, 4]],+ ["a b", [3, 0, 2]],+ ])("byte_fallback on: %j", (text, ids) => {+ expect(encodeWithData(make(true), text)).toEqual(ids);+ });++ it.each<[string, number[]]>([+ ["Dž", [0, 4]],+ ["DžDž", [0, 4, 4]],+ ["😀", [0, 4]],+ ["a b", [3, 0, 2]],+ ])("byte_fallback off keeps <unk>: %j", (text, ids) => {+ expect(encodeWithData(make(false), text)).toEqual(ids);+ });+});++ // Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).+ // Added tokens are cut out of the text before it is tokenized: whitespace runs and placeholders in the+ // ModernBERT checkpoint, HTML tags and control tokens in the Gemma one.+ describe("added tokens (HF parity)", () => {+ const added = (id: number, content: string, o: Record<string, unknown> = {}) =>+ ({ id, content, single_word: false, lstrip: false, rstrip: false, normalized: false, special: false, ...o });+ const byteLevel = parseTokenizerJson({+ normalizer: { type: "NFC" },+ pre_tokenizer: { type: "ByteLevel", add_prefix_space: false, use_regex: true },+ model: {+ vocab: { a: 0, b: 1, c: 2, "Ġ": 3, "Ċ": 4, "Ġa": 5, "Ġb": 6, "|": 7 },+ merges: ["Ġ a", "Ġ b"],+ },+ added_tokens: [+ added(8, " ", { normalized: true }), added(9, " ", { normalized: true }),+ added(10, "|||X|||", { normalized: true }),+ added(11, "[SEP]", { special: true }), added(12, "[MASK]", { special: true, lstrip: true }),+ ],+ })!;+ const metaspace = parseTokenizerJson({+ normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },+ pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },+ model: {+ vocab: { "▁": 0, a: 1, b: 2, "▁a": 3, "▁b": 4, "<unk>": 5, c: 6, "▁c": 7 },+ merges: ["▁ a", "▁ b", "▁ c"],+ },+ added_tokens: [added(8, "<t>"), added(9, "<eos>", { special: true })],+ })!;++ it.each<[string, number[]]>([+ ["a b", [0, 6]],+ ["a b", [0, 8, 1]],+ ["a b", [0, 9, 1]],+ ["a b", [0, 9, 6]],+ [" a", [8, 0]],+ ["a ", [0, 8]],+ ["a|||X|||b", [0, 10, 1]],+ ["a[SEP]b", [0, 11, 1]],+ ["a [MASK] b", [0, 12, 6]],+ ["a [SEP] b", [0, 3, 11, 6]],+ ["a\n b", [0, 4, 8, 1]],+ ["ab ca", [0, 1, 8, 2, 0]],+ ])("ByteLevel: %j", (text, ids) => {+ expect(encodeWithData(byteLevel, text)).toEqual(ids);+ });++ it.each<[string, number[]]>([+ ["a<t>b", [3, 8, 4]],+ ["<t>", [8]],+ ["a <t> b", [3, 0, 8, 4]],+ ["<t><t>", [8, 8]],+ ["a<eos>b", [3, 9, 4]],+ ["a b", [3, 0, 4]],+ ["<t>a", [8, 3]],+ ["a<t>", [3, 8]],+ ["a <eos>", [3, 0, 9]],+ ])("Metaspace: %j", (text, ids) => {+ expect(encodeWithData(metaspace, text)).toEqual(ids);+ });+ });+describe("model-ml/tokenizer.json parity (gated on file presence)", () => {it("reproduces reference id-sequences byte-identical", () => {if (!existsSync(mlPath)) return;