Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): fall back to byte tokens for characters the vocab lacks, as Hugging Face does
divit aggarwalea5875fe31d4
diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.tsindex 89730610..f54c2575 100644--- a/laya-ts/src/tokenizer.ts+++ b/laya-ts/src/tokenizer.ts@@ -27,6 +27,8 @@ export interface TokenizerData {maskToken: string;/** Normalizer Replace rules (pattern -> content) applied in order before pre-tokenizing. */replaces: Array<[string, string]>;+ /** A character the vocab lacks becomes its `<0xNN>` byte tokens instead of unk (HF byte_fallback). */+ byteFallback?: boolean;}/** Metaspace word-boundary marker (HF SentencePiece-style replacement for ' '). */@@ -258,6 +260,7 @@ export function metaspaceEncode(text: string,unkId?: number,replaces: ReadonlyArray<readonly [string, string]> = [[" ", METASPACE_REPLACEMENT]],+ byteFallback = false,): number[] {const unk = unkId ?? vocab.get("<unk>") ?? vocab.get("[UNK]") ?? CHECKPOINT_IDS.unk;if (!text) return [];@@ -265,7 +268,18 @@ export function metaspaceEncode(for (const [from, to] of replaces) t = t.split(from).join(to);const out: number[] = [];const push = (piece: string): void => {- for (const tok of bpeWord(Array.from(piece), merges)) out.push(vocab.get(tok) ?? unk);+ for (const tok of bpeWord(Array.from(piece), merges)) {+ const id = vocab.get(tok);+ if (id !== undefined) {+ out.push(id);+ continue;+ }+ const bytes = byteFallback+ ? Array.from((sharedEncoder ??= new TextEncoder()).encode(tok), (b) => vocab.get(`<0x${b.toString(16).toUpperCase().padStart(2, "0")}>`))+ : [];+ if (bytes.length > 0 && bytes.every((b) => b !== undefined)) out.push(...(bytes as number[]));+ else out.push(unk);+ }};for (const seg of t.split(/(\n+)/)) {if (!seg) continue;@@ -284,7 +298,7 @@ export function metaspaceEncode(/** Dispatch to the Metaspace or GPT-2/ByteLevel encoder based on the parsed pre-tokenizer. */export function encodeWithData(data: TokenizerData, text: string): number[] {return data.kind === "metaspace"- ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces)+ ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces, data.byteFallback): bpeEncode(data.vocab, data.merges, text);}@@ -323,7 +337,7 @@ function collectReplaces(node: unknown, out: Array<[string, string]>): void {export function parseTokenizerJson(raw: unknown): TokenizerData | null {try {const r = raw as {- model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };+ model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]>; byte_fallback?: boolean };normalizer?: unknown;pre_tokenizer?: unknown;added_tokens?: Array<{ id?: number; content?: string }>;@@ -362,6 +376,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {kind,maskToken: mask.token,replaces,+ byteFallback: r.model?.byte_fallback === true,};} catch {return null;diff --git a/laya-ts/tests/tokenizer-parity.test.ts b/laya-ts/tests/tokenizer-parity.test.tsindex 8eeea468..6f632b3a 100644--- a/laya-ts/tests/tokenizer-parity.test.ts+++ b/laya-ts/tests/tokenizer-parity.test.ts@@ -65,6 +65,49 @@ describe("parseTokenizerJson specials aliases", () => {});});+// Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).+// With byte_fallback a character the vocab lacks becomes its UTF-8 `<0xNN>` tokens, or <unk> if any of+// those tokens is missing too. The Gemma checkpoint sets it: a Hangul syllable or CJK ideograph outside+// the 256k vocab reached the model as <unk> here.+describe("byte_fallback (HF parity)", () => {+ const make = (byteFallback: boolean) => parseTokenizerJson({+ normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },+ pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },+ model: {+ byte_fallback: byteFallback,+ vocab: {+ "▁": 0, a: 1, b: 2, "▁a": 3, "<unk>": 4,+ "<0xC7>": 5, "<0x85>": 6, "<0xF0>": 7, "<0x9F>": 8, "<0x98>": 9, "<0x80>": 10, "<0xE4>": 11,+ },+ merges: ["▁ a"],+ },+ })!;++ it.each<[string, number[]]>([+ ["Dž", [0, 5, 6]],+ ["aDžb", [3, 5, 6, 2]],+ ["DžDž", [0, 5, 6, 5, 6]],+ ["a Dž", [3, 0, 5, 6]],+ ["😀", [0, 7, 8, 9, 10]],+ ["a😀b", [3, 7, 8, 9, 10, 2]],+ ["é", [0, 4]],+ ["aéb", [3, 4, 2]],+ ["中", [0, 4]],+ ["a b", [3, 0, 2]],+ ])("byte_fallback on: %j", (text, ids) => {+ expect(encodeWithData(make(true), text)).toEqual(ids);+ });++ it.each<[string, number[]]>([+ ["Dž", [0, 4]],+ ["DžDž", [0, 4, 4]],+ ["😀", [0, 4]],+ ["a b", [3, 0, 2]],+ ])("byte_fallback off keeps <unk>: %j", (text, ids) => {+ expect(encodeWithData(make(false), text)).toEqual(ids);+ });+});+describe("model-ml/tokenizer.json parity (gated on file presence)", () => {it("reproduces reference id-sequences byte-identical", () => {if (!existsSync(mlPath)) return;