Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ts): cut added tokens out of the text before tokenizing, as Hugging Face does
divit aggarwal6517cc11ac69
diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.tsindex 89730610..203bd2fd 100644--- a/laya-ts/src/tokenizer.ts+++ b/laya-ts/src/tokenizer.ts@@ -27,8 +27,12 @@ export interface TokenizerData {maskToken: string;/** Normalizer Replace rules (pattern -> content) applied in order before pre-tokenizing. */replaces: Array<[string, string]>;+ /** Tokens cut out of the text before it is tokenized (HF added_tokens). */+ added?: AddedToken[];}+export interface AddedToken { content: string; id: number; normalized: boolean; lstrip: boolean; rstrip: boolean }+/** Metaspace word-boundary marker (HF SentencePiece-style replacement for ' '). */export const METASPACE_REPLACEMENT = "▁";@@ -281,13 +285,87 @@ export function metaspaceEncode(return out;}-/** Dispatch to the Metaspace or GPT-2/ByteLevel encoder based on the parsed pre-tokenizer. */-export function encodeWithData(data: TokenizerData, text: string): number[] {+function encodeSegment(data: TokenizerData, text: string): number[] {return data.kind === "metaspace"? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces): bpeEncode(data.vocab, data.merges, text);}+/** Regex source for `words`, longest match first, as a prefix tree: one branch per next character+ * rather than one alternative per token, so a scan does not try every token at every position. */+function trieSource(words: string[]): string {+ type Node = { kids: Map<string, Node>; end: boolean };+ const root: Node = { kids: new Map(), end: false };+ for (const w of words) {+ let n = root;+ for (const ch of w) {+ let k = n.kids.get(ch);+ if (!k) n.kids.set(ch, (k = { kids: new Map(), end: false }));+ n = k;+ }+ n.end = true;+ }+ const walk = (n: Node): string => {+ const alts = [...n.kids].map(([c, k]) => c.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + walk(k));+ if (!alts.length) return "";+ const body = alts.length === 1 ? alts[0] : "(?:" + alts.join("|") + ")";+ return n.end ? "(?:" + body + ")?" : body;+ };+ return walk(root);+}++type AddedPhase = { re: RegExp; byContent: Map<string, AddedToken>; normalized: boolean };+const addedPhases = new WeakMap<TokenizerData, AddedPhase[]>();++/** HF matches added tokens that are not `normalized` on the raw text first, then the `normalized` ones+ * on what is left. */+function phasesOf(data: TokenizerData): AddedPhase[] {+ let phases = addedPhases.get(data);+ if (!phases) {+ phases = [false, true].flatMap((normalized) => {+ const toks = (data.added ?? []).filter((t) => t.normalized === normalized && t.content);+ if (!toks.length) return [];+ const re = new RegExp(trieSource(toks.map((t) => t.content)), "gu");+ return [{ re, byContent: new Map(toks.map((t) => [t.content, t])), normalized }];+ });+ addedPhases.set(data, phases);+ }+ return phases;+}++function splitOnAdded(text: string, { re, byContent }: AddedPhase): Array<string | AddedToken> {+ const out: Array<string | AddedToken> = [];+ let last = 0;+ for (const m of text.matchAll(re)) {+ const tok = byContent.get(m[0])!;+ let before = text.slice(last, m.index);+ if (tok.lstrip) before = before.trimEnd();+ if (before) out.push(before);+ out.push(tok);+ last = m.index + m[0].length;+ if (tok.rstrip) while (last < text.length && /\s/u.test(text[last])) last++;+ }+ if (last < text.length) out.push(text.slice(last));+ return out;+}++/** Dispatch to the Metaspace or GPT-2/ByteLevel encoder, after cutting out added tokens as HF does. */+export function encodeWithData(data: TokenizerData, text: string): number[] {+ let pieces: Array<string | AddedToken> = [text];+ for (const phase of phasesOf(data)) {+ pieces = pieces.flatMap((p) => {+ if (typeof p !== "string") return [p];+ return splitOnAdded(phase.normalized && data.kind === "bytelevel" ? p.normalize("NFC") : p, phase);+ });+ }+ const out: number[] = [];+ for (const p of pieces) {+ if (typeof p === "string") out.push(...encodeSegment(data, p));+ else out.push(p.id);+ }+ return out;+}+function childNodes(node: unknown): unknown[] {if (!node || typeof node !== "object") return [];const o = node as Record<string, unknown>;@@ -326,7 +404,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };normalizer?: unknown;pre_tokenizer?: unknown;- added_tokens?: Array<{ id?: number; content?: string }>;+ added_tokens?: Array<{ id?: number; content?: string; normalized?: boolean; lstrip?: boolean; rstrip?: boolean }>;};const vocabObj = r?.model?.vocab;if (!vocabObj || typeof vocabObj !== "object") return null;@@ -337,8 +415,15 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {if (pair.length >= 2) merges.set(pair[0] + " " + pair[1], i);}const added = new Map<string, number>();+ const addedTokens: AddedToken[] = [];for (const t of r.added_tokens ?? []) {- if (typeof t?.content === "string" && typeof t?.id === "number") added.set(t.content, t.id);+ if (typeof t?.content === "string" && typeof t?.id === "number") {+ added.set(t.content, t.id);+ addedTokens.push({+ content: t.content, id: t.id,+ normalized: t.normalized === true, lstrip: t.lstrip === true, rstrip: t.rstrip === true,+ });+ }}const pick = (aliases: readonly string[], fb: number): { id: number; token: string } => {for (const a of aliases) {@@ -362,6 +447,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {kind,maskToken: mask.token,replaces,+ added: addedTokens,};} catch {return null;diff --git a/laya-ts/tests/tokenizer-parity.test.ts b/laya-ts/tests/tokenizer-parity.test.tsindex 8eeea468..90c8d977 100644--- a/laya-ts/tests/tokenizer-parity.test.ts+++ b/laya-ts/tests/tokenizer-parity.test.ts@@ -65,6 +65,67 @@ describe("parseTokenizerJson specials aliases", () => {});});+// Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).+// Added tokens are cut out of the text before it is tokenized: whitespace runs and placeholders in the+// ModernBERT checkpoint, HTML tags and control tokens in the Gemma one.+describe("added tokens (HF parity)", () => {+ const added = (id: number, content: string, o: Record<string, unknown> = {}) =>+ ({ id, content, single_word: false, lstrip: false, rstrip: false, normalized: false, special: false, ...o });+ const byteLevel = parseTokenizerJson({+ normalizer: { type: "NFC" },+ pre_tokenizer: { type: "ByteLevel", add_prefix_space: false, use_regex: true },+ model: {+ vocab: { a: 0, b: 1, c: 2, "Ġ": 3, "Ċ": 4, "Ġa": 5, "Ġb": 6, "|": 7 },+ merges: ["Ġ a", "Ġ b"],+ },+ added_tokens: [+ added(8, " ", { normalized: true }), added(9, " ", { normalized: true }),+ added(10, "|||X|||", { normalized: true }),+ added(11, "[SEP]", { special: true }), added(12, "[MASK]", { special: true, lstrip: true }),+ ],+ })!;+ const metaspace = parseTokenizerJson({+ normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },+ pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },+ model: {+ vocab: { "▁": 0, a: 1, b: 2, "▁a": 3, "▁b": 4, "<unk>": 5, c: 6, "▁c": 7 },+ merges: ["▁ a", "▁ b", "▁ c"],+ },+ added_tokens: [added(8, "<t>"), added(9, "<eos>", { special: true })],+ })!;++ it.each<[string, number[]]>([+ ["a b", [0, 6]],+ ["a b", [0, 8, 1]],+ ["a b", [0, 9, 1]],+ ["a b", [0, 9, 6]],+ [" a", [8, 0]],+ ["a ", [0, 8]],+ ["a|||X|||b", [0, 10, 1]],+ ["a[SEP]b", [0, 11, 1]],+ ["a [MASK] b", [0, 12, 6]],+ ["a [SEP] b", [0, 3, 11, 6]],+ ["a\n b", [0, 4, 8, 1]],+ ["ab ca", [0, 1, 8, 2, 0]],+ ])("ByteLevel: %j", (text, ids) => {+ expect(encodeWithData(byteLevel, text)).toEqual(ids);+ });++ it.each<[string, number[]]>([+ ["a<t>b", [3, 8, 4]],+ ["<t>", [8]],+ ["a <t> b", [3, 0, 8, 4]],+ ["<t><t>", [8, 8]],+ ["a<eos>b", [3, 9, 4]],+ ["a b", [3, 0, 4]],+ ["<t>a", [8, 3]],+ ["a<t>", [3, 8]],+ ["a <eos>", [3, 0, 9]],+ ])("Metaspace: %j", (text, ids) => {+ expect(encodeWithData(metaspace, text)).toEqual(ids);+ });+});+describe("model-ml/tokenizer.json parity (gated on file presence)", () => {it("reproduces reference id-sequences byte-identical", () => {if (!existsSync(mlPath)) return;