Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): cut added tokens out of the text before tokenizing, as Hugging Face does

divit aggarwal6517cc11ac69
diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.ts
index 89730610..203bd2fd 100644
--- a/laya-ts/src/tokenizer.ts
+++ b/laya-ts/src/tokenizer.ts
@@ -27,8 +27,12 @@ export interface TokenizerData {
maskToken: string;
/** Normalizer Replace rules (pattern -> content) applied in order before pre-tokenizing. */
replaces: Array<[string, string]>;
+ /** Tokens cut out of the text before it is tokenized (HF added_tokens). */
+ added?: AddedToken[];
}
+export interface AddedToken { content: string; id: number; normalized: boolean; lstrip: boolean; rstrip: boolean }
+
/** Metaspace word-boundary marker (HF SentencePiece-style replacement for ' '). */
export const METASPACE_REPLACEMENT = "▁";
@@ -281,13 +285,87 @@ export function metaspaceEncode(
return out;
}
-/** Dispatch to the Metaspace or GPT-2/ByteLevel encoder based on the parsed pre-tokenizer. */
-export function encodeWithData(data: TokenizerData, text: string): number[] {
+function encodeSegment(data: TokenizerData, text: string): number[] {
return data.kind === "metaspace"
? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces)
: bpeEncode(data.vocab, data.merges, text);
}
+/** Regex source for `words`, longest match first, as a prefix tree: one branch per next character
+ * rather than one alternative per token, so a scan does not try every token at every position. */
+function trieSource(words: string[]): string {
+ type Node = { kids: Map<string, Node>; end: boolean };
+ const root: Node = { kids: new Map(), end: false };
+ for (const w of words) {
+ let n = root;
+ for (const ch of w) {
+ let k = n.kids.get(ch);
+ if (!k) n.kids.set(ch, (k = { kids: new Map(), end: false }));
+ n = k;
+ }
+ n.end = true;
+ }
+ const walk = (n: Node): string => {
+ const alts = [...n.kids].map(([c, k]) => c.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + walk(k));
+ if (!alts.length) return "";
+ const body = alts.length === 1 ? alts[0] : "(?:" + alts.join("|") + ")";
+ return n.end ? "(?:" + body + ")?" : body;
+ };
+ return walk(root);
+}
+
+type AddedPhase = { re: RegExp; byContent: Map<string, AddedToken>; normalized: boolean };
+const addedPhases = new WeakMap<TokenizerData, AddedPhase[]>();
+
+/** HF matches added tokens that are not `normalized` on the raw text first, then the `normalized` ones
+ * on what is left. */
+function phasesOf(data: TokenizerData): AddedPhase[] {
+ let phases = addedPhases.get(data);
+ if (!phases) {
+ phases = [false, true].flatMap((normalized) => {
+ const toks = (data.added ?? []).filter((t) => t.normalized === normalized && t.content);
+ if (!toks.length) return [];
+ const re = new RegExp(trieSource(toks.map((t) => t.content)), "gu");
+ return [{ re, byContent: new Map(toks.map((t) => [t.content, t])), normalized }];
+ });
+ addedPhases.set(data, phases);
+ }
+ return phases;
+}
+
+function splitOnAdded(text: string, { re, byContent }: AddedPhase): Array<string | AddedToken> {
+ const out: Array<string | AddedToken> = [];
+ let last = 0;
+ for (const m of text.matchAll(re)) {
+ const tok = byContent.get(m[0])!;
+ let before = text.slice(last, m.index);
+ if (tok.lstrip) before = before.trimEnd();
+ if (before) out.push(before);
+ out.push(tok);
+ last = m.index + m[0].length;
+ if (tok.rstrip) while (last < text.length && /\s/u.test(text[last])) last++;
+ }
+ if (last < text.length) out.push(text.slice(last));
+ return out;
+}
+
+/** Dispatch to the Metaspace or GPT-2/ByteLevel encoder, after cutting out added tokens as HF does. */
+export function encodeWithData(data: TokenizerData, text: string): number[] {
+ let pieces: Array<string | AddedToken> = [text];
+ for (const phase of phasesOf(data)) {
+ pieces = pieces.flatMap((p) => {
+ if (typeof p !== "string") return [p];
+ return splitOnAdded(phase.normalized && data.kind === "bytelevel" ? p.normalize("NFC") : p, phase);
+ });
+ }
+ const out: number[] = [];
+ for (const p of pieces) {
+ if (typeof p === "string") out.push(...encodeSegment(data, p));
+ else out.push(p.id);
+ }
+ return out;
+}
+
function childNodes(node: unknown): unknown[] {
if (!node || typeof node !== "object") return [];
const o = node as Record<string, unknown>;
@@ -326,7 +404,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {
model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };
normalizer?: unknown;
pre_tokenizer?: unknown;
- added_tokens?: Array<{ id?: number; content?: string }>;
+ added_tokens?: Array<{ id?: number; content?: string; normalized?: boolean; lstrip?: boolean; rstrip?: boolean }>;
};
const vocabObj = r?.model?.vocab;
if (!vocabObj || typeof vocabObj !== "object") return null;
@@ -337,8 +415,15 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {
if (pair.length >= 2) merges.set(pair[0] + " " + pair[1], i);
}
const added = new Map<string, number>();
+ const addedTokens: AddedToken[] = [];
for (const t of r.added_tokens ?? []) {
- if (typeof t?.content === "string" && typeof t?.id === "number") added.set(t.content, t.id);
+ if (typeof t?.content === "string" && typeof t?.id === "number") {
+ added.set(t.content, t.id);
+ addedTokens.push({
+ content: t.content, id: t.id,
+ normalized: t.normalized === true, lstrip: t.lstrip === true, rstrip: t.rstrip === true,
+ });
+ }
}
const pick = (aliases: readonly string[], fb: number): { id: number; token: string } => {
for (const a of aliases) {
@@ -362,6 +447,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {
kind,
maskToken: mask.token,
replaces,
+ added: addedTokens,
};
} catch {
return null;
diff --git a/laya-ts/tests/tokenizer-parity.test.ts b/laya-ts/tests/tokenizer-parity.test.ts
index 8eeea468..90c8d977 100644
--- a/laya-ts/tests/tokenizer-parity.test.ts
+++ b/laya-ts/tests/tokenizer-parity.test.ts
@@ -65,6 +65,67 @@ describe("parseTokenizerJson specials aliases", () => {
});
});
+// Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).
+// Added tokens are cut out of the text before it is tokenized: whitespace runs and placeholders in the
+// ModernBERT checkpoint, HTML tags and control tokens in the Gemma one.
+describe("added tokens (HF parity)", () => {
+ const added = (id: number, content: string, o: Record<string, unknown> = {}) =>
+ ({ id, content, single_word: false, lstrip: false, rstrip: false, normalized: false, special: false, ...o });
+ const byteLevel = parseTokenizerJson({
+ normalizer: { type: "NFC" },
+ pre_tokenizer: { type: "ByteLevel", add_prefix_space: false, use_regex: true },
+ model: {
+ vocab: { a: 0, b: 1, c: 2, "Ġ": 3, "Ċ": 4, "Ġa": 5, "Ġb": 6, "|": 7 },
+ merges: ["Ġ a", "Ġ b"],
+ },
+ added_tokens: [
+ added(8, " ", { normalized: true }), added(9, " ", { normalized: true }),
+ added(10, "|||X|||", { normalized: true }),
+ added(11, "[SEP]", { special: true }), added(12, "[MASK]", { special: true, lstrip: true }),
+ ],
+ })!;
+ const metaspace = parseTokenizerJson({
+ normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },
+ pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },
+ model: {
+ vocab: { "▁": 0, a: 1, b: 2, "▁a": 3, "▁b": 4, "<unk>": 5, c: 6, "▁c": 7 },
+ merges: ["▁ a", "▁ b", "▁ c"],
+ },
+ added_tokens: [added(8, "<t>"), added(9, "<eos>", { special: true })],
+ })!;
+
+ it.each<[string, number[]]>([
+ ["a b", [0, 6]],
+ ["a b", [0, 8, 1]],
+ ["a b", [0, 9, 1]],
+ ["a b", [0, 9, 6]],
+ [" a", [8, 0]],
+ ["a ", [0, 8]],
+ ["a|||X|||b", [0, 10, 1]],
+ ["a[SEP]b", [0, 11, 1]],
+ ["a [MASK] b", [0, 12, 6]],
+ ["a [SEP] b", [0, 3, 11, 6]],
+ ["a\n b", [0, 4, 8, 1]],
+ ["ab ca", [0, 1, 8, 2, 0]],
+ ])("ByteLevel: %j", (text, ids) => {
+ expect(encodeWithData(byteLevel, text)).toEqual(ids);
+ });
+
+ it.each<[string, number[]]>([
+ ["a<t>b", [3, 8, 4]],
+ ["<t>", [8]],
+ ["a <t> b", [3, 0, 8, 4]],
+ ["<t><t>", [8, 8]],
+ ["a<eos>b", [3, 9, 4]],
+ ["a b", [3, 0, 4]],
+ ["<t>a", [8, 3]],
+ ["a<t>", [3, 8]],
+ ["a <eos>", [3, 0, 9]],
+ ])("Metaspace: %j", (text, ids) => {
+ expect(encodeWithData(metaspace, text)).toEqual(ids);
+ });
+});
+
describe("model-ml/tokenizer.json parity (gated on file presence)", () => {
it("reproduces reference id-sequences byte-identical", () => {
if (!existsSync(mlPath)) return;