Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #527 from Divit-aggarwal/fix/ts-tokenizer-added-tokens

Nandakishor7a75c291144f
diff --cc laya-ts/src/tokenizer.ts
index f54c2575,203bd2fd..3e460d0d
--- a/laya-ts/src/tokenizer.ts
+++ b/laya-ts/src/tokenizer.ts
@@@ -27,10 -27,12 +27,14 @@@ export interface TokenizerData
maskToken: string;
/** Normalizer Replace rules (pattern -> content) applied in order before pre-tokenizing. */
replaces: Array<[string, string]>;
+ /** A character the vocab lacks becomes its `<0xNN>` byte tokens instead of unk (HF byte_fallback). */
+ byteFallback?: boolean;
+ /** Tokens cut out of the text before it is tokenized (HF added_tokens). */
+ added?: AddedToken[];
}
+ export interface AddedToken { content: string; id: number; normalized: boolean; lstrip: boolean; rstrip: boolean }
+
/** Metaspace word-boundary marker (HF SentencePiece-style replacement for ' '). */
export const METASPACE_REPLACEMENT = "▁";
@@@ -295,10 -285,9 +299,9 @@@ export function metaspaceEncode
return out;
}
- /** Dispatch to the Metaspace or GPT-2/ByteLevel encoder based on the parsed pre-tokenizer. */
- export function encodeWithData(data: TokenizerData, text: string): number[] {
+ function encodeSegment(data: TokenizerData, text: string): number[] {
return data.kind === "metaspace"
- ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces)
+ ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces, data.byteFallback)
: bpeEncode(data.vocab, data.merges, text);
}
@@@ -337,10 -401,10 +415,10 @@@ function collectReplaces(node: unknown
export function parseTokenizerJson(raw: unknown): TokenizerData | null {
try {
const r = raw as {
- model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };
+ model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]>; byte_fallback?: boolean };
normalizer?: unknown;
pre_tokenizer?: unknown;
- added_tokens?: Array<{ id?: number; content?: string }>;
+ added_tokens?: Array<{ id?: number; content?: string; normalized?: boolean; lstrip?: boolean; rstrip?: boolean }>;
};
const vocabObj = r?.model?.vocab;
if (!vocabObj || typeof vocabObj !== "object") return null;
@@@ -376,7 -447,7 +461,8 @@@
kind,
maskToken: mask.token,
replaces,
+ byteFallback: r.model?.byte_fallback === true,
+ added: addedTokens,
};
} catch {
return null;
diff --cc laya-ts/tests/tokenizer-parity.test.ts
index 6f632b3a,90c8d977..0191f33a
--- a/laya-ts/tests/tokenizer-parity.test.ts
+++ b/laya-ts/tests/tokenizer-parity.test.ts
@@@ -65,49 -65,67 +65,110 @@@ describe("parseTokenizerJson specials a
});
});
+// Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).
+// With byte_fallback a character the vocab lacks becomes its UTF-8 `<0xNN>` tokens, or <unk> if any of
+// those tokens is missing too. The Gemma checkpoint sets it: a Hangul syllable or CJK ideograph outside
+// the 256k vocab reached the model as <unk> here.
+describe("byte_fallback (HF parity)", () => {
+ const make = (byteFallback: boolean) => parseTokenizerJson({
+ normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },
+ pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },
+ model: {
+ byte_fallback: byteFallback,
+ vocab: {
+ "▁": 0, a: 1, b: 2, "▁a": 3, "<unk>": 4,
+ "<0xC7>": 5, "<0x85>": 6, "<0xF0>": 7, "<0x9F>": 8, "<0x98>": 9, "<0x80>": 10, "<0xE4>": 11,
+ },
+ merges: ["▁ a"],
+ },
+ })!;
+
+ it.each<[string, number[]]>([
+ ["Dž", [0, 5, 6]],
+ ["aDžb", [3, 5, 6, 2]],
+ ["DžDž", [0, 5, 6, 5, 6]],
+ ["a Dž", [3, 0, 5, 6]],
+ ["😀", [0, 7, 8, 9, 10]],
+ ["a😀b", [3, 7, 8, 9, 10, 2]],
+ ["é", [0, 4]],
+ ["aéb", [3, 4, 2]],
+ ["中", [0, 4]],
+ ["a b", [3, 0, 2]],
+ ])("byte_fallback on: %j", (text, ids) => {
+ expect(encodeWithData(make(true), text)).toEqual(ids);
+ });
+
+ it.each<[string, number[]]>([
+ ["Dž", [0, 4]],
+ ["DžDž", [0, 4, 4]],
+ ["😀", [0, 4]],
+ ["a b", [3, 0, 2]],
+ ])("byte_fallback off keeps <unk>: %j", (text, ids) => {
+ expect(encodeWithData(make(false), text)).toEqual(ids);
+ });
+});
+
+ // Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).
+ // Added tokens are cut out of the text before it is tokenized: whitespace runs and placeholders in the
+ // ModernBERT checkpoint, HTML tags and control tokens in the Gemma one.
+ describe("added tokens (HF parity)", () => {
+ const added = (id: number, content: string, o: Record<string, unknown> = {}) =>
+ ({ id, content, single_word: false, lstrip: false, rstrip: false, normalized: false, special: false, ...o });
+ const byteLevel = parseTokenizerJson({
+ normalizer: { type: "NFC" },
+ pre_tokenizer: { type: "ByteLevel", add_prefix_space: false, use_regex: true },
+ model: {
+ vocab: { a: 0, b: 1, c: 2, "Ġ": 3, "Ċ": 4, "Ġa": 5, "Ġb": 6, "|": 7 },
+ merges: ["Ġ a", "Ġ b"],
+ },
+ added_tokens: [
+ added(8, " ", { normalized: true }), added(9, " ", { normalized: true }),
+ added(10, "|||X|||", { normalized: true }),
+ added(11, "[SEP]", { special: true }), added(12, "[MASK]", { special: true, lstrip: true }),
+ ],
+ })!;
+ const metaspace = parseTokenizerJson({
+ normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },
+ pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },
+ model: {
+ vocab: { "▁": 0, a: 1, b: 2, "▁a": 3, "▁b": 4, "<unk>": 5, c: 6, "▁c": 7 },
+ merges: ["▁ a", "▁ b", "▁ c"],
+ },
+ added_tokens: [added(8, "<t>"), added(9, "<eos>", { special: true })],
+ })!;
+
+ it.each<[string, number[]]>([
+ ["a b", [0, 6]],
+ ["a b", [0, 8, 1]],
+ ["a b", [0, 9, 1]],
+ ["a b", [0, 9, 6]],
+ [" a", [8, 0]],
+ ["a ", [0, 8]],
+ ["a|||X|||b", [0, 10, 1]],
+ ["a[SEP]b", [0, 11, 1]],
+ ["a [MASK] b", [0, 12, 6]],
+ ["a [SEP] b", [0, 3, 11, 6]],
+ ["a\n b", [0, 4, 8, 1]],
+ ["ab ca", [0, 1, 8, 2, 0]],
+ ])("ByteLevel: %j", (text, ids) => {
+ expect(encodeWithData(byteLevel, text)).toEqual(ids);
+ });
+
+ it.each<[string, number[]]>([
+ ["a<t>b", [3, 8, 4]],
+ ["<t>", [8]],
+ ["a <t> b", [3, 0, 8, 4]],
+ ["<t><t>", [8, 8]],
+ ["a<eos>b", [3, 9, 4]],
+ ["a b", [3, 0, 4]],
+ ["<t>a", [8, 3]],
+ ["a<t>", [3, 8]],
+ ["a <eos>", [3, 0, 9]],
+ ])("Metaspace: %j", (text, ids) => {
+ expect(encodeWithData(metaspace, text)).toEqual(ids);
+ });
+ });
+
describe("model-ml/tokenizer.json parity (gated on file presence)", () => {
it("reproduces reference id-sequences byte-identical", () => {
if (!existsSync(mlPath)) return;