Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ts): fall back to byte tokens for characters the vocab lacks, as Hugging Face does

divit aggarwalea5875fe31d4
diff --git a/laya-ts/src/tokenizer.ts b/laya-ts/src/tokenizer.ts
index 89730610..f54c2575 100644
--- a/laya-ts/src/tokenizer.ts
+++ b/laya-ts/src/tokenizer.ts
@@ -27,6 +27,8 @@ export interface TokenizerData {
maskToken: string;
/** Normalizer Replace rules (pattern -> content) applied in order before pre-tokenizing. */
replaces: Array<[string, string]>;
+ /** A character the vocab lacks becomes its `<0xNN>` byte tokens instead of unk (HF byte_fallback). */
+ byteFallback?: boolean;
}
/** Metaspace word-boundary marker (HF SentencePiece-style replacement for ' '). */
@@ -258,6 +260,7 @@ export function metaspaceEncode(
text: string,
unkId?: number,
replaces: ReadonlyArray<readonly [string, string]> = [[" ", METASPACE_REPLACEMENT]],
+ byteFallback = false,
): number[] {
const unk = unkId ?? vocab.get("<unk>") ?? vocab.get("[UNK]") ?? CHECKPOINT_IDS.unk;
if (!text) return [];
@@ -265,7 +268,18 @@ export function metaspaceEncode(
for (const [from, to] of replaces) t = t.split(from).join(to);
const out: number[] = [];
const push = (piece: string): void => {
- for (const tok of bpeWord(Array.from(piece), merges)) out.push(vocab.get(tok) ?? unk);
+ for (const tok of bpeWord(Array.from(piece), merges)) {
+ const id = vocab.get(tok);
+ if (id !== undefined) {
+ out.push(id);
+ continue;
+ }
+ const bytes = byteFallback
+ ? Array.from((sharedEncoder ??= new TextEncoder()).encode(tok), (b) => vocab.get(`<0x${b.toString(16).toUpperCase().padStart(2, "0")}>`))
+ : [];
+ if (bytes.length > 0 && bytes.every((b) => b !== undefined)) out.push(...(bytes as number[]));
+ else out.push(unk);
+ }
};
for (const seg of t.split(/(\n+)/)) {
if (!seg) continue;
@@ -284,7 +298,7 @@ export function metaspaceEncode(
/** Dispatch to the Metaspace or GPT-2/ByteLevel encoder based on the parsed pre-tokenizer. */
export function encodeWithData(data: TokenizerData, text: string): number[] {
return data.kind === "metaspace"
- ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces)
+ ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces, data.byteFallback)
: bpeEncode(data.vocab, data.merges, text);
}
@@ -323,7 +337,7 @@ function collectReplaces(node: unknown, out: Array<[string, string]>): void {
export function parseTokenizerJson(raw: unknown): TokenizerData | null {
try {
const r = raw as {
- model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };
+ model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]>; byte_fallback?: boolean };
normalizer?: unknown;
pre_tokenizer?: unknown;
added_tokens?: Array<{ id?: number; content?: string }>;
@@ -362,6 +376,7 @@ export function parseTokenizerJson(raw: unknown): TokenizerData | null {
kind,
maskToken: mask.token,
replaces,
+ byteFallback: r.model?.byte_fallback === true,
};
} catch {
return null;
diff --git a/laya-ts/tests/tokenizer-parity.test.ts b/laya-ts/tests/tokenizer-parity.test.ts
index 8eeea468..6f632b3a 100644
--- a/laya-ts/tests/tokenizer-parity.test.ts
+++ b/laya-ts/tests/tokenizer-parity.test.ts
@@ -65,6 +65,49 @@ describe("parseTokenizerJson specials aliases", () => {
});
});
+// Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).
+// With byte_fallback a character the vocab lacks becomes its UTF-8 `<0xNN>` tokens, or <unk> if any of
+// those tokens is missing too. The Gemma checkpoint sets it: a Hangul syllable or CJK ideograph outside
+// the 256k vocab reached the model as <unk> here.
+describe("byte_fallback (HF parity)", () => {
+ const make = (byteFallback: boolean) => parseTokenizerJson({
+ normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },
+ pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },
+ model: {
+ byte_fallback: byteFallback,
+ vocab: {
+ "▁": 0, a: 1, b: 2, "▁a": 3, "<unk>": 4,
+ "<0xC7>": 5, "<0x85>": 6, "<0xF0>": 7, "<0x9F>": 8, "<0x98>": 9, "<0x80>": 10, "<0xE4>": 11,
+ },
+ merges: ["▁ a"],
+ },
+ })!;
+
+ it.each<[string, number[]]>([
+ ["Dž", [0, 5, 6]],
+ ["aDžb", [3, 5, 6, 2]],
+ ["DžDž", [0, 5, 6, 5, 6]],
+ ["a Dž", [3, 0, 5, 6]],
+ ["😀", [0, 7, 8, 9, 10]],
+ ["a😀b", [3, 7, 8, 9, 10, 2]],
+ ["é", [0, 4]],
+ ["aéb", [3, 4, 2]],
+ ["中", [0, 4]],
+ ["a b", [3, 0, 2]],
+ ])("byte_fallback on: %j", (text, ids) => {
+ expect(encodeWithData(make(true), text)).toEqual(ids);
+ });
+
+ it.each<[string, number[]]>([
+ ["Dž", [0, 4]],
+ ["DžDž", [0, 4, 4]],
+ ["😀", [0, 4]],
+ ["a b", [3, 0, 2]],
+ ])("byte_fallback off keeps <unk>: %j", (text, ids) => {
+ expect(encodeWithData(make(false), text)).toEqual(ids);
+ });
+});
+
describe("model-ml/tokenizer.json parity (gated on file presence)", () => {
it("reproduces reference id-sequences byte-identical", () => {
if (!existsSync(mlPath)) return;