Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
// laya-ts/tests/tokenizer-parity.test.ts — Gemma-style (Metaspace BPE) parity.
// Reference vectors from transformers AutoTokenizer on model-ml/tokenizer.json,
// add_special_tokens=False. Hardcoded so the suite passes on any machine; when the
// real model-ml/tokenizer.json is present it is loaded and checked byte-identical.
import { describe, expect, it } from "vitest";
import { existsSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { encodeWithData, metaspaceEncode, parseTokenizerJson } from "../src/tokenizer.js";
const EN = "choice question: Which department should handle this request?";
const EN_IDS = [6241, 2872, 235292, 12236, 9888, 1412, 6589, 736, 3853, 235336];
const HI = "मुझसे दो बार शुल्क लिया गया";
const HI_IDS = [39728, 238385, 20579, 51484, 99744, 144905, 228834, 158850, 44370];
const here = dirname(fileURLToPath(import.meta.url));
const mlPath = resolve(here, "../../model-ml/tokenizer.json");
describe("metaspaceEncode (synthetic)", () => {
const vocab = new Map([
["▁hi", 10], ["▁", 11], ["h", 12], ["i", 13], ["<unk>", 3],
]);
const joins = new Map([["▁ h", 0], ["▁h i", 1]]);
it("prepends one marker, splits keeping marks, no lowercase", () => {
expect(metaspaceEncode(vocab, joins, "hi")).toEqual([10]);
expect(metaspaceEncode(vocab, joins, " hi")).toEqual([10]);
expect(metaspaceEncode(vocab, joins, " hi")).toEqual([11, 10]);
expect(metaspaceEncode(vocab, new Map(), "")).toEqual([]);
expect(metaspaceEncode(vocab, joins, "HI")).toEqual([11, 3, 3]);
});
it("unknown piece -> unk id", () => {
expect(metaspaceEncode(vocab, joins, "z")).toEqual([11, 3]);
});
});
describe("parseTokenizerJson specials aliases", () => {
it("resolves Gemma <bos>/<eos>/<pad>/<mask>/<unk> (cls=2 sep=1 pad=0 mask=4 unk=3)", () => {
const data = parseTokenizerJson({
model: { vocab: { "▁a": 10 }, merges: [] },
normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },
pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },
added_tokens: [
{ id: 0, content: "<pad>" }, { id: 1, content: "<eos>" }, { id: 2, content: "<bos>" },
{ id: 3, content: "<unk>" }, { id: 4, content: "<mask>" },
],
})!;
expect(data.ids).toEqual({ cls: 2, sep: 1, mask: 4, pad: 0, unk: 3 });
expect(data.kind).toBe("metaspace");
expect(data.maskToken).toBe("<mask>");
expect(data.replaces).toEqual([[" ", "▁"]]);
});
it("keeps ModernBERT [CLS]/[SEP]/[PAD]/[MASK]/[UNK] on the ByteLevel path", () => {
const data = parseTokenizerJson({
model: { vocab: { hello: 1 }, merges: ["h e"] },
pre_tokenizer: { type: "ByteLevel" },
added_tokens: [
{ id: 50280, content: "[UNK]" }, { id: 50281, content: "[CLS]" },
{ id: 50282, content: "[SEP]" }, { id: 50283, content: "[PAD]" },
{ id: 50284, content: "[MASK]" },
],
})!;
expect(data.ids).toEqual({ cls: 50281, sep: 50282, mask: 50284, pad: 50283, unk: 50280 });
expect(data.kind).toBe("bytelevel");
expect(data.maskToken).toBe("[MASK]");
});
});
// Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).
// With byte_fallback a character the vocab lacks becomes its UTF-8 `<0xNN>` tokens, or <unk> if any of
// those tokens is missing too. The Gemma checkpoint sets it: a Hangul syllable or CJK ideograph outside
// the 256k vocab reached the model as <unk> here.
describe("byte_fallback (HF parity)", () => {
const make = (byteFallback: boolean) => parseTokenizerJson({
normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },
pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },
model: {
byte_fallback: byteFallback,
vocab: {
"▁": 0, a: 1, b: 2, "▁a": 3, "<unk>": 4,
"<0xC7>": 5, "<0x85>": 6, "<0xF0>": 7, "<0x9F>": 8, "<0x98>": 9, "<0x80>": 10, "<0xE4>": 11,
},
merges: ["▁ a"],
},
})!;
it.each<[string, number[]]>([
["Dž", [0, 5, 6]],
["aDžb", [3, 5, 6, 2]],
["DžDž", [0, 5, 6, 5, 6]],
["a Dž", [3, 0, 5, 6]],
["😀", [0, 7, 8, 9, 10]],
["a😀b", [3, 7, 8, 9, 10, 2]],
["é", [0, 4]],
["aéb", [3, 4, 2]],
["中", [0, 4]],
["a b", [3, 0, 2]],
])("byte_fallback on: %j", (text, ids) => {
expect(encodeWithData(make(true), text)).toEqual(ids);
});
it.each<[string, number[]]>([
["Dž", [0, 4]],
["DžDž", [0, 4, 4]],
["😀", [0, 4]],
["a b", [3, 0, 2]],
])("byte_fallback off keeps <unk>: %j", (text, ids) => {
expect(encodeWithData(make(false), text)).toEqual(ids);
});
});
// Expected ids are what Hugging Face `tokenizers` returns for the same JSON (add_special_tokens=False).
// Added tokens are cut out of the text before it is tokenized: whitespace runs and placeholders in the
// ModernBERT checkpoint, HTML tags and control tokens in the Gemma one.
describe("added tokens (HF parity)", () => {
const added = (id: number, content: string, o: Record<string, unknown> = {}) =>
({ id, content, single_word: false, lstrip: false, rstrip: false, normalized: false, special: false, ...o });
const byteLevel = parseTokenizerJson({
normalizer: { type: "NFC" },
pre_tokenizer: { type: "ByteLevel", add_prefix_space: false, use_regex: true },
model: {
vocab: { a: 0, b: 1, c: 2, "Ġ": 3, "Ċ": 4, "Ġa": 5, "Ġb": 6, "|": 7 },
merges: ["Ġ a", "Ġ b"],
},
added_tokens: [
added(8, " ", { normalized: true }), added(9, " ", { normalized: true }),
added(10, "|||X|||", { normalized: true }),
added(11, "[SEP]", { special: true }), added(12, "[MASK]", { special: true, lstrip: true }),
],
})!;
const metaspace = parseTokenizerJson({
normalizer: { type: "Replace", pattern: { String: " " }, content: "▁" },
pre_tokenizer: { type: "Metaspace", replacement: "▁", prepend_scheme: "always", split: true },
model: {
vocab: { "▁": 0, a: 1, b: 2, "▁a": 3, "▁b": 4, "<unk>": 5, c: 6, "▁c": 7 },
merges: ["▁ a", "▁ b", "▁ c"],
},
added_tokens: [added(8, "<t>"), added(9, "<eos>", { special: true })],
})!;
it.each<[string, number[]]>([
["a b", [0, 6]],
["a b", [0, 8, 1]],
["a b", [0, 9, 1]],
["a b", [0, 9, 6]],
[" a", [8, 0]],
["a ", [0, 8]],
["a|||X|||b", [0, 10, 1]],
["a[SEP]b", [0, 11, 1]],
["a [MASK] b", [0, 12, 6]],
["a [SEP] b", [0, 3, 11, 6]],
["a\n b", [0, 4, 8, 1]],
["ab ca", [0, 1, 8, 2, 0]],
])("ByteLevel: %j", (text, ids) => {
expect(encodeWithData(byteLevel, text)).toEqual(ids);
});
it.each<[string, number[]]>([
["a<t>b", [3, 8, 4]],
["<t>", [8]],
["a <t> b", [3, 0, 8, 4]],
["<t><t>", [8, 8]],
["a<eos>b", [3, 9, 4]],
["a b", [3, 0, 4]],
["<t>a", [8, 3]],
["a<t>", [3, 8]],
["a <eos>", [3, 0, 9]],
])("Metaspace: %j", (text, ids) => {
expect(encodeWithData(metaspace, text)).toEqual(ids);
});
});
describe("model-ml/tokenizer.json parity (gated on file presence)", () => {
it("reproduces reference id-sequences byte-identical", () => {
if (!existsSync(mlPath)) return;
const raw = JSON.parse(readFileSync(mlPath, "utf8"));
const data = parseTokenizerJson(raw)!;
expect(data.kind).toBe("metaspace");
expect(data.ids).toMatchObject({ cls: 2, sep: 1, mask: 4, pad: 0, unk: 3 });
expect(data.maskToken).toBe("<mask>");
expect(encodeWithData(data, EN)).toEqual(EN_IDS);
expect(encodeWithData(data, HI)).toEqual(HI_IDS);
});
});