Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

24.0 KB
Raw Blame
export interface TokenizerLike {
  readonly clsId: number; readonly sepId: number;
  readonly maskId: number; readonly padId: number;
  readonly maskToken: string;
  encode(text: string): number[];
  /** Decode without removing special tokens; needed only when predictLong splits the state. */
  decode?(ids: readonly number[]): string;
}

/** Special ids of the laya ModernBERT checkpoint (HF added_tokens). */
export const CHECKPOINT_IDS = { cls: 50281, sep: 50282, mask: 50284, pad: 50283, unk: 50280 } as const;

/** Alias lookup order per special: ModernBERT `[X]` names first, Gemma `<x>` names after. */
export const SPECIAL_ALIASES = {
  cls: ["[CLS]", "<bos>", "<s>"],
  sep: ["[SEP]", "<eos>", "</s>"],
  pad: ["[PAD]", "<pad>"],
  mask: ["[MASK]", "<mask>"],
  unk: ["[UNK]", "<unk>"],
} as const;

export interface TokenizerIds { cls: number; sep: number; mask: number; pad: number; unk: number }
export type PreTokenizerKind = "metaspace" | "bytelevel";
export interface TokenizerData {
  vocab: Map<string, number>;
  merges: Map<string, number>;
  ids: TokenizerIds;
  kind: PreTokenizerKind;
  maskToken: string;
  /** Normalizer Replace rules (pattern -> content) applied in order before pre-tokenizing. */
  replaces: Array<[string, string]>;
  /** A character the vocab lacks becomes its `<0xNN>` byte tokens instead of unk (HF byte_fallback). */
  byteFallback?: boolean;
  /** Tokens cut out of the text before it is tokenized (HF added_tokens). */
  added?: AddedToken[];
  /** Absent when tokenizer.json uses a decoder this runtime does not implement. */
  decode?: (ids: readonly number[]) => string;
}

export interface AddedToken { content: string; id: number; normalized: boolean; lstrip: boolean; rstrip: boolean }

/** Metaspace word-boundary marker (HF SentencePiece-style replacement for ' '). */
export const METASPACE_REPLACEMENT = "▁";

/** GPT-2 byte<->unicode table (same mapping as HF ByteLevel pre-tokenizer). */
function byteUnicodeMaps(): { b2u: Map<number, string>; u2b: Map<string, number> } {
  const b2u = new Map<number, string>();
  const u2b = new Map<string, number>();
  const extra = (n: number): number => (n < 0x100 ? n + 0x100 : n);
  const ranges: Array<[number, number]> = [[0x21, 0x7e], [0xa1, 0xac], [0xae, 0xff]];
  let k = 0;
  const inRange = (b: number): boolean => ranges.some(([lo, hi]) => b >= lo && b <= hi);
  for (let b = 0; b < 256; b++) {
    const cp = inRange(b) ? b : extra(k++);
    b2u.set(b, String.fromCodePoint(cp));
    u2b.set(String.fromCodePoint(cp), b);
  }
  return { b2u, u2b };
}

let cached: { b2u: Map<number, string>; u2b: Map<string, number> } | null = null;
function maps(): { b2u: Map<number, string>; u2b: Map<string, number> } {
  if (!cached) cached = byteUnicodeMaps();
  return cached;
}

/** GPT-2 pre-tokenizer split (HF ByteLevel use_regex=true). */
const GPT2_SPLIT = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;

/** Piece length at or above which `bpeWord` uses the heap.
 *
 * Its job is to leave the pieces prose is made of on code that did not change. English prose
 * pieces measure a five-byte median with the checkpoint tokenizer (p90 eight, longest eleven),
 * so essentially all of them take the inline rescan below -- byte for byte the loop that
 * shipped before this change, reached in exactly one call as before. That is a regression-proofing choice, not a tuned one.
 *
 * It is deliberately not the break-even point. Per call the two forms cross over nearer six to
 * eight bytes, and swept end to end at 0, 4, 6, 8, 12, 16, 24 and 32 the difference on 8 000
 * characters of English stayed inside +/-5% at every setting -- below what the measurement can
 * resolve. With the threshold in place English provably cannot regress, because it does not
 * reach the new code at all; without it the cost was under 1% and indistinguishable from noise
 * in 200 interleaved rounds. Both are defensible; this one needs no benchmark to believe.
 *
 * What is not noise is what happens past this line, where the rescan is quadratic and the heap
 * is the only form that stays usable.
 */
const HEAP_MIN_LEN = 32;

/** Merge one word's chars greedily by merge rank: lowest rank wins, leftmost breaks a tie.
 *
 * Short pieces take the rescan inlined below, the loop that shipped before this change; long
 * ones take the heap, which is the only form that stays usable when a piece is an entire state.
 * Both forms follow the same rule and return the same tokens -- the tests assert that at every
 * length from 0 to 200 across the boundary.
 *
 * This has to stay near-linear, because the piece reaching it can be an entire state. Both
 * encoders that call it cut only at spaces: `metaspaceEncode` at the metaspace marker and at
 * newline runs, `bpeEncode` on GPT2_SPLIT, whose ` ?\p{L}+` takes a maximal run of letters. A
 * Chinese or Japanese state has no spaces, so it arrives here as one piece -- and this used to
 * rescan the whole word for its best pair and rebuild the array once per merge, which is
 * O(n^2). 8000 unspaced characters cost 1.41 s (Chinese) and 1.60 s (Japanese) through
 * `metaspaceEncode` with mmBERT's tokenizer, against 1.9 ms for English prose of the same
 * length, growing 4x per doubling. Both encoders are synchronous, so that was the whole event
 * loop rather than one slow request.
 *
 * The word is a doubly linked list over the original slots, so merging is a relink rather than
 * an array rebuild, and the candidate pairs live in a binary min-heap keyed on (rank, slot). A
 * slot only ever absorbs the neighbour to its right, so slot indices stay in text order and
 * ordering heap ties by slot index is exactly the leftmost tie-break a left-to-right rescan
 * gets for free.
 *
 * One merge invalidates at most two pairs: the merged slot's own, and the one starting at its
 * left neighbour. Deleting from a heap costs more than re-pushing, so each slot carries a
 * version counter and a popped entry whose version has moved on is skipped -- standard lazy
 * deletion. A matching version is itself proof the pair still has the rank it was pushed with,
 * because `version` moves whenever the slot's text, its right neighbour, or that neighbour's
 * text changes; there is no need to look the rank up again on the way out.
 *
 * Three details are what make this pay rather than merely scale, and together they are worth
 * 1.2-1.3x over the same algorithm written the obvious way. The heap lives in three
 * `Int32Array`s instead of a JS array of tuples, so pushing costs no allocation. Its capacity
 * is derived rather than guessed, so it never grows in practice -- see the note on it below.
 * And both sifts move the hole instead of swapping, one write per level rather than three.
 *
 * Merge *selection* is O(log n), so the call is O(n log n) for the token lengths a learned
 * table produces. `offer` still builds the key `tok[slot] + " " + tok[j]`, which costs the
 * length of those tokens, so a contrived table that grows a single token to O(n) stays
 * quadratic in key construction alone -- measured on such a table it grows 4x or worse per
 * doubling and is only 2.8-3.9x faster than the rescan. Real tables cap token length far below
 * n, which is why the checkpoint tokenizers measure linear here.
 */
function bpeWord(chars: string[], rank: Map<string, number>): string[] {
  if (chars.length >= HEAP_MIN_LEN) return bpeWordHeap(chars, rank);
  // The rescan stays inline rather than in its own function, so a short piece costs exactly one
  // call as it did before -- routing it through a helper added a second call frame, and at ~10 ns
  // over the 2 824 pieces in 8 000 characters of English prose that measured as a real 0.8%.
  let word = chars.slice();
  if (word.length <= 1) return word;
  for (;;) {
    let best = Infinity, idx = -1;
    for (let i = 0; i < word.length - 1; i++) {
      const r = rank.get(word[i] + " " + word[i + 1]);
      if (r !== undefined && r < best) { best = r; idx = i; }
    }
    if (idx < 0) return word;
    word = [...word.slice(0, idx), word[idx] + word[idx + 1], ...word.slice(idx + 2)];
  }
}

/** The same rule in O(n log n): pairs chosen from a min-heap over a linked list of the slots. */
function bpeWordHeap(chars: string[], rank: Map<string, number>): string[] {
  const n = chars.length;
  const tok = chars.slice();
  if (n <= 1) return tok;

  const next = new Int32Array(n), prev = new Int32Array(n);
  const version = new Int32Array(n);
  const dead = new Uint8Array(n);
  for (let i = 0; i < n; i++) {
    prev[i] = i - 1;
    next[i] = i + 1 < n ? i + 1 : -1;
  }

  // 3n is a bound, not a hope: at most n-1 pairs are offered up front, every merge kills exactly
  // one slot so there are at most n-1 merges, and each offers at most two pairs -- 3n-3 pushes
  // worst case. Measured over 120 000 inputs, including tables built to maximise pushes, the
  // high-water mark is 2n. The grow below still exists because an out-of-bounds typed-array
  // write is silently dropped in JS, which would cost a merge and emit wrong tokens with no
  // error at all: too quiet a failure to leave resting on a proof. It costs under 1%.
  let heapCap = 3 * n;
  let heapRank = new Int32Array(heapCap), heapSlot = new Int32Array(heapCap), heapVer = new Int32Array(heapCap);
  let heapLen = 0;

  /** Offer the pair starting at `slot`, when it has a right neighbour and a rank. Sifts up by
   *  moving the hole down toward the root, writing the new entry once at the end. */
  const offer = (slot: number): void => {
    const j = next[slot];
    if (j < 0) return;
    const r = rank.get(tok[slot] + " " + tok[j]);
    if (r === undefined) return;
    if (heapLen === heapCap) {
      heapCap *= 2;
      const r2 = new Int32Array(heapCap); r2.set(heapRank); heapRank = r2;
      const s2 = new Int32Array(heapCap); s2.set(heapSlot); heapSlot = s2;
      const v2 = new Int32Array(heapCap); v2.set(heapVer); heapVer = v2;
    }
    const ver = version[slot];
    let c = heapLen++;
    while (c > 0) {
      const p = (c - 1) >> 1;
      if (heapRank[p] < r || (heapRank[p] === r && heapSlot[p] < slot)) break;
      heapRank[c] = heapRank[p]; heapSlot[c] = heapSlot[p]; heapVer[c] = heapVer[p];
      c = p;
    }
    heapRank[c] = r; heapSlot[c] = slot; heapVer[c] = ver;
  };

  for (let i = 0; i < n - 1; i++) offer(i);

  while (heapLen > 0) {
    const slot = heapSlot[0], ver = heapVer[0];

    // Pop: take the last entry and sift it down through the hole left at the root.
    const lastRank = heapRank[--heapLen], lastSlot = heapSlot[heapLen], lastVer = heapVer[heapLen];
    if (heapLen > 0) {
      let c = 0;
      for (;;) {
        const l = 2 * c + 1;
        if (l >= heapLen) break;
        const r = l + 1;
        let m = l;
        if (r < heapLen && (heapRank[r] < heapRank[l]
            || (heapRank[r] === heapRank[l] && heapSlot[r] < heapSlot[l]))) m = r;
        if (heapRank[m] > lastRank || (heapRank[m] === lastRank && heapSlot[m] > lastSlot)) break;
        heapRank[c] = heapRank[m]; heapSlot[c] = heapSlot[m]; heapVer[c] = heapVer[m];
        c = m;
      }
      heapRank[c] = lastRank; heapSlot[c] = lastSlot; heapVer[c] = lastVer;
    }

    if (dead[slot] || version[slot] !== ver) continue;   // superseded by a later merge
    const j = next[slot];
    if (j < 0 || dead[j]) continue;

    tok[slot] = tok[slot] + tok[j];
    dead[j] = 1;
    const k = next[j];
    next[slot] = k;
    if (k >= 0) prev[k] = slot;

    version[slot]++;
    offer(slot);
    const p = prev[slot];
    if (p >= 0) {
      version[p]++;
      offer(p);
    }
  }

  const out: string[] = [];
  for (let i = 0; i >= 0; i = next[i]) out.push(tok[i]);
  return out;}

/** Byte-level BPE encode: NFC-normalize, NO lowercasing, GPT-2 byte map + rank-order merges. */
let sharedEncoder: TextEncoder | null = null;
export function bpeEncode(vocab: Map<string, number>, merges: Map<string, number>, text: string): number[] {
  const { b2u } = maps();
  const unkId = vocab.get("[UNK]") ?? CHECKPOINT_IDS.unk;
  const out: number[] = [];
  const enc = (sharedEncoder ??= new TextEncoder());
  const parts = text.normalize("NFC").match(GPT2_SPLIT);
  if (!parts) return out;
  for (const piece of parts) {
    const chars: string[] = [];
    for (const b of enc.encode(piece)) chars.push(b2u.get(b) ?? "");
    for (const tok of bpeWord(chars, merges)) out.push(vocab.get(tok) ?? unkId);
  }
  return out;
}

/** Metaspace (SentencePiece-style) BPE encode: unicode chars, NO byte map, NO lowercasing.
 * Normalizer replaces run first, then one marker is ensured at text start, then the text
 * is cut into words at each marker (marker kept as word prefix) with maximal `\n` runs
 * as their own pieces. Each piece is BPE-merged; leftover unknown pieces map to unk. */
export function metaspaceEncode(
  vocab: Map<string, number>,
  merges: Map<string, number>,
  text: string,
  unkId?: number,
  replaces: ReadonlyArray<readonly [string, string]> = [[" ", METASPACE_REPLACEMENT]],
  byteFallback = false,
): number[] {
  const unk = unkId ?? vocab.get("<unk>") ?? vocab.get("[UNK]") ?? CHECKPOINT_IDS.unk;
  if (!text) return [];
  let t = text;
  for (const [from, to] of replaces) t = t.split(from).join(to);
  const out: number[] = [];
  const push = (piece: string): void => {
    for (const tok of bpeWord(Array.from(piece), merges)) {
      const id = vocab.get(tok);
      if (id !== undefined) {
        out.push(id);
        continue;
      }
      const bytes = byteFallback
        ? Array.from((sharedEncoder ??= new TextEncoder()).encode(tok), (b) => vocab.get(`<0x${b.toString(16).toUpperCase().padStart(2, "0")}>`))
        : [];
      if (bytes.length > 0 && bytes.every((b) => b !== undefined)) out.push(...(bytes as number[]));
      else out.push(unk);
    }
  };
  for (const seg of t.split(/(\n+)/)) {
    if (!seg) continue;
    if (seg[0] === "\n") {
      push(seg);
    } else {
      const w = seg.startsWith(METASPACE_REPLACEMENT) ? seg : METASPACE_REPLACEMENT + seg;
      for (const chunk of w.split(METASPACE_REPLACEMENT).slice(1)) {
        push(chunk ? METASPACE_REPLACEMENT + chunk : METASPACE_REPLACEMENT);
      }
    }
  }
  return out;
}

function encodeSegment(data: TokenizerData, text: string): number[] {
  return data.kind === "metaspace"
    ? metaspaceEncode(data.vocab, data.merges, text, data.ids.unk, data.replaces, data.byteFallback)
    : bpeEncode(data.vocab, data.merges, text);
}

/** Regex source for `words`, longest match first, as a prefix tree: one branch per next character
 *  rather than one alternative per token, so a scan does not try every token at every position. */
function trieSource(words: string[]): string {
  type Node = { kids: Map<string, Node>; end: boolean };
  const root: Node = { kids: new Map(), end: false };
  for (const w of words) {
    let n = root;
    for (const ch of w) {
      let k = n.kids.get(ch);
      if (!k) n.kids.set(ch, (k = { kids: new Map(), end: false }));
      n = k;
    }
    n.end = true;
  }
  const walk = (n: Node): string => {
    const alts = [...n.kids].map(([c, k]) => c.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + walk(k));
    if (!alts.length) return "";
    const body = alts.length === 1 ? alts[0] : "(?:" + alts.join("|") + ")";
    return n.end ? "(?:" + body + ")?" : body;
  };
  return walk(root);
}

type AddedPhase = { re: RegExp; byContent: Map<string, AddedToken>; normalized: boolean };
const addedPhases = new WeakMap<TokenizerData, AddedPhase[]>();

/** HF matches added tokens that are not `normalized` on the raw text first, then the `normalized` ones
 *  on what is left. */
function phasesOf(data: TokenizerData): AddedPhase[] {
  let phases = addedPhases.get(data);
  if (!phases) {
    phases = [false, true].flatMap((normalized) => {
      const toks = (data.added ?? []).filter((t) => t.normalized === normalized && t.content);
      if (!toks.length) return [];
      const re = new RegExp(trieSource(toks.map((t) => t.content)), "gu");
      return [{ re, byContent: new Map(toks.map((t) => [t.content, t])), normalized }];
    });
    addedPhases.set(data, phases);
  }
  return phases;
}

function splitOnAdded(text: string, { re, byContent }: AddedPhase): Array<string | AddedToken> {
  const out: Array<string | AddedToken> = [];
  let last = 0;
  for (const m of text.matchAll(re)) {
    const tok = byContent.get(m[0])!;
    let before = text.slice(last, m.index);
    if (tok.lstrip) before = before.trimEnd();
    if (before) out.push(before);
    out.push(tok);
    last = m.index + m[0].length;
    if (tok.rstrip) while (last < text.length && /\s/u.test(text[last])) last++;
  }
  if (last < text.length) out.push(text.slice(last));
  return out;
}

/** Dispatch to the Metaspace or GPT-2/ByteLevel encoder, after cutting out added tokens as HF does. */
export function encodeWithData(data: TokenizerData, text: string): number[] {
  let pieces: Array<string | AddedToken> = [text];
  for (const phase of phasesOf(data)) {
    pieces = pieces.flatMap((p) => {
      if (typeof p !== "string") return [p];
      return splitOnAdded(phase.normalized && data.kind === "bytelevel" ? p.normalize("NFC") : p, phase);
    });
  }
  const out: number[] = [];
  for (const p of pieces) {
    if (typeof p === "string") out.push(...encodeSegment(data, p));
    else out.push(p.id);
  }
  return out;
}

function childNodes(node: unknown): unknown[] {
  if (!node || typeof node !== "object") return [];
  const o = node as Record<string, unknown>;
  const out: unknown[] = [];
  for (const k of ["normalizers", "pre_tokenizers", "decoders"]) {
    const v = o[k];
    if (Array.isArray(v)) out.push(...v);
  }
  return out;
}

/** True when a normalizer/pre-tokenizer/decoder node (or nested Sequence member) has a type. */
function hasNodeType(node: unknown, want: string): boolean {
  if (!node || typeof node !== "object") return false;
  if ((node as Record<string, unknown>)["type"] === want) return true;
  return childNodes(node).some((c) => hasNodeType(c, want));
}

/** Collect normalizer Replace rules ({pattern: {String}, content}) in order. */
function collectReplaces(node: unknown, out: Array<[string, string]>): void {
  if (!node || typeof node !== "object") return;
  const o = node as Record<string, unknown>;
  if (o["type"] === "Replace") {
    const pat = o["pattern"] as Record<string, unknown> | undefined;
    const from = pat?.["String"];
    const to = o["content"];
    if (typeof from === "string" && typeof to === "string") out.push([from, to]);
  }
  for (const c of childNodes(node)) collectReplaces(c, out);
}

/** Compile the two checkpoint decoders, retaining HF's skip_special_tokens=False behavior. */
function checkpointDecoder(
  raw: unknown, vocab: Map<string, number>, added: AddedToken[],
): ((ids: readonly number[]) => string) | undefined {
  if (!raw || typeof raw !== "object") return undefined;
  const decoder = raw as Record<string, unknown>;
  let decodeTokens: (tokens: string[]) => string;
  // ignoreBOM preserves U+FEFF in a decoded window, just as the HF decoder does.
  const utf8 = new TextDecoder("utf-8", { ignoreBOM: true });
  if (decoder.type === "ByteLevel") {
    decodeTokens = (tokens) => {
      const bytes: number[] = [];
      const { u2b } = maps();
      for (const token of tokens) {
        const mapped = Array.from(token, (ch) => u2b.get(ch));
        // HF falls back to the whole token's UTF-8 bytes if any character is unmapped.
        const part = mapped.every((b) => b !== undefined)
          ? mapped as number[] : (sharedEncoder ??= new TextEncoder()).encode(token);
        for (const b of part) bytes.push(b);
      }
      return utf8.decode(new Uint8Array(bytes));
    };
  } else if (decoder.type === "Sequence" && Array.isArray(decoder.decoders)) {
    const steps = decoder.decoders as Array<Record<string, unknown> | null>;
    const pattern = steps[0]?.pattern as Record<string, unknown> | undefined;
    if (steps.length !== 3 || steps[0]?.type !== "Replace"
        || typeof pattern?.String !== "string" || pattern.String.length === 0
        || typeof steps[0].content !== "string"
        || steps[1]?.type !== "ByteFallback" || steps[2]?.type !== "Fuse") return undefined;
    const from = pattern.String, to = steps[0].content;
    const strict = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
    decodeTokens = (tokens) => {
      const out: string[] = [], bytes: number[] = [];
      const flush = (): void => {
        if (bytes.length === 0) return;
        // ByteFallback rejects an entire invalid run, including any otherwise valid prefix.
        try { out.push(strict.decode(new Uint8Array(bytes))); }
        catch { out.push("\uFFFD".repeat(bytes.length)); }
        bytes.length = 0;
      };
      for (const token of tokens) {
        const replaced = token.split(from).join(to);
        const match = /^<0x([\dA-Fa-f]{2})>$/.exec(replaced);
        if (match) bytes.push(parseInt(match[1], 16));
        else { flush(); out.push(replaced); }
      }
      flush();
      return out.join("");
    };
  } else return undefined;
  const byId = new Map(Array.from(vocab, ([token, id]) => [id, token]));
  for (const token of added) byId.set(token.id, token.content);
  return (ids) => decodeTokens(ids.flatMap((id) => {
    const token = byId.get(id);
    return token === undefined ? [] : [token];
  }));
}

/** Parse an HF tokenizer.json ({model vocab/merges, normalizer, pre_tokenizer, added_tokens}). */
export function parseTokenizerJson(raw: unknown): TokenizerData | null {
  try {
    const r = raw as {
      model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]>; byte_fallback?: boolean };
      normalizer?: unknown;
      pre_tokenizer?: unknown;
      decoder?: unknown;
      added_tokens?: Array<{ id?: number; content?: string; normalized?: boolean; lstrip?: boolean; rstrip?: boolean }>;
    };
    const vocabObj = r?.model?.vocab;
    if (!vocabObj || typeof vocabObj !== "object") return null;
    const vocab = new Map(Object.entries(vocabObj));
    const merges = new Map<string, number>();
    for (const [i, m] of (r.model?.merges ?? []).entries()) {
      const pair = typeof m === "string" ? m.split(" ") : m;
      if (pair.length >= 2) merges.set(pair[0] + " " + pair[1], i);
    }
    const added = new Map<string, number>();
    const addedTokens: AddedToken[] = [];
    for (const t of r.added_tokens ?? []) {
      if (typeof t?.content === "string" && typeof t?.id === "number") {
        added.set(t.content, t.id);
        addedTokens.push({
          content: t.content, id: t.id,
          normalized: t.normalized === true, lstrip: t.lstrip === true, rstrip: t.rstrip === true,
        });
      }
    }
    const pick = (aliases: readonly string[], fb: number): { id: number; token: string } => {
      for (const a of aliases) {
        const v = added.get(a) ?? vocab.get(a);
        if (v !== undefined) return { id: v, token: a };
      }
      return { id: fb, token: aliases[0] };
    };
    const cls = pick(SPECIAL_ALIASES.cls, CHECKPOINT_IDS.cls);
    const sep = pick(SPECIAL_ALIASES.sep, CHECKPOINT_IDS.sep);
    const mask = pick(SPECIAL_ALIASES.mask, CHECKPOINT_IDS.mask);
    const pad = pick(SPECIAL_ALIASES.pad, CHECKPOINT_IDS.pad);
    const unk = pick(SPECIAL_ALIASES.unk, CHECKPOINT_IDS.unk);
    const kind: PreTokenizerKind = hasNodeType(r?.pre_tokenizer, "Metaspace") ? "metaspace" : "bytelevel";
    const replaces: Array<[string, string]> = [];
    collectReplaces(r?.normalizer, replaces);
    if (kind === "metaspace" && replaces.length === 0) replaces.push([" ", METASPACE_REPLACEMENT]);
    return {
      vocab, merges,
      ids: { cls: cls.id, sep: sep.id, mask: mask.id, pad: pad.id, unk: unk.id },
      kind,
      maskToken: mask.token,
      replaces,
      byteFallback: r.model?.byte_fallback === true,
      added: addedTokens,
      decode: checkpointDecoder(r.decoder, vocab, addedTokens),
    };
  } catch {
    return null;
  }
}

/** Load an HF tokenizer.json from a local path (node) or URL into vocab/merges/ids. */
export async function loadTokenizerJson(pathOrUrl: string): Promise<TokenizerData | null> {
  let raw: unknown;
  if (/^https?:\/\//.test(pathOrUrl)) {
    const res = await fetch(pathOrUrl);
    if (!res.ok) return null;
    raw = await res.json();
  } else {
    const fs: typeof import("node:fs/promises") = await import("node:fs/promises");
    raw = JSON.parse(await fs.readFile(pathOrUrl, "utf8"));
  }
  return parseTokenizerJson(raw);
}