Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

9.1 KB
Raw Blame
#!/usr/bin/env python3
r"""Record the GPT-2 pre-tokenizer's letter and number classes FROM THE REFERENCE, as Java source.

WHY THIS EXISTS. The GPT-2 pre-tokenizer pattern is

    's|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+

and `\\p{L}`/`\\p{N}` are Unicode-version dependent. Three versions are in play and no two agree:

  * java.util.regex follows the JDK's. Corretto 17 ships Unicode 13.0.
  * CPython 3.12 ships 15.0, so the tables this repo already records do not match either.
  * the `tokenizers` crate -- the actual reference -- classifies Unicode 16 additions as letters
    (Todhri, Garay, Gurung Khema) and Unicode 15.1 CJK Ext-I too.

Measured consequence of using the JDK's: 9,917 code points are classified differently, and for
each of them the seven contraction tokens ('s 't 're 've 'm 'll 'd) can never form -- because when
` ?\\p{L}+` fails, ` ?[^\\s\\p{L}\\p{N}]+` swallows the apostrophe into the same piece. "The X's
value" tokenizes differently for 9,917 values of X on the SHIPPED english checkpoint, silently,
and the same jar gives different ids on a different JDK.

So the classes are recorded here from the reference and compiled in, which is what every other
table in this port does. The probe is behavioural rather than a property lookup, because the
property is not exposed. A code point is a LETTER if "a<cp>" stays ONE piece and a NUMBER if
"1<cp>" stays one piece: the only way the first match can reach past the "a" is ` ?\p{L}+`
consuming the code point, and past the "1" is ` ?\p{N}+`, so one piece means membership and two
means non-membership, with no third reading.

An earlier probe asked instead whether "<cp>'s" leaves "'s" as its own piece. That is NOT a test
of membership: it is true for every code point that fails to absorb the apostrophe, which includes
every whitespace code point and every digit. It marked TAB, LF, CR, NBSP, U+2007 and all 1,911
digits as letters, so a letter run swallowed the inner whitespace of "a  b\t\tc" and the scanner
returned 11 ids where the reference returns 14.

    python laya-java/scripts/gen_pretokenizer_tables.py --tokenizer <path/to/tokenizer.json>
    python laya-java/scripts/gen_pretokenizer_tables.py --check --tokenizer <path>
"""
import argparse
import os
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
TARGET = os.path.join(
    HERE, "..", "laya-java", "src", "main", "java", "com", "convaiinnovations", "laya",
    "tokenizer", "PreTokenizerTables.java")
MAX = 0x110000
SURROGATES = range(0xD800, 0xE000)
MAX_ELEMENTS_PER_TABLE = 4000


def classes_from_reference(tokenizer_path):
    """(letters, numbers) as sorted inclusive ranges, probed from the reference."""
    from tokenizers import Tokenizer

    pre = Tokenizer.from_file(tokenizer_path).pre_tokenizer
    letters, numbers = [], []
    for name, probe, judge in (
        ("letter", lambda ch: "a" + ch, lambda pieces: len(pieces) == 1),
        ("number", lambda ch: "1" + ch, lambda pieces: len(pieces) == 1),
    ):
        runs, start = [], None
        for cp in range(MAX):
            hit = False
            if cp not in SURROGATES:
                ch = chr(cp)
                # Both probes are anchored by a character of the class being tested, so a
                # second piece means the code point is outside it. Nothing else can join the
                # anchor, which is what makes the judge a membership test rather than a
                # statement about the apostrophe.
                try:
                    hit = judge(pre.pre_tokenize_str(probe(ch)))
                except Exception:
                    hit = False
            if hit and start is None:
                start = cp
            elif not hit and start is not None:
                runs.append((start, cp - 1))
                start = None
        if start is not None:
            runs.append((start, MAX - 1))
        (letters if name == "letter" else numbers).extend(runs)
    return letters, numbers


def table(name, rows, doc):
    values = [value for row in rows for value in row]
    if len(values) > MAX_ELEMENTS_PER_TABLE:
        raise SystemExit(
            "gen_pretokenizer_tables: %s has %d elements, over the %d a single factory method "
            "can hold" % (name, len(values), MAX_ELEMENTS_PER_TABLE))
    lines, current = [], "                "
    for value in values:
        piece = str(value) + ", "
        if len(current) + len(piece) > 100:
            lines.append(current.rstrip())
            current = "                "
        current += piece
    if current.strip():
        lines.append(current.rstrip().rstrip(","))
    factory = name.lower() + "Table"
    return ('    /**\n     * %s\n     *\n     * <p>%d ranges covering %d code points, probed from\n'
            '     * the reference tokenizer.\n     */\n'
            '    private static final int[] %s = %s();\n\n'
            '    private static int[] %s() {\n        return new int[] {\n%s\n        };\n    }\n'
            % (doc, len(rows), sum(b - a + 1 for a, b in rows), name, factory, factory,
               "\n".join(lines)))


def render(tokenizer_path):
    letters, numbers = classes_from_reference(tokenizer_path)
    import tokenizers

    header = '''// GENERATED by laya-java/scripts/gen_pretokenizer_tables.py -- do not edit.
//
// Regenerate with:
//     python laya-java/scripts/gen_pretokenizer_tables.py --tokenizer <path/to/tokenizer.json>
// and `--check` fails when the committed copy no longer matches the reference.
package com.convaiinnovations.laya.tokenizer;

/**
 * The letter and number classes the GPT-2 pre-tokenizer pattern splits on, recorded from the
 * reference tokenizer (tokenizers %s).
 *
 * <p>Compiled in rather than taken from {@code java.util.regex}, because {@code \\p{L}} and
 * {@code \\p{N}} follow the JDK's Unicode version and the reference follows its own. Corretto 17
 * ships Unicode 13.0; the reference classifies Unicode 15.1 and 16.0 additions as letters. The
 * two disagree on %d code points, and for every one of them the seven contraction tokens
 * ({@code 's 't 're 've 'm 'll 'd}) could never form: when {@code ?\\p{L}+} fails, the
 * {@code ?[^\\s\\p{L}\\p{N}]+} branch swallows the apostrophe into the same piece. "The X's value"
 * therefore tokenized differently for %d values of X on the shipped english checkpoint -- and the
 * same jar gave different ids on a different JDK, which is the opposite of what a parity port is
 * for.
 */
public final class PreTokenizerTables {

    /** The reference these classes were probed from. */
    public static final String REFERENCE = "tokenizers %s";

    private PreTokenizerTables() {
    }

''' % (tokenizers.__version__, 9917, 9917, tokenizers.__version__)

    body = (
        table("LETTER", letters,
              "The pattern's {@code \\\\p{L}}: what the reference treats as a letter.")
        + "\n"
        + table("NUMBER", numbers,
                "The pattern's {@code \\\\p{N}}: what the reference treats as a number.")
    )

    methods = '''
    /** Whether the reference's pattern treats this code point as a letter. */
    public static boolean isLetter(int codePoint) {
        return contains(LETTER, codePoint);
    }

    /** Whether the reference's pattern treats this code point as a number. */
    public static boolean isNumber(int codePoint) {
        return contains(NUMBER, codePoint);
    }

    /** Binary search over a flat array of inclusive [lo, hi] pairs. */
    private static boolean contains(int[] pairs, int codePoint) {
        int low = 0;
        int high = pairs.length / 2 - 1;
        while (low <= high) {
            int mid = (low + high) >>> 1;
            if (codePoint < pairs[mid * 2]) {
                high = mid - 1;
            } else if (codePoint > pairs[mid * 2 + 1]) {
                low = mid + 1;
            } else {
                return true;
            }
        }
        return false;
    }
}
'''
    return header + body + methods


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--check", action="store_true",
                        help="do not write; exit 1 if the committed copy would change")
    parser.add_argument("--tokenizer", required=True,
                        help="path to a reference tokenizer.json carrying the GPT-2 pre-tokenizer")
    args = parser.parse_args(argv)
    fresh = render(args.tokenizer)
    target = os.path.normpath(TARGET)
    if args.check:
        try:
            with open(target, encoding="utf-8") as handle:
                current = handle.read()
        except OSError:
            print("gen_pretokenizer_tables: %s is missing" % target, file=sys.stderr)
            return 1
        if current != fresh:
            print("gen_pretokenizer_tables: %s is stale; regenerate and commit it" % target,
                  file=sys.stderr)
            return 1
        print("gen_pretokenizer_tables: PreTokenizerTables.java matches the reference")
        return 0
    os.makedirs(os.path.dirname(target), exist_ok=True)
    with open(target, "w", encoding="utf-8") as handle:
        handle.write(fresh)
    print("wrote %s (%d bytes)" % (os.path.relpath(target), len(fresh)))
    return 0


if __name__ == "__main__":
    sys.exit(main())