Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
4.3 KB
Raw Blame
#!/usr/bin/env python3
"""Generate c/gliner_unicode.h: the parts of Python's str that GLiNER2 reads text with.

The gliner2 package splits a text into words with a Python regex
(processing/word_splitter.py, WhitespaceTokenSplitter) and lower-cases every
word with str.lower(). Reproducing that in C needs Python's own character
classes, not another library's:

  word        \\w of a str pattern: str.isalnum() or "_"
  space       \\s of a str pattern: str.isspace()
  lower       str.lower() one code point at a time (U+0130 lowers to two code
              points and capital sigma depends on its neighbours: both are
              handled in gliner_decide.c, not here)
  ignorable   Case_Ignorable, and
  cased       Cased, for the final-sigma rule str.lower() applies
              (Unicode 3.13 Final_Sigma)

Python exposes no Cased or Case_Ignorable predicate, so both are read off
str.lower() itself: for a code point c, "c" + capital sigma lowers to a final
sigma exactly when c is cased and not case-ignorable, and "A" + c + capital
sigma does when c is case-ignorable or cased. The three outcomes tell the
classes apart wherever the rule can see them (a case-ignorable code point is
always skipped, so whether it is also cased never matters).

The tables follow the Unicode version of the Python that runs this script,
which the header records; regenerate with the Python the reference runs on.

    python3 tools/gen_gliner_unicode.py > gliner_unicode.h
"""
import sys
import unicodedata

SIGMA = "Σ"


def ranges(predicate):
    out, start = [], None
    for cp in range(0x110000):
        if 0xD800 <= cp <= 0xDFFF:
            hit = False
        else:
            hit = predicate(chr(cp))
        if hit and start is None:
            start = cp
        elif not hit and start is not None:
            out.append((start, cp - 1))
            start = None
    if start is not None:
        out.append((start, 0x10FFFF))
    return out


def final_sigma_after(text):
    return text.lower().endswith("ς")


def cased_not_ignorable(ch):
    return final_sigma_after(ch + SIGMA)


def ignorable(ch):
    return final_sigma_after("A" + ch + SIGMA) and not final_sigma_after(ch + SIGMA)


def emit_ranges(name, values):
    lines = [f"static const GlUniRange {name}[{len(values)}] = {{"]
    row = []
    for lo, hi in values:
        row.append(f"{{0x{lo:x}, 0x{hi:x}}}")
        if len(row) == 6:
            lines.append("    " + ", ".join(row) + ",")
            row = []
    if row:
        lines.append("    " + ", ".join(row) + ",")
    lines.append("};")
    return lines


def main():
    word = ranges(lambda ch: ch.isalnum() or ch == "_")
    space = ranges(lambda ch: ch.isspace())
    cased = ranges(cased_not_ignorable)
    ignore = ranges(ignorable)
    lower = []
    for cp in range(0x110000):
        if 0xD800 <= cp <= 0xDFFF or cp in (0x130, 0x3A3):
            continue
        low = chr(cp).lower()
        if low != chr(cp):
            if len(low) != 1:
                sys.exit(f"U+{cp:04X} lowers to {len(low)} code points: extend the generator")
            lower.append((cp, ord(low)))
    out = [
        "/* GENERATED by tools/gen_gliner_unicode.py -- do not edit by hand.",
        f" * Python {sys.version.split()[0]}, Unicode Character Database {unicodedata.unidata_version}.",
        " * Python's \\w and \\s, str.lower() and the classes of its final-sigma rule,",
        " * as the gliner2 word splitter reads text (gliner_decide.c). */",
        "#ifndef COLI_GLINER_UNICODE_H",
        "#define COLI_GLINER_UNICODE_H",
        "#include <stdint.h>",
        "typedef struct { uint32_t lo, hi; } GlUniRange;",
        "typedef struct { uint32_t cp, lower; } GlUniLower;",
    ]
    out += emit_ranges("gl_uni_word", word)
    out += emit_ranges("gl_uni_space", space)
    out += emit_ranges("gl_uni_cased", cased)
    out += emit_ranges("gl_uni_ignorable", ignore)
    out.append(f"static const GlUniLower gl_uni_lower[{len(lower)}] = {{")
    row = []
    for cp, low in lower:
        row.append(f"{{0x{cp:x}, 0x{low:x}}}")
        if len(row) == 6:
            out.append("    " + ", ".join(row) + ",")
            row = []
    if row:
        out.append("    " + ", ".join(row) + ",")
    out.append("};")
    out.append("#endif")
    sys.stdout.write("\n".join(out) + "\n")
    return 0


if __name__ == "__main__":
    sys.exit(main())