Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
#!/usr/bin/env python3
"""Generate c/gliner_unicode.h: the parts of Python's str that GLiNER2 reads text with.
The gliner2 package splits a text into words with a Python regex
(processing/word_splitter.py, WhitespaceTokenSplitter) and lower-cases every
word with str.lower(). Reproducing that in C needs Python's own character
classes, not another library's:
word \\w of a str pattern: str.isalnum() or "_"
space \\s of a str pattern: str.isspace()
lower str.lower() one code point at a time (U+0130 lowers to two code
points and capital sigma depends on its neighbours: both are
handled in gliner_decide.c, not here)
ignorable Case_Ignorable, and
cased Cased, for the final-sigma rule str.lower() applies
(Unicode 3.13 Final_Sigma)
Python exposes no Cased or Case_Ignorable predicate, so both are read off
str.lower() itself: for a code point c, "c" + capital sigma lowers to a final
sigma exactly when c is cased and not case-ignorable, and "A" + c + capital
sigma does when c is case-ignorable or cased. The three outcomes tell the
classes apart wherever the rule can see them (a case-ignorable code point is
always skipped, so whether it is also cased never matters).
The tables follow the Unicode version of the Python that runs this script,
which the header records; regenerate with the Python the reference runs on.
python3 tools/gen_gliner_unicode.py > gliner_unicode.h
"""
import sys
import unicodedata
SIGMA = "Σ"
def ranges(predicate):
out, start = [], None
for cp in range(0x110000):
if 0xD800 <= cp <= 0xDFFF:
hit = False
else:
hit = predicate(chr(cp))
if hit and start is None:
start = cp
elif not hit and start is not None:
out.append((start, cp - 1))
start = None
if start is not None:
out.append((start, 0x10FFFF))
return out
def final_sigma_after(text):
return text.lower().endswith("ς")
def cased_not_ignorable(ch):
return final_sigma_after(ch + SIGMA)
def ignorable(ch):
return final_sigma_after("A" + ch + SIGMA) and not final_sigma_after(ch + SIGMA)
def emit_ranges(name, values):
lines = [f"static const GlUniRange {name}[{len(values)}] = {{"]
row = []
for lo, hi in values:
row.append(f"{{0x{lo:x}, 0x{hi:x}}}")
if len(row) == 6:
lines.append(" " + ", ".join(row) + ",")
row = []
if row:
lines.append(" " + ", ".join(row) + ",")
lines.append("};")
return lines
def main():
word = ranges(lambda ch: ch.isalnum() or ch == "_")
space = ranges(lambda ch: ch.isspace())
cased = ranges(cased_not_ignorable)
ignore = ranges(ignorable)
lower = []
for cp in range(0x110000):
if 0xD800 <= cp <= 0xDFFF or cp in (0x130, 0x3A3):
continue
low = chr(cp).lower()
if low != chr(cp):
if len(low) != 1:
sys.exit(f"U+{cp:04X} lowers to {len(low)} code points: extend the generator")
lower.append((cp, ord(low)))
out = [
"/* GENERATED by tools/gen_gliner_unicode.py -- do not edit by hand.",
f" * Python {sys.version.split()[0]}, Unicode Character Database {unicodedata.unidata_version}.",
" * Python's \\w and \\s, str.lower() and the classes of its final-sigma rule,",
" * as the gliner2 word splitter reads text (gliner_decide.c). */",
"#ifndef COLI_GLINER_UNICODE_H",
"#define COLI_GLINER_UNICODE_H",
"#include <stdint.h>",
"typedef struct { uint32_t lo, hi; } GlUniRange;",
"typedef struct { uint32_t cp, lower; } GlUniLower;",
]
out += emit_ranges("gl_uni_word", word)
out += emit_ranges("gl_uni_space", space)
out += emit_ranges("gl_uni_cased", cased)
out += emit_ranges("gl_uni_ignorable", ignore)
out.append(f"static const GlUniLower gl_uni_lower[{len(lower)}] = {{")
row = []
for cp, low in lower:
row.append(f"{{0x{cp:x}, 0x{low:x}}}")
if len(row) == 6:
out.append(" " + ", ".join(row) + ",")
row = []
if row:
out.append(" " + ", ".join(row) + ",")
out.append("};")
out.append("#endif")
sys.stdout.write("\n".join(out) + "\n")
return 0
if __name__ == "__main__":
sys.exit(main())