Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
"""Generates tok_unicode_deepseek.h: the Unicode classes the DeepSeek V4 / V4.1
pre-tokenizer needs on top of tok_unicode.h (\\p{L}, \\p{N}, \\s):
- \\p{M} marks (category starting with 'M')
- \\p{P} punctuation (category starting with 'P')
- \\p{S} symbols (category starting with 'S')
Same range format and the same uni_in() lookup as tok_unicode.h. Run once:
python3 tools/gen_unicode_deepseek.py > tok_unicode_deepseek.h
"""
import unicodedata
def ranges(pred):
out, lo = [], None
for cp in range(0x110000):
if 0xD800 <= cp <= 0xDFFF or not pred(cp):
if lo is not None:
out.append((lo, cp - 1))
lo = None
continue
if lo is None:
lo = cp
if lo is not None:
out.append((lo, 0x10FFFF))
return out
def cat(cp):
return unicodedata.category(chr(cp))
def emit(name, rs):
print(f"static const uint32_t {name}[][2] = {{")
for i in range(0, len(rs), 6):
print(" " + "".join(f"{{0x{lo:X},0x{hi:X}}}," for lo, hi in rs[i:i + 6]))
print("};")
print(f"static const int {name}_n = {len(rs)};\n")
print(f"/* GENERATED by tools/gen_unicode_deepseek.py (Unicode {unicodedata.unidata_version})"
" -- do not edit by hand. */")
print("#ifndef TOK_UNICODE_DEEPSEEK_H\n#define TOK_UNICODE_DEEPSEEK_H\n#include <stdint.h>\n"
"#include \"tok_unicode.h\"\n")
emit("uni_Mk", ranges(lambda c: cat(c).startswith("M")))
emit("uni_Pu", ranges(lambda c: cat(c).startswith("P")))
emit("uni_Sy", ranges(lambda c: cat(c).startswith("S")))
print("static inline int is_Mk(uint32_t c){ return uni_in(uni_Mk,uni_Mk_n,c); }")
print("static inline int is_Pu(uint32_t c){ return uni_in(uni_Pu,uni_Pu_n,c); }")
print("static inline int is_Sy(uint32_t c){ return uni_in(uni_Sy,uni_Sy_n,c); }")
print("#endif")