Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
1.8 KB
Raw Blame
"""Generates tok_unicode_deepseek.h: the Unicode classes the DeepSeek V4 / V4.1
pre-tokenizer needs on top of tok_unicode.h (\\p{L}, \\p{N}, \\s):
  - \\p{M}  marks        (category starting with 'M')
  - \\p{P}  punctuation  (category starting with 'P')
  - \\p{S}  symbols      (category starting with 'S')
Same range format and the same uni_in() lookup as tok_unicode.h. Run once:
  python3 tools/gen_unicode_deepseek.py > tok_unicode_deepseek.h
"""
import unicodedata


def ranges(pred):
    out, lo = [], None
    for cp in range(0x110000):
        if 0xD800 <= cp <= 0xDFFF or not pred(cp):
            if lo is not None:
                out.append((lo, cp - 1))
                lo = None
            continue
        if lo is None:
            lo = cp
    if lo is not None:
        out.append((lo, 0x10FFFF))
    return out


def cat(cp):
    return unicodedata.category(chr(cp))


def emit(name, rs):
    print(f"static const uint32_t {name}[][2] = {{")
    for i in range(0, len(rs), 6):
        print("    " + "".join(f"{{0x{lo:X},0x{hi:X}}}," for lo, hi in rs[i:i + 6]))
    print("};")
    print(f"static const int {name}_n = {len(rs)};\n")


print(f"/* GENERATED by tools/gen_unicode_deepseek.py (Unicode {unicodedata.unidata_version})"
      " -- do not edit by hand. */")
print("#ifndef TOK_UNICODE_DEEPSEEK_H\n#define TOK_UNICODE_DEEPSEEK_H\n#include <stdint.h>\n"
      "#include \"tok_unicode.h\"\n")
emit("uni_Mk", ranges(lambda c: cat(c).startswith("M")))
emit("uni_Pu", ranges(lambda c: cat(c).startswith("P")))
emit("uni_Sy", ranges(lambda c: cat(c).startswith("S")))
print("static inline int is_Mk(uint32_t c){ return uni_in(uni_Mk,uni_Mk_n,c); }")
print("static inline int is_Pu(uint32_t c){ return uni_in(uni_Pu,uni_Pu_n,c); }")
print("static inline int is_Sy(uint32_t c){ return uni_in(uni_Sy,uni_Sy_n,c); }")
print("#endif")