Skip to content

LordOS — a free, fast and secure GNU/Linux distribution based on Debian, with its own desktop environment: display server and Wayland compositor in C, a React UI engine on QuickJS, and private native apps.

25.1 KB
Raw Blame
// gen-unicode.mjs — generates server/src/ui/unidata.h from the Unicode Character Database
// (UCD) files. These are range tables for:
//   - bidi class (UAX #9), mirrored pairs and bracket pairs → bidi.c
//   - joining type for Arabic, Syriac, N'Ko, Mongolian… (ArabicShaping) → init/medi/fina forms
//   - script (Scripts.txt) → OpenType tag and fallback font
//   - combining class (canonical mark order) and default-ignorable characters
//   - Indic categories (syllabic and positional, grouped the way HarfBuzz does), the Khmer and
//     Myanmar ones, the Universal Shaping Engine ones (Tibetan, Sinhala, Balinese, Javanese…), marks,
//     canonical decompositions of the complex scripts and composition exclusions
//   - line-break classes that matter for Chinese, Japanese and Korean (UAX #14)
//
//   node tools/gen-unicode.mjs <UCD folder>
// The folder needs (from https://www.unicode.org/Public/<version>/ucd/):
//   extracted/DerivedBidiClass.txt  extracted/DerivedJoiningType.txt  BidiMirroring.txt
//   BidiBrackets.txt  Scripts.txt  PropertyValueAliases.txt  LineBreak.txt
//   extracted/DerivedCombiningClass.txt  DerivedCoreProperties.txt
//   IndicSyllabicCategory.txt  IndicPositionalCategory.txt  UnicodeData.txt  CompositionExclusions.txt
//   ArabicShaping.txt, and from HarfBuzz (src/ms-use/): IndicSyllabicCategory-Additional.txt
//   IndicPositionalCategory-Additional.txt (Universal Shaping Engine categories)
// (the extracted/ ones may be in the same folder).
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const root = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
const ucd = process.argv[2];
if (!ucd) { console.error('usage: gen-unicode.mjs <UCD folder>'); process.exit(1); }
const read = (name) => {
  for (const p of [path.join(ucd, name), path.join(ucd, 'extracted', name)])
    if (fs.existsSync(p)) return fs.readFileSync(p, 'utf8');
  throw new Error(`missing ${name}`);
};
const version = (read('DerivedBidiClass.txt').match(/DerivedBidiClass-([\d.]+)\.txt/) || [])[1] || '?';

// Lines "XXXX..YYYY ; value" (and the @missing ones, which give the default values).
function parse(text, { missing = true } = {}) {
  const out = [];
  for (const line of text.split('\n')) {
    let m = missing && line.match(/^# @missing: ([0-9A-F]+)\.\.([0-9A-F]+); (\S+)/);
    if (m) { out.push({ lo: parseInt(m[1], 16), hi: parseInt(m[2], 16), v: m[3], miss: true }); continue; }
    const body = line.split('#')[0].trim();
    if (!body) continue;
    const f = body.split(';').map((s) => s.trim());
    const [a, b] = f[0].split('..');
    out.push({ lo: parseInt(a, 16), hi: parseInt(b || a, 16), v: f[1], f });
  }
  return out;
}

// Assigns values to a sparse range map: first the @missing ones (from most general to most
// specific, as they appear in the file), then the explicit ones.
function ranges(entries, map, deflt) {
  const val = new Map();
  const set = (lo, hi, v) => { for (let c = lo; c <= hi; c++) val.set(c, v); };
  for (const e of entries.filter((e) => e.miss)) set(e.lo, e.hi, map(e.v));
  for (const e of entries.filter((e) => !e.miss)) set(e.lo, e.hi, map(e.v));
  const out = [];
  let cur = null;
  for (let c = 0; c <= 0x10ffff; c++) {
    const v = val.has(c) ? val.get(c) : deflt;
    if (cur && cur.v === v && cur.hi === c - 1) cur.hi = c;
    else { if (cur && cur.v !== deflt) out.push(cur); cur = { lo: c, hi: c, v }; }
  }
  if (cur && cur.v !== deflt) out.push(cur);
  return out;
}

const hex = (n) => `0x${n.toString(16).toUpperCase()}`;
// The enums go on top; the tables are only compiled in unicode.c (UNIDATA_TABLES).
let enums = '';
let h = '';

// ---- bidi ----
const BIDI = ['L', 'R', 'AL', 'EN', 'ES', 'ET', 'AN', 'CS', 'NSM', 'BN', 'B', 'S', 'WS', 'ON',
  'LRE', 'LRO', 'RLE', 'RLO', 'PDF', 'LRI', 'RLI', 'FSI', 'PDI'];
const BIDI_LONG = { Left_To_Right: 'L', Right_To_Left: 'R', Arabic_Letter: 'AL', European_Terminator: 'ET',
  Boundary_Neutral: 'BN' };
const bidi = ranges(parse(read('DerivedBidiClass.txt')), (v) => BIDI.indexOf(BIDI_LONG[v] || v), 0);
enums += `enum { ${BIDI.map((c) => `BIDI_${c}`).join(', ')} };\n`;
h += `static const UniRange uni_bidi[] = {\n${bidi.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, BIDI_${BIDI[r.v]} },`).join('\n')}\n};\n\n`;

const mirror = parse(read('BidiMirroring.txt'), { missing: false }).map((e) => ({ a: e.lo, b: parseInt(e.f[1], 16) }));
h += `static const UniPair uni_mirror[] = {\n${mirror.map((p) => `    { ${hex(p.a)}, ${hex(p.b)} },`).join('\n')}\n};\n\n`;

const brackets = parse(read('BidiBrackets.txt'), { missing: false }).map((e) => ({ a: e.lo, b: parseInt(e.f[1], 16), open: e.f[2] === 'o' }));
h += `static const UniBracket uni_brackets[] = {\n${brackets.map((p) => `    { ${hex(p.a)}, ${hex(p.b)}, ${p.open ? 1 : 0} },`).join('\n')}\n};\n\n`;

// ---- joining (Arabic and other joining scripts) ----
const JOIN = ['U', 'R', 'L', 'D', 'C', 'T'];
const JOIN_LONG = { Non_Joining: 'U', Right_Joining: 'R', Left_Joining: 'L', Dual_Joining: 'D', Join_Causing: 'C', Transparent: 'T' };
const join = ranges(parse(read('DerivedJoiningType.txt')), (v) => JOIN.indexOf(JOIN_LONG[v] || v), 0);
enums += `enum { ${JOIN.map((c) => `JOIN_${c}`).join(', ')} };\n`;
h += `static const UniRange uni_join[] = {\n${join.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, JOIN_${JOIN[r.v]} },`).join('\n')}\n};\n\n`;

// ---- scripts: ISO 15924 code and OpenType tag ----
const alias = {}, longName = {};
for (const line of read('PropertyValueAliases.txt').split('\n')) {
  const m = line.match(/^sc\s*;\s*(\w+)\s*;\s*(\w+)/);
  if (m) { alias[m[2]] = m[1]; longName[m[1]] = m[2]; }
}
const scripts = ['Zyyy'];                       // 0 = Common
const sidx = (name) => {
  const code = name === 'Common' ? 'Zyyy' : alias[name] || name;
  let i = scripts.indexOf(code);
  if (i < 0) { scripts.push(code); i = scripts.length - 1; }
  return i;
};
sidx('Inherited');                              // 1 = Inherited
const script = ranges(parse(read('Scripts.txt'), { missing: false }), sidx, 0);
// OpenType tags: the lowercase code, except those the specification defines differently.
const OT = { Zyyy: 'DFLT', Zinh: 'DFLT', Hira: 'kana', Kana: 'kana', Hrkt: 'kana', Hani: 'hani', Hang: 'hang', Laoo: 'lao ',
  Yiii: 'yi  ', Nkoo: 'nko ', Vaii: 'vai ' };
enums += `#define SCRIPT_COMMON 0\n#define SCRIPT_INHERITED 1\n`;
for (const k of ['Latn', 'Grek', 'Cyrl', 'Arab', 'Hebr', 'Syrc', 'Thaa', 'Nkoo', 'Mong', 'Hani', 'Hira', 'Kana', 'Hang', 'Thai', 'Laoo', 'Khmr', 'Mymr'])
  enums += `#define SCRIPT_${k.toUpperCase()} ${scripts.indexOf(k)}\n`;
enums += `#define SCRIPT_COUNT ${scripts.length}\n`;
h += `static const char uni_script_tag[][5] = {\n${scripts.map((s) => `    "${OT[s] || s.toLowerCase()}",   /* ${s} */`).join('\n')}\n};\n\n`;
// English name with spaces (to look up the "Noto Sans <name>" font).
h += `static const char *const uni_script_name[] = {\n${scripts.map((s) => `    "${(longName[s] || s).replace(/_/g, ' ')}",`).join('\n')}\n};\n\n`;
h += `static const UniRange uni_script[] = {\n${script.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, ${r.v} },`).join('\n')}\n};\n\n`;

// ---- line breaking: only what scripts without spaces need (simplified UAX #14) ----
// ID: can break before and after (ideographs, kana, hangul syllables).
// CL: no break before (closing, 、。). OP: no break after (opening). NS: no break
// before (ー, small ぁ, ・). EX/IS: no break before (!? and separators). SA: Southeast Asian
// scripts without spaces between words (Thai, Lao, Khmer, Myanmar…): broken
// by dictionary (Thai) or by syllable (textlayout.c).
const LB = ['XX', 'ID', 'CL', 'OP', 'NS', 'EX', 'IS', 'SA'];
const LB_MAP = { ID: 'ID', H2: 'ID', H3: 'ID', JL: 'ID', JV: 'ID', JT: 'ID', CL: 'CL', CP: 'CL', OP: 'OP', NS: 'NS', CJ: 'NS',
  EX: 'EX', IS: 'IS', SA: 'SA' };
const lb = ranges(parse(read('LineBreak.txt')), (v) => LB.indexOf(LB_MAP[v] || 'XX'), 0);
enums += `enum { ${LB.map((c) => `LB_${c}`).join(', ')} };\n`;
h += `static const UniRange uni_linebreak[] = {\n${lb.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, LB_${LB[r.v]} },`).join('\n')}\n};\n\n`;

// ---- marks: canonical combining class; invisibles (ZWJ, direction marks…) ----
const ccc = ranges(parse(read('DerivedCombiningClass.txt')), (v) => Number(v) || 0, 0);
h += `static const UniRange uni_ccc[] = {\n${ccc.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, ${r.v} },`).join('\n')}\n};\n\n`;
const ign = ranges(parse(read('DerivedCoreProperties.txt'), { missing: false }).filter((e) => e.v === 'Default_Ignorable_Code_Point'), () => 1, 0);
h += `static const UniRange uni_ignorable[] = {\n${ign.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, 1 },`).join('\n')}\n};\n\n`;


// ---- Indic scripts (Devanagari … Malayalam): category and position, as in HarfBuzz ----
const IC = ['X', 'C', 'V', 'N', 'H', 'ZWNJ', 'ZWJ', 'M', 'SM', 'A', 'PLACEHOLDER', 'DOTTEDCIRCLE', 'RS', 'Repha', 'Ra', 'CM', 'Symbol', 'CS'];
const IC_MAP = { Avagraha: 'Symbol', Bindu: 'SM', Brahmi_Joining_Number: 'PLACEHOLDER', Cantillation_Mark: 'A',
  Consonant: 'C', Consonant_Dead: 'C', Consonant_Final: 'CM', Consonant_Head_Letter: 'C', Consonant_Initial_Postfixed: 'C',
  Consonant_Killer: 'M', Consonant_Medial: 'CM', Consonant_Placeholder: 'PLACEHOLDER', Consonant_Preceding_Repha: 'Repha',
  Consonant_Prefixed: 'X', Consonant_Subjoined: 'CM', Consonant_Succeeding_Repha: 'CM', Consonant_With_Stacker: 'CS',
  Gemination_Mark: 'SM', Invisible_Stacker: 'H', Joiner: 'ZWJ', Modifying_Letter: 'X', Non_Joiner: 'ZWNJ', Nukta: 'N',
  Number: 'PLACEHOLDER', Number_Joiner: 'PLACEHOLDER', Pure_Killer: 'M', Register_Shifter: 'RS', Syllable_Modifier: 'SM',
  Tone_Letter: 'X', Tone_Mark: 'N', Virama: 'H', Visarga: 'SM', Vowel: 'V', Vowel_Dependent: 'M', Vowel_Independent: 'V' };
const IC_OVERRIDE = { 0x0930: 'Ra', 0x09B0: 'Ra', 0x09F0: 'Ra', 0x0A30: 'Ra', 0x0AB0: 'Ra', 0x0B30: 'Ra', 0x0BB0: 'Ra',
  0x0C30: 'Ra', 0x0CB0: 'Ra', 0x0D30: 'Ra', 0x0953: 'SM', 0x0954: 'SM', 0x0A72: 'C', 0x0A73: 'C', 0x1CF5: 'C', 0x1CF6: 'C',
  0x0CF1: 'CS', 0x0CF2: 'CS', 0x00A0: 'PLACEHOLDER', 0x00D7: 'PLACEHOLDER', 0x2010: 'PLACEHOLDER', 0x2011: 'PLACEHOLDER',
  0x2012: 'PLACEHOLDER', 0x2013: 'PLACEHOLDER', 0x2014: 'PLACEHOLDER', 0x2212: 'PLACEHOLDER', 0x25CC: 'DOTTEDCIRCLE',
  0x200C: 'ZWNJ', 0x200D: 'ZWJ' };
for (let c = 0x1CE2; c <= 0x1CE8; c++) IC_OVERRIDE[c] = 'A';
IC_OVERRIDE[0x1CED] = 'A'; IC_OVERRIDE[0x1CF8] = 'A'; IC_OVERRIDE[0x1CF9] = 'A';
for (let c = 0xA8F2; c <= 0xA8F7; c++) IC_OVERRIDE[c] = 'Symbol';
const indicRange = (c) => (c >= 0x0900 && c <= 0x0DFF) || (c >= 0x1CD0 && c <= 0x1CFF) || (c >= 0xA8E0 && c <= 0xA8FF) || IC_OVERRIDE[c];
const icVal = new Map();
for (const e of parse(read('IndicSyllabicCategory.txt'), { missing: false }))
  for (let c = e.lo; c <= e.hi; c++) if (indicRange(c)) icVal.set(c, IC.indexOf(IC_MAP[e.v] || 'X'));
for (const [c, v] of Object.entries(IC_OVERRIDE)) icVal.set(Number(c), IC.indexOf(v));
const IP = ['END', 'PRE_C', 'ABOVE_C', 'BELOW_C', 'POST_C', 'AFTER_MAIN', 'PRE_M'];
const IP_MAP = { Not_Applicable: 'END', Left: 'PRE_C', Top: 'ABOVE_C', Bottom: 'BELOW_C', Right: 'POST_C', Bottom_And_Right: 'POST_C',
  Left_And_Right: 'POST_C', Top_And_Bottom: 'BELOW_C', Top_And_Bottom_And_Left: 'BELOW_C', Top_And_Bottom_And_Right: 'POST_C',
  Top_And_Left: 'ABOVE_C', Top_And_Left_And_Right: 'POST_C', Top_And_Right: 'POST_C', Overstruck: 'AFTER_MAIN', Visual_Order_Left: 'PRE_M' };
const ipVal = new Map();
for (const e of parse(read('IndicPositionalCategory.txt'), { missing: false }))
  for (let c = e.lo; c <= e.hi; c++) if (indicRange(c)) ipVal.set(c, IP.indexOf(IP_MAP[e.v] || 'END'));
const toRanges = (m) => {
  const keys = [...m.keys()].sort((a, b) => a - b), out = [];
  for (const c of keys) {
    const v = m.get(c), last = out[out.length - 1];
    if (last && last.hi === c - 1 && last.v === v) last.hi = c; else out.push({ lo: c, hi: c, v });
  }
  return out.filter((r) => r.v !== 0);
};
const icR = toRanges(icVal), ipR = toRanges(ipVal);
enums += `enum { ${IC.map((c) => `IC_${c}`).join(', ')} };\n`;
enums += `enum { ${IP.map((c) => `IP_${c}`).join(', ')} };\n`;
h += `static const UniRange uni_indic_cat[] = {\n${icR.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, IC_${IC[r.v]} },`).join('\n')}\n};\n\n`;
h += `static const UniRange uni_indic_pos[] = {\n${ipR.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, IP_${IP[r.v]} },`).join('\n')}\n};\n\n`;

// two-part canonical (not compatibility) decompositions of the complex scripts
// (Indic, Tibetan, Myanmar, Khmer, Balinese, Brahmi and family…), and which ones are not
// recomposed (क़ = क + ़ stays separate). Sorted by character (binary search).
const complexRange = (c) => (c >= 0x0900 && c <= 0x0DFF) || (c >= 0x0F00 && c <= 0x109F) || (c >= 0x1780 && c <= 0x17FF) ||
  (c >= 0x1B00 && c <= 0x1BFF) || (c >= 0xA800 && c <= 0xABFF) || (c >= 0x11000 && c <= 0x11FFF);
const unicodeData = read('UnicodeData.txt').split('\n').filter(Boolean).map((line) => line.split(';'));
const decomp = [];
for (const f of unicodeData) {
  const c = parseInt(f[0], 16);
  if (!complexRange(c) || !f[5] || f[5].startsWith('<')) continue;
  const parts = f[5].split(' ').map((x) => parseInt(x, 16));
  if (parts.length === 2) decomp.push([c, parts[0], parts[1]]);
}
const excl = new Set();
for (const line of read('CompositionExclusions.txt').split('\n')) {
  const m = line.match(/^([0-9A-F]+)/);
  if (m) excl.add(parseInt(m[1], 16));
}
// those that decompose into something starting with a mark (non-initial) are not composed either
const cccOf = new Map(unicodeData.map((f) => [parseInt(f[0], 16), Number(f[3]) || 0]));
for (const [c, a] of decomp) if (cccOf.get(a)) excl.add(c);
h += `typedef struct { uint32_t cp, a, b; uint8_t excluded; } UniDecomp;\n`;
h += `static const UniDecomp uni_decomp[] = {\n${decomp.map(([c, a, b]) => `    { ${hex(c)}, ${hex(a)}, ${hex(b)}, ${excl.has(c) ? 1 : 0} },`).join('\n')}\n};\n\n`;

// ---- marks (general category Mn, Mc, Me) ----
const gcOf = new Map();
for (const f of unicodeData) gcOf.set(parseInt(f[0], 16), f[2]);
const markMap = new Map();
for (const [c, gc] of gcOf) if (gc === 'Mn' || gc === 'Mc' || gc === 'Me') markMap.set(c, 1);
const markR = toRanges(markMap);
h += `static const UniRange uni_mark[] = {\n${markR.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, 1 },`).join('\n')}\n};\n\n`;

// ---- Khmer and Myanmar: categories like those in HarfBuzz's Indic table (gen-indic-table.py) ----
const SEA = ['X', 'C', 'V', 'N', 'H', 'ZWNJ', 'ZWJ', 'M', 'SM', 'A', 'VD', 'PLACEHOLDER', 'DOTTEDCIRCLE', 'RS', 'MPst',
  'Repha', 'Ra', 'CM', 'Symbol', 'CS', 'SMPst', 'VAbv', 'VBlw', 'VPre', 'VPst', 'Robatic', 'Xgroup', 'Ygroup',
  'As', 'MH', 'MR', 'MW', 'MY', 'PT', 'VS', 'ML'];
const seaBlock = (c) => (c >= 0x1780 && c <= 0x17FF) ? 'Khmer' : (c >= 0x1000 && c <= 0x109F) || (c >= 0xA9E0 && c <= 0xA9FF) ||
  (c >= 0xAA60 && c <= 0xAA7F) || (c >= 0x116D0 && c <= 0x116FF) ? 'Myanmar' :
  c <= 0xFF || (c >= 0x2000 && c <= 0x209F) || (c >= 0x1CD0 && c <= 0x1CFF) || (c >= 0xA8E0 && c <= 0xA8FF) ? 'other' : null;
const iscAll = new Map(), ipcAll = new Map();
for (const e of parse(read('IndicSyllabicCategory.txt'), { missing: false })) for (let c = e.lo; c <= e.hi; c++) iscAll.set(c, e.v);
for (const e of parse(read('IndicPositionalCategory.txt'), { missing: false })) for (let c = e.lo; c <= e.hi; c++) ipcAll.set(c, e.v);
const SEA_OVERRIDE = { 0x2015: 'PLACEHOLDER', 0x2022: 'PLACEHOLDER', 0x25FB: 'PLACEHOLDER', 0x25FC: 'PLACEHOLDER',
  0x25FD: 'PLACEHOLDER', 0x25FE: 'PLACEHOLDER', 0x25CC: 'DOTTEDCIRCLE', 0x00A0: 'PLACEHOLDER',
  0x179A: 'Ra', 0x17CC: 'Robatic', 0x17C9: 'Robatic', 0x17CA: 'Robatic', 0x17C6: 'Xgroup', 0x17CB: 'Xgroup', 0x17CD: 'Xgroup',
  0x17CE: 'Xgroup', 0x17CF: 'Xgroup', 0x17D0: 'Xgroup', 0x17D1: 'Xgroup', 0x17C7: 'Ygroup', 0x17C8: 'Ygroup', 0x17DD: 'Ygroup',
  0x17D3: 'Ygroup', 0x17D9: 'PLACEHOLDER',
  0x104E: 'C', 0x1004: 'Ra', 0x101B: 'Ra', 0x105A: 'Ra', 0x1032: 'A', 0x1036: 'A', 0x103A: 'As', 0x103E: 'MH', 0x1060: 'ML',
  0x103C: 'MR', 0x103D: 'MW', 0x1082: 'MW', 0x103B: 'MY', 0x105E: 'MY', 0x105F: 'MY', 0x1063: 'PT', 0x1064: 'PT', 0x1069: 'PT',
  0x106A: 'PT', 0x106B: 'PT', 0x106C: 'PT', 0x106D: 'PT', 0xAA7B: 'PT', 0x1038: 'SM', 0x1087: 'SM', 0x1088: 'SM', 0x1089: 'SM',
  0x108A: 'SM', 0x108B: 'SM', 0x108C: 'SM', 0x108D: 'SM', 0x108F: 'SM', 0x109A: 'SM', 0x109B: 'SM', 0x109C: 'SM', 0x104A: 'PLACEHOLDER' };
for (let c = 0xFE00; c <= 0xFE0F; c++) SEA_OVERRIDE[c] = 'VS';
const SEA_POS = { Left: 'VPre', Top: 'VAbv', Bottom: 'VBlw', Right: 'VPst', Bottom_And_Right: 'VPst', Left_And_Right: 'VPst',
  Top_And_Bottom: 'VBlw', Top_And_Bottom_And_Left: 'VBlw', Top_And_Bottom_And_Right: 'VPst', Top_And_Left: 'VAbv',
  Top_And_Left_And_Right: 'VPst', Top_And_Right: 'VPst' };
const seaVal = new Map();
for (const c of new Set([...iscAll.keys(), ...Object.keys(SEA_OVERRIDE).map(Number)])) {
  const block = seaBlock(c);
  if (!block && !SEA_OVERRIDE[c]) continue;
  const ipc = ipcAll.get(c) || 'Not_Applicable';
  let cat = IC_MAP[iscAll.get(c)] || 'X';
  if (cat === 'SM' && ipc === 'Not_Applicable') cat = 'SMPst';
  if (SEA_OVERRIDE[c]) cat = SEA_OVERRIDE[c];
  if ((cat === 'M' || cat === 'MPst') && (block === 'Khmer' || block === 'Myanmar')) cat = SEA_POS[ipc] || cat;
  seaVal.set(c, SEA.indexOf(cat));
}
const seaR = toRanges(seaVal);
enums += `enum { ${SEA.map((c) => `SEA_${c}`).join(', ')} };\n#define SEA_NAMES ${SEA.map((c) => `"${c}"`).join(', ')}\n`;
h += `static const UniRange uni_sea[] = {\n${seaR.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, SEA_${SEA[r.v]} },`).join('\n')}\n};\n\n`;

// ---- Universal Shaping Engine: categories as in HarfBuzz's gen-use-table.py ----
// Also needs ArabicShaping.txt and the two USE data files that ship with HarfBuzz
// (src/ms-use/IndicSyllabicCategory-Additional.txt and IndicPositionalCategory-Additional.txt).
const useIsc = new Map(iscAll), useIpc = new Map(ipcAll), ajt = new Map(), udi = new Set();
const inUse = new Set([...iscAll.keys(), ...ipcAll.keys()]);
for (const e of parse(read('ArabicShaping.txt'), { missing: false })) for (let c = e.lo; c <= e.hi; c++) { ajt.set(c, e.f[2]); inUse.add(c); }
for (const e of parse(read('DerivedCoreProperties.txt'), { missing: false }).filter((e) => e.v === 'Default_Ignorable_Code_Point'))
  for (let c = e.lo; c <= e.hi; c++) { udi.add(c); inUse.add(c); }
for (const e of parse(read('IndicSyllabicCategory-Additional.txt'), { missing: false }))
  for (let c = e.lo; c <= e.hi; c++) { useIsc.set(c, e.v === 'Consonant_Final_Modifier' ? 'Syllable_Modifier' : e.v); inUse.add(c); }
for (const e of parse(read('IndicPositionalCategory-Additional.txt'), { missing: false }))
  for (let c = e.lo; c <= e.hi; c++) { useIpc.set(c, e.v === 'NA' ? 'Not_Applicable' : e.v); inUse.add(c); }
const scriptName = new Map();
for (const e of parse(read('Scripts.txt'), { missing: false })) for (let c = e.lo; c <= e.hi; c++) scriptName.set(c, e.v);
const USE_DISABLED = new Set(['Arabic', 'Lao', 'Samaritan', 'Syriac', 'Thai']);
const USE_RULES = [
  ['B', (U, I, D, G, J) => ['Number', 'Consonant', 'Consonant_Head_Letter', 'Tone_Letter', 'Vowel_Independent'].includes(I) ||
    (['C', 'D', 'L', 'R'].includes(J) && I !== 'Joiner') ||
    (G === 'Lo' && ['Avagraha', 'Bindu', 'Consonant_Final', 'Consonant_Medial', 'Consonant_Subjoined', 'Vowel', 'Vowel_Dependent'].includes(I))],
  ['N', (U, I) => I === 'Brahmi_Joining_Number'],
  ['GB', (U, I) => I === 'Consonant_Placeholder' || [0x2015, 0x2022, 0x25FB, 0x25FC, 0x25FD, 0x25FE].includes(U)],
  ['CGJ', (U, I, D, G) => I === 'Joiner' || (D && ['Mc', 'Me', 'Mn'].includes(G))],
  ['F', (U, I, D, G) => (I === 'Consonant_Final' && G !== 'Lo') || I === 'Consonant_Succeeding_Repha'],
  ['FM', (U, I) => I === 'Syllable_Modifier'],
  ['M', (U, I, D, G) => (I === 'Consonant_Medial' && G !== 'Lo') || I === 'Consonant_Initial_Postfixed'],
  ['CM', (U, I) => ['Nukta', 'Gemination_Mark', 'Consonant_Killer'].includes(I)],
  ['SUB', (U, I, D, G) => I === 'Consonant_Subjoined' && G !== 'Lo'],
  ['CS', (U, I) => I === 'Consonant_With_Stacker'],
  ['H', (U, I) => I === 'Virama' && U !== 0x0DCA],
  ['HVM', (U) => U === 0x0DCA],
  ['HN', (U, I) => I === 'Number_Joiner'],
  ['IS', (U, I) => I === 'Invisible_Stacker' && U !== 0x1A60],
  ['G', (U, I) => I === 'Hieroglyph'],
  ['HM', (U, I) => I === 'Hieroglyph_Modifier'],
  ['HR', (U, I) => I === 'Hieroglyph_Mirror'],
  ['J', (U, I) => I === 'Hieroglyph_Joiner'],
  ['SB', (U, I) => I === 'Hieroglyph_Mark_Begin' || I === 'Hieroglyph_Segment_Begin'],
  ['SE', (U, I) => I === 'Hieroglyph_Mark_End' || I === 'Hieroglyph_Segment_End'],
  ['ZWNJ', (U, I) => I === 'Non_Joiner'],
  ['O', null],                                  // below: depends on the others
  ['RK', (U, I) => I === 'Reordering_Killer'],
  ['R', (U, I) => I === 'Consonant_Preceding_Repha' || I === 'Consonant_Prefixed'],
  ['Sk', (U) => U === 0x1A60],
  ['SM', (U, I) => I === 'Symbol_Modifier'],
  ['V', (U, I, D, G) => I === 'Pure_Killer' || (G !== 'Lo' && (I === 'Vowel' || I === 'Vowel_Dependent'))],
  ['VM', (U, I, D, G) => ['Tone_Mark', 'Cantillation_Mark', 'Register_Shifter', 'Visarga'].includes(I) || (G !== 'Lo' && I === 'Bindu')],
  ['WJ', null],
];
const useRule = Object.fromEntries(USE_RULES);
useRule.WJ = (U, I, D, G) => (D && ![0x115F, 0x1160, 0x3164, 0xFFA0, 0x1BCA0, 0x1BCA1, 0x1BCA2, 0x1BCA3].includes(U) && I === 'Other' &&
  !useRule.CGJ(U, I, D, G)) || G === 'Cn';
useRule.O = (U, I, D, G, J) => (G === 'Po' || ['Consonant_Dead', 'Joiner', 'Modifying_Letter', 'Other'].includes(I)) &&
  !useRule.B(U, I, D, G, J) && !useRule.GB(U, I, D, G, J) && !useRule.CGJ(U, I, D, G, J) && !useRule.SM(U, I, D, G, J) && !useRule.WJ(U, I, D, G, J);
const USE_POS = {
  F: { Abv: ['Top'], Blw: ['Bottom'], Pst: ['Right'] },
  M: { Abv: ['Top'], Blw: ['Bottom', 'Bottom_And_Left', 'Bottom_And_Right'], Pst: ['Right'], Pre: ['Left', 'Top_And_Bottom_And_Left'] },
  CM: { Abv: ['Top'], Blw: ['Bottom', 'Overstruck'] },
  V: { Abv: ['Top', 'Top_And_Bottom', 'Top_And_Bottom_And_Right', 'Top_And_Right'], Blw: ['Bottom', 'Overstruck', 'Bottom_And_Right'],
    Pst: ['Right'], Pre: ['Left', 'Top_And_Left', 'Top_And_Left_And_Right', 'Left_And_Right'] },
  VM: { Abv: ['Top'], Blw: ['Bottom', 'Overstruck'], Pst: ['Right'], Pre: ['Left'] },
  SM: { Abv: ['Top'], Blw: ['Bottom'] },
  FM: { Abv: ['Top'], Blw: ['Bottom'], Pst: ['Not_Applicable'] },
};
const USE = ['O', 'B', 'N', 'GB', 'CGJ', 'SUB', 'H', 'HN', 'ZWNJ', 'WJ', 'R', 'CS', 'IS', 'Sk', 'G', 'J', 'SB', 'SE', 'HVM', 'HM', 'HR', 'RK',
  'FAbv', 'FBlw', 'FPst', 'MAbv', 'MBlw', 'MPst', 'MPre', 'CMAbv', 'CMBlw', 'VAbv', 'VBlw', 'VPst', 'VPre', 'VMAbv', 'VMBlw', 'VMPst',
  'VMPre', 'SMAbv', 'SMBlw', 'FMAbv', 'FMBlw', 'FMPst'];
const useVal = new Map();
let useConflicts = 0;
for (const U of inUse) {
  const sc = scriptName.get(U) || 'Unknown';
  if (USE_DISABLED.has(sc)) continue;
  const G = gcOf.get(U) || 'Cn';
  if (G === 'Cn') continue;                     // unassigned in this Unicode version
  let I = useIsc.get(U) || 'Other', P = useIpc.get(U) || 'Not_Applicable';
  const D = udi.has(U), J = ajt.get(U) || 'X';
  if (U >= 0x1CE2 && U <= 0x1CE8) I = 'Cantillation_Mark';
  if ((U >= 0x0F18 && U <= 0x0F19) || (U >= 0x0F3E && U <= 0x0F3F)) I = 'Vowel_Dependent';
  if (U === 0x1CED) I = 'Tone_Mark';
  const hits = USE_RULES.map(([k]) => k).filter((k) => useRule[k](U, I, D, G, J));
  if (hits.length !== 1) { useConflicts++; if (!hits.length) continue; }
  let cat = hits[0];
  if ([0x11302, 0x11303, 0x114C1].includes(U)) P = 'Top';
  const pm = USE_POS[cat];
  if (pm) {
    const suf = Object.keys(pm).filter((k) => pm[k].includes(P));
    if (suf.length !== 1) { useConflicts++; continue; }
    cat += suf[0];
  }
  if (cat !== 'O') useVal.set(U, USE.indexOf(cat));
}
const useR = toRanges(useVal);
enums += `enum { ${USE.map((c) => `USE_${c}`).join(', ')}, USE_COUNT };\n#define USE_NAMES ${USE.map((c) => `"${c}"`).join(', ')}\n`;
h += `static const UniRange uni_use[] = {\n${useR.map((r) => `    { ${hex(r.lo)}, ${hex(r.hi)}, USE_${USE[r.v]} },`).join('\n')}\n};\n\n`;

h = `/* unidata.h — generated by tools/gen-unicode.mjs from Unicode ${version}. Do not edit.
 * Sorted range tables; absent values are the default value (0). The tables
 * are only compiled in unicode.c; everything else uses the functions in unicode.h. */
#ifndef LORD_UNIDATA_H
#define LORD_UNIDATA_H

#include <stdint.h>

#define UNIDATA_VERSION "${version}"

${enums}
#ifdef UNIDATA_TABLES

typedef struct { uint32_t lo, hi; uint8_t v; } UniRange;
typedef struct { uint32_t a, b; } UniPair;
typedef struct { uint32_t a, b; uint8_t open; } UniBracket;

` + h + `/* Value of cp in a range table (0 if absent). */
static inline uint8_t uni_lookup(const UniRange *t, int n, uint32_t cp) {
    int lo = 0, hi = n - 1;
    while (lo <= hi) {
        int mid = (lo + hi) / 2;
        if (cp < t[mid].lo) hi = mid - 1;
        else if (cp > t[mid].hi) lo = mid + 1;
        else return t[mid].v;
    }
    return 0;
}
#define UNI_N(t) ((int)(sizeof t / sizeof *t))

#endif /* UNIDATA_TABLES */
#endif
`;
const out = path.join(root, 'server/src/ui/unidata.h');
fs.writeFileSync(out, h);
console.log(`${out}: Unicode ${version}, Indic ${icR.length}/${ipR.length}/${decomp.length}, Khmer/Myanmar ${seaR.length}, USE ${useR.length} (${useConflicts} without a unique category), marks ${markR.length}, bidi ${bidi.length} ranges, ccc ${ccc.length}, invisibles ${ign.length}, joining ${join.length}, scripts ${scripts.length}/${script.length}, line break ${lb.length}, mirrors ${mirror.length}, brackets ${brackets.length}`);