Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

4.5 KB
Raw Blame
"""Rebuild OFL web subsets; not required for normal docs builds.

python -m pip install fonttools==4.66.1 brotli==1.2.0
python scripts/build-reading-fonts.py --source-dir /path/to/originals
Source filenames, URLs, versions and hashes: fonts/reading/sources.json.
"""
import argparse
import hashlib
import json
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path

import brotli
from fontTools import subset
from fontTools.ttLib import TTFont

DOCS = Path(__file__).resolve().parents[1]
OUT = DOCS / '.vitepress/theme/fonts/reading'


def unicode_ranges(points):
    runs = []
    for point in sorted(points):
        if runs and point == runs[-1][1] + 1:
            runs[-1][1] = point
        else:
            runs.append([point, point])
    return ','.join(f'U+{a:X}' + (f'-{b:X}' if b != a else '') for a, b in runs)


def latin(point):
    return (point <= 0x24f or 0x300 <= point <= 0x36f
            or 0x2000 <= point <= 0x206f or 0x20a0 <= point <= 0x214f)


def build_one(job):
    source, family, slug, group, points = job
    # Moderate Brotli quality keeps maintenance practical; the font conversion
    # remains lossless. Glyph outlines and OpenType layout features are retained.
    compress = brotli.compress
    brotli.compress = lambda data, **kw: compress(data, **{**kw, 'quality': 5})
    try:
        font = TTFont(source, recalcTimestamp=False)
        postscript = family.replace(' ', '') + '-Regular'
        names = {1: family, 2: 'Regular', 3: postscript, 4: family,
                 6: postscript, 16: family, 17: 'Regular',
                 21: family, 22: 'Regular'}
        for record in font['name'].names:
            if record.nameID in names:
                record.string = names[record.nameID].encode(record.getEncoding())
        options = subset.Options()
        options.layout_features = ['*']
        options.name_IDs = ['*']
        options.name_languages = ['*']
        worker = subset.Subsetter(options=options)
        worker.populate(unicodes=points)
        worker.subset(font)
        # Shaping closure may retain mirrored characters as well as requested glyphs.
        assert set(points) <= set(font.getBestCmap()), (slug, group, 'missing glyphs')
        font.flavor = 'woff2'
        filename = f'{slug}-{group}.woff2'
        font.save(OUT / filename)
        return family, filename, unicode_ranges(points), (OUT / filename).stat().st_size
    finally:
        brotli.compress = compress


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--source-dir', type=Path, required=True)
    parser.add_argument('--jobs', type=int, default=4)
    args = parser.parse_args()
    sources = json.loads((OUT / 'sources.json').read_text())
    # Common document characters form one hot subset; rare characters remain
    # available in bounded Unicode blocks. New prose never needs a font rebuild.
    common = {ord(ch) for locale in ['en', 'zh']
              for path in (DOCS / locale).rglob('*.md')
              for ch in path.read_text(encoding='utf-8')}
    jobs = []
    for info in sources['subsets']:
        source = args.source_dir / info['source_file']
        assert hashlib.sha256(source.read_bytes()).hexdigest() == info['sha256'], source
        coverage = set(TTFont(source).getBestCmap())
        groups = {'latin': {cp for cp in coverage if latin(cp)}}
        groups['common'] = (coverage & common) - groups['latin']
        for cp in sorted(coverage - set.union(*groups.values())):
            groups.setdefault(f'u{cp // 2048 * 2048:05x}', set()).add(cp)
        assert sum(map(len, groups.values())) == len(coverage)
        for group, points in groups.items():
            if points:
                jobs.append((str(source), info['family'], info['slug'], group, points))
    with ProcessPoolExecutor(max_workers=args.jobs) as executor:
        results = list(executor.map(build_one, jobs))
    css = ['/* Generated by docs/scripts/build-reading-fonts.py; OFL licenses beside assets. */']
    for family, filename, ranges, size in results:
        css.append(f'''@font-face {{
  font-family: '{family}';
  font-style: normal;
  font-weight: 400;
  font-display: swap;
  src: url('./{filename}') format('woff2');
  unicode-range: {ranges};
}}''')
    (OUT / 'subsets.css').write_text('\n\n'.join(css) + '\n')
    print(f'{len(results)} files; {sum(row[3] for row in results):,} bytes total')
    for _, filename, _, size in results:
        if '-latin.' in filename or '-common.' in filename:
            print(f'{filename}: {size:,} bytes')


if __name__ == '__main__':
    main()