Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""Rebuild OFL web subsets; not required for normal docs builds.
python -m pip install fonttools==4.66.1 brotli==1.2.0
python scripts/build-reading-fonts.py --source-dir /path/to/originals
Source filenames, URLs, versions and hashes: fonts/reading/sources.json.
"""
import argparse
import hashlib
import json
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path
import brotli
from fontTools import subset
from fontTools.ttLib import TTFont
DOCS = Path(__file__).resolve().parents[1]
OUT = DOCS / '.vitepress/theme/fonts/reading'
def unicode_ranges(points):
runs = []
for point in sorted(points):
if runs and point == runs[-1][1] + 1:
runs[-1][1] = point
else:
runs.append([point, point])
return ','.join(f'U+{a:X}' + (f'-{b:X}' if b != a else '') for a, b in runs)
def latin(point):
return (point <= 0x24f or 0x300 <= point <= 0x36f
or 0x2000 <= point <= 0x206f or 0x20a0 <= point <= 0x214f)
def build_one(job):
source, family, slug, group, points = job
# Moderate Brotli quality keeps maintenance practical; the font conversion
# remains lossless. Glyph outlines and OpenType layout features are retained.
compress = brotli.compress
brotli.compress = lambda data, **kw: compress(data, **{**kw, 'quality': 5})
try:
font = TTFont(source, recalcTimestamp=False)
postscript = family.replace(' ', '') + '-Regular'
names = {1: family, 2: 'Regular', 3: postscript, 4: family,
6: postscript, 16: family, 17: 'Regular',
21: family, 22: 'Regular'}
for record in font['name'].names:
if record.nameID in names:
record.string = names[record.nameID].encode(record.getEncoding())
options = subset.Options()
options.layout_features = ['*']
options.name_IDs = ['*']
options.name_languages = ['*']
worker = subset.Subsetter(options=options)
worker.populate(unicodes=points)
worker.subset(font)
# Shaping closure may retain mirrored characters as well as requested glyphs.
assert set(points) <= set(font.getBestCmap()), (slug, group, 'missing glyphs')
font.flavor = 'woff2'
filename = f'{slug}-{group}.woff2'
font.save(OUT / filename)
return family, filename, unicode_ranges(points), (OUT / filename).stat().st_size
finally:
brotli.compress = compress
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source-dir', type=Path, required=True)
parser.add_argument('--jobs', type=int, default=4)
args = parser.parse_args()
sources = json.loads((OUT / 'sources.json').read_text())
# Common document characters form one hot subset; rare characters remain
# available in bounded Unicode blocks. New prose never needs a font rebuild.
common = {ord(ch) for locale in ['en', 'zh']
for path in (DOCS / locale).rglob('*.md')
for ch in path.read_text(encoding='utf-8')}
jobs = []
for info in sources['subsets']:
source = args.source_dir / info['source_file']
assert hashlib.sha256(source.read_bytes()).hexdigest() == info['sha256'], source
coverage = set(TTFont(source).getBestCmap())
groups = {'latin': {cp for cp in coverage if latin(cp)}}
groups['common'] = (coverage & common) - groups['latin']
for cp in sorted(coverage - set.union(*groups.values())):
groups.setdefault(f'u{cp // 2048 * 2048:05x}', set()).add(cp)
assert sum(map(len, groups.values())) == len(coverage)
for group, points in groups.items():
if points:
jobs.append((str(source), info['family'], info['slug'], group, points))
with ProcessPoolExecutor(max_workers=args.jobs) as executor:
results = list(executor.map(build_one, jobs))
css = ['/* Generated by docs/scripts/build-reading-fonts.py; OFL licenses beside assets. */']
for family, filename, ranges, size in results:
css.append(f'''@font-face {{
font-family: '{family}';
font-style: normal;
font-weight: 400;
font-display: swap;
src: url('./{filename}') format('woff2');
unicode-range: {ranges};
}}''')
(OUT / 'subsets.css').write_text('\n\n'.join(css) + '\n')
print(f'{len(results)} files; {sum(row[3] for row in results):,} bytes total')
for _, filename, _, size in results:
if '-latin.' in filename or '-common.' in filename:
print(f'{filename}: {size:,} bytes')
if __name__ == '__main__':
main()