The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.
Python
Add typesetting data model
Sina Atalay1ab366506060
diff --git a/src/manimgx/drawing/typesetting.py b/src/manimgx/drawing/typesetting.pynew file mode 100644index 00000000..1268694d--- /dev/null+++ b/src/manimgx/drawing/typesetting.py@@ -0,0 +1,244 @@+"""Typst source → its layout, typeset in this process (`_engine.typeset`) and cached on disk.+A layout is one row per item (glyph or shape) in document order, each shape's curves, and each+labelled group's items; the layout owns each glyph's immutable outline and carets."""++import functools+import os+import pickle+import tempfile+from dataclasses import dataclass+from importlib.resources import files+from importlib.util import find_spec+from pathlib import Path++import numpy as np++from manimgx import _engine+from manimgx.caches import Memo, forgets+from manimgx.drawing.geometry import Shape++# The fonts manimgx ships, its font packages' (Noto): with Typst's own, every face a document is+# set in or falls back to, so text is the same on every machine; a system font only by name. The+# CJK one is not installed in the browser+FONTS = tuple(+ str(files(package))+ for package in ("manimgx_fonts", "manimgx_fonts_cjk")+ if find_spec(package) is not None+)+TYPST_COMPILATION_FONT_SIZE = 10+TYPST_TEMPLATE = (+ "#set page(width: auto, height: auto, margin: 0pt, fill: none)\n#set text(size:"+ " {text_size}pt)\n{preamble}\n"+)+TypstError = _engine.TypstError+"""The error raised when Typst can't typeset a document, or LaTeX can't be converted to+Typst: its message is Typst's (or mitex's), each error with the bytes of the source it+is at, counted from the start of the whole document typeset — a page setup and the+preamble come before the code."""++# a row's columns (see rust/engine/src/typeset.rs): a glyph's origin is its source node+# (start, end) and the bytes of the node's text its cluster draws (start, end)+KIND, KEY, PLACEMENT, FILL, STROKE, WIDTH, ADVANCE, NODE, DRAWN = (+ 0,+ 1,+ slice(2, 8),+ slice(8, 12),+ slice(12, 16),+ 16,+ 17,+ slice(18, 20),+ slice(20, 22),+)+ROW = 23+GLYPH, SHAPE = 0.0, 1.0+++type Glyph = tuple[int, Shape | None, bytes]+++@dataclass(frozen=True, slots=True)+class Layout:+ """A typeset drawing: `rows` (n × ROW, glyphs with outlines and shapes), `shapes`+ (cubic points, y up), `labels` (a label and its rows, including empty groups),+ `body` (where the body starts in the source, in bytes), and `glyphs` (the+ immutable outline and caret records its rows name)."""++ rows: np.ndarray+ shapes: list[np.ndarray]+ labels: list[tuple[str, list[int]]]+ body: int+ glyphs: dict[int, Glyph]+++# Reuse under disjoint native (<2^48) or content (>=2^52) keys. The bound counts+# aliases; either can be forgotten independently. Live layouts own their records.+_GLYPHS: Memo[int, Glyph] = Memo(1 << 12)+_CACHE = Path(tempfile.gettempdir()) / "manimgx-layouts"+# what made a layout, part of its key: the entry's format, and the engine's own file (a rebuilt+# engine never reads a layout an older one made)+_ENGINE = Path(_engine.__file__).stat()+_VERSION = f"layout 4, engine {_ENGINE.st_size} {_ENGINE.st_mtime_ns}".encode()+_CONTENT = 1 << 52 # glyph keys by content: exact in float64, apart from the engine's+type Stored = tuple[+ np.ndarray,+ list[np.ndarray],+ list[tuple[str, list[int]]],+ int,+ dict[int, tuple[bytes, bytes]],+]+++def typst_string(text: str) -> str:+ """Write `text` as a Typst string literal, which Typst reads back as exactly `text`.++ Args:+ text: Any text.++ Returns:+ The literal, its quotes included.+ """+ escaped = (+ text.replace("\\", "\\\\")+ .replace('"', '\\"')+ .replace("\n", "\\n")+ .replace("\r", "\\r")+ )+ return f'"{escaped}"'+++def typeset(+ body: str,+ preamble: str = "",+ text_size: float = TYPST_COMPILATION_FONT_SIZE,+ font_paths: list[str | Path] | None = None,+ package_path: str | Path | None = None,+) -> Layout:+ """Typeset a Typst document, and read its layout off Typst's frames.++ The document is set in the fonts manimgx ships (Typst's own and the Noto faces),+ after those in `font_paths`; a system font is read only for a family the document+ names that none of them has. A layout is kept on disk, under the system's temporary+ directory (`manimgx-layouts`): the same document, with the same fonts and packages,+ is read back, not typeset again, in any process.++ Args:+ body: The document's body: Typst markup.+ preamble: Typst code set before the body.+ text_size: The text's size, in points.+ font_paths: Directories of font files, searched, with their subdirectories,+ before the fonts manimgx ships.+ package_path: The directory Typst packages are imported from, besides mitex+ (the engine's); None for none but mitex.+ """+ prefix = TYPST_TEMPLATE.format(text_size=text_size, preamble=preamble)+ source = prefix + body + "\n"+ fonts = (*(str(p) for p in font_paths or ()), *FONTS)+ packages = None if package_path is None else str(package_path)+ depends = _files(fonts if packages is None else (*fonts, packages))+ key = _engine.digest(_VERSION, b"\0", source.encode(), b"\0", depends)+ path = _CACHE / f"{key:016x}.layout"+ stored = _read(path)+ if stored is None:+ stored = _typeset(source, fonts, packages)+ _write(path, stored)+ rows, shapes, labels, _, glyphs = stored+ records = {+ key: _glyph(key, points, carets) for key, (points, carets) in glyphs.items()+ }+ return Layout(rows, shapes, labels, len(prefix.encode()), records)+++def _glyph(key: int, points: bytes, carets: bytes) -> Glyph:+ """Decode one immutable record, reusing it while remembered."""+ record = _GLYPHS.get(key)+ if record is None:+ array = np.frombuffer(points).reshape(-1, 3)+ record = _GLYPHS.keep(+ key, (key, Shape(array.copy()) if len(array) else None, carets)+ )+ return record+++@forgets+def _files(paths: tuple[str, ...]) -> bytes:+ """The files a layout depends on besides its source (fonts, packages), as paths, sizes and+ times, once per process."""+ found = []+ for root in map(Path, paths):+ found.append(f"{root}\0".encode())+ within = sorted(root.rglob("*")) if root.is_dir() else [root]+ for file in within:+ if file.is_file():+ stat = file.stat()+ found.append(f"{file}\0{stat.st_size}\0{stat.st_mtime_ns}\0".encode())+ return _engine.digest(*found).to_bytes(8, "little")+++def _read(path: Path) -> Stored | None:+ """A stored layout, if there is one and the fonts it was set with haven't changed."""+ try:+ stored: Stored = pickle.loads(path.read_bytes())+ except (OSError, pickle.UnpicklingError, EOFError, ValueError):+ return None+ system = stored[3]+ if system and system != _engine.system_fonts_fingerprint():+ return None+ return stored+++def _write(path: Path, stored: Stored) -> None:+ """Store a layout: written whole, then renamed, so a reader never sees half of one."""+ partial = path.with_suffix(f".{os.getpid()}.tmp")+ try:+ _CACHE.mkdir(parents=True, exist_ok=True)+ partial.write_bytes(pickle.dumps(stored, protocol=5))+ os.replace(partial, path)+ except OSError:+ pass+++def _typeset(source: str, fonts: tuple[str, ...], packages: str | None) -> Stored:+ """Typeset with Typst; every glyph's key by content, with its outline and carets along."""+ raw, shapes, labels, system = _engine.typeset(source, fonts, packages)+ rows = np.frombuffer(raw).reshape(-1, ROW).copy()+ glyphs = rows[:, KIND] == GLYPH+ keys = rows[glyphs, KEY].astype(np.int64).tolist()+ held = {k: record for k in set(keys) if (record := _GLYPHS.get(k)) is not None}+ new = sorted(set(keys) - held.keys())+ for k, data in zip(new, _engine.glyph_outlines(new)):+ carets = np.array(_engine.ligature_carets(k), dtype="<f8").tobytes()+ content = _CONTENT | (+ _engine.digest(+ len(data).to_bytes(8, "little"),+ data,+ carets,+ )+ & (_CONTENT - 1)+ )+ held[k] = _GLYPHS.keep(k, _glyph(content, data, carets))+ records = {record[0]: record for record in held.values()}+ rows[glyphs, KEY] = [held[k][0] for k in keys]+ drawn = [+ i+ for i, row in enumerate(rows)+ if row[KIND] != GLYPH or records[int(row[KEY])][1] is not None+ ]+ if len(drawn) != len(rows):+ row_of = {old: new for new, old in enumerate(drawn)}+ rows = rows[drawn]+ labels = [+ (label, [row_of[i] for i in members if i in row_of])+ for label, members in labels+ ]+ glyphs = {+ key: (b"" if shape is None else shape.array.tobytes(), carets)+ for key, shape, carets in records.values()+ }+ return (+ rows,+ [np.frombuffer(s).reshape(-1, 3) for s in shapes],+ labels,+ _engine.system_fonts_fingerprint() if system else 0,+ glyphs,+ )diff --git a/tests/unit/drawing/test_typesetting.py b/tests/unit/drawing/test_typesetting.pynew file mode 100644index 00000000..2e9e0322--- /dev/null+++ b/tests/unit/drawing/test_typesetting.py@@ -0,0 +1,363 @@+"""A document is the same however it was set, and what it holds is its own.++- Set with nothing remembered and no layout on disk, read back from disk, set warm (a copy of+ what was made, or made anew with its glyphs remembered), or set while the glyph memo keeps one+ record at a time, a document draws the same, its groups (its labels, a paragraph's whole text)+ name the same parts, its own, and its glyphs keep their keys and cut the same; so do its copy,+ saved state and target once everything is forgotten, and with their glyphs' keys reassigned.+ A document on disk is read, not set again, and what the engine answered can change afterwards+ without changing a document.+- A glyph cuts at its font's carets, else into equal parts of its advance. (No font manimgx ships+ has carets: a made-up font has them.)+- Documents share the glyphs they remember: one outline each; cutting and painting one leaves+ the others as they were, and a cut glyph lets its uncut outline go.+- A glyph without an outline is no part, and labels lose it; a paragraph's blank lines are+ empty lines.+- Text is set in the fonts manimgx ships, the same on every machine: every script finds a face+ there (no .notdef glyph), and a system font is read only for a family a text names.+"""++import gc+import re+import weakref+from collections.abc import Callable, Iterator, Sequence+from pathlib import Path+from typing import ClassVar, NamedTuple++import numpy as np+import pytest+from tests.oracles import look++import manimgx as m+from manimgx import _engine, caches+from manimgx.drawing import typesetting as tc+from manimgx.drawing.geometry import Shape+from manimgx.mobject import _PROTOTYPES+from manimgx.mobjects.text import TypstGlyph, _cut++type Document = m.Typst | m.Paragraph+type Answer = tuple[bytes, list[bytes], list[tuple[str, list[int]]], bool]+++class Engine:+ """The engine, counting the documents it is asked to set; or, `made_up`, a font with+ ligature carets: "first", "second" and "missing" are a glyph each, a square 1000 units wide+ (its advance), which the font cuts at 250, at 750, and nowhere."""++ WORDS: ClassVar = {"first": 981, "second": 982, "missing": 983}+ OUTLINE = m.Square(1000).shift([500, 500, 0]).points.tobytes()++ def __init__(self, monkeypatch: pytest.MonkeyPatch, made_up: bool) -> None:+ self.asked = 0+ self.carets = {981: [250.0], 982: [750.0], 983: []}+ real = _engine.typeset++ def typeset(+ source: str, fonts: Sequence[str], packages: str | None = None+ ) -> Answer:+ self.asked += 1+ return self.made_up(source) if made_up else real(source, fonts, packages)++ monkeypatch.setattr(_engine, "typeset", typeset)+ if made_up:+ monkeypatch.setattr(+ _engine, "glyph_outlines", lambda keys: [self.OUTLINE for _ in keys]+ )+ monkeypatch.setattr(_engine, "ligature_carets", self.carets.__getitem__)++ def made_up(self, source: str) -> Answer:+ keys = [self.WORDS[word] for word in re.findall("|".join(self.WORDS), source)]+ rows = np.zeros((len(keys), tc.ROW))+ rows[:, tc.KEY] = keys+ rows[:, tc.PLACEMENT] = [0.01, 0, 0, 0.01, 0, 0]+ rows[:, tc.FILL] = [0, 0, 0, 1]+ rows[:, tc.STROKE] = [-1, -1, -1, -1]+ rows[:, tc.ADVANCE] = 1000+ return rows.tobytes(), [], [], False++ def change(self) -> None:+ """Change what it answered: the made-up font's carets."""+ for carets in self.carets.values():+ carets[:] = [900.0] * len(carets)+++class Case(NamedTuple):+ make: Callable[[], Document]+ named: dict[str, list[int]] | None = None+ """What its groups name, as places in its family, where the case says."""+ cuts: tuple[float, ...] = ()+ """Where the made-up font cuts each glyph, as a fraction of its width; none: the fonts+ manimgx ships."""+++DOCUMENTS = {+ "a ligature colored inside": Case(+ lambda: m.Text("office", t2c={"f": m.RED, "i": m.BLUE})+ ),+ "labels, empty and repeated": Case(+ lambda: m.Typst("#box[#box[x ] <same> y] <same> #box[ ] <empty>"),+ # every group of a label, as one: the inner `same` adds x again, after y+ named={"same": [2, 1], "empty": []},+ ),+ "blank lines": Case(lambda: m.Paragraph("", "one", "", "two", "")),+ "math": Case(lambda: m.MathTex(r"\frac{a}{b} = \sqrt{x}")),+ "a font with carets": Case(+ lambda: m.Typst("first second missing"), cuts=(0.25, 0.75, 0.5)+ ),+}+++def glyphs(mob: m.Mobject) -> Iterator[TypstGlyph]:+ """Its glyphs, uncut (a cut glyph is the group of its pieces)."""+ for part in mob.submobjects:+ if not isinstance(part, TypstGlyph):+ yield from glyphs(part)+ elif not part.submobjects:+ yield part+++def outline(glyph: TypstGlyph) -> Shape:+ ((_, shape),) = glyph._geometry.terms+ return shape+++def cut(glyph: TypstGlyph) -> TypstGlyph:+ """A copy of the glyph, cut in two: red, then blue."""+ piece = glyph.copy()+ _cut(piece, [m.RED, m.BLUE])+ return piece+++def named(doc: Document) -> dict[str, list[int]]:+ """The parts its groups name, as places in its family (-1: a part not its own)."""+ places = {id(part): k for k, part in enumerate(doc.get_family())}+ groups = (+ {"lines": doc.lines_text}+ if isinstance(doc, m.Paragraph)+ else {label: doc.select(label) for label in doc.labels}+ )+ return {+ name: [places.get(id(part), -1) for part in group.submobjects]+ for name, group in groups.items()+ }+++class Seen(NamedTuple):+ """What a document shows, what its groups name, and its glyphs' keys and cuts."""++ look: list[tuple[object, ...]]+ named: dict[str, list[int]]+ keys: list[int]+ cuts: list[list[tuple[object, ...]]]+++def seen(doc: Document) -> Seen:+ keys = [glyph.key for glyph in glyphs(doc)]+ return Seen(look(doc), named(doc), keys, [look(cut(g)) for g in glyphs(doc)])++[email protected]("name", DOCUMENTS)+def test_a_document_is_the_same_however_it_was_set(+ name: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ case = DOCUMENTS[name]+ engine = Engine(monkeypatch, made_up=bool(case.cuts))+ limit = tc._GLYPHS.limit++ def made(directory: str, *, tight: bool = False, forget: bool = True) -> Document:+ monkeypatch.setattr(tc, "_CACHE", tmp_path / directory)+ monkeypatch.setattr(tc._GLYPHS, "limit", 1 if tight else limit)+ if forget:+ caches.clear()+ return case.make()++ first = made("layouts") # nothing remembered, nothing on disk+ sets = {"tight": made("tight layouts", tight=True)}+ asked = engine.asked+ engine.change()+ sets["from disk"] = made("layouts")+ sets["warm"] = made("layouts", forget=False) # a copy of what it made+ _PROTOTYPES.clear()+ sets["glyphs remembered"] = made("layouts", forget=False) # made anew+ sets["from disk, tight"] = made("layouts", tight=True)+ assert engine.asked == asked, "a document on disk is read, not set again"+ remembered, disk = glyphs(sets["glyphs remembered"]), glyphs(sets["from disk"])+ assert all(outline(a) is outline(b) for a, b in zip(remembered, disk, strict=True))+ replicas = [first.copy(), first.save_state().saved_state, first.generate_target()]+ rekeyed = first.copy()+ for glyph in glyphs(rekeyed):+ glyph.key = -1+ caches.clear()++ expected = seen(first)+ for doc in (*sets.values(), *replicas):+ assert isinstance(doc, (m.Typst, m.Paragraph))+ assert seen(doc) == expected+ assert seen(rekeyed) == expected._replace(keys=[-1] * len(expected.keys))+ if case.named is not None:+ assert named(first) == case.named+ for glyph, at in zip(glyphs(first), case.cuts, strict=bool(case.cuts)):+ left, right = cut(glyph)+ x = glyph.get_left()[0] + at * glyph.width+ assert left.get_right()[0] == pytest.approx(x)+ assert right.get_left()[0] == pytest.approx(x)++ others = [*sets.values(), *replicas, rekeyed]+ before = [look(doc) for doc in others]+ for glyph in glyphs(first):+ _cut(glyph, [m.RED, m.BLUE])+ first.set_color(m.YELLOW)+ assert [look(doc) for doc in others] == before+++def test_a_cut_glyph_lets_its_uncut_outline_go() -> None:+ (glyph,) = glyphs(m.Text("A"))+ uncut = weakref.ref(outline(glyph).array)+ caches.clear()+ assert uncut() is not None, "a live glyph holds its outline"+ _cut(glyph, [m.RED, m.BLUE])+ caches.clear()+ gc.collect()+ assert uncut() is None+++def test_a_glyph_without_an_outline_is_no_part_and_labels_lose_it(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ # No outline is the sole omission criterion: advance, opacity and a shape's+ # height must not decide whether the item exists. An image-only font glyph is+ # omitted like a blank, even if it has a large advance.+ rows = np.zeros((8, tc.ROW))+ rows[:, tc.PLACEMENT] = [1, 0, 0, 1, 0, 0]+ rows[:, tc.FILL] = [0, 0, 0, 1]+ rows[:, tc.STROKE] = [-1, -1, -1, -1]+ rows[:, tc.KEY] = [981, 982, 981, 982, 0, 982, 981, 981]+ rows[:, tc.ADVANCE] = [999, 0, 30, 10, 0, 1, 20, 30]+ rows[4, tc.KIND] = tc.SHAPE+ rows[3, tc.FILL] = [1, 1, 1, 0]+ square = np.array([[0.0, 0, 0], [0, 1, 0], [1, 1, 0], [1, 0, 0]])+ horizontal = np.array([[0.0, 0, 0], [1, 0, 0], [2, 0, 0], [3, 0, 0]])+ labels = [+ ("empty", [0, 2, 6, 7]),+ ("duplicate", [1, 1, 2, 3]),+ ("same", [4, 5]),+ ("same", [0, 1]),+ ("outer", list(range(8))),+ ]+ raw = rows.tobytes()+ calls = 0++ def typeset(+ source: str, fonts: Sequence[str], packages: str | None = None+ ) -> Answer:+ nonlocal calls+ calls += 1+ assert calls == 1, "the second construction must read its disk cache"+ return raw, [horizontal.tobytes()], labels, False++ def outlines(keys: Sequence[int]) -> list[bytes]:+ return [b"" if key == 981 else square.tobytes() for key in keys]++ def carets(key: int) -> list[float]:+ return [0.5] if key == 982 else []++ monkeypatch.setattr(tc, "_CACHE", tmp_path)+ monkeypatch.setattr(tc._engine, "typeset", typeset)+ monkeypatch.setattr(tc._engine, "glyph_outlines", outlines)+ monkeypatch.setattr(tc._engine, "ligature_carets", carets)+ expected = rows[[1, 3, 4, 5]].copy()+ for _ in range(2):+ # A new process has no outline or native-key table when it reads the file.+ caches.clear()+ layout = tc.typeset("synthetic drawing")+ key = int(layout.rows[0, tc.KEY])+ expected[[0, 1, 3], tc.KEY] = key+ assert layout.rows.tobytes() == expected.tobytes()+ assert layout.labels == [+ ("empty", []),+ ("duplicate", [0, 0, 1]),+ ("same", [2, 3]),+ ("same", [0]),+ ("outer", [0, 1, 2, 3]),+ ]+ assert layout.shapes[0].tobytes() == horizontal.tobytes()+ _, shape, caret_bytes = layout.glyphs[key]+ assert shape is not None+ assert shape.array.tobytes() == square.tobytes()+ assert tuple(np.frombuffer(caret_bytes, dtype="<f8")) == (0.5,)+ assert rows.tobytes() == raw+ assert calls == 1+++def test_marks_a_package_draws_come_from_the_element_around_them() -> None:+ # mitex's radical and fraction bar are drawn in its package's code, which the document+ # doesn't hold: they come from the document's innermost element around them, the+ # equation; what the document writes comes from its own node.+ preamble = (+ '#import "@preview/mitex:0.2.7": mitex-scope\n'+ "#let (mitexsqrt, frac) = (mitex-scope.mitexsqrt, mitex-scope.frac)\n"+ )+ body = "$mitexsqrt(x) + frac(1, 2)$ y"+ layout = tc.typeset(body, preamble)+ origins = [+ (+ "glyph" if row[tc.KIND] == tc.GLYPH else "shape",+ body[+ int(row[tc.NODE][0]) - layout.body : int(row[tc.NODE][1]) - layout.body+ ],+ tuple(row[tc.DRAWN]) == (-1, -1),+ )+ for row in layout.rows+ ]+ equation = body[: body.rindex("$") + 1]+ assert origins == [+ ("glyph", equation, True), # the radical+ ("shape", equation, True), # its bar+ ("glyph", "x", False),+ ("glyph", "+", False),+ ("glyph", "1", False),+ ("glyph", "2", False),+ ("shape", equation, True), # the fraction's bar+ ("glyph", "y", False),+ ]+++def test_blank_lines_are_empty_lines() -> None:+ paragraph = m.Paragraph("", "one", "", "two", "")+ assert [len(line) for line in paragraph] == [0, 3, 0, 3, 0]+++def _set(text: m.Text) -> tuple[list[int], bool]:+ """The glyph ids the engine set a text in, and whether it read the system's fonts."""+ preamble = tc.TYPST_TEMPLATE.format(+ text_size=tc.TYPST_COMPILATION_FONT_SIZE, preamble=text.typst_preamble+ )+ rows, _, _, system = _engine.typeset(preamble + text.typst_code, tc.FONTS)+ table = np.frombuffer(rows).reshape(-1, tc.ROW)+ keys = table[table[:, tc.KIND] == tc.GLYPH, tc.KEY].astype(np.int64)+ return [int(k) % 65536 for k in keys], system+++ ("text", "font"),+ [ # multiple_fonts' texts+ ("வணக்கம்", "sans-serif"),+ ("日本へようこそ", ""),+ ("Здравствуйте मस नम म ", "sans-serif"),+ ("नमस्ते", "sans-serif"),+ ("صباح الخير \n تشرفت بمقابلتك", "sans-serif"),+ ("臂猿「黛比」帶著孩子", "sans-serif"),+ ],+)+def test_every_script_is_set_in_a_font_manimgx_ships(text: str, font: str) -> None:+ ids, system = _set(m.Text(text, font=font))+ assert ids+ assert 0 not in ids # .notdef: a character no font has+ assert not system+++def test_a_system_font_is_read_only_by_name() -> None:+ assert _set(m.Text("Hi", font="Open Sans"))[1]