The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.
Python
Fix release checks and tabulate benchmark comparisons
Sina Atalay1d8ed4e3f1b6
diff --git a/README.md b/README.mdindex b604d243..3a6ec3f9 100644--- a/README.md+++ b/README.md@@ -55,48 +55,56 @@**Compare the same scene.** Linked rings at 2.5 seconds, rendered by each engine.-<div class="mx-benchmark-quality" role="group" aria-label="Linked rings quality comparison across five engines">- <figure>- <figcaption>ManimGX</figcaption>- <picture class="mx-benchmark-quality__frame">- <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manimgx.png">- <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manimgx-light.png">- <img src="https://manimgx.academa.ai/images/benchmark-quality/manimgx-light.png" alt="Eight linked rings rendered by ManimGX at 2.5 seconds" width="1920" height="1080" loading="lazy">- </picture>- </figure>- <figure>- <figcaption>ManimCE</figcaption>- <picture class="mx-benchmark-quality__frame">- <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manim_ce.png">- <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manim_ce-light.png">- <img src="https://manimgx.academa.ai/images/benchmark-quality/manim_ce-light.png" alt="Eight linked rings rendered by ManimCE at 2.5 seconds" width="1920" height="1080" loading="lazy">- </picture>- </figure>- <figure>- <figcaption>ManimGL</figcaption>- <picture class="mx-benchmark-quality__frame">- <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manimgl.png">- <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manimgl-light.png">- <img src="https://manimgx.academa.ai/images/benchmark-quality/manimgl-light.png" alt="Eight linked rings rendered by ManimGL at 2.5 seconds" width="1920" height="1080" loading="lazy">- </picture>- </figure>- <figure>- <figcaption>Blender Workbench</figcaption>- <picture class="mx-benchmark-quality__frame">- <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/blender_workbench.png">- <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/blender_workbench-light.png">- <img src="https://manimgx.academa.ai/images/benchmark-quality/blender_workbench-light.png" alt="Eight linked rings rendered by Blender Workbench at 2.5 seconds" width="1920" height="1080" loading="lazy">- </picture>- </figure>- <figure>- <figcaption>Blender EEVEE</figcaption>- <picture class="mx-benchmark-quality__frame">- <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/blender_eevee.png">- <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/blender_eevee-light.png">- <img src="https://manimgx.academa.ai/images/benchmark-quality/blender_eevee-light.png" alt="Eight linked rings rendered by Blender EEVEE at 2.5 seconds" width="1920" height="1080" loading="lazy">- </picture>- </figure>-</div>+<table class="mx-benchmark-quality" aria-label="Linked rings quality comparison across five engines">+ <thead>+ <tr>+ <th scope="col">ManimGX</th>+ <th scope="col">ManimCE</th>+ <th scope="col">ManimGL</th>+ <th scope="col">Blender Workbench</th>+ <th scope="col">Blender EEVEE</th>+ </tr>+ </thead>+ <tbody>+ <tr>+ <td>+ <picture class="mx-benchmark-quality__frame">+ <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manimgx.png">+ <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manimgx-light.png">+ <img src="https://manimgx.academa.ai/images/benchmark-quality/manimgx-light.png" alt="Eight linked rings rendered by ManimGX at 2.5 seconds" width="1920" loading="lazy">+ </picture>+ </td>+ <td>+ <picture class="mx-benchmark-quality__frame">+ <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manim_ce.png">+ <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manim_ce-light.png">+ <img src="https://manimgx.academa.ai/images/benchmark-quality/manim_ce-light.png" alt="Eight linked rings rendered by ManimCE at 2.5 seconds" width="1920" loading="lazy">+ </picture>+ </td>+ <td>+ <picture class="mx-benchmark-quality__frame">+ <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manimgl.png">+ <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/manimgl-light.png">+ <img src="https://manimgx.academa.ai/images/benchmark-quality/manimgl-light.png" alt="Eight linked rings rendered by ManimGL at 2.5 seconds" width="1920" loading="lazy">+ </picture>+ </td>+ <td>+ <picture class="mx-benchmark-quality__frame">+ <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/blender_workbench.png">+ <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/blender_workbench-light.png">+ <img src="https://manimgx.academa.ai/images/benchmark-quality/blender_workbench-light.png" alt="Eight linked rings rendered by Blender Workbench at 2.5 seconds" width="1920" loading="lazy">+ </picture>+ </td>+ <td>+ <picture class="mx-benchmark-quality__frame">+ <source media="(prefers-color-scheme: dark)" srcset="https://manimgx.academa.ai/images/benchmark-quality/blender_eevee.png">+ <source media="(prefers-color-scheme: light)" srcset="https://manimgx.academa.ai/images/benchmark-quality/blender_eevee-light.png">+ <img src="https://manimgx.academa.ai/images/benchmark-quality/blender_eevee-light.png" alt="Eight linked rings rendered by Blender EEVEE at 2.5 seconds" width="1920" loading="lazy">+ </picture>+ </td>+ </tr>+ </tbody>+</table>## Get starteddiff --git a/docs/content/stylesheets/manimgx.css b/docs/content/stylesheets/manimgx.cssindex 71b8274f..a35d8e74 100644--- a/docs/content/stylesheets/manimgx.css+++ b/docs/content/stylesheets/manimgx.css@@ -895,30 +895,20 @@ html {/* Identical viewports onto original 1080p benchmark frames; no image resampling. */.mx-benchmark-quality {- display: grid;- grid-template-columns: repeat(5, minmax(0, 1fr));- gap: 0;+ table-layout: fixed;+ border-collapse: collapse;width: min(100%, 80vw);margin: 1rem auto 2rem;}-.md-typeset .mx-benchmark-quality figure {- display: block;- width: 100%;- max-width: none;- margin: 0;- min-width: 0;+.mx-benchmark-quality td {+ padding: 0;}-.md-typeset .mx-benchmark-quality figcaption {- display: flex;- align-items: center;- justify-content: center;+.mx-benchmark-quality th {height: 3.2em;- max-width: none;- margin: 0 0 0.5rem;+ padding: 0 0 0.5rem;font-size: clamp(0.55rem, 1.5vw, 0.7rem);- font-style: normal;font-weight: 600;line-height: 1.25;text-align: center;diff --git a/scripts/benchmark/run.py b/scripts/benchmark/run.pyindex cf0a037c..8bd61f09 100644--- a/scripts/benchmark/run.py+++ b/scripts/benchmark/run.py@@ -220,6 +220,8 @@ def run_one(tool: str, scene: str, repeat: int, args: argparse.Namespace) -> Runif args.dry_run:result.status = "planned"return result+ if not hasattr(os, "wait4"):+ raise RuntimeError("the benchmark runner measures CPU time on macOS and Linux")env = os.environ | {"BENCH_ENGINE": tool,"BENCH_SCENE": scene,@@ -264,7 +266,9 @@ def run_one(tool: str, scene: str, repeat: int, args: argparse.Namespace) -> Runwith contextlib.suppress(ProcessLookupError):os.killpg(process.pid, signal.SIGKILL)result.status = "timeout" if timed_out else "failed"- result.error = Path(result.log).read_text(errors="replace")[-4000:]+ result.error = Path(result.log).read_text(encoding="utf-8", errors="replace")[+ -4000:+ ]return resulttry:videos = [diff --git a/scripts/benchmark/scenes/ffmpeg.py b/scripts/benchmark/scenes/ffmpeg.pyindex 34012c82..abe6407a 100755--- a/scripts/benchmark/scenes/ffmpeg.py+++ b/scripts/benchmark/scenes/ffmpeg.py@@ -19,6 +19,6 @@ command = [args[-1],]Path(os.environ["MANIMGX_BENCH_FFMPEG_LOG"]).write_text(- json.dumps(command, indent=2) + "\n"+ json.dumps(command, indent=2) + "\n", encoding="utf-8")os.execvp(command[0], command)diff --git a/scripts/benchmark/scenes/linked_rings_blender.py b/scripts/benchmark/scenes/linked_rings_blender.pyindex ebe54452..b7cdb796 100644--- a/scripts/benchmark/scenes/linked_rings_blender.py+++ b/scripts/benchmark/scenes/linked_rings_blender.py@@ -103,7 +103,7 @@ else:f"light[{i}].col.{axis} {intensity:.8f}",f"light[{i}].spec.{axis} {0.12 * intensity:.8f}",]- studio.write_text("\n".join(lines) + "\n")+ studio.write_text("\n".join(lines) + "\n", encoding="utf-8")loaded = bpy.context.preferences.studio_lights.load(str(studio), "STUDIO")shading = scene.display.shadingshading.light = "STUDIO"diff --git a/tests/benchmarks/test_comparison.py b/tests/benchmarks/test_comparison.pyindex 28e9e436..75a6c007 100644--- a/tests/benchmarks/test_comparison.py+++ b/tests/benchmarks/test_comparison.py@@ -11,6 +11,28 @@ from scripts.benchmark import chart, runfrom scripts.benchmark.scenes.suite_data import ring_pose+def test_unsupported_cpu_accounting_never_starts_a_process(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.delattr(run.os, "wait4", raising=False)+ monkeypatch.setattr(run, "command", lambda *args: [sys.executable, "-c", "pass"])++ def launch(*args: object, **kwargs: object) -> None:+ pytest.fail("the unsupported benchmark started a process")++ monkeypatch.setattr(run.subprocess, "Popen", launch)+ with pytest.raises(RuntimeError, match="macOS and Linux"):+ run.run_one(+ "manimgx",+ "linked_rings",+ 1,+ argparse.Namespace(out=tmp_path, dry_run=False),+ )+++ not hasattr(run.os, "wait4"), reason="CPU timing runs on macOS and Linux"+)@pytest.mark.parametrize("timeout", [False, True])def test_unsuccessful_processes_are_retained(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, timeout: bool@@ -41,18 +63,18 @@ def test_a_run_never_overwrites_an_existing_run(tmp_path: Path) -> None:@pytest.mark.parametrize("tool", run.TOOLS)-def test_encoder_options_must_match_fastest_preset_and_crf18(+def test_encoder_options_must_match_fastest_preset_and_crf23(tmp_path: Path, tool: str) -> None:path = tmp_path / "out.mp4"options = (- "crf=18.0 subme=1 me=dia bframes=0"+ "crf=23.0 subme=1 me=dia bframes=0"if tool.startswith("blender_")- else "crf=18.0 cabac=0 subme=0 bframes=0"+ else "crf=23.0 cabac=0 subme=0 bframes=0")path.write_bytes(f"x264 - core 165 - options: {options}\x00".encode())- assert "crf=18.0" in run.encoder_evidence(path, tool)- path.write_bytes(path.read_bytes().replace(b"crf=18.0", b"crf=23.0"))+ assert "crf=23.0" in run.encoder_evidence(path, tool)+ path.write_bytes(path.read_bytes().replace(b"crf=23.0", b"crf=18.0"))with pytest.raises(ValueError, match="Unexpected encoder"):run.encoder_evidence(path, tool)@@ -88,7 +110,7 @@ def test_totals_require_every_scene_and_repeat() -> None:def test_recorded_totals_match_every_run() -> None:- data = json.loads((run.HERE / "results.json").read_text())+ data = json.loads((run.HERE / "results.json").read_text(encoding="utf-8"))rows = [run.Run(r["scene"],diff --git a/tests/integration/test_release_reference.py b/tests/integration/test_release_reference.pyindex a2477a6c..baec0bd3 100644--- a/tests/integration/test_release_reference.py+++ b/tests/integration/test_release_reference.py@@ -48,9 +48,9 @@ def test_release_selects_authenticated_compatible_wheel_and_round_trips(assert current.tag == "v0.1.0"assert current.wheel().filename == WHEELmanifest = tmp_path / "release.json"- manifest.write_text(json.dumps(asdict(current)))+ manifest.write_text(json.dumps(asdict(current)), encoding="utf-8")assert release.Release.read(manifest) == current- manifest.write_text("null\n")+ manifest.write_text("null\n", encoding="utf-8")assert release.Release.read(manifest) is None@@ -133,7 +133,10 @@ def test_bootstrap_renders_and_checks_the_actual_export(actual = reference.compare(case, output)assert calls == [(case, "manimgx", {"mp4": True, "log": output / "actual.log"})]assert isinstance(actual.frames, Failure) is (outcome != "success")- assert json.loads((output / "reference.json").read_text())["release"] is None+ assert (+ json.loads((output / "reference.json").read_text(encoding="utf-8"))["release"]+ is None+ )def test_released_wheel_is_verified_and_compared_once_per_worker(@@ -167,7 +170,8 @@ def test_released_wheel_is_verified_and_compared_once_per_worker(result = reference.compare(Case(name), output)assert result.frames == Failure("detected regression")assert (- json.loads((output / "release.json").read_text())["sha256"] == wheel.sha256+ json.loads((output / "release.json").read_text(encoding="utf-8"))["sha256"]+ == wheel.sha256)assert len(acquired) == 1assert len(prepared) == 1diff --git a/tests/integration/test_sound.py b/tests/integration/test_sound.pyindex 8b8a2502..e927240f 100644--- a/tests/integration/test_sound.py+++ b/tests/integration/test_sound.py@@ -174,7 +174,7 @@ def test_audio_files_start_and_end_where_they_declare(def test_a_sound_manimgx_does_not_decode_is_named(tmp_path: Path) -> None:data = encoded(tmp_path / "dolby.mkv", "ac3", "stereo", clicks(2))with pytest.raises(- ValueError, match="its sound is ac3, which manimgx doesn't decode"+ ValueError, match="its sound is ac3, which ManimGX doesn't decode"):decode_audio(data, RATE)diff --git a/tests/integration/test_take.py b/tests/integration/test_take.pyindex f671b3d6..fca26016 100644--- a/tests/integration/test_take.py+++ b/tests/integration/test_take.py@@ -168,7 +168,7 @@ def test_without_a_gpu_every_film_is_a_take(monkeypatch: pytest.MonkeyPatch) ->def test_a_take_is_not_also_a_video(tmp_path: Path) -> None:- with pytest.raises(ValueError, match="drawn by manimgx's player"):+ with pytest.raises(ValueError, match="drawn by ManimGX's player"):Held().render(tmp_path / "held.mp4", take=lambda _: None)