Skip to content

The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.

Python
16.4 KB
Raw Blame
"""A film's video: decoded, it shows the frames the film drew, in their colors — the engine's
conversion to NV12, x264 (built into the engine), and what the MP4 says of its colors agree.
"""

from pathlib import Path
from unittest.mock import Mock, patch

import av
import numpy as np
import pytest
from tests.integration.corpus.frames import decode
from tests.integration.corpus.run_manimgx import mp4_frames

import manimgx as m
from manimgx._engine import Player
from manimgx.rendering import feed
from manimgx.rendering.feed import Feeder
from manimgx.rendering.film import Cut, Frame, X264Preset

SIZE = (320, 180)
pytestmark = pytest.mark.config(pixel_width=SIZE[0], pixel_height=SIZE[1])


class Filled(m.Scene):
    def construct(self) -> None:
        self.add(m.Rectangle(width=10, height=6, color=m.ORANGE, fill_opacity=1))
        self.play(m.Create(m.Circle(color=m.TEAL)))
        self.wait(0.5)


def test_the_video_shows_the_frames_drawn(tmp_path: Path) -> None:
    drawn: list[np.ndarray] = []

    def sink(frame: Frame) -> None:
        pixels = np.frombuffer(frame.pixels(), np.uint8).reshape(SIZE[1], SIZE[0], 4)
        drawn.append(pixels[:, :, :3].astype(int))

    # Minimize compression loss: this bounds color conversion, independently of the
    # size/quality tradeoff chosen by the default encoder policy in test_encoding.
    Filled().render(tmp_path / "filled.mp4", frames=sink, preset="ultrafast", crf=1)
    shown = [frame.astype(int) for frame in decode(tmp_path / "filled.mp4", SIZE)]
    assert len(shown) == len(drawn)  # a held frame is one picture that lasts
    for picture, frame in zip(shown, drawn, strict=True):
        assert np.abs(picture - frame).mean() < 1.0
    # inside the rectangle, away from the circle's stroke: its orange, within a level or two
    assert np.abs(shown[-1][90, 60] - drawn[-1][90, 60]).max() <= 2


@pytest.mark.parametrize("preset", ["ultrafast", "slow"])
def test_each_picture_shows_at_its_frame(tmp_path: Path, preset: X264Preset) -> None:
    # a slow preset encodes B-frames: pictures decoded after the picture they are shown
    # before, so the MP4 says when each is shown apart from when it is decoded
    starts: list[int] = []
    video = tmp_path / "filled.mp4"
    Filled().render(
        video, frames=lambda frame: starts.append(frame.index), preset=preset
    )
    with av.open(str(video)) as container:
        stream = container.streams.video[0]
        assert stream.time_base is not None
        shown = [
            frame.pts * stream.time_base * int(m.config.frame_rate)
            for frame in container.decode(stream)
            if frame.pts is not None
        ]
    assert shown == starts


@pytest.mark.parametrize("pixels", [False, True])
@pytest.mark.parametrize("cut", [False, True])
def test_video_and_pixel_sinks_share_one_draw(
    tmp_path: Path, pixels: bool, cut: bool
) -> None:
    player = Mock(wraps=Player(*SIZE))
    sent: list[Frame] = []

    def sink(frame: Frame) -> None:
        sent.append(frame)
        if pixels:
            assert len(frame.pixels()) == SIZE[0] * SIZE[1] * 4
        if cut and len(sent) == 3:
            raise Cut

    video = tmp_path / "shared.mp4"
    with patch("manimgx.rendering.feed.Player", return_value=player):
        Filled().render(video, frames=sink)
    assert player.push.call_count == len(sent)
    assert all(
        call.kwargs.get("capture", False) == pixels
        for call in player.push.call_args_list
    )
    player.render.assert_not_called()
    assert len(list(decode(video, SIZE))) == len(sent)
    # A frame retained by a metadata consumer must no longer try to export it when read.
    assert len(sent[-1].pixels()) == SIZE[0] * SIZE[1] * 4
    assert player.push.call_count == len(sent)


@pytest.mark.parametrize("queued_layers", [False, True])
@pytest.mark.parametrize("samples", [1, 4])
def test_captured_exports_preserve_pixels_and_video_across_pending_draws(
    tmp_path: Path, queued_layers: bool, samples: int
) -> None:
    # RGBA rows require padding; NV12 edges are partial macroblocks.
    width, height = 130, 74
    capture, plain, direct = (Player(width, height, samples) for _ in range(3))
    feeders = [Feeder(width, height, player) for player in (capture, plain, direct)]
    camera = m.Camera(three_d=True)
    # Cover the picture in see-through mesh layers: the first draw must grow its fragment
    # lists. Capturing must return the successful retry, including after older queued draws.
    layers: list[m.Mobject] = [
        m.MeshMobject(
            np.array([[-8, -5, z], [8, -5, z], [8, 5, z], [-8, 5, z]]),
            np.array([[0, 1, 2], [0, 2, 3]]),
            color=m.BLUE if i % 2 else m.RED,
            fill_opacity=0.2,
        )
        for i, z in enumerate(np.linspace(-0.2, 0.2, 12))
    ]
    captured_video, plain_video = tmp_path / "captured.mp4", tmp_path / "plain.mp4"
    expected_frames: list[np.ndarray] = []
    capture.begin_export(str(captured_video), 30)
    plain.begin_export(str(plain_video), 30)
    for index, wanted in enumerate([False, False, False, True, True, False, True]):
        objects = (
            layers
            if (index >= 3) != queued_layers
            else [m.Square(fill_opacity=1).shift((index % 3) * m.RIGHT)]
        )
        prepared = [feeder.frame(camera, objects) for feeder in feeders]
        expected = direct.render(*prepared[2])
        assert any(expected[0::4])  # geometry is visible, including the mesh layers
        expected_frames.append(
            np.frombuffer(expected, np.uint8)
            .reshape(height, width, 4)[:, :, :3]
            .astype(int)
        )
        view, records, cameras = prepared[0]
        actual = capture.push(view, records, cameras=cameras, capture=wanted)
        assert actual == (expected if wanted else None)
        view, records, cameras = prepared[1]
        assert plain.push(view, records, cameras=cameras) is None
    capture.end_export()
    plain.end_export()
    assert captured_video.read_bytes() == plain_video.read_bytes()
    shown = [frame.astype(int) for frame in decode(captured_video, (width, height))]
    assert len(shown) == len(expected_frames)
    for picture, frame in zip(shown, expected_frames, strict=True):
        assert np.abs(picture - frame).mean() < 2.0


@pytest.mark.config(frame_rate=30)
@pytest.mark.parametrize("pressed", [False, True])
@pytest.mark.parametrize("resource", ["points", "mesh", "rows", "texture"])
def test_eviction_preserves_queued_transparency_retries(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, resource: str, pressed: bool
) -> None:
    # The feeder's cadence, small, so the scene is short: a mark every 4 frames, and every 12
    # a sweep of what no frame has shown for 8.
    monkeypatch.setattr(feed, "MARK", 4)
    monkeypatch.setattr(feed, "KEEP", 8)
    monkeypatch.setattr(feed, "SWEEP", 12)

    class Recurring(m.ThreeDScene):
        def construct(self) -> None:
            rng = np.random.default_rng(4)
            points = rng.uniform(-3, 3, (60, 3))
            rows = rng.random((60, 4))
            rows[:, 3] = 1
            pixels = rng.integers(0, 256, (16, 32, 4), dtype=np.uint8)
            pixels[:, :, 3] = 255
            cloud = m.PMobject(stroke_width=4).add_points(points, rgbas=rows)
            mesh = m.MeshMobject(points, np.arange(60).reshape(-1, 3))
            image = m.ImageMobject(pixels).scale_to_fit_width(4)
            target = {"points": cloud, "mesh": mesh, "rows": cloud, "texture": image}[
                resource
            ]
            layers = [
                m.MeshMobject(
                    np.array([[-8, -5, z], [8, -5, z], [8, 5, z], [-8, 5, z]]),
                    np.array([[0, 1, 2], [0, 2, 3]]),
                    color=m.BLUE if i % 2 else m.RED,
                    fill_opacity=1,
                )
                for i, z in enumerate(np.linspace(-0.2, 0.2, 12))
            ]

            def update(_: m.Mobject) -> None:
                frame = self.film.feeder.frames
                # Reuse an old upload between the feeder's marks, just before its sweep.
                # Twelve transparent layers force a retry after that sweep has run.
                repeated = frame in (1, 10)
                for layer in layers:
                    layer.set_fill(opacity=0.2 if frame == 10 else 1)
                if resource in ("points", "mesh"):
                    target.points = points if repeated else rng.uniform(-3, 3, (60, 3))
                elif resource == "rows":
                    colors = rows if repeated else rng.random((60, 4))
                    if not repeated:
                        colors[:, 3] = 1
                    cloud.fill_rgbas = colors
                else:
                    texture = (
                        pixels
                        if repeated
                        else rng.integers(0, 256, pixels.shape, dtype=np.uint8)
                    )
                    if not repeated:
                        texture[:, :, 3] = 255
                    texture.flags.writeable = False
                    image.paint = image.paint.but(texture=texture)

            target.add_updater(update)
            self.add(*layers, target)
            self.wait(0.6)

    reference, swept = tmp_path / "reference.mp4", tmp_path / "swept.mp4"
    with monkeypatch.context() as disabled:
        disabled.setattr(feed, "SWEEP", 10_000)
        disabled.setattr(feed, "BUDGET", 1 << 40)
        Recurring().render(reference)
    if pressed:
        monkeypatch.setattr(feed, "BUDGET", 0)
    film = Recurring().render(swept)
    assert film.feeder.evictions > 0
    assert swept.read_bytes() == reference.read_bytes()


@pytest.mark.parametrize("auxiliary", [False, True])
def test_evicting_what_a_frame_in_flight_draws_waits_for_its_retries(
    tmp_path: Path, auxiliary: bool
) -> None:
    def film(path: Path, evicting: bool) -> None:
        player = Player(*SIZE)
        feeder = Feeder(*SIZE, player)
        camera = m.Camera(three_d=True)
        # see-through layers overflow the first frame's lists: it is drawn again later, from
        # what it draws, the cloud's and the image's uploads among it
        layers = [
            m.MeshMobject(
                np.array([[-8, -5, z], [8, -5, z], [8, 5, z], [-8, 5, z]]),
                np.array([[0, 1, 2], [0, 2, 3]]),
                color=m.BLUE if i % 2 else m.RED,
                fill_opacity=0.2,
            )
            for i, z in enumerate(np.linspace(-0.2, 0.2, 12))
        ]
        cloud = m.PMobject(stroke_width=4).add_points(
            np.random.default_rng(7).uniform(-3, 3, (20, 3)),
            rgbas=np.random.default_rng(8).random((20, 4)),
        )
        image = m.ImageMobject(np.full((2, 2, 4), 128, np.uint8))
        player.begin_export(str(path), 30)
        view, records, _ = feeder.frame(camera, [*layers, cloud, image])
        if auxiliary:
            empty_view, empty_records, _ = feeder.frame(camera, [])
            player.push(
                empty_view, empty_records, cameras=[(123, *SIZE, view, records)]
            )
        else:
            player.push(view, records)
        if evicting:
            player.evict(
                [
                    key
                    for row in np.frombuffer(records, feed.RECORD)
                    for key in (int(row[name]) for name in (*feed.KEYS, "texture"))
                    if key
                ]
            )
        player.push(*feeder.frame(camera, [m.Square(fill_opacity=1)])[:2])
        player.end_export()

    reference, evicted = tmp_path / "reference.mp4", tmp_path / "evicted.mp4"
    film(reference, False)
    film(evicted, True)
    assert evicted.read_bytes() == reference.read_bytes()


@pytest.mark.config(frame_rate=10)
@pytest.mark.parametrize(
    "story", ["empty", "static", "hold", "off_grid", "sections", "cut"]
)
@pytest.mark.parametrize("preset", ["ultrafast", "slow"])
def test_video_duration_covers_the_complete_film(
    tmp_path: Path, story: str, preset: X264Preset
) -> None:
    """Every callback hold, including the closing frame, survives MP4 sample timing."""

    class Story(m.Scene):
        def construct(self) -> None:
            if story == "empty":
                return
            dot = m.Dot()
            self.add(dot)
            if story == "static":
                return
            if story == "hold":
                self.wait(1)
            else:
                self.play(dot.animate.shift(m.RIGHT), run_time=0.137)
                if story == "sections":
                    self.next_section("held")
                self.wait(0.53)

    sent: list[tuple[int, int]] = []

    def observe(frame: Frame) -> None:
        sent.append((frame.index, frame.repeat))
        if story == "cut" and len(sent) == 2:
            raise Cut

    path = tmp_path / "timing.mp4"
    film = Story().render(path, frames=observe, preset=preset)
    covered = 0
    for index, repeat in sent:
        assert index == covered
        assert repeat > 0
        covered += repeat
    assert mp4_frames(path) == film.frame_count == covered
    with av.open(str(path)) as container:
        stream = container.streams.video[0]
        assert stream.time_base is not None
        shown = [
            (frame.pts * stream.time_base * 10, frame.duration * stream.time_base * 10)
            for frame in container.decode(stream)
            if frame.pts is not None
        ]
    assert shown == sent
    if story in {"empty", "static"}:
        assert covered == 1
    if story in {"hold", "sections"}:
        assert any(repeat > 1 for _, repeat in sent)


@pytest.mark.parametrize("sound", [False, True])
@pytest.mark.parametrize(
    ("bframes", "pyramid"), [(0, "none"), (3, "none"), (3, "normal")]
)
@pytest.mark.parametrize(
    "durations",
    [
        [1] * 9 + [60],
        [40] + [1] * 9,
        [1, 8, 1, 1, 32, 1, 1, 2, 1, 16],
    ],
    ids=["last-hold", "first-hold", "mixed-holds"],
)
def test_reordered_video_preserves_each_pictures_duration(
    tmp_path: Path, bframes: int, pyramid: str, durations: list[int], sound: bool
) -> None:
    player = Player(*SIZE)
    feeder, camera = Feeder(*SIZE, player), m.Camera()
    path = tmp_path / "reordered.mp4"
    player.begin_export(
        str(path),
        10,
        "slow",
        options=[
            ("bframes", str(bframes)),
            ("b-adapt", "0"),
            ("b-pyramid", pyramid),
            ("scenecut", "0"),
        ],
    )
    expected, at = [], 0
    for index, duration in enumerate(durations):
        view, records, cameras = feeder.frame(
            camera, [m.Dot().shift((index / 10) * m.RIGHT)]
        )
        player.push(view, records, repeat=duration, cameras=cameras, key=index == 5)
        expected.append((at, duration))
        at += duration
    player.end_export(np.zeros(at * 4800, np.float32).tobytes() if sound else None)
    with av.open(str(path)) as container:
        stream = container.streams.video[0]
        assert stream.time_base is not None
        assert stream.duration is not None
        assert stream.duration * stream.time_base * 10 == at
        actual = [
            (frame.pts * stream.time_base * 10, frame.duration * stream.time_base * 10)
            for frame in container.decode(stream)
            if frame.pts is not None
        ]
    assert actual == expected
    # These controls really exercise reordering, including B-pyramid reference pictures.
    with av.open(str(path)) as container:
        order = [
            packet.pts
            for packet in container.demux(video=0)
            if packet.size and packet.pts is not None
        ]
    assert len(order) == len(durations)
    assert (order != sorted(order)) == bool(bframes)
    if sound:
        with av.open(str(path)) as container:
            audio = container.streams.audio[0]
            assert audio.duration == at * 4800
            audio_packets = [packet for packet in container.demux(audio) if packet.size]
        with av.open(str(path)) as container:
            video_packets = [
                packet for packet in container.demux(video=0) if packet.size
            ]
        # Every decoding predecessor arrives before the audio chunk containing the
        # earliest picture still needing it, even if it owns a long display hold.
        for i, packet in enumerate(video_packets):
            deadline = min(order[i:]) // 10
            audible = next(
                p
                for p in audio_packets
                if p.pts is not None and p.pts >= deadline * 48000
            )
            assert packet.pos is not None
            assert audible.pos is not None
            assert packet.pos < audible.pos