Skip to content

armgit / manim

Public Imported from GitHub
0

A community-maintained Python framework for creating mathematical animations.

Refactored graphical unit testing system, and implemented multi frames tests. (#1802)

Hugues Devimeux7e6991e7aec9
diff --git a/manim/renderer/cairo_renderer.py b/manim/renderer/cairo_renderer.py
index 537b86b2f..fc7da7a95 100644
--- a/manim/renderer/cairo_renderer.py
+++ b/manim/renderer/cairo_renderer.py
@@ -57,11 +57,18 @@ class CairoRenderer:
time: time elapsed since initialisation of scene.
"""
- def __init__(self, camera_class=None, skip_animations=False, **kwargs):
+ def __init__(
+ self,
+ file_writer_class=SceneFileWriter,
+ camera_class=None,
+ skip_animations=False,
+ **kwargs,
+ ):
# All of the following are set to EITHER the value passed via kwargs,
# OR the value stored in the global config dict at the time of
# _instance construction_.
self.file_writer = None
+ self._file_writer_class = file_writer_class
camera_cls = camera_class if camera_class is not None else Camera
self.camera = camera_cls()
self._original_skipping_status = skip_animations
@@ -72,7 +79,7 @@ class CairoRenderer:
self.static_image = None
def init_scene(self, scene):
- self.file_writer = SceneFileWriter(
+ self.file_writer = self._file_writer_class(
self,
scene.__class__.__name__,
)
diff --git a/manim/renderer/opengl_renderer.py b/manim/renderer/opengl_renderer.py
index d0688a1fc..9c9d77e12 100644
--- a/manim/renderer/opengl_renderer.py
+++ b/manim/renderer/opengl_renderer.py
@@ -203,9 +203,10 @@ JOINT_TYPE_MAP = {
class OpenGLRenderer:
- def __init__(self, skip_animations=False):
+ def __init__(self, file_writer_class=SceneFileWriter, skip_animations=False):
# Measured in pixel widths, used for vector graphics
self.anti_alias_width = 1.5
+ self._file_writer_class = file_writer_class
self._original_skipping_status = skip_animations
self.skip_animations = skip_animations
@@ -220,7 +221,7 @@ class OpenGLRenderer:
def init_scene(self, scene):
self.partial_movie_files = []
- self.file_writer = SceneFileWriter(
+ self.file_writer = self._file_writer_class(
self,
scene.__class__.__name__,
)
diff --git a/tests/conftest.py b/tests/conftest.py
index f90a4d850..79f4b266b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -17,6 +17,12 @@ def pytest_addoption(parser):
default=False,
help="Will show a visual comparison if a graphical unit test fails.",
)
+ parser.addoption(
+ "--set_test",
+ action="store_true",
+ default=False,
+ help="Will create the control data for EACH running tests. ",
+ )
def pytest_configure(config):
diff --git a/tests/test_graphical_units/testing/_frames_testers.py b/tests/test_graphical_units/testing/_frames_testers.py
new file mode 100644
index 000000000..432f8e92b
--- /dev/null
+++ b/tests/test_graphical_units/testing/_frames_testers.py
@@ -0,0 +1,73 @@
+import contextlib
+from pathlib import Path
+
+import numpy as np
+
+from manim import logger
+
+from ._show_diff import show_diff_helper
+
+
+class _FramesTester:
+ def __init__(self, file_path: Path, show_diff=False) -> None:
+ self._file_path = file_path
+ self._show_diff = show_diff
+ self._frames: np.ndarray
+ self._number_frames: int = 0
+ self._frames_compared = 0
+
+ @contextlib.contextmanager
+ def testing(self):
+ with np.load(self._file_path) as data:
+ self._frames = data["frame_data"]
+ # For backward compatibility, when the control data contains only one frame (<= v0.8.0)
+ if len(self._frames.shape) != 4:
+ self._frames = np.expand_dims(self._frames, axis=0)
+ self._number_frames = np.ma.size(self._frames, axis=0)
+ yield
+ assert self._frames_compared == self._number_frames, (
+ f"The scene tested contained {self._frames_compared} frames, "
+ f"when there are {self._number_frames} control frames for this test."
+ )
+
+ def check_frame(self, frame_number: int, frame: np.ndarray):
+ assert frame_number < self._number_frames, (
+ f"The tested scene is at frame number {frame_number} "
+ f"when there are {self._number_frames} control frames."
+ )
+ try:
+ np.testing.assert_array_equal(
+ self._frames[frame_number],
+ frame,
+ err_msg=f"Frame no {frame_number}. You can use --show_diff to visually show the difference.",
+ verbose=False,
+ )
+ self._frames_compared += 1
+ except AssertionError as e:
+ if self._show_diff:
+ show_diff_helper(frame_number, frame, self._frames[frame_number])
+ raise e
+
+
+class _ControlDataWriter(_FramesTester):
+ def __init__(self, file_path: Path) -> None:
+ self.file_path = file_path
+ self.frames = np.empty((0, 480, 854, 4))
+ self._number_frames_written: int = 0
+
+ # Actually write a frame.
+ def check_frame(self, index: int, frame: np.ndarray):
+ frame = frame[np.newaxis, ...]
+ self.frames = np.concatenate((self.frames, frame))
+ self._number_frames_written += 1
+
+ @contextlib.contextmanager
+ def testing(self):
+ yield
+ self.save_contol_data()
+
+ def save_contol_data(self):
+ np.savez_compressed(self.file_path, frame_data=self.frames)
+ logger.info(
+ f"{self._number_frames_written} control frames saved in {self.file_path}"
+ )
diff --git a/tests/test_graphical_units/testing/_show_diff.py b/tests/test_graphical_units/testing/_show_diff.py
new file mode 100644
index 000000000..8c0383325
--- /dev/null
+++ b/tests/test_graphical_units/testing/_show_diff.py
@@ -0,0 +1,38 @@
+import numpy as np
+
+
+def show_diff_helper(
+ frame_number: int, frame_data: np.ndarray, expected_frame_data: np.ndarray
+):
+ """Will visually display with matplotlib differences between frame generated and the one expected."""
+ import matplotlib.gridspec as gridspec
+ import matplotlib.pyplot as plt
+
+ gs = gridspec.GridSpec(2, 2)
+ fig = plt.figure()
+ fig.suptitle(f"Test difference summary at frame {frame_number}", fontsize=16)
+
+ ax = fig.add_subplot(gs[0, 0])
+ ax.imshow(frame_data)
+ ax.set_title("Generated :")
+
+ ax = fig.add_subplot(gs[0, 1])
+ ax.imshow(expected_frame_data)
+ ax.set_title("Expected :")
+
+ ax = fig.add_subplot(gs[1, :])
+ diff_im = expected_frame_data.copy()
+ diff_im = np.where(
+ frame_data != np.array([0, 0, 0, 255]),
+ np.array([0, 255, 0, 255], dtype="uint8"),
+ np.array([0, 0, 0, 255], dtype="uint8"),
+ ) # Set any non-black pixels to green
+ np.putmask(
+ diff_im,
+ expected_frame_data != frame_data,
+ np.array([255, 0, 0, 255], dtype="uint8"),
+ ) # Set any different pixels to red
+ ax.imshow(diff_im, interpolation="nearest")
+ ax.set_title("Differences summary : (green = same, red = different)")
+
+ plt.show()
diff --git a/tests/test_graphical_units/testing/_test_class_makers.py b/tests/test_graphical_units/testing/_test_class_makers.py
new file mode 100644
index 000000000..5b5b8bfd1
--- /dev/null
+++ b/tests/test_graphical_units/testing/_test_class_makers.py
@@ -0,0 +1,72 @@
+from typing import Callable, Type
+
+from manim.renderer.cairo_renderer import CairoRenderer
+from manim.scene.scene import Scene
+from manim.scene.scene_file_writer import SceneFileWriter
+
+from ._frames_testers import _FramesTester
+
+
+def _make_test_scene_class(
+ base_scene: Type[Scene],
+ construct_test: Callable[[Scene], None],
+ test_renderer,
+) -> Type[Scene]:
+ class _TestedScene(base_scene):
+ def __init__(self, *args, **kwargs):
+ super().__init__(renderer=test_renderer, *args, **kwargs)
+
+ def construct(self):
+ construct_test(self)
+
+ # Manim hack to render the very last frame (normally the last frame is not the very end of the animation)
+ self.update_to_time(self.get_run_time(self.animations))
+ self.renderer.render(self, 1, self.moving_mobjects)
+
+ return _TestedScene
+
+
+def _make_test_renderer_class(from_renderer):
+ # Just for inheritance.
+ class _TestRenderer(from_renderer):
+ pass
+
+ return _TestRenderer
+
+
+class DummySceneFileWriter(SceneFileWriter):
+ """Delegate of SceneFileWriter used to test the frames."""
+
+ def __init__(self, renderer, scene_name, **kwargs):
+ super().__init__(renderer, scene_name, **kwargs)
+ self.i = 0
+
+ def init_output_directories(self, scene_name):
+ pass
+
+ def add_partial_movie_file(self, hash_animation):
+ pass
+
+ def begin_animation(self, allow_write=True):
+ pass
+
+ def end_animation(self, allow_write):
+ pass
+
+ def combine_movie_files(self, partial_movie_files):
+ pass
+
+ def clean_cache(self):
+ pass
+
+ def write_frame(self, frame_or_renderer):
+ self.i += 1
+
+
+def _make_scene_file_writer_class(tester: _FramesTester) -> Type[SceneFileWriter]:
+ class TestSceneFileWriter(DummySceneFileWriter):
+ def write_frame(self, frame_or_renderer):
+ tester.check_frame(self.i, frame_or_renderer)
+ super().write_frame(frame_or_renderer)
+
+ return TestSceneFileWriter
diff --git a/tests/test_graphical_units/testing/config_graphical_tests_monoframe.cfg b/tests/test_graphical_units/testing/config_graphical_tests_monoframe.cfg
new file mode 100644
index 000000000..2034b502d
--- /dev/null
+++ b/tests/test_graphical_units/testing/config_graphical_tests_monoframe.cfg
@@ -0,0 +1,7 @@
+# Custom manim configuration used to run tests when only using the last frame.
+[CLI]
+
+frame_rate = 6
+pixel_height = 480
+pixel_width = 854
+disable_caching = True
diff --git a/tests/test_graphical_units/testing/config_graphical_tests_multiframes.cfg b/tests/test_graphical_units/testing/config_graphical_tests_multiframes.cfg
new file mode 100644
index 000000000..d2d90c5be
--- /dev/null
+++ b/tests/test_graphical_units/testing/config_graphical_tests_multiframes.cfg
@@ -0,0 +1,7 @@
+# Custom manim configuration used to run tests when testing several frames.
+[CLI]
+
+frame_rate = 6
+pixel_height = 480
+pixel_width = 854
+disable_caching = True
diff --git a/tests/test_graphical_units/testing/frames_comparison.py b/tests/test_graphical_units/testing/frames_comparison.py
new file mode 100644
index 000000000..37e3b9788
--- /dev/null
+++ b/tests/test_graphical_units/testing/frames_comparison.py
@@ -0,0 +1,219 @@
+import functools
+import inspect
+import itertools
+from pathlib import Path
+from typing import Any, Callable, Type
+
+from _pytest.fixtures import FixtureRequest
+from _pytest.mark.structures import Mark
+
+from manim import Scene, config
+from manim._config import tempconfig
+from manim._config.utils import ManimConfig
+from manim.renderer.cairo_renderer import CairoRenderer
+
+from ._frames_testers import _ControlDataWriter, _FramesTester
+from ._test_class_makers import (
+ DummySceneFileWriter,
+ _make_scene_file_writer_class,
+ _make_test_renderer_class,
+ _make_test_scene_class,
+)
+
+SCENE_PARAMETER_NAME = "scene"
+_tests_root_dir_path = Path(__file__).absolute().parents[2]
+PATH_CONTROL_DATA = _tests_root_dir_path / Path("control_data", "graphical_units_data")
+
+
+def frames_comparison(
+ test_name: str,
+ module_name: str,
+ last_frame: bool = True,
+ renderer_class=CairoRenderer,
+ base_scene=Scene,
+ **custom_config,
+):
+ """Compares the frames generated by the test with control frames
+ specified with the ``test_name`` and ``module_name``.
+
+ If there is no control frames for this test, the test will fail. To generate
+ control frames for a given test, pass ``--set_test`` flag to pytest
+ while running the test.
+
+ Parameters
+ ----------
+ test_name
+ The name of the test.
+ module_name
+ The module which the test belongs to.
+ last_frame
+ whether the test should test the last frame, by default True.
+ renderer_class
+ The base renderer to use (OpenGLRenderer/CairoRenderer), by default CairoRenderer
+ base_scene
+ The base class for the scene (ThreeDScene, etc.), by default Scene
+
+ .. warning::
+ By default, last_frame is True, which means that only the last frame is tested.
+ If the scene has a moving animation, then the test must set last_frame to False.
+ """
+
+ def decorator(tested_scene_construct):
+ if (
+ SCENE_PARAMETER_NAME
+ not in inspect.getfullargspec(tested_scene_construct).args
+ ):
+ raise Exception(
+ f"Invalid graphical test function test function : must have '{SCENE_PARAMETER_NAME}'as one of the parameters."
+ )
+
+ # Exclude "scene" from the argument list of the signature.
+ old_sig = inspect.signature(
+ functools.partial(tested_scene_construct, scene=None)
+ )
+
+ @functools.wraps(tested_scene_construct)
+ # The "request" parameter is meant to be used as a fixture by pytest. See below.
+ def wrapper(*args, request: FixtureRequest, tmp_path, **kwargs):
+ # Wraps the test_function to a construct method, to "freeze" the eventual additional arguments (parametrizations fixtures).
+ construct = functools.partial(tested_scene_construct, *args, **kwargs)
+
+ # Kwargs contains the eventual parametrization arguments.
+ # This modify the test_name so the it is defined by the parametrization arguments too.
+ # Ex : if "length" is parametrized from 0 to 20, the kwargs will be with once with {"length" : 1}, etc.
+ test_name_with_param = test_name + "_".join(
+ map(lambda tup: f"{str(tup[0])}:{str(tup[1])}", kwargs.items())
+ )
+
+ setting_test = request.config.getoption("--set_test")
+ real_test = _make_test_comparing_frames(
+ file_path=_control_data_path(
+ module_name, test_name_with_param, setting_test
+ ),
+ base_scene=base_scene,
+ construct=construct,
+ renderer_class=renderer_class,
+ is_set_test_data_test=setting_test,
+ last_frame=last_frame,
+ show_diff=request.config.getoption("--show_diff"),
+ )
+
+ config_tests = _config_test(last_frame)
+
+ config_tests["text_dir"] = tmp_path
+ config_tests["tex_dir"] = tmp_path
+
+ if last_frame:
+ config["frame_rate"] = 1
+ config["dry_run"] = True
+
+ # Isolate the config used for the test, to avoid a modifying the global config during the test run.
+ with tempconfig({**custom_config, **config_tests}):
+ real_test()
+
+ parameters = list(old_sig.parameters.values())
+ # Adds "request" param into the signature of the wrapper, to use the associated pytest fixture.
+ # This fixture is needed to have access to flags value and pytest's config. See above.
+ if "request" not in old_sig.parameters:
+ parameters += [inspect.Parameter("request", inspect.Parameter.KEYWORD_ONLY)]
+ if "tmp_path" not in old_sig.parameters:
+ parameters += [
+ inspect.Parameter("tmp_path", inspect.Parameter.KEYWORD_ONLY)
+ ]
+ new_sig = old_sig.replace(parameters=parameters)
+ wrapper.__signature__ = new_sig
+
+ # Reach a bit into pytest internals to hoist the marks from our wrapped
+ # function.
+ setattr(wrapper, "pytestmark", [])
+ new_marks = getattr(tested_scene_construct, "pytestmark", [])
+ wrapper.pytestmark = new_marks
+ return wrapper
+
+ return decorator
+
+
+def _make_test_comparing_frames(
+ file_path: Path,
+ base_scene: Type[Scene],
+ construct: Callable[[Scene], None],
+ renderer_class, # Renderer type, there is no superclass renderer yet .....
+ is_set_test_data_test: bool,
+ last_frame: bool,
+ show_diff: bool,
+) -> Callable[[], None]:
+ """Create the real pytest test that will fail if the frames mismatch.
+
+ Parameters
+ ----------
+ file_path : Path
+ The path of the control frames.
+ base_scene : Type[Scene]
+ The base scene class.
+ construct : Callable[[Scene], None]
+ The construct method (= the test function)
+ renderer_class : [type]
+ The renderer base class.
+ show_diff : bool
+ whether to visually show_diff (see --show_diff)
+
+ Returns
+ -------
+ Callable[[], None]
+ The pytest test.
+ """
+
+ if is_set_test_data_test:
+ frames_tester = _ControlDataWriter(file_path)
+ else:
+ frames_tester = _FramesTester(file_path, show_diff=show_diff)
+
+ file_writer_class = (
+ _make_scene_file_writer_class(frames_tester)
+ if not last_frame
+ else DummySceneFileWriter
+ )
+ testRenderer = _make_test_renderer_class(renderer_class)
+
+ def real_test():
+ with frames_tester.testing():
+ sceneTested = _make_test_scene_class(
+ base_scene=base_scene,
+ construct_test=construct,
+ test_renderer=testRenderer(file_writer_class=file_writer_class),
+ )
+ scene_tested = sceneTested(skip_animations=True)
+ scene_tested.render()
+ if last_frame:
+ frames_tester.check_frame(-1, scene_tested.renderer.get_frame())
+
+ return real_test
+
+
+def _control_data_path(module_name: str, test_name: str, setting_test: bool) -> Path:
+ path = PATH_CONTROL_DATA / module_name
+ if setting_test:
+ # Create the directory if not existing.
+ path.mkdir(exist_ok=True)
+ if not setting_test and not path.exists():
+ raise Exception(f"The control frames directory can't be found in {path}")
+ path = (path / test_name).with_suffix(".npz")
+ if not setting_test and not path.is_file():
+ raise Exception(
+ f"The control frame for the test {test_name} cannot be found in {path.parent}. "
+ "Make sure you generated the control frames first."
+ )
+ return path
+
+
+def _config_test(last_frame: bool) -> ManimConfig:
+ return ManimConfig().digest_file(
+ str(
+ Path(__file__).parent
+ / (
+ "config_graphical_tests_monoframe.cfg"
+ if last_frame
+ else "config_graphical_tests_multiframes.cfg"
+ )
+ )
+ )