A community-maintained Python framework for creating mathematical animations.
Refactored graphical unit testing system, and implemented multi frames tests. (#1802)
Hugues Devimeux7e6991e7aec9
diff --git a/manim/renderer/cairo_renderer.py b/manim/renderer/cairo_renderer.pyindex 537b86b2f..fc7da7a95 100644--- a/manim/renderer/cairo_renderer.py+++ b/manim/renderer/cairo_renderer.py@@ -57,11 +57,18 @@ class CairoRenderer:time: time elapsed since initialisation of scene."""- def __init__(self, camera_class=None, skip_animations=False, **kwargs):+ def __init__(+ self,+ file_writer_class=SceneFileWriter,+ camera_class=None,+ skip_animations=False,+ **kwargs,+ ):# All of the following are set to EITHER the value passed via kwargs,# OR the value stored in the global config dict at the time of# _instance construction_.self.file_writer = None+ self._file_writer_class = file_writer_classcamera_cls = camera_class if camera_class is not None else Cameraself.camera = camera_cls()self._original_skipping_status = skip_animations@@ -72,7 +79,7 @@ class CairoRenderer:self.static_image = Nonedef init_scene(self, scene):- self.file_writer = SceneFileWriter(+ self.file_writer = self._file_writer_class(self,scene.__class__.__name__,)diff --git a/manim/renderer/opengl_renderer.py b/manim/renderer/opengl_renderer.pyindex d0688a1fc..9c9d77e12 100644--- a/manim/renderer/opengl_renderer.py+++ b/manim/renderer/opengl_renderer.py@@ -203,9 +203,10 @@ JOINT_TYPE_MAP = {class OpenGLRenderer:- def __init__(self, skip_animations=False):+ def __init__(self, file_writer_class=SceneFileWriter, skip_animations=False):# Measured in pixel widths, used for vector graphicsself.anti_alias_width = 1.5+ self._file_writer_class = file_writer_classself._original_skipping_status = skip_animationsself.skip_animations = skip_animations@@ -220,7 +221,7 @@ class OpenGLRenderer:def init_scene(self, scene):self.partial_movie_files = []- self.file_writer = SceneFileWriter(+ self.file_writer = self._file_writer_class(self,scene.__class__.__name__,)diff --git a/tests/conftest.py b/tests/conftest.pyindex f90a4d850..79f4b266b 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -17,6 +17,12 @@ def pytest_addoption(parser):default=False,help="Will show a visual comparison if a graphical unit test fails.",)+ parser.addoption(+ "--set_test",+ action="store_true",+ default=False,+ help="Will create the control data for EACH running tests. ",+ )def pytest_configure(config):diff --git a/tests/test_graphical_units/testing/_frames_testers.py b/tests/test_graphical_units/testing/_frames_testers.pynew file mode 100644index 000000000..432f8e92b--- /dev/null+++ b/tests/test_graphical_units/testing/_frames_testers.py@@ -0,0 +1,73 @@+import contextlib+from pathlib import Path++import numpy as np++from manim import logger++from ._show_diff import show_diff_helper+++class _FramesTester:+ def __init__(self, file_path: Path, show_diff=False) -> None:+ self._file_path = file_path+ self._show_diff = show_diff+ self._frames: np.ndarray+ self._number_frames: int = 0+ self._frames_compared = 0++ @contextlib.contextmanager+ def testing(self):+ with np.load(self._file_path) as data:+ self._frames = data["frame_data"]+ # For backward compatibility, when the control data contains only one frame (<= v0.8.0)+ if len(self._frames.shape) != 4:+ self._frames = np.expand_dims(self._frames, axis=0)+ self._number_frames = np.ma.size(self._frames, axis=0)+ yield+ assert self._frames_compared == self._number_frames, (+ f"The scene tested contained {self._frames_compared} frames, "+ f"when there are {self._number_frames} control frames for this test."+ )++ def check_frame(self, frame_number: int, frame: np.ndarray):+ assert frame_number < self._number_frames, (+ f"The tested scene is at frame number {frame_number} "+ f"when there are {self._number_frames} control frames."+ )+ try:+ np.testing.assert_array_equal(+ self._frames[frame_number],+ frame,+ err_msg=f"Frame no {frame_number}. You can use --show_diff to visually show the difference.",+ verbose=False,+ )+ self._frames_compared += 1+ except AssertionError as e:+ if self._show_diff:+ show_diff_helper(frame_number, frame, self._frames[frame_number])+ raise e+++class _ControlDataWriter(_FramesTester):+ def __init__(self, file_path: Path) -> None:+ self.file_path = file_path+ self.frames = np.empty((0, 480, 854, 4))+ self._number_frames_written: int = 0++ # Actually write a frame.+ def check_frame(self, index: int, frame: np.ndarray):+ frame = frame[np.newaxis, ...]+ self.frames = np.concatenate((self.frames, frame))+ self._number_frames_written += 1++ @contextlib.contextmanager+ def testing(self):+ yield+ self.save_contol_data()++ def save_contol_data(self):+ np.savez_compressed(self.file_path, frame_data=self.frames)+ logger.info(+ f"{self._number_frames_written} control frames saved in {self.file_path}"+ )diff --git a/tests/test_graphical_units/testing/_show_diff.py b/tests/test_graphical_units/testing/_show_diff.pynew file mode 100644index 000000000..8c0383325--- /dev/null+++ b/tests/test_graphical_units/testing/_show_diff.py@@ -0,0 +1,38 @@+import numpy as np+++def show_diff_helper(+ frame_number: int, frame_data: np.ndarray, expected_frame_data: np.ndarray+):+ """Will visually display with matplotlib differences between frame generated and the one expected."""+ import matplotlib.gridspec as gridspec+ import matplotlib.pyplot as plt++ gs = gridspec.GridSpec(2, 2)+ fig = plt.figure()+ fig.suptitle(f"Test difference summary at frame {frame_number}", fontsize=16)++ ax = fig.add_subplot(gs[0, 0])+ ax.imshow(frame_data)+ ax.set_title("Generated :")++ ax = fig.add_subplot(gs[0, 1])+ ax.imshow(expected_frame_data)+ ax.set_title("Expected :")++ ax = fig.add_subplot(gs[1, :])+ diff_im = expected_frame_data.copy()+ diff_im = np.where(+ frame_data != np.array([0, 0, 0, 255]),+ np.array([0, 255, 0, 255], dtype="uint8"),+ np.array([0, 0, 0, 255], dtype="uint8"),+ ) # Set any non-black pixels to green+ np.putmask(+ diff_im,+ expected_frame_data != frame_data,+ np.array([255, 0, 0, 255], dtype="uint8"),+ ) # Set any different pixels to red+ ax.imshow(diff_im, interpolation="nearest")+ ax.set_title("Differences summary : (green = same, red = different)")++ plt.show()diff --git a/tests/test_graphical_units/testing/_test_class_makers.py b/tests/test_graphical_units/testing/_test_class_makers.pynew file mode 100644index 000000000..5b5b8bfd1--- /dev/null+++ b/tests/test_graphical_units/testing/_test_class_makers.py@@ -0,0 +1,72 @@+from typing import Callable, Type++from manim.renderer.cairo_renderer import CairoRenderer+from manim.scene.scene import Scene+from manim.scene.scene_file_writer import SceneFileWriter++from ._frames_testers import _FramesTester+++def _make_test_scene_class(+ base_scene: Type[Scene],+ construct_test: Callable[[Scene], None],+ test_renderer,+) -> Type[Scene]:+ class _TestedScene(base_scene):+ def __init__(self, *args, **kwargs):+ super().__init__(renderer=test_renderer, *args, **kwargs)++ def construct(self):+ construct_test(self)++ # Manim hack to render the very last frame (normally the last frame is not the very end of the animation)+ self.update_to_time(self.get_run_time(self.animations))+ self.renderer.render(self, 1, self.moving_mobjects)++ return _TestedScene+++def _make_test_renderer_class(from_renderer):+ # Just for inheritance.+ class _TestRenderer(from_renderer):+ pass++ return _TestRenderer+++class DummySceneFileWriter(SceneFileWriter):+ """Delegate of SceneFileWriter used to test the frames."""++ def __init__(self, renderer, scene_name, **kwargs):+ super().__init__(renderer, scene_name, **kwargs)+ self.i = 0++ def init_output_directories(self, scene_name):+ pass++ def add_partial_movie_file(self, hash_animation):+ pass++ def begin_animation(self, allow_write=True):+ pass++ def end_animation(self, allow_write):+ pass++ def combine_movie_files(self, partial_movie_files):+ pass++ def clean_cache(self):+ pass++ def write_frame(self, frame_or_renderer):+ self.i += 1+++def _make_scene_file_writer_class(tester: _FramesTester) -> Type[SceneFileWriter]:+ class TestSceneFileWriter(DummySceneFileWriter):+ def write_frame(self, frame_or_renderer):+ tester.check_frame(self.i, frame_or_renderer)+ super().write_frame(frame_or_renderer)++ return TestSceneFileWriterdiff --git a/tests/test_graphical_units/testing/config_graphical_tests_monoframe.cfg b/tests/test_graphical_units/testing/config_graphical_tests_monoframe.cfgnew file mode 100644index 000000000..2034b502d--- /dev/null+++ b/tests/test_graphical_units/testing/config_graphical_tests_monoframe.cfg@@ -0,0 +1,7 @@+# Custom manim configuration used to run tests when only using the last frame.+[CLI]++frame_rate = 6+pixel_height = 480+pixel_width = 854+disable_caching = Truediff --git a/tests/test_graphical_units/testing/config_graphical_tests_multiframes.cfg b/tests/test_graphical_units/testing/config_graphical_tests_multiframes.cfgnew file mode 100644index 000000000..d2d90c5be--- /dev/null+++ b/tests/test_graphical_units/testing/config_graphical_tests_multiframes.cfg@@ -0,0 +1,7 @@+# Custom manim configuration used to run tests when testing several frames.+[CLI]++frame_rate = 6+pixel_height = 480+pixel_width = 854+disable_caching = Truediff --git a/tests/test_graphical_units/testing/frames_comparison.py b/tests/test_graphical_units/testing/frames_comparison.pynew file mode 100644index 000000000..37e3b9788--- /dev/null+++ b/tests/test_graphical_units/testing/frames_comparison.py@@ -0,0 +1,219 @@+import functools+import inspect+import itertools+from pathlib import Path+from typing import Any, Callable, Type++from _pytest.fixtures import FixtureRequest+from _pytest.mark.structures import Mark++from manim import Scene, config+from manim._config import tempconfig+from manim._config.utils import ManimConfig+from manim.renderer.cairo_renderer import CairoRenderer++from ._frames_testers import _ControlDataWriter, _FramesTester+from ._test_class_makers import (+ DummySceneFileWriter,+ _make_scene_file_writer_class,+ _make_test_renderer_class,+ _make_test_scene_class,+)++SCENE_PARAMETER_NAME = "scene"+_tests_root_dir_path = Path(__file__).absolute().parents[2]+PATH_CONTROL_DATA = _tests_root_dir_path / Path("control_data", "graphical_units_data")+++def frames_comparison(+ test_name: str,+ module_name: str,+ last_frame: bool = True,+ renderer_class=CairoRenderer,+ base_scene=Scene,+ **custom_config,+):+ """Compares the frames generated by the test with control frames+ specified with the ``test_name`` and ``module_name``.++ If there is no control frames for this test, the test will fail. To generate+ control frames for a given test, pass ``--set_test`` flag to pytest+ while running the test.++ Parameters+ ----------+ test_name+ The name of the test.+ module_name+ The module which the test belongs to.+ last_frame+ whether the test should test the last frame, by default True.+ renderer_class+ The base renderer to use (OpenGLRenderer/CairoRenderer), by default CairoRenderer+ base_scene+ The base class for the scene (ThreeDScene, etc.), by default Scene++ .. warning::+ By default, last_frame is True, which means that only the last frame is tested.+ If the scene has a moving animation, then the test must set last_frame to False.+ """++ def decorator(tested_scene_construct):+ if (+ SCENE_PARAMETER_NAME+ not in inspect.getfullargspec(tested_scene_construct).args+ ):+ raise Exception(+ f"Invalid graphical test function test function : must have '{SCENE_PARAMETER_NAME}'as one of the parameters."+ )++ # Exclude "scene" from the argument list of the signature.+ old_sig = inspect.signature(+ functools.partial(tested_scene_construct, scene=None)+ )++ @functools.wraps(tested_scene_construct)+ # The "request" parameter is meant to be used as a fixture by pytest. See below.+ def wrapper(*args, request: FixtureRequest, tmp_path, **kwargs):+ # Wraps the test_function to a construct method, to "freeze" the eventual additional arguments (parametrizations fixtures).+ construct = functools.partial(tested_scene_construct, *args, **kwargs)++ # Kwargs contains the eventual parametrization arguments.+ # This modify the test_name so the it is defined by the parametrization arguments too.+ # Ex : if "length" is parametrized from 0 to 20, the kwargs will be with once with {"length" : 1}, etc.+ test_name_with_param = test_name + "_".join(+ map(lambda tup: f"{str(tup[0])}:{str(tup[1])}", kwargs.items())+ )++ setting_test = request.config.getoption("--set_test")+ real_test = _make_test_comparing_frames(+ file_path=_control_data_path(+ module_name, test_name_with_param, setting_test+ ),+ base_scene=base_scene,+ construct=construct,+ renderer_class=renderer_class,+ is_set_test_data_test=setting_test,+ last_frame=last_frame,+ show_diff=request.config.getoption("--show_diff"),+ )++ config_tests = _config_test(last_frame)++ config_tests["text_dir"] = tmp_path+ config_tests["tex_dir"] = tmp_path++ if last_frame:+ config["frame_rate"] = 1+ config["dry_run"] = True++ # Isolate the config used for the test, to avoid a modifying the global config during the test run.+ with tempconfig({**custom_config, **config_tests}):+ real_test()++ parameters = list(old_sig.parameters.values())+ # Adds "request" param into the signature of the wrapper, to use the associated pytest fixture.+ # This fixture is needed to have access to flags value and pytest's config. See above.+ if "request" not in old_sig.parameters:+ parameters += [inspect.Parameter("request", inspect.Parameter.KEYWORD_ONLY)]+ if "tmp_path" not in old_sig.parameters:+ parameters += [+ inspect.Parameter("tmp_path", inspect.Parameter.KEYWORD_ONLY)+ ]+ new_sig = old_sig.replace(parameters=parameters)+ wrapper.__signature__ = new_sig++ # Reach a bit into pytest internals to hoist the marks from our wrapped+ # function.+ setattr(wrapper, "pytestmark", [])+ new_marks = getattr(tested_scene_construct, "pytestmark", [])+ wrapper.pytestmark = new_marks+ return wrapper++ return decorator+++def _make_test_comparing_frames(+ file_path: Path,+ base_scene: Type[Scene],+ construct: Callable[[Scene], None],+ renderer_class, # Renderer type, there is no superclass renderer yet .....+ is_set_test_data_test: bool,+ last_frame: bool,+ show_diff: bool,+) -> Callable[[], None]:+ """Create the real pytest test that will fail if the frames mismatch.++ Parameters+ ----------+ file_path : Path+ The path of the control frames.+ base_scene : Type[Scene]+ The base scene class.+ construct : Callable[[Scene], None]+ The construct method (= the test function)+ renderer_class : [type]+ The renderer base class.+ show_diff : bool+ whether to visually show_diff (see --show_diff)++ Returns+ -------+ Callable[[], None]+ The pytest test.+ """++ if is_set_test_data_test:+ frames_tester = _ControlDataWriter(file_path)+ else:+ frames_tester = _FramesTester(file_path, show_diff=show_diff)++ file_writer_class = (+ _make_scene_file_writer_class(frames_tester)+ if not last_frame+ else DummySceneFileWriter+ )+ testRenderer = _make_test_renderer_class(renderer_class)++ def real_test():+ with frames_tester.testing():+ sceneTested = _make_test_scene_class(+ base_scene=base_scene,+ construct_test=construct,+ test_renderer=testRenderer(file_writer_class=file_writer_class),+ )+ scene_tested = sceneTested(skip_animations=True)+ scene_tested.render()+ if last_frame:+ frames_tester.check_frame(-1, scene_tested.renderer.get_frame())++ return real_test+++def _control_data_path(module_name: str, test_name: str, setting_test: bool) -> Path:+ path = PATH_CONTROL_DATA / module_name+ if setting_test:+ # Create the directory if not existing.+ path.mkdir(exist_ok=True)+ if not setting_test and not path.exists():+ raise Exception(f"The control frames directory can't be found in {path}")+ path = (path / test_name).with_suffix(".npz")+ if not setting_test and not path.is_file():+ raise Exception(+ f"The control frame for the test {test_name} cannot be found in {path.parent}. "+ "Make sure you generated the control frames first."+ )+ return path+++def _config_test(last_frame: bool) -> ManimConfig:+ return ManimConfig().digest_file(+ str(+ Path(__file__).parent+ / (+ "config_graphical_tests_monoframe.cfg"+ if last_frame+ else "config_graphical_tests_multiframes.cfg"+ )+ )+ )