A community-maintained Python framework for creating mathematical animations.
perf(cache): hash NumPy arrays without JSON expansion (#4883)
Dex Hunter5ce17190d4d1
diff --git a/manim/utils/hashing.py b/manim/utils/hashing.pyindex c5c65cce8..0b0e88cd4 100644--- a/manim/utils/hashing.py+++ b/manim/utils/hashing.py@@ -3,6 +3,7 @@from __future__ import annotationsimport copy+import hashlibimport inspectimport jsonimport zlib@@ -34,6 +35,58 @@ KEYS_TO_FILTER_OUT = {}+def _canonical_field_title(title: Any) -> Any:+ if title is None:+ return None+ title_type = type(title)+ return [f"{title_type.__module__}.{title_type.__qualname__}", repr(title)]+++def _canonical_dtype_descriptor(dtype: np.dtype) -> Any:+ canonical = dtype.newbyteorder("<")+ if canonical.names is not None:+ return [+ "struct",+ [+ [+ name,+ _canonical_dtype_descriptor(canonical.fields[name][0]),+ canonical.fields[name][1],+ _canonical_field_title(canonical.fields[name][2])+ if len(canonical.fields[name]) > 2+ else None,+ ]+ for name in canonical.names or ()+ ],+ canonical.itemsize,+ canonical.alignment,+ canonical.isalignedstruct,+ ]+ if canonical.subdtype:+ base, shape = canonical.subdtype+ return ["subarray", _canonical_dtype_descriptor(base), list(shape)]+ return canonical.str+++def _hash_ndarray(array: np.ndarray) -> str:+ descriptor = _canonical_dtype_descriptor(array.dtype)+ canonical_dtype = array.dtype.newbyteorder("<")+ digest = hashlib.sha256()+ if canonical_dtype.names is not None:+ for name in canonical_dtype.names or ():+ field_hash = _hash_ndarray(np.asarray(array[name])).encode()+ digest.update(len(field_hash).to_bytes(8, "little"))+ digest.update(field_hash)+ else:+ canonical = np.ascontiguousarray(+ array.astype(canonical_dtype, copy=False),+ )+ digest.update(canonical)+ encoded_descriptor = json.dumps(descriptor, separators=(",", ":"))+ # This compacts array content before the existing CRC32 play-key pipeline.+ return f"NDARRAY:{encoded_descriptor}:{array.shape}:{digest.hexdigest()}"++class _Memoizer:"""Implements the memoization logic to optimize the hashing procedure and preventthe circular references within iterable processed.@@ -180,6 +233,15 @@ class _Memoizer:class _CustomEncoder(json.JSONEncoder):+ def __init__(+ self,+ *args: Any,+ include_pixel_array: bool = False,+ **kwargs: Any,+ ) -> None:+ super().__init__(*args, **kwargs)+ self.include_pixel_array = include_pixel_array+def default(self, obj: Any) -> Any:"""This method is used to serialize objects to JSON format.@@ -223,11 +285,12 @@ class _CustomEncoder(json.JSONEncoder):code = ""return self._cleaned_iterable({"code": code, "nonlocals": cvardict})elif isinstance(obj, np.ndarray):- if obj.size > 1000:- obj = np.resize(obj, (100, 100))- return f"TRUNCATED ARRAY: {repr(obj)}"- # We return the repr and not a list to avoid the JSONEncoder to iterate over it.- return repr(obj)+ if obj.__class__ is not np.ndarray or obj.dtype.hasobject:+ if obj.size > 1000:+ obj = np.resize(obj, (100, 100))+ return f"TRUNCATED ARRAY: {repr(obj)}"+ return repr(obj)+ return _hash_ndarray(obj)elif hasattr(obj, "__dict__"):temp = obj.__dict__# MappingProxy is scene-caching nightmare. It contains all of the object methods and attributes. We skip it as the mechanism will at some point process the object, but instantiated.@@ -278,7 +341,9 @@ class _CustomEncoder(json.JSONEncoder):processed_dict = {}for k, v in dct.items():v = _Memoizer.check_already_processed(v)- if k in KEYS_TO_FILTER_OUT:+ if k in KEYS_TO_FILTER_OUT and not (+ self.include_pixel_array and k == "pixel_array"+ ):continue# We check if the k is of the right format (supported by JSON)if not isinstance(k, (str, int, float, bool)) and k is not None:@@ -320,20 +385,26 @@ class _CustomEncoder(json.JSONEncoder):return super().encode(obj)-def get_json(obj: Any) -> str:+def get_json(obj: Any, *, include_pixel_array: bool = False) -> str:"""Recursively serialize `object` to JSON using the :class:`CustomEncoder` class.Parameters----------objThe dict to flatten+ include_pixel_array+ Whether to include pixel arrays encountered while flattening the object.Returns-------:class:`str`The flattened object"""- return json.dumps(obj, cls=_CustomEncoder)+ return json.dumps(+ obj,+ cls=_CustomEncoder,+ include_pixel_array=include_pixel_array,+ )def get_hash_from_play_call(@@ -367,8 +438,13 @@ def get_hash_from_play_call(t_start = perf_counter()_Memoizer.mark_as_processed(scene_object)camera_json = get_json(camera_object)- animations_list_json = [get_json(x) for x in sorted(animations_list, key=str)]- current_mobjects_list_json = [get_json(x) for x in current_mobjects_list]+ animations_list_json = [+ get_json(animation, include_pixel_array=True)+ for animation in sorted(animations_list, key=str)+ ]+ current_mobjects_list_json = [+ get_json(mobject, include_pixel_array=True) for mobject in current_mobjects_list+ ]hash_camera, hash_animations, hash_current_mobjects = (zlib.crc32(repr(json_val).encode())for json_val in [camera_json, animations_list_json, current_mobjects_list_json]diff --git a/tests/module/utils/test_hashing.py b/tests/module/utils/test_hashing.pyindex 53925782f..912f5344d 100644--- a/tests/module/utils/test_hashing.py+++ b/tests/module/utils/test_hashing.py@@ -3,10 +3,11 @@ from __future__ import annotationsimport jsonfrom zlib import crc32+import numpy as npimport pytestimport manim.utils.hashing as hashing-from manim import Square+from manim import ImageMobject, SquareALREADY_PROCESSED_PLACEHOLDER = hashing._Memoizer.ALREADY_PROCESSED_PLACEHOLDER@@ -115,11 +116,196 @@ def test_JSON_with_circular_references():def test_JSON_with_big_np_array():- import numpy as np-a = np.zeros((1000, 1000))- o_ser = hashing.get_json(a)- assert "TRUNCATED ARRAY" in o_ser+ serialized = hashing.get_json(a)+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(a.copy()) == serialized++ a[500, 500] = 1+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(a) != serialized+++def test_JSON_with_equivalent_np_array_layouts():+ logical = np.arange(24, dtype=np.float64).reshape(4, 6) / 3+ serialized = hashing.get_json(np.array(logical, order="C"))++ for equivalent in [+ np.array(logical, order="F"),+ logical.astype(logical.dtype.newbyteorder(">")),+ ]:+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(equivalent) == serialized++ backing = np.empty((4, 12), dtype=np.float64)+ backing[:, ::2] = logical+ backing[:, 1::2] = -1+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(backing[:, ::2]) == serialized+++def test_JSON_with_out_of_order_and_overlapping_structured_arrays():+ for dtype in [+ np.dtype(+ {+ "names": ["late", "early"],+ "formats": ["<i4", "<i2"],+ "offsets": [8, 0],+ "itemsize": 12,+ }+ ),+ np.dtype(+ {+ "names": ["wide", "narrow"],+ "formats": ["<u4", "<u2"],+ "offsets": [0, 2],+ "itemsize": 4,+ }+ ),+ ]:+ array = np.zeros(8, dtype=dtype)+ first_field = dtype.names[0]+ array[first_field] = np.arange(8) + 100+ hashing._Memoizer.reset_already_processed()+ serialized = hashing.get_json(array)++ changed = array.copy()+ changed[first_field][4] += 1+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(changed) != serialized+++def test_JSON_preserves_structured_dtype_identity_without_hashing_padding():+ compact_dtype = np.dtype([("value", "<i4")])+ padded_dtype = np.dtype(+ {+ "names": ["value"],+ "formats": ["<i4"],+ "offsets": [0],+ "itemsize": 12,+ }+ )+ compact = np.zeros(4, dtype=compact_dtype)+ padded = np.zeros(4, dtype=padded_dtype)+ compact["value"] = padded["value"] = np.arange(4)++ compact_json = hashing.get_json(compact)+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(padded) != compact_json++ shifted_dtype = np.dtype(+ {+ "names": ["value"],+ "formats": ["<i4"],+ "offsets": [4],+ "itemsize": 12,+ }+ )+ shifted = np.zeros(4, dtype=shifted_dtype)+ shifted["value"] = np.arange(4)+ hashing._Memoizer.reset_already_processed()+ shifted_json = hashing.get_json(shifted)+ hashing._Memoizer.reset_already_processed()+ assert shifted_json != hashing.get_json(padded)++ alternate_padding = padded.copy()+ alternate_padding.view(np.uint8).reshape(4, 12)[:, 4:] = 0xFF+ hashing._Memoizer.reset_already_processed()+ padded_json = hashing.get_json(padded)+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(alternate_padding) == padded_json++ titled = np.zeros(4, dtype=np.dtype([(("title", "value"), "<i4")]))+ titled["value"] = np.arange(4)+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(titled) != compact_json++ aligned_dtype = np.dtype([("flag", "u1"), ("value", "<u4")], align=True)+ unaligned_dtype = np.dtype(+ {+ "names": ["flag", "value"],+ "formats": ["u1", "<u4"],+ "offsets": [0, 4],+ "itemsize": 8,+ },+ align=False,+ )+ aligned = np.zeros(4, dtype=aligned_dtype)+ unaligned = np.zeros(4, dtype=unaligned_dtype)+ aligned["value"] = unaligned["value"] = np.arange(4)+ hashing._Memoizer.reset_already_processed()+ aligned_json = hashing.get_json(aligned)+ hashing._Memoizer.reset_already_processed()+ assert aligned_json != hashing.get_json(unaligned)++ hashing._Memoizer.reset_already_processed()+ empty_struct_json = hashing.get_json(np.empty(4, dtype=np.dtype([])))+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(np.empty(4, dtype=np.dtype("V0"))) != empty_struct_json++ empty_padded_dtype = np.dtype({"names": [], "formats": [], "itemsize": 8})+ empty_padded = np.zeros(4, dtype=empty_padded_dtype)+ alternate_empty_padding = empty_padded.copy()+ alternate_empty_padding.view(np.uint8)[:] = 0xFF+ hashing._Memoizer.reset_already_processed()+ empty_padded_json = hashing.get_json(empty_padded)+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(alternate_empty_padding) == empty_padded_json+ hashing._Memoizer.reset_already_processed()+ assert hashing.get_json(np.zeros(4, dtype="V8")) != empty_padded_json++ bytes_title = np.zeros(4, dtype=np.dtype([((b"title", "value"), "<i4")]))+ scalar_title = np.zeros(+ 4,+ dtype=np.dtype([((np.int64(7), "value"), "<i4")]),+ )+ bytes_title["value"] = scalar_title["value"] = np.arange(4)+ hashing._Memoizer.reset_already_processed()+ bytes_title_json = hashing.get_json(bytes_title)+ hashing._Memoizer.reset_already_processed()+ assert bytes_title_json != hashing.get_json(scalar_title)+++def test_play_hash_includes_mobject_pixels_but_not_camera_pixels():+ class HashableObject:+ def __init__(self, name: str, pixels: np.ndarray) -> None:+ self.name = name+ self.pixel_array = pixels++ def __str__(self) -> str:+ return self.name++ scene = HashableObject("scene", np.arange(8, dtype=np.uint8))+ camera = HashableObject("camera", np.arange(8, dtype=np.uint8))+ mobject = ImageMobject(np.zeros((8, 8, 4), dtype=np.uint8))++ original = hashing.get_hash_from_play_call(scene, camera, [], [mobject])+ mobject.pixel_array[4, 4, 0] ^= 1+ assert hashing.get_hash_from_play_call(scene, camera, [], [mobject]) != original++ mobject.pixel_array[4, 4, 0] ^= 1+ camera.pixel_array[0] ^= 1+ assert hashing.get_hash_from_play_call(scene, camera, [], [mobject]) == original+++def test_play_hash_keeps_distinct_mobjects_with_equal_python_hashes():+ class CollidingObject:+ def __init__(self, name: str) -> None:+ self.name = name++ def __str__(self) -> str:+ return self.name++ def __hash__(self) -> int:+ return 1++ scene = CollidingObject("scene")+ camera = CollidingObject("camera")+ mobjects = [CollidingObject("first"), CollidingObject("second")]++ original = hashing.get_hash_from_play_call(scene, camera, [], mobjects)+ mobjects[1].name = "changed"+ assert hashing.get_hash_from_play_call(scene, camera, [], mobjects) != originaldef test_JSON_with_tuple():