Skip to content

armgit / manim

Public Imported from GitHub
0

A community-maintained Python framework for creating mathematical animations.

perf(cache): hash NumPy arrays without JSON expansion (#4883)

Dex Hunter5ce17190d4d1
diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py
index c5c65cce8..0b0e88cd4 100644
--- a/manim/utils/hashing.py
+++ b/manim/utils/hashing.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import copy
+import hashlib
import inspect
import json
import zlib
@@ -34,6 +35,58 @@ KEYS_TO_FILTER_OUT = {
}
+def _canonical_field_title(title: Any) -> Any:
+ if title is None:
+ return None
+ title_type = type(title)
+ return [f"{title_type.__module__}.{title_type.__qualname__}", repr(title)]
+
+
+def _canonical_dtype_descriptor(dtype: np.dtype) -> Any:
+ canonical = dtype.newbyteorder("<")
+ if canonical.names is not None:
+ return [
+ "struct",
+ [
+ [
+ name,
+ _canonical_dtype_descriptor(canonical.fields[name][0]),
+ canonical.fields[name][1],
+ _canonical_field_title(canonical.fields[name][2])
+ if len(canonical.fields[name]) > 2
+ else None,
+ ]
+ for name in canonical.names or ()
+ ],
+ canonical.itemsize,
+ canonical.alignment,
+ canonical.isalignedstruct,
+ ]
+ if canonical.subdtype:
+ base, shape = canonical.subdtype
+ return ["subarray", _canonical_dtype_descriptor(base), list(shape)]
+ return canonical.str
+
+
+def _hash_ndarray(array: np.ndarray) -> str:
+ descriptor = _canonical_dtype_descriptor(array.dtype)
+ canonical_dtype = array.dtype.newbyteorder("<")
+ digest = hashlib.sha256()
+ if canonical_dtype.names is not None:
+ for name in canonical_dtype.names or ():
+ field_hash = _hash_ndarray(np.asarray(array[name])).encode()
+ digest.update(len(field_hash).to_bytes(8, "little"))
+ digest.update(field_hash)
+ else:
+ canonical = np.ascontiguousarray(
+ array.astype(canonical_dtype, copy=False),
+ )
+ digest.update(canonical)
+ encoded_descriptor = json.dumps(descriptor, separators=(",", ":"))
+ # This compacts array content before the existing CRC32 play-key pipeline.
+ return f"NDARRAY:{encoded_descriptor}:{array.shape}:{digest.hexdigest()}"
+
+
class _Memoizer:
"""Implements the memoization logic to optimize the hashing procedure and prevent
the circular references within iterable processed.
@@ -180,6 +233,15 @@ class _Memoizer:
class _CustomEncoder(json.JSONEncoder):
+ def __init__(
+ self,
+ *args: Any,
+ include_pixel_array: bool = False,
+ **kwargs: Any,
+ ) -> None:
+ super().__init__(*args, **kwargs)
+ self.include_pixel_array = include_pixel_array
+
def default(self, obj: Any) -> Any:
"""This method is used to serialize objects to JSON format.
@@ -223,11 +285,12 @@ class _CustomEncoder(json.JSONEncoder):
code = ""
return self._cleaned_iterable({"code": code, "nonlocals": cvardict})
elif isinstance(obj, np.ndarray):
- if obj.size > 1000:
- obj = np.resize(obj, (100, 100))
- return f"TRUNCATED ARRAY: {repr(obj)}"
- # We return the repr and not a list to avoid the JSONEncoder to iterate over it.
- return repr(obj)
+ if obj.__class__ is not np.ndarray or obj.dtype.hasobject:
+ if obj.size > 1000:
+ obj = np.resize(obj, (100, 100))
+ return f"TRUNCATED ARRAY: {repr(obj)}"
+ return repr(obj)
+ return _hash_ndarray(obj)
elif hasattr(obj, "__dict__"):
temp = obj.__dict__
# MappingProxy is scene-caching nightmare. It contains all of the object methods and attributes. We skip it as the mechanism will at some point process the object, but instantiated.
@@ -278,7 +341,9 @@ class _CustomEncoder(json.JSONEncoder):
processed_dict = {}
for k, v in dct.items():
v = _Memoizer.check_already_processed(v)
- if k in KEYS_TO_FILTER_OUT:
+ if k in KEYS_TO_FILTER_OUT and not (
+ self.include_pixel_array and k == "pixel_array"
+ ):
continue
# We check if the k is of the right format (supported by JSON)
if not isinstance(k, (str, int, float, bool)) and k is not None:
@@ -320,20 +385,26 @@ class _CustomEncoder(json.JSONEncoder):
return super().encode(obj)
-def get_json(obj: Any) -> str:
+def get_json(obj: Any, *, include_pixel_array: bool = False) -> str:
"""Recursively serialize `object` to JSON using the :class:`CustomEncoder` class.
Parameters
----------
obj
The dict to flatten
+ include_pixel_array
+ Whether to include pixel arrays encountered while flattening the object.
Returns
-------
:class:`str`
The flattened object
"""
- return json.dumps(obj, cls=_CustomEncoder)
+ return json.dumps(
+ obj,
+ cls=_CustomEncoder,
+ include_pixel_array=include_pixel_array,
+ )
def get_hash_from_play_call(
@@ -367,8 +438,13 @@ def get_hash_from_play_call(
t_start = perf_counter()
_Memoizer.mark_as_processed(scene_object)
camera_json = get_json(camera_object)
- animations_list_json = [get_json(x) for x in sorted(animations_list, key=str)]
- current_mobjects_list_json = [get_json(x) for x in current_mobjects_list]
+ animations_list_json = [
+ get_json(animation, include_pixel_array=True)
+ for animation in sorted(animations_list, key=str)
+ ]
+ current_mobjects_list_json = [
+ get_json(mobject, include_pixel_array=True) for mobject in current_mobjects_list
+ ]
hash_camera, hash_animations, hash_current_mobjects = (
zlib.crc32(repr(json_val).encode())
for json_val in [camera_json, animations_list_json, current_mobjects_list_json]
diff --git a/tests/module/utils/test_hashing.py b/tests/module/utils/test_hashing.py
index 53925782f..912f5344d 100644
--- a/tests/module/utils/test_hashing.py
+++ b/tests/module/utils/test_hashing.py
@@ -3,10 +3,11 @@ from __future__ import annotations
import json
from zlib import crc32
+import numpy as np
import pytest
import manim.utils.hashing as hashing
-from manim import Square
+from manim import ImageMobject, Square
ALREADY_PROCESSED_PLACEHOLDER = hashing._Memoizer.ALREADY_PROCESSED_PLACEHOLDER
@@ -115,11 +116,196 @@ def test_JSON_with_circular_references():
def test_JSON_with_big_np_array():
- import numpy as np
-
a = np.zeros((1000, 1000))
- o_ser = hashing.get_json(a)
- assert "TRUNCATED ARRAY" in o_ser
+ serialized = hashing.get_json(a)
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(a.copy()) == serialized
+
+ a[500, 500] = 1
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(a) != serialized
+
+
+def test_JSON_with_equivalent_np_array_layouts():
+ logical = np.arange(24, dtype=np.float64).reshape(4, 6) / 3
+ serialized = hashing.get_json(np.array(logical, order="C"))
+
+ for equivalent in [
+ np.array(logical, order="F"),
+ logical.astype(logical.dtype.newbyteorder(">")),
+ ]:
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(equivalent) == serialized
+
+ backing = np.empty((4, 12), dtype=np.float64)
+ backing[:, ::2] = logical
+ backing[:, 1::2] = -1
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(backing[:, ::2]) == serialized
+
+
+def test_JSON_with_out_of_order_and_overlapping_structured_arrays():
+ for dtype in [
+ np.dtype(
+ {
+ "names": ["late", "early"],
+ "formats": ["<i4", "<i2"],
+ "offsets": [8, 0],
+ "itemsize": 12,
+ }
+ ),
+ np.dtype(
+ {
+ "names": ["wide", "narrow"],
+ "formats": ["<u4", "<u2"],
+ "offsets": [0, 2],
+ "itemsize": 4,
+ }
+ ),
+ ]:
+ array = np.zeros(8, dtype=dtype)
+ first_field = dtype.names[0]
+ array[first_field] = np.arange(8) + 100
+ hashing._Memoizer.reset_already_processed()
+ serialized = hashing.get_json(array)
+
+ changed = array.copy()
+ changed[first_field][4] += 1
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(changed) != serialized
+
+
+def test_JSON_preserves_structured_dtype_identity_without_hashing_padding():
+ compact_dtype = np.dtype([("value", "<i4")])
+ padded_dtype = np.dtype(
+ {
+ "names": ["value"],
+ "formats": ["<i4"],
+ "offsets": [0],
+ "itemsize": 12,
+ }
+ )
+ compact = np.zeros(4, dtype=compact_dtype)
+ padded = np.zeros(4, dtype=padded_dtype)
+ compact["value"] = padded["value"] = np.arange(4)
+
+ compact_json = hashing.get_json(compact)
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(padded) != compact_json
+
+ shifted_dtype = np.dtype(
+ {
+ "names": ["value"],
+ "formats": ["<i4"],
+ "offsets": [4],
+ "itemsize": 12,
+ }
+ )
+ shifted = np.zeros(4, dtype=shifted_dtype)
+ shifted["value"] = np.arange(4)
+ hashing._Memoizer.reset_already_processed()
+ shifted_json = hashing.get_json(shifted)
+ hashing._Memoizer.reset_already_processed()
+ assert shifted_json != hashing.get_json(padded)
+
+ alternate_padding = padded.copy()
+ alternate_padding.view(np.uint8).reshape(4, 12)[:, 4:] = 0xFF
+ hashing._Memoizer.reset_already_processed()
+ padded_json = hashing.get_json(padded)
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(alternate_padding) == padded_json
+
+ titled = np.zeros(4, dtype=np.dtype([(("title", "value"), "<i4")]))
+ titled["value"] = np.arange(4)
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(titled) != compact_json
+
+ aligned_dtype = np.dtype([("flag", "u1"), ("value", "<u4")], align=True)
+ unaligned_dtype = np.dtype(
+ {
+ "names": ["flag", "value"],
+ "formats": ["u1", "<u4"],
+ "offsets": [0, 4],
+ "itemsize": 8,
+ },
+ align=False,
+ )
+ aligned = np.zeros(4, dtype=aligned_dtype)
+ unaligned = np.zeros(4, dtype=unaligned_dtype)
+ aligned["value"] = unaligned["value"] = np.arange(4)
+ hashing._Memoizer.reset_already_processed()
+ aligned_json = hashing.get_json(aligned)
+ hashing._Memoizer.reset_already_processed()
+ assert aligned_json != hashing.get_json(unaligned)
+
+ hashing._Memoizer.reset_already_processed()
+ empty_struct_json = hashing.get_json(np.empty(4, dtype=np.dtype([])))
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(np.empty(4, dtype=np.dtype("V0"))) != empty_struct_json
+
+ empty_padded_dtype = np.dtype({"names": [], "formats": [], "itemsize": 8})
+ empty_padded = np.zeros(4, dtype=empty_padded_dtype)
+ alternate_empty_padding = empty_padded.copy()
+ alternate_empty_padding.view(np.uint8)[:] = 0xFF
+ hashing._Memoizer.reset_already_processed()
+ empty_padded_json = hashing.get_json(empty_padded)
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(alternate_empty_padding) == empty_padded_json
+ hashing._Memoizer.reset_already_processed()
+ assert hashing.get_json(np.zeros(4, dtype="V8")) != empty_padded_json
+
+ bytes_title = np.zeros(4, dtype=np.dtype([((b"title", "value"), "<i4")]))
+ scalar_title = np.zeros(
+ 4,
+ dtype=np.dtype([((np.int64(7), "value"), "<i4")]),
+ )
+ bytes_title["value"] = scalar_title["value"] = np.arange(4)
+ hashing._Memoizer.reset_already_processed()
+ bytes_title_json = hashing.get_json(bytes_title)
+ hashing._Memoizer.reset_already_processed()
+ assert bytes_title_json != hashing.get_json(scalar_title)
+
+
+def test_play_hash_includes_mobject_pixels_but_not_camera_pixels():
+ class HashableObject:
+ def __init__(self, name: str, pixels: np.ndarray) -> None:
+ self.name = name
+ self.pixel_array = pixels
+
+ def __str__(self) -> str:
+ return self.name
+
+ scene = HashableObject("scene", np.arange(8, dtype=np.uint8))
+ camera = HashableObject("camera", np.arange(8, dtype=np.uint8))
+ mobject = ImageMobject(np.zeros((8, 8, 4), dtype=np.uint8))
+
+ original = hashing.get_hash_from_play_call(scene, camera, [], [mobject])
+ mobject.pixel_array[4, 4, 0] ^= 1
+ assert hashing.get_hash_from_play_call(scene, camera, [], [mobject]) != original
+
+ mobject.pixel_array[4, 4, 0] ^= 1
+ camera.pixel_array[0] ^= 1
+ assert hashing.get_hash_from_play_call(scene, camera, [], [mobject]) == original
+
+
+def test_play_hash_keeps_distinct_mobjects_with_equal_python_hashes():
+ class CollidingObject:
+ def __init__(self, name: str) -> None:
+ self.name = name
+
+ def __str__(self) -> str:
+ return self.name
+
+ def __hash__(self) -> int:
+ return 1
+
+ scene = CollidingObject("scene")
+ camera = CollidingObject("camera")
+ mobjects = [CollidingObject("first"), CollidingObject("second")]
+
+ original = hashing.get_hash_from_play_call(scene, camera, [], mobjects)
+ mobjects[1].name = "changed"
+ assert hashing.get_hash_from_play_call(scene, camera, [], mobjects) != original
def test_JSON_with_tuple():