Skip to content

armgit / manim

Public Imported from GitHub
0

A community-maintained Python framework for creating mathematical animations.

Add key function parameter for unhashables to `list_update` and `list_difference_update` (#4975)

nikolajmunk22aa85f47c9d
diff --git a/manim/utils/iterables.py b/manim/utils/iterables.py
index 66708353a..cdebd1f12 100644
--- a/manim/utils/iterables.py
+++ b/manim/utils/iterables.py
@@ -35,7 +35,8 @@ import numpy as np
T = TypeVar("T")
U = TypeVar("U")
F = TypeVar("F", np.float64, np.int_)
-H = TypeVar("H", bound=Hashable)
+H1 = TypeVar("H1", bound=Hashable)
+H2 = TypeVar("H2", bound=Hashable)
if TYPE_CHECKING:
@@ -133,24 +134,91 @@ def concatenate_lists(*list_of_lists: Iterable[T]) -> list[T]:
return [item for lst in list_of_lists for item in lst]
-def list_difference_update(l1: Iterable[T], l2: Iterable[U]) -> list[T]:
- """Returns a list containing all the elements of l1 not in l2.
+@overload
+def list_difference_update(
+ l1: Iterable[H1], l2: Iterable[H2], *, key: None = None
+) -> list[H1]: ...
+@overload
+def list_difference_update(
+ l1: Iterable[T],
+ l2: Iterable[U],
+ *,
+ key: Callable[[T | U], Hashable],
+) -> list[T]: ...
+def list_difference_update(
+ l1: Iterable[T],
+ l2: Iterable[U],
+ *,
+ key: Callable[[T | U], Hashable] | None = None,
+) -> list[T]:
+ """Returns a list containing all the elements of ``l1`` which are not present
+ in ``l2``.
+
+ An optional key function can be provided to specify which elements
+ should be considered equal.
+
+ Parameters
+ ----------
+ l1
+ The first iterable.
+ l2
+ The second iterable.
+ key
+ A key function which provides a value used to determine element equality. The
+ default value of ``None`` means that regular hashable equality will be used.
Examples
--------
.. code-block:: pycon
- >>> list_difference_update([1, 2, 3, 4], [2, 4])
- [1, 3]
+ >>> list_difference_update([1, 2, 3, 4, 1], [2, 4])
+ [1, 3, 1]
+ >>> list_difference_update(["a", "b", "A", "C"], ["A", "D"], key=str.lower)
+ ['b', 'C']
"""
- l2 = set(l2)
- return [e for e in l1 if e not in l2]
+ if key is None:
+ if not isinstance(l2, (set, dict, frozenset)):
+ # l2 is not a set-like object, so convert it to a set for faster lookups
+ l2 = set(l2)
+ return [e for e in l1 if e not in l2]
+
+ # Use provided key function to determine uniqueness
+ l2_keys = {key(e) for e in l2}
+ return [e for e in l1 if key(e) not in l2_keys]
-def list_update(l1: Iterable[T], l2: Iterable[U]) -> list[T | U]:
- """Used instead of ``set.update()`` to maintain order,
- making sure duplicates are removed from l1, not l2.
- Removes overlap of l1 and l2 and then concatenates l2 unchanged.
+@overload
+def list_update(
+ l1: Iterable[H1], l2: Iterable[H2], *, key: None = None
+) -> list[H1 | H2]: ...
+@overload
+def list_update(
+ l1: Iterable[T],
+ l2: Iterable[U],
+ *,
+ key: Callable[[T | U], Hashable],
+) -> list[T | U]: ...
+def list_update(
+ l1: Iterable[T],
+ l2: Iterable[U],
+ *,
+ key: Callable[[T | U], Hashable] | None = None,
+) -> list[T | U]:
+ """Returns a new list containing all the elements of ``l1`` which are not present in
+ ``l2``, followed by all the elements of ``l2``. Element order is preserved.
+
+ An optional key function can be provided to specify which elements
+ should be considered equal.
+
+ Parameters
+ ----------
+ l1
+ The first iterable.
+ l2
+ The second iterable.
+ key
+ A key function which provides a value used to determine element equality. The
+ default value of ``None`` means that regular hashable equality will be used.
Examples
--------
@@ -158,9 +226,12 @@ def list_update(l1: Iterable[T], l2: Iterable[U]) -> list[T | U]:
>>> list_update([1, 2, 3], [2, 4, 4])
[1, 3, 2, 4, 4]
+ >>> list_update(["a", "b", "c", "A", "B", "C"], ["A", "b", "D"], key=str.lower)
+ ['c', 'C', 'A', 'b', 'D']
"""
- l2 = list(l2)
- return list_difference_update(l1, l2) + cast(list[T | U], l2)
+ if not isinstance(l2, list):
+ l2 = list(l2)
+ return list_difference_update(l1, l2, key=key) + cast(list[T | U], l2)
@overload
@@ -256,7 +327,7 @@ def make_even_by_cycling(
)
-def remove_list_redundancies(lst: Reversible[H]) -> list[H]:
+def remove_list_redundancies(lst: Reversible[H1]) -> list[H1]:
"""Used instead of ``list(set(l))`` to maintain order.
Keeps the last occurrence of each element.
"""
diff --git a/tests/module/utils/test_iterables.py b/tests/module/utils/test_iterables.py
index 6f52d7441..888533cf5 100644
--- a/tests/module/utils/test_iterables.py
+++ b/tests/module/utils/test_iterables.py
@@ -2,34 +2,209 @@ import pytest
from manim.utils.iterables import list_difference_update, list_update
+ORDERED_ITERABLE_TYPES = [
+ # Common non-list iterables
+ pytest.param(tuple, id="tuple"),
+ pytest.param(lambda values: dict.fromkeys(values).keys(), id="dictkeys"),
+ # One-shot iterables
+ pytest.param(iter, id="iter"),
+ pytest.param(
+ lambda values: (value for value in values),
+ id="generator",
+ ),
+]
-def test_list_difference_update_removes_matching_items():
- assert list_difference_update([1, 2, 3, 4], [2, 4]) == [1, 3]
+STANDARD_LIST_DIFFERENCE_CASES = [
+ ([1, 2, 3, 4], [4, 2], [1, 3]),
+ ([], [1, 2], []),
+ ([1, 2, 3], [], [1, 2, 3]),
+ # Disjoint input
+ ([1, 2, 3], [4, 5], [1, 2, 3]),
+ # Input with same elements
+ ([1, 2, 3], [3, 2, 1], []),
+ # Non-int input with equality semantics
+ (["a", "b", "A", "C", "D"], ["A", "d"], ["a", "b", "C", "D"]),
+ # Input without equality semantics
+ ([ob1 := object(), ob2 := object(), ob3 := object()], [ob1, ob3], [ob2]),
+]
+# Reuse cases, since list_update is equivalent to list_difference_update(l1, l2) + l2
+STANDARD_LIST_UPDATE_CASES = [
+ (l1, l2, expected + l2) for l1, l2, expected in STANDARD_LIST_DIFFERENCE_CASES
+]
-def test_list_difference_update_preserves_duplicates_and_order():
+
[email protected](("l1", "l2", "expected"), STANDARD_LIST_DIFFERENCE_CASES)
+def test_list_difference_update_removes_matching_items(l1, l2, expected):
+ assert list_difference_update(l1, l2) == expected
+
+
[email protected]("l1_type", ORDERED_ITERABLE_TYPES)
[email protected]("l2_type", ORDERED_ITERABLE_TYPES)
+def test_list_difference_update_accepts_iterables(l1_type, l2_type):
+ l1, l2, expected = STANDARD_LIST_DIFFERENCE_CASES[0]
+ assert list_difference_update(l1_type(l1), l2_type(l2)) == expected
+
+
+def test_list_difference_update_preserves_l1_duplicates_and_order():
assert list_difference_update([3, 1, 3, 2, 1], [1]) == [3, 3, 2]
- ("l1", "l2", "expected"),
- [([1, 2, 3, 4], [2, 4], [1, 3]), ([], [1, 2], [])],
-)
-def test_list_difference_update_removes_elements(l1, l2, expected):
- assert list_difference_update(l1, l2) == expected
+def test_list_difference_update_accepts_duplicates_in_l2():
+ assert list_difference_update([1, 2, 3], [2, 4, 4]) == [1, 3]
def test_list_difference_update_preserves_l1_order_and_duplicates():
assert list_difference_update([3, 1, 3, 2, 1], [1]) == [3, 3, 2]
+def test_list_difference_update_with_unordered_l1_removes_matching_items():
+ """Test that matching items are correctly removed, even if there is no order to
+ preserve in l1.
+ """
+ l1 = {(0, 4), (1, 3), (3, "A")}
+ result = list_difference_update(l1, [(5, "B"), (1, 3)])
+ assert sorted(result) == [(0, 4), (3, "A")]
+
+
[email protected](("l1", "l2", "expected"), STANDARD_LIST_DIFFERENCE_CASES)
+def test_list_difference_update_with_key_none_uses_default_equality(l1, l2, expected):
+ assert list_difference_update(l1, l2, key=None) == expected
+
+
[email protected](("l1", "l2", "expected"), STANDARD_LIST_DIFFERENCE_CASES)
+def test_list_difference_update_with_no_key_function_uses_default_equality(
+ l1, l2, expected
+):
+ assert list_difference_update(l1, l2) == expected
+
+
[email protected](("l1", "l2", "expected"), STANDARD_LIST_DIFFERENCE_CASES)
+def test_list_difference_update_with_identity_function_uses_default_equality(
+ l1, l2, expected
+):
+ """Test that it is possible to pass through the element itself and use default
+ equality semantics.
+ """
+ assert list_difference_update(l1, l2, key=lambda x: x) == expected
+
+
+ ("l1", "l2", "key", "expected"),
+ [
+ (["a", "b", "C", "A", "C", "a"], ["A", "D"], str.lower, ["b", "C", "C"]),
+ ([1, 2, 11, 111, 12, 2, 20, 1], [2, 4], lambda x: x % 10, [1, 11, 111, 20, 1]),
+ # Unhashable input
+ ([1, [1], 2, 3, [3], [1]], [[1], 3], str, [1, 2, [3]]),
+ ],
+)
+def test_list_difference_update_with_key_function_matches_unequal_items(
+ l1, l2, key, expected
+):
+ """Test that items in l1 are removed if they match items in l2 according to the key
+ function, even if they are not equal.
+ """
+ assert list_difference_update(l1, l2, key=key) == expected
+
+
[email protected]("l1_type", ORDERED_ITERABLE_TYPES)
[email protected]("l2_type", ORDERED_ITERABLE_TYPES)
+def test_list_difference_update_with_key_function_accepts_iterables(l1_type, l2_type):
+ l1, l2, expected = (["a", "b", "C", "A", "a"], ["A", "D"], ["b", "C"])
+ assert list_difference_update(l1_type(l1), l2_type(l2), key=str.lower) == expected
+
+
@pytest.mark.parametrize(
- ("l1", "l2", "expected"),
- [([1, 2, 3], [2, 4], [1, 3, 2, 4]), ([], [1, 2], [1, 2])],
+ ("l1", "l2", "key", "expected"),
+ [
+ ([1, "1", 1.0, 2, "2.0", "A"], [1, "2"], str, [1.0, "2.0", "A"]),
+ ([1, "1", 1.0, "1.0", 1.5, 2, "3"], [1, "2", "3.0"], float, [1.5]),
+ ],
)
+def test_list_difference_update_with_key_function_matches_different_types(
+ l1, l2, key, expected
+):
+ assert list_difference_update(l1, l2, key=key) == expected
+
+
[email protected](("l1", "l2", "expected"), STANDARD_LIST_UPDATE_CASES)
def test_list_update_removes_overlap_and_appends_l2(l1, l2, expected):
assert list_update(l1, l2) == expected
[email protected]("l1_type", ORDERED_ITERABLE_TYPES)
[email protected]("l2_type", ORDERED_ITERABLE_TYPES)
+def test_list_update_accepts_iterables(l1_type, l2_type):
+ l1, l2, expected = STANDARD_LIST_UPDATE_CASES[0]
+ assert list_update(l1_type(l1), l2_type(l2)) == expected
+
+
+def test_list_update_preserves_l1_duplicates_and_order():
+ assert list_update([3, 1, 3, 2, 1], [1]) == [3, 3, 2, 1]
+
+
def test_list_update_preserves_duplicates_in_l2():
assert list_update([1, 2, 3], [2, 4, 4]) == [1, 3, 2, 4, 4]
+
+
+def test_list_update_with_unordered_l1_removes_matching_items():
+ """Test that matching items are correctly removed, even if there is no order to
+ preserve in l1.
+ """
+ l1 = {(0, 4), (1, 3), (3, "A")}
+ result = list_update(l1, [(5, "B"), (1, 3)])
+ assert sorted(result) == [(0, 4), (1, 3), (3, "A"), (5, "B")]
+ assert result[-2:] == [(5, "B"), (1, 3)] # l2 is appended to the end of the result
+
+
[email protected](("l1", "l2", "expected"), STANDARD_LIST_UPDATE_CASES)
+def test_list_update_with_key_none_uses_default_equality(l1, l2, expected):
+ assert list_update(l1, l2, key=None) == expected
+
+
[email protected](("l1", "l2", "expected"), STANDARD_LIST_UPDATE_CASES)
+def test_list_update_with_identity_function_uses_default_equality(l1, l2, expected):
+ """Test that it is possible to pass through the element itself and use default
+ equality semantics.
+ """
+ assert list_update(l1, l2, key=lambda x: x) == expected
+
+
+ ("l1", "l2", "key", "expected"),
+ [
+ (["a", "b", "A", "C"], ["A", "B", "D"], str.lower, ["C", "A", "B", "D"]),
+ ([1, 2, 11, 111, 12, 2, 1], [2, 4], lambda x: x % 10, [1, 11, 111, 1, 2, 4]),
+ # Unhashable input
+ ([1, [1], 2, 3, [3]], [[1], 3], str, [1, 2, [3], [1], 3]),
+ ],
+)
+def test_list_update_with_key_function_matches_unequal_items(l1, l2, key, expected):
+ """Test that items in l1 are removed if they match items in l2 according to the key
+ function, even if they are not equal.
+ """
+ assert list_update(l1, l2, key=key) == expected
+
+
[email protected]("l1_type", ORDERED_ITERABLE_TYPES)
[email protected]("l2_type", ORDERED_ITERABLE_TYPES)
+def test_list_update_with_key_function_accepts_iterables(l1_type, l2_type):
+ """Test that items in l1 are removed if they match items in l2 according to the key
+ function, even if they are not equal.
+ """
+ l1, l2, expected = (["a", "b", "C", "A", "a"], ["A", "D"], ["b", "C", "A", "D"])
+ assert list_update(l1_type(l1), l2_type(l2), key=str.lower) == expected
+
+
+ ("l1", "l2", "key", "expected"),
+ [
+ ([1, "1", 1.0, 2, "2.0", "A"], [1, "2"], str, [1.0, "2.0", "A", 1, "2"]),
+ (["1", 1.0, "1.0", 1.5, 2, "3"], [1, "2", "3.0"], float, [1.5, 1, "2", "3.0"]),
+ ],
+)
+def test_list_update_with_key_function_matches_different_types(l1, l2, key, expected):
+ """Test that items in l1 are removed if they match items in l2 according to the key
+ function, even if they are not equal.
+ """
+ assert list_update(l1, l2, key=key) == expected