π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
fix(geometry): align a multi-axis matrix batch in transform_points with the points' batch axes (#6476)
sarankumard484f29e8cec
diff --git a/changelog.d/6476.fixed.md b/changelog.d/6476.fixed.mdnew file mode 100644index 000000000..116f8cfe0--- /dev/null+++ b/changelog.d/6476.fixed.md@@ -0,0 +1 @@+`transform_points` raises a `ValueError` for a transformation batch with several axes, `(*, D+1, D+1)`, that does not line up with the leading batch axes of the points once its size-1 axes are left out. Such a batch used to be paired with the point sets by count, which returned points transformed by the wrong matrices, for example `(2, 2)` matrices with `(2, 4, N, D)` points, or failed inside `bmm`. Batches that line up, such as `(B, 1)` or `(1, B)` with `(B, C, N, D)` points, give the same results as before. Four layouts that main paired correctly only by coincidence now raise too, such as `(1, 2)` matrices with `(1, 1, 2, N, D)` points; give the matrix batch the same axes as the points, `(1, 1, 2)`, instead.diff --git a/kornia/geometry/linalg.py b/kornia/geometry/linalg.pyindex 44eec61d7..6a900ed52 100644--- a/kornia/geometry/linalg.py+++ b/kornia/geometry/linalg.py@@ -220,6 +220,24 @@ def relative_transformation(trans_01: torch.Tensor, trans_02: torch.Tensor) -> treturn trans_12+def _matrix_batch_pairs_with_points(trans_batch: list[int], points_batch: list[int]) -> bool:+ """Whether a matrix batch with several axes lines up with the leading axes of the points' batch.++ ``transform_points`` pairs the flattened matrix batch with the leading points' axes in order. That pairing is right+ when the matrix batch equals those leading axes, or does once its size-1 axes are left out, and wrong otherwise.+ """+ stripped = list(trans_batch)+ while len(stripped) > 0 and stripped[-1] == 1:+ stripped.pop()+ if len(stripped) == 0:+ return True+ nonunit: list[int] = []+ for size in trans_batch:+ if size != 1:+ nonunit.append(size)+ return list(points_batch[: len(stripped)]) == stripped or list(points_batch[: len(nonunit)]) == nonunit++def transform_points(trans_01: torch.Tensor, points_1: torch.Tensor) -> torch.Tensor:r"""Apply transformations to a set of points.@@ -231,7 +249,9 @@ def transform_points(trans_01: torch.Tensor, points_1: torch.Tensor) -> torch.TeArgs:trans_01: tensor for transformations of shape- ``(D+1, D+1)`` or ``(B, D+1, D+1)``.+ ``(D+1, D+1)`` or ``(B, D+1, D+1)``. A batch with several axes, ``(*, D+1, D+1)``, pairs with the+ leading batch axes of ``points_1`` in order once its size-1 axes are left out, so ``(B, 1, D+1, D+1)``+ and ``(1, B, D+1, D+1)`` both apply matrix ``b`` to ``points_1[b]``; any other multi-axis batch raises.points_1: tensor of points of shape ``(N, D)`` or ``(B, N, D)``.Returns:@@ -275,6 +295,13 @@ def transform_points(trans_01: torch.Tensor, points_1: torch.Tensor) -> torch.Teif points_1.shape[-2] == 0:return points_1.to(out_dtype)+ # A matrix batch with several axes pairs with the points' leading batch axes in order (kornia#6473).+ if trans_01.ndim > 3 and not _matrix_batch_pairs_with_points(list(trans_01.shape[:-2]), list(points_1.shape[:-2])):+ raise ValueError(+ "A matrix batch with several axes must match the leading batch axes of the points, size-1 matrix axes "+ f"aside. Got {trans_01.shape} and {points_1.shape}"+ )+# We reshape to BxNxD in case we get more dimensions, e.g., MxBxNxDshape_inp = list(points_1.shape)points_1 = points_1.reshape(-1, points_1.shape[-2], points_1.shape[-1])diff --git a/tests/geometry/camera/test_pinhole.py b/tests/geometry/camera/test_pinhole.pyindex 21385e113..6e048ed56 100644--- a/tests/geometry/camera/test_pinhole.py+++ b/tests/geometry/camera/test_pinhole.py@@ -1117,7 +1117,8 @@ class TestPinholeCamera(BaseTester):cam = kornia.geometry.camera.PinholeCamera(k, e, h, w)assert cam.fx.shape == (1, 2)self.assert_close(cam.fx, torch.full((1, 2), 100.0, device=device, dtype=dtype))- with pytest.raises(RuntimeError):+ # A (1, 2) camera batch does not line up with a (1, 5, 3) point set (#6473).+ with pytest.raises(ValueError, match="leading batch axes of the points"):cam.project(torch.ones(1, 5, 3, device=device, dtype=dtype))def test_rank4_construction_does_not_warn_under_trace_4266(self, device, dtype):diff --git a/tests/geometry/test_linalg.py b/tests/geometry/test_linalg.pyindex 227a11ec0..99d2def14 100644--- a/tests/geometry/test_linalg.py+++ b/tests/geometry/test_linalg.py@@ -110,6 +110,40 @@ class TestTransformPoints(BaseTester):with pytest.raises(ValueError, match="Input batch size must be the same"):kgl.transform_points(transform, points)+ @pytest.mark.parametrize(+ "trans_batch, points_batch, expected",+ [+ ((2, 1), (2, 3), [[0, 0, 0], [1, 1, 1]]), # trailing size-1 axis, as in (B, 1, D+1, D+1) callers+ ((1, 2), (2, 3), [[0, 0, 0], [1, 1, 1]]), # leading size-1 axis: matrix b for points[b]+ ((1, 2), (2, 2), [[0, 0], [1, 1]]), # the same with C == B+ ((2, 3), (2, 3), [[0, 1, 2], [3, 4, 5]]), # matrix batch equal to the points' batch+ ((2, 1, 3), (2, 3), [[0, 1, 2], [3, 4, 5]]), # a size-1 axis in the middle+ ((1, 2), (1, 2), [[0, 1]]), # equal, size-1 axis included+ ],+ )+ def test_multi_axis_matrix_batch_pairs_with_leading_point_axes_6473(+ self, trans_batch, points_batch, expected, device, dtype+ ):+ # Matrix k translates x by k, so the x of each transformed point is the flat index of its matrix.+ trans = torch.eye(4, device=device, dtype=dtype).repeat(*trans_batch, 1, 1)+ trans.view(-1, 4, 4)[:, 0, 3] = torch.arange(trans.view(-1, 4, 4).shape[0], device=device, dtype=dtype)+ points = torch.zeros(*points_batch, 5, 3, device=device, dtype=dtype)+ out = kgl.transform_points(trans, points)+ assert out.shape == points.shape+ expected_x = torch.tensor(expected, device=device, dtype=dtype)[..., None].expand(*points_batch, 5)+ self.assert_close(out[..., 0], expected_x, atol=0, rtol=0)++ @pytest.mark.parametrize(+ "trans_batch, points_batch",+ [((2, 2), (2, 4)), ((1, 2), (3, 2)), ((2, 2), (2, 3)), ((2, 2), (2, 1, 2)), ((1, 3), (2, 3))],+ )+ def test_multi_axis_matrix_batch_that_does_not_line_up_raises_6473(self, trans_batch, points_batch, device, dtype):+ # These used to pair matrices with point sets by count and return wrong points, or fail inside bmm.+ trans = torch.eye(4, device=device, dtype=dtype).repeat(*trans_batch, 1, 1)+ points = torch.zeros(*points_batch, 1, 3, device=device, dtype=dtype)+ with pytest.raises(ValueError, match="leading batch axes of the points"):+ kgl.transform_points(trans, points)+@pytest.mark.parametrize("batch_size", [1, 2, 5])@pytest.mark.parametrize("num_points", [2, 3, 5])@pytest.mark.parametrize("num_dims", [2, 3])