Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(geometry): align a multi-axis matrix batch in transform_points with the points' batch axes (#6476)

sarankumard484f29e8cec
diff --git a/changelog.d/6476.fixed.md b/changelog.d/6476.fixed.md
new file mode 100644
index 000000000..116f8cfe0
--- /dev/null
+++ b/changelog.d/6476.fixed.md
@@ -0,0 +1 @@
+`transform_points` raises a `ValueError` for a transformation batch with several axes, `(*, D+1, D+1)`, that does not line up with the leading batch axes of the points once its size-1 axes are left out. Such a batch used to be paired with the point sets by count, which returned points transformed by the wrong matrices, for example `(2, 2)` matrices with `(2, 4, N, D)` points, or failed inside `bmm`. Batches that line up, such as `(B, 1)` or `(1, B)` with `(B, C, N, D)` points, give the same results as before. Four layouts that main paired correctly only by coincidence now raise too, such as `(1, 2)` matrices with `(1, 1, 2, N, D)` points; give the matrix batch the same axes as the points, `(1, 1, 2)`, instead.
diff --git a/kornia/geometry/linalg.py b/kornia/geometry/linalg.py
index 44eec61d7..6a900ed52 100644
--- a/kornia/geometry/linalg.py
+++ b/kornia/geometry/linalg.py
@@ -220,6 +220,24 @@ def relative_transformation(trans_01: torch.Tensor, trans_02: torch.Tensor) -> t
return trans_12
+def _matrix_batch_pairs_with_points(trans_batch: list[int], points_batch: list[int]) -> bool:
+ """Whether a matrix batch with several axes lines up with the leading axes of the points' batch.
+
+ ``transform_points`` pairs the flattened matrix batch with the leading points' axes in order. That pairing is right
+ when the matrix batch equals those leading axes, or does once its size-1 axes are left out, and wrong otherwise.
+ """
+ stripped = list(trans_batch)
+ while len(stripped) > 0 and stripped[-1] == 1:
+ stripped.pop()
+ if len(stripped) == 0:
+ return True
+ nonunit: list[int] = []
+ for size in trans_batch:
+ if size != 1:
+ nonunit.append(size)
+ return list(points_batch[: len(stripped)]) == stripped or list(points_batch[: len(nonunit)]) == nonunit
+
+
def transform_points(trans_01: torch.Tensor, points_1: torch.Tensor) -> torch.Tensor:
r"""Apply transformations to a set of points.
@@ -231,7 +249,9 @@ def transform_points(trans_01: torch.Tensor, points_1: torch.Tensor) -> torch.Te
Args:
trans_01: tensor for transformations of shape
- ``(D+1, D+1)`` or ``(B, D+1, D+1)``.
+ ``(D+1, D+1)`` or ``(B, D+1, D+1)``. A batch with several axes, ``(*, D+1, D+1)``, pairs with the
+ leading batch axes of ``points_1`` in order once its size-1 axes are left out, so ``(B, 1, D+1, D+1)``
+ and ``(1, B, D+1, D+1)`` both apply matrix ``b`` to ``points_1[b]``; any other multi-axis batch raises.
points_1: tensor of points of shape ``(N, D)`` or ``(B, N, D)``.
Returns:
@@ -275,6 +295,13 @@ def transform_points(trans_01: torch.Tensor, points_1: torch.Tensor) -> torch.Te
if points_1.shape[-2] == 0:
return points_1.to(out_dtype)
+ # A matrix batch with several axes pairs with the points' leading batch axes in order (kornia#6473).
+ if trans_01.ndim > 3 and not _matrix_batch_pairs_with_points(list(trans_01.shape[:-2]), list(points_1.shape[:-2])):
+ raise ValueError(
+ "A matrix batch with several axes must match the leading batch axes of the points, size-1 matrix axes "
+ f"aside. Got {trans_01.shape} and {points_1.shape}"
+ )
+
# We reshape to BxNxD in case we get more dimensions, e.g., MxBxNxD
shape_inp = list(points_1.shape)
points_1 = points_1.reshape(-1, points_1.shape[-2], points_1.shape[-1])
diff --git a/tests/geometry/camera/test_pinhole.py b/tests/geometry/camera/test_pinhole.py
index 21385e113..6e048ed56 100644
--- a/tests/geometry/camera/test_pinhole.py
+++ b/tests/geometry/camera/test_pinhole.py
@@ -1117,7 +1117,8 @@ class TestPinholeCamera(BaseTester):
cam = kornia.geometry.camera.PinholeCamera(k, e, h, w)
assert cam.fx.shape == (1, 2)
self.assert_close(cam.fx, torch.full((1, 2), 100.0, device=device, dtype=dtype))
- with pytest.raises(RuntimeError):
+ # A (1, 2) camera batch does not line up with a (1, 5, 3) point set (#6473).
+ with pytest.raises(ValueError, match="leading batch axes of the points"):
cam.project(torch.ones(1, 5, 3, device=device, dtype=dtype))
def test_rank4_construction_does_not_warn_under_trace_4266(self, device, dtype):
diff --git a/tests/geometry/test_linalg.py b/tests/geometry/test_linalg.py
index 227a11ec0..99d2def14 100644
--- a/tests/geometry/test_linalg.py
+++ b/tests/geometry/test_linalg.py
@@ -110,6 +110,40 @@ class TestTransformPoints(BaseTester):
with pytest.raises(ValueError, match="Input batch size must be the same"):
kgl.transform_points(transform, points)
+ @pytest.mark.parametrize(
+ "trans_batch, points_batch, expected",
+ [
+ ((2, 1), (2, 3), [[0, 0, 0], [1, 1, 1]]), # trailing size-1 axis, as in (B, 1, D+1, D+1) callers
+ ((1, 2), (2, 3), [[0, 0, 0], [1, 1, 1]]), # leading size-1 axis: matrix b for points[b]
+ ((1, 2), (2, 2), [[0, 0], [1, 1]]), # the same with C == B
+ ((2, 3), (2, 3), [[0, 1, 2], [3, 4, 5]]), # matrix batch equal to the points' batch
+ ((2, 1, 3), (2, 3), [[0, 1, 2], [3, 4, 5]]), # a size-1 axis in the middle
+ ((1, 2), (1, 2), [[0, 1]]), # equal, size-1 axis included
+ ],
+ )
+ def test_multi_axis_matrix_batch_pairs_with_leading_point_axes_6473(
+ self, trans_batch, points_batch, expected, device, dtype
+ ):
+ # Matrix k translates x by k, so the x of each transformed point is the flat index of its matrix.
+ trans = torch.eye(4, device=device, dtype=dtype).repeat(*trans_batch, 1, 1)
+ trans.view(-1, 4, 4)[:, 0, 3] = torch.arange(trans.view(-1, 4, 4).shape[0], device=device, dtype=dtype)
+ points = torch.zeros(*points_batch, 5, 3, device=device, dtype=dtype)
+ out = kgl.transform_points(trans, points)
+ assert out.shape == points.shape
+ expected_x = torch.tensor(expected, device=device, dtype=dtype)[..., None].expand(*points_batch, 5)
+ self.assert_close(out[..., 0], expected_x, atol=0, rtol=0)
+
+ @pytest.mark.parametrize(
+ "trans_batch, points_batch",
+ [((2, 2), (2, 4)), ((1, 2), (3, 2)), ((2, 2), (2, 3)), ((2, 2), (2, 1, 2)), ((1, 3), (2, 3))],
+ )
+ def test_multi_axis_matrix_batch_that_does_not_line_up_raises_6473(self, trans_batch, points_batch, device, dtype):
+ # These used to pair matrices with point sets by count and return wrong points, or fail inside bmm.
+ trans = torch.eye(4, device=device, dtype=dtype).repeat(*trans_batch, 1, 1)
+ points = torch.zeros(*points_batch, 1, 3, device=device, dtype=dtype)
+ with pytest.raises(ValueError, match="leading batch axes of the points"):
+ kgl.transform_points(trans, points)
+
@pytest.mark.parametrize("batch_size", [1, 2, 5])
@pytest.mark.parametrize("num_points", [2, 3, 5])
@pytest.mark.parametrize("num_dims", [2, 3])