π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
import torch
from torch._dynamo.exc import BackendCompilerFailed, InternalTorchDynamoError, Unsupported, UserError
import kornia.augmentation as K
from kornia.geometry.keypoints import Keypoints, Keypoints3D, VideoKeypoints
from kornia.geometry.linalg import transform_points
from testing.base import BaseTester, supports_bilinear_2d_grid_sample
@pytest.mark.parametrize("container,coordinates", [(Keypoints, 2), (Keypoints3D, 3)])
class TestKeypointsIndexing(BaseTester):
@pytest.mark.parametrize("num_points", [1, 4])
@pytest.mark.parametrize("key_kind", ["first", "last", "scalar_tensor"])
@pytest.mark.parametrize("strided", [False, True])
def test_scalar_unbatched(self, container, coordinates, num_points, key_kind, strided, device, dtype):
data = torch.arange(num_points * coordinates, device=device, dtype=dtype).reshape(num_points, coordinates)
if strided:
data = data.t().contiguous().t()
key = {"first": 0, "last": -1, "scalar_tensor": torch.tensor(-1, device=device)}[key_kind]
keypoints = container(data)
selected = keypoints[key]
assert type(selected) is container
assert selected.shape == (1, coordinates)
assert selected.data.dtype == dtype
assert selected.data.device == device
self.assert_close(selected.to_tensor(), data[key].unsqueeze(0), atol=0, rtol=0)
selected.data.add_(1)
self.assert_close(keypoints.data[key], selected.data[0], atol=0, rtol=0)
keypoints[key] = container(torch.full((1, coordinates), 7.0, device=device, dtype=dtype))
self.assert_close(data[key], torch.full((coordinates,), 7.0, device=device, dtype=dtype))
@pytest.mark.parametrize("batched", [False, True])
def test_existing_selection(self, container, coordinates, batched, device, dtype):
shape = (2, 4, coordinates) if batched else (4, coordinates)
data = torch.arange(8 * coordinates if batched else 4 * coordinates, device=device, dtype=dtype).reshape(shape)
keypoints = container(data)
for key in [
slice(None, 1),
slice(0, 0),
torch.tensor([1, 0], device=device),
torch.arange(data.shape[0], device=device) % 2 == 0,
]:
self.assert_close(keypoints[key].data, data[key], atol=0, rtol=0)
if batched:
self.assert_close(keypoints[0].data, data[0], atol=0, rtol=0)
empty = container(data[:0])
assert empty[:].shape == data[:0].shape
with pytest.raises(IndexError):
empty[0]
@pytest.mark.parametrize("key", [(slice(None), 0), (Ellipsis, 0)])
def test_coordinate_selection_raises(self, container, coordinates, key, device, dtype):
# one coordinate of every point is not a point, even when there are as many points as coordinates
data = torch.arange(coordinates * coordinates, device=device, dtype=dtype).reshape(coordinates, coordinates)
with pytest.raises(ValueError, match="Keypoints shape must be"):
container(data)[key]
def test_gradcheck_scalar(self, container, coordinates, device):
data = torch.arange(3 * coordinates, device=device, dtype=torch.float64).reshape(3, coordinates)
self.gradcheck(lambda x: container(x)[-1].data, (data.requires_grad_(),))
@pytest.mark.parametrize("scalar_tensor", [False, True])
def test_dynamo_scalar(
self, container, coordinates, scalar_tensor, device, dtype, torch_optimizer, optimizer_backend
):
data = torch.arange(4 * coordinates, device=device, dtype=dtype).reshape(4, coordinates)
key = torch.tensor(-1, device=device) if scalar_tensor else -1
def select(x, index):
return container(x)[index].data
expected = data[-1:]
self.assert_close(select(data, key), expected, atol=0, rtol=0)
compiled = torch_optimizer(select, fullgraph=True)
try:
actual = compiled(data, key)
except UserError as exc:
if (
scalar_tensor
and device.type == "cuda"
and optimizer_backend == "inductor"
and "Could not extract specialized integer from data-dependent expression" in str(exc)
):
pytest.xfail("Dynamo cannot specialize a CUDA scalar tensor index, including raw tensor indexing")
raise
except InternalTorchDynamoError as exc:
if scalar_tensor and device.type == "cuda" and "PendingUnbackedSymbolNotFound" in str(exc):
# torch 2.14 and 2.15 raise this for raw ``x[index]`` with a 0-d CUDA index as well.
pytest.xfail("Dynamo cannot trace a CUDA scalar tensor index, including raw tensor indexing")
raise
except Unsupported as exc:
if torch.__version__.startswith("2.5.") and "UserDefinedClassVariable" in str(exc):
pytest.xfail("PyTorch 2.5 cannot compile the keypoint type(self) constructor")
raise
except BackendCompilerFailed as exc:
if scalar_tensor and optimizer_backend == "inductor" and "GuardOnDataDependentSymNode" in str(exc):
pytest.xfail("Inductor cannot specialize a scalar tensor index, including raw tensor indexing")
raise
assert actual.shape == (1, coordinates)
self.assert_close(actual, expected, atol=0, rtol=0)
class TestKeypoints(BaseTester):
def test_smoke(self, device, dtype):
data = torch.rand(10, 2, device=device, dtype=dtype)
kp = Keypoints(data)
assert isinstance(kp, Keypoints)
def test_cardinality(self, device, dtype):
data = torch.rand(10, 2, device=device, dtype=dtype)
kp = Keypoints(data)
assert kp.shape == (10, 2)
def test_batched(self, device, dtype):
data = torch.rand(3, 10, 2, device=device, dtype=dtype)
kp = Keypoints(data)
assert kp.shape == (3, 10, 2)
assert kp._is_batched is True
def test_unbatched(self, device, dtype):
data = torch.rand(10, 2, device=device, dtype=dtype)
kp = Keypoints(data)
assert kp._is_batched is False
def test_device_dtype(self, device, dtype):
data = torch.rand(5, 2, device=device, dtype=dtype)
kp = Keypoints(data)
assert kp.device == device
assert kp.dtype == dtype
def test_from_tensor(self, device, dtype):
data = torch.rand(5, 2, device=device, dtype=dtype)
kp = Keypoints.from_tensor(data)
assert kp.shape == data.shape
def test_to_tensor(self, device, dtype):
data = torch.rand(5, 2, device=device, dtype=dtype)
kp = Keypoints(data)
out = kp.to_tensor()
assert out.shape == data.shape
self.assert_close(out, data)
def test_clone(self, device, dtype):
data = torch.rand(5, 2, device=device, dtype=dtype)
kp = Keypoints(data)
kp2 = kp.clone()
self.assert_close(kp.data, kp2.data)
kp2._data[0, 0] = 999.0
assert not torch.allclose(kp.data, kp2.data)
def test_getitem(self, device, dtype):
data = torch.rand(10, 2, device=device, dtype=dtype)
kp = Keypoints(data)
kp2 = kp[:5]
assert kp2.shape == (5, 2)
def test_setitem(self, device, dtype):
data = torch.rand(10, 2, device=device, dtype=dtype)
kp = Keypoints(data)
new_data = torch.zeros(5, 2, device=device, dtype=dtype)
new_kp = Keypoints(new_data)
kp[:5] = new_kp
self.assert_close(kp.data[:5], new_data)
def test_transform_keypoints(self, device, dtype):
# Use batched keypoints (B, N, 2) with batched M (B, 3, 3)
data = torch.tensor([[[1.0, 0.0], [0.0, 1.0]]], device=device, dtype=dtype) # (1, 2, 2)
kp = Keypoints(data)
M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # (1, 3, 3)
M[0, 0, 2] = 2.0 # translate x by 2
M[0, 1, 2] = 3.0 # translate y by 3
kp_t = kp.transform_keypoints(M)
expected = torch.tensor([[[3.0, 3.0], [2.0, 4.0]]], device=device, dtype=dtype)
self.assert_close(kp_t.data, expected)
def test_transform_keypoints_inplace(self, device, dtype):
data = torch.tensor([[[1.0, 0.0]]], device=device, dtype=dtype) # (1, 1, 2)
kp = Keypoints(data)
M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # (1, 3, 3)
M[0, 0, 2] = 1.0
kp.transform_keypoints_(M)
expected = torch.tensor([[[2.0, 0.0]]], device=device, dtype=dtype)
self.assert_close(kp.data, expected)
def test_transform_keypoints_batched(self, device, dtype):
data = torch.ones(2, 4, 2, device=device, dtype=dtype)
kp = Keypoints(data)
M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0).expand(2, -1, -1).clone()
M[:, 0, 2] = 5.0
kp_t = kp.transform_keypoints(M)
assert kp_t.shape == (2, 4, 2)
self.assert_close(kp_t.data[..., 0], torch.full((2, 4), 6.0, device=device, dtype=dtype))
def test_pad(self, device, dtype):
data = torch.zeros(2, 4, 2, device=device, dtype=dtype)
kp = Keypoints(data)
padding = torch.tensor([[1.0, 0.0, 2.0, 0.0], [0.0, 0.0, 3.0, 0.0]], device=device, dtype=dtype)
kp.pad(padding)
# x += left_pad, y += top_pad
self.assert_close(kp.data[0, :, 0], torch.full((4,), 1.0, device=device, dtype=dtype))
self.assert_close(kp.data[1, :, 0], torch.zeros(4, device=device, dtype=dtype))
self.assert_close(kp.data[0, :, 1], torch.full((4,), 2.0, device=device, dtype=dtype))
def test_unpad(self, device, dtype):
data = torch.ones(2, 4, 2, device=device, dtype=dtype) * 5.0
kp = Keypoints(data)
padding = torch.tensor([[1.0, 0.0, 2.0, 0.0], [0.0, 0.0, 0.0, 0.0]], device=device, dtype=dtype)
kp.unpad(padding)
self.assert_close(kp.data[0, :, 0], torch.full((4,), 4.0, device=device, dtype=dtype))
self.assert_close(kp.data[0, :, 1], torch.full((4,), 3.0, device=device, dtype=dtype))
def test_pad_unpad_unbatched_5021(self, device, dtype):
# kornia#5021: an unbatched (N, 2) container indexes as (N,), so the (1, 1) padding column
# broadcast the in-place update to (1, N) and raised "output with shape [2] doesn't match the
# broadcast shape [1, 2]". The result must match the singleton-batch container.
data = torch.tensor([[8.0, 2.0], [3.0, 5.0]], device=device, dtype=dtype)
padding = torch.tensor([[3.0, 100.0, 7.0, 1000.0]], device=device, dtype=dtype)
padded = Keypoints(data.clone()).pad(padding)
self.assert_close(padded.data, torch.tensor([[11.0, 9.0], [6.0, 12.0]], device=device, dtype=dtype))
self.assert_close(padded.data, Keypoints(data[None].clone()).pad(padding).data[0])
unpadded = Keypoints(data.clone()).unpad(padding)
self.assert_close(unpadded.data, torch.tensor([[5.0, -5.0], [0.0, -2.0]], device=device, dtype=dtype))
self.assert_close(unpadded.data, Keypoints(data[None].clone()).unpad(padding).data[0])
self.assert_close(Keypoints(data.clone()).pad(padding).unpad(padding).data, data)
def test_pad_padding_on_cpu(self, device, dtype):
# kornia#5021: like Boxes.pad, the padding is moved to the keypoints' device, so a CPU
# padding_size works for keypoints on an accelerator.
padding = torch.tensor([[1.0, 0.0, 2.0, 0.0]], dtype=dtype)
for data in (torch.zeros(3, 2, device=device, dtype=dtype), torch.zeros(1, 3, 2, device=device, dtype=dtype)):
kp = Keypoints(data.clone()).pad(padding)
assert kp.device == device
self.assert_close(kp.data[..., 0], torch.full(data.shape[:-1], 1.0, device=device, dtype=dtype))
self.assert_close(kp.data[..., 1], torch.full(data.shape[:-1], 2.0, device=device, dtype=dtype))
def test_index_put(self, device, dtype):
data = torch.zeros(10, 2, device=device, dtype=dtype)
kp = Keypoints(data)
new_vals = torch.ones(3, 2, device=device, dtype=dtype)
idx = (torch.tensor([0, 1, 2], device=device),)
kp2 = kp.index_put(idx, new_vals)
self.assert_close(kp2.data[:3], new_vals)
def test_index_put_inplace(self, device, dtype):
data = torch.zeros(10, 2, device=device, dtype=dtype)
kp = Keypoints(data)
new_vals = torch.ones(3, 2, device=device, dtype=dtype)
idx = (torch.tensor([0, 1, 2], device=device),)
kp.index_put(idx, new_vals, inplace=True)
self.assert_close(kp.data[:3], new_vals)
def test_type(self, device, dtype):
if device.type == "mps":
pytest.skip("MPS does not support float64")
data = torch.rand(5, 2, device=device, dtype=torch.float32)
kp = Keypoints(data)
kp.type(torch.float64)
assert kp.dtype == torch.float64
def test_not_implemented(self, device, dtype):
kp = Keypoints(torch.rand(2, 5, 2, device=device, dtype=dtype))
with pytest.raises(NotImplementedError, match="from a list of tensors is not implemented"):
Keypoints([torch.rand(3, 2, device=device, dtype=dtype), torch.rand(5, 2, device=device, dtype=dtype)])
with pytest.raises(NotImplementedError, match=r"`Keypoints\.to_tensor\(as_padded_sequence=True\)`"):
kp.to_tensor(as_padded_sequence=True)
def test_exception(self, device, dtype):
with pytest.raises(TypeError):
Keypoints("not a tensor")
with pytest.raises(ValueError):
Keypoints(torch.tensor([1, 2, 3], dtype=torch.int32))
with pytest.raises(ValueError):
Keypoints(torch.rand(3, 3, device=device, dtype=dtype))
with pytest.raises(ValueError):
Keypoints(torch.rand(3, 4, 2, 2, device=device, dtype=dtype))
def test_transform_exception(self, device, dtype):
kp = Keypoints(torch.rand(5, 2, device=device, dtype=dtype))
with pytest.raises(ValueError):
kp.transform_keypoints(torch.eye(4, device=device, dtype=dtype))
def test_pad_exception(self, device, dtype):
kp = Keypoints(torch.rand(2, 4, 2, device=device, dtype=dtype))
with pytest.raises(RuntimeError):
kp.pad(torch.zeros(2, 3, device=device, dtype=dtype))
# an unbatched container carries a single image, so it takes exactly one padding row
unbatched = Keypoints(torch.rand(4, 2, device=device, dtype=dtype))
with pytest.raises(RuntimeError, match="one row"):
unbatched.pad(torch.zeros(2, 4, device=device, dtype=dtype))
with pytest.raises(RuntimeError, match="one row"):
unbatched.unpad(torch.zeros(2, 4, device=device, dtype=dtype))
# zero rows as well: a `> 1` check would let it through to an IndexError instead
with pytest.raises(RuntimeError, match="one row"):
unbatched.pad(torch.zeros(0, 4, device=device, dtype=dtype))
def test_int_input_raises_by_default(self, device, dtype):
with pytest.raises(ValueError):
Keypoints(torch.ones(5, 2, device=device, dtype=torch.int32))
def test_int_input_converted_when_not_raising(self, device, dtype):
data = torch.ones(5, 2, device=device, dtype=torch.int32)
kp = Keypoints(data, raise_if_not_floating_point=False)
assert kp.dtype == torch.float32
def test_gradcheck(self, device):
data = torch.rand(1, 5, 2, device=device, dtype=torch.float64, requires_grad=True)
M = torch.eye(3, device=device, dtype=torch.float64).unsqueeze(0)
M[0, 0, 2] = 1.0
def fn(x):
return Keypoints(x).transform_keypoints(M).data
self.gradcheck(fn, (data,))
def test_dynamo(self, device, dtype, torch_optimizer):
data = torch.rand(1, 5, 2, device=device, dtype=dtype)
M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0)
def fn(x):
return Keypoints(x).transform_keypoints(M).data
op = torch_optimizer(fn)
self.assert_close(op(data), fn(data))
def test_smoke_jit(self, device, dtype):
pass # Keypoints is not a nn.Module, jit test not applicable
def test_module(self, device, dtype):
pass # Keypoints is not a nn.Module
class TestVideoKeypoints(BaseTester):
@pytest.mark.parametrize("layout", ["transposed", "expanded", "strided"])
def test_noncontiguous_roundtrip(self, layout, device, dtype):
data = torch.arange(48, device=device, dtype=dtype).reshape(2, 3, 4, 2)
if layout == "transposed":
data = data.transpose(0, 1)
elif layout == "expanded":
data = data[:1].expand(2, -1, -1, -1)
else:
data = data[:, ::2]
assert not data.is_contiguous()
keypoints = VideoKeypoints.from_tensor(data)
self.assert_close(keypoints.data, data.reshape(-1, 4, 2))
self.assert_close(keypoints.to_tensor(), data)
@pytest.mark.parametrize("shape", [(2, 3, 0, 2), (0, 3, 4, 2)])
def test_empty_roundtrip(self, shape, device, dtype):
data = torch.empty(shape, device=device, dtype=dtype)
keypoints = VideoKeypoints.from_tensor(data)
assert keypoints.data.shape == (shape[0] * shape[1], shape[2], 2)
assert keypoints.clone().to_tensor().shape == data.shape
self.assert_close(keypoints.to_tensor(), data)
def test_smoke(self, device, dtype):
data = torch.rand(2, 5, 10, 2, device=device, dtype=dtype)
vkp = VideoKeypoints.from_tensor(data)
assert isinstance(vkp, VideoKeypoints)
def test_cardinality(self, device, dtype):
B, T, N = 2, 5, 10
data = torch.rand(B, T, N, 2, device=device, dtype=dtype)
vkp = VideoKeypoints.from_tensor(data)
assert vkp.temporal_channel_size == T
out = vkp.to_tensor()
assert out.shape == (B, T, N, 2)
def test_to_tensor_roundtrip(self, device, dtype):
data = torch.rand(2, 4, 8, 2, device=device, dtype=dtype)
vkp = VideoKeypoints.from_tensor(data)
out = vkp.to_tensor()
self.assert_close(out, data)
def test_clone(self, device, dtype):
data = torch.rand(2, 4, 8, 2, device=device, dtype=dtype)
vkp = VideoKeypoints.from_tensor(data)
vkp2 = vkp.clone()
self.assert_close(vkp.to_tensor(), vkp2.to_tensor())
assert vkp2.temporal_channel_size == vkp.temporal_channel_size
def test_transform_keypoints(self, device, dtype):
B, T, N = 1, 3, 5
data = torch.ones(B, T, N, 2, device=device, dtype=dtype)
vkp = VideoKeypoints.from_tensor(data)
# After from_tensor, internal shape is (B*T, N, 2); need M with batch size B*T or 1
M = torch.eye(3, device=device, dtype=dtype).unsqueeze(0) # (1, 3, 3) broadcasts
out = vkp.transform_keypoints(M)
assert isinstance(out, VideoKeypoints)
assert out.temporal_channel_size == T
@pytest.mark.parametrize(
"key, videos",
[(0, [0]), (-1, [2]), (slice(0, 2), [0, 1]), (slice(1, None), [1, 2]), ("index", [2, 0]), ("mask", [0, 2])],
)
def test_getitem_selects_videos_6375(self, key, videos, device, dtype):
# As VideoBoxes (#4249): indexing selects whole videos on the video-batch axis and keeps the temporal axis.
# The inherited Keypoints.__getitem__ indexed frames and dropped temporal_channel_size, so to_tensor() raised.
data = torch.arange(3 * 4 * 5 * 2, device=device, dtype=dtype).reshape(3, 4, 5, 2)
vkp = VideoKeypoints.from_tensor(data)
if key == "index":
key = torch.tensor([2, 0], device=device)
elif key == "mask":
key = torch.tensor([True, False, True], device=device)
out = vkp[key]
assert isinstance(out, VideoKeypoints)
assert out.temporal_channel_size == 4
self.assert_close(out.to_tensor(), data[videos])
def test_setitem_writes_videos_6375(self, device, dtype):
# from_tensor shares the storage of a contiguous tensor, so build the expected values before writing.
data = torch.zeros(3, 4, 5, 2, device=device, dtype=dtype)
expected = data.clone()
expected[1] = 1
vkp = VideoKeypoints.from_tensor(data.clone())
vkp[1] = VideoKeypoints.from_tensor(torch.ones(1, 4, 5, 2, device=device, dtype=dtype))
self.assert_close(vkp.to_tensor(), expected)
vkp[torch.tensor([True, False, True], device=device)] = vkp[torch.tensor([1, 1], device=device)]
self.assert_close(vkp.to_tensor(), torch.ones_like(data))
# A value with another number of frames per video is rejected rather than regrouped.
with pytest.raises(ValueError, match="4 frames"):
vkp[0] = VideoKeypoints.from_tensor(torch.ones(2, 2, 5, 2, device=device, dtype=dtype))
def test_exception(self, device, dtype):
with pytest.raises(ValueError):
VideoKeypoints.from_tensor(torch.rand(5, 2, device=device, dtype=dtype))
with pytest.raises(ValueError):
VideoKeypoints.from_tensor(torch.rand(2, 5, 10, 3, device=device, dtype=dtype))
def test_gradcheck(self, device):
data = torch.rand(3, 2, 2, 2, device=device, dtype=torch.float64)
matrix = torch.eye(3, device=device, dtype=torch.float64)[None]
def transform(x):
return VideoKeypoints.from_tensor(x.transpose(0, 1)).transform_keypoints(matrix).to_tensor()
self.gradcheck(transform, (data,))
def test_dynamo(self, device, dtype, torch_optimizer):
data = torch.rand(3, 2, 4, 2, device=device, dtype=dtype)
def roundtrip(x):
return VideoKeypoints.from_tensor(x.transpose(0, 1)).to_tensor()
self.assert_close(torch_optimizer(roundtrip, fullgraph=True)(data), data.transpose(0, 1))
def test_smoke_jit(self, device, dtype):
pass
def test_module(self, device, dtype):
pass
def test_exception_in_base(self, device, dtype):
pass
class TestKeypoints3D(BaseTester):
def test_smoke(self, device, dtype):
data = torch.rand(10, 3, device=device, dtype=dtype)
kp = Keypoints3D(data)
assert isinstance(kp, Keypoints3D)
def test_cardinality(self, device, dtype):
data = torch.rand(10, 3, device=device, dtype=dtype)
kp = Keypoints3D(data)
assert kp.shape == (10, 3)
def test_batched(self, device, dtype):
data = torch.rand(3, 10, 3, device=device, dtype=dtype)
kp = Keypoints3D(data)
assert kp.shape == (3, 10, 3)
assert kp._is_batched is True
def test_unbatched(self, device, dtype):
data = torch.rand(10, 3, device=device, dtype=dtype)
kp = Keypoints3D(data)
assert kp._is_batched is False
def test_from_tensor(self, device, dtype):
data = torch.rand(5, 3, device=device, dtype=dtype)
kp = Keypoints3D.from_tensor(data)
assert kp.shape == data.shape
def test_to_tensor(self, device, dtype):
data = torch.rand(5, 3, device=device, dtype=dtype)
kp = Keypoints3D(data)
out = kp.to_tensor()
self.assert_close(out, data)
def test_clone(self, device, dtype):
data = torch.rand(5, 3, device=device, dtype=dtype)
kp = Keypoints3D(data)
kp2 = kp.clone()
self.assert_close(kp.data, kp2.data)
kp2._data[0, 0] = 999.0
assert not torch.allclose(kp.data, kp2.data)
def test_getitem(self, device, dtype):
data = torch.rand(10, 3, device=device, dtype=dtype)
kp = Keypoints3D(data)
kp2 = kp[:5]
assert kp2.shape == (5, 3)
def test_setitem(self, device, dtype):
data = torch.rand(10, 3, device=device, dtype=dtype)
kp = Keypoints3D(data)
new_data = torch.zeros(5, 3, device=device, dtype=dtype)
new_kp = Keypoints3D(new_data)
kp[:5] = new_kp
self.assert_close(kp.data[:5], new_data)
@pytest.mark.parametrize(
"data_shape, matrix_shape",
[((2, 5, 3), (2, 4, 4)), ((2, 5, 3), (4, 4)), ((5, 3), (4, 4)), ((5, 3), (1, 4, 4))],
)
def test_transform_keypoints(self, data_shape, matrix_shape, device, dtype):
data = torch.rand(data_shape).to(device, dtype)
M = torch.rand(matrix_shape).to(device, dtype)
kp = Keypoints3D(data)
kp_t = kp.transform_keypoints(M)
assert kp_t is not kp
assert kp_t.data.device == device
assert kp_t.data.dtype == dtype
self.assert_close(kp_t.data, transform_points(M, data), rtol=0, atol=0)
def test_transform_keypoints_inplace(self, device, dtype):
data = torch.rand(2, 5, 3).to(device, dtype)
original = data.clone()
M = torch.rand(2, 4, 4).to(device, dtype)
expected = transform_points(M, data)
kp = Keypoints3D(data)
assert kp.transform_keypoints(M, inplace=True) is kp
self.assert_close(kp.data, expected, rtol=0, atol=0)
kp = Keypoints3D(data)
assert kp.transform_keypoints_(M) is kp
self.assert_close(kp.data, expected, rtol=0, atol=0)
self.assert_close(data, original, rtol=0, atol=0)
@pytest.mark.parametrize("method", ["pad", "unpad"])
def test_pad_unpad_deprecated(self, method, device, dtype):
kp = Keypoints3D(torch.rand(5, 3, device=device, dtype=dtype))
with (
pytest.warns(DeprecationWarning, match=rf"`Keypoints3D\.{method}` is deprecated") as record,
pytest.raises(NotImplementedError, match=rf"`Keypoints3D\.{method}` is not implemented"),
):
getattr(kp, method)(torch.zeros(1, 6, device=device, dtype=dtype))
assert len(record) == 1
assert record[0].filename == __file__
def test_not_implemented(self, device, dtype):
kp = Keypoints3D(torch.rand(5, 3, device=device, dtype=dtype))
with pytest.raises(NotImplementedError, match=r"`Keypoints3D\.pad` is not implemented"):
kp.pad(torch.zeros(1, 6, device=device, dtype=dtype))
with pytest.raises(NotImplementedError, match=r"`Keypoints3D\.unpad` is not implemented"):
kp.unpad(torch.zeros(1, 6, device=device, dtype=dtype))
with pytest.raises(NotImplementedError, match="from a list of tensors is not implemented"):
Keypoints3D([torch.rand(3, 3, device=device, dtype=dtype), torch.rand(5, 3, device=device, dtype=dtype)])
with pytest.raises(NotImplementedError, match=r"`Keypoints3D\.to_tensor\(as_padded_sequence=True\)`"):
kp.to_tensor(as_padded_sequence=True)
def test_exception(self, device, dtype):
with pytest.raises(TypeError):
Keypoints3D("not a tensor")
with pytest.raises(ValueError):
Keypoints3D(torch.tensor([1, 2, 3], dtype=torch.int32))
with pytest.raises(ValueError):
Keypoints3D(torch.rand(3, 2, device=device, dtype=dtype))
def test_transform_exception(self, device, dtype):
kp = Keypoints3D(torch.rand(5, 3, device=device, dtype=dtype))
with pytest.raises(ValueError, match=r"must be \(4, 4\) or \(B, 4, 4\)"):
kp.transform_keypoints(torch.eye(3, device=device, dtype=dtype))
def test_int_input_converted_when_not_raising(self, device, dtype):
data = torch.ones(5, 3, device=device, dtype=torch.int32)
kp = Keypoints3D(data, raise_if_not_floating_point=False)
assert kp.data.dtype == torch.float32
def test_gradcheck(self, device):
data = torch.rand(1, 5, 3, device=device, dtype=torch.float64, requires_grad=True)
M = torch.eye(4, device=device, dtype=torch.float64).unsqueeze(0)
M[0, 0, 3] = 1.0
def fn(x):
return Keypoints3D(x).transform_keypoints(M).data
self.gradcheck(fn, (data,))
def test_dynamo(self, device, dtype, torch_optimizer):
data = torch.rand(1, 5, 3, device=device, dtype=dtype)
M = torch.eye(4, device=device, dtype=dtype).unsqueeze(0)
def fn(x):
return Keypoints3D(x).transform_keypoints(M).data
op = torch_optimizer(fn)
self.assert_close(op(data), fn(data))
def test_smoke_jit(self, device, dtype):
pass
def test_module(self, device, dtype):
pass
@pytest.mark.usefixtures("restore_torch_rng")
class TestConventionsKeypoints(BaseTester):
"""Pins for the coordinate, transform and aliasing conventions of :class:`Keypoints`."""
@staticmethod
def _value_at(image, xy):
# The image value at the pixel nearest to an (x, y) keypoint, or None when that pixel is outside the image.
x, y = (int(v) for v in xy.round().tolist())
if 0 <= x < image.shape[-1] and 0 <= y < image.shape[-2]:
return float(image[0, 0, y, x])
return None
@pytest.mark.parametrize(
"make_augmentation",
[
pytest.param(lambda: K.RandomAffine(degrees=(90.0, 90.0), p=1.0), id="rotate90"),
pytest.param(lambda: K.RandomAffine(degrees=0.0, translate=(0.2, 0.0), p=1.0), id="translate_x"),
],
)
def test_convention_keypoints_are_xy_pixel_coordinates(self, make_augmentation, device, dtype):
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip(f"this torch build has no bilinear 2D grid_sample kernel for {dtype} on {device.type}")
# A keypoint is (x, y) in pixels: x indexes the columns (W) and y the rows (H). Image-content oracle: the
# single bright pixel at row 2, column 8 of a 7 x 11 image is the keypoint (8, 2), and wherever the
# augmentation moves that pixel, the transformed keypoint lands on it.
height, width, row, col = 7, 11, 2, 8
image = torch.zeros(1, 1, height, width, device=device, dtype=dtype)
image[0, 0, row, col] = 1.0
keypoints = torch.tensor([[[col, row]]], device=device, dtype=dtype)
torch.manual_seed(0)
augmentation = K.AugmentationSequential(make_augmentation(), data_keys=["input", "keypoints"])
out_image, out_keypoints = augmentation(image, keypoints)
landed = tuple(int(v) for v in out_keypoints[0, 0].round().tolist())
brightest = int(out_image[0, 0].flatten().argmax())
# rotate90 moves the pixel to (6, 6) and the seeded x-translation by +1.18 px to (9, 2)
assert (brightest % width, brightest // width) == landed
assert landed != (col, row)
value = self._value_at(out_image, out_keypoints[0, 0])
assert value is not None
assert value > 0.5
# Relabel control: the same pixel read as (row, col) = (2, 8) leaves the image or lands on a dark pixel
# ((0, 0) after rotate90, row 8 of a 7-row image after the translation).
torch.manual_seed(0)
augmentation = K.AugmentationSequential(make_augmentation(), data_keys=["input", "keypoints"])
_, out_relabelled = augmentation(image, keypoints.flip(-1))
value = self._value_at(out_image, out_relabelled[0, 0])
assert value is None or value < 0.5
def test_convention_keypoints_transform_is_column_vector_then_divided(self, device, dtype):
# transform_keypoints maps a point p to M @ [x, y, 1]^T (a column vector, M on the left) and divides by the
# third component. M != M^T and the projective row is non-zero, so for the first point the row-vector
# reading [x, y, 1] @ M of the affine matrix ([0.2452, -0.0065]) and the undivided projective result
# ([6.2, 3.2]) are far from the literals.
points = torch.tensor([[8.0, 2.0], [3.0, 5.0]], device=device, dtype=dtype)
affine = torch.tensor([[0.9, -0.3, 4.0], [0.2, 1.1, -1.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype)
projective = torch.tensor([[1.0, 0.1, -2.0], [0.05, 0.9, 1.0], [0.01, 0.02, 1.0]], device=device, dtype=dtype)
expected_affine = torch.tensor([[10.6, 2.8], [5.2, 5.1]], device=device, dtype=dtype)
self.assert_close(Keypoints(points.clone()).transform_keypoints(affine).data, expected_affine)
# M @ [x, y, 1]^T = (6.2, 3.2, 1.12) and (1.5, 5.65, 1.13)
expected_projective = torch.tensor(
[[6.2 / 1.12, 3.2 / 1.12], [1.5 / 1.13, 5.65 / 1.13]], device=device, dtype=dtype
)
self.assert_close(Keypoints(points.clone()).transform_keypoints(projective).data, expected_projective)
batched = Keypoints(points[None].clone()).transform_keypoints(projective[None])
self.assert_close(batched.data, expected_projective[None])
def test_convention_keypoints_transform_inplace_rebinds_copy_does_not_alias(self, device, dtype):
# inplace=False returns a new Keypoints on new storage, so writing into the result reaches neither the
# original container nor the caller's tensor. inplace=True, and transform_keypoints_, return self after
# rebinding its data to the transformed tensor, so the caller's tensor keeps the old coordinates.
original = torch.tensor([[[8.0, 2.0], [3.0, 5.0]]], device=device, dtype=dtype)
affine = torch.tensor([[0.9, -0.3, 4.0], [0.2, 1.1, -1.0], [0.0, 0.0, 1.0]], device=device, dtype=dtype)
expected = torch.tensor([[[10.6, 2.8], [5.2, 5.1]]], device=device, dtype=dtype)
caller = original.clone()
kp = Keypoints(caller)
out = kp.transform_keypoints(affine)
assert out is not kp
self.assert_close(out.data, expected)
out.data.fill_(999.0)
assert kp.data is caller
self.assert_close(caller, original)
caller = original.clone()
kp = Keypoints(caller)
assert kp.transform_keypoints(affine, inplace=True) is kp
self.assert_close(kp.data, expected)
self.assert_close(caller, original)
caller = original.clone()
kp = Keypoints(caller)
assert kp.transform_keypoints_(affine) is kp
self.assert_close(kp.data, expected)
self.assert_close(caller, original)
@pytest.mark.parametrize("layout", ["batched", "unbatched", "strided"])
def test_convention_keypoints_wrap_without_copy_and_edits_write_through(self, layout, device, dtype):
# The constructor wraps the caller's tensor without copying it, a non-contiguous view included, and pad /
# unpad shift that tensor in place and return self. padding_size rows are (left, right, top, bottom):
# x += left and y += top. The four distinct padding values make a slot mix-up visible. clone() gives
# independent storage. Item assignment and index_put(inplace=True) also write into the caller's tensor.
original = torch.tensor([[8.0, 2.0], [3.0, 5.0]], device=device, dtype=dtype)
padded = torch.tensor([[11.0, 9.0], [6.0, 12.0]], device=device, dtype=dtype)
padding = torch.tensor([[3.0, 100.0, 7.0, 1000.0]], device=device, dtype=dtype)
if layout == "batched":
caller = original[None].clone()
elif layout == "unbatched":
caller = original.clone()
else:
caller = original.t().contiguous().t()
assert not caller.is_contiguous()
kp = Keypoints(caller)
assert kp.pad(padding) is kp
self.assert_close(caller.reshape(2, 2), padded)
assert kp.unpad(padding) is kp
self.assert_close(caller.reshape(2, 2), original)
independent = kp.clone()
independent.pad(padding)
self.assert_close(independent.data.reshape(2, 2), padded)
self.assert_close(caller.reshape(2, 2), original)
second_point = (torch.tensor([1], device=device),)
if layout == "batched":
second_point = (torch.tensor([0], device=device), *second_point)
kp.index_put(second_point, torch.tensor([100.0, 200.0], device=device, dtype=dtype), inplace=True)
kp[..., :1, :] = Keypoints(torch.tensor([[70.0, 60.0]], device=device, dtype=dtype))
edited = torch.tensor([[70.0, 60.0], [100.0, 200.0]], device=device, dtype=dtype)
self.assert_close(caller.reshape(2, 2), edited)
def test_convention_keypoints_single_padding_row_broadcasts_batch(self, device, dtype):
caller = torch.tensor([[[8.0, 2.0], [3.0, 5.0]], [[1.0, 4.0], [6.0, 7.0]]], device=device, dtype=dtype)
original = caller.clone()
padding = torch.tensor([[3.0, 100.0, 7.0, 1000.0]], device=device, dtype=dtype)
keypoints = Keypoints(caller)
assert keypoints.pad(padding) is keypoints
expected = torch.tensor([[[11.0, 9.0], [6.0, 12.0]], [[4.0, 11.0], [9.0, 14.0]]], device=device, dtype=dtype)
self.assert_close(caller, expected)
assert keypoints.unpad(padding) is keypoints
self.assert_close(caller, original)
@pytest.mark.parametrize(
"path",
[
"keypoints_list_input",
"keypoints_from_tensor_list",
"keypoints_to_tensor_padded_sequence",
"keypoints3d_list_input",
"keypoints3d_from_tensor_list",
"keypoints3d_to_tensor_padded_sequence",
"keypoints3d_pad",
"keypoints3d_unpad",
],
)
def test_wart_keypoints_documented_paths_not_implemented_5023(self, path, device, dtype):
# Wart pin (#5023): the API offers list input (the runtime list branch), to_tensor(as_padded_sequence=True)
# and the deprecated Keypoints3D pad / unpad methods, and every one of these paths raises
# NotImplementedError. Implementing a path, or removing it from the API, flips its case. The message is
# not asserted, so rewording it leaves every case green.
kp2d = torch.tensor([[[8.0, 2.0], [3.0, 5.0]]], device=device, dtype=dtype)
kp3d = torch.tensor([[[8.0, 2.0, 4.0], [3.0, 5.0, 1.0]]], device=device, dtype=dtype)
calls = {
"keypoints_list_input": lambda: Keypoints([kp2d[0], kp2d[0, :1]]),
"keypoints_from_tensor_list": lambda: Keypoints.from_tensor([kp2d[0], kp2d[0, :1]]),
"keypoints_to_tensor_padded_sequence": lambda: Keypoints(kp2d).to_tensor(as_padded_sequence=True),
"keypoints3d_list_input": lambda: Keypoints3D([kp3d[0], kp3d[0, :1]]),
"keypoints3d_from_tensor_list": lambda: Keypoints3D.from_tensor([kp3d[0], kp3d[0, :1]]),
"keypoints3d_to_tensor_padded_sequence": lambda: Keypoints3D(kp3d).to_tensor(as_padded_sequence=True),
"keypoints3d_pad": lambda: Keypoints3D(kp3d).pad(torch.zeros(1, 6, device=device, dtype=dtype)),
"keypoints3d_unpad": lambda: Keypoints3D(kp3d).unpad(torch.zeros(1, 6, device=device, dtype=dtype)),
}
with pytest.raises(NotImplementedError):
calls[path]()