π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import annotations
import pytest
import torch
import kornia.augmentation as K
from kornia.constants import Resample
from kornia.geometry.transform import get_perspective_transform
from testing.base import BaseTester
class TestResizeMatrix(BaseTester):
@staticmethod
def _ramp(height, width, device, dtype):
y, x = torch.meshgrid(
torch.arange(height, device=device, dtype=dtype),
torch.arange(width, device=device, dtype=dtype),
indexing="ij",
)
return torch.stack((x, y)).unsqueeze(0).repeat(2, 1, 1, 1)
@staticmethod
def _augmentation(kind, size, align_corners, device, dtype):
if kind == "resize":
aug = K.Resize(size, align_corners=align_corners)
elif kind == "longest":
aug = K.LongestMaxSize(size[1], align_corners=align_corners)
elif kind == "smallest":
aug = K.SmallestMaxSize(size[0], align_corners=align_corners)
else:
aug = K.RandomResizedCrop(size, align_corners=align_corners, cropping_mode="slice")
# The True path solves in the parameter dtype; use double for a tight double control.
aug.set_rng_device_and_dtype(device, torch.float64 if dtype == torch.float64 else torch.float32)
return aug
@pytest.mark.parametrize("kind", ["resize", "longest", "smallest", "crop"])
@pytest.mark.parametrize("size", [(10, 14), (3, 4), (5, 7), (5, 11)])
@pytest.mark.parametrize("align_corners", [False, True])
def test_matrix_predicts_sampled_coordinates_4804(self, device, dtype, kind, size, align_corners):
image = self._ramp(9, 12, device, dtype) if kind == "crop" else self._ramp(5, 7, device, dtype)
aug = self._augmentation(kind, size, align_corners, device, dtype)
params = aug.forward_parameters(image.shape)
if kind == "crop":
# Distinct origins and crop sizes expose translation signs and per-row scaling.
params["src"] = image.new_tensor([[[1, 2], [7, 2], [7, 6], [1, 6]], [[3, 1], [10, 1], [10, 6], [3, 6]]])
out = aug(image, params=params)
assert aug._transform_matrix is None # construction remains lazy
matrix = aug.transform_matrix
assert matrix.dtype == dtype
assert matrix.device == image.device
assert matrix.shape == (2, 3, 3)
assert aug.transform_matrix is matrix
src = params["src"].to(image)
origin = src[:, 0]
extent = src[:, 2] - origin
h, w = out.shape[-2:]
lengths = image.new_tensor([w, h])
scales = (lengths - 1) / extent if align_corners else lengths / (extent + 1)
offsets = -origin * scales
if not align_corners:
offsets = offsets + (scales - 1) / 2
expected = torch.eye(3, device=device, dtype=dtype).repeat(2, 1, 1)
expected[:, 0, 0], expected[:, 1, 1] = scales.unbind(-1)
expected[:, :2, 2] = offsets
tolerance = {"atol": 1e-12, "rtol": 1e-12} if dtype == torch.float64 else {}
self.assert_close(matrix, expected, **tolerance)
# Two ramps report the source x and y actually sampled. Invert the recorded
# matrix, then account for interpolate's clamping to the crop boundary.
grid = self._ramp(h, w, device, dtype).flatten(2)
homogeneous = torch.cat((grid, torch.ones_like(grid[:, :1])), dim=1)
work_dtype = torch.float64 if dtype == torch.float64 else torch.float32
predicted = (torch.linalg.inv(matrix.to(work_dtype)) @ homogeneous.to(work_dtype))[:, :2]
predicted = predicted.maximum(origin.unsqueeze(-1)).minimum((origin + extent).unsqueeze(-1))
self.assert_close(out.flatten(2), predicted.to(dtype), **tolerance)
@pytest.mark.parametrize("kind", ["resize", "crop"])
@pytest.mark.parametrize("shape,size", [((1, 7), (3, 4)), ((5, 1), (2, 3)), ((5, 7), (1, 1)), ((1, 1), (1, 1))])
def test_singleton_half_pixel_matrix_4804(self, device, dtype, kind, shape, size):
image = self._ramp(*shape, device, dtype)
aug = self._augmentation(kind, size, False, device, dtype)
params = aug.forward_parameters(image.shape)
h, w = shape
params["src"] = image.new_tensor([[[0, 0], [w - 1, 0], [w - 1, h - 1], [0, h - 1]]]).repeat(2, 1, 1)
out = aug(image, params=params)
sx, sy = size[1] / w, size[0] / h
expected = image.new_tensor([[sx, 0, (sx - 1) / 2], [0, sy, (sy - 1) / 2], [0, 0, 1]])
self.assert_close(aug.transform_matrix, expected.expand(2, -1, -1))
self.assert_close(out, torch.nn.functional.interpolate(image, size, mode="bilinear", align_corners=False))
@pytest.mark.parametrize("kind", ["resize", "longest", "smallest", "crop"])
def test_sequential_annotations_follow_pixels_4804(self, device, dtype, kind):
image = self._ramp(8, 12, device, dtype)
aug = self._augmentation(kind, (4, 6), False, device, dtype)
seq = K.AugmentationSequential(aug, data_keys=["input", "keypoints", "bbox"])
params = seq.forward_parameters(image.shape)
if kind == "crop":
params[0].data["src"] = image.new_tensor(
[[[2, 1], [9, 1], [9, 6], [2, 6]], [[1, 2], [8, 2], [8, 7], [1, 7]]]
)
# Select interior output pixels and derive their source centres analytically.
src = params[0].data["src"].to(image)
origin, extent = src[:, 0], src[:, 2] - src[:, 0] + 1
target = image.new_tensor([[[1, 1], [4, 1], [4, 2], [1, 2]]]).repeat(2, 1, 1)
points = (target + 0.5) * (extent / image.new_tensor([6, 4])).unsqueeze(1) - 0.5 + origin.unsqueeze(1)
output, mapped, boxes = seq(image, points, points.unsqueeze(1), params=params)
self.assert_close(mapped, target)
self.assert_close(boxes, target.unsqueeze(1))
self.assert_close(output[:, :, [1, 1, 2, 2], [1, 4, 4, 1]].transpose(1, 2), points)
@pytest.mark.parametrize("align_corners", [False, True])
@pytest.mark.parametrize("kind", ["resize", "crop"])
def test_bicubic_matrix_and_image_4804(self, device, dtype, kind, align_corners):
image = self._ramp(5, 7, device, dtype)
aug = self._augmentation(kind, (8, 11), align_corners, device, dtype)
params = aug.forward_parameters(image.shape)
params["src"] = params["src"].new_tensor([[[0, 0], [6, 0], [6, 4], [0, 4]]]).repeat(2, 1, 1)
aug(image, params=params)
bilinear_matrix = aug.transform_matrix.clone()
out = aug(image, params=params, resample=Resample.BICUBIC)
self.assert_close(aug.transform_matrix, bilinear_matrix, atol=0, rtol=0)
expected = torch.nn.functional.interpolate(image, (8, 11), mode="bicubic", align_corners=align_corners)
self.assert_close(out, expected)
@pytest.mark.parametrize("align_corners", [False, True])
def test_resample_keeps_corner_geometry_4804(self, device, dtype, align_corners):
image = self._ramp(8, 12, device, dtype)
aug = K.RandomResizedCrop((4, 6), cropping_mode="resample", align_corners=align_corners)
params = aug.forward_parameters(image.shape)
output = aug(image, params=params)
expected = get_perspective_transform(params["src"].to(image), params["dst"].to(image))
self.assert_close(aug.transform_matrix, expected, atol=0, rtol=0)
# Slice/resample agree for corner alignment; False intentionally uses distinct grids.
if align_corners and dtype in (torch.float32, torch.float64):
# Half-precision warp grids have additional rounding beyond interpolate.
sliced = K.RandomResizedCrop((4, 6), align_corners=True)(image, params=params)
self.assert_close(output, sliced)
@pytest.mark.parametrize("kind", ["resize", "crop"])
@pytest.mark.parametrize("resample,align_corners", [("bilinear", True)])
def test_unchanged_matrix_paths(self, device, dtype, kind, resample, align_corners):
image = self._ramp(5, 7, device, dtype)
aug = self._augmentation(kind, (8, 11), align_corners, device, dtype)
params = aug.forward_parameters(image.shape)
if kind == "crop":
# Random singleton crops make the unchanged corner-aligned solve singular.
params["src"] = params["src"].new_tensor(
[[[1, 1], [5, 1], [5, 4], [1, 4]], [[2, 0], [6, 0], [6, 3], [2, 3]]]
)
aug(image, params=params, resample=Resample.get(resample))
src, dst = aug._params["src"], aug._params["dst"]
if kind == "crop":
src, dst = src.to(image), dst.to(image)
expected = get_perspective_transform(src, dst).to(image)
self.assert_close(aug.transform_matrix, expected, atol=0, rtol=0)
@pytest.mark.parametrize("kind", ["resize", "crop"])
def test_gradcheck(self, device, kind):
image = self._ramp(3, 4, device, torch.float64)[:1]
points = image.new_tensor([[[1.25, 1.5]]])
aug = self._augmentation(kind, (2, 3), False, device, torch.float64)
seq = K.AugmentationSequential(aug, data_keys=["input", "keypoints"])
params = seq.forward_parameters(image.shape)
if kind == "crop":
params[0].data["src"] = image.new_tensor([[[0, 0], [3, 0], [3, 2], [0, 2]]])
self.gradcheck(lambda x, p: tuple(seq(x, p, params=params)), (image, points))
@pytest.mark.parametrize("kind", ["resize", "longest", "smallest"])
@pytest.mark.parametrize("align_corners", [False, True])
def test_inverse_restores_ramp_4804(self, device, dtype, kind, align_corners):
# Bilinear resampling reproduces a ramp, so warping a 2x upscale back with the recorded matrix restores
# the interior exactly. The corner-to-corner matrix at align_corners=False missed by 0.21 px here.
image = self._ramp(9, 13, device, dtype)
aug = self._augmentation(kind, (18, 26), align_corners, device, dtype)
restored = aug.inverse(aug(image))
# Two half-precision warps round by up to one bfloat16 ulp at 12 (0.0625), below the 0.21 px defect.
tolerances = {torch.float64: (1e-12, 1e-12), torch.float16: (0.05, 0), torch.bfloat16: (0.1, 0)}
atol, rtol = tolerances.get(dtype, (1e-4, 1e-4))
self.assert_close(restored[..., 1:-1, 1:-1], image[..., 1:-1, 1:-1], atol=atol, rtol=rtol)