π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
import torch
from kornia.geometry.transform import elastic_transform2d
from testing.base import BaseTester, supports_2d_border_padding
def _row_spread_std(out: torch.Tensor, reference: torch.Tensor) -> torch.Tensor:
# weighted standard deviation (over interior rows) of the displaced energy,
# i.e. how far the affected region spreads vertically.
energy = (out[0, 0, 1:-1] - reference[0, 0, 1:-1]).abs().sum(dim=1)
rows = torch.arange(energy.shape[0], dtype=energy.dtype, device=energy.device)
weights = energy / energy.sum()
mean = (weights * rows).sum()
return (weights * (rows - mean) ** 2).sum().sqrt()
class TestElasticTransform(BaseTester):
@pytest.mark.parametrize("padding_mode", ["zeros", "border", "reflection"])
@pytest.mark.parametrize("align_corners", [False, True])
@pytest.mark.parametrize("mode", ["bilinear", "nearest"])
@pytest.mark.parametrize("axis", [0, 1])
@pytest.mark.parametrize("direction", [-1, 1])
def test_displacement_outside_image(self, device, dtype, padding_mode, align_corners, mode, axis, direction):
if padding_mode == "border" and not supports_2d_border_padding(device):
pytest.skip("MPS 2D grid_sample lacks 'border' padding")
image = torch.arange(1, 6, device=device, dtype=dtype).view(1, 1, 1, 5).expand(2, 2, 5, 5).clone()
rows = {
"zeros": [3.0, 4.0, 5.0, 0.0, 0.0],
"border": [3.0, 4.0, 5.0, 5.0, 5.0],
"reflection": [3.0, 4.0, 5.0, 4.0, 3.0] if align_corners else [3.0, 4.0, 5.0, 5.0, 4.0],
}
expected = torch.tensor(rows[padding_mode], device=device, dtype=dtype).view(1, 1, 1, 5).expand_as(image)
if direction == -1:
image, expected = image.flip(-1), expected.flip(-1)
if axis == 1:
image, expected = image.transpose(-2, -1), expected.transpose(-2, -1)
noise = torch.zeros(2, 2, 5, 5, device=device, dtype=dtype)
noise[:, axis] = direction
# A constant displacement of two pixels; the 1-tap Gaussian leaves the field unchanged.
scale = 1.0 if align_corners else 0.8
output = elastic_transform2d(
image,
noise,
kernel_size=(1, 1),
sigma=(1.0, 1.0),
alpha=(scale, scale),
align_corners=align_corners,
mode=mode,
padding_mode=padding_mode,
)
# A rounded float16 grid can land just inside the zero-padding boundary (5 * 2**-9 here).
tolerance = {"rtol": 1e-3, "atol": 1e-2} if dtype == torch.float16 else {}
self.assert_close(output, expected, **tolerance)
@pytest.mark.parametrize("align_corners", [False, True])
def test_zero_padding_fractional_boundary(self, device, align_corners):
image = torch.ones(1, 1, 5, 5, device=device)
noise = torch.zeros(1, 2, 5, 5, device=device)
noise[:, 0] = 1.0
# Three quarters of a pixel beyond the final centre leaves one quarter of its bilinear weight.
scale = 0.375 if align_corners else 0.3
output = elastic_transform2d(
image,
noise,
kernel_size=(1, 1),
sigma=(1.0, 1.0),
alpha=(scale, 0.0),
align_corners=align_corners,
padding_mode="zeros",
)
expected = torch.ones_like(image)
expected[..., -1] = 0.25
self.assert_close(output, expected)
@pytest.mark.parametrize("align_corners", [False, True])
def test_reflected_displacement_gradient(self, device, align_corners):
image = torch.arange(1, 6, device=device, dtype=torch.float32).view(1, 1, 1, 5).expand(1, 1, 5, 5)
noise = torch.zeros(1, 2, 5, 5, device=device)
noise[:, 0] = 1.0
noise.requires_grad_()
# The last column samples strictly inside the reflected image, beyond normalized coordinate 1.
scale = 1.15 if align_corners else 0.92
output = elastic_transform2d(
image,
noise,
kernel_size=(1, 1),
sigma=(1.0, 1.0),
alpha=(scale, 0.0),
align_corners=align_corners,
padding_mode="reflection",
)
gradient = torch.autograd.grad(output[..., -1].sum(), noise)[0]
expected = torch.zeros_like(noise)
expected[:, 0, :, -1] = -2.3
self.assert_close(gradient, expected)
def test_smoke(self, device, dtype):
image = torch.rand(1, 4, 5, 5, device=device, dtype=dtype)
noise = torch.rand(1, 2, 5, 5, device=device, dtype=dtype)
assert elastic_transform2d(image, noise) is not None
@pytest.mark.parametrize("batch, channels, height, width", [(1, 3, 3, 4), (2, 2, 2, 4), (1, 5, 4, 1)])
def test_cardinality(self, device, dtype, batch, channels, height, width):
shape = batch, channels, height, width
img = torch.ones(shape, device=device, dtype=dtype)
noise = torch.ones((batch, 2, height, width), device=device, dtype=dtype)
assert elastic_transform2d(img, noise).shape == shape
def test_exception(self, device, dtype):
from kornia.core.exceptions import ShapeError, TypeCheckError
ex = torch.ones(1, device=device, dtype=dtype)
with pytest.raises(TypeCheckError) as errinfo:
elastic_transform2d([0.0], ex)
assert "Type mismatch: expected Tensor" in str(errinfo.value)
with pytest.raises(TypeCheckError) as errinfo:
elastic_transform2d(ex, 1)
assert "Type mismatch: expected Tensor" in str(errinfo.value)
img = torch.ones(1, 1, 1, device=device, dtype=dtype)
noise = torch.ones(1, 2, 1, 1, device=device, dtype=dtype)
with pytest.raises(ShapeError) as errinfo:
elastic_transform2d(img, noise)
assert "Shape dimension mismatch" in str(errinfo.value)
img = torch.ones(1, 1, 1, 1, device=device, dtype=dtype)
noise = torch.ones(2, 1, 1, device=device, dtype=dtype)
with pytest.raises(ShapeError) as errinfo:
elastic_transform2d(img, noise)
assert "Shape dimension mismatch" in str(errinfo.value)
img = torch.ones(1, 1, 1, 1, device=device, dtype=dtype)
noise = torch.ones(1, 3, 1, 1, device=device, dtype=dtype)
with pytest.raises(RuntimeError) as errinfo:
elastic_transform2d(img, noise)
assert "The size of tensor a (2) must match the size of tensor b (3)" in str(errinfo.value)
@pytest.mark.parametrize(
"kernel_size, sigma, alpha",
[
((3, 3), (4.0, 4.0), (32.0, 32.0)),
((5, 3), (4.0, 8.0), (16.0, 32.0)),
((5, 5), torch.tensor([2.0, 8.0]), torch.tensor([16.0, 64.0])),
],
)
def test_valid_paramters(self, device, dtype, kernel_size, sigma, alpha):
image = torch.rand(1, 4, 5, 5, device=device, dtype=dtype)
noise = torch.rand(1, 2, 5, 5, device=device, dtype=dtype)
if isinstance(sigma, torch.Tensor):
sigma = sigma.to(device, dtype)
if isinstance(alpha, torch.Tensor):
alpha = alpha.to(device, dtype)
assert elastic_transform2d(image, noise, kernel_size, sigma, alpha) is not None
def test_values(self, device, dtype):
image = torch.tensor(
[[[[0.0018, 0.7521, 0.7550], [0.2053, 0.4249, 0.1369], [0.1027, 0.3992, 0.8773]]]],
device=device,
dtype=dtype,
)
noise = torch.ones(1, 2, 3, 3, device=device, dtype=dtype)
expected = torch.tensor(
[[[[0.0062, 0.7506, 0.7487], [0.2058, 0.4235, 0.1397], [0.1036, 0.3996, 0.8693]]]],
device=device,
dtype=dtype,
)
actual = elastic_transform2d(image, noise)
# bfloat16's 7-bit mantissa makes the reference values intentionally rounded more
# coarsely than float16/float32; retain a tolerance that covers that representation.
tolerance = 1e-2 if dtype is torch.bfloat16 else 1e-3
self.assert_close(actual, expected, atol=tolerance, rtol=tolerance)
@pytest.mark.parametrize("align_corners", [False, True])
def test_zero_displacement_is_identity(self, device, dtype, align_corners):
"""Zero displacement should preserve the input for either grid convention."""
image = torch.arange(16.0, device=device, dtype=dtype).reshape(1, 1, 4, 4)
noise = torch.zeros(1, 2, 4, 4, device=device, dtype=dtype)
output = elastic_transform2d(
image,
noise,
kernel_size=(3, 3),
sigma=(1.0, 1.0),
alpha=(1.0, 1.0),
align_corners=align_corners,
)
self.assert_close(output, image)
@pytest.mark.parametrize("requires_grad", [True, False])
def test_gradcheck(self, device, dtype, requires_grad):
image = torch.rand(1, 1, 3, 3, device=device, dtype=torch.float64, requires_grad=requires_grad)
noise = torch.rand(1, 2, 3, 3, device=device, dtype=torch.float64, requires_grad=not requires_grad)
assert self.gradcheck(
elastic_transform2d, (image, noise), raise_exception=True, fast_mode=True, nondet_tol=1e-4
)
def test_convention_noise_channel0_is_x_positive_shifts_left(self, device, dtype):
# noise channel 0 is the x-sampling-offset channel; a positive constant value there
# shifts image content LEFT (not right, and not channel 1/y). test_values uses a
# symmetric (both-channels-equal) noise tensor and so cannot discriminate this.
image = torch.zeros(1, 1, 5, 5, device=device, dtype=dtype)
image[0, 0, 2, 2] = 1.0
noise = torch.zeros(1, 2, 5, 5, device=device, dtype=dtype)
noise[0, 0] = 0.5
# Snippet used to generate expected:
# img = torch.zeros(1, 1, 5, 5); img[0, 0, 2, 2] = 1.0
# noise = torch.zeros(1, 2, 5, 5); noise[0, 0] = 0.5
# out = elastic_transform2d(img, noise, kernel_size=(3, 3), sigma=(1., 1.), alpha=(1., 1.))
# (out[0, 0] > 0.1).nonzero() -> [[2, 1]]
out = elastic_transform2d(image, noise, kernel_size=(3, 3), sigma=(1.0, 1.0), alpha=(1.0, 1.0))
assert (out[0, 0] > 0.1).nonzero().tolist() == [[2, 1]]
def test_convention_alpha_xy_order(self, device, dtype):
# alpha is (x, y): alpha[0] scales the x-displacement, unlike kernel_size and sigma, which
# are (y, x) (test_convention_sigma_yx_order).
image = torch.zeros(1, 1, 9, 9, device=device, dtype=dtype)
image[0, 0, 4, 4] = 1.0
noise = torch.zeros(1, 2, 9, 9, device=device, dtype=dtype)
noise[0, 0] = 0.2
out_a = elastic_transform2d(image, noise, kernel_size=(3, 3), sigma=(1.0, 1.0), alpha=(2.0, 0.5))
out_b = elastic_transform2d(image, noise, kernel_size=(3, 3), sigma=(1.0, 1.0), alpha=(0.5, 2.0))
# Snippet used to generate expected (requires only this module):
# image = torch.zeros(1, 1, 9, 9); image[0, 0, 4, 4] = 1.0
# noise = torch.zeros(1, 2, 9, 9); noise[0, 0] = 0.2
# out_a = elastic_transform2d(image, noise, kernel_size=(3, 3), sigma=(1.0, 1.0), alpha=(2.0, 0.5))
# out_b = elastic_transform2d(image, noise, kernel_size=(3, 3), sigma=(1.0, 1.0), alpha=(0.5, 2.0))
# (out_a[0, 0] > 0.05).nonzero().tolist() / (out_b[0, 0] > 0.05).nonzero().tolist()
# alpha=(2.0, 0.5): larger alpha[0] -> larger x-displacement -> marker moves further left.
assert (out_a[0, 0] > 0.05).nonzero().tolist() == [[4, 2], [4, 3]]
# alpha=(0.5, 2.0): smaller alpha[0] -> smaller x-displacement -> marker moves less.
assert (out_b[0, 0] > 0.05).nonzero().tolist() == [[4, 3], [4, 4]]
def test_convention_sigma_yx_order(self, device, dtype):
# sigma is genuinely (sigma_y, sigma_x): an impulse in the x-displacement noise channel,
# blurred with a strongly anisotropic sigma, spreads along ROWS when sigma[0] is large and
# is confined to a single row when sigma[1] is large instead -- pinning sigma[0] to the y
# (row) direction (matching the docstring; see test_convention_alpha_xy_order for the
# contrasting alpha order).
#
# Pinned as an ORDERING PROPERTY (row-wise spread of the affected region), not as exact
# threshold-crossing pixel lists: compute the per-row total displaced energy, then the
# weighted standard deviation of that energy across rows (how far the effect spreads
# vertically). sigma=(3.0, 0.3) must spread markedly wider across rows than sigma=(0.3, 3.0).
#
# Snippet used to generate expected (requires only this module):
# N, c = 21, 10
# image = torch.zeros(1, 1, N, N); image[0, 0, :, c] = 1.0
# noise = torch.zeros(1, 2, N, N); noise[0, 0, c, c] = 1.0
# kwargs = dict(kernel_size=(9, 9), alpha=(3.0, 0.0))
# out_y = elastic_transform2d(image, noise, sigma=(3.0, 0.3), **kwargs)
# out_x = elastic_transform2d(image, noise, sigma=(0.3, 3.0), **kwargs)
# def row_std(out):
# energy = (out[0, 0, 1:-1] - image[0, 0, 1:-1]).abs().sum(dim=1)
# rows = torch.arange(energy.shape[0], dtype=energy.dtype)
# w = energy / energy.sum()
# mean = (w * rows).sum()
# return (w * (rows - mean) ** 2).sum().sqrt().item()
# row_std(out_y), row_std(out_x) # -> 2.582, 0.144 (ratio ~18x)
n_sigma, center = 21, 10
sigma_image = torch.zeros(1, 1, n_sigma, n_sigma, device=device, dtype=dtype)
sigma_image[0, 0, :, center] = 1.0
sigma_noise = torch.zeros(1, 2, n_sigma, n_sigma, device=device, dtype=dtype)
sigma_noise[0, 0, center, center] = 1.0
sigma_kwargs = {"kernel_size": (9, 9), "alpha": (3.0, 0.0)}
out_sigma_y_wide = elastic_transform2d(sigma_image, sigma_noise, sigma=(3.0, 0.3), **sigma_kwargs)
out_sigma_x_wide = elastic_transform2d(sigma_image, sigma_noise, sigma=(0.3, 3.0), **sigma_kwargs)
spread_y_wide = _row_spread_std(out_sigma_y_wide, sigma_image)
spread_x_wide = _row_spread_std(out_sigma_x_wide, sigma_image)
# sigma=(3.0, 0.3) (large sigma_y) spreads markedly wider across rows than
# sigma=(0.3, 3.0) (large sigma_x) -- a robust ordering property, not an exact pixel list.
assert spread_y_wide > 5 * spread_x_wide
def test_convention_padding_mode_border_differs_from_zeros(self, device, dtype):
# Displacements beyond the image use zeros by default, or the edge pixel with 'border'.
image = torch.zeros(1, 1, 5, 5, device=device, dtype=dtype)
image[0, 0, 2, 4] = 5.0
noise = torch.zeros(1, 2, 5, 5, device=device, dtype=dtype)
noise[0, 0] = 1.0
kwargs = {"kernel_size": (3, 3), "sigma": (1.0, 1.0), "alpha": (4.0, 0.0)}
# Snippet used to generate expected (requires only this module):
# image = torch.zeros(1, 1, 5, 5); image[0, 0, 2, 4] = 5.0
# noise = torch.zeros(1, 2, 5, 5); noise[0, 0] = 1.0
# kwargs = dict(kernel_size=(3, 3), sigma=(1.0, 1.0), alpha=(4.0, 0.0))
# elastic_transform2d(image, noise, **kwargs)[0, 0, 2] # default row
# elastic_transform2d(image, noise, padding_mode="border", **kwargs)[0, 0, 2] # 'border' row
out_default = elastic_transform2d(image, noise, **kwargs)
expected_zeros_row = torch.zeros(5, device=device, dtype=dtype)
self.assert_close(out_default[0, 0, 2], expected_zeros_row, rtol=1e-2, atol=1e-2)
if not supports_2d_border_padding(device):
pytest.skip("MPS 2D grid_sample lacks 'border' padding")
out_border = elastic_transform2d(image, noise, padding_mode="border", **kwargs)
expected_border_row = torch.tensor([5.0, 5.0, 5.0, 5.0, 5.0], device=device, dtype=dtype)
self.assert_close(out_border[0, 0, 2], expected_border_row, rtol=1e-2, atol=1e-2)