π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
import torch
import kornia
from testing.base import (
DYNAMIC_EXPORT_UNAVAILABLE_REASON,
DYNAMO_UNAVAILABLE_REASON,
assert_close,
dynamic_export_is_available,
dynamo_is_available,
supports_bilinear_2d_grid_sample,
supports_bilinear_3d_grid_sample,
)
def test_create_meshgrid(device, dtype):
height, width = 4, 6
normalized_coordinates = False
# create the meshgrid and verify shape
grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates, device=device, dtype=dtype)
assert grid.device == device
assert grid.dtype == dtype
assert grid.shape == (1, height, width, 2)
# check grid corner values
assert_close(grid[0, 0, 0], torch.tensor((0.0, 0.0), device=device, dtype=dtype), atol=0.0, rtol=0.0)
assert_close(
grid[0, height - 1, width - 1],
torch.tensor((width - 1, height - 1), device=device, dtype=dtype),
atol=0.0,
rtol=0.0,
)
def test_normalize_pixel_grid(device, dtype):
# generate input data
height, width = 2, 4
# create points grid
grid_norm = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device, dtype=dtype)
assert grid_norm.device == device
assert grid_norm.dtype == dtype
grid_norm = torch.unsqueeze(grid_norm, dim=0)
grid_pix = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=False, device=device, dtype=dtype)
assert grid_pix.device == device
assert grid_pix.dtype == dtype
grid_pix = torch.unsqueeze(grid_pix, dim=0)
# grid from pixel space to normalized
norm_trans_pix = kornia.geometry.conversions.normal_transform_pixel(
height, width, device=device, dtype=dtype
) # 1x3x3
pix_trans_norm = torch.tensor([[[1.5, 0.0, 1.5], [0.0, 0.5, 0.5], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
# transform grids
grid_pix_to_norm = kornia.geometry.linalg.transform_points(norm_trans_pix, grid_pix)
grid_norm_to_pix = kornia.geometry.linalg.transform_points(pix_trans_norm, grid_norm)
assert_close(grid_pix, grid_norm_to_pix)
assert_close(grid_norm, grid_pix_to_norm)
@pytest.mark.parametrize(("height", "width"), [(1, 4), (4, 1), (1, 1)])
def test_normalized_meshgrid_singleton_axis_is_centered(height, width, device, dtype):
grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device, dtype=dtype)
if height == 1:
assert_close(grid[..., 1], torch.zeros_like(grid[..., 1]), atol=0.0, rtol=0.0)
if width == 1:
assert_close(grid[..., 0], torch.zeros_like(grid[..., 0]), atol=0.0, rtol=0.0)
@pytest.mark.parametrize(("height", "width"), [(1, 4), (4, 1), (1, 1), (4, 6)])
@pytest.mark.parametrize("grid_dtype", [torch.int32, torch.int64])
def test_normalized_meshgrid_integer_dtype_promotes_uniformly(height, width, grid_dtype, device):
# An integer ``dtype`` makes the non-singleton branch promote to the default float dtype
# (integer division), so the singleton branch must promote identically or torch.meshgrid
# rejects the mixed pair. Eager and tracing take different code paths here; this pins that
# they agree, and that a singleton axis is still centred once promoted.
grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True, device=device, dtype=grid_dtype)
assert grid.dtype == torch.get_default_dtype()
if height == 1:
assert_close(grid[..., 1], torch.zeros_like(grid[..., 1]), atol=0.0, rtol=0.0)
if width == 1:
assert_close(grid[..., 0], torch.zeros_like(grid[..., 0]), atol=0.0, rtol=0.0)
@pytest.mark.parametrize("grid_dtype", [torch.int32, torch.int64])
@pytest.mark.parametrize(("trace_height", "runtime_height"), [(2, 1), (1, 2)])
def test_normalized_meshgrid_integer_trace_crosses_singleton_boundary(grid_dtype, trace_height, runtime_height, device):
class IntegerMeshGrid(torch.nn.Module):
def forward(self, image):
return kornia.geometry.create_meshgrid(
image.shape[-2],
image.shape[-1],
normalized_coordinates=True,
device=image.device,
dtype=grid_dtype,
)
example = torch.zeros(1, 1, trace_height, 4, device=device)
runtime = torch.zeros(1, 1, runtime_height, 4, device=device)
traced = torch.jit.trace(IntegerMeshGrid(), example)
assert_close(traced(runtime), IntegerMeshGrid()(runtime), atol=0.0, rtol=0.0)
@pytest.mark.parametrize(("trace_height", "runtime_height"), [(2, 1), (1, 2)])
def test_normalized_meshgrid_trace_crosses_singleton_boundary(trace_height, runtime_height, device, dtype):
class MeshGrid(torch.nn.Module):
def forward(self, image):
return kornia.geometry.create_meshgrid(
image.shape[-2], image.shape[-1], normalized_coordinates=True, device=image.device, dtype=image.dtype
)
example = torch.zeros(1, 1, trace_height, 4, device=device, dtype=dtype)
runtime = torch.zeros(1, 1, runtime_height, 4, device=device, dtype=dtype)
traced = torch.jit.trace(MeshGrid(), example)
assert_close(traced(runtime), MeshGrid()(runtime), atol=0.0, rtol=0.0)
@pytest.mark.parametrize("is_3d", [False, True], ids=["2d", "3d"])
@pytest.mark.parametrize("size", [258, 300, 1000, 2050, 3000])
def test_normalized_meshgrid_trace_matches_eager_at_unrepresentable_sizes(is_3d, size, device, dtype):
"""A size whose predecessor does not fit the coordinate dtype must not round the divisor.
``bfloat16`` holds 299 only as 300, so casting ``size - 1`` into the coordinate dtype before
dividing shifts every normalized coordinate -- up to 0.0078 in bfloat16, 0.00098 in float16.
Eager divides by a Python ``int``, so the traced graph has to divide against the unrounded
size too and narrow only after normalization. Every size the singleton-boundary tests above use is
representable in all four dtypes, so none of them can catch this.
The sizes here are the ones whose predecessor is *not* exactly representable in bfloat16
(all five) or float16 (2050 and 3000). CPU eager supplies the correctly rounded reference;
exact comparison at every dtype preserves the half-precision regression check.
"""
class MeshGrid(torch.nn.Module):
def forward(self, image):
if is_3d:
return kornia.geometry.create_meshgrid3d(
image.shape[-3],
image.shape[-2],
image.shape[-1],
normalized_coordinates=True,
device=image.device,
dtype=image.dtype,
)
return kornia.geometry.create_meshgrid(
image.shape[-2], image.shape[-1], normalized_coordinates=True, device=image.device, dtype=image.dtype
)
shape = (1, 1, 2, size, 4) if is_3d else (1, 1, size, 4)
image = torch.zeros(*shape, device=device, dtype=dtype)
traced = torch.jit.trace(MeshGrid(), image)
# CUDA eager's Python-scalar division multiplies by a host-computed reciprocal, which can
# round differently from the traced tensor division (#4195).
assert_close(traced(image).cpu(), MeshGrid()(image.cpu()), atol=0.0, rtol=0.0)
@pytest.mark.parametrize("normalized_coordinates", [False, True], ids=["pixel", "normalized"])
@pytest.mark.parametrize("is_3d", [False, True], ids=["2d", "3d"])
@pytest.mark.skipif(not dynamic_export_is_available(), reason=DYNAMIC_EXPORT_UNAVAILABLE_REASON)
def test_meshgrid_export_crosses_singleton_boundary(is_3d, normalized_coordinates):
class MeshGrid(torch.nn.Module):
def forward(self, image):
if is_3d:
return kornia.geometry.create_meshgrid3d(
image.shape[-3],
image.shape[-2],
image.shape[-1],
normalized_coordinates=normalized_coordinates,
device=image.device,
)
return kornia.geometry.create_meshgrid(
image.shape[-2],
image.shape[-1],
normalized_coordinates=normalized_coordinates,
device=image.device,
)
image_shape = (1, 1, 2, 3, 4) if is_3d else (1, 1, 2, 4)
example = torch.zeros(image_shape)
exported = torch.export.export(
MeshGrid(),
(example,),
dynamic_shapes=({2: torch.export.Dim("singleton_axis", min=1, max=8)},),
).module()
for runtime_size in (1, 5):
runtime = torch.zeros(*image_shape[:2], runtime_size, *image_shape[3:])
assert_close(exported(runtime), MeshGrid()(runtime), atol=0.0, rtol=0.0)
@pytest.mark.parametrize("default_dtype", [torch.float32, torch.float64])
@pytest.mark.parametrize("is_3d", [False, True], ids=["2d", "3d"])
@pytest.mark.skipif(not dynamo_is_available(), reason=DYNAMO_UNAVAILABLE_REASON)
def test_pixel_meshgrid_default_dtype_matches_compile(is_3d, default_dtype):
class MeshGrid(torch.nn.Module):
def forward(self, image):
if is_3d:
return kornia.geometry.create_meshgrid3d(
image.shape[-3], image.shape[-2], image.shape[-1], False, device=image.device
)
return kornia.geometry.create_meshgrid(image.shape[-2], image.shape[-1], False, device=image.device)
image = torch.zeros((1, 1, 2, 3, 4) if is_3d else (1, 1, 3, 4), dtype=torch.float32)
previous_dtype = torch.get_default_dtype()
try:
torch.set_default_dtype(default_dtype)
expected = MeshGrid()(image)
actual = torch.compile(MeshGrid(), fullgraph=True)(image)
finally:
torch.set_default_dtype(previous_dtype)
assert actual.dtype == expected.dtype == default_dtype
assert_close(actual, expected, atol=0.0, rtol=0.0)
@pytest.mark.parametrize("is_3d", [False, True], ids=["2d", "3d"])
def test_pixel_meshgrid_scripted_ramp_follows_the_runtime_default_dtype(is_3d):
"""A scripted grid with no explicit dtype must track ``torch.get_default_dtype()`` per call.
The default dtype is read off an empty tensor because TorchScript has no
``aten::get_default_dtype``. A factory call with no graph input is constant-folded by
TorchScript's optimizing executor once profiling completes, which froze the ramp at whatever
the default happened to be on the first call -- ``linspace(0, size - 1, size)`` never folded
because it consumed ``size``. Passing ``device`` restores that dependency. The first call here
is deliberately made under a *different* default from the assertion, since the freeze only
shows from the second call onwards. No "compile"/"dynamo" in the name on purpose: this is
plain TorchScript and must run in the ordinary test jobs, which deselect those by name.
"""
op = kornia.geometry.create_meshgrid3d if is_3d else kornia.geometry.create_meshgrid
args = (2, 3, 4) if is_3d else (3, 4)
scripted = torch.jit.script(op)
previous_dtype = torch.get_default_dtype()
try:
torch.set_default_dtype(torch.float32)
assert scripted(*args).dtype == torch.float32
torch.set_default_dtype(torch.float64)
assert scripted(*args).dtype == torch.float64 == op(*args).dtype
assert scripted(*args).dtype == torch.float64, "the scripted ramp froze its default dtype"
finally:
torch.set_default_dtype(previous_dtype)
@pytest.mark.parametrize("grid_dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("is_3d", [False, True], ids=["2d", "3d"])
def test_pixel_meshgrid_half_dtype_matches_compile(is_3d, grid_dtype):
"""A half-precision pixel ramp must not change when the grid is built under ``torch.compile``.
``linspace`` rounds its endpoint into the coordinate dtype and fills the upper half of the ramp
backwards from that rounded value, so it lands one ulp off the correctly rounded column index --
``linspace(0, 299, 300)`` gives 258 at index 257 where bfloat16 holds 257 as 256. Inductor lowers
the same call to a per-index computation that rounds correctly, so the eager and compiled grids
disagreed by up to one ulp of the width. The sizes here are the first at which the two diverge:
258 for bfloat16 and 2050 for float16.
"""
class MeshGrid(torch.nn.Module):
def forward(self, image):
if is_3d:
return kornia.geometry.create_meshgrid3d(
image.shape[-3], image.shape[-2], image.shape[-1], False, device=image.device, dtype=image.dtype
)
return kornia.geometry.create_meshgrid(
image.shape[-2], image.shape[-1], False, device=image.device, dtype=image.dtype
)
size = 258 if grid_dtype == torch.bfloat16 else 2050
shape = (1, 1, 2, 2, size) if is_3d else (1, 1, 2, size)
image = torch.zeros(*shape, dtype=grid_dtype)
expected = MeshGrid()(image)
actual = torch.compile(MeshGrid(), fullgraph=True)(image)
assert actual.dtype == expected.dtype == grid_dtype
assert_close(actual, expected, atol=0.0, rtol=0.0)
@pytest.mark.parametrize("grid_dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("is_3d", [False, True], ids=["2d", "3d"])
def test_normalized_meshgrid_half_dtype_matches_compile(is_3d, grid_dtype, device):
"""A half-precision normalized grid must not change when it is built under ``torch.compile``.
Eager evaluates ``(xs / (size - 1) - 0.5) * 2`` as three half-precision ops and rounds after
each; inductor computes the chain in float32 and rounds once on store. Eager also materializes
the half ramp before the capture branch widens it, while inductor folds that narrow-then-widen
round trip away and keeps the exact index, so above ``2 ** p`` the two divided different
numerators. Building and normalizing the ramp in float32 and narrowing once removes both, and
is a no-op at float32/float64, which never widen.
The sizes are the first at which the pixel ramps themselves diverge, so the normalized error
on top of them is the one under test. At float16 on CUDA a residue of at most one ulp of the
coordinate dtype survives, because ATen's and triton's float32 division are not bit-identical;
bfloat16's 8-bit significand absorbs it and the CPU kernels agree exactly.
"""
class MeshGrid(torch.nn.Module):
def forward(self, image):
if is_3d:
return kornia.geometry.create_meshgrid3d(
image.shape[-3], image.shape[-2], image.shape[-1], True, device=image.device, dtype=image.dtype
)
return kornia.geometry.create_meshgrid(
image.shape[-2], image.shape[-1], True, device=image.device, dtype=image.dtype
)
size = 300 if grid_dtype == torch.bfloat16 else 3000
shape = (1, 1, 2, 2, size) if is_3d else (1, 1, 2, size)
image = torch.zeros(*shape, dtype=grid_dtype, device=device)
expected = MeshGrid()(image)
actual = torch.compile(MeshGrid(), fullgraph=True)(image)
assert actual.dtype == expected.dtype == grid_dtype
inexact_division = device.type == "cuda" and grid_dtype == torch.float16
assert_close(actual, expected, atol=torch.finfo(grid_dtype).eps if inexact_division else 0.0, rtol=0.0)
def test_create_meshgrid_align_corners(device, dtype):
# align_corners selects which grid_sample normalization the grid is built for: True puts pixel
# centers 0 and W-1 at -1 and +1, False puts the outer pixel EDGES there (#3904).
height, width = 2, 4
grid_true = kornia.geometry.create_meshgrid(height, width, True, device=device, dtype=dtype, align_corners=True)
grid_false = kornia.geometry.create_meshgrid(height, width, True, device=device, dtype=dtype, align_corners=False)
# align_corners=True is the default, so the bare call must match it
assert_close(kornia.geometry.create_meshgrid(height, width, True, device=device, dtype=dtype), grid_true)
assert_close(grid_true[0, :, 0, 0], torch.tensor([-1.0, -1.0], device=device, dtype=dtype))
assert_close(grid_true[0, :, -1, 0], torch.tensor([1.0, 1.0], device=device, dtype=dtype))
# False: x centers land at (2x + 1)/W - 1, i.e. -0.75 and 0.75 for W = 4
assert_close(grid_false[0, :, 0, 0], torch.tensor([-0.75, -0.75], device=device, dtype=dtype))
assert_close(grid_false[0, :, -1, 0], torch.tensor([0.75, 0.75], device=device, dtype=dtype))
# align_corners is only about the normalization, so it is a no-op for pixel coordinates
assert_close(
kornia.geometry.create_meshgrid(height, width, False, device=device, dtype=dtype, align_corners=True),
kornia.geometry.create_meshgrid(height, width, False, device=device, dtype=dtype, align_corners=False),
)
def test_create_meshgrid3d(device, dtype):
depth, height, width = 5, 4, 6
normalized_coordinates = False
# create the meshgrid and verify shape
grid = kornia.geometry.create_meshgrid3d(depth, height, width, normalized_coordinates, device=device, dtype=dtype)
assert grid.device == device
assert grid.dtype == dtype
assert grid.shape == (1, depth, height, width, 3)
# check grid corner values
assert_close(grid[0, 0, 0, 0], torch.tensor((0.0, 0.0, 0.0), device=device, dtype=dtype), atol=0.0, rtol=0.0)
assert_close(
grid[0, depth - 1, height - 1, width - 1],
torch.tensor((depth - 1, width - 1, height - 1), device=device, dtype=dtype),
atol=0.0,
rtol=0.0,
)
@pytest.mark.parametrize(("depth", "height", "width", "axis"), [(1, 4, 6, 0), (5, 1, 6, 2), (5, 4, 1, 1)])
def test_normalized_meshgrid3d_singleton_axis_is_centered(depth, height, width, axis, device, dtype):
grid = kornia.geometry.create_meshgrid3d(
depth, height, width, normalized_coordinates=True, device=device, dtype=dtype
)
assert_close(grid[..., axis], torch.zeros_like(grid[..., axis]), atol=0.0, rtol=0.0)
@pytest.mark.parametrize(("trace_depth", "runtime_depth"), [(2, 1), (1, 2)])
def test_normalized_meshgrid3d_trace_crosses_singleton_boundary(trace_depth, runtime_depth, device, dtype):
class MeshGrid3d(torch.nn.Module):
def forward(self, volume):
return kornia.geometry.create_meshgrid3d(
volume.shape[-3],
volume.shape[-2],
volume.shape[-1],
normalized_coordinates=True,
device=volume.device,
dtype=volume.dtype,
)
example = torch.zeros(1, 1, trace_depth, 3, 4, device=device, dtype=dtype)
runtime = torch.zeros(1, 1, runtime_depth, 3, 4, device=device, dtype=dtype)
traced = torch.jit.trace(MeshGrid3d(), example)
assert_close(traced(runtime), MeshGrid3d()(runtime), atol=0.0, rtol=0.0)
def test_convention_create_meshgrid_align_corners_matches_grid_sample(device, dtype):
# A normalized grid samples an image as the identity only under grid_sample's own align_corners flag: True puts
# pixel centres 0 and W - 1 at -1 and +1, False puts the outer pixel edges there. Crossing the flags shifts every
# sample. H != W, so neither axis can stand in for the other.
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip(f"torch has no bilinear 2D grid_sample kernel for {device.type} {dtype}")
generator = torch.Generator(device="cpu").manual_seed(0)
image = torch.rand(1, 1, 3, 5, generator=generator).to(device, dtype)
for align_corners in (True, False):
grid = kornia.geometry.create_meshgrid(3, 5, device=device, dtype=dtype, align_corners=align_corners)
same = torch.nn.functional.grid_sample(image, grid, align_corners=align_corners)
crossed = torch.nn.functional.grid_sample(image, grid, align_corners=not align_corners)
assert_close(same, image)
assert (crossed - image).abs().max() > 0.1
def test_convention_create_meshgrid3d_reorder_for_grid_sample(device, dtype):
# The last axis of create_meshgrid3d is (d, x, y) -- depth, column, row -- while grid_sample reads (x, y, z), so
# the default corner-aligned grid samples a volume as the identity after the [1, 2, 0] reorder.
# D != H != W; the permuted volume is the relabel control.
if not supports_bilinear_3d_grid_sample(device, dtype):
pytest.skip(f"torch has no bilinear 3D grid_sample kernel for {device.type} {dtype}")
pixel = kornia.geometry.create_meshgrid3d(2, 3, 4, normalized_coordinates=False, device=device, dtype=dtype)
# (d, h, w) = (1, 2, 3) holds (d, x, y) = (1, 3, 2)
assert_close(pixel[0, 1, 2, 3], torch.tensor([1.0, 3.0, 2.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
generator = torch.Generator(device="cpu").manual_seed(0)
volume = torch.rand(1, 1, 2, 3, 4, generator=generator).to(device, dtype)
grid = kornia.geometry.create_meshgrid3d(2, 3, 4, device=device, dtype=dtype)
reordered = torch.nn.functional.grid_sample(volume, grid[..., [1, 2, 0]], align_corners=True)
as_is = torch.nn.functional.grid_sample(volume, grid, align_corners=True)
assert_close(reordered, volume)
assert (as_is - volume).abs().max() > 0.1
permuted = volume.permute(0, 1, 4, 2, 3).contiguous() # (D, H, W) = (4, 2, 3)
grid = kornia.geometry.create_meshgrid3d(4, 2, 3, device=device, dtype=dtype)
assert_close(torch.nn.functional.grid_sample(permuted, grid[..., [1, 2, 0]], align_corners=True), permuted)
@pytest.mark.xfail(
strict=True, raises=TypeError, reason="#4503: create_meshgrid3d cannot produce an align_corners=False grid"
)
def test_wart_create_meshgrid3d_lacks_align_corners_false_4503(device, dtype):
# Correct half-pixel coordinates for (D, H, W) = (2, 3, 4), in Kornia's (d, x, y) order.
grid = kornia.geometry.create_meshgrid3d(2, 3, 4, device=device, dtype=dtype, align_corners=False)
assert_close(grid[0, 0, 0, 0], torch.tensor([-0.5, -0.75, -2 / 3], device=device, dtype=dtype))
assert_close(grid[0, -1, -1, -1], torch.tensor([0.5, 0.75, 2 / 3], device=device, dtype=dtype))