π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import io
import math
import warnings
import pytest
import torch
import kornia
from kornia.core.exceptions import ShapeError
from kornia.geometry import bbox as bbox_module
from kornia.geometry.bbox import (
bbox_generator,
bbox_generator3d,
bbox_to_mask,
bbox_to_mask3d,
infer_bbox_shape,
infer_bbox_shape3d,
nms,
transform_bbox,
validate_bbox,
validate_bbox3d,
)
from kornia.geometry.boxes import Boxes3D
from kornia.geometry.transform.crop3d import crop_and_resize3d, crop_by_boxes3d
from testing.base import DYNAMO_UNAVAILABLE_REASON, BaseTester, dynamo_is_available
class TestBbox2D(BaseTester):
def test_wart_infer_bbox_shape_is_inclusive_3934(self, device, dtype):
# Wart pin for kornia#3934: a raw box spanning x=1..4 and y=1..2 reports
# (height, width) = (2, 4) because infer_bbox_shape adds one on both axes.
# The non-square extents also pin the (heights, widths) return order.
boxes = torch.tensor([[[1.0, 1.0], [4.0, 1.0], [4.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
height, width = infer_bbox_shape(boxes)
self.assert_close(height, torch.tensor([2.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
self.assert_close(width, torch.tensor([4.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
def test_convention_infer_bbox_shape_reads_fixed_vertex_indices(self, device, dtype):
# Convention pin: the extents come from fixed vertex indices, not from a
# maximum - minimum reduction. Swapping the top-left and top-right vertices in
# x therefore yields a negative width, where max - min + 1 would give 5.
# The -3 is 1 - 5 + 1; the inclusive offset is tracked in kornia#3934.
boxes = torch.tensor([[[5.0, 2.0], [1.0, 2.0], [1.0, 3.0], [5.0, 3.0]]], device=device, dtype=dtype)
height, width = infer_bbox_shape(boxes)
self.assert_close(height, torch.tensor([2.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
self.assert_close(width, torch.tensor([-3.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
def test_convention_infer_bbox_shape_preserves_generated_zero_extent(self, device, dtype):
# bbox_generator represents a zero width by placing the top-right vertex one
# unit left of the top-left vertex. RandomCutMixV2 relies on the fixed-index
# reading recovering width 0; a maximum - minimum reduction would return 2.
# The recovered zero depends on the inclusive +1 tracked in kornia#3934.
x_start = torch.tensor([3.0], device=device, dtype=dtype)
y_start = torch.tensor([4.0], device=device, dtype=dtype)
boxes = bbox_generator(x_start, y_start, torch.zeros_like(x_start), torch.full_like(y_start, 5.0))
height, width = infer_bbox_shape(boxes)
self.assert_close(height, torch.tensor([5.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
self.assert_close(width, torch.tensor([0.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
@pytest.mark.parametrize("num_boxes", [2, 3])
def test_infer_bbox_shape_rejects_rank4_4180(self, device, dtype, num_boxes):
boxes = torch.zeros(1, num_boxes, 4, 2, device=device, dtype=dtype)
with pytest.raises(ShapeError, match="expected 3 dimensions, got 4"):
infer_bbox_shape(boxes)
@pytest.mark.parametrize("num_boxes", [2, 3])
def test_bbox_to_mask_rejects_rank4_4180(self, device, dtype, num_boxes):
boxes = torch.zeros(1, num_boxes, 4, 2, device=device, dtype=dtype)
with pytest.raises(ShapeError, match="expected 3 dimensions, got 4"):
bbox_to_mask(boxes, size=(5, 5))
def test_convention_validate_bbox_checks_parallelograms_and_accepts_contiguous_batched_boxes(self, device, dtype):
# The validator only compares the top and bottom edge vectors at fixed vertex
# indices. A cyclically ordered sheared parallelogram and rotated rectangle
# both return True, while the trapezoid fails. The leading dimensions also
# pin contiguous rank-4 input.
sheared_parallelogram = torch.tensor(
[[[[0.0, 0.0], [2.0, 0.0], [3.0, 1.0], [1.0, 1.0]]]], device=device, dtype=dtype
)
assert validate_bbox(sheared_parallelogram) is True
rotated_rectangle = torch.tensor(
[[[[0.0, 0.0], [3.0, 4.0], [-1.0, 7.0], [-4.0, 3.0]]]], device=device, dtype=dtype
)
assert validate_bbox(rotated_rectangle) is True
relabeled_rectangle = torch.tensor(
[[[[0.0, 0.0], [3.0, 0.0], [0.0, 2.0], [3.0, 2.0]]]], device=device, dtype=dtype
)
with pytest.warns(FutureWarning, match="validate_bbox"):
assert validate_bbox(relabeled_rectangle) is False
trapezoid = torch.tensor([[[[0.0, 0.0], [10.0, 0.0], [10.0, 5.0], [3.0, 5.0]]]], device=device, dtype=dtype)
with pytest.warns(FutureWarning, match="validate_bbox"):
assert validate_bbox(trapezoid) is False
# The edge vectors share their x component and differ only in y, so only the second comparison rejects it.
tilted_bottom = torch.tensor([[[[0.0, 0.0], [2.0, 0.0], [2.0, 2.0], [0.0, 1.0]]]], device=device, dtype=dtype)
with pytest.warns(FutureWarning, match="validate_bbox"):
assert validate_bbox(tilted_bottom) is False
def test_convention_validate_bbox_accepts_noncontiguous_rank4_layout_4174(self, device, dtype):
# kornia#4174: a rank-4 input whose leading dimensions cannot be merged by view returns a boolean.
boxes = torch.zeros(2, 3, 4, 2, device=device, dtype=dtype).transpose(0, 1)
assert not boxes.is_contiguous()
assert validate_bbox(boxes) is True
sliced = torch.zeros(2, 4, 4, 2, device=device, dtype=dtype)[:, 1:]
assert not sliced.is_contiguous()
assert validate_bbox(sliced) is True
expanded = torch.zeros(1, 3, 4, 2, device=device, dtype=dtype).expand(2, -1, -1, -1)
assert expanded.stride(0) == 0
assert validate_bbox(expanded) is True
def test_convention_validate_bbox_rejects_non_finite_coordinates_4238(self, device, dtype):
# Pin kornia#4238: NaN and infinite coordinates must not pass validation,
# including when an invalid row is mixed with a valid row.
boxes = torch.tensor(
[
[[0.0, 0.0], [4.0, 0.0], [4.0, 4.0], [0.0, 4.0]],
[[1.0, 1.0], [5.0, 1.0], [5.0, 5.0], [1.0, 5.0]],
],
device=device,
dtype=dtype,
)
for coordinate_index in range(8):
for non_finite in [float("nan"), float("inf"), float("-inf")]:
invalid_boxes = boxes.clone()
invalid_boxes[1].reshape(-1)[coordinate_index] = non_finite
with pytest.warns(FutureWarning, match="validate_bbox"):
assert validate_bbox(invalid_boxes) is False
def test_wart_validate_bbox_returns_false_where_validate_bbox3d_raises_4013(self, device, dtype):
# Wart pin for kornia#4013: for the same invalid shape, the 2D validator
# returns False while the 3D validator raises. The False result is deprecated and
# warns until a later release turns it into the AssertionError the 3D validator raises.
invalid_shape = torch.rand(1, 3, 3, device=device, dtype=dtype)
with pytest.warns(FutureWarning, match="validate_bbox"):
assert validate_bbox(invalid_shape) is False
with pytest.raises(AssertionError):
validate_bbox3d(invalid_shape)
non_cube = torch.tensor(
[
[
[0.0, 0.0, 0.0],
[1.0, 0.0, 0.0],
[1.0, 1.0, 0.0],
[0.0, 1.0, 0.0],
[0.0, 0.0, 1.0],
[2.0, 0.0, 1.0],
[2.0, 1.0, 1.0],
[0.0, 1.0, 1.0],
]
],
device=device,
dtype=dtype,
)
with pytest.raises(AssertionError):
validate_bbox3d(non_cube)
def test_validate_bbox_warns_where_it_returns_false_4013(self, device, dtype):
# kornia#4013: every input for which validate_bbox returns False warns, pointing at the caller, and still
# returns False; a valid box warns nothing.
valid = torch.tensor([[[0.0, 0.0], [3.0, 0.0], [3.0, 2.0], [0.0, 2.0]]], device=device, dtype=dtype)
non_rectangle = valid.clone()
non_rectangle[0, 3, 1] = 3.0
nan_box = valid.clone()
nan_box[0, 2, 0] = float("nan")
wrong_shape = torch.rand(1, 3, 2, device=device, dtype=dtype)
for boxes in (non_rectangle, nan_box, wrong_shape):
with pytest.warns(FutureWarning, match="validate_bbox") as record:
assert validate_bbox(boxes) is False
assert [w.filename for w in record] == [__file__]
with warnings.catch_warnings():
warnings.simplefilter("error")
assert validate_bbox(valid) is True
assert validate_bbox(valid[None]) is True
def test_smoke(self, device, dtype):
# Sample two points of the rectangle
points = torch.rand(1, 4, device=device, dtype=dtype)
# Fill according missing points
bbox = torch.zeros(1, 4, 2, device=device, dtype=dtype)
bbox[0, 0] = points[0][:2]
bbox[0, 1, 0] = points[0][2]
bbox[0, 1, 1] = points[0][1]
bbox[0, 2] = points[0][2:]
bbox[0, 3, 0] = points[0][0]
bbox[0, 3, 1] = points[0][3]
# Validate
assert validate_bbox(bbox)
def test_bounding_boxes_dim_inferring(self, device, dtype):
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
h, w = infer_bbox_shape(boxes)
assert (h, w) == (2, 3)
def test_bounding_boxes_dim_inferring_batch(self, device, dtype):
boxes = torch.tensor(
[[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]], [[2.0, 2.0], [4.0, 2.0], [4.0, 3.0], [2.0, 3.0]]],
device=device,
dtype=dtype,
)
h, w = infer_bbox_shape(boxes)
assert (h.unique().item(), w.unique().item()) == (2, 3)
def test_gradcheck(self, device):
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=torch.float64)
self.gradcheck(infer_bbox_shape, (boxes,))
def test_dynamo(self, device, dtype, torch_optimizer):
# Define script
op = infer_bbox_shape
op_optimized = torch_optimizer(op)
# Define input
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
# Run
expected = op(boxes)
actual = op_optimized(boxes)
# Compare
self.assert_close(actual, expected)
def test_jit(self, device, dtype):
# Test with valid rectangular box
boxes = torch.tensor([[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]], device=device, dtype=dtype)
# JIT compile the validate_bbox function
scripted_fn = torch.jit.script(validate_bbox)
# Test with valid box
self.assert_close(scripted_fn(boxes), validate_bbox(boxes))
# Test with a valid sheared parallelogram
boxes_sheared = torch.tensor([[[0.0, 0.0], [2.0, 0.0], [3.0, 1.0], [1.0, 1.0]]], device=device, dtype=dtype)
self.assert_close(scripted_fn(boxes_sheared), validate_bbox(boxes_sheared))
# Test with invalid shape: the scripted function returns False without the eager deprecation warning
boxes_wrong_shape = torch.rand(1, 3, 2, device=device, dtype=dtype)
with warnings.catch_warnings():
warnings.simplefilter("error")
assert scripted_fn(boxes_wrong_shape) is False
boxes_non_rectangle = torch.tensor(
[[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 2.0]]], device=device, dtype=dtype
)
with warnings.catch_warnings():
warnings.simplefilter("error")
assert scripted_fn(boxes_non_rectangle) is False
def test_convention_bbox_to_mask_takes_width_then_height_and_reads_two_vertices_4014(self, device, dtype):
# Convention pin (kornia#4014 records the argument-order split with Boxes.to_mask, now repaired through
# `size`): bbox_to_mask(boxes, size=(height, width)) returns (B, height, width), fills through the
# top-left (index 0) and bottom-right (index 2) vertices inclusively, ignores the two
# other vertices, keeps the input dtype, and carries no gradient path.
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
mask = bbox_to_mask(boxes, size=(3, 5))
assert mask.shape == (1, 3, 5)
assert mask.dtype == dtype
expected = torch.tensor(
[[[0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 1.0, 1.0, 1.0, 0.0], [0.0, 1.0, 1.0, 1.0, 0.0]]],
device=device,
dtype=dtype,
)
self.assert_close(mask, expected, atol=0.0, rtol=0.0)
assert bbox_to_mask(boxes, size=(5, 3)).shape == (1, 5, 3)
corrupted = boxes.clone()
corrupted[0, 1] = 9.0
corrupted[0, 3] = -9.0
self.assert_close(bbox_to_mask(corrupted, size=(3, 5)), expected, atol=0.0, rtol=0.0)
assert bbox_to_mask(boxes.to(torch.int64), size=(3, 5)).dtype == torch.int64
assert not bbox_to_mask(boxes.clone().requires_grad_(), size=(3, 5)).requires_grad
# One deprecated call stays in the pin, because the opposite order is what #4014 was filed about.
with pytest.warns(DeprecationWarning, match=r"size=\(height, width\)"):
self.assert_close(bbox_to_mask(boxes, 5, 3), expected, atol=0.0, rtol=0.0)
def test_bbox_to_mask_size_matches_the_deprecated_pair_bitwise_4014(self, device, dtype):
# The repair must not move a single pixel: on a non-square size the new form has to be
# bitwise equal to what the deprecated pair produced before.
boxes = torch.tensor([[[1.0, 1.0], [4.0, 1.0], [4.0, 3.0], [1.0, 3.0]]], device=device, dtype=dtype)
with pytest.warns(DeprecationWarning, match=r"size=\(height, width\)"):
deprecated = bbox_to_mask(boxes, 7, 4) # width 7, height 4
canonical = bbox_to_mask(boxes, size=(4, 7))
assert canonical.shape == deprecated.shape == (1, 4, 7)
assert torch.equal(canonical, deprecated)
def test_bbox_to_mask_size_form_warns_nothing_4014(self, device, dtype):
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
with warnings.catch_warnings():
warnings.simplefilter("error")
bbox_to_mask(boxes, size=(3, 5))
@pytest.mark.parametrize("form", ["positional", "keyword"])
def test_bbox_to_mask_deprecated_forms_warn_and_agree_4014(self, device, dtype, form):
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
expected = bbox_to_mask(boxes, size=(3, 5))
with pytest.warns(DeprecationWarning, match=r"size=\(height, width\)") as record:
got = bbox_to_mask(boxes, 5, 3) if form == "positional" else bbox_to_mask(boxes, width=5, height=3)
assert torch.equal(got, expected)
# The warning names the caller's line, not a line inside kornia.
assert [w.filename for w in record] == [__file__]
def test_bbox_to_mask_rejects_both_forms_and_neither_4014(self, device, dtype):
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
with pytest.raises(ValueError, match="Pass either size=\\(height, width\\) or the deprecated width"):
bbox_to_mask(boxes, 5, 3, (3, 5))
with pytest.raises(ValueError, match="not both"):
bbox_to_mask(boxes, height=3, size=(3, 5))
with pytest.raises(TypeError, match="missing required argument 'size'"):
bbox_to_mask(boxes)
with pytest.raises(TypeError, match="missing required argument 'size'"):
bbox_to_mask(boxes, 5)
def test_bbox_to_mask_stays_scriptable_for_both_size_forms_4014(self, device, dtype):
# `size` is an ordinary fourth parameter, not a keyword-only one, precisely because
# torch.jit.script(bbox_to_mask) works today and TorchScript rejects keyword-only defaults.
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
scripted = torch.jit.script(bbox_to_mask)
assert torch.equal(scripted(boxes, size=(3, 5)), bbox_to_mask(boxes, size=(3, 5)))
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
assert torch.equal(scripted(boxes, 5, 3), bbox_to_mask(boxes, 5, 3))
def test_dynamo_bbox_to_mask_is_fullgraph_for_both_forms_4014(self, device, dtype, torch_optimizer):
boxes = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
eager = bbox_to_mask(boxes, size=(3, 5))
assert torch.equal(torch_optimizer(bbox_to_mask, fullgraph=True)(boxes, size=(3, 5)), eager)
# The deprecated path must compile without a graph break too: the warning is suppressed
# while capturing, so nothing here raises a Python-level call into `warnings`.
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
assert torch.equal(torch_optimizer(bbox_to_mask, fullgraph=True)(boxes, 5, 3), eager)
def test_wart_bbox_to_mask_compares_raw_floats_inclusively_4015(self, device, dtype):
# Wart pin for kornia#4015: the fractional box [1.4, 3.0] x [2.0, 2.6] covers the integer
# columns 2..3 of row 2 under inclusive raw-float comparison, so the integer edges x=3 and
# y=2 are inside. The [1.4, 3.6] x [1.4, 2.6] box in test_boxes.py covers two pixels here
# and twelve under Boxes.to_mask's rounding of the exclusive export.
boxes = torch.tensor([[[1.4, 2.0], [3.0, 2.0], [3.0, 2.6], [1.4, 2.6]]], device=device, dtype=dtype)
expected = torch.zeros(1, 5, 6, device=device, dtype=dtype)
expected[0, 2, 2:4] = 1.0
self.assert_close(bbox_to_mask(boxes, size=(5, 6)), expected, atol=0.0, rtol=0.0)
fractional = torch.tensor([[[1.4, 1.4], [3.6, 1.4], [3.6, 2.6], [1.4, 2.6]]], device=device, dtype=dtype)
assert bbox_to_mask(fractional, size=(5, 6)).sum().item() == 2.0
@pytest.mark.parametrize("half_dtype", [torch.float16, torch.bfloat16])
def test_bbox_to_mask_position_grid_is_exact_for_half_boxes_on_a_large_image(self, device, half_dtype):
# float16/bfloat16 can only represent consecutive integers exactly up to 2048 px /
# 256 px; beyond that, adjacent pixel positions collapse onto the same value.
# bbox_to_mask's position grid used to be built at boxes.dtype, so a half-precision
# `boxes` on an image past that threshold silently mismasked rows/columns near the
# collapse. Compare against the float32 result for the same region on a 4096x4096
# image -- well past the threshold on both axes for either half dtype. Coordinates
# 1000 / 3072 / 4080 are exactly representable in both half dtypes, so the half box
# covers exactly the same rectangle as the float32 one.
boxes32 = torch.tensor(
[[[1000.0, 1000.0], [4080.0, 1000.0], [4080.0, 3072.0], [1000.0, 3072.0]]], device=device
)
boxes_half = boxes32.to(half_dtype)
mask32 = bbox_to_mask(boxes32, size=(4096, 4096))
mask_half = bbox_to_mask(boxes_half, size=(4096, 4096))
assert mask_half.dtype == half_dtype
assert torch.equal(mask_half.bool(), mask32.bool())
def test_convention_bbox_generator_far_corner_is_start_plus_size_minus_one_3934(self, device, dtype):
# Convention pin (kornia#3934 tracks the inclusive arithmetic): width 3 from x=1 places
# the right edge at x=3, which infer_bbox_shape reads back as width 3. A scalar input is a
# batch of one, dtype and device are preserved, mixed dtypes raise, and the output keeps a
# gradient path to its inputs.
start = torch.tensor([1.0], device=device, dtype=dtype)
size = torch.tensor([3.0], device=device, dtype=dtype)
boxes = bbox_generator(start, start, size, size)
expected = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 3.0], [1.0, 3.0]]], device=device, dtype=dtype)
self.assert_close(boxes, expected, atol=0.0, rtol=0.0)
heights, widths = infer_bbox_shape(boxes)
self.assert_close(heights, size, atol=0.0, rtol=0.0)
self.assert_close(widths, size, atol=0.0, rtol=0.0)
assert boxes.device == start.device
self.assert_close(bbox_generator(start[0], start[0], size[0], size[0]), expected, atol=0.0, rtol=0.0)
assert bbox_generator(*(t.clone().requires_grad_() for t in (start, start, size, size))).requires_grad
other = torch.float32 if dtype == torch.float16 else torch.float16
with pytest.raises(AssertionError, match="same dtype"):
bbox_generator(start, start, size.to(other), size)
class TestTransformBoxes2D(BaseTester):
def test_convention_transform_bbox_restores_vector_endpoints(self, device, dtype):
# Under a horizontal flip, restoration sorts xyxy coordinates while
# polygon vertices retain their transformed cyclic order.
matrix = torch.tensor([[[-1.0, 0.0, 10.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
vector = torch.tensor([[1.0, 1.0, 3.0, 2.0]], device=device, dtype=dtype)
polygon = torch.tensor([[[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]]], device=device, dtype=dtype)
self.assert_close(
transform_bbox(matrix, vector), torch.tensor([[7.0, 1.0, 9.0, 2.0]], device=device, dtype=dtype)
)
self.assert_close(
transform_bbox(matrix, polygon),
torch.tensor([[[[9.0, 1.0], [7.0, 1.0], [7.0, 2.0], [9.0, 2.0]]]], device=device, dtype=dtype),
)
def test_transform_boxes(self, device, dtype):
boxes = torch.tensor([[139.2640, 103.0150, 397.3120, 410.5225]], device=device, dtype=dtype)
expected = torch.tensor([[114.6880, 103.0150, 372.7360, 410.5225]], device=device, dtype=dtype)
trans_mat = torch.tensor([[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
out = transform_bbox(trans_mat, boxes, restore_coordinates=True)
self.assert_close(out, expected, atol=1e-4, rtol=1e-4)
def test_transform_multiple_boxes(self, device, dtype):
boxes = torch.tensor(
[
[139.2640, 103.0150, 397.3120, 410.5225],
[1.0240, 80.5547, 512.0000, 512.0000],
[165.2053, 262.1440, 510.6347, 508.9280],
[119.8080, 144.2067, 257.0240, 410.1292],
],
device=device,
dtype=dtype,
)
boxes = boxes.repeat(2, 1, 1) # 2 x 4 x 4 two images 4 boxes each
expected = torch.tensor(
[
[
[114.6880, 103.0150, 372.7360, 410.5225],
[0.0000, 80.5547, 510.9760, 512.0000],
[1.3652, 262.1440, 346.7947, 508.9280],
[254.9760, 144.2067, 392.1920, 410.1292],
],
[
[139.2640, 103.0150, 397.3120, 410.5225],
[1.0240, 80.5547, 512.0000, 512.0000],
[165.2053, 262.1440, 510.6347, 508.9280],
[119.8080, 144.2067, 257.0240, 410.1292],
],
],
device=device,
dtype=dtype,
)
trans_mat = torch.tensor(
[
[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]],
[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]],
],
device=device,
dtype=dtype,
)
out = transform_bbox(trans_mat, boxes, restore_coordinates=True)
self.assert_close(out, expected, atol=1e-4, rtol=1e-4)
@pytest.mark.parametrize(
"matrix_batch,boxes_shape",
[
(1, (0, 4)),
(0, (0, 4)),
(0, (0, 5, 4)),
(1, (0, 5, 4)),
(2, (2, 0, 4)),
(0, (0, 5, 4, 2)),
(1, (0, 5, 4, 2)),
],
)
@pytest.mark.parametrize("mode", ["xyxy", "xywh"])
@pytest.mark.parametrize("restore_coordinates", [True, False])
def test_empty_boxes(self, matrix_batch, boxes_shape, mode, restore_coordinates, device, dtype):
# view(B, -1, 2) cannot infer -1 for zero elements, so zero boxes or an empty batch raised in reshape.
if mode == "xywh" and len(boxes_shape) == 4:
pytest.skip("polygons are given as vertices, not as xywh")
trans_mat = torch.eye(3, device=device, dtype=dtype).expand(matrix_batch, 3, 3)
boxes = torch.empty(boxes_shape, device=device, dtype=dtype, requires_grad=True)
out = transform_bbox(trans_mat, boxes, mode=mode, restore_coordinates=restore_coordinates)
assert out.shape == boxes.shape
assert out.device == boxes.device
assert out.dtype == dtype
out.sum().backward()
assert boxes.grad is not None
assert boxes.grad.shape == boxes.shape
@pytest.mark.device_agnostic
@pytest.mark.parametrize("boxes_shape", [(3, 4), (2, 3, 4), (2, 3, 4, 2)])
def test_legacy_onnx_dynamic_axes_match_eager(self, boxes_shape, dtype):
# The traced box count must stay dynamic: an export at one size has to run at another.
if dtype != torch.float32:
pytest.skip("the exported graph is checked once, in float32")
pytest.importorskip("onnx")
ort = pytest.importorskip("onnxruntime")
def sample(batch, num_boxes):
shape = (num_boxes, *boxes_shape[1:]) if len(boxes_shape) == 2 else (batch, num_boxes, *boxes_shape[2:])
boxes = torch.arange(math.prod(shape), dtype=dtype).reshape(shape) % 7
trans_mat = torch.tensor([[-1.0, 0.1, 9.0], [0.2, 1.0, -2.0], [0.0, 0.0, 1.0]], dtype=dtype)
return trans_mat.repeat(num_boxes if len(shape) == 2 else batch, 1, 1), boxes
class _TransformBbox(torch.nn.Module):
def forward(self, trans_mat: torch.Tensor, boxes: torch.Tensor) -> torch.Tensor:
return transform_bbox(trans_mat, boxes)
model = _TransformBbox()
if len(boxes_shape) == 2:
axes = {"trans_mat": {0: "num_boxes"}, "boxes": {0: "num_boxes"}}
else:
axes = {"trans_mat": {0: "batch"}, "boxes": {0: "batch", 1: "num_boxes"}}
buf = io.BytesIO()
with warnings.catch_warnings():
warnings.simplefilter("ignore")
torch.onnx.export(
model,
sample(2, 3),
buf, # ty: ignore[invalid-argument-type] # The legacy exporter accepts BytesIO.
input_names=["trans_mat", "boxes"],
opset_version=18,
dynamic_axes=axes,
dynamo=False,
)
session = ort.InferenceSession(buf.getvalue(), providers=["CPUExecutionProvider"])
for trans_mat, boxes in (sample(2, 3), sample(4, 6)):
actual = session.run(None, {"trans_mat": trans_mat.numpy(), "boxes": boxes.numpy()})[0]
self.assert_close(torch.from_numpy(actual), model(trans_mat, boxes))
def test_transform_boxes_wh(self, device, dtype):
boxes = torch.tensor(
[
[139.2640, 103.0150, 258.0480, 307.5075],
[1.0240, 80.5547, 510.9760, 431.4453],
[165.2053, 262.1440, 345.4293, 246.7840],
[119.8080, 144.2067, 137.2160, 265.9225],
],
device=device,
dtype=dtype,
)
boxes_before = boxes.clone()
expected = torch.tensor(
[
[114.6880, 103.0150, 258.0480, 307.5075],
[0.0000, 80.5547, 510.9760, 431.4453],
[1.3654, 262.1440, 345.4293, 246.7840],
[254.9760, 144.2067, 137.2160, 265.9225],
],
device=device,
dtype=dtype,
)
trans_mat = torch.tensor([[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
out = transform_bbox(trans_mat, boxes, mode="xywh", restore_coordinates=True)
self.assert_close(out, expected, atol=1e-4, rtol=1e-4)
assert torch.equal(boxes, boxes_before)
def test_gradcheck(self, device):
boxes = torch.tensor(
[
[139.2640, 103.0150, 258.0480, 307.5075],
[1.0240, 80.5547, 510.9760, 431.4453],
[165.2053, 262.1440, 345.4293, 246.7840],
[119.8080, 144.2067, 137.2160, 265.9225],
],
device=device,
dtype=torch.float64,
)
trans_mat = torch.tensor(
[[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=torch.float64
)
self.gradcheck(transform_bbox, (trans_mat, boxes, "xyxy", True))
def test_dynamo(self, device, dtype, torch_optimizer):
boxes = torch.tensor([[139.2640, 103.0150, 258.0480, 307.5075]], device=device, dtype=dtype)
trans_mat = torch.tensor([[[-1.0, 0.0, 512.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
args = (boxes, trans_mat)
op = kornia.geometry.transform_points
op_optimized = torch_optimizer(op)
self.assert_close(op(*args), op_optimized(*args))
class TestBbox3D(BaseTester):
def test_convention_validate_bbox3d_rejects_non_finite_coordinates_4258(self, device, dtype):
# kornia#4258, the 3D counterpart of #4238: a non-finite coordinate is a False predicate result, not an
# AssertionError.
boxes = torch.tensor(
[
[
[0.0, 0.0, 0.0],
[4.0, 0.0, 0.0],
[4.0, 4.0, 0.0],
[0.0, 4.0, 0.0],
[0.0, 0.0, 4.0],
[4.0, 0.0, 4.0],
[4.0, 4.0, 4.0],
[0.0, 4.0, 4.0],
]
],
device=device,
dtype=dtype,
)
assert validate_bbox3d(boxes) is True
for coordinate_index in range(24):
for non_finite in [float("nan"), float("inf"), float("-inf")]:
invalid_boxes = boxes.clone()
invalid_boxes[0].reshape(-1)[coordinate_index] = non_finite
with pytest.warns(FutureWarning, match="validate_bbox3d"):
assert validate_bbox3d(invalid_boxes) is False
# the same holds for the (B, N, 8, 3) rank-4 layout
with pytest.warns(FutureWarning, match="validate_bbox3d"):
assert validate_bbox3d(invalid_boxes[None]) is False
def test_convention_consumers_still_raise_on_non_finite_coordinates_4258(self, device, dtype):
# validate_bbox3d's four internal callers use it for its raise and discard the return value, so each
# raises on a non-finite box itself.
box = torch.tensor(
[
[
[0.0, 0.0, 0.0],
[4.0, 0.0, 0.0],
[4.0, 4.0, 0.0],
[0.0, 4.0, 0.0],
[0.0, 0.0, 4.0],
[4.0, 0.0, 4.0],
[4.0, 4.0, 4.0],
[0.0, 4.0, 4.0],
]
],
device=device,
dtype=dtype,
)
invalid = box.clone()
invalid[0, 7, 1] = float("nan")
# The callers raise themselves, so validate_bbox3d's deprecation warning (kornia#4013) is not theirs to
# emit: a FutureWarning here would surface as an error instead of the AssertionError.
warnings.simplefilter("error", FutureWarning)
with pytest.raises(AssertionError, match="finite"):
infer_bbox_shape3d(invalid)
with pytest.raises(AssertionError, match="finite"):
bbox_to_mask3d(invalid, (6, 6, 6))
# The other two callers are the src_box and dst_box of _crop_by_boxes3d_to_size. crop_by_boxes3d sizes
# its output from the dst box, so a non-finite src box reaches only that guard.
volume = torch.arange(64, device=device, dtype=dtype).reshape(1, 1, 4, 4, 4)
with pytest.raises(AssertionError, match="finite"):
crop_by_boxes3d(volume, invalid, box)
with pytest.raises(AssertionError, match="finite"):
crop_and_resize3d(volume, invalid, (2, 2, 2))
# the valid box is untouched
assert len(infer_bbox_shape3d(box)) == 3
assert bbox_to_mask3d(box, (6, 6, 6)).shape == (1, 1, 6, 6, 6)
assert crop_by_boxes3d(volume, box, box).shape == (1, 1, 5, 5, 5)
assert crop_and_resize3d(volume, box, (2, 2, 2)).shape == (1, 1, 2, 2, 2)
def test_generator_scalar_inputs(self, device, dtype):
args = [torch.tensor(value, device=device, dtype=dtype) for value in (1, 2, 3, 4, 5, 6)]
boxes = bbox_generator3d(*args)
assert boxes.shape == (1, 8, 3)
self.assert_close(boxes[0, 0], torch.tensor([1, 2, 3], device=device, dtype=dtype))
self.assert_close(boxes[0, 6], torch.tensor([4, 6, 8], device=device, dtype=dtype))
def test_smoke(self, device, dtype):
# Sample two points of the 3d rect
points = torch.rand(1, 6, device=device, dtype=dtype)
# Fill according missing points
bbox = torch.zeros(1, 8, 3, device=device, dtype=dtype)
bbox[0, 0] = points[0][:3]
bbox[0, 1, 0] = points[0][3]
bbox[0, 1, 1] = points[0][1]
bbox[0, 1, 2] = points[0][2]
bbox[0, 2, 0] = points[0][3]
bbox[0, 2, 1] = points[0][4]
bbox[0, 2, 2] = points[0][2]
bbox[0, 3, 0] = points[0][0]
bbox[0, 3, 1] = points[0][4]
bbox[0, 3, 2] = points[0][2]
bbox[0, 4, 0] = points[0][0]
bbox[0, 4, 1] = points[0][1]
bbox[0, 4, 2] = points[0][5]
bbox[0, 5, 0] = points[0][3]
bbox[0, 5, 1] = points[0][1]
bbox[0, 5, 2] = points[0][5]
bbox[0, 6] = points[0][3:]
bbox[0, 7, 0] = points[0][0]
bbox[0, 7, 1] = points[0][4]
bbox[0, 7, 2] = points[0][5]
# Validate
assert validate_bbox3d(bbox)
def test_convention_validate_bbox3d_accepts_noncontiguous_rank4_layout_4174(self, device, dtype):
# kornia#4174: a rank-4 input whose leading dimensions cannot be merged by view returns a boolean.
boxes = torch.zeros(2, 3, 8, 3, device=device, dtype=dtype).transpose(0, 1)
assert not boxes.is_contiguous()
assert validate_bbox3d(boxes) is True
def test_bounding_boxes_dim_inferring(self, device, dtype):
boxes = torch.tensor(
[
[[0, 1, 2], [10, 1, 2], [10, 21, 2], [0, 21, 2], [0, 1, 32], [10, 1, 32], [10, 21, 32], [0, 21, 32]],
[[3, 4, 5], [43, 4, 5], [43, 54, 5], [3, 54, 5], [3, 4, 65], [43, 4, 65], [43, 54, 65], [3, 54, 65]],
],
device=device,
dtype=dtype,
) # 2x8x3
d, h, w = infer_bbox_shape3d(boxes)
self.assert_close(d, torch.tensor([31.0, 61.0], device=device, dtype=dtype))
self.assert_close(h, torch.tensor([21.0, 51.0], device=device, dtype=dtype))
self.assert_close(w, torch.tensor([11.0, 41.0], device=device, dtype=dtype))
def test_gradcheck(self, device):
boxes = torch.tensor(
[
[
[0.0, 1.0, 2.0],
[10, 1, 2],
[10, 21, 2],
[0, 21, 2],
[0, 1, 32],
[10, 1, 32],
[10, 21, 32],
[0, 21, 32],
]
],
device=device,
dtype=torch.float64,
)
self.gradcheck(infer_bbox_shape3d, (boxes,))
def test_dynamo(self, device, dtype, torch_optimizer):
# Define script
op = infer_bbox_shape3d
op_script = torch_optimizer(op)
boxes = torch.tensor(
[[[0, 0, 1], [3, 0, 1], [3, 2, 1], [0, 2, 1], [0, 0, 3], [3, 0, 3], [3, 2, 3], [0, 2, 3]]],
device=device,
dtype=dtype,
) # 1x8x3
actual = op_script(boxes)
expected = op(boxes)
self.assert_close(actual, expected)
@staticmethod
def _unit_cuboid(device, dtype) -> torch.Tensor:
# Inclusive vertices for x in 1..3, y in 1..3, z in 1..2: width 3, height 3, depth 2.
return torch.tensor(
[
[
[1.0, 1.0, 1.0],
[3.0, 1.0, 1.0],
[3.0, 3.0, 1.0],
[1.0, 3.0, 1.0],
[1.0, 1.0, 2.0],
[3.0, 1.0, 2.0],
[3.0, 3.0, 2.0],
[1.0, 3.0, 2.0],
]
],
device=device,
dtype=dtype,
)
def test_wart_validate_bbox3d_checks_equal_edge_extents_not_cuboids_4013(self, device, dtype, monkeypatch):
# Wart pin for kornia#4013 (the 3D validator raises where the 2D one returns False, and its one
# False result, a non-finite coordinate, warns that it will raise too): the check compares
# the inclusive x extents of the four left-to-right edges, the y extents of the two diagonals
# of the front face and of the back face, and the z extents of the four front-to-back edges,
# so a sheared parallelepiped and a zero-extent box pass, and only disagreeing extents raise.
# Under graph capture the extent check is skipped and the shape check alone returns True.
cuboid = self._unit_cuboid(device, dtype)
assert validate_bbox3d(cuboid) is True
assert validate_bbox3d(cuboid[None]) is True
assert validate_bbox3d(torch.zeros(1, 8, 3, device=device, dtype=dtype)) is True
sheared = cuboid.clone()
sheared[0, 4:, 0] += 1.0
assert validate_bbox3d(sheared) is True
for extent, expected in zip(infer_bbox_shape3d(sheared), (2.0, 3.0, 3.0)):
self.assert_close(extent, torch.tensor([expected], device=device, dtype=dtype), atol=0.0, rtol=0.0)
unequal = cuboid.clone()
unequal[0, 1, 0] = 5.0
with warnings.catch_warnings():
warnings.simplefilter("error")
with pytest.raises(AssertionError, match="widths"):
validate_bbox3d(unequal)
with pytest.raises(AssertionError, match="shape"):
validate_bbox3d(cuboid[0])
nan_box = cuboid.clone()
nan_box[0, 5, 2] = float("nan")
with pytest.warns(FutureWarning, match="validate_bbox3d") as record:
assert validate_bbox3d(nan_box) is False
assert [w.filename for w in record] == [__file__]
monkeypatch.setattr(bbox_module, "is_compiling", lambda: True)
assert validate_bbox3d(unequal) is True
with pytest.raises(AssertionError, match="shape"):
validate_bbox3d(cuboid[0])
@pytest.mark.skipif(not dynamo_is_available(), reason=DYNAMO_UNAVAILABLE_REASON)
@pytest.mark.parametrize(
"op",
[lambda boxes: torch.stack(infer_bbox_shape3d(boxes)), lambda boxes: bbox_to_mask3d(boxes, (4, 5, 6))],
ids=["infer_bbox_shape3d", "bbox_to_mask3d"],
)
def test_value_checks_skipped_under_any_capture_5037(self, op, device, dtype):
# validate_bbox3d checks the edge extents on the host, which a fullgraph trace cannot do, so any capture skips
# the check, not only export (#5037). Eager still raises on unequal edges; a traced call does not.
from torch._dynamo.testing import CompileCounter
cuboid = self._unit_cuboid(device, dtype)
unequal = cuboid.clone()
unequal[0, 1, 0] = 5.0
counter = CompileCounter()
torch._dynamo.reset()
traced = torch.compile(op, backend=counter, fullgraph=True)
self.assert_close(traced(cuboid), op(cuboid), atol=0.0, rtol=0.0)
assert counter.frame_count == 1
with pytest.raises(AssertionError, match="widths"):
op(unequal)
assert traced(unequal).shape == op(cuboid).shape
def test_validate_bbox3d_scripted_keeps_the_value_checks_5037(self, device, dtype):
# TorchScript cannot call is_compiling(); a scripted validate_bbox3d keeps the value checks, as eager does.
cuboid = self._unit_cuboid(device, dtype)
unequal = cuboid.clone()
unequal[0, 1, 0] = 5.0
scripted = torch.jit.script(validate_bbox3d)
assert scripted(cuboid) is True
with pytest.raises(torch.jit.Error, match="widths"):
scripted(unequal)
def test_convention_infer_bbox_shape3d_returns_depth_height_width_inclusive_3934(self, device, dtype):
# Convention pin (kornia#3934 tracks the inclusive arithmetic): the tuple order is
# (depths, heights, widths) and each extent is one larger than its coordinate span.
boxes = torch.tensor(
[
[
[1.0, 2.0, 3.0],
[4.0, 2.0, 3.0],
[4.0, 4.0, 3.0],
[1.0, 4.0, 3.0],
[1.0, 2.0, 7.0],
[4.0, 2.0, 7.0],
[4.0, 4.0, 7.0],
[1.0, 4.0, 7.0],
]
],
device=device,
dtype=dtype,
)
depths, heights, widths = infer_bbox_shape3d(boxes)
self.assert_close(depths, torch.tensor([5.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
self.assert_close(heights, torch.tensor([3.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
self.assert_close(widths, torch.tensor([4.0], device=device, dtype=dtype), atol=0.0, rtol=0.0)
@pytest.mark.parametrize("num_boxes", [1, 8])
def test_convention_rank4_input_is_rejected_by_the_3d_helpers_4248(self, device, dtype, num_boxes):
# kornia#4248, the 3D twin of kornia#4180: validate_bbox3d accepts (B, N, 8, 3), while infer_bbox_shape3d
# and bbox_to_mask3d reject it with a ShapeError.
boxes = self._unit_cuboid(device, dtype).expand(num_boxes, 8, 3)[None].contiguous()
assert validate_bbox3d(boxes) is True
for call in (lambda: infer_bbox_shape3d(boxes), lambda: bbox_to_mask3d(boxes, (4, 5, 5))):
with pytest.raises(ShapeError):
call()
def test_bbox_generator3d_matches_bbox_generator_extent_4018(self, device, dtype):
# kornia#4018: 3D sizes are inclusive, just like the 2D generator. A size of 3 from x=1 spans x=1..3.
start = torch.tensor([1.0], device=device, dtype=dtype)
size = torch.tensor([3.0], device=device, dtype=dtype)
boxes = bbox_generator3d(start, start, start, size, size, size)
expected_x = torch.tensor([[1.0, 3.0, 3.0, 1.0, 1.0, 3.0, 3.0, 1.0]], device=device, dtype=dtype)
self.assert_close(boxes[..., 0], expected_x, atol=0.0, rtol=0.0)
for extent in infer_bbox_shape3d(boxes):
self.assert_close(extent, size, atol=0.0, rtol=0.0)
for extent in infer_bbox_shape(bbox_generator(start, start, size, size)):
self.assert_close(extent, size, atol=0.0, rtol=0.0)
def test_wart_bbox_to_mask3d_truncates_fractional_coordinates_4015(self, device, dtype):
# Wart pin for kornia#4015: x in [1.6, 3.4] truncates to 1..3 and y, z in [1.6, 2.4] to
# 1..2, filled inclusively: twelve voxels, where Boxes3D.to_mask fills two (test_boxes.py).
boxes = torch.tensor(
[
[
[1.6, 1.6, 1.6],
[3.4, 1.6, 1.6],
[3.4, 2.4, 1.6],
[1.6, 2.4, 1.6],
[1.6, 1.6, 2.4],
[3.4, 1.6, 2.4],
[3.4, 2.4, 2.4],
[1.6, 2.4, 2.4],
]
],
device=device,
dtype=dtype,
)
expected = torch.zeros(1, 1, 5, 5, 6, device=device, dtype=dtype)
expected[0, 0, 1:3, 1:3, 1:4] = 1.0
self.assert_close(bbox_to_mask3d(boxes, (5, 5, 6)), expected, atol=0.0, rtol=0.0)
def test_bbox_to_mask3d_preserves_input_dtype_4250(self, device, dtype):
# kornia#4250: bbox_to_mask3d returns a (B, 1, D, H, W) mask in the input dtype, like bbox_to_mask (which
# has no channel axis). Neither carries a gradient.
cuboid = self._unit_cuboid(device, dtype)
mask = bbox_to_mask3d(cuboid, (4, 5, 6))
assert mask.shape == (1, 1, 4, 5, 6)
assert mask.dtype == dtype
assert bbox_to_mask3d(cuboid.repeat(2, 1, 1), (4, 5, 6)).shape == (2, 1, 4, 5, 6)
assert bbox_to_mask3d(cuboid.to(torch.int64), (4, 5, 6)).dtype == torch.int64
assert not bbox_to_mask3d(cuboid.clone().requires_grad_(), (4, 5, 6)).requires_grad
square = torch.tensor([[[1.0, 1.0], [3.0, 1.0], [3.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype)
assert bbox_to_mask(square, size=(5, 6)).dtype == dtype
@pytest.mark.parametrize("case", ["full_width", "full_height", "full_depth", "overhang"])
def test_convention_bbox_to_mask3d_intersects_the_axis_ranges_for_a_full_or_overhanging_axis_box_4255(
self, device, dtype, case
):
# kornia#4255: a box whose slab covers (or overhangs) a whole axis of the (4, 4, 5) volume fills the
# intersection of the three axis ranges, exactly as Boxes3D.to_mask does. The interior box is a control.
xyzxyz_plus, region = {
"full_width": ([0.0, 1.0, 1.0, 4.0, 2.0, 2.0], (slice(1, 3), slice(1, 3), slice(None))),
"full_height": ([1.0, 0.0, 1.0, 2.0, 3.0, 2.0], (slice(1, 3), slice(None), slice(1, 3))),
"full_depth": ([1.0, 1.0, 0.0, 2.0, 2.0, 3.0], (slice(None), slice(1, 3), slice(1, 3))),
"overhang": ([-1.0, 1.0, 1.0, 5.0, 2.0, 2.0], (slice(1, 3), slice(1, 3), slice(None))),
}[case]
boxes = Boxes3D.from_tensor(torch.tensor([xyzxyz_plus], device=device, dtype=dtype), mode="xyzxyz_plus")
expected = torch.zeros(1, 1, 4, 4, 5, device=device, dtype=dtype)
expected[(0, 0, *region)] = 1.0
self.assert_close(bbox_to_mask3d(boxes.data, (4, 4, 5)), expected, atol=0.0, rtol=0.0)
self.assert_close(boxes.to_mask(4, 4, 5).to(dtype), expected[:, 0], atol=0.0, rtol=0.0)
interior = Boxes3D.from_tensor(
torch.tensor([[1.0, 1.0, 1.0, 2.0, 2.0, 2.0]], device=device, dtype=dtype), mode="xyzxyz_plus"
)
assert bbox_to_mask3d(interior.data, (4, 4, 5)).sum().item() == interior.to_mask(4, 4, 5).sum().item() == 8.0
class TestNMS(BaseTester):
@pytest.mark.parametrize(
("side", "offset", "expected"),
[(200, 0, [0, 2]), (512, 128, [0, 1, 2])],
)
def test_image_sized_boxes(self, device, dtype, side, offset, expected):
# Identical boxes have IoU 1; offset 512-pixel boxes have IoU 9/23.
boxes = torch.tensor(
[[0, 0, side, side], [offset, offset, side + offset, side + offset], [600, 0, 640, 40]],
device=device,
dtype=dtype,
)
scores = torch.tensor([0.875, 0.75, 0.5], device=device, dtype=dtype)
original = boxes.clone()
expected = torch.tensor(expected, device=device, dtype=torch.long)
for _ in range(2):
self.assert_close(nms(boxes, scores, 0.5), expected, atol=0, rtol=0)
self.assert_close(boxes, original, atol=0, rtol=0)
@pytest.mark.parametrize(
("box_dtype", "side", "origin"),
[
(torch.uint8, 100, 0),
(torch.int8, 100, -100),
(torch.int16, 200, 0),
(torch.int32, 200, 2**24),
(torch.int32, 50_000, 0), # each area exceeds 2**31 - 1
(torch.int64, 200, 2**40),
],
)
def test_integer_boxes(self, device, box_dtype, side, origin):
boxes = torch.tensor(
[[origin, origin, origin + side, origin + side]] * 2
+ [[origin + side - 20, origin + side - 20, origin + side - 10, origin + side - 10]],
device=device,
dtype=box_dtype,
)
scores = torch.tensor([0.875, 0.75, 0.5], device=device)
expected = torch.tensor([0, 2], device=device, dtype=torch.long)
self.assert_close(nms(boxes, scores, 0.5), expected, atol=0, rtol=0)
def test_iou_just_above_threshold(self, device, dtype):
# The exact IoU is 8479 / 16793 = 0.5049, so 0.5 suppresses the second box. Computed in bfloat16, the areas,
# intersection and union round to an IoU of exactly 0.5, which kept it. The coordinates are exact in any dtype.
boxes = torch.tensor([[0, 0, 153, 162], [14, 40, 173, 202]], device=device, dtype=dtype)
scores = torch.tensor([0.875, 0.75], device=device, dtype=dtype)
expected = torch.tensor([0], device=device, dtype=torch.long)
self.assert_close(nms(boxes, scores, 0.5), expected, atol=0, rtol=0)
@pytest.mark.parametrize(
"boxes",
[
[[10, 10, 10, 20], [100, 100, 100, 200]], # zero width, far apart
[[10, 10, 20, 10], [10, 100, 20, 100]], # zero height, far apart
[[10, 10, 10, 10], [50, 50, 50, 50]], # two points
[[10, 10, 10, 20], [10, 10, 10, 20]], # identical zero-width boxes
],
)
@pytest.mark.parametrize("iou_threshold", [0.0, 0.5])
def test_zero_area_boxes_have_zero_iou(self, boxes, iou_threshold, device, dtype):
# The intersection and the union of two zero-area boxes are both 0. Their IoU is 0, as in torchvision, not
# 0 / 0 = NaN, which compared as an overlap and suppressed the lower-scored box. At a threshold of 0, any
# positive IoU would suppress it too.
boxes = torch.tensor(boxes, device=device, dtype=dtype)
scores = torch.tensor([0.875, 0.75], device=device, dtype=dtype)
expected = torch.tensor([0, 1], device=device, dtype=torch.long)
self.assert_close(nms(boxes, scores, iou_threshold), expected, atol=0, rtol=0)
self.assert_close(nms(boxes.to(torch.int32), scores, iou_threshold), expected, atol=0, rtol=0)
@pytest.mark.parametrize("iou_threshold", [0.0, 0.5])
def test_lower_scored_nan_box_does_not_suppress_or_get_suppressed(self, iou_threshold, device, dtype):
# A NaN IoU is not above the threshold, so the lower-scored NaN box is kept. It does not suppress any box.
boxes = torch.tensor([[0, 0, 10, 10], [100, 100, float("nan"), 110]], device=device, dtype=dtype)
scores = torch.tensor([0.875, 0.75], device=device, dtype=dtype)
expected = torch.tensor([0, 1], device=device, dtype=torch.long)
self.assert_close(nms(boxes, scores, iou_threshold), expected, atol=0, rtol=0)
@pytest.mark.parametrize("iou_threshold", [0.0, 0.5])
def test_highest_scored_nan_box_does_not_suppress_valid_boxes(self, iou_threshold, device, dtype):
boxes = torch.tensor(
[[100, 100, float("nan"), 110], [0, 0, 10, 10], [200, 200, 210, 210]], device=device, dtype=dtype
)
scores = torch.tensor([0.875, 0.75, 0.625], device=device, dtype=dtype)
expected = torch.tensor([0, 1, 2], device=device, dtype=torch.long)
self.assert_close(nms(boxes, scores, iou_threshold), expected, atol=0, rtol=0)
def test_zero_area_box_next_to_a_box(self, device, dtype):
# A zero-width box inside a box with a positive area has IoU 0 with it, and a box with a positive area still
# suppresses its duplicate. The last box, a point, is then compared with the zero-width box: their union is
# empty, so their IoU is 0 and the point is kept (before #6361 the 0 / 0 suppressed it).
boxes = torch.tensor(
[[0, 0, 20, 20], [10, 5, 10, 15], [0, 0, 20, 20], [40, 40, 40, 40]], device=device, dtype=dtype
)
scores = torch.tensor([0.875, 0.75, 0.625, 0.5], device=device, dtype=dtype)
expected = torch.tensor([0, 1, 3], device=device, dtype=torch.long)
self.assert_close(nms(boxes, scores, 0.5), expected, atol=0, rtol=0)
def test_convention_nms_uses_exclusive_xyxy_iou_4008(self, device, dtype):
# The boxes overlap by 1 / 4 under exclusive xyxy arithmetic. At 0.3, NMS
# keeps both; inclusive corner arithmetic would compute 4 / 9 and suppress
# the lower-scored box. The literal also pins descending-score index order.
boxes = torch.tensor([[0.0, 0.0, 1.0, 1.0], [0.0, 0.0, 2.0, 2.0]], device=device, dtype=dtype)
scores = torch.tensor([0.8, 0.9], device=device, dtype=dtype)
actual = nms(boxes, scores, iou_threshold=0.3)
self.assert_close(actual, torch.tensor([1, 0], device=device, dtype=torch.long), atol=0.0, rtol=0.0)
def test_dynamo(self, device, dtype, torch_optimizer):
# A default torch.compile cannot capture the data-dependent loop in one graph and runs nms eagerly: the same
# indices, which keep the descending-score order.
boxes = torch.tensor(
[[10, 10, 20, 20], [15, 5, 15, 25], [100, 100, 150, 150], [100, 100, 150, 150]], device=device, dtype=dtype
)
scores = torch.tensor([0.9, 0.8, 0.7, 0.95], device=device, dtype=dtype)
expected = torch.tensor([3, 0, 1], device=device, dtype=torch.long)
self.assert_close(nms(boxes, scores, 0.5), expected, atol=0, rtol=0)
self.assert_close(torch_optimizer(nms)(boxes, scores, 0.5), expected, atol=0, rtol=0)
def test_empty(self, device):
boxes = torch.empty((0, 4), device=device)
scores = torch.empty((0,), device=device)
actual = nms(boxes, scores, iou_threshold=0.8)
assert actual.shape == (0,)
assert actual.dtype == torch.long
assert actual.device == boxes.device
def test_smoke(self, device, dtype):
boxes = torch.tensor(
[
[10.0, 10.0, 20.0, 20.0],
[15.0, 5.0, 15.0, 25.0],
[100.0, 100.0, 200.0, 200.0],
[100.0, 100.0, 200.0, 200.0],
],
device=device,
dtype=dtype,
)
scores = torch.tensor([0.9, 0.8, 0.7, 0.9], device=device, dtype=dtype)
expected = torch.tensor([0, 3, 1], device=device, dtype=torch.long)
actual = nms(boxes, scores, iou_threshold=0.8)
self.assert_close(actual, expected)
@pytest.mark.parametrize(
("boxes_shape", "scores_shape"),
[
((3, 5), (3,)),
((2, 3, 4), (2,)),
],
)
def test_invalid_boxes_shape(self, boxes_shape, scores_shape, device, dtype):
boxes = torch.zeros(boxes_shape, device=device, dtype=dtype)
scores = torch.zeros(scores_shape, device=device, dtype=dtype)
with pytest.raises(ValueError, match="boxes expected as Nx4"):
nms(boxes, scores, iou_threshold=0.8)