π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
fix(geometry): clip boxes left of or above the image in crop_by_indices (#6322)
Debanjan Sarkardcbad0b365e7
diff --git a/changelog.d/6322.fixed.md b/changelog.d/6322.fixed.mdnew file mode 100644index 000000000..5a6931580--- /dev/null+++ b/changelog.d/6322.fixed.md@@ -0,0 +1 @@+`kornia.geometry.transform.crop_by_indices` no longer reads pixels from the opposite edge for a box that starts left of or above the image. The part of the box outside the image is clipped, as past the right or bottom edge, and `shape_compensation` then applies; with `"pad"` the padding goes on the side where the box overflowed. The `torch.compile` path gives the same result. `CenterCrop2D` in its default slice mode now zero-pads a size larger than the input, as its resample mode does, instead of stretching the last row or column.diff --git a/kornia/geometry/transform/_crop.py b/kornia/geometry/transform/_crop.pyindex e077200e6..ac6f95835 100644--- a/kornia/geometry/transform/_crop.py+++ b/kornia/geometry/transform/_crop.py@@ -105,13 +105,13 @@ def _compiled_slice_resize(# Nearest normally uses float32 index arithmetic; handle ATen's CPU-double exception below.coordinate_dtype = torch.float64 if input.dtype == torch.float64 and mode != "nearest" else torch.float32src = src.to(device=input.device, dtype=torch.long)- # Match Python slicing for negative and out-of-bounds replay coordinates.+ # Clip out-of-bounds coordinates to the image, as crop_by_indices does: a negative start must not wrap (#6312).x0, x1 = src[:, 0, 0:1], src[:, 1, 0:1] + 1y0, y1 = src[:, 0, 1:2], src[:, 3, 1:2] + 1- x0 = torch.where(x0 < 0, x0 + width, x0).clamp(0, width).to(coordinate_dtype)- x1 = torch.where(x1 < 0, x1 + width, x1).clamp(0, width).to(coordinate_dtype)- y0 = torch.where(y0 < 0, y0 + height, y0).clamp(0, height).to(coordinate_dtype)- y1 = torch.where(y1 < 0, y1 + height, y1).clamp(0, height).to(coordinate_dtype)+ x0 = x0.clamp(0, width).to(coordinate_dtype)+ x1 = x1.clamp(0, width).to(coordinate_dtype)+ y0 = y0.clamp(0, height).to(coordinate_dtype)+ y1 = y1.clamp(0, height).to(coordinate_dtype)x = _resize_coordinates(x0, x1, width, size[1], mode, align_corners)y = _resize_coordinates(y0, y1, height, size[0], mode, align_corners)if mode == "nearest" and input.device.type == "cpu" and input.dtype == torch.float64 and sum(size) > 128:@@ -147,8 +147,10 @@ def _compiled_slice_resize(flat = work.reshape(batch, channels, height * width)def gather(x_index: torch.Tensor, y_index: torch.Tensor) -> torch.Tensor:- x_index = (x0 + x_index.clamp(min=0)).minimum(x1 - 1).long()- y_index = (y0 + y_index.clamp(min=0)).minimum(y1 - 1).long()+ # ``clamp(min=0)``: a box entirely left of or above the image clips to an empty interval, whose ``x1 - 1`` is+ # -1 and would read the previous row; it reads the image edge instead, as a box past the right edge does.+ x_index = (x0 + x_index.clamp(min=0)).minimum(x1 - 1).clamp(min=0).long()+ y_index = (y0 + y_index.clamp(min=0)).minimum(y1 - 1).clamp(min=0).long()indices = (y_index.unsqueeze(-1) * width + x_index.unsqueeze(-2)).reshape(batch, 1, -1)return flat.gather(2, indices.expand(-1, channels, -1)).reshape(batch, channels, *size)diff --git a/kornia/geometry/transform/crop2d.py b/kornia/geometry/transform/crop2d.pyindex 67b31a589..cb1e0087d 100644--- a/kornia/geometry/transform/crop2d.py+++ b/kornia/geometry/transform/crop2d.py@@ -471,6 +471,9 @@ def crop_by_indices(- ``shape_compensation`` (``'resize'`` by default) applies whenever the croppedslice does not match ``size``, whether or not ``src_box`` is identical acrossthe batch β each row's output depends only on its own box.+ - the part of a box outside the image is clipped on every side, a box that starts left of or above the+ image included, and ``shape_compensation`` then applies. With ``'pad'`` the padding goes on the side+ where the box overflowed, so every kept pixel stays at its position in the box... note::Under :func:`torch.compile` a non-empty batch with ``size`` given, ``'bilinear'``,@@ -550,6 +553,14 @@ def crop_by_indices(# stacked tensor) instead of a ``unique`` per column plus a device-to-host ``int(...)`` inside# every loop iteration β the coordinates index Python-level slicing, so they must be host ints.x1l, x2l, y1l, y2l = torch.stack([x1, x2, y1, y2], dim=0).tolist()+ # A negative index would wrap to the far edge, so the part of a box left of or above the image is clipped,+ # like the part past the right or bottom edge, which the slice clips. ``'pad'`` pads it back on that side.+ left = [max(-v, 0) for v in x1l]+ top = [max(-v, 0) for v in y1l]+ x1l, x2l = [max(v, 0) for v in x1l], [max(v, 0) for v in x2l]+ y1l, y2l = [max(v, 0) for v in y1l], [max(v, 0) for v in y2l]+ # With 'pad' a clipped box needs that padding even when its clipped slice happens to have the requested size.+ shifted = [shape_compensation != "resize" and (left[i] > 0 or top[i] > 0) for i in range(B)]# ``B > 0``: an empty batch has no first box to compare against, and takes the general path below,# which returns an empty ``(0, C, *size)`` tensor (#4429).@@ -559,9 +570,11 @@ def crop_by_indices(and x2l.count(x2l[0]) == Band y1l.count(y1l[0]) == Band y2l.count(y2l[0]) == B+ and left.count(left[0]) == B+ and top.count(top[0]) == B):out = input_tensor[..., y1l[0] : y2l[0], x1l[0] : x2l[0]]- if size is not None and out.shape[-2:] != size:+ if size is not None and (out.shape[-2:] != size or shifted[0]):if shape_compensation == "resize":return resize(out,@@ -571,7 +584,7 @@ def crop_by_indices(side="short",antialias=antialias,)- return F.pad(out, [0, size[1] - out.shape[-1], 0, size[0] - out.shape[-2]])+ return _pad_to_size(out, size, left[0], top[0])if size is None:h, w = infer_bbox_shape(src)@@ -596,12 +609,17 @@ def crop_by_indices(# ``contiguous`` keeps the result in the default memory format, as the ``torch.empty`` below does,# when the input is channels-last. MPS keeps the per-row copies: on an M1 the ``cat`` of strided# views ran at 0.75-0.96x their speed without gradients (B = 4-64, 256Β² -> 128Β², #4987).- if B > 0 and input_tensor.device.type != "mps" and all(crop.shape[-2:] == size for crop in slices):+ if (+ B > 0+ and input_tensor.device.type != "mps"+ and not any(shifted)+ and all(crop.shape[-2:] == size for crop in slices)+ ):return torch.cat(slices).contiguous()out = torch.empty(B, C, *size, device=input_tensor.device, dtype=input_tensor.dtype) if rows is None else Nonecrops: list[torch.Tensor] = []for i, _out in enumerate(slices):- if _out.shape[-2:] != size:+ if _out.shape[-2:] != size or shifted[i]:if shape_compensation == "resize":_out = resize(_out,@@ -612,7 +630,7 @@ def crop_by_indices(antialias=antialias,)else:- _out = F.pad(_out, [0, size[1] - _out.shape[-1], 0, size[0] - _out.shape[-2]])+ _out = _pad_to_size(_out, size, left[i], top[i])if out is None:crops.append(_out)else:@@ -622,6 +640,14 @@ def crop_by_indices(return out+def _pad_to_size(crop: torch.Tensor, size: Tuple[int, int], left: int, top: int) -> torch.Tensor:+ # ``left`` and ``top`` are the box columns and rows clipped before the image, so every kept pixel stays at its+ # position in the box; the rest of ``size`` is padded (or, for a larger slice, trimmed) at the right and bottom.+ # A box entirely outside the image clips more than ``size``: it is all padding, as past the right edge.+ left, top = min(left, size[1]), min(top, size[0])+ return F.pad(crop, [left, size[1] - crop.shape[-1] - left, top, size[0] - crop.shape[-2] - top])++def _crop_by_indices_export(input_tensor: torch.Tensor,src_box: torch.Tensor,@@ -749,6 +775,12 @@ class CenterCrop2D(nn.Module):)if self.flags["cropping_mode"] == "slice": # uses advanced slicing to crop+ if dst_h > src_h or dst_w > src_w:+ # A size larger than the input: zero-pad around the image, as the resample mode does, then slice.+ # The offsets are Python ints from the shapes, so this compiles with fullgraph=True.+ left, top = max(-start_x, 0), max(-start_y, 0)+ padded = F.pad(input, [left, max(end_x + 1 - src_w, 0), top, max(end_y + 1 - src_h, 0)])+ return padded[..., start_y + top : end_y + 1 + top, start_x + left : end_x + 1 + left].contiguous()return crop_by_indices(input, self.points_src.expand(batch_size, -1, -1).to(input), self.flags["size"])raise NotImplementedError(f"Not supported type: {self.flags['cropping_mode']}.")diff --git a/tests/geometry/transform/test_crop2d.py b/tests/geometry/transform/test_crop2d.pyindex f307af7eb..cced67193 100644--- a/tests/geometry/transform/test_crop2d.py+++ b/tests/geometry/transform/test_crop2d.py@@ -392,6 +392,28 @@ class TestCenterCrop(BaseTester):reflected = kornia.geometry.transform.center_crop(small, (4, 4), padding_mode="reflection", align_corners=False)self.assert_close(reflected, replicated)+ # A 4x5 input: overflow on the right only, on every side, with odd margins, and on one axis while the other crops.+ _larger_sizes_6312 = [(4, 6), (6, 6), (5, 7), (2, 7), (7, 3)]++ @pytest.mark.parametrize("size", _larger_sizes_6312)+ def test_center_crop2d_slice_larger_than_input_matches_resample_6312(self, size, device, dtype):+ # The slice mode wrapped the negative start and stretched the wrong pixels; it must zero-pad as resample does.+ if not supports_bilinear_2d_grid_sample(device, dtype):+ pytest.skip("bilinear 2D grid_sample is unavailable for this device and dtype")+ inp = torch.arange(1.0, 21.0, device=device, dtype=dtype).view(1, 1, 4, 5).repeat(2, 3, 1, 1)+ out_slice = kornia.geometry.transform.CenterCrop2D(size)(inp)+ out_resample = kornia.geometry.transform.CenterCrop2D(size, cropping_mode="resample")(inp)+ assert out_slice.shape == (2, 3, *size)+ self.assert_close(out_slice, out_resample)++ @pytest.mark.parametrize("size", _larger_sizes_6312)+ def test_dynamo_center_crop2d_slice_larger_than_input_6312(self, size, device, dtype, torch_optimizer):+ if not supports_bilinear_2d_grid_sample(device, dtype):+ pytest.skip("bilinear 2D grid_sample is unavailable for this device and dtype")+ inp = torch.arange(1.0, 21.0, device=device, dtype=dtype).view(1, 1, 4, 5)+ out = torch_optimizer(kornia.geometry.transform.CenterCrop2D(size), fullgraph=True)(inp)+ self.assert_close(out, kornia.geometry.transform.CenterCrop2D(size, cropping_mode="resample")(inp))+class TestCropByBoxes(BaseTester):def test_crop_by_boxes_no_resizing(self, device, dtype):@@ -958,6 +980,99 @@ class TestCropByIndices(BaseTester):kornia.geometry.transform.crop_by_indices, (inp, src_box, (3, 4)), requires_grad=(True, False, False))+ # (x1, y1, x2, y2) boxes on a 5x6 image: left of it, above it, and both (#6312).+ _outside_boxes_6312 = {"left": (-2, 1, 3, 3), "above": (1, -1, 4, 2), "left_and_above": (-1, -2, 2, 1)}++ @staticmethod+ def _box_6312(x1, y1, x2, y2, batch, device, dtype):+ box = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]+ return torch.tensor([box] * batch, device=device, dtype=dtype)++ @pytest.mark.parametrize("where", ["left", "above", "left_and_above"])+ def test_box_outside_the_image_pads_in_place_6312(self, where, device, dtype):+ # A negative start used to wrap to the far edge of the axis. With 'pad' the box must give what+ # crop_by_boxes gives: the image pixels at their positions in the box and zeros outside the image.+ x1, y1, x2, y2 = self._outside_boxes_6312[where]+ size = (y2 - y1 + 1, x2 - x1 + 1)+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6).repeat(2, 1, 1, 1)+ src_box = self._box_6312(x1, y1, x2, y2, 2, device, torch.int64)+ dst_box = self._box_6312(0, 0, size[1] - 1, size[0] - 1, 2, device, dtype)++ out = kornia.geometry.transform.crop_by_indices(inp, src_box, size=size, shape_compensation="pad")++ expected = kornia.geometry.transform.crop_by_boxes(inp, src_box.to(dtype), dst_box)+ self.assert_close(out, expected)++ @pytest.mark.parametrize("where", ["left", "above", "left_and_above"])+ def test_box_outside_the_image_resizes_the_clipped_slice_6312(self, where, device, dtype):+ # With 'resize' the part inside the image is resized to `size`, as for a box past the right or bottom edge.+ x1, y1, x2, y2 = self._outside_boxes_6312[where]+ size = (y2 - y1 + 1, x2 - x1 + 1)+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6)+ src_box = self._box_6312(x1, y1, x2, y2, 1, device, torch.int64)++ out = kornia.geometry.transform.crop_by_indices(inp, src_box, size=size)++ clipped = inp[..., max(y1, 0) : y2 + 1, max(x1, 0) : x2 + 1]+ expected = kornia.geometry.transform.resize(clipped, size, align_corners=None, side="short")+ self.assert_close(out, expected)++ @pytest.mark.parametrize(+ "box", [(-5, 0, -2, 2), (0, -6, 3, -3), (7, 0, 9, 2), (0, 6, 3, 8)], ids=["left", "above", "right", "below"]+ )+ def test_box_entirely_outside_the_image_pads_to_zeros_6312(self, box, device, dtype):+ # With 'pad' a box with no pixel in the image is all padding, on whichever side it lies.+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6)+ src_box = self._box_6312(*box, 1, device, torch.int64)+ out = kornia.geometry.transform.crop_by_indices(inp, src_box, size=(3, 4), shape_compensation="pad")+ self.assert_close(out, torch.zeros(1, 1, 3, 4, device=device, dtype=dtype))++ def test_clipped_slice_of_the_requested_size_is_still_padded_6312(self, device, dtype):+ # x = -1..2 is 4 wide; clipped to x = 0..2 it is 3 wide, the requested width, yet the pixels must keep their+ # box positions: a zero for x = -1, then x = 0 and 1, with x = 2 trimmed. Both the identical-box path and the+ # per-row path are checked.+ inp = torch.arange(1.0, 13.0, device=device, dtype=dtype).view(1, 1, 2, 6)+ box = self._box_6312(-1, 0, 2, 0, 1, device, torch.int64)+ expected = torch.tensor([[[[0.0, 1.0, 2.0]]]], device=device, dtype=dtype)+ out = kornia.geometry.transform.crop_by_indices(inp, box, size=(1, 3), shape_compensation="pad")+ self.assert_close(out, expected, atol=0.0, rtol=0.0)++ in_bounds = self._box_6312(0, 1, 2, 1, 1, device, torch.int64)+ out = kornia.geometry.transform.crop_by_indices(+ inp.repeat(2, 1, 1, 1), torch.cat([box, in_bounds]), size=(1, 3), shape_compensation="pad"+ )+ self.assert_close(out[:1], expected, atol=0.0, rtol=0.0)+ self.assert_close(out[1:], inp[..., 1:2, 0:3], atol=0.0, rtol=0.0)++ def test_boxes_with_different_negative_starts_pad_per_row_6312(self, device, dtype):+ # Both starts clip to x = 0, so the boxes look alike once clipped; each row must keep its own left padding.+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6).repeat(2, 1, 1, 1)+ src_box = torch.cat(+ [self._box_6312(-1, 0, 3, 2, 1, device, torch.int64), self._box_6312(-2, 0, 2, 2, 1, device, torch.int64)]+ )+ dst_box = self._box_6312(0, 0, 4, 2, 2, device, dtype)++ out = kornia.geometry.transform.crop_by_indices(inp, src_box, size=(3, 5), shape_compensation="pad")++ self.assert_close(out, kornia.geometry.transform.crop_by_boxes(inp, src_box.to(dtype), dst_box))++ @pytest.mark.parametrize("interpolation", ["bilinear", "bicubic", "nearest"])+ @pytest.mark.parametrize("where", ["left", "above", "left_and_above"])+ def test_dynamo_box_outside_the_image_6312(self, where, interpolation, device, dtype, torch_optimizer):+ # The fullgraph gather emulated the wrap, so it agreed with the wrong eager result; it must resize the+ # clipped slice instead.+ x1, y1, x2, y2 = self._outside_boxes_6312[where]+ size = (y2 - y1 + 2, x2 - x1 + 2)+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6)+ src_box = self._box_6312(x1, y1, x2, y2, 1, device, torch.int64)+ op = kornia.geometry.transform.crop_by_indices++ actual = torch_optimizer(op, fullgraph=True)(inp, src_box, size=size, interpolation=interpolation)++ clipped = inp[..., max(y1, 0) : y2 + 1, max(x1, 0) : x2 + 1]+ expected = kornia.geometry.transform.resize(clipped, size, interpolation=interpolation, side="short")+ self.assert_close(actual, expected, rtol=1e-4, atol=1e-4)+class TestCropSizeValidation:"""Tests that crop functions properly reject invalid size arguments."""@@ -1000,3 +1115,18 @@ class TestSliceResizeFullgraphPathEager(BaseTester):)actual = _compiled_slice_resize(image, src, (13, 29), mode, align_corners)self.assert_close(actual, expected, rtol=1e-12, atol=1e-12)++ @pytest.mark.parametrize("mode", ["bilinear", "bicubic", "nearest"])+ def test_box_entirely_left_of_the_image_reads_its_own_rows_6312(self, mode):+ # The eager path raises for the empty slice of a box with no pixel in the image; the gather cannot raise+ # without a host sync. Its empty interval read the last pixel of the previous row; it must stay in the box's+ # rows and read the image edge, as a box entirely past the right edge reads the last column.+ from kornia.geometry.transform._crop import _compiled_slice_resize++ image = torch.arange(1.0, 31.0, dtype=torch.float64).view(1, 1, 5, 6)+ left = torch.tensor([[[-5, 2], [-2, 2], [-2, 3], [-5, 3]]])+ right = torch.tensor([[[8, 2], [10, 2], [10, 3], [8, 3]]])+ out_left = _compiled_slice_resize(image, left, (2, 4), mode, None)+ out_right = _compiled_slice_resize(image, right, (2, 4), mode, None)+ self.assert_close(out_left, image[..., 2:4, :1].expand(-1, -1, -1, 4))+ self.assert_close(out_right, image[..., 2:4, -1:].expand(-1, -1, -1, 4))