Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(geometry): clip boxes left of or above the image in crop_by_indices (#6322)

Debanjan Sarkardcbad0b365e7
diff --git a/changelog.d/6322.fixed.md b/changelog.d/6322.fixed.md
new file mode 100644
index 000000000..5a6931580
--- /dev/null
+++ b/changelog.d/6322.fixed.md
@@ -0,0 +1 @@
+`kornia.geometry.transform.crop_by_indices` no longer reads pixels from the opposite edge for a box that starts left of or above the image. The part of the box outside the image is clipped, as past the right or bottom edge, and `shape_compensation` then applies; with `"pad"` the padding goes on the side where the box overflowed. The `torch.compile` path gives the same result. `CenterCrop2D` in its default slice mode now zero-pads a size larger than the input, as its resample mode does, instead of stretching the last row or column.
diff --git a/kornia/geometry/transform/_crop.py b/kornia/geometry/transform/_crop.py
index e077200e6..ac6f95835 100644
--- a/kornia/geometry/transform/_crop.py
+++ b/kornia/geometry/transform/_crop.py
@@ -105,13 +105,13 @@ def _compiled_slice_resize(
# Nearest normally uses float32 index arithmetic; handle ATen's CPU-double exception below.
coordinate_dtype = torch.float64 if input.dtype == torch.float64 and mode != "nearest" else torch.float32
src = src.to(device=input.device, dtype=torch.long)
- # Match Python slicing for negative and out-of-bounds replay coordinates.
+ # Clip out-of-bounds coordinates to the image, as crop_by_indices does: a negative start must not wrap (#6312).
x0, x1 = src[:, 0, 0:1], src[:, 1, 0:1] + 1
y0, y1 = src[:, 0, 1:2], src[:, 3, 1:2] + 1
- x0 = torch.where(x0 < 0, x0 + width, x0).clamp(0, width).to(coordinate_dtype)
- x1 = torch.where(x1 < 0, x1 + width, x1).clamp(0, width).to(coordinate_dtype)
- y0 = torch.where(y0 < 0, y0 + height, y0).clamp(0, height).to(coordinate_dtype)
- y1 = torch.where(y1 < 0, y1 + height, y1).clamp(0, height).to(coordinate_dtype)
+ x0 = x0.clamp(0, width).to(coordinate_dtype)
+ x1 = x1.clamp(0, width).to(coordinate_dtype)
+ y0 = y0.clamp(0, height).to(coordinate_dtype)
+ y1 = y1.clamp(0, height).to(coordinate_dtype)
x = _resize_coordinates(x0, x1, width, size[1], mode, align_corners)
y = _resize_coordinates(y0, y1, height, size[0], mode, align_corners)
if mode == "nearest" and input.device.type == "cpu" and input.dtype == torch.float64 and sum(size) > 128:
@@ -147,8 +147,10 @@ def _compiled_slice_resize(
flat = work.reshape(batch, channels, height * width)
def gather(x_index: torch.Tensor, y_index: torch.Tensor) -> torch.Tensor:
- x_index = (x0 + x_index.clamp(min=0)).minimum(x1 - 1).long()
- y_index = (y0 + y_index.clamp(min=0)).minimum(y1 - 1).long()
+ # ``clamp(min=0)``: a box entirely left of or above the image clips to an empty interval, whose ``x1 - 1`` is
+ # -1 and would read the previous row; it reads the image edge instead, as a box past the right edge does.
+ x_index = (x0 + x_index.clamp(min=0)).minimum(x1 - 1).clamp(min=0).long()
+ y_index = (y0 + y_index.clamp(min=0)).minimum(y1 - 1).clamp(min=0).long()
indices = (y_index.unsqueeze(-1) * width + x_index.unsqueeze(-2)).reshape(batch, 1, -1)
return flat.gather(2, indices.expand(-1, channels, -1)).reshape(batch, channels, *size)
diff --git a/kornia/geometry/transform/crop2d.py b/kornia/geometry/transform/crop2d.py
index 67b31a589..cb1e0087d 100644
--- a/kornia/geometry/transform/crop2d.py
+++ b/kornia/geometry/transform/crop2d.py
@@ -471,6 +471,9 @@ def crop_by_indices(
- ``shape_compensation`` (``'resize'`` by default) applies whenever the cropped
slice does not match ``size``, whether or not ``src_box`` is identical across
the batch β€” each row's output depends only on its own box.
+ - the part of a box outside the image is clipped on every side, a box that starts left of or above the
+ image included, and ``shape_compensation`` then applies. With ``'pad'`` the padding goes on the side
+ where the box overflowed, so every kept pixel stays at its position in the box.
.. note::
Under :func:`torch.compile` a non-empty batch with ``size`` given, ``'bilinear'``,
@@ -550,6 +553,14 @@ def crop_by_indices(
# stacked tensor) instead of a ``unique`` per column plus a device-to-host ``int(...)`` inside
# every loop iteration β€” the coordinates index Python-level slicing, so they must be host ints.
x1l, x2l, y1l, y2l = torch.stack([x1, x2, y1, y2], dim=0).tolist()
+ # A negative index would wrap to the far edge, so the part of a box left of or above the image is clipped,
+ # like the part past the right or bottom edge, which the slice clips. ``'pad'`` pads it back on that side.
+ left = [max(-v, 0) for v in x1l]
+ top = [max(-v, 0) for v in y1l]
+ x1l, x2l = [max(v, 0) for v in x1l], [max(v, 0) for v in x2l]
+ y1l, y2l = [max(v, 0) for v in y1l], [max(v, 0) for v in y2l]
+ # With 'pad' a clipped box needs that padding even when its clipped slice happens to have the requested size.
+ shifted = [shape_compensation != "resize" and (left[i] > 0 or top[i] > 0) for i in range(B)]
# ``B > 0``: an empty batch has no first box to compare against, and takes the general path below,
# which returns an empty ``(0, C, *size)`` tensor (#4429).
@@ -559,9 +570,11 @@ def crop_by_indices(
and x2l.count(x2l[0]) == B
and y1l.count(y1l[0]) == B
and y2l.count(y2l[0]) == B
+ and left.count(left[0]) == B
+ and top.count(top[0]) == B
):
out = input_tensor[..., y1l[0] : y2l[0], x1l[0] : x2l[0]]
- if size is not None and out.shape[-2:] != size:
+ if size is not None and (out.shape[-2:] != size or shifted[0]):
if shape_compensation == "resize":
return resize(
out,
@@ -571,7 +584,7 @@ def crop_by_indices(
side="short",
antialias=antialias,
)
- return F.pad(out, [0, size[1] - out.shape[-1], 0, size[0] - out.shape[-2]])
+ return _pad_to_size(out, size, left[0], top[0])
if size is None:
h, w = infer_bbox_shape(src)
@@ -596,12 +609,17 @@ def crop_by_indices(
# ``contiguous`` keeps the result in the default memory format, as the ``torch.empty`` below does,
# when the input is channels-last. MPS keeps the per-row copies: on an M1 the ``cat`` of strided
# views ran at 0.75-0.96x their speed without gradients (B = 4-64, 256Β² -> 128Β², #4987).
- if B > 0 and input_tensor.device.type != "mps" and all(crop.shape[-2:] == size for crop in slices):
+ if (
+ B > 0
+ and input_tensor.device.type != "mps"
+ and not any(shifted)
+ and all(crop.shape[-2:] == size for crop in slices)
+ ):
return torch.cat(slices).contiguous()
out = torch.empty(B, C, *size, device=input_tensor.device, dtype=input_tensor.dtype) if rows is None else None
crops: list[torch.Tensor] = []
for i, _out in enumerate(slices):
- if _out.shape[-2:] != size:
+ if _out.shape[-2:] != size or shifted[i]:
if shape_compensation == "resize":
_out = resize(
_out,
@@ -612,7 +630,7 @@ def crop_by_indices(
antialias=antialias,
)
else:
- _out = F.pad(_out, [0, size[1] - _out.shape[-1], 0, size[0] - _out.shape[-2]])
+ _out = _pad_to_size(_out, size, left[i], top[i])
if out is None:
crops.append(_out)
else:
@@ -622,6 +640,14 @@ def crop_by_indices(
return out
+def _pad_to_size(crop: torch.Tensor, size: Tuple[int, int], left: int, top: int) -> torch.Tensor:
+ # ``left`` and ``top`` are the box columns and rows clipped before the image, so every kept pixel stays at its
+ # position in the box; the rest of ``size`` is padded (or, for a larger slice, trimmed) at the right and bottom.
+ # A box entirely outside the image clips more than ``size``: it is all padding, as past the right edge.
+ left, top = min(left, size[1]), min(top, size[0])
+ return F.pad(crop, [left, size[1] - crop.shape[-1] - left, top, size[0] - crop.shape[-2] - top])
+
+
def _crop_by_indices_export(
input_tensor: torch.Tensor,
src_box: torch.Tensor,
@@ -749,6 +775,12 @@ class CenterCrop2D(nn.Module):
)
if self.flags["cropping_mode"] == "slice": # uses advanced slicing to crop
+ if dst_h > src_h or dst_w > src_w:
+ # A size larger than the input: zero-pad around the image, as the resample mode does, then slice.
+ # The offsets are Python ints from the shapes, so this compiles with fullgraph=True.
+ left, top = max(-start_x, 0), max(-start_y, 0)
+ padded = F.pad(input, [left, max(end_x + 1 - src_w, 0), top, max(end_y + 1 - src_h, 0)])
+ return padded[..., start_y + top : end_y + 1 + top, start_x + left : end_x + 1 + left].contiguous()
return crop_by_indices(input, self.points_src.expand(batch_size, -1, -1).to(input), self.flags["size"])
raise NotImplementedError(f"Not supported type: {self.flags['cropping_mode']}.")
diff --git a/tests/geometry/transform/test_crop2d.py b/tests/geometry/transform/test_crop2d.py
index f307af7eb..cced67193 100644
--- a/tests/geometry/transform/test_crop2d.py
+++ b/tests/geometry/transform/test_crop2d.py
@@ -392,6 +392,28 @@ class TestCenterCrop(BaseTester):
reflected = kornia.geometry.transform.center_crop(small, (4, 4), padding_mode="reflection", align_corners=False)
self.assert_close(reflected, replicated)
+ # A 4x5 input: overflow on the right only, on every side, with odd margins, and on one axis while the other crops.
+ _larger_sizes_6312 = [(4, 6), (6, 6), (5, 7), (2, 7), (7, 3)]
+
+ @pytest.mark.parametrize("size", _larger_sizes_6312)
+ def test_center_crop2d_slice_larger_than_input_matches_resample_6312(self, size, device, dtype):
+ # The slice mode wrapped the negative start and stretched the wrong pixels; it must zero-pad as resample does.
+ if not supports_bilinear_2d_grid_sample(device, dtype):
+ pytest.skip("bilinear 2D grid_sample is unavailable for this device and dtype")
+ inp = torch.arange(1.0, 21.0, device=device, dtype=dtype).view(1, 1, 4, 5).repeat(2, 3, 1, 1)
+ out_slice = kornia.geometry.transform.CenterCrop2D(size)(inp)
+ out_resample = kornia.geometry.transform.CenterCrop2D(size, cropping_mode="resample")(inp)
+ assert out_slice.shape == (2, 3, *size)
+ self.assert_close(out_slice, out_resample)
+
+ @pytest.mark.parametrize("size", _larger_sizes_6312)
+ def test_dynamo_center_crop2d_slice_larger_than_input_6312(self, size, device, dtype, torch_optimizer):
+ if not supports_bilinear_2d_grid_sample(device, dtype):
+ pytest.skip("bilinear 2D grid_sample is unavailable for this device and dtype")
+ inp = torch.arange(1.0, 21.0, device=device, dtype=dtype).view(1, 1, 4, 5)
+ out = torch_optimizer(kornia.geometry.transform.CenterCrop2D(size), fullgraph=True)(inp)
+ self.assert_close(out, kornia.geometry.transform.CenterCrop2D(size, cropping_mode="resample")(inp))
+
class TestCropByBoxes(BaseTester):
def test_crop_by_boxes_no_resizing(self, device, dtype):
@@ -958,6 +980,99 @@ class TestCropByIndices(BaseTester):
kornia.geometry.transform.crop_by_indices, (inp, src_box, (3, 4)), requires_grad=(True, False, False)
)
+ # (x1, y1, x2, y2) boxes on a 5x6 image: left of it, above it, and both (#6312).
+ _outside_boxes_6312 = {"left": (-2, 1, 3, 3), "above": (1, -1, 4, 2), "left_and_above": (-1, -2, 2, 1)}
+
+ @staticmethod
+ def _box_6312(x1, y1, x2, y2, batch, device, dtype):
+ box = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
+ return torch.tensor([box] * batch, device=device, dtype=dtype)
+
+ @pytest.mark.parametrize("where", ["left", "above", "left_and_above"])
+ def test_box_outside_the_image_pads_in_place_6312(self, where, device, dtype):
+ # A negative start used to wrap to the far edge of the axis. With 'pad' the box must give what
+ # crop_by_boxes gives: the image pixels at their positions in the box and zeros outside the image.
+ x1, y1, x2, y2 = self._outside_boxes_6312[where]
+ size = (y2 - y1 + 1, x2 - x1 + 1)
+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6).repeat(2, 1, 1, 1)
+ src_box = self._box_6312(x1, y1, x2, y2, 2, device, torch.int64)
+ dst_box = self._box_6312(0, 0, size[1] - 1, size[0] - 1, 2, device, dtype)
+
+ out = kornia.geometry.transform.crop_by_indices(inp, src_box, size=size, shape_compensation="pad")
+
+ expected = kornia.geometry.transform.crop_by_boxes(inp, src_box.to(dtype), dst_box)
+ self.assert_close(out, expected)
+
+ @pytest.mark.parametrize("where", ["left", "above", "left_and_above"])
+ def test_box_outside_the_image_resizes_the_clipped_slice_6312(self, where, device, dtype):
+ # With 'resize' the part inside the image is resized to `size`, as for a box past the right or bottom edge.
+ x1, y1, x2, y2 = self._outside_boxes_6312[where]
+ size = (y2 - y1 + 1, x2 - x1 + 1)
+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6)
+ src_box = self._box_6312(x1, y1, x2, y2, 1, device, torch.int64)
+
+ out = kornia.geometry.transform.crop_by_indices(inp, src_box, size=size)
+
+ clipped = inp[..., max(y1, 0) : y2 + 1, max(x1, 0) : x2 + 1]
+ expected = kornia.geometry.transform.resize(clipped, size, align_corners=None, side="short")
+ self.assert_close(out, expected)
+
+ @pytest.mark.parametrize(
+ "box", [(-5, 0, -2, 2), (0, -6, 3, -3), (7, 0, 9, 2), (0, 6, 3, 8)], ids=["left", "above", "right", "below"]
+ )
+ def test_box_entirely_outside_the_image_pads_to_zeros_6312(self, box, device, dtype):
+ # With 'pad' a box with no pixel in the image is all padding, on whichever side it lies.
+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6)
+ src_box = self._box_6312(*box, 1, device, torch.int64)
+ out = kornia.geometry.transform.crop_by_indices(inp, src_box, size=(3, 4), shape_compensation="pad")
+ self.assert_close(out, torch.zeros(1, 1, 3, 4, device=device, dtype=dtype))
+
+ def test_clipped_slice_of_the_requested_size_is_still_padded_6312(self, device, dtype):
+ # x = -1..2 is 4 wide; clipped to x = 0..2 it is 3 wide, the requested width, yet the pixels must keep their
+ # box positions: a zero for x = -1, then x = 0 and 1, with x = 2 trimmed. Both the identical-box path and the
+ # per-row path are checked.
+ inp = torch.arange(1.0, 13.0, device=device, dtype=dtype).view(1, 1, 2, 6)
+ box = self._box_6312(-1, 0, 2, 0, 1, device, torch.int64)
+ expected = torch.tensor([[[[0.0, 1.0, 2.0]]]], device=device, dtype=dtype)
+ out = kornia.geometry.transform.crop_by_indices(inp, box, size=(1, 3), shape_compensation="pad")
+ self.assert_close(out, expected, atol=0.0, rtol=0.0)
+
+ in_bounds = self._box_6312(0, 1, 2, 1, 1, device, torch.int64)
+ out = kornia.geometry.transform.crop_by_indices(
+ inp.repeat(2, 1, 1, 1), torch.cat([box, in_bounds]), size=(1, 3), shape_compensation="pad"
+ )
+ self.assert_close(out[:1], expected, atol=0.0, rtol=0.0)
+ self.assert_close(out[1:], inp[..., 1:2, 0:3], atol=0.0, rtol=0.0)
+
+ def test_boxes_with_different_negative_starts_pad_per_row_6312(self, device, dtype):
+ # Both starts clip to x = 0, so the boxes look alike once clipped; each row must keep its own left padding.
+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6).repeat(2, 1, 1, 1)
+ src_box = torch.cat(
+ [self._box_6312(-1, 0, 3, 2, 1, device, torch.int64), self._box_6312(-2, 0, 2, 2, 1, device, torch.int64)]
+ )
+ dst_box = self._box_6312(0, 0, 4, 2, 2, device, dtype)
+
+ out = kornia.geometry.transform.crop_by_indices(inp, src_box, size=(3, 5), shape_compensation="pad")
+
+ self.assert_close(out, kornia.geometry.transform.crop_by_boxes(inp, src_box.to(dtype), dst_box))
+
+ @pytest.mark.parametrize("interpolation", ["bilinear", "bicubic", "nearest"])
+ @pytest.mark.parametrize("where", ["left", "above", "left_and_above"])
+ def test_dynamo_box_outside_the_image_6312(self, where, interpolation, device, dtype, torch_optimizer):
+ # The fullgraph gather emulated the wrap, so it agreed with the wrong eager result; it must resize the
+ # clipped slice instead.
+ x1, y1, x2, y2 = self._outside_boxes_6312[where]
+ size = (y2 - y1 + 2, x2 - x1 + 2)
+ inp = torch.arange(1.0, 31.0, device=device, dtype=dtype).view(1, 1, 5, 6)
+ src_box = self._box_6312(x1, y1, x2, y2, 1, device, torch.int64)
+ op = kornia.geometry.transform.crop_by_indices
+
+ actual = torch_optimizer(op, fullgraph=True)(inp, src_box, size=size, interpolation=interpolation)
+
+ clipped = inp[..., max(y1, 0) : y2 + 1, max(x1, 0) : x2 + 1]
+ expected = kornia.geometry.transform.resize(clipped, size, interpolation=interpolation, side="short")
+ self.assert_close(actual, expected, rtol=1e-4, atol=1e-4)
+
class TestCropSizeValidation:
"""Tests that crop functions properly reject invalid size arguments."""
@@ -1000,3 +1115,18 @@ class TestSliceResizeFullgraphPathEager(BaseTester):
)
actual = _compiled_slice_resize(image, src, (13, 29), mode, align_corners)
self.assert_close(actual, expected, rtol=1e-12, atol=1e-12)
+
+ @pytest.mark.parametrize("mode", ["bilinear", "bicubic", "nearest"])
+ def test_box_entirely_left_of_the_image_reads_its_own_rows_6312(self, mode):
+ # The eager path raises for the empty slice of a box with no pixel in the image; the gather cannot raise
+ # without a host sync. Its empty interval read the last pixel of the previous row; it must stay in the box's
+ # rows and read the image edge, as a box entirely past the right edge reads the last column.
+ from kornia.geometry.transform._crop import _compiled_slice_resize
+
+ image = torch.arange(1.0, 31.0, dtype=torch.float64).view(1, 1, 5, 6)
+ left = torch.tensor([[[-5, 2], [-2, 2], [-2, 3], [-5, 3]]])
+ right = torch.tensor([[[8, 2], [10, 2], [10, 3], [8, 3]]])
+ out_left = _compiled_slice_resize(image, left, (2, 4), mode, None)
+ out_right = _compiled_slice_resize(image, right, (2, 4), mode, None)
+ self.assert_close(out_left, image[..., 2:4, :1].expand(-1, -1, -1, 4))
+ self.assert_close(out_right, image[..., 2:4, -1:].expand(-1, -1, -1, 4))