π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from functools import partial
from unittest.mock import patch
import numpy as np
import pytest
import torch
import kornia
import kornia.augmentation as K
from kornia.augmentation.container.augment import _NON_RIGID_WARPS
from kornia.augmentation.container.base import ParamItem
from kornia.constants import BorderType, DataKey, Resample
from kornia.core._compat import torch_version_lt
from kornia.geometry.bbox import bbox_to_mask
from kornia.geometry.boxes import Boxes
from kornia.geometry.transform import resize
from testing.augmentation.utils import oversized_random_crop_params, reproducibility_test
from testing.base import BaseTester, assert_close, supports_bilinear_2d_grid_sample
class TestAugmentationSequential:
@pytest.mark.parametrize("as_dict", [False, True])
def test_numpy_annotations_are_not_scaled_like_images(self, as_dict):
image = np.zeros((8, 9, 3), dtype=np.uint8)
mask = np.zeros((8, 9), dtype=np.int64)
mask[2:5, 3:6] = 3
boxes = np.array([[[3.0, 2.0, 6.0, 5.0]]], dtype=np.float32)
keypoints = np.array([[[4.0, 3.0]]], dtype=np.float32)
data_keys = ["input", "mask", "bbox_xyxy", "keypoints"]
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=None if as_dict else data_keys)
if as_dict:
output = aug(dict(zip(data_keys, (image, mask, boxes, keypoints))))
out_mask, out_boxes, out_keypoints = output["mask"], output["bbox_xyxy"], output["keypoints"]
else:
_, out_mask, out_boxes, out_keypoints = aug(image, mask, boxes, keypoints)
assert out_mask.dtype == torch.int64
assert set(out_mask.unique().tolist()) == {0, 3}
assert torch.equal(out_boxes, torch.from_numpy(boxes))
assert torch.equal(out_keypoints, torch.from_numpy(keypoints))
@pytest.mark.parametrize(
"data_keys,error_type,match",
[
("input", KeyError, "'i' is not a valid DataKey"),
("image", KeyError, "'i' is not a valid DataKey"),
(["mask", "input"], NotImplementedError, "first input must be DataKey.IMAGE"),
(["input", "bbox_yxyx"], KeyError, "'bbox_yxyx' is not a valid DataKey"),
([0, 10], ValueError, "10 is not a valid DataKey"),
([BorderType.REFLECT], TypeError, "DataKey"),
],
)
@pytest.mark.parametrize("augmentation_list", [K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0)])
def test_exception(self, augmentation_list, data_keys, error_type, match, device, dtype):
with pytest.raises(error_type, match=match):
K.AugmentationSequential(augmentation_list, data_keys=data_keys)
@pytest.mark.slow
@pytest.mark.parametrize("same_on_batch", [True, False])
@pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 10, True, False])
@pytest.mark.parametrize("inp", [torch.randn(1, 3, 1000, 500), torch.randn(3, 1000, 500)])
def test_mixup(self, inp, random_apply, same_on_batch, device, dtype):
inp = torch.as_tensor(inp, device=device, dtype=dtype)
aug = K.AugmentationSequential(
K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)),
K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0),
K.RandomAffine(360, p=1.0),
K.RandomMixUpV2(p=1.0),
data_keys=["input"],
random_apply=random_apply,
same_on_batch=same_on_batch,
)
out = aug(inp)
assert out.shape[-3:] == inp.shape[-3:]
reproducibility_test(inp, aug)
def test_mixup_cutmix_only(self, device, dtype):
mixup = K.RandomMixUpV2(p=1.0, data_keys=["input"])
cutmix = K.RandomCutMixV2(p=1.0, data_keys=["input"])
aug = K.AugmentationSequential(
mixup,
cutmix,
data_keys=["input"],
random_apply=1,
)
input = torch.randn(2, 3, 224, 224, device=device, dtype=dtype)
out_input = aug(input)
assert out_input.shape == input.shape
@pytest.mark.parametrize("as_dict", [False, True])
@pytest.mark.parametrize("as_path", [False, True])
@pytest.mark.parametrize("mode", ["L", "P"])
def test_pil_masks_keep_label_values(self, tmp_path, as_dict, as_path, mode):
Image = pytest.importorskip("PIL.Image")
image = np.zeros((8, 9, 3), dtype=np.uint8)
labels = np.zeros((8, 9), dtype=np.uint8)
labels[2:5, 1:3] = 3
mask = Image.fromarray(labels, mode=mode)
if mode == "P":
mask.putpalette([0, 0, 0, 128, 0, 0, 0, 128, 0, 128, 128, 0] + [0] * 756)
if as_path:
mask_path = tmp_path / "mask.png"
mask.save(mask_path)
mask = str(mask_path)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=None if as_dict else ["input", "mask"])
if as_dict:
output = aug({"input": image, "mask": mask})
out_mask = output["mask"]
else:
_, out_mask = aug(image, mask)
assert out_mask.shape == (1, 1, 8, 9)
assert out_mask.dtype == torch.uint8
assert set(out_mask.unique().tolist()) == {0, 3}
def test_numpy_mask_with_incompatible_shape_raises(self):
image = np.zeros((2, 8, 9, 3), dtype=np.uint8)
mask = np.zeros((2, 8, 9), dtype=np.uint8)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=["input", "mask"])
with pytest.raises(ValueError, match="Image and mask must have matching spatial dimensions"):
aug(image, mask)
@pytest.mark.parametrize("mode", ["I;16", "I;16B"])
def test_pil_16bit_masks_keep_label_values(self, mode):
Image = pytest.importorskip("PIL.Image")
image = np.zeros((8, 9, 3), dtype=np.uint8)
labels = np.zeros((8, 9), dtype=np.uint16)
labels[2:5, 1:3] = 1000
mask = Image.frombytes(mode, (9, 8), labels.astype("<u2" if mode == "I;16" else ">u2").tobytes())
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
_, out_mask = aug(image, mask)
assert out_mask.dtype == torch.uint16
expected = torch.from_numpy(np.ascontiguousarray(labels[:, ::-1]).astype(np.int32))[None, None]
assert torch.equal(out_mask.to(torch.int32), expected)
def test_tensor_mask_does_not_need_pillow(self, monkeypatch):
# Pillow is optional: converting PIL masks must not import it for a tensor mask.
import sys
from kornia.core.external import PILImage as lazy_pil
monkeypatch.setattr(lazy_pil, "module", None)
monkeypatch.setitem(sys.modules, "PIL.Image", None)
image, mask = torch.rand(2, 3, 8, 9), torch.rand(2, 1, 8, 9)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
_, out_mask = aug(image, mask)
assert torch.equal(out_mask, mask.flip(-1))
@pytest.mark.parametrize(
("image_shape", "mask_shape", "batch"),
[
((2, 3, 8, 9), (3, 1, 8, 9), 2),
((3, 8, 9), (3, 1, 8, 9), 1),
((3, 8, 9), (1, 1, 8, 10), 1),
((2, 3, 8, 9), (9, 9), 2),
((3, 8, 9), (8, 10), 1),
((8, 9), (1, 1, 8, 10), 1),
((8, 9), (2, 1, 8, 9), 1),
],
)
def test_mask_with_incompatible_shape_raises(self, image_shape, mask_shape, batch):
image, mask = torch.zeros(image_shape), torch.zeros(mask_shape)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
with pytest.raises(ValueError, match=rf"compatible batch sizes \(1 or {batch}\)"):
aug(image, mask)
def test_pil_rgb_mask_with_a_single_image(self):
Image = pytest.importorskip("PIL.Image")
labels = np.zeros((8, 9, 3), dtype=np.uint8)
labels[2:5, 1:3] = (1, 2, 3)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=["input", "mask"])
_, out_mask = aug(np.zeros((8, 9, 3), dtype=np.uint8), Image.fromarray(labels, mode="RGB"))
assert torch.equal(out_mask, torch.from_numpy(labels).permute(2, 0, 1)[None])
def test_single_mask_with_a_batch_is_broadcast_not_rejected(self):
image, mask = torch.zeros(2, 3, 8, 9), torch.rand(1, 1, 8, 9)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
_, out_mask = aug(image, mask)
assert torch.equal(out_mask, mask.flip(-1).expand(2, -1, -1, -1))
@pytest.mark.parametrize("mask_shape", [(1, 1, 8, 8), (1, 8, 8), (3, 8, 8)])
@pytest.mark.parametrize(
"make_child",
[
lambda: K.RandomHorizontalFlip(p=1.0),
lambda: K.RandomHorizontalFlip(p=0.5),
lambda: K.RandomCrop((4, 4), p=1.0),
lambda: K.RandomAffine(30.0, p=1.0, resample="nearest"),
],
ids=["flip", "flip_half", "crop", "affine"],
)
def test_single_mask_follows_each_samples_own_transform(self, make_child, mask_shape):
# Every image is a copy of the mask, so each output mask must equal its own augmented image. A mask that was
# transformed with sample 0's draw only (flip, crop) or that made the warp fail (affine) breaks this.
mask = torch.rand(*mask_shape)
image = mask.reshape(1, mask.shape[-3], 8, 8).expand(4, -1, -1, -1).clone()
for seed in range(5):
torch.manual_seed(seed)
aug = K.AugmentationSequential(make_child(), data_keys=["input", "mask"])
out_image, out_mask = aug(image, mask)
assert out_mask.shape == out_image.shape
assert_close(out_mask, out_image)
@pytest.mark.parametrize("mask_shape", [(1, 1, 8, 8), (1, 8, 8)])
def test_single_mask_equals_an_explicit_full_batch_mask(self, mask_shape):
image = torch.rand(3, 3, 8, 8)
mask = (torch.rand(*mask_shape) > 0.5).float()
outputs = []
for candidate in (mask, mask.expand(3, *mask.shape[1:]).clone()):
torch.manual_seed(7)
aug = K.AugmentationSequential(
K.RandomAffine(30.0, p=1.0), K.RandomHorizontalFlip(p=0.5), data_keys=["input", "mask"]
)
outputs.append(aug(image, candidate)[1])
assert outputs[0].shape == (3, 1, 8, 8)
assert_close(outputs[0], outputs[1])
@pytest.mark.parametrize("mask_shape", [(2, 1, 8, 8), (2, 8, 8)])
def test_full_batch_mask_is_not_repeated(self, mask_shape):
# A leading size equal to the image batch is one mask per image, in both the 3D and the 4D layout.
mask = torch.rand(*mask_shape)
image = mask.reshape(2, 1, 8, 8).expand(-1, 3, -1, -1).clone()
torch.manual_seed(0)
aug = K.AugmentationSequential(K.RandomCrop((4, 4), p=1.0), data_keys=["input", "mask"])
out_image, out_mask = aug(image, mask)
assert out_mask.shape == (2, 1, 4, 4)
assert_close(out_mask, out_image[:, :1])
def test_single_mask_broadcast_keeps_the_callers_mask_and_inverts(self):
image, mask = torch.rand(2, 3, 8, 8), torch.rand(1, 1, 8, 8)
original = mask.clone()
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
out_image, out_mask = aug(image, mask)
_, inv_mask = aug.inverse(out_image, out_mask)
assert torch.equal(mask, original)
assert_close(inv_mask, original.expand(2, -1, -1, -1))
def test_single_mask_broadcast_through_a_dict_and_keepdim(self):
image, mask = torch.rand(2, 3, 8, 8), torch.rand(1, 8, 8)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None, keepdim=True)
out = aug({"image": image, "mask": mask})
assert out["mask"].shape == (2, 8, 8)
def test_dict_output_leaves_unrecognized_keys_alone_5695(self, device, dtype):
# A dictionary key that is not a data key holds a pass-through value: converting it like an
# image would move a (B, C, H, W) tensor to channels-last, which is not what the caller passed in.
image = torch.rand(1, 3, 8, 8, device=device, dtype=dtype)
aux = torch.rand(1, 3, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=None)
out = aug({"image": image, "aux": aux}, output_type="numpy")
assert out["aux"].shape == (1, 3, 8, 8)
def test_dict_output_keeps_a_three_dimensional_mask_batch_first_5695(self, device, dtype):
# A (B, H, W) mask returned under keepdim=True is batch-first, like the image: it gets its channel axis
# back instead of being read as (C, H, W) and permuted to (H, W, B).
image = torch.rand(2, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(2, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=None, keepdim=True)
out = aug({"image": image, "mask": mask}, output_type="numpy")
assert out["mask"].shape == (2, 8, 8, 1)
numpy_dtype = torch.float32 if dtype == torch.bfloat16 else dtype
np.testing.assert_array_equal(out["mask"][..., 0], mask.cpu().to(numpy_dtype).numpy())
def test_positional_output_converts_each_tensor_of_a_mask_list_5695(self, device, dtype):
# A mask given as a per-sample list of tensors comes back as a list of arrays for
# output_type != "pt", each converted like the image.
image = torch.rand(2, 3, 8, 8, device=device, dtype=dtype)
masks = [
torch.rand(1, 1, 8, 8, device=device, dtype=dtype),
torch.rand(1, 1, 8, 8, device=device, dtype=dtype),
]
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=["input", "mask"])
_, out_masks = aug(image, masks, output_type="numpy")
assert [type(m) for m in out_masks] == [np.ndarray, np.ndarray]
assert out_masks[0].shape == (1, 8, 8, 1)
assert out_masks[1].shape == (1, 8, 8, 1)
@pytest.mark.parametrize("batch", [1, 2, 3])
def test_mask_output_converts_per_sample_like_the_image_5695(self, batch, device, dtype):
# Every sample's mask is its own PIL image, as every sample's image is, and the NumPy mask has the layout
# the mask input reads, so feeding it back in gives the same mask.
image = torch.rand(batch, 3, 6, 8, device=device, dtype=dtype)
mask = (torch.rand(batch, 1, 6, 8, device=device, dtype=dtype) > 0.5).to(dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
pil_image, pil_mask = aug(image, mask, output_type="pil")
assert len(pil_image) == len(pil_mask) == batch
for sample, flipped in zip(pil_mask, mask.flip(-1)):
assert sample.mode == "L"
np.testing.assert_array_equal(np.asarray(sample), (flipped[0] * 255).to(torch.uint8).cpu().numpy())
_, np_mask = aug(image, mask, output_type="numpy")
assert np_mask.shape == (batch, 6, 8, 1)
_, fed_back = aug(image.cpu(), np_mask)
assert_close(fed_back.to(device=device, dtype=dtype), mask)
@pytest.mark.parametrize(
"image_shape, mask_shape, expected",
[
((2, 3, 6, 8), (2, 6, 8), (2, 6, 8, 1)), # (B, H, W): the leading size is the image batch
((1, 3, 6, 8), (2, 6, 8), (6, 8, 2)), # one (C, H, W) mask next to a batch of one
((3, 6, 8), (3, 6, 8), (6, 8, 3)), # one (C, H, W) mask next to an unbatched image
],
)
def test_keepdim_three_dimensional_mask_reads_like_the_input_5695(
self, image_shape, mask_shape, expected, device, dtype
):
# A 3D mask is (B, H, W) when its leading size is the batch of a batched image, as on the way in;
# otherwise it is one (C, H, W) mask and converts like an unbatched image.
image = torch.rand(*image_shape, device=device, dtype=dtype)
mask = torch.rand(*mask_shape, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=["input", "mask"], keepdim=True)
_, out = aug(image, mask, output_type="numpy")
assert out.shape == expected
def test_single_mask_with_an_unbatched_image_stays_single(self):
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
_, out_mask = aug(torch.rand(3, 8, 8), torch.rand(1, 1, 8, 8))
assert out_mask.shape == (1, 1, 8, 8)
@pytest.mark.parametrize("keepdim", [None, True])
def test_single_hw_mask_is_broadcast_5611(self, keepdim, device, dtype):
# An (H, W) tensor mask is one mask for every image, like the NumPy (H, W) mask it may come from.
mask = torch.rand(8, 8, device=device, dtype=dtype)
image = mask.expand(4, 3, 8, 8).clone()
aug = K.AugmentationSequential(
K.RandomAffine(30.0, p=1.0, resample="nearest"), data_keys=["input", "mask"], keepdim=keepdim
)
out_image, out_mask = aug(image, mask)
assert out_mask.shape == ((4, 8, 8) if keepdim else (4, 1, 8, 8))
assert_close(out_mask.reshape(4, 1, 8, 8), out_image[:, :1])
def test_multi_channel_mask_with_a_batch_one_image_is_not_repeated(self, device, dtype):
# With B == 1 there is nothing to repeat: a (C, H, W) mask keeps its shape under keepdim=True.
image = torch.rand(1, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(2, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"], keepdim=True)
_, out_mask = aug(image, mask)
assert_close(out_mask, mask.flip(-1))
def test_video_single_mask_follows_each_samples_own_flip_5624(self, device, dtype):
# Entry point 1 of #5624: a (1, T, C, H, W) mask next to a (B, T, C, H, W) video; each output mask
# must equal its own sample's augmented image, and the test uses a fixed per-sample batch_prob so the
# fixture is deterministic and not dependent on the global RNG.
mask = torch.rand(1, 4, 1, 8, 8, device=device, dtype=dtype)
image = mask.expand(2, -1, 3, -1, -1).clone()
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomHorizontalFlip(p=0.5), data_format="BTCHW"), data_keys=["input", "mask"]
)
out_image, out_mask = aug(image, mask)
aug._params[0].data[0].data["batch_prob"] = torch.tensor(
[True, False, True, False, False, True, False, True], device=device, dtype=torch.bool
)
out_image, out_mask = aug(image, mask, params=aug._params)
assert out_mask.shape == (2, 4, 1, 8, 8)
assert_close(out_mask, out_image[:, :, :1])
@pytest.mark.parametrize("data_format", ["BTCHW", "BCTHW"])
def test_video_single_mask_equals_an_explicit_full_batch_mask_5624(self, data_format, device, dtype):
# Entry point 1 of #5624, RandomAffine: a single mask must match an already-expanded one, not just avoid
# the grid_sampler batch-size error. BCTHW puts the frames on dim 2, so its frame check must not read the
# channel axis (1 mask channel against 3 image channels).
channels_first = data_format == "BCTHW"
image = torch.rand(*((2, 3, 4) if channels_first else (2, 4, 3)), 8, 8, device=device, dtype=dtype)
mask = torch.rand(*((1, 1, 4) if channels_first else (1, 4, 1)), 8, 8, device=device, dtype=dtype)
outputs = []
for candidate in (mask, mask.expand(2, -1, -1, -1, -1).clone()):
torch.manual_seed(5)
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomAffine(30.0, p=1.0), data_format=data_format), data_keys=["input", "mask"]
)
outputs.append(aug(image, candidate)[1])
assert outputs[0].shape == (2, *mask.shape[1:])
assert_close(outputs[0], outputs[1])
@pytest.mark.parametrize("nested", [False, True])
def test_video_replay_single_mask_uses_the_video_batch_5624(self, nested, device, dtype):
# Entry point 2 of #5624 for a video: with same_on_frame=False each child records B * T rows in
# forward_input_shape, so a mask-only replay must repeat a single mask to the video batch B that
# VideoSequential records, also through a nested container.
video = torch.rand(3, 4, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(1, 4, 1, 8, 8, device=device, dtype=dtype)
child = K.RandomAffine(30.0, p=1.0)
aug = K.AugmentationSequential(
K.VideoSequential(K.ImageSequential(child) if nested else child, same_on_frame=False),
data_keys=["input", "mask"],
)
aug(video, mask.expand(3, -1, -1, -1, -1).clone())
out_single = aug(mask, params=aug._params, data_keys=["mask"])
out_full = aug(mask.expand(3, -1, -1, -1, -1).clone(), params=aug._params, data_keys=["mask"])
assert out_single.shape == (3, 4, 1, 8, 8)
assert_close(out_single, out_full)
def test_replay_single_mask_follows_each_samples_own_flip_5624(self, device, dtype):
# Entry point 2 of #5624: a mask replayed alone (no image) with params recorded for batch > 1 must
# match an already-expanded replay, and the fixture uses a fixed batch_prob pattern so the result is
# deterministic and independent of global RNG state.
image = torch.rand(3, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(1, 1, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.5), data_keys=["input", "mask"])
aug(image, mask.expand(3, -1, -1, -1).clone())
aug._params[0].data["batch_prob"] = torch.tensor([True, False, True], device=device, dtype=torch.bool)
out_mask = aug(mask, params=aug._params, data_keys=["mask"])
expected = aug(mask.expand(3, -1, -1, -1).clone(), params=aug._params, data_keys=["mask"])
assert out_mask.shape == expected.shape
assert_close(out_mask, expected)
def test_replay_single_mask_equals_an_explicit_full_batch_mask_5624(self, device, dtype):
# Entry point 2 of #5624, RandomAffine: a replayed single mask must match an already-expanded one, not
# just avoid the grid_sampler batch-size error.
image = torch.rand(3, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(1, 1, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomAffine(30.0, p=1.0), data_keys=["input", "mask"])
aug(image, mask.expand(3, -1, -1, -1).clone())
out_single = aug(mask, params=aug._params, data_keys=["mask"])
out_full = aug(mask.expand(3, -1, -1, -1).clone(), params=aug._params, data_keys=["mask"])
assert out_single.shape == (3, 1, 8, 8)
assert_close(out_single, out_full)
def test_video_mask_batch_mismatch_raises_5624(self, device, dtype):
# A mask batch that is neither 1 nor the video batch is rejected, as for a 4-D image (#5618).
image = torch.rand(2, 4, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(3, 4, 1, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomHorizontalFlip(p=1.0), data_format="BTCHW"), data_keys=["input", "mask"]
)
with pytest.raises(ValueError, match="batch"):
aug(image, mask)
def test_video_mask_frame_count_mismatch_raises_5624(self, device, dtype):
image = torch.rand(2, 4, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(2, 3, 1, 8, 8, device=device, dtype=dtype) # 3 frames, the video has 4
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomHorizontalFlip(p=1.0), data_format="BTCHW"), data_keys=["input", "mask"]
)
with pytest.raises(ValueError, match="frames"):
aug(image, mask)
def test_replay_mask_batch_mismatch_raises_5624(self, device, dtype):
# A replayed mask batch that is neither 1 nor the recorded batch is rejected the same way.
image = torch.rand(3, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(1, 1, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
aug(image, mask.expand(3, -1, -1, -1).clone())
bad_mask = torch.rand(2, 1, 8, 8, device=device, dtype=dtype)
with pytest.raises(ValueError, match="batch"):
aug(bad_mask, params=aug._params, data_keys=["mask"])
@pytest.mark.parametrize(
"data_format,mask_shape",
[("BTCHW", (8, 8)), ("BTCHW", (4, 8, 8)), ("BTCHW", (1, 2, 4, 1, 8, 8)), ("BCTHW", (2, 4, 8, 8))],
)
def test_video_ambiguous_mask_rank_raises_5624(self, data_format, mask_shape, device, dtype):
# Next to a video a mask is 5D, or (B, T, H, W) for a BTCHW video; any other rank is ambiguous (missing
# channel? missing batch? a plain 2D mask?), so it must raise here rather than fail later inside
# VideoSequential's own reshape. A BCTHW video reads its frames from dim 2, which a 4D mask does not have.
shape = (2, 4, 3, 8, 8) if data_format == "BTCHW" else (2, 3, 4, 8, 8)
image = torch.rand(*shape, device=device, dtype=dtype)
mask = torch.rand(*mask_shape, device=device, dtype=dtype)
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomHorizontalFlip(p=1.0), data_format=data_format), data_keys=["input", "mask"]
)
with pytest.raises(ValueError, match="5D"):
aug(image, mask)
@pytest.mark.parametrize("mask_batch", [1, 2])
def test_video_bthw_mask_equals_the_5d_mask_5624(self, mask_batch, device, dtype):
# A BTCHW video reads a (B, T, H, W) mask with one channel and repeats a single (1, T, H, W) mask like a 5D
# one. The default same_on_frame=True also needs the batch_prob rule of #5598 for a 4D mask.
video = torch.rand(2, 4, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(mask_batch, 4, 8, 8, device=device, dtype=dtype)
outputs = []
for candidate in (mask, mask.expand(2, -1, -1, -1)[:, :, None].clone()):
torch.manual_seed(5)
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomAffine(30.0, p=1.0), same_on_frame=False), data_keys=["input", "mask"]
)
outputs.append(aug(video, candidate)[1])
assert outputs[0].shape == (2, 4, 1, 8, 8)
assert_close(outputs[0], outputs[1])
def test_5d_mask_without_a_video_raises_5624(self, device, dtype):
# The mirror case: a 5D mask is only meaningful next to a video. Next to a plain 4D image it is
# ambiguous, so it must raise here rather than fail later with a confusing shape error.
image = torch.rand(2, 3, 8, 8, device=device, dtype=dtype)
mask = torch.rand(1, 4, 1, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
with pytest.raises(ValueError, match="5D"):
aug(image, mask)
def test_3d_augmentation_replay_mask_keeps_its_own_error_5624(self, device, dtype):
# A 3D augmentation's own "not yet supported" mask error must still surface on replay, not the
# video/5D-mask ValueError meant for a video (3D volumes and videos are both 5D).
volume = torch.rand(2, 1, 4, 8, 8, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip3D(p=1.0), data_keys=["input", "mask"])
aug(volume, data_keys=["input"])
mask = torch.rand(2, 1, 4, 8, 8, device=device, dtype=dtype)
with pytest.raises(NotImplementedError, match="3d mask"):
aug(mask, params=aug._params, data_keys=["mask"])
def test_call_time_data_keys_are_restored_after_forward_exception(self, device, dtype):
image = torch.rand(1, 3, 16, 20, device=device, dtype=dtype)
mask = torch.ones(1, 1, 16, 20, device=device, dtype=dtype)
aug = K.AugmentationSequential(
K.RandomThinPlateSpline(p=1.0),
data_keys=["input"],
)
with pytest.raises(NotImplementedError):
aug(image, mask, data_keys=["input", "mask"])
assert aug.transform_op.data_keys == aug.data_keys
# A later call without call-time data_keys must use the container defaults.
out = aug(image)
assert out.shape == image.shape
def test_call_time_data_keys_are_restored_after_inverse_exception(self, device, dtype):
image = torch.rand(1, 1, 4, 8, 8, device=device, dtype=dtype)
class_label = torch.tensor([1], device=device)
aug = K.AugmentationSequential(
K.RandomHorizontalFlip3D(p=1.0),
data_keys=["input"],
)
outputs = aug(image, class_label, data_keys=["input", "class"])
with pytest.raises(NotImplementedError, match="3d inverse"):
aug.inverse(*outputs, data_keys=["input", "class"])
assert aug.transform_op.data_keys == aug.data_keys
# The next keyless call must still use the container defaults.
out = aug(image)
assert out.shape == image.shape
def test_call_time_data_keys_are_restored_after_successful_inverse(self, device, dtype):
# A successful ``inverse`` with call-time keys must not leave them behind either (#5136).
image = torch.rand(1, 3, 8, 10, device=device, dtype=dtype)
mask = torch.ones(1, 1, 8, 10, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input"])
out_image, out_mask = aug(image, mask, data_keys=["input", "mask"])
aug.inverse(out_image, out_mask, data_keys=["input", "mask"])
assert aug.transform_op.data_keys == aug.data_keys
out = aug(image)
assert out.shape == image.shape
def test_video(self, device, dtype):
input = torch.randn(2, 3, 5, 6, device=device, dtype=dtype)[None]
bbox = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], device=device, dtype=dtype).expand(
2, 1, -1, -1
)[None]
points = torch.tensor([[[1.0, 1.0]]], device=device, dtype=dtype).expand(2, -1, -1)[None]
aug_list = K.AugmentationSequential(
K.VideoSequential(
kornia.augmentation.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), kornia.augmentation.RandomAffine(360, p=1.0)
),
data_keys=["input", "mask", "bbox", "keypoints"],
)
out = aug_list(input, input, bbox, points)
assert out[0].shape == input.shape
assert out[1].shape == input.shape
assert out[2].shape == bbox.shape
assert out[3].shape == points.shape
out_inv = aug_list.inverse(*out)
assert out_inv[0].shape == input.shape
assert out_inv[1].shape == input.shape
assert out_inv[2].shape == bbox.shape
assert out_inv[3].shape == points.shape
def test_3d_augmentations(self, device, dtype):
input = torch.randn(2, 2, 3, 5, 6, device=device, dtype=dtype)
aug_list = K.AugmentationSequential(
K.RandomAffine3D(360.0, p=1.0), K.RandomHorizontalFlip3D(p=1.0), data_keys=["input"]
)
out = aug_list(input)
assert out.shape == input.shape
def test_3d_augmentation_rejects_4d_input(self, device, dtype):
input = torch.randn(2, 3, 5, 6, device=device, dtype=dtype)
aug_list = K.AugmentationSequential(K.RandomAffine3D(360.0, p=1.0), data_keys=["input"])
with pytest.raises(
RuntimeError,
match=r"Input size must have a shape of either \(D, H, W\) or \(B, C, D, H, W\)",
):
aug_list(input)
def test_3d_augmentation_rejects_4d_input_when_replaying_params(self, device, dtype):
input_5d = torch.randn(1, 2, 3, 5, 6, device=device, dtype=dtype)
aug_list = K.AugmentationSequential(K.RandomAffine3D(360.0, p=1.0), data_keys=["input"])
aug_list(input_5d)
params = aug_list._params
input_4d = torch.randn(2, 3, 5, 6, device=device, dtype=dtype)
with pytest.raises(
RuntimeError,
match=r"Input size must have a shape of either \(D, H, W\) or \(B, C, D, H, W\)",
):
aug_list(input_4d, params=params)
def test_identity_matrix_3d(self, device, dtype):
input = torch.rand(2, 1, 3, 4, 5, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomDepthicalFlip3D(p=1.0))
assert aug.contains_3d_augmentation
matrix = aug.identity_matrix(input)
assert matrix.shape == (2, 4, 4)
expected = torch.eye(4, device=device, dtype=dtype).expand(2, -1, -1)
assert_close(matrix, expected)
@pytest.mark.parametrize("image_dtype", [torch.float16, torch.float32, torch.float64, torch.bfloat16])
def test_mixed_image_bbox_dtypes(self, device, image_dtype):
# Regression test for https://github.com/kornia/kornia/issues/3705 and #3706:
# bbox stays in fp32 while the image uses a half/double compute dtype.
if device.type == "mps" and image_dtype == torch.float64:
pytest.skip("MPS does not support float64")
if device.type == "cpu" and image_dtype in (torch.float16, torch.bfloat16) and torch_version_lt(2, 6, 0):
pytest.skip("PyTorch <2.6 has no CPU Half/BFloat16 grid_sample kernel")
torch.manual_seed(0)
img = torch.rand(2, 3, 32, 32, device=device, dtype=image_dtype)
bb = torch.tensor(
[[[[4.0, 4.0], [12.0, 4.0], [12.0, 12.0], [4.0, 12.0]]]] * 2,
device=device,
dtype=torch.float32,
)
aug = K.AugmentationSequential(K.RandomAffine(degrees=10, p=1.0), data_keys=["image", "bbox"])
out_img, out_bb = aug(img, bb)
assert out_img.dtype == image_dtype
assert out_bb.dtype == torch.float32
assert out_img.shape == img.shape
assert out_bb.shape == bb.shape
def test_random_flips(self, device, dtype):
inp = torch.randn(1, 3, 255, 510, device=device, dtype=dtype)
bbox = torch.tensor([[[177, 5], [330, 5], [330, 125], [177, 125]]], device=device, dtype=dtype)
expected_bbox_vertical_flip = torch.tensor(
[[[177, 129], [330, 129], [330, 249], [177, 249]]], device=device, dtype=dtype
)
expected_bbox_horizontal_flip = torch.tensor(
[[[179, 5], [332, 5], [332, 125], [179, 125]]], device=device, dtype=dtype
)
aug_ver = K.AugmentationSequential(
K.RandomVerticalFlip(p=1.0), data_keys=["input", "bbox"], same_on_batch=False
)
aug_hor = K.AugmentationSequential(
K.RandomHorizontalFlip(p=1.0), data_keys=["image", "bbox"], same_on_batch=False
)
out_ver = aug_ver(inp.clone(), bbox.clone())
out_hor = aug_hor(inp.clone(), bbox.clone())
assert_close(out_ver[1], expected_bbox_vertical_flip)
assert_close(out_hor[1], expected_bbox_horizontal_flip)
def test_with_mosaic(self, device, dtype):
width, height = 100, 100
crop_width, crop_height = 3, 3
input = torch.randn(3, 3, width, height, device=device, dtype=dtype)
bbox = torch.tensor(
[[[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]]], device=device, dtype=dtype
).expand(3, -1, -1)
aug = K.AugmentationSequential(
K.RandomCrop((crop_width, crop_height), padding=1, cropping_mode="resample", fill=0),
K.RandomHorizontalFlip(p=1.0),
K.RandomMosaic(p=1.0),
data_keys=["input", "bbox_xyxy"],
)
reproducibility_test((input, bbox), aug)
@pytest.mark.parametrize("num_boxes", [1, 2])
def test_convention_padded_random_crop_accepts_rank2_bboxes_4244(self, num_boxes, device, dtype):
# kornia#4244: RandomCrop's padded path routed bounding boxes through Boxes.pad and
# Boxes.unpad until #4801. A rank-2 (N, 4) bbox for a single image builds an *unbatched* Boxes
# container, which those methods crashed on with "output with shape [1, 4] doesn't match
# the broadcast shape [1, 1, 4]". This is the reproducer from the issue's follow-up, and it
# is the only public route to that container: a rank-3 (B, N, 4) bbox builds a batched one,
# which never crashed. Both box counts are covered because clamp failed differently at N=1
# (IndexError) and N=2 (RuntimeError) on the unbatched path.
input = torch.rand(1, 3, 8, 8, device=device, dtype=dtype)
bbox = torch.tensor([[1.0, 1.0, 4.0, 4.0], [2.0, 2.0, 5.0, 6.0]], device=device, dtype=dtype)[:num_boxes]
aug = K.AugmentationSequential(K.RandomCrop((6, 6), padding=1, p=1.0), data_keys=["input", "bbox_xyxy"])
out_input, out_bbox = aug(input, bbox)
assert out_input.shape == (1, 3, 6, 6)
# the rank of the caller's bbox is preserved on the way out
assert out_bbox.shape == (num_boxes, 4)
assert torch.isfinite(out_bbox).all()
@pytest.mark.parametrize("batch_size", [1, 2])
def test_padded_random_crop_batched_bboxes(self, batch_size, device, dtype):
# The batched (B, N, 4) companion of the pin above. It builds a batched Boxes container,
# which always worked, so it passes on either side of the fix and is kept only as a guard
# that the rank-2 repair did not disturb the batched route.
input = torch.rand(batch_size, 3, 8, 8, device=device, dtype=dtype)
bbox = torch.tensor([[[1.0, 1.0, 4.0, 4.0]]], device=device, dtype=dtype).expand(batch_size, -1, -1)
aug = K.AugmentationSequential(K.RandomCrop((6, 6), padding=1, p=1.0), data_keys=["input", "bbox_xyxy"])
out_input, out_bbox = aug(input, bbox)
assert out_input.shape == (batch_size, 3, 6, 6)
assert out_bbox.shape == (batch_size, 1, 4)
assert torch.isfinite(out_bbox).all()
@pytest.mark.parametrize("padding", [None, 2])
def test_random_crop_accepts_unbatched_keypoints_5021(self, padding, device, dtype):
# kornia#5021: RandomCrop routes keypoints through Keypoints.pad, with or without padding. A
# rank-2 (N, 2) keypoints tensor for a single image builds an *unbatched* Keypoints container,
# which pad crashed on with "output with shape [2] doesn't match the broadcast shape [1, 2]".
# The same crop on the singleton-batch (1, N, 2) keypoints always worked and is the reference.
input = torch.rand(1, 1, 7, 11, device=device, dtype=dtype)
keypoints = torch.tensor([[8.0, 2.0], [3.0, 5.0]], device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomCrop((5, 9), padding=padding, p=1.0), data_keys=["input", "keypoints"])
torch.manual_seed(0)
_, expected = aug(input, keypoints[None])
torch.manual_seed(0)
out_input, out_keypoints = aug(input, keypoints)
assert out_input.shape == (1, 1, 5, 9)
# the rank of the caller's keypoints is preserved on the way out
assert out_keypoints.shape == (2, 2)
assert_close(out_keypoints, expected[0])
def test_random_crops_and_flips(self, device, dtype):
width, height = 100, 100
crop_width, crop_height = 3, 3
input = torch.randn(3, 3, width, height, device=device, dtype=dtype)
bbox = torch.tensor(
[[[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]]], device=device, dtype=dtype
).expand(3, -1, -1)
aug = K.AugmentationSequential(
K.RandomCrop((crop_width, crop_height), padding=1, cropping_mode="resample", fill=0),
K.RandomHorizontalFlip(p=1.0),
data_keys=["input", "bbox_xyxy"],
)
reproducibility_test((input, bbox), aug)
_params = aug.forward_parameters(input.shape)
# specifying the crop locations allows us to compute by hand the expected outputs
crop_locations = torch.tensor(
[[1.0, 2.0], [1.0, 1.0], [2.0, 0.0]],
device=_params[0].data["src"].device,
dtype=_params[0].data["src"].dtype,
)
crops = crop_locations.expand(4, -1, -1).permute(1, 0, 2).clone()
crops[:, 1:3, 0] += crop_width - 1
crops[:, 2:4, 1] += crop_height - 1
_params[0].data["src"] = crops
# expected output bboxes after crop for specified crop locations and crop size (3,3)
expected_out_bbox = torch.tensor(
[
[[1.0, 0.0, 2.0, 1.0], [0.0, -1.0, 1.0, 1.0], [0.0, -1.0, 2.0, 0.0]],
[[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]],
[[0.0, 2.0, 1.0, 3.0], [-1.0, 1.0, 0.0, 3.0], [-1.0, 1.0, 1.0, 2.0]],
],
device=device,
dtype=dtype,
)
# horizontally flip boxes based on crop width
xmins = expected_out_bbox[..., 0].clone()
xmaxs = expected_out_bbox[..., 2].clone()
expected_out_bbox[..., 0] = crop_width - xmaxs - 1
expected_out_bbox[..., 2] = crop_width - xmins - 1
out = aug(input, bbox, params=_params)
assert out[1].shape == bbox.shape
assert_close(out[1], expected_out_bbox, atol=1e-4, rtol=1e-4)
out_inv = aug.inverse(*out)
assert out_inv[1].shape == bbox.shape
assert_close(out_inv[1], bbox, atol=1e-4, rtol=1e-4)
def test_random_erasing(self, device, dtype):
fill_value = 0.5
input = torch.randn(3, 3, 100, 100, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomErasing(p=1.0, value=fill_value), data_keys=["image", "mask"])
reproducibility_test((input, input), aug)
out = aug(input, input)
assert torch.all(out[1][out[0] == fill_value] == 0.0)
def test_resize(self, device, dtype):
size = 50
input = torch.randn(3, 3, 100, 100, device=device, dtype=dtype)
mask = torch.randn(3, 1, 100, 100, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.Resize((size, size), p=1.0), data_keys=["input", "mask"])
reproducibility_test((input, mask), aug)
out = aug(input, mask)
assert out[0].shape == (3, 3, size, size)
assert out[1].shape == (3, 1, size, size)
def test_random_crops(self, device, dtype):
# Test with relaxed tolerance for platform-specific numerical precision
torch.manual_seed(233)
input = torch.randn(3, 3, 3, 3, device=device, dtype=dtype)
bbox = torch.tensor(
[[[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]]], device=device, dtype=dtype
).expand(3, -1, -1)
points = torch.tensor([[[0.0, 0.0], [1.0, 1.0]]], device=device, dtype=dtype).expand(3, -1, -1)
aug = K.AugmentationSequential(
K.RandomCrop((3, 3), padding=1, cropping_mode="resample", fill=0),
K.RandomAffine((360.0, 360.0), p=1.0),
data_keys=["input", "mask", "bbox_xyxy", "keypoints"],
extra_args={},
)
reproducibility_test((input, input, bbox, points), aug)
_params = aug.forward_parameters(input.shape)
# specifying the crops allows us to compute by hand the expected outputs
_params[0].data["src"] = torch.tensor(
[
[[1.0, 2.0], [3.0, 2.0], [3.0, 4.0], [1.0, 4.0]],
[[1.0, 1.0], [3.0, 1.0], [3.0, 3.0], [1.0, 3.0]],
[[2.0, 0.0], [4.0, 0.0], [4.0, 2.0], [2.0, 2.0]],
],
device=_params[0].data["src"].device,
dtype=_params[0].data["src"].dtype,
)
expected_out_bbox = torch.tensor(
[
[[1.0, 0.0, 2.0, 1.0], [0.0, -1.0, 1.0, 1.0], [0.0, -1.0, 2.0, 0.0]],
[[1.0, 1.0, 2.0, 2.0], [0.0, 0.0, 1.0, 2.0], [0.0, 0.0, 2.0, 1.0]],
[[0.0, 2.0, 1.0, 3.0], [-1.0, 1.0, 0.0, 3.0], [-1.0, 1.0, 1.0, 2.0]],
],
device=device,
dtype=dtype,
)
expected_out_points = torch.tensor(
[[[0.0, -1.0], [1.0, 0.0]], [[0.0, 0.0], [1.0, 1.0]], [[-1.0, 1.0], [0.0, 2.0]]], device=device, dtype=dtype
)
out = aug(input, input, bbox, points, params=_params)
assert out[0].shape == (3, 3, 3, 3)
assert_close(out[0], out[1], atol=1e-4, rtol=1e-4)
assert out[2].shape == bbox.shape
assert_close(out[2], expected_out_bbox, atol=1e-3, rtol=1e-3)
assert out[3].shape == points.shape
assert_close(out[3], expected_out_points, atol=1e-4, rtol=1e-4)
out_inv = aug.inverse(*out)
assert out_inv[0].shape == input.shape
assert_close(out_inv[0], out_inv[1], atol=1e-4, rtol=1e-4)
assert out_inv[2].shape == bbox.shape
assert_close(out_inv[2], bbox, atol=1e-3, rtol=1e-3)
assert out_inv[3].shape == points.shape
assert_close(out_inv[3], points, atol=1e-4, rtol=1e-4)
def test_random_resized_crop(self, device, dtype):
size = 50
input = torch.randn(3, 3, 100, 100, device=device, dtype=dtype)
mask = torch.randn(3, 1, 100, 100, device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomResizedCrop((size, size), p=1.0), data_keys=["input", "mask"])
reproducibility_test((input, mask), aug)
out = aug(input, mask)
assert out[0].shape == (3, 3, size, size)
assert out[1].shape == (3, 1, size, size)
@pytest.mark.parametrize(
"bbox",
[
[
torch.tensor([[1, 5, 2, 7], [0, 3, 9, 9]]),
torch.tensor([[1, 5, 2, 7], [0, 3, 9, 9], [0, 5, 8, 7]]),
torch.empty((0, 4)),
],
torch.empty((3, 0, 4)),
torch.tensor([[[1, 5, 2, 7], [0, 3, 9, 9]], [[1, 5, 2, 7], [0, 3, 9, 9]], [[0, 5, 8, 7], [0, 2, 5, 5]]]),
],
)
@pytest.mark.parametrize(
"augmentation",
[K.RandomCrop((30, 30), padding=1, pad_if_needed=True, cropping_mode="resample", fill=0), K.Resize((30, 30))],
)
def test_bbox(self, bbox, augmentation, device, dtype):
img = torch.rand((3, 3, 10, 10), device=device, dtype=dtype)
if isinstance(bbox, list):
for i, b in enumerate(bbox):
bbox[i] = b.to(device=device, dtype=dtype)
else:
bbox = bbox.to(device=device, dtype=dtype)
inputs = [img, bbox]
aug = K.AugmentationSequential(augmentation, data_keys=["input", "bbox_xyxy"])
transformed = aug(*inputs)
assert len(transformed) == len(inputs)
bboxes_transformed = transformed[-1]
assert len(bboxes_transformed) == len(bbox)
assert bboxes_transformed.__class__ == bbox.__class__
for i in range(len(bbox)):
assert len(bboxes_transformed[i]) == len(bbox[i])
def test_class(self, device, dtype):
img = torch.zeros((5, 1, 5, 5))
labels = torch.randint(0, 10, size=(5, 1))
aug = K.AugmentationSequential(K.RandomCrop((3, 3), pad_if_needed=True), data_keys=["input", "class"])
_, out_labels = aug(img, labels)
assert labels is out_labels
@pytest.mark.slow
@pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 10, True, False])
def test_forward_and_inverse(self, random_apply, device, dtype):
inp = torch.randn(1, 3, 1000, 500, device=device, dtype=dtype)
bbox = torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype)
keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype)
mask = bbox_to_mask(
torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), size=(1000, 500)
)[:, None]
aug = K.AugmentationSequential(
K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)),
K.AugmentationSequential(
K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0),
K.RandomAffine(360, p=1.0),
K.RandomAffine(360, p=1.0),
data_keys=["input", "mask", "bbox", "keypoints"],
),
K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0),
K.RandomAffine(360, p=1.0),
data_keys=["input", "mask", "bbox", "keypoints"],
random_apply=random_apply,
)
out = aug(inp, mask, bbox, keypoints)
assert out[0].shape == inp.shape
assert out[1].shape == mask.shape
assert out[2].shape == bbox.shape
assert out[3].shape == keypoints.shape
assert set(out[1].unique().tolist()).issubset(set(mask.unique().tolist()))
out_inv = aug.inverse(*out)
assert out_inv[0].shape == inp.shape
assert out_inv[1].shape == mask.shape
assert out_inv[2].shape == bbox.shape
assert out_inv[3].shape == keypoints.shape
assert set(out_inv[1].unique().tolist()).issubset(set(mask.unique().tolist()))
if random_apply is False:
reproducibility_test((inp, mask, bbox, keypoints), aug)
@pytest.mark.slow
def test_individual_forward_and_inverse(self, device, dtype):
inp = torch.randn(1, 3, 1000, 500, device=device, dtype=dtype)
bbox = torch.tensor([[[[355, 10], [660, 10], [660, 250], [355, 250]]]], device=device, dtype=dtype)
keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype)
mask = bbox_to_mask(
torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), size=(1000, 500)
)[:, None]
crop_size = (200, 200)
aug = K.AugmentationSequential(
K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)),
K.AugmentationSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)),
K.RandomAffine(360, p=1.0),
K.RandomCrop(crop_size, padding=1, cropping_mode="resample", fill=0),
data_keys=["input", "mask", "bbox", "keypoints"],
extra_args={},
)
# NOTE: Mask data with nearest not passing reproducibility check under float64.
reproducibility_test((inp, mask, bbox, keypoints), aug)
out = aug(inp, mask, bbox, keypoints)
assert out[0].shape == (*inp.shape[:2], *crop_size)
assert out[1].shape == (*mask.shape[:2], *crop_size)
assert out[2].shape == bbox.shape
assert out[3].shape == keypoints.shape
out_inv = aug.inverse(*out)
assert out_inv[0].shape == inp.shape
assert out_inv[1].shape == mask.shape
assert out_inv[2].shape == bbox.shape
assert out_inv[3].shape == keypoints.shape
aug = K.AugmentationSequential(K.RandomAffine(360, p=1.0))
assert aug(inp, data_keys=["input"]).shape == inp.shape
aug = K.AugmentationSequential(K.RandomAffine(360, p=1.0))
assert aug(inp, data_keys=["input"]).shape == inp.shape
assert aug(mask, data_keys=["mask"], params=aug._params).shape == mask.shape
assert aug.inverse(inp, data_keys=["input"]).shape == inp.shape
assert aug.inverse(bbox, data_keys=["bbox"]).shape == bbox.shape
assert aug.inverse(keypoints, data_keys=["keypoints"]).shape == keypoints.shape
assert aug.inverse(mask, data_keys=["mask"]).shape == mask.shape
@pytest.mark.slow
@pytest.mark.parametrize("random_apply", [2, (1, 1), (2,), 10, True, False])
def test_forward_and_inverse_return_transform(self, random_apply, device, dtype):
inp = torch.randn(1, 3, 1000, 500, device=device, dtype=dtype)
bbox = torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype)
keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype)
mask = bbox_to_mask(
torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), size=(1000, 500)
)[:, None]
aug = K.AugmentationSequential(
K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)),
K.AugmentationSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)),
K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0),
K.RandomAffine(360, p=1.0),
data_keys=["input", "mask", "bbox", "keypoints"],
random_apply=random_apply,
extra_args={},
)
out = aug(inp, mask, bbox, keypoints)
assert out[0].shape == inp.shape
assert out[1].shape == mask.shape
assert out[2].shape == bbox.shape
assert out[3].shape == keypoints.shape
reproducibility_test((inp, mask, bbox, keypoints), aug)
out_inv = aug.inverse(*out)
assert out_inv[0].shape == inp.shape
assert out_inv[1].shape == mask.shape
assert out_inv[2].shape == bbox.shape
assert out_inv[3].shape == keypoints.shape
@pytest.mark.slow
@pytest.mark.parametrize("random_apply", [1, (2, 2), (1, 2), (2,), 10, True, False])
def test_inverse_and_forward_return_transform(self, random_apply, device, dtype):
inp = torch.randn(1, 3, 1000, 500, device=device, dtype=dtype)
bbox = torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype)
bbox_2 = [
# torch.tensor([[[355, 10], [660, 10], [660, 250], [355, 250]]], device=device, dtype=dtype),
torch.tensor(
[[[355, 10], [660, 10], [660, 250], [355, 250]], [[355, 10], [660, 10], [660, 250], [355, 250]]],
device=device,
dtype=dtype,
)
]
bbox_wh = torch.tensor([[[30, 40, 100, 100]]], device=device, dtype=dtype)
bbox_wh_2 = [
# torch.tensor([[30, 40, 100, 100]], device=device, dtype=dtype),
torch.tensor([[30, 40, 100, 100], [30, 40, 100, 100]], device=device, dtype=dtype)
]
keypoints = torch.tensor([[[465, 115], [545, 116]]], device=device, dtype=dtype)
mask = bbox_to_mask(
torch.tensor([[[155, 0], [900, 0], [900, 400], [155, 400]]], device=device, dtype=dtype), size=(1000, 500)
)[:, None]
aug = K.AugmentationSequential(
K.ImageSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)),
K.AugmentationSequential(K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0)),
K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0),
K.RandomAffine(360, p=1.0),
data_keys=["input", "mask", "bbox", "keypoints", "bbox", "BBOX_XYWH", "BBOX_XYWH"],
random_apply=random_apply,
)
with pytest.raises(ValueError, match=r"No parameters available for inversing"):
aug.inverse(inp, mask, bbox, keypoints, bbox_2, bbox_wh, bbox_wh_2)
out = aug(inp, mask, bbox, keypoints, bbox_2, bbox_wh, bbox_wh_2)
assert out[0].shape == inp.shape
assert out[1].shape == mask.shape
assert out[2].shape == bbox.shape
assert out[3].shape == keypoints.shape
if random_apply is False:
reproducibility_test((inp, mask, bbox, keypoints, bbox_2, bbox_wh, bbox_wh_2), aug)
@pytest.mark.skip(reason="turn off due to Union Type")
def test_jit(self, device, dtype):
B, C, H, W = 2, 3, 4, 4
img = torch.ones(B, C, H, W, device=device, dtype=dtype)
op = K.AugmentationSequential(
K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0), K.RandomAffine(360, p=1.0), same_on_batch=True
)
op_jit = torch.jit.script(op)
assert_close(op(img), op_jit(img))
@pytest.mark.parametrize("batch_prob", [[True, True], [False, True], [False, False]])
@pytest.mark.parametrize("box", ["bbox", "bbox_xyxy", "bbox_xywh"])
def test_autocast(self, batch_prob, box, device, dtype):
if not hasattr(torch, "autocast"):
pytest.skip("PyTorch version without autocast support")
def mock_forward_parameters_sequential(batch_shape, cls, batch_prob):
named_modules = cls.get_forward_sequence()
params = []
for name, module in named_modules:
if isinstance(module, (K.base._AugmentationBase, K.MixAugmentationBaseV2, K.ImageSequential)):
with patch.object(module, "__batch_prob_generator__", return_value=batch_prob):
mod_param = module.forward_parameters(batch_shape)
param = ParamItem(name, mod_param)
else:
param = ParamItem(name, None)
batch_shape = K.container.image._get_new_batch_shape(param, batch_shape)
params.append(param)
return params
tfs = (K.RandomAffine(0.5, (0.1, 0.5), (0.5, 1.5), 1.2, p=1.0), K.RandomGaussianBlur((3, 3), (0.1, 3), p=1))
data_keys = ["input", "mask", box, "keypoints"]
aug = K.AugmentationSequential(*tfs, data_keys=data_keys, random_apply=True)
bs = len(batch_prob)
imgs = torch.rand(bs, 3, 7, 4, dtype=dtype, device=device)
if box == "bbox":
bb = torch.tensor([[[1.0, 1.0], [2.0, 1.0], [2.0, 2.0], [1.0, 2.0]]], dtype=dtype, device=device).expand(
bs, 1, -1, -1
)
else:
bb = torch.rand(bs, 1, 4, dtype=dtype, device=device)
msk = torch.zeros_like(imgs)
msk[..., 3:, 2] = 1.0
points = torch.rand(bs, 1, 2, dtype=dtype, device=device)
to_apply = torch.tensor(batch_prob, device=device)
fwd_params = partial(mock_forward_parameters_sequential, cls=aug, batch_prob=to_apply)
with patch.object(aug, "forward_parameters", fwd_params):
params = aug.forward_parameters(imgs.shape)
with torch.autocast(device.type):
outputs = aug(imgs, msk, bb, points, params=params)
assert outputs[0].dtype == dtype, "Output image dtype should match the input dtype"
assert outputs[1].dtype == dtype, "Output mask dtype should match the input dtype"
assert outputs[2].dtype == dtype, "Output box dtype should match the input dtype"
assert outputs[3].dtype == dtype, "Output keypoints dtype should match the input dtype"
class _FailingInverseCrop(K.RandomResizedCrop):
# A crop whose image and mask inverse raises, to check what an exception inside inverse leaves behind.
def inverse_transform(self, *args, **kwargs):
raise NotImplementedError("inverse_transform failed")
@pytest.mark.usefixtures("restore_torch_rng")
class TestConventionAugmentationSequential(BaseTester):
"""Convention checks and pins for documented `AugmentationSequential` limitations."""
@pytest.mark.parametrize("num_points", [0, 2])
def test_video_keypoints_noncontiguous_and_empty(self, num_points, device, dtype):
image = torch.arange(36, device=device, dtype=dtype).reshape(1, 1, 1, 6, 6).expand(2, 3, -1, -1, -1)
points = torch.arange(3 * 2 * num_points * 2, device=device, dtype=dtype).reshape(3, 2, num_points, 2)
points = (points % 6).transpose(0, 1)
expected = points.clone()
expected[..., 0] = 5 - expected[..., 0]
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomHorizontalFlip(p=1.0), data_format="BTCHW"),
data_keys=["input", "keypoints"],
)
out_image, out_points = aug(image, points)
self.assert_close(out_image, image.flip(-1))
assert out_points.shape == points.shape
self.assert_close(out_points, expected)
_, restored = aug.inverse(out_image, out_points)
self.assert_close(restored, points)
@pytest.mark.parametrize("num_boxes", [0, 2])
def test_video_boxes_noncontiguous_and_empty(self, num_boxes, device, dtype):
image = torch.arange(36, device=device, dtype=dtype).reshape(1, 1, 1, 6, 6).expand(2, 3, -1, -1, -1)
corners = torch.tensor([[1.0, 1.0], [3.0, 1.0], [3.0, 4.0], [1.0, 4.0]], device=device, dtype=dtype)
shift = torch.arange(6, device=device, dtype=dtype).reshape(3, 2, 1, 1, 1) * 0.25
boxes = (corners + shift).expand(3, 2, num_boxes, 4, 2).transpose(0, 1)
expected = boxes.clone()
expected[..., 0] = 5 - expected[..., 0]
expected = expected[..., [1, 0, 3, 2], :]
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomHorizontalFlip(p=1.0), data_format="BTCHW"),
data_keys=["input", "bbox"],
)
out_image, out_boxes = aug(image, boxes)
assert out_boxes.shape == boxes.shape
self.assert_close(out_boxes, expected)
_, restored = aug.inverse(out_image, out_boxes)
self.assert_close(restored, boxes)
def test_video_boxes_inverse_with_partial_gate_5871(self, device, dtype):
torch.manual_seed(0)
corners = torch.tensor([[1.0, 1.0], [3.0, 1.0], [3.0, 4.0], [1.0, 4.0]], device=device, dtype=dtype)
boxes = corners.expand(4, 3, 2, 4, 2).clone()
aug = K.AugmentationSequential(
K.VideoSequential(K.RandomHorizontalFlip(p=0.5), data_format="BTCHW"),
data_keys=["input", "bbox"],
)
image, out_boxes = aug(torch.rand(4, 3, 1, 6, 6, device=device, dtype=dtype), boxes)
# The gate must mix applied and skipped frames, otherwise the round-trip proves nothing.
flipped = (out_boxes.data.view(4, 3, 2, 4, 2) - boxes).abs().sum(dim=(-1, -2, -3)) > 0
assert bool(flipped.any()), flipped.tolist()
assert not bool(flipped.all()), flipped.tolist()
_, restored = aug.inverse(image, out_boxes)
self.assert_close(restored, boxes)
@pytest.mark.parametrize("batch_size", [2, 3])
@pytest.mark.parametrize("keepdim,child_keepdim", [(True, False), (None, True), (False, True)])
@pytest.mark.parametrize("mask_layout", ["BHW", "BCHW"])
@pytest.mark.parametrize("augmentation", ["flip", "affine", "crop"])
def test_inverse_mask_shape_5597(
self, batch_size, keepdim, child_keepdim, mask_layout, augmentation, device, dtype
):
image = torch.rand(batch_size, 3, 8, 10, device=device, dtype=dtype)
mask = (torch.arange(batch_size * 80, device=device).reshape(batch_size, 8, 10) % 3 == 0).to(dtype)
if mask_layout == "BCHW":
mask = mask.unsqueeze(1)
if augmentation == "flip":
child = K.RandomHorizontalFlip(p=1.0, keepdim=child_keepdim)
elif augmentation == "affine":
child = K.RandomAffine((-60.0, 60.0), p=1.0, keepdim=child_keepdim)
else:
child = K.RandomCrop((5, 7), cropping_mode="resample", p=1.0, keepdim=child_keepdim)
seq = K.AugmentationSequential(child, data_keys=["image", "mask"], keepdim=keepdim)
out_image, out_mask = seq(image, mask)
inv_image, inv_mask = seq.inverse(out_image, out_mask)
expected_mask = mask.unsqueeze(1) if mask_layout == "BHW" and keepdim is False else mask
assert inv_mask.shape == expected_mask.shape
assert inv_mask.dtype == mask.dtype
assert inv_mask.device == mask.device
assert inv_image.shape == image.shape
assert inv_image.dtype == image.dtype
assert inv_image.device == image.device
if augmentation == "flip":
self.assert_close(inv_image, image, atol=0, rtol=0)
self.assert_close(inv_mask, expected_mask, atol=0, rtol=0)
# The established BCHW path is a reference for lossy warps; replay exactly the same parameters.
reference_mask = out_mask.unsqueeze(1) if out_mask.ndim == 3 else out_mask
reference_image, reference_mask = seq.inverse(out_image, reference_mask)
self.assert_close(inv_image, reference_image, atol=0, rtol=0)
self.assert_close(inv_mask.reshape(reference_mask.shape), reference_mask, atol=0, rtol=0)
@pytest.mark.parametrize("batch_prob", [[True, True, True], [False, True, False], [False, False, False]])
@pytest.mark.parametrize("mask_dtype", [torch.bool, torch.int64])
def test_inverse_bhw_mask_gates_and_dtype_5597(self, batch_prob, mask_dtype, device, dtype):
image = torch.rand(3, 2, 6, 8, device=device, dtype=dtype)
mask = (torch.arange(3 * 48, device=device).reshape(3, 6, 8) % 3 == 0).to(mask_dtype)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.5), data_keys=["image", "mask"], keepdim=True)
params = seq.forward_parameters(image.shape)
params[0].data["batch_prob"] = torch.tensor(batch_prob, device=device)
inv_image, inv_mask = seq.inverse(*seq(image, mask, params=params))
self.assert_close(inv_image, image, atol=0, rtol=0)
self.assert_close(inv_mask, mask, atol=0, rtol=0)
assert inv_mask.dtype == mask_dtype
assert inv_mask.device == mask.device
@pytest.mark.parametrize("shape", [(6, 8), (2, 6, 8), (1, 2, 6, 8)])
@pytest.mark.parametrize("keepdim", [True, False])
def test_inverse_unbatched_image_and_mask_shape_5597(self, shape, keepdim, device, dtype):
image = torch.rand(shape, device=device, dtype=dtype)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["image", "mask"], keepdim=keepdim)
inv_image, inv_mask = seq.inverse(*seq(image, image.clone()))
expected = image if keepdim else image.reshape(1, -1, 6, 8)
self.assert_close(inv_image, expected, atol=0, rtol=0)
self.assert_close(inv_mask, expected, atol=0, rtol=0)
@pytest.mark.parametrize("keepdim", [False, None])
def test_inverse_bhw_mask_without_keepdim_5597(self, keepdim, device, dtype):
# A (B, H, W) mask that is not a forward output (a prediction, say) is read per sample, as the forward
# reads it, and comes back in the batch form (B, 1, H, W) when keepdim is off.
image = torch.arange(3 * 2 * 6 * 8, device=device, dtype=dtype).reshape(3, 2, 6, 8) / 288
mask = (torch.arange(3 * 6 * 8, device=device).reshape(3, 6, 8) % 3 == 0).to(dtype)
seq = K.AugmentationSequential(
K.RandomAffine((-60.0, 60.0), p=1.0), data_keys=["image", "mask"], keepdim=keepdim
)
out_image = seq(image, data_keys=["image"])
_, inv_mask = seq.inverse(out_image, mask, data_keys=["image", "mask"])
_, reference = seq.inverse(out_image, mask.unsqueeze(1), data_keys=["image", "mask"])
assert inv_mask.shape == (3, 1, 6, 8)
self.assert_close(inv_mask, reference, atol=0, rtol=0)
@pytest.mark.parametrize("mask_first", [True, False])
def test_inverse_mask_exception_preserves_next_image_5290(self, mask_first, device, dtype):
image = torch.arange(16 * 20, device=device, dtype=dtype).reshape(1, 1, 16, 20) / 320
mask = (image > 0.5).to(dtype)
seq = K.AugmentationSequential(_FailingInverseCrop((12, 14), p=1.0), data_keys=["input", "mask"])
reference = K.AugmentationSequential(_FailingInverseCrop((12, 14), p=1.0), data_keys=["input", "mask"])
out_image, out_mask = seq(image, mask)
params = seq._params
original_flags = dict(seq[0].flags)
inverse_args = (out_mask, out_image) if mask_first else (out_mask,)
inverse_keys = ["mask", "input"] if mask_first else ["mask"]
with pytest.raises(NotImplementedError, match="inverse_transform failed"):
seq.inverse(*inverse_args, data_keys=inverse_keys)
# Explicit keys keep this independent of the container's separate data_keys restoration bug.
actual, _ = seq(image, mask, params=params, data_keys=["input", "mask"])
expected, _ = reference(image, mask, params=params)
self.assert_close(actual, expected, atol=0, rtol=0)
assert seq[0].flags == original_flags
@pytest.mark.parametrize("align_corners", [None, False, True])
@pytest.mark.parametrize("shared_flags", [True, False])
def test_direct_inverse_mask_exception_preserves_flags_5290(self, align_corners, shared_flags, device, dtype):
aug = _FailingInverseCrop((4, 6), align_corners=True, p=1.0)
mask = aug(torch.ones(1, 1, 6, 8, device=device, dtype=dtype))
flags = aug.flags if shared_flags else dict(aug.flags)
original_flags = dict(flags)
with pytest.raises(NotImplementedError, match="inverse_transform failed"):
aug.inverse_masks(mask, aug._params, flags, transform=aug.transform_matrix, align_corners=align_corners)
assert flags["align_corners"] == original_flags["align_corners"]
assert flags == original_flags
assert aug.flags == original_flags
@pytest.mark.parametrize("align_corners", [None, False, True])
@pytest.mark.parametrize("resample", [None, Resample.BILINEAR])
@pytest.mark.parametrize("mask_layout", ["BCHW", "BHW"])
def test_inverse_mask_preserves_sampling_overrides_5290(self, align_corners, resample, mask_layout, device, dtype):
aug = K.RandomAffine((17.0, 17.0), align_corners=True, p=1.0, keepdim=True)
mask = (torch.arange(3 * 6 * 8, device=device).reshape(3, 1, 6, 8) % 3 == 0).to(dtype)
aug(mask)
transform = torch.linalg.inv(aug.transform_matrix.to(torch.float32)).to(dtype)
original_flags = dict(aug.flags)
kwargs = {"align_corners": align_corners}
if resample is not None:
kwargs["resample"] = resample
input_mask = mask[:, 0] if mask_layout == "BHW" else mask
actual = aug.inverse_masks(input_mask, aug._params, aug.flags, transform=transform, **kwargs)
expected_flags = dict(original_flags)
expected_flags["resample"] = Resample.NEAREST if resample is None else resample
expected_flags["align_corners"] = True if align_corners is None else align_corners
expected = aug.inverse_inputs(mask, aug._params, expected_flags, transform=transform)
if mask_layout == "BHW":
expected = expected[:, 0]
self.assert_close(actual, expected, atol=0, rtol=0)
if resample is None:
assert ((actual == 0) | (actual == 1)).all()
else:
assert ((actual > 0) & (actual < 1)).any()
assert aug.flags == original_flags
assert kwargs["align_corners"] is align_corners
def test_convention_flip_is_integer_centre_inclusive_for_every_data_key(self, device, dtype):
# Convention pin: a horizontal flip maps column x to W - 1 - x, and a vertical flip row y to H - 1 - y,
# for the image, the mask, keypoints and boxes alike - inclusive pixel coordinates about the integer
# centre, the same rule `kornia.geometry.transform.hflip` / `vflip` follow.
# H = 3, W = 4 (asymmetric); hot pixel (row 0, col 1), off both centre lines; keypoint (x=1, y=2).
# Snippet used to generate expected: this body. hflip:
# pixel -> (row 0, col 2), keypoint -> (2, 2), bbox_xyxy [0,0,1,1] -> [2, 0, 3, 1], bbox vertices ->
# [[2,0],[3,0],[3,1],[2,1]]. vflip: pixel -> (row 2, col 1), keypoint -> (1, 0),
# bbox_xyxy -> [0, 1, 1, 2]. An exclusive reading (x' = W - x) would give keypoint (3, 2) and
# box [3, 0, 4, 1].
img = torch.zeros(1, 1, 3, 4, device=device, dtype=dtype)
img[0, 0, 0, 1] = 1.0
kpts = torch.tensor([[[1.0, 2.0]]], device=device, dtype=dtype)
boxes = torch.tensor([[[0.0, 0.0, 1.0, 1.0]]], device=device, dtype=dtype)
vertices = torch.tensor([[[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]]]], device=device, dtype=dtype)
keys = ["input", "mask", "keypoints", "bbox_xyxy", "bbox"]
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=keys)
out_img, out_mask, out_kpts, out_boxes, out_vertices = aug(img, img.clone(), kpts, boxes, vertices)
expected_img = torch.zeros(1, 1, 3, 4, device=device, dtype=dtype)
expected_img[0, 0, 0, 2] = 1.0
self.assert_close(out_img, expected_img)
self.assert_close(out_mask, out_img)
self.assert_close(out_kpts, torch.tensor([[[2.0, 2.0]]], device=device, dtype=dtype))
self.assert_close(out_boxes, torch.tensor([[[2.0, 0.0, 3.0, 1.0]]], device=device, dtype=dtype))
self.assert_close(
out_vertices,
torch.tensor([[[[2.0, 0.0], [3.0, 0.0], [3.0, 1.0], [2.0, 1.0]]]], device=device, dtype=dtype),
)
aug = K.AugmentationSequential(K.RandomVerticalFlip(p=1.0), data_keys=keys[:-1])
out_img, out_mask, out_kpts, out_boxes = aug(img, img.clone(), kpts, boxes)
expected_img = torch.zeros(1, 1, 3, 4, device=device, dtype=dtype)
expected_img[0, 0, 2, 1] = 1.0
self.assert_close(out_img, expected_img)
self.assert_close(out_mask, out_img)
self.assert_close(out_kpts, torch.tensor([[[1.0, 0.0]]], device=device, dtype=dtype))
self.assert_close(out_boxes, torch.tensor([[[0.0, 1.0, 1.0, 2.0]]], device=device, dtype=dtype))
def test_convention_data_key_shapes_are_validated(self, device, dtype):
# Convention pin: each data key has one accepted layout - `bbox` is (B, N, 4, 2) vertices, `bbox_xyxy`
# is (B, N, 4) corners, `keypoints` is (B, N, 2) - and feeding one layout under the other key raises
# `ValueError` naming the expected shape and the box mode.
# Snippet used to generate expected: this body:
# "Boxes shape must be (N, 4, 2) or (B, N, 4, 2) when vertices_plus mode. Got torch.Size([1, 1, 4])."
# "Boxes shape must be (N, 4) or (B, N, 4) when xyxy_plus mode. Got torch.Size([1, 1, 4, 2])."
# "Keypoints shape must be (N, 2) or (B, N, 2). Got torch.Size([1, 2, 3])."
img = torch.rand(1, 1, 3, 4, device=device, dtype=dtype)
cases = [
("bbox", torch.rand(1, 1, 4, device=device, dtype=dtype), "must be .N, 4, 2."),
("bbox_xyxy", torch.rand(1, 1, 4, 2, device=device, dtype=dtype), "must be .N, 4."),
("keypoints", torch.rand(1, 2, 3, device=device, dtype=dtype), "Keypoints shape must be"),
]
for key, wrong, message in cases:
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", key])
with pytest.raises(ValueError, match=message):
aug(img, wrong)
# the accepted layouts pass, including the degenerate N = 0 case
for key, shape in [("bbox", (2, 0, 4, 2)), ("bbox_xyxy", (2, 0, 4)), ("keypoints", (2, 0, 2))]:
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", key])
empty = torch.rand(*shape, device=device, dtype=dtype)
out = aug(torch.rand(2, 1, 3, 4, device=device, dtype=dtype), empty)
assert out[1].shape == shape
# `mask` is the one key whose rank changes: (B, H, W) in, (B, 1, H, W) out
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
out = aug(img, torch.rand(1, 3, 4, device=device, dtype=dtype))
assert out[1].shape == (1, 1, 3, 4)
def test_per_sample_mask_lists_follow_mixed_gates_4477(self, device, dtype):
image = torch.arange(24, device=device, dtype=dtype).reshape(2, 1, 3, 4)
masks = [image[:1].clone(), image[1:].expand(-1, 2, -1, -1).clone()]
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.5), data_keys=["input", "mask"])
params = [
ParamItem(
"RandomHorizontalFlip_0",
{"batch_prob": torch.tensor([1.0, 0.0]), "forward_input_shape": torch.tensor(image.shape)},
)
]
out_image, out_masks = seq(image, masks, params=params)
self.assert_close(out_masks[0], out_image[:1])
self.assert_close(out_masks[1], out_image[1:].expand(-1, 2, -1, -1))
_, restored = seq.inverse(out_image, out_masks, params=params)
for actual, expected in zip(restored, masks):
self.assert_close(actual, expected)
@pytest.mark.parametrize("cropping_mode", ["slice", "resample"])
def test_per_sample_mask_lists_use_own_crop_window_4477(self, cropping_mode, device, dtype):
if cropping_mode == "resample" and not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("The image crop path requires grid_sample for the selected dtype.")
image = torch.arange(16, device=device, dtype=dtype).reshape(1, 1, 4, 4).repeat(2, 1, 1, 1)
masks = [image[:1].clone(), image[1:].expand(-1, 2, -1, -1).clone()]
crop = K.AugmentationSequential(
K.RandomCrop((2, 2), p=1.0, cropping_mode=cropping_mode, resample="nearest"), data_keys=["input", "mask"]
)
params = crop.forward_parameters(image.shape)
params[0].data["src"] = torch.tensor(
[[[0.0, 0.0], [1.0, 0.0], [1.0, 1.0], [0.0, 1.0]], [[2.0, 2.0], [3.0, 2.0], [3.0, 3.0], [2.0, 3.0]]],
device=device,
dtype=dtype,
)
output, out_masks = crop(image, masks, params=params)
self.assert_close(output[0], image[0, :, :2, :2])
self.assert_close(output[1], image[1, :, 2:, 2:])
self.assert_close(out_masks[0], output[:1])
self.assert_close(out_masks[1], output[1:].expand(-1, 2, -1, -1))
@pytest.mark.parametrize("nested", [False, True])
def test_per_sample_mask_lists_warp_replay_and_inverse_4477(self, nested, device, dtype):
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("The image warp path requires grid_sample for the selected dtype.")
image = torch.arange(3 * 8 * 8, device=device, dtype=dtype).reshape(3, 1, 8, 8)
masks = [image[i : i + 1].expand(-1, i + 1, -1, -1).clone() for i in range(3)]
rotation = K.RandomRotation((-90.0, 90.0), p=0.5, resample="nearest")
child = K.ImageSequential(rotation) if nested else rotation
seq = K.AugmentationSequential(child, data_keys=["input", "mask"])
params = seq.forward_parameters(image.shape)
draw = params[0].data[0].data if nested else params[0].data
draw["degrees"] = torch.tensor([-90.0, 90.0, 90.0], device=device, dtype=dtype)
draw["batch_prob"] = torch.tensor([1.0, 0.0, 1.0], device=device, dtype=dtype)
output, out_masks = seq(image, masks, params=params)
for i, mask in enumerate(out_masks):
self.assert_close(mask, output[i : i + 1].expand(-1, i + 1, -1, -1))
# A later image draw must not change replay or inverse of these saved params.
seq(image, data_keys=["input"])
replay_image, replay_masks = seq(image, masks, params=params)
self.assert_close(replay_image, output)
for actual, expected in zip(replay_masks, out_masks):
self.assert_close(actual, expected)
_, restored = seq.inverse(output, out_masks, params=params)
for actual, expected in zip(restored, masks):
self.assert_close(actual, expected)
@pytest.mark.parametrize("rank", [2, 3, 4])
@pytest.mark.parametrize("keepdim", [False, True])
def test_per_sample_mask_list_ranks_4477(self, rank, keepdim, device, dtype):
image = torch.arange(24, device=device, dtype=dtype).reshape(2, 1, 3, 4)
entries = [image[i : i + 1].clone() for i in range(2)]
if rank < 4:
entries = [m.squeeze(0) for m in entries]
if rank == 2:
entries = [m.squeeze(0) for m in entries]
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"], keepdim=keepdim)
output, masks = seq(image, entries)
for i, mask in enumerate(masks):
assert mask.ndim == (rank if keepdim else 4)
self.assert_close(mask.reshape(1, 1, 3, 4), output[i : i + 1])
@pytest.mark.parametrize("padding", [0, 1])
def test_mask_list_crop_inverse_matches_tensor_masks_4477(self, padding, device, dtype):
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("The image crop path requires grid_sample for the selected dtype.")
image = torch.arange(2 * 6 * 8, device=device, dtype=dtype).reshape(2, 1, 6, 8)
entries = [image[:1].clone(), image[1:].expand(-1, 2, -1, -1).clone()]
seq = K.AugmentationSequential(
K.RandomCrop((4, 4), padding=padding, p=1.0, cropping_mode="resample", resample="nearest"),
data_keys=["input", "mask"],
)
output, masks = seq(image, entries)
params = seq._params
_, tensor_mask = seq(image, image.clone(), params=params)
for i, mask in enumerate(masks):
self.assert_close(mask, tensor_mask[i : i + 1].expand(-1, i + 1, -1, -1))
_, inverse_list = seq.inverse(output, masks, params=params)
_, inverse_tensor = seq.inverse(output, tensor_mask, params=params)
for i, mask in enumerate(inverse_list):
self.assert_close(mask, inverse_tensor[i : i + 1].expand(-1, i + 1, -1, -1))
def test_empty_mask_list_4477(self, device, dtype):
image = torch.rand(0, 1, 4, 6, device=device, dtype=dtype)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
output, masks = seq(image, [])
assert output.shape == image.shape
assert masks == []
restored, masks = seq.inverse(output, masks)
assert restored.shape == image.shape
assert masks == []
@pytest.mark.parametrize("layout", ["short", "long", "full_batch", "spatial", "rank1", "rank5"])
def test_invalid_mask_list_layout_4477(self, layout, device, dtype):
image = torch.rand(2, 1, 4, 6, device=device, dtype=dtype)
entries = [image[:1].clone(), image[1:].clone()]
if layout == "short":
entries = entries[:1]
elif layout == "long":
entries.append(image[:1].clone())
elif layout == "full_batch":
entries = [image.clone(), image.clone()]
elif layout == "rank1":
entries[1] = entries[1].flatten()
elif layout == "rank5":
entries[1] = entries[1][None]
else:
entries[1] = entries[1][..., :2, :]
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
with pytest.raises(ValueError, match="Mask list"):
seq(image, entries)
@pytest.mark.parametrize("entry", [None, 1, "mask", [1]])
def test_invalid_mask_list_entry_type_4477(self, entry, device, dtype):
image = torch.rand(2, 1, 4, 6, device=device, dtype=dtype)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
with pytest.raises(ValueError, match=r"Mask list entry 1 must be a torch\.Tensor"):
seq(image, [image[:1].clone(), entry])
@pytest.mark.parametrize("use_cached_params", [False, True])
@pytest.mark.parametrize("nested", [False, True])
@pytest.mark.parametrize("layout", ["short", "long", "full_batch", "non_tensor"])
def test_invalid_mask_list_image_less_inverse_4477(self, layout, nested, use_cached_params, device, dtype):
image = torch.arange(48, device=device, dtype=dtype).reshape(2, 1, 4, 6)
flip = K.RandomHorizontalFlip(p=1.0)
seq = K.AugmentationSequential(K.ImageSequential(flip) if nested else flip, data_keys=["input", "mask"])
_, masks = seq(image, [image[:1].clone(), image[1:].clone()])
params = seq._params
if layout == "short":
masks = masks[:1]
elif layout == "long":
masks.append(masks[0].clone())
elif layout == "full_batch":
masks = [image.clone(), image.clone()]
else:
masks[1] = None
with pytest.raises(ValueError, match="Mask list"):
seq.inverse(masks, params=None if use_cached_params else params, data_keys=["mask"])
assert seq.transform_op.data_keys == seq.data_keys
@pytest.mark.parametrize("use_cached_params", [False, True])
@pytest.mark.parametrize("nested", [False, True])
@pytest.mark.parametrize("batch_size", [0, 2])
def test_mask_list_image_less_inverse_4477(self, batch_size, nested, use_cached_params, device, dtype):
image = torch.arange(batch_size * 24, device=device, dtype=dtype).reshape(batch_size, 1, 4, 6)
entries = [image[i : i + 1].expand(-1, i + 1, -1, -1).clone() for i in range(batch_size)]
flip = K.RandomHorizontalFlip(p=1.0)
seq = K.AugmentationSequential(
K.ImageSequential(flip) if nested else flip, K.RandomVerticalFlip(p=1.0), data_keys=["input", "mask"]
)
_, masks = seq(image, entries)
params = seq._params
if not use_cached_params:
# Explicit parameters must win over the metadata of a later call with another batch size.
seq(torch.zeros(3, 1, 4, 6, device=device, dtype=dtype), data_keys=["input"])
restored = seq.inverse(masks, params=None if use_cached_params else params, data_keys=["mask"])
assert len(restored) == batch_size
for actual, expected in zip(restored, entries):
self.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.parametrize("layout", ["short", "long", "full_batch", "non_tensor", "spatial", "rank1", "rank5"])
def test_mask_list_beside_rank2_image_validates_before_children_4477(self, layout, device, dtype):
image = torch.zeros(4, 6, device=device, dtype=dtype)
masks = [image.clone()]
if layout == "short":
masks = []
elif layout == "long":
masks.append(image.clone())
elif layout == "full_batch":
masks = [image.expand(2, 1, 4, 6).clone()]
elif layout == "non_tensor":
masks = [None]
elif layout == "spatial":
masks = [image[:2].clone()]
elif layout == "rank1":
masks = [image.flatten()]
else:
masks = [image.reshape(1, 1, 1, 4, 6)]
child = K.RandomHorizontalFlip(p=1.0)
seq = K.AugmentationSequential(child, data_keys=["input", "mask"])
with (
patch.object(child, "forward", side_effect=AssertionError("Child must not run before validation")),
pytest.raises(ValueError, match="Mask list"),
):
seq(image, masks)
def test_mask_list_beside_rank2_image_matches_tensor_mask_4477(self, device, dtype):
image = torch.arange(24, device=device, dtype=dtype).reshape(4, 6)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
output, tensor_mask = seq(image, image.clone())
list_output, masks = seq(image, [image.clone()], params=seq._params)
self.assert_close(list_output, output)
assert len(masks) == 1
self.assert_close(masks[0], tensor_mask)
def test_mask_list_nested_inverse_runs_children_in_reverse_4477(self, device, dtype):
# A quarter turn and a flip do not commute, so the inverse order is visible.
image = torch.arange(2 * 36, device=device, dtype=dtype).reshape(2, 1, 6, 6)
entries = [image[:1].clone(), image[1:].expand(-1, 2, -1, -1).clone()]
seq = K.AugmentationSequential(
K.ImageSequential(K.RandomRotation90((1, 1), p=1.0), K.RandomHorizontalFlip(p=1.0)),
data_keys=["input", "mask"],
)
output, masks = seq(image, entries)
_, restored = seq.inverse(output, masks)
for actual, expected in zip(restored, entries):
self.assert_close(actual, expected)
def test_mask_list_uses_mask_fill_for_per_channel_image_fill_4477(self, device, dtype):
image = torch.arange(2 * 3 * 4 * 6, device=device, dtype=dtype).reshape(2, 3, 4, 6)
mask = torch.arange(2 * 4 * 6, device=device, dtype=dtype).reshape(2, 1, 4, 6) + 1
seq = K.AugmentationSequential(
K.RandomCrop((4, 6), padding=2, fill=(7.0, 8.0, 9.0), p=1.0), data_keys=["input", "mask"]
)
_, tensor_mask = seq(image, mask)
_, masks = seq(image, [mask[:1].clone(), mask[1:].clone()], params=seq._params)
for i, entry in enumerate(masks):
self.assert_close(entry, tensor_mask[i : i + 1])
@pytest.mark.parametrize(
("fill_value", "mask_override", "mask_padding"),
[
((0.2, 0.5, 0.9), None, 0.0), # a per-channel image fill pads the mask with background
((0.2, 0.5, 0.9), 0.5, 0.5), # unless the mask sets its own fill_value
(0.7, None, 0.7), # a scalar fill applies to both, as before
],
)
def test_random_affine_mask_fill_follows_the_mask_rule_6021(
self, fill_value, mask_override, mask_padding, device, dtype
):
# The mask rule read only ``flags["fill"]``; RandomAffine stores ``fill_value``, so a per-channel image
# fill turned a one-channel mask into three channels holding the image colours.
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("The image warp path requires grid_sample for the selected dtype.")
image = torch.rand(2, 3, 8, 10, device=device, dtype=dtype)
mask = torch.ones(2, 1, 8, 10, device=device, dtype=dtype)
fill = torch.tensor(fill_value, device=device, dtype=dtype)
extra_args = None if mask_override is None else {DataKey.MASK: {"fill_value": torch.tensor(mask_override)}}
seq = K.AugmentationSequential(
K.RandomAffine((30.0, 30.0), padding_mode="fill", fill_value=fill, p=1.0),
data_keys=["input", "mask"],
extra_args=extra_args,
)
out_image, out_mask = seq(image, mask)
assert out_image.shape == image.shape
assert out_mask.shape == mask.shape
padding = out_mask != 1.0
assert padding.any()
self.assert_close(out_mask[padding], torch.full_like(out_mask[padding], mask_padding))
@pytest.mark.parametrize("as_list", [False, True])
def test_random_affine_mask_fill_rule_on_inverse_6021(self, as_list, device, dtype):
# The inverse warps masks with the module's flags as well, so the rule must hold there too: a per-channel
# image fill would turn a one-channel mask into three channels, or fail the fill_value channel check.
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("The image warp path requires grid_sample for the selected dtype.")
image = torch.rand(2, 3, 8, 10, device=device, dtype=dtype)
mask = torch.ones(2, 1, 8, 10, device=device, dtype=dtype)
fill = torch.tensor([0.2, 0.5, 0.9], device=device, dtype=dtype)
seq = K.AugmentationSequential(
K.RandomAffine((30.0, 30.0), padding_mode="fill", fill_value=fill, p=1.0), data_keys=["input", "mask"]
)
out_image, out_mask = seq(image, list(mask) if as_list else mask)
_, inv_mask = seq.inverse(out_image, out_mask)
inv_mask = torch.cat(inv_mask) if as_list else inv_mask
assert inv_mask.shape == mask.shape
padding = inv_mask != 1.0
assert padding.any()
self.assert_close(inv_mask[padding], torch.zeros_like(inv_mask[padding]))
def test_mask_list_replay_keeps_the_saved_draw_4477(self, device, dtype):
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("The image warp path requires grid_sample for the selected dtype.")
image = torch.arange(2 * 64, device=device, dtype=dtype).reshape(2, 1, 8, 8)
entries = [image[:1].clone(), image[1:].clone()]
seq = K.AugmentationSequential(K.RandomRotation((90.0, 90.0), p=1.0), data_keys=["input", "mask"])
output, masks = seq(image, entries)
params = seq._params
assert params[0].data["forward_input_shape"].tolist() == list(image.shape)
assert seq[0]._params["batch_prob"].shape == (2,)
_, expected = seq.inverse(output, masks, params=params)
assert seq[0]._params["batch_prob"].shape == (2,)
seq(image, data_keys=["input"]) # a newer draw: the inverse below recomputes the saved matrix
keypoints = torch.zeros(2, 1, 2, device=device, dtype=dtype)
restored, _ = seq.inverse(masks, keypoints, params=params, data_keys=["mask", "keypoints"])
for actual, wanted in zip(restored, expected):
self.assert_close(actual, wanted)
@pytest.mark.parametrize("children", [1, 2])
def test_mask_list_as_the_only_data_key_4477(self, children, device, dtype):
image = torch.arange(2 * 24, device=device, dtype=dtype).reshape(2, 1, 4, 6)
entries = [image[:1].clone(), image[1:].expand(-1, 2, -1, -1).clone()]
modules = [K.RandomHorizontalFlip(p=1.0), K.RandomVerticalFlip(p=1.0)][:children]
seq = K.AugmentationSequential(*modules, data_keys=["input", "mask"])
_, masks = seq(image, entries)
replayed = seq(entries, data_keys=["mask"], params=seq._params)
assert isinstance(replayed, list)
assert len(replayed) == 2
for actual, expected in zip(replayed, masks):
self.assert_close(actual, expected)
restored = seq.inverse(masks, data_keys=["mask"])
assert isinstance(restored, list)
assert len(restored) == 2
for actual, expected in zip(restored, entries):
self.assert_close(actual, expected)
def test_mask_list_inverse_without_image_validates_layout_4477(self, device, dtype):
image = torch.rand(2, 1, 4, 6, device=device, dtype=dtype)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
_, masks = seq(image, [image[:1].clone(), image[1:].clone()])
keypoints = torch.zeros(2, 1, 2, device=device, dtype=dtype)
with pytest.raises(ValueError, match="Mask list"):
seq.inverse([*masks, masks[0]], keypoints, data_keys=["mask", "keypoints"])
@pytest.mark.parametrize("policy_name", ["trivial", "rand", "auto"])
def test_mask_list_under_auto_operations_matches_tensor_masks_4477(self, policy_name, device, dtype):
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("The image warp path requires grid_sample for the selected dtype.")
torch.manual_seed(0)
image = torch.rand(2, 3, 8, 8).to(device, dtype) # CPU draw: the policy angle below is device-independent
mask = torch.arange(2 * 64, device=device, dtype=dtype).reshape(2, 1, 8, 8)
if policy_name == "trivial":
policy = K.auto.TrivialAugment(policy=[[("rotate", -30.0, 30.0)]])
elif policy_name == "rand":
policy = K.auto.RandAugment(1, 15, policy=[[("rotate", -30.0, 30.0)]])
else:
policy = K.auto.AutoAugment(policy=[[("rotate", 1.0, 9)]])
seq = K.AugmentationSequential(policy, data_keys=["input", "mask"])
output, tensor_mask = seq(image, mask)
params = seq._params
_, masks = seq(image, [mask[:1].clone(), mask[1:].clone()], params=params)
_, restored_tensor = seq.inverse(output, tensor_mask, params=params)
_, restored = seq.inverse(output, masks, params=params)
assert not torch.equal(tensor_mask, mask)
for i in range(2):
self.assert_close(masks[i], tensor_mask[i : i + 1])
self.assert_close(restored[i], restored_tensor[i : i + 1])
@pytest.mark.parametrize("child", ["video", "mix", "cutmix", "mosaic", "jigsaw", "patch", "3d"])
def test_mask_list_unsupported_children_raise_4477(self, child, device, dtype):
image = torch.rand(2, 3, 4, 4, device=device, dtype=dtype)
entries = [torch.zeros(1, 1, 4, 4, device=device, dtype=dtype) for _ in range(2)]
if child == "video":
image = torch.rand(2, 2, 3, 4, 4, device=device, dtype=dtype)
module = K.VideoSequential(K.RandomHorizontalFlip(p=1.0), data_format="BTCHW")
elif child == "mix":
module = K.RandomMixUpV2(p=1.0)
elif child == "cutmix":
module = K.RandomCutMixV2(p=1.0)
elif child == "mosaic":
module = K.RandomMosaic(p=1.0)
elif child == "jigsaw":
module = K.RandomJigsaw((2, 2), p=1.0)
elif child == "patch":
module = K.PatchSequential(K.RandomHorizontalFlip(p=1.0), grid_size=(2, 2), random_apply=True)
else:
image = torch.rand(2, 1, 2, 4, 4, device=device, dtype=dtype)
entries = [torch.zeros(1, 1, 2, 4, 4, device=device, dtype=dtype) for _ in range(2)]
module = K.RandomHorizontalFlip3D(p=1.0)
seq = K.AugmentationSequential(module, data_keys=["input", "mask"])
with pytest.raises(NotImplementedError):
seq(image, entries)
output = seq(image, data_keys=["input"])
if child not in ("mix", "cutmix", "mosaic", "jigsaw"):
with pytest.raises(NotImplementedError):
seq.inverse(output, entries, data_keys=["input", "mask"])
with pytest.raises(NotImplementedError):
seq.inverse(entries, data_keys=["mask"])
@pytest.mark.parametrize("mask_only", [False, True])
def test_mask_list_transplantation_rejected_before_parameter_calculation_4477(self, mask_only, device, dtype):
image = torch.arange(2 * 16, device=device, dtype=dtype).reshape(2, 1, 4, 4)
mask = torch.zeros(2, 1, 4, 4, device=device, dtype=torch.long)
mask[:, :, :2, :2] = 1
seq = K.AugmentationSequential(K.RandomTransplantation(p=1.0), data_keys=["input", "mask"])
# Tensor masks remain supported, and provide a saved draw for mask-only replay.
output, tensor_mask = seq(image.clone(), mask.clone())
assert output.shape == image.shape
assert tensor_mask.shape == mask.shape
entries = [mask[:1].clone(), mask[1:].clone()]
args = (entries,) if mask_only else (image, entries)
kwargs = {"params": seq._params, "data_keys": ["mask"]} if mask_only else {}
with pytest.raises(NotImplementedError, match="RandomTransplantation does not support mask lists"):
seq(*args, **kwargs)
def test_mask_list_passes_through_intensity_patch_sequential_4477(self, device, dtype):
image = torch.rand(2, 3, 4, 4, device=device, dtype=dtype)
entries = [torch.rand(1, 1, 4, 4, device=device, dtype=dtype) for _ in range(2)]
patch = K.PatchSequential(K.RandomGaussianBlur((3, 3), (0.5, 0.5), p=1.0), grid_size=(2, 2), random_apply=True)
seq = K.AugmentationSequential(patch, data_keys=["input", "mask"])
output, masks = seq(image, entries)
_, restored = seq.inverse(output, masks)
for actual, restored_entry, expected in zip(masks, restored, entries):
self.assert_close(actual, expected)
self.assert_close(restored_entry, expected)
def test_mix_augmentation_inverse_raises_4693(self, device, dtype):
image = torch.stack([torch.full((2, 4, 6), float(i + 1), device=device, dtype=dtype) for i in range(3)])
mask = torch.zeros(3, 4, 6, device=device, dtype=torch.long)
for i in range(3):
mask[i, :2, :3] = i + 1
seq = K.AugmentationSequential(
K.RandomTransplantation(p=1.0),
data_keys=["input", "mask"],
)
torch.manual_seed(7)
output = seq(image, mask)
assert not torch.equal(output[0], image)
with pytest.raises(RuntimeError, match="Inverse for RandomTransplantation is not supported"):
seq.inverse(*output)
def test_mix_augmentation_3d_inverse_raises_4693(self, device, dtype):
image = torch.stack([torch.full((2, 3, 4, 5), float(i + 1), device=device, dtype=dtype) for i in range(3)])
mask = torch.zeros(3, 3, 4, 5, device=device, dtype=torch.long)
for i in range(3):
mask[i, :2, :2, :3] = i + 1
seq = K.AugmentationSequential(
K.RandomTransplantation3D(p=1.0),
data_keys=["input", "mask"],
)
torch.manual_seed(7)
output = seq(image, mask)
assert not torch.equal(output[0], image)
with pytest.raises(RuntimeError, match="Inverse for RandomTransplantation3D is not supported"):
seq.inverse(*output)
def test_mix_children_dispatch_annotation_keys_4493(self, device, dtype):
# #4493: the container dispatches a mix child's annotation keys to the child's own handlers: RandomMosaic
# transforms boxes (container xyxy_plus path), and unsupported keys raise NotImplementedError as a direct
# call does. A class key raises from the container.
from kornia.geometry.boxes import Boxes
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
boxes = torch.tensor([[[0.0, 0.0, 2.0, 2.0]], [[1.0, 1.0, 3.0, 3.0]]], device=device, dtype=dtype)
mask = torch.arange(2, device=device, dtype=dtype).reshape(2, 1, 1, 1).expand(2, 1, 16, 16).clone()
keypoints = torch.tensor([[[1.0, 1.0]], [[2.0, 2.0]]], device=device, dtype=dtype)
# RandomMosaic boxes are transformed inside the container.
seq = K.AugmentationSequential(K.RandomMosaic(p=1.0), data_keys=["input", "bbox_xyxy"])
torch.manual_seed(0)
out_image, out_boxes = seq(image, boxes)
assert not torch.equal(out_image, image)
assert not torch.equal(out_boxes, boxes)
params = seq._params[0].data
mosaic = K.RandomMosaic(p=1.0)
expected = mosaic.transform_boxes(Boxes.from_tensor(boxes, mode="xyxy_plus"), params, mosaic.flags).to_tensor(
mode="xyxy_plus"
)
self.assert_close(out_boxes, expected, rtol=0, atol=0)
# Unsupported annotation keys raise, matching a direct call (no silent passthrough).
for factory, key, payload in (
(lambda: K.RandomMixUpV2(p=1.0), "mask", mask),
(lambda: K.RandomMixUpV2(p=1.0), "bbox_xyxy", boxes),
(lambda: K.RandomMixUpV2(p=1.0), "keypoints", keypoints),
(lambda: K.RandomMosaic(p=1.0), "mask", mask),
(lambda: K.RandomMosaic(p=1.0), "keypoints", keypoints),
(lambda: K.RandomCutMixV2(p=1.0), "mask", mask),
(lambda: K.RandomJigsaw(p=1.0), "mask", mask),
):
with pytest.raises(NotImplementedError):
K.AugmentationSequential(factory(), data_keys=["input", key])(image, payload)
with pytest.raises(NotImplementedError, match="class labels"):
K.AugmentationSequential(K.RandomMixUpV2(p=1.0), data_keys=["input", "class"])(
image, torch.tensor([0, 1], device=device)
)
def test_convention_dictionary_mask_before_image_keeps_float64_precision_4478(self):
# Convention pin for kornia#4478: a mask is converted with this call's image dtype wherever it sits in the
# dictionary, so these float64 values, which float32 cannot hold, come back bit for bit in both insertion
# orders.
image = torch.zeros(1, 1, 1, 2, dtype=torch.float64)
mask = torch.tensor([1.0 + 2**-30, 2.0 + 2**-29], dtype=torch.float64).reshape_as(image)
early_mask = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None)
image_first = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None)
expected = mask.flip(-1)
early_output = early_mask({"mask": mask, "image": image})["mask"]
assert early_output.dtype == torch.float64
assert torch.equal(early_output, expected)
assert torch.equal(image_first({"image": image, "mask": mask})["mask"], expected)
@pytest.mark.parametrize("input_kind", ["pil", "path", "numpy"])
def test_dictionary_image_accepts_pil_path_and_numpy_5852(self, input_kind, tmp_path):
import numpy as np
from PIL import Image as PILImage
image = PILImage.new("RGB", (5, 4), color=(200, 100, 25))
path = tmp_path / "image.png"
image.save(path)
value = {"pil": image, "path": str(path), "numpy": np.array(image)}[input_kind]
positional = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=["input"])(value)
dictionary = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=None)({"input": value})
assert torch.equal(dictionary["input"], positional)
@pytest.mark.parametrize("key", ["input", "image"])
def test_dictionary_image_missing_path_raises_5852(self, key, tmp_path):
# The key marks the string as an image path, so a missing file is reported as such.
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.0), data_keys=None)
with pytest.raises(FileNotFoundError):
aug({key: str(tmp_path / "missing.png")})
@pytest.mark.parametrize("mask_dtypes", [(torch.int64, torch.bool), (torch.bool, torch.int64)])
def test_convention_each_mask_keeps_its_own_dtype_4478(self, mask_dtypes, device, dtype):
# Convention pin for kornia#4478: each mask output comes back in the dtype of its own argument, so labels
# 2, 3 and 5 survive next to a boolean mask. Both argument orders are covered.
image = torch.arange(24, device=device, dtype=dtype).reshape(2, 1, 3, 4)
labels = torch.tensor([0, 2, 3, 5], device=device).reshape(1, 1, 1, 4).expand(2, 1, 3, 4)
first, second = (labels.to(mask_dtype) for mask_dtype in mask_dtypes)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask", "mask"])
_, out_first, out_second = seq(image, first, second)
assert (out_first.dtype, out_second.dtype) == mask_dtypes
assert torch.equal(out_first, first.flip(-1))
assert torch.equal(out_second, second.flip(-1))
integer_output = out_first if mask_dtypes[0] == torch.int64 else out_second
assert integer_output.unique().tolist() == [0, 2, 3, 5] # the labels survive next to a boolean mask
assert seq.mask_dtype == mask_dtypes[1] # the attribute still records the last mask argument's dtype
@pytest.mark.parametrize("mask_dtypes", [(torch.int64, torch.bool), (torch.bool, torch.int64)])
def test_convention_each_list_mask_element_keeps_its_own_dtype_4478(self, mask_dtypes, device, dtype):
# Convention pin for kornia#4478: a list mask comes back per element in each entry's own dtype. Both
# element orders are covered.
image = torch.arange(16, device=device, dtype=dtype).reshape(2, 1, 2, 4)
labels = torch.tensor([0, 2, 3, 5], device=device).reshape(1, 1, 1, 4).expand(1, 1, 2, 4)
entries = [labels.to(mask_dtype) for mask_dtype in mask_dtypes]
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
_, out = seq(image, entries)
assert [m.dtype for m in out] == list(mask_dtypes)
for out_entry, entry in zip(out, entries):
assert torch.equal(out_entry, entry.flip(-1))
assert seq.mask_dtype == mask_dtypes[0] # the attribute records the first element of the last list mask
def test_list_mask_inverse_undoes_the_forward_pass_4716(self, device, dtype):
# kornia#4716: forward accepted a list mask and returned one, but inverse raised
# "'list' object has no attribute 'dtype'". It now inverts element by element, and each
# entry comes back in its own dtype, as in the forward pass (#4478).
image = torch.arange(48, device=device, dtype=dtype).reshape(2, 1, 4, 6)
labels = torch.arange(24, device=device).reshape(1, 1, 4, 6).expand(2, 1, -1, -1) % 5
masks = [labels[:1].clone(), (labels[1:] > 2)]
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
out_image, out_masks = seq(image, masks)
restored_image, restored = seq.inverse(out_image, out_masks)
self.assert_close(restored_image, image)
assert isinstance(restored, list)
assert len(restored) == 2
assert [m.dtype for m in restored] == [torch.int64, torch.bool]
for entry, original in zip(restored, masks):
assert torch.equal(entry, original)
def test_list_mask_inverse_uses_each_entrys_forward_gate_4716(self, device, dtype):
# The forward pass gives list entry i the gate of sample i. The inverse has to
# reuse that same gate, or a mixed-probability batch would come back half re-flipped.
image = torch.arange(24, device=device, dtype=dtype).reshape(2, 1, 3, 4)
masks = [image[:1].clone(), image[1:].expand(-1, 2, -1, -1).clone()]
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.5), data_keys=["input", "mask"])
params = [
ParamItem(
"RandomHorizontalFlip_0",
{"batch_prob": torch.tensor([1.0, 0.0]), "forward_input_shape": torch.tensor(image.shape)},
)
]
out_image, out_masks = seq(image, masks, params=params)
_, restored = seq.inverse(out_image, out_masks, params=params)
self.assert_close(restored[0], masks[0])
self.assert_close(restored[1], masks[1])
def test_list_mask_inverse_through_a_nested_sequential_4716(self, device, dtype):
image = torch.arange(24, device=device, dtype=dtype).reshape(1, 1, 4, 6).repeat(2, 1, 1, 1)
masks = [image[:1].clone(), image[1:].clone() * 2]
seq = K.AugmentationSequential(
K.ImageSequential(K.RandomHorizontalFlip(p=1.0), K.RandomVerticalFlip(p=1.0)),
data_keys=["input", "mask"],
)
out_image, out_masks = seq(image, masks)
self.assert_close(out_masks[0], image[:1].flip(-1).flip(-2))
_, restored = seq.inverse(out_image, out_masks)
for entry, original in zip(restored, masks):
self.assert_close(entry, original)
def test_list_mask_inverse_applies_the_inverse_matrix_4716(self, device, dtype):
# A flip is its own inverse, so a flip round trip cannot tell the inverse matrix from the forward one.
# A quarter turn can: applying the forward matrix again gives a half turn instead of the identity.
image = torch.rand(2, 1, 8, 8, device=device, dtype=dtype)
labels = torch.arange(64, device=device).reshape(1, 1, 8, 8).expand(2, 1, -1, -1) % 7
masks = [labels[:1].clone(), labels[1:] > 3]
seq = K.AugmentationSequential(K.RandomRotation(degrees=(90.0, 90.0), p=1.0), data_keys=["input", "mask"])
out_image, out_masks = seq(image, masks)
assert not torch.equal(out_masks[0], labels)
_, restored = seq.inverse(out_image, out_masks)
assert [m.dtype for m in restored] == [torch.int64, torch.bool]
for entry, original in zip(restored, masks):
assert torch.equal(entry, original)
@pytest.mark.parametrize("key", ["bbox_xyxy", "bbox_xywh"])
@pytest.mark.parametrize("suffix", ["", "_2", "-a"])
def test_dictionary_coordinate_box_keys_4483(self, key, suffix, device, dtype):
image = torch.arange(20, device=device, dtype=dtype).reshape(1, 1, 4, 5)
boxes = torch.tensor([[[0.0, 0.0, 2.0, 2.0]]], device=device, dtype=dtype)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None)
output = seq({"image": image, key + suffix: boxes})
expected = [2.0, 0.0, 4.0, 2.0] if key == "bbox_xyxy" else [3.0, 0.0, 2.0, 2.0]
self.assert_close(output[key + suffix], torch.tensor([[expected]], device=device, dtype=dtype))
restored = seq.inverse(output)
self.assert_close(restored[key + suffix], boxes)
def test_dictionary_class_alias_4483(self):
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None)
keys, metadata = seq._read_datakeys_from_dict(("input", "class", "class-a", "label", "label-a"))
assert keys == [DataKey.INPUT, DataKey.LABEL, DataKey.LABEL, DataKey.LABEL, DataKey.LABEL]
assert metadata == []
@pytest.mark.parametrize("cropping_mode", ["slice", "resample"])
@pytest.mark.parametrize("align_corners", [None, False, True])
def test_resized_crop_mask_accepts_any_align_corners_4802(self, cropping_mode, align_corners, device, dtype):
# #4802: slice mode drops `align_corners` for nearest resampling, so every override works in both modes.
image = torch.arange(16, device=device, dtype=dtype).reshape(1, 1, 4, 4) / 16
mask = torch.arange(16, device=device, dtype=dtype).reshape(1, 1, 4, 4).remainder(2)
seq = K.AugmentationSequential(
K.RandomResizedCrop((2, 2), scale=(1.0, 1.0), ratio=(1.0, 1.0), cropping_mode=cropping_mode, p=1.0),
data_keys=["input", "mask"],
extra_args={DataKey.MASK: {"resample": Resample.NEAREST, "align_corners": align_corners}},
)
out_image, out_mask = seq(image, mask)
assert out_image.shape == out_mask.shape == (1, 1, 2, 2)
assert set(out_mask.unique().tolist()).issubset({0.0, 1.0})
def test_convention_masks_keep_labels_and_add_padding_fill_through_a_rotation(self, device, dtype):
# Convention pin: masks are resampled with nearest interpolation, so a {2, 3} mask still holds those
# labels after a 45 degree affine without intermediate values. The zero padding fill is also present,
# and the mask dtype is preserved, `bool` included.
# The claim is checked one parameter away from the pin's fixture: B = 2 and B = 1 both hold.
for batch in (2, 1):
aug = K.AugmentationSequential(K.RandomAffine(degrees=(45.0, 45.0), p=1.0), data_keys=["input", "mask"])
mask = torch.full((batch, 1, 6, 8), 2.0, device=device, dtype=dtype)
mask[:, :, 1:4, 2:6] = 3.0
out_mask = aug(torch.rand(batch, 3, 6, 8, device=device, dtype=dtype), mask)[1]
assert sorted(out_mask.unique().tolist()) == [0.0, 2.0, 3.0]
assert out_mask.dtype == dtype
assert (out_mask - mask).abs().max().item() == 2.0 # the rotation moved the mask
aug = K.AugmentationSequential(K.RandomAffine(degrees=(45.0, 45.0), p=1.0), data_keys=["input", "mask"])
bool_mask = torch.zeros(2, 1, 6, 8, device=device, dtype=torch.bool)
bool_mask[:, :, 1:4, 2:6] = True
assert aug(torch.rand(2, 3, 6, 8, device=device, dtype=dtype), bool_mask)[1].dtype == torch.bool
@pytest.mark.parametrize("mask_dtype", [torch.float32, torch.int64, torch.bool])
def test_resize_antialias_preserves_mask_labels_4479(self, mask_dtype, device, dtype):
yy, xx = torch.meshgrid(
torch.arange(12, device=device),
torch.arange(16, device=device),
indexing="ij",
)
blocks = ((yy // 2) + (xx // 2)) % 2
mask = blocks.bool() if mask_dtype == torch.bool else (2 + blocks).to(mask_dtype)
mask = mask[None, None]
image = torch.rand(1, 3, 12, 16, device=device, dtype=dtype)
without_antialias = K.AugmentationSequential(
K.Resize((6, 8), antialias=False),
data_keys=["input", "mask"],
)
with_antialias = K.AugmentationSequential(
K.Resize((6, 8), antialias=True),
data_keys=["input", "mask"],
)
expected_mask = without_antialias(image, mask)[1]
out_image, out_mask = with_antialias(image, mask)
expected_image = resize(image, (6, 8), "bilinear", align_corners=True, antialias=True)
assert out_mask.dtype == mask_dtype
assert torch.equal(out_mask, expected_mask)
assert set(out_mask.unique().tolist()) == set(mask.unique().tolist())
self.assert_close(out_image, expected_image)
def test_resize_mask_explicit_antialias_override_4479(self, device, dtype):
yy, xx = torch.meshgrid(
torch.arange(12, device=device),
torch.arange(16, device=device),
indexing="ij",
)
mask = (((yy // 2) + (xx // 2)) % 2).to(dtype)[None, None]
image = torch.rand(1, 3, 12, 16, device=device, dtype=dtype)
aug = K.AugmentationSequential(
K.Resize((6, 8), antialias=True),
data_keys=["input", "mask"],
extra_args={
DataKey.MASK: {
"resample": Resample.BILINEAR,
"align_corners": True,
"antialias": True,
}
},
)
out_mask = aug(image, mask)[1]
expected = resize(mask, (6, 8), "bilinear", align_corners=True, antialias=True)
without_antialias = resize(mask, (6, 8), "bilinear", align_corners=True, antialias=False)
self.assert_close(out_mask, expected)
assert not torch.equal(out_mask, without_antialias)
def test_convention_boxes_follow_the_xyxy_plus_convention(self, device, dtype):
# Convention pin: the container's box arithmetic is `Boxes`' inclusive `xyxy_plus` mode, for a scaling
# op as well as for a flip - `Boxes.from_tensor(..., mode="xyxy_plus").transform_boxes_(M)` reproduces
# the container's output exactly, while `mode="xyxy"` (exclusive) does not.
# Two asymmetric boxes on a 6x8 image, resized to (3, 4) => M = diag(3/7, 2/5) in xyxy_plus space.
# Snippet used to generate expected: this body. Container
# [[0.0, 0.0, 1.2857143, 0.8]], [[0.85714293, 0.4, 2.1428573, 1.6]] == xyxy_plus;
# xyxy gives [[0.0, 0.0, 1.8571429, 1.4]], [[0.85714293, 0.4, 2.7142859, 2.2]].
boxes = torch.tensor([[[0.0, 0.0, 3.0, 2.0]], [[2.0, 1.0, 5.0, 4.0]]], device=device, dtype=dtype)
aug = K.AugmentationSequential(K.Resize((3, 4)), data_keys=["input", "bbox_xyxy"])
out_boxes = aug(torch.rand(2, 3, 6, 8, device=device, dtype=dtype), boxes.clone())[1]
matrix = aug.transform_matrix
inclusive = Boxes.from_tensor(boxes.clone(), mode="xyxy_plus")
inclusive.transform_boxes_(matrix)
self.assert_close(out_boxes, inclusive.to_tensor(mode="xyxy_plus"))
exclusive = Boxes.from_tensor(boxes.clone(), mode="xyxy")
exclusive.transform_boxes_(matrix)
assert not torch.allclose(out_boxes, exclusive.to_tensor(mode="xyxy"))
def test_convention_inverse_restores_keypoints_and_loses_rotated_box_corners(self, device, dtype):
# Convention pin: `.inverse()` restores keypoints moved by the geometric chain, while non-axis-aligned
# rotations cannot restore tensor boxes. Forward turns both `bbox_xyxy` and vertex `bbox` formats into
# axis-aligned enclosures, losing the original corners before inverse is called.
torch.manual_seed(0)
aug = K.AugmentationSequential(K.RandomAffine(degrees=(45.0, 45.0), p=1.0), data_keys=["input", "keypoints"])
img = torch.rand(2, 3, 6, 8, device=device, dtype=dtype)
kpts = torch.tensor([[[1.0, 2.0]], [[3.0, 4.0]]], device=device, dtype=dtype)
out_img, out_kpts = aug(img, kpts)
assert (out_kpts - kpts).abs().max().item() > 1.0 # the forward really moved them
restored = aug.inverse(out_img, out_kpts)[1]
self.assert_close(restored, kpts)
boxes_by_key = {
"bbox_xyxy": torch.tensor([[[1.0, 2.0, 4.0, 5.0]]], device=device, dtype=dtype),
"bbox": torch.tensor([[[[1.0, 2.0], [4.0, 2.0], [4.0, 5.0], [1.0, 5.0]]]], device=device, dtype=dtype),
}
for key, boxes in boxes_by_key.items():
box_aug = K.AugmentationSequential(K.RandomAffine(degrees=(45.0, 45.0), p=1.0), data_keys=["input", key])
out_img, out_boxes = box_aug(torch.rand(1, 3, 8, 8, device=device, dtype=dtype), boxes)
restored_boxes = box_aug.inverse(out_img, out_boxes)[1]
assert torch.isfinite(restored_boxes).all()
if key == "bbox":
restored_corners = restored_boxes[0, 0]
original_corners = boxes[0, 0]
else:
restored_corners = Boxes.from_tensor(restored_boxes, mode="xyxy_plus").to_tensor(mode="vertices_plus")[
0, 0
]
original_corners = Boxes.from_tensor(boxes, mode="xyxy_plus").to_tensor(mode="vertices_plus")[0, 0]
assert (restored_corners.min(dim=0).values < original_corners.min(dim=0).values - 1).all()
assert (restored_corners.max(dim=0).values > original_corners.max(dim=0).values + 1).all()
@pytest.mark.parametrize("nested_first", [False, True])
@pytest.mark.parametrize("warmed", [False, True])
@pytest.mark.parametrize("outer_cls", [K.AugmentationSequential, K.ImageSequential])
def test_convention_nested_container_matrix_4476(self, nested_first, warmed, outer_cls, device, dtype):
nested = K.AugmentationSequential(K.RandomVerticalFlip(p=1.0), data_keys=["input"])
if warmed:
nested(torch.rand(2, 1, 8, 10, device=device, dtype=dtype))
_ = nested.transform_matrix
horizontal = K.RandomHorizontalFlip(p=1.0)
children = [nested, horizontal] if nested_first else [horizontal, nested]
outer = outer_cls(*children)
image = torch.arange(24, device=device, dtype=dtype).reshape(1, 1, 4, 6)
output = outer(image)
params = outer._params
self.assert_close(output, image.flip((-2, -1)))
expected = torch.tensor([[[-1.0, 0.0, 5.0], [0.0, -1.0, 3.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
if outer_cls is K.AugmentationSequential:
self.assert_close(outer.transform_matrix, expected)
self.assert_close(outer.get_transformation_matrix(image, params=params), expected)
self.assert_close(nested.transform_matrix[:, 1, 2], torch.tensor([3.0], device=device, dtype=dtype))
self.assert_close(outer(image, params=params), output)
self.assert_close(outer.inverse(output, params=params), image)
@pytest.mark.parametrize(
"outer_augmentation,expected_points",
[
(K.RandomHorizontalFlip(p=1.0), [[[4.0, 5.0]], [[3.0, 4.0]]]),
(K.RandomRotation((90.0, 90.0), p=1.0), [[[0.0, 1.0]], [[1.0, 2.0]]]),
],
)
@pytest.mark.parametrize("nested_cls", [K.AugmentationSequential, K.ImageSequential])
def test_nested_container_matrix_matches_keypoints_4476(
self, outer_augmentation, expected_points, nested_cls, device, dtype
):
inner = nested_cls(K.RandomVerticalFlip(p=1.0))
nested = nested_cls(inner)
outer = K.AugmentationSequential(outer_augmentation, nested, data_keys=["input", "keypoints"])
image = torch.rand(2, 1, 6, 6, device=device, dtype=dtype)
points = torch.tensor([[[1.0, 0.0]], [[2.0, 1.0]]], device=device, dtype=dtype)
_, output_points = outer(image, points)
homogeneous = torch.cat((points, torch.ones_like(points[..., :1])), dim=-1)
matrix_points = homogeneous @ outer.transform_matrix.transpose(-2, -1)
self.assert_close(matrix_points[..., :2], output_points)
self.assert_close(output_points, torch.tensor(expected_points, device=device, dtype=dtype))
@pytest.mark.parametrize("nested_first", [False, True])
def test_convention_nested_non_rigid_container_is_skipped_4476(self, nested_first, device, dtype):
# A nested container whose children are all non-rigid records no matrix and is skipped, in either order and
# under the outer ``rigid`` mode too: the nested container's own ``silent`` mode skipped its children.
image = torch.rand(1, 1, 6, 8, device=device, dtype=dtype)
expected = torch.tensor([[[-1.0, 0.0, 7.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
def children():
nested = K.AugmentationSequential(K.RandomThinPlateSpline(p=1.0))
horizontal = K.RandomHorizontalFlip(p=1.0)
return [nested, horizontal] if nested_first else [horizontal, nested]
outer = K.AugmentationSequential(*children())
outer(image)
self.assert_close(outer.transform_matrix, expected)
image_outer = K.ImageSequential(*children())
image_outer(image)
self.assert_close(image_outer.get_transformation_matrix(image, params=image_outer._params), expected)
rigid_outer = K.AugmentationSequential(*children(), transformation_matrix_mode="rigid")
rigid_outer(image)
self.assert_close(rigid_outer.transform_matrix, expected)
def test_nested_container_records_its_params_4476(self, device, dtype):
nested = K.AugmentationSequential(K.RandomVerticalFlip(p=1.0))
outer = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), nested)
image = torch.rand(1, 1, 4, 6, device=device, dtype=dtype)
output = outer(image)
self.assert_close(nested.inverse(output), image.flip(-1))
@pytest.mark.parametrize("nested_first", [False, True])
@pytest.mark.parametrize("mode", ["silent", "silence"])
def test_nested_image_sequence_matrix_4476(self, nested_first, mode, device, dtype):
nested = K.ImageSequential(K.RandomVerticalFlip(p=1.0))
horizontal = K.RandomHorizontalFlip(p=1.0)
children = [nested, horizontal] if nested_first else [horizontal, nested]
outer = K.AugmentationSequential(*children, data_keys=["input", "keypoints"], transformation_matrix_mode=mode)
for batch, height, width in [(2, 8, 10), (1, 4, 6)]:
image = torch.arange(batch * height * width, device=device, dtype=dtype).reshape(batch, 1, height, width)
points = torch.tensor([[[1.0, 0.0]]], device=device, dtype=dtype).expand(batch, -1, -1)
output, moved = outer(image, points)
params = outer._params
expected = torch.tensor([[[width - 2.0, height - 1.0]]], device=device, dtype=dtype).expand(batch, -1, -1)
self.assert_close(output, image.flip((-2, -1)))
self.assert_close(moved, expected)
homogeneous = torch.cat((points, torch.ones_like(points[..., :1])), dim=-1)
self.assert_close((homogeneous @ outer.transform_matrix.transpose(-2, -1))[..., :2], expected)
replay = outer(image, points, params=params)
self.assert_close(replay[0], output)
self.assert_close(replay[1], moved)
self.assert_close(outer.inverse(*replay, params=params)[1], points)
@pytest.mark.parametrize("nested_container", [False, True])
def test_repeated_nested_image_member_matrix_4476(self, nested_container, device, dtype):
image = torch.arange(24, device=device, dtype=dtype).reshape(1, 1, 4, 6)
horizontal = K.RandomHorizontalFlip(p=0.5)
child = K.AugmentationSequential(horizontal) if nested_container else horizontal
nested = K.ImageSequential(child, random_apply=2)
outer = K.AugmentationSequential(nested)
first = horizontal.forward_parameters(image.shape)
first["batch_prob"].fill_(1.0)
second = horizontal.forward_parameters(image.shape)
second["batch_prob"].zero_()
calls = [
ParamItem("AugmentationSequential_0", [ParamItem("RandomHorizontalFlip_0", p)])
if nested_container
else ParamItem("RandomHorizontalFlip_0", p)
for p in (first, second)
]
params = [ParamItem("ImageSequential_0", calls)]
output = outer(image, params=params)
self.assert_close(output, image.flip(-1))
expected = torch.tensor([[[-1.0, 0.0, 5.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
self.assert_close(outer.transform_matrix, expected)
self.assert_close(outer(image, params=params), output)
self.assert_close(outer.transform_matrix, expected)
@pytest.mark.parametrize("repeated_plain", [False, True])
@pytest.mark.parametrize("override", [False, True])
def test_repeated_nested_subclass_matrix_4476(self, repeated_plain, override, device, dtype):
class CustomSequential(K.AugmentationSequential):
def transform_inputs(self, input, params, extra_args=None):
output = super().transform_inputs(input, params, extra_args)
if override:
vertical = input.new_tensor([[[1.0, 0.0, 0.0], [0.0, -1.0, 3.0], [0.0, 0.0, 1.0]]])
self._transform_matrix = vertical @ self.transform_matrix
output = output.flip(-2)
return output
image = torch.arange(24, device=device, dtype=dtype).reshape(1, 1, 4, 6)
horizontal = K.RandomHorizontalFlip(p=0.5)
child = CustomSequential(horizontal)
nested = K.ImageSequential(child, random_apply=2)
if repeated_plain:
nested = K.ImageSequential(K.ImageSequential(child), random_apply=2)
outer = K.AugmentationSequential(nested)
first = horizontal.forward_parameters(image.shape)
first["batch_prob"].fill_(1.0)
second = horizontal.forward_parameters(image.shape)
second["batch_prob"].zero_()
for draws in ((first, second), (second, first)):
calls = [ParamItem("CustomSequential_0", [ParamItem("RandomHorizontalFlip_0", p)]) for p in draws]
if repeated_plain:
calls = [ParamItem("ImageSequential_0", [call]) for call in calls]
params = [ParamItem("ImageSequential_0", calls)]
expected = image.new_tensor([[[-1.0, 0.0, 5.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]])
for _ in range(2):
rng = torch.get_rng_state()
self.assert_close(outer(image, params=params), image.flip(-1))
self.assert_close(outer.transform_matrix, expected)
self.assert_close(torch.get_rng_state(), rng, rtol=0, atol=0)
def test_nested_image_annotation_only_replay_4476(self, device, dtype):
horizontal = K.RandomHorizontalFlip(p=1.0)
outer = K.AugmentationSequential(K.ImageSequential(horizontal))
draw = horizontal.forward_parameters(torch.Size((1, 1, 4, 6)))
params = [ParamItem("ImageSequential_0", [ParamItem("RandomHorizontalFlip_0", draw)])]
points = torch.tensor([[[1.0, 0.0]]], device=device, dtype=dtype)
expected = points.new_tensor([[[-1.0, 0.0, 5.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]])
for _ in range(2):
self.assert_close(outer(points, params=params, data_keys=["keypoints"]), points.new_tensor([[[4.0, 0.0]]]))
self.assert_close(outer.transform_matrix, expected)
@pytest.mark.parametrize("mode", ["silent", "skip"])
def test_nested_image_sequence_modes_4476(self, mode, device, dtype):
nested = K.ImageSequential(K.RandomVerticalFlip(p=1.0), K.RandomThinPlateSpline(p=0.0))
outer = K.AugmentationSequential(nested, K.RandomHorizontalFlip(p=1.0), transformation_matrix_mode=mode)
image = torch.arange(24, device=device, dtype=dtype).reshape(1, 1, 4, 6)
self.assert_close(outer(image), image.flip((-2, -1)))
if mode == "skip":
assert outer.transform_matrix is None
else:
expected = torch.tensor([[[-1.0, 0.0, 5.0], [0.0, -1.0, 3.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
self.assert_close(outer.transform_matrix, expected)
def test_nested_image_sequence_keeps_mix_labels_4476(self, device, dtype):
outer = K.AugmentationSequential(K.ImageSequential(K.RandomMixUpV2(p=1.0)), data_keys=["input", "class"])
image = torch.rand(2, 1, 4, 6, device=device, dtype=dtype)
labels = torch.tensor([0, 1], device=device)
_, result = outer(image, labels)
self.assert_close(result, labels, rtol=0, atol=0)
def test_nested_image_sequence_keeps_inner_skip_mode_4476(self, device, dtype):
nested = K.ImageSequential(
K.AugmentationSequential(K.RandomVerticalFlip(p=1.0), transformation_matrix_mode="skip"),
K.RandomHorizontalFlip(p=1.0),
)
outer = K.AugmentationSequential(nested)
image = torch.rand(1, 1, 4, 6, device=device, dtype=dtype)
self.assert_close(outer(image), image.flip((-2, -1)))
expected = torch.tensor([[[-1.0, 0.0, 5.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
self.assert_close(outer.transform_matrix, expected)
# An annotation-only replay reads the matrix from the parameters and keeps the inner skip mode too.
points = torch.tensor([[[1.0, 0.0]]], device=device, dtype=dtype)
self.assert_close(
outer(points, params=outer._params, data_keys=["keypoints"]), points.new_tensor([[[4.0, 3.0]]])
)
self.assert_close(outer.transform_matrix, expected)
def test_nested_image_sequence_3d_matrix_4476(self, device, dtype):
outer = K.AugmentationSequential(
K.RandomVerticalFlip3D(p=1.0), K.ImageSequential(K.RandomHorizontalFlip3D(p=1.0))
)
image = torch.rand(1, 1, 3, 4, 6, device=device, dtype=dtype)
self.assert_close(outer(image), image.flip((-2, -1)))
expected = torch.tensor(
[[[-1.0, 0.0, 0.0, 5.0], [0.0, -1.0, 0.0, 3.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]],
device=device,
dtype=dtype,
)
self.assert_close(outer.transform_matrix, expected)
def test_repeated_nested_resize_keeps_input_overrides_4476(self, device, dtype):
nested = K.ImageSequential(K.Resize((4, 6)), random_apply=2)
outer = K.AugmentationSequential(nested, extra_args={DataKey.INPUT: {"align_corners": False}})
image = torch.rand(1, 1, 8, 10, device=device, dtype=dtype)
outer(image)
expected = torch.tensor([[[0.6, 0.0, -0.2], [0.0, 0.5, -0.25], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
self.assert_close(outer.transform_matrix, expected)
def test_convention_nested_rigid_container_checks_its_children_4476(self, device, dtype):
# A nested container records its matrix in its own mode, so ``rigid`` rejects a non-rigid child as a direct
# call does.
nested = K.AugmentationSequential(K.RandomThinPlateSpline(p=1.0), transformation_matrix_mode="rigid")
outer = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), nested)
with pytest.raises(RuntimeError, match="under `rigid` computation mode"):
outer(torch.rand(1, 1, 6, 8, device=device, dtype=dtype))
@pytest.mark.parametrize("crop_cls", [K.CenterCrop, K.RandomCrop, K.RandomResizedCrop])
def test_convention_slice_crop_inverse_restores_the_canvas(self, crop_cls, device, dtype):
# Slice-mode crops invert like resample-mode ones: the output is warped back onto the input canvas.
seq = K.AugmentationSequential(crop_cls((4, 6), p=1.0), data_keys=["input"])
image = torch.rand(1, 3, 6, 8, device=device, dtype=dtype)
out = seq(image)
restored = seq.inverse(out)
assert restored.shape == image.shape
if crop_cls is not K.RandomResizedCrop:
left, top = seq[0]._params["src"][0, 0].long().tolist()
self.assert_close(restored[..., top : top + 4, left : left + 6], out)
def test_convention_3d_inverse_raises_for_geometry_and_skips_intensity(self, device, dtype):
# Enough voxels in two populated histogram bins force an actual equalization in every dtype.
image = torch.full((1, 1, 8, 8, 8), 0.2, device=device, dtype=dtype)
image[:, :, :4] = 0.7
intensity = K.AugmentationSequential(K.RandomEqualize3D(p=1.0), data_keys=["input"])
output = intensity(image)
assert not torch.equal(output, image)
self.assert_close(intensity.inverse(output), output, atol=0, rtol=0)
geometric = K.AugmentationSequential(K.RandomHorizontalFlip3D(p=1.0), data_keys=["input"])
with pytest.raises(NotImplementedError, match="3d inverse"):
geometric.inverse(geometric(image))
@pytest.mark.parametrize("as_list", [False, True])
@pytest.mark.parametrize("operation", ["flip", "resize", "affine"])
def test_convention_integer_mask_labels_survive_4478(self, as_list, operation, device, dtype):
if operation == "affine" and not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("The image path requires bilinear grid_sample for the selected dtype.")
image = torch.ones(1, 1, 2, 3, device=device, dtype=dtype)
mask = torch.tensor([0, 257, 2049, 26001, -2049, 65535], device=device).reshape(1, 1, 2, 3)
children = {
"flip": K.RandomHorizontalFlip(p=1.0),
"resize": K.Resize((4, 6)),
"affine": K.RandomAffine(degrees=(0.0, 0.0), p=1.0),
}
seq = K.AugmentationSequential(children[operation], data_keys=["input", "mask"])
expected = mask.flip(-1) if operation == "flip" else mask
if operation == "resize":
expected = mask.repeat_interleave(2, -2).repeat_interleave(2, -1)
output = seq(image, [mask] if as_list else mask)[1]
output = output[0] if as_list else output
assert output.dtype == mask.dtype
assert torch.equal(output, expected)
if operation == "flip":
inverse = seq.inverse(*seq(image, [mask] if as_list else mask))[1]
inverse = inverse[0] if as_list else inverse
assert torch.equal(inverse, mask)
def test_convention_inexact_integer_mask_labels_raise_4478(self, dtype):
# CPU assertions report immediately; invalid asynchronous accelerator assertions can poison the stream.
working = torch.float64 if dtype == torch.float64 else torch.float32
label = 2**53 + 1 if working == torch.float64 else 2**24 + 1
image = torch.ones(1, 1, 2, 2, dtype=dtype)
mask = torch.full_like(image, label, dtype=torch.int64)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
with pytest.raises(RuntimeError, match=r"Integer mask labels.*exactly representable"):
seq(image, mask)
@pytest.mark.parametrize("mask_dtype", [torch.int32, torch.uint32, torch.int64, torch.uint64])
def test_integer_mask_boundary_labels_4478(self, dtype, mask_dtype):
# Keep invalid assertions on CPU: accelerator assertion failures can poison the stream.
limits = torch.iinfo(mask_dtype)
precision = 53 if dtype == torch.float64 else 24
exponent = limits.bits - int(limits.min < 0)
# The largest representable integer below 2**exponent has spacing 2**(exponent - precision).
last_exact = limits.max if exponent <= precision else 2**exponent - 2 ** (exponent - precision)
image = torch.ones(1, 1, 2, 2, dtype=dtype)
mask = torch.tensor([limits.min, last_exact, 0, limits.min], dtype=mask_dtype).reshape_as(image)
expected = torch.tensor([last_exact, limits.min, limits.min, 0], dtype=mask_dtype).reshape_as(image)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
assert torch.equal(seq(image, mask)[1], expected)
if exponent > precision:
with pytest.raises(RuntimeError, match=r"Integer mask labels.*exactly representable"):
seq(image, torch.full_like(mask, limits.max))
@pytest.mark.parametrize("mask_dtype", [torch.bool, torch.uint8, torch.int16, torch.int32, torch.int64])
def test_integer_mask_partial_gate_and_replay_4478(self, mask_dtype, device, dtype):
labels = [0, 1, 2] if mask_dtype in (torch.bool, torch.uint8) else [257, -2049, 26001]
mask = torch.tensor(labels, device=device, dtype=mask_dtype).reshape(1, 1, 1, 3).repeat(2, 1, 2, 1)
image = torch.ones(2, 1, 2, 3, device=device, dtype=dtype)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=0.5), data_keys=["input", "mask"])
seq(image, mask)
params = seq._params
params[0].data["batch_prob"] = torch.tensor([1.0, 0.0], device=device, dtype=dtype)
expected = mask.clone()
expected[0] = mask[0].flip(-1)
output = seq(image, mask, params=params)[1]
assert output.dtype == mask_dtype
assert torch.equal(output, expected)
replay = seq(mask, params=params, data_keys=["mask"])
assert torch.equal(replay, expected)
def test_integer_masks_keep_labels_under_autocast_4478(self):
image = torch.ones(1, 1, 2, 3)
mask = torch.tensor([0, 257, 2049, 26001, -2049, 65535]).reshape(1, 1, 2, 3)
seq = K.AugmentationSequential(K.RandomAffine(degrees=(0.0, 0.0), p=1.0), data_keys=["input", "mask"])
with torch.autocast("cpu", dtype=torch.bfloat16):
output = seq(image, mask)[1]
assert torch.equal(output, mask)
def test_integer_mask_cast_dynamo_4478(self, device, dtype, torch_optimizer):
# Exercise the dtype conversion and validation with a real fullgraph compiler, without RNG capture.
mask = torch.tensor([0, 257, 2049, 26001], device=device).reshape(1, 1, 2, 2)
op = partial(K.AugmentationSequential._cast_mask, dtype=dtype)
compiled = torch_optimizer(op, fullgraph=True)
self.assert_close(compiled(mask), op(mask), atol=0, rtol=0)
if device.type == "cpu":
label = 2**53 + 1 if dtype == torch.float64 else 2**24 + 1
for invalid in (label, torch.iinfo(torch.int64).max):
with pytest.raises(RuntimeError, match="Integer mask labels"):
compiled(torch.full_like(mask, invalid))
def test_integer_mask_float64_accepts_larger_exact_labels_4478(self):
image = torch.ones(1, 1, 2, 2, dtype=torch.float64)
mask = torch.tensor([2**24 + 1, -(2**24 + 1), 2**53, -(2**53)]).reshape(1, 1, 2, 2)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
assert torch.equal(seq(image, mask)[1], mask.flip(-1))
@pytest.mark.parametrize("padding", [0, 2])
@pytest.mark.parametrize("half", [torch.float16, torch.bfloat16])
def test_integer_mask_resample_crop_next_to_a_half_image_4478(self, half, padding, device):
# The integer mask works in float32 and the crop matrix comes from the half-precision image: the resample
# crop must combine them in one dtype and still cut the labels out exactly.
if not supports_bilinear_2d_grid_sample(device, half):
pytest.skip("The image path requires bilinear grid_sample for the selected dtype.")
image = torch.ones(2, 1, 8, 10, device=device, dtype=half)
mask = (torch.arange(160, device=device) * 9 + 2049).reshape(2, 1, 8, 10)
crop = K.RandomCrop((5, 6), padding=padding, cropping_mode="resample", p=1.0)
seq = K.AugmentationSequential(crop, data_keys=["input", "mask"])
out_image, out_mask = seq(image, mask)
padded = torch.nn.functional.pad(mask, [padding] * 4)
for sample, corner in zip(range(2), seq._params[0].data["src"][:, 0].long().tolist()):
x0, y0 = corner
assert torch.equal(out_mask[sample], padded[sample, :, y0 : y0 + 5, x0 : x0 + 6])
assert seq.inverse(out_image, out_mask)[1].dtype == mask.dtype
@pytest.mark.parametrize("mask_dtype", [None, torch.int64, torch.bool])
def test_convention_empty_batch_with_mask_4478(self, mask_dtype, device, dtype):
# Convention pin for kornia#4478: an empty batch with a mask returns an empty mask, forward and inverse,
# in the mask's own dtype.
image = torch.empty(0, 1, 2, 2, device=device, dtype=dtype)
mask = image.clone() if mask_dtype is None else torch.empty(0, 1, 2, 2, device=device, dtype=mask_dtype)
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
out_image, out_mask = seq(image, mask)
assert out_image.shape == out_mask.shape == (0, 1, 2, 2)
assert out_mask.dtype == mask.dtype
restored_image, restored_mask = seq.inverse(out_image, out_mask)
assert restored_image.shape == restored_mask.shape == (0, 1, 2, 2)
assert restored_mask.dtype == mask.dtype
@pytest.mark.parametrize(
"make_op", [lambda: K.Resize((4, 5)), lambda: K.CenterCrop((4, 5)), lambda: K.RandomCrop((4, 5), p=1.0)]
)
@pytest.mark.parametrize("as_dict", [False, True])
@pytest.mark.parametrize("mask_dtype", [torch.int64, torch.bool])
def test_empty_batch_shape_changing_mask_follows_image_4429(self, make_op, as_dict, mask_dtype, device, dtype):
image = torch.empty(0, 3, 8, 10, device=device, dtype=dtype)
mask = torch.empty(0, 1, 8, 10, device=device, dtype=mask_dtype)
seq = K.AugmentationSequential(make_op(), data_keys=None if as_dict else ["input", "mask"])
if as_dict:
result = seq({"input": image, "mask": mask})
out_image, out_mask = result["input"], result["mask"]
else:
out_image, out_mask = seq(image, mask)
assert out_image.shape == (0, 3, 4, 5)
assert out_mask.shape == (0, 1, 4, 5)
assert out_image.device == out_mask.device == device
assert out_image.dtype == dtype
assert out_mask.dtype == mask_dtype
@pytest.mark.parametrize(
"make_op",
[
lambda p: K.Resize((4, 5), p=p),
lambda p: K.CenterCrop((4, 5), cropping_mode="resample", p=p),
lambda p: K.RandomCrop((4, 5), cropping_mode="resample", p=p),
],
ids=["resize", "center-crop", "crop"],
)
@pytest.mark.parametrize("p", [0.5, 1.0])
def test_empty_batch_mask_round_trip_follows_image_4429(self, make_op, p, device, dtype):
# An empty batch takes the configured size under every gate (#4429): the mask follows the image onto that
# canvas, and the inverse restores both.
image = torch.empty(0, 3, 8, 10, device=device, dtype=dtype)
mask = torch.empty(0, 1, 8, 10, device=device, dtype=torch.int64)
seq = K.AugmentationSequential(make_op(p), data_keys=["input", "mask"])
out_image, out_mask = seq(image, mask)
assert out_image.shape == (0, 3, 4, 5)
assert out_mask.shape == (0, 1, 4, 5)
restored_image, restored_mask = seq.inverse(out_image, out_mask)
assert restored_image.shape == image.shape
assert restored_mask.shape == mask.shape
assert restored_mask.dtype == mask.dtype
def test_dictionary_preserves_metadata_and_input_4483(self, device, dtype):
seq = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None)
metadata = {"imagenet_id": 7, "maskrcnn_boxes": "unchanged", "labelled_image": None, "note": "retained"}
image = torch.arange(4, device=device, dtype=dtype).reshape(1, 1, 2, 2)
data = {"note": metadata["note"], "image": image, **metadata}
original = dict(data)
output = seq(data)
assert data.keys() == original.keys()
for key, value in original.items():
assert data[key] is value
self.assert_close(output["image"], image.flip(-1))
for key, value in metadata.items():
assert output[key] is value
restored = seq.inverse(output)
self.assert_close(restored["image"], image)
for key, value in metadata.items():
assert restored[key] is value
assert output[key] is value
def test_dictionary_key_boundaries_4483(self):
seq = K.AugmentationSequential(data_keys=None)
names = ("image", "IMAGE-left", "input_2", "mask_2", "keypoints-right", "bbox_xyxy_2", "bbox_xywh-a", "bbox_2")
keys, metadata = seq._read_datakeys_from_dict(names)
assert keys == [
DataKey.INPUT,
DataKey.INPUT,
DataKey.INPUT,
DataKey.MASK,
DataKey.KEYPOINTS,
DataKey.BBOX_XYXY,
DataKey.BBOX_XYWH,
DataKey.BBOX,
]
assert metadata == []
names = (
"imagenet_id",
"maskrcnn_boxes",
"labelled_image",
"keypoint",
"classification",
"inputsize",
"images",
"masks",
"labels",
"bboxes",
"inputs",
"image2",
"imageLeft",
"image.2",
"keypoints2",
)
keys, metadata = seq._read_datakeys_from_dict(names)
assert keys == []
assert metadata == list(names)
keys, metadata = seq._read_datakeys_from_dict(
("bbox_xyxy2", "bbox_xywh2", "class", "class_id", "class_weights", "class-names")
)
assert keys == [DataKey.BBOX, DataKey.BBOX, DataKey.LABEL, DataKey.LABEL, DataKey.LABEL, DataKey.LABEL]
assert metadata == []
def test_numpy_mask_is_read_channels_last_without_scaling_5236(self):
# A NumPy mask is laid out like a NumPy image, (H, W) or (H, W, C), and keeps its labels: an (8, 9, 1) mask
# becomes (1, 1, 8, 9) and flips with the image. The label block sits off the flip axis (columns 1-2 go to
# 6-7), and a mask read as (C, H, W) = (8, 9, 1) would come back with another shape.
image = np.zeros((8, 9, 3), dtype=np.uint8)
mask = np.zeros((8, 9, 1), dtype=np.int64)
mask[2:5, 1:3] = 3
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
_, out_mask = aug(image, mask)
expected = torch.zeros(1, 1, 8, 9, dtype=torch.int64)
expected[..., 2:5, 6:8] = 3
assert torch.equal(out_mask, expected)
def test_numpy_annotations_follow_the_image_to_the_module_device_5207(self):
# A stateful container converts a NumPy image onto its device; NumPy masks, keypoints and boxes go to the
# same device and keep their dtype, so a mask keeps its labels. The meta device stands in for an accelerator.
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
aug.register_buffer("reference", torch.empty((), device="meta", dtype=torch.float64))
mask = aug._convert_non_image(np.zeros((8, 9, 1), dtype=np.int64), DataKey.MASK)
keypoints = aug._convert_non_image(np.zeros((1, 2, 2), dtype=np.float32), DataKey.KEYPOINTS)
boxes = aug._convert_non_image(np.zeros((1, 1, 4), dtype=np.float32), DataKey.BBOX_XYXY)
image = aug.to_tensor(np.zeros((8, 9, 3), dtype=np.uint8))
assert [x.device.type for x in (image, mask, keypoints, boxes)] == ["meta"] * 4
assert [x.dtype for x in (image, mask, keypoints, boxes)] == [torch.float64, torch.int64] + [torch.float32] * 2
def test_numpy_image_and_mask_run_on_the_module_device_5207(self, device):
# RandomBrightness holds buffers, so the container's NumPy image moves to its device; the mask must follow
# it, or the flip mixes devices.
image = np.zeros((8, 9, 3), dtype=np.uint8)
mask = np.zeros((8, 9), dtype=np.int64)
mask[2:5, 1:3] = 3
aug = K.AugmentationSequential(
K.RandomBrightness((1.0, 1.0), p=1.0), K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"]
).to(device)
out_image, out_mask = aug(image, mask)
expected = torch.zeros(1, 1, 8, 9, dtype=torch.int64)
expected[..., 2:5, 6:8] = 3
assert out_image.device == out_mask.device == torch.device(device)
assert torch.equal(out_mask.cpu(), expected)
def test_argument_without_a_data_key_raises(self):
# ``__call__`` converts NumPy arguments by data key; an argument with no key must still reach ``forward``,
# which rejects the count, instead of being dropped.
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input"])
with pytest.raises(AssertionError, match="number of inputs must align"):
aug(torch.zeros(1, 3, 8, 9), torch.zeros(1, 1, 8, 9))
def test_convention_same_on_batch_none_does_not_override_a_child(self, device, dtype):
# Convention pin: `AugmentationSequential(same_on_batch=None)` - the default - keeps whatever each
# child was built with, while `True` and `False` overwrite the child's own setting in both directions.
# `keepdim` follows the same three-state rule.
# Snippet used to generate expected: this body, seed 0, B = 4:
# None over a same_on_batch=True child -> one distinct angle (36.5752067565918 four times);
# False over the same child -> four distinct angles; True over a same_on_batch=False child -> one.
# keepdim: None over a keepdim=True child -> (3, 6, 8), False -> (1, 3, 6, 8), True over False -> (3, 6, 8).
def angles(container_flag, child_flag):
torch.manual_seed(0)
child = K.RandomAffine(degrees=(10.0, 90.0), p=1.0, same_on_batch=child_flag)
seq = K.AugmentationSequential(child, data_keys=["input"], same_on_batch=container_flag)
seq(torch.rand(4, 3, 6, 8, device=device, dtype=dtype))
return child._params["angle"].unique().numel()
assert angles(None, True) == 1
assert angles(False, True) == 4
assert angles(True, False) == 1
assert angles(None, False) == 4
def shape(container_flag, child_flag):
seq = K.AugmentationSequential(
K.RandomHorizontalFlip(p=1.0, keepdim=child_flag), data_keys=["input"], keepdim=container_flag
)
return tuple(seq(torch.rand(3, 6, 8, device=device, dtype=dtype)).shape)
assert shape(None, True) == (3, 6, 8)
assert shape(False, True) == (1, 3, 6, 8)
assert shape(True, False) == (3, 6, 8)
@pytest.mark.parametrize("factory", [lambda: K.RandomAffine(30.0, p=1.0), lambda: K.RandomElasticTransform(p=1.0)])
def test_extra_args_mask_resample_accepts_a_string_or_int_4815(self, factory, device, dtype):
# #4815: the container normalizes a `resample` override as the constructors do.
image = torch.rand(1, 1, 6, 8, device=device, dtype=dtype)
masks = []
for resample in (Resample.NEAREST, "nearest", "NEAREST", 0):
seq = K.AugmentationSequential(
factory(), data_keys=["input", "mask"], extra_args={DataKey.MASK: {"resample": resample}}
)
assert seq.extra_args[DataKey.MASK]["resample"] is Resample.NEAREST
torch.manual_seed(0)
masks.append(seq(image, image.clone())[1])
for mask in masks[1:]:
assert torch.equal(mask, masks[0])
def test_extra_args_rejects_an_unknown_resample_4815(self):
with pytest.raises(KeyError):
K.AugmentationSequential(
K.RandomAffine(30.0, p=1.0), data_keys=["input", "mask"], extra_args={DataKey.MASK: {"resample": "foo"}}
)
def test_extra_args_mask_override_reaches_the_sampler_4419(self, device, dtype):
# #4419: both halves of `extra_args[DataKey.MASK]` reach the sampler; `RandomElasticTransform`, which has
# its own mask path, honours both too. The align_corners fixture samples outside the frame with
# padding_mode="reflection", where the two conventions differ (delta 1.0). The elastic fixture uses a
# checkerboard mask and a displacement large enough to keep the bilinear-vs-nearest delta well above 0.1.
def mask_of(aug_factory, extra):
torch.manual_seed(0)
aug = K.AugmentationSequential(aug_factory(), data_keys=["input", "mask"], extra_args=extra)
img = torch.rand(1, 3, 6, 8, device=device, dtype=dtype)
mask = torch.zeros(1, 1, 6, 8, device=device, dtype=dtype)
mask[:, :, 1:4, 2:6] = 1.0
return aug(img, mask)[1]
bilinear = {DataKey.MASK: {"resample": Resample.BILINEAR, "align_corners": None}}
align = {DataKey.MASK: {"resample": Resample.NEAREST, "align_corners": True}}
affine = lambda: K.RandomAffine(degrees=(45.0, 45.0), p=1.0) # noqa: E731
assert (mask_of(affine, bilinear) - mask_of(affine, None)).abs().max().item() > 0.0
reflected = lambda: K.RandomAffine(degrees=(45.0, 45.0), p=1.0, padding_mode="reflection") # noqa: E731
assert (mask_of(reflected, align) - mask_of(reflected, None)).abs().max().item() == 1.0
def elastic_mask_of(extra):
torch.manual_seed(0)
aug = K.AugmentationSequential(
K.RandomElasticTransform(alpha=(5.0, 5.0), sigma=(4.0, 4.0), p=1.0),
data_keys=["input", "mask"],
extra_args=extra,
)
img = torch.rand(1, 3, 6, 8, device=device, dtype=dtype)
yy, xx = torch.meshgrid(torch.arange(6, device=device), torch.arange(8, device=device), indexing="ij")
mask = ((yy + xx) % 2).to(dtype).expand(1, 1, 6, 8).clone()
return aug(img, mask)[1]
assert (elastic_mask_of(bilinear) - elastic_mask_of(None)).abs().max().item() > 0.1
# a user dict replaces the container's default wholesale rather than merging into it
default_args = K.AugmentationSequential(affine(), data_keys=["input", "mask"]).extra_args
assert default_args == {DataKey.MASK: {"resample": Resample.NEAREST, "align_corners": None}}
replaced = K.AugmentationSequential(
affine(), data_keys=["input", "mask"], extra_args={DataKey.MASK: {"align_corners": True}}
).extra_args
assert replaced == {DataKey.MASK: {"align_corners": True}}
@pytest.mark.parametrize(
"factory",
[
lambda: K.RandomAffine(degrees=(45.0, 45.0), p=1.0),
lambda: K.RandomRotation(degrees=(45.0, 45.0), p=1.0),
lambda: K.RandomPerspective(0.5, p=1.0),
lambda: K.RandomShear((20.0, 20.0), p=1.0),
lambda: K.Resize((9, 13)),
lambda: K.RandomResizedCrop((9, 13), p=1.0),
],
ids=["RandomAffine", "RandomRotation", "RandomPerspective", "RandomShear", "Resize", "RandomResizedCrop"],
)
def test_extra_args_mask_resample_is_honoured_and_nearest_stays_the_default_4419(self, factory, device, dtype):
# #4419: a bilinear `resample` under `extra_args[DataKey.MASK]` reaches the sampler on the base 2D
# geometric mask path, as it already did on `inverse` and on `RandomElasticTransform`. Without one --
# the container's default dict, or a user dict that leaves `resample` out -- masks stay nearest, so a
# binary mask stays binary even though these classes resample images bilinearly. A checkerboard mask,
# so that any bilinear resample produces in-between values wherever the crop or warp lands.
def mask_of(extra):
torch.manual_seed(0)
aug = K.AugmentationSequential(factory(), data_keys=["input", "mask"], extra_args=extra)
img = torch.rand(1, 3, 6, 8, device=device, dtype=dtype)
yy, xx = torch.meshgrid(torch.arange(6, device=device), torch.arange(8, device=device), indexing="ij")
mask = ((yy + xx) % 2).to(dtype).expand(1, 1, 6, 8).clone()
return aug(img, mask)[1]
def is_binary(t):
return bool(((t == 0) | (t == 1)).all())
default = mask_of(None)
assert is_binary(default)
assert is_binary(mask_of({DataKey.MASK: {"align_corners": None}}))
bilinear = mask_of({DataKey.MASK: {"resample": Resample.BILINEAR, "align_corners": None}})
assert not is_binary(bilinear)
assert (bilinear - default).abs().max().item() > 0.0
def test_direct_transform_masks_call_without_resample_stays_nearest_4419(self, device, dtype):
# The module's own `resample` is for images. A direct call that does not choose one must not pick it
# up for the mask.
torch.manual_seed(0)
aug = K.RandomAffine(degrees=(45.0, 45.0), p=1.0)
assert aug.flags["resample"] == Resample.BILINEAR
img = torch.rand(1, 3, 6, 8, device=device, dtype=dtype)
mask = torch.zeros(1, 1, 6, 8, device=device, dtype=dtype)
mask[:, :, 1:4, 2:6] = 1.0
aug(img)
out = aug.transform_masks(mask, params=aug._params, flags=aug.flags, transform=aug.transform_matrix)
assert bool(((out == 0) | (out == 1)).all())
assert aug.flags["resample"] == Resample.BILINEAR
@pytest.mark.parametrize("augmentation", ["RandomThinPlateSpline", "RandomFisheye"])
@pytest.mark.parametrize("p", [0.0, 1.0])
def test_unsupported_non_rigid_mask_reports_augmentation_4420(self, augmentation, p, device, dtype):
if augmentation == "RandomThinPlateSpline":
aug = K.RandomThinPlateSpline(p=p)
else:
aug = K.RandomFisheye(
torch.tensor([-0.3, 0.3], device=device, dtype=dtype),
torch.tensor([-0.3, 0.3], device=device, dtype=dtype),
torch.tensor([0.9, 1.0], device=device, dtype=dtype),
p=p,
)
image = torch.rand(2, 3, 6, 8, device=device, dtype=dtype)
mask = torch.zeros(2, 1, 6, 8, device=device, dtype=dtype)
with pytest.raises(NotImplementedError, match=rf"{augmentation}.*mask.*4420"):
K.AugmentationSequential(aug, data_keys=["input", "mask"])(image, mask)
@pytest.mark.parametrize("augmentation", ["RandomThinPlateSpline", "RandomFisheye"])
@pytest.mark.parametrize("method", ["apply_transform_mask", "apply_non_transform_mask"])
def test_unsupported_non_rigid_mask_hooks_report_augmentation_4420(self, augmentation, method, device, dtype):
# The container calls the transform hook first even at p=0, so exercise the skipped-input hook directly.
if augmentation == "RandomThinPlateSpline":
aug = K.RandomThinPlateSpline()
else:
aug = K.RandomFisheye(
torch.tensor([-0.3, 0.3], device=device, dtype=dtype),
torch.tensor([-0.3, 0.3], device=device, dtype=dtype),
torch.tensor([0.9, 1.0], device=device, dtype=dtype),
)
mask = torch.zeros(2, 1, 6, 8, device=device, dtype=dtype)
with pytest.raises(NotImplementedError, match=rf"{augmentation}.*mask.*4420"):
getattr(aug, method)(mask, {}, aug.flags)
@staticmethod
def _non_rigid_warp(augmentation, p, device, dtype):
if augmentation == "RandomElasticTransform":
return K.RandomElasticTransform(p=p)
if augmentation == "RandomThinPlateSpline":
return K.RandomThinPlateSpline(p=p)
return K.RandomFisheye(
torch.tensor([-0.3, 0.3], device=device, dtype=dtype),
torch.tensor([-0.3, 0.3], device=device, dtype=dtype),
torch.tensor([0.9, 1.0], device=device, dtype=dtype),
p=p,
)
@pytest.mark.parametrize("augmentation", ["RandomElasticTransform", "RandomThinPlateSpline", "RandomFisheye"])
@pytest.mark.parametrize("data_key", ["keypoints", "bbox", "bbox_xyxy", "bbox_xywh"])
@pytest.mark.parametrize("p", [0.0, 1.0])
def test_non_rigid_augmentation_refuses_keypoints_and_boxes_4420(self, augmentation, data_key, p, device, dtype):
# A non-rigid warp has no transformation matrix, so keypoints and boxes used to come back unchanged while the
# image moved. The container refuses them instead, at any `p`, until the warp moves them too.
image = torch.rand(2, 3, 6, 8, device=device, dtype=dtype)
if data_key == "keypoints":
coordinates = torch.tensor([[[1.0, 2.0]], [[3.0, 4.0]]], device=device, dtype=dtype)
elif data_key == "bbox":
corners = torch.tensor([[0.0, 0.0], [3.0, 0.0], [3.0, 2.0], [0.0, 2.0]], device=device, dtype=dtype)
coordinates = corners.expand(2, 1, 4, 2).clone()
else:
coordinates = torch.tensor([[[0.0, 0.0, 3.0, 2.0]], [[2.0, 1.0, 3.0, 3.0]]], device=device, dtype=dtype)
aug = K.AugmentationSequential(
self._non_rigid_warp(augmentation, p, device, dtype), data_keys=["input", data_key]
)
with pytest.raises(NotImplementedError, match=rf"{augmentation} does not support {data_key} inputs.*4420"):
aug(image, coordinates)
@pytest.mark.parametrize("arrangement", ["random_apply", "nested", "video"])
def test_non_rigid_refusal_reads_the_pipeline_not_the_draw_4420(self, arrangement, device, dtype):
# The refusal does not depend on which children a call draws: a `random_apply` weight of zero never draws
# the warp, yet the call raises, so a pipeline cannot pass on some batches and fail on others. A warp one
# container down, or applied frame by frame, is refused too.
flip = K.RandomHorizontalFlip(p=1.0)
warp = K.RandomElasticTransform(p=1.0)
image = torch.rand(2, 3, 6, 8, device=device, dtype=dtype)
keypoints = torch.tensor([[[1.0, 2.0]], [[3.0, 4.0]]], device=device, dtype=dtype)
if arrangement == "random_apply":
aug = K.AugmentationSequential(
flip, warp, data_keys=["input", "keypoints"], random_apply=1, random_apply_weights=[1.0, 0.0]
)
elif arrangement == "nested":
aug = K.AugmentationSequential(flip, K.ImageSequential(warp), data_keys=["input", "keypoints"])
else:
aug = K.AugmentationSequential(K.VideoSequential(flip, warp), data_keys=["input", "keypoints"])
image = torch.rand(2, 3, 3, 6, 8, device=device, dtype=dtype)
keypoints = keypoints.unsqueeze(1).expand(2, 3, 1, 2).clone()
with pytest.raises(NotImplementedError, match=r"RandomElasticTransform does not support keypoints.*4420"):
aug(image, keypoints)
def test_non_rigid_refusal_lists_every_unhandled_augmentation_4420(self):
# The container moves keypoints and boxes through rigid, intensity and mix children only; a shipped 2D
# augmentation outside them would return the coordinates unchanged, so it must be refused.
handled = (K.RigidAffineAugmentationBase2D, K.IntensityAugmentationBase2D, K.MixAugmentationBaseV2)
unhandled = {
cls
for cls in vars(K).values()
if isinstance(cls, type)
and issubclass(cls, K.AugmentationBase2D)
and cls is not K.AugmentationBase2D
and not issubclass(cls, handled)
}
assert unhandled == set(_NON_RIGID_WARPS)
def test_elastic_box_hook_refuses_boxes_4420(self, device, dtype):
# A direct call reaches the hook that used to return the boxes unchanged.
aug = K.RandomElasticTransform(p=1.0)
boxes = Boxes.from_tensor(torch.tensor([[[0.0, 0.0, 3.0, 2.0]]], device=device, dtype=dtype))
with pytest.raises(NotImplementedError, match=r"RandomElasticTransform does not support box inputs.*4420"):
aug.apply_transform_box(boxes, {}, aug.flags)
def test_convention_non_rigid_augmentation_masks_4420(self, device, dtype):
# Masks are untouched by the keypoint and box refusal: `RandomElasticTransform` resamples a mask through
# the image's displacement field (nearest), and `RandomThinPlateSpline` / `RandomFisheye` raise on one.
# Seed zero and a checkerboard with a stronger displacement expose both the image and mask warp.
torch.manual_seed(0)
aug = K.AugmentationSequential(
K.RandomElasticTransform(p=1.0, alpha=(5.0, 5.0), sigma=(4.0, 4.0)), data_keys=["input", "mask"]
)
img = torch.rand(2, 3, 6, 8, device=device, dtype=dtype)
yy, xx = torch.meshgrid(torch.arange(6, device=device), torch.arange(8, device=device), indexing="ij")
mask = ((yy + xx) % 2).to(dtype).expand(2, 1, 6, 8).clone()
out_img, out_mask = aug(img, mask)
assert (out_img - img).abs().max().item() > 0.1 # the image really moved
assert (out_mask != mask).sum().item() > 0 # and so did the mask, through the same displacement field
for factory in (
lambda: K.RandomThinPlateSpline(p=1.0),
lambda: K.RandomFisheye(
torch.tensor([-0.3, 0.3], device=device, dtype=dtype),
torch.tensor([-0.3, 0.3], device=device, dtype=dtype),
torch.tensor([0.9, 1.0], device=device, dtype=dtype),
p=1.0,
),
):
with pytest.raises(NotImplementedError):
K.AugmentationSequential(factory(), data_keys=["input", "mask"])(img, mask)
@pytest.mark.parametrize("data_style", ["image", "list", "dict", "dict_mask_first"])
def test_show_and_save_render_a_detached_image_4835(self, data_style, tmp_path, device, dtype):
# `.show()` / `.save()` render the cache through `.numpy()`, so it must hold the augmented image
# alone and detached: the raw forward output keeps the autograd graph and, for several data keys,
# is a list or a dict. It stays on the input device; the helpers move it to the CPU themselves.
from PIL import Image as PILImage
image = torch.rand(2, 3, 6, 8, device=device, dtype=dtype, requires_grad=True)
mask = torch.rand(2, 1, 6, 8, device=device, dtype=dtype)
if data_style == "image":
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0))
out_image = aug(image)
elif data_style == "list":
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
out_image, _ = aug(image, mask)
elif data_style == "dict":
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None)
out_image = aug({"image": image, "mask": mask})["image"]
else: # the image is not the first key, so the cache must pick it by its position
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None)
out_image = aug({"mask": mask, "image": image})["image"]
assert out_image.requires_grad # the returned image still carries the graph
cached = aug._output_image
assert isinstance(cached, torch.Tensor)
assert not cached.requires_grad
assert cached.device == out_image.device # no device-to-host copy in the forward pass
self.assert_close(cached, out_image.detach())
assert isinstance(aug.show(display=False), PILImage.Image)
path = tmp_path / "augmented.jpg"
aug.save(name=str(path))
assert path.is_file()
@pytest.mark.parametrize("no_image_style", ["dict", "data_keys"])
def test_show_and_save_reject_a_call_that_carried_no_image_4835(self, no_image_style, tmp_path, device, dtype):
# A call whose data keys hold no `DataKey.INPUT` augments no image, so there is nothing to
# render: neither the mask that came out nor the raw container output may be cached. Both
# helpers report the empty cache instead. `forward` only accepts such a call when `params`
# are supplied, since it otherwise has no image to sample them from.
mask = torch.rand(2, 1, 6, 8, device=device, dtype=dtype)
if no_image_style == "dict":
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=None)
params = aug.forward_parameters((2, 3, 6, 8))
out = aug({"mask": mask}, params=params)
assert set(out) == {"mask"}
else:
aug = K.AugmentationSequential(K.RandomHorizontalFlip(p=1.0), data_keys=["input", "mask"])
params = aug.forward_parameters((2, 3, 6, 8))
out = aug(mask, data_keys=["mask"], params=params)
assert isinstance(out, torch.Tensor)
assert aug._output_image is None
for helper in (lambda: aug.show(display=False), lambda: aug.save(name=str(tmp_path / "none.jpg"))):
with pytest.raises(ValueError, match="No pre-computed images found"):
helper()
class TestAugmentationSequentialReplay:
@pytest.mark.parametrize("edit_in_place", [False, True])
def test_annotation_replay_uses_edited_params(self, device, dtype, edit_in_place):
with torch.random.fork_rng(devices=[device] if device.type == "cuda" else []):
torch.manual_seed(0)
image = torch.rand(2, 1, 16, 16, device=device, dtype=dtype)
keypoints = torch.tensor([[[3.0, 4.0]], [[3.0, 4.0]]], device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomAffine(30.0, p=0.5), data_keys=["input", "keypoints"])
params = aug.forward_parameters(image.shape)
first_gate = torch.tensor([True, False], device=device)
second_gate = torch.tensor([False, True], device=device)
params[0].data["batch_prob"] = first_gate.clone()
aug(image, keypoints, params=params)
if edit_in_place:
params[0].data["batch_prob"][:] = second_gate
else:
params[0].data["batch_prob"] = second_gate
replayed = aug(keypoints, params=params, data_keys=["keypoints"])
moved = [(replayed[i] - keypoints[i]).abs().max() > 1e-4 for i in range(2)]
assert [bool(value) for value in moved] == [False, True]
def test_annotation_replay_after_repeated_replacements(self, device, dtype):
# Each replacement frees the previous tensor, and the next one can take its address, so a cache keyed by
# ``id()`` would see the tensor recorded by the image call again. Repeat to make an address reuse likely.
image = torch.rand(2, 1, 16, 16, device=device, dtype=dtype)
keypoints = torch.tensor([[[3.0, 4.0]], [[9.0, 12.0]]], device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomAffine(30.0, p=1.0), data_keys=["input", "keypoints"])
for _ in range(10):
params = aug.forward_parameters(image.shape)
aug(image, keypoints, params=params)
params[0].data["angle"] = params[0].data["angle"] * 0
params[0].data["angle"] = params[0].data["angle"] + 10.0
replayed = aug(keypoints, params=params, data_keys=["keypoints"])
_, expected = aug(image, keypoints, params=params)
assert_close(replayed, expected)
def test_annotation_replay_with_inference_tensors(self, monkeypatch):
# Inference tensors have no version counter: a replaced value is still seen, and the annotations of one call
# still share the image's matrix instead of recomputing one each.
calls = []
generate = K.RandomAffine.generate_transformation_matrix
def counted(self, *args, **kwargs):
calls.append(None)
return generate(self, *args, **kwargs)
monkeypatch.setattr(K.RandomAffine, "generate_transformation_matrix", counted)
with torch.inference_mode():
image = torch.rand(2, 1, 16, 16)
keypoints = torch.tensor([[[3.0, 4.0]], [[3.0, 4.0]]])
boxes = torch.tensor([[[2.0, 4.0, 6.0, 8.0]]]).expand(2, -1, -1)
data_keys = ["input", "mask", "bbox_xyxy", "keypoints"]
aug = K.AugmentationSequential(K.RandomAffine(30.0, p=0.5), data_keys=data_keys)
params = aug.forward_parameters(image.shape)
params[0].data["batch_prob"] = torch.tensor([True, False])
aug(image, image, boxes, keypoints, params=params)
assert len(calls) == 1
params[0].data["batch_prob"] = torch.tensor([False, True])
replayed = aug(keypoints, params=params, data_keys=["keypoints"])
moved = [(replayed[i] - keypoints[i]).abs().max() > 1e-4 for i in range(2)]
assert [bool(value) for value in moved] == [False, True]
def test_annotation_replay_uses_matrices_from_params(self, device, dtype):
with torch.random.fork_rng(devices=[device] if device.type == "cuda" else []):
torch.manual_seed(0)
image = torch.rand(1, 1, 16, 16, device=device, dtype=dtype)
mask = torch.zeros_like(image)
mask[..., 4:8, 2:6] = 1
boxes = torch.tensor([[[2.0, 4.0, 6.0, 8.0]]], device=device, dtype=dtype)
keypoints = torch.tensor([[[3.0, 5.0]]], device=device, dtype=dtype)
data_keys = ["input", "mask", "bbox_xyxy", "keypoints"]
aug = K.AugmentationSequential(K.RandomAffine(45.0, translate=(0.2, 0.2), p=1.0), data_keys=data_keys)
first = aug(image, mask, boxes, keypoints)
params = aug._params
expected_inverse = aug.inverse(*first, params=params)
aug(
torch.rand(2, 1, 16, 16, device=device, dtype=dtype),
mask.expand(2, -1, -1, -1),
boxes.expand(2, -1, -1),
keypoints.expand(2, -1, -1),
)
replayed = aug(mask, boxes, keypoints, params=params, data_keys=data_keys[1:])
assert_close(replayed[0], first[1])
assert_close(replayed[1], first[2])
assert_close(replayed[2], first[3])
actual_inverse = aug.inverse(*first, params=params)
for actual, expected in zip(actual_inverse, expected_inverse):
assert_close(actual, expected)
def test_annotations_reuse_the_matrix_of_the_image_call(self, device, dtype, monkeypatch):
# Within one call the image records its matrix next to the params dict it was given, and the mask, box and
# keypoint handlers reuse it instead of recomputing one matrix each. The inverse recomputes the image's matrix
# once, as an image-only inverse does, and its annotation handlers reuse the recorded one.
calls = []
generate = K.RandomAffine.generate_transformation_matrix
def counted(self, *args, **kwargs):
calls.append(None)
return generate(self, *args, **kwargs)
monkeypatch.setattr(K.RandomAffine, "generate_transformation_matrix", counted)
image = torch.zeros(2, 1, 16, 16, device=device, dtype=dtype)
boxes = torch.tensor([[[2.0, 4.0, 6.0, 8.0]]], device=device, dtype=dtype).expand(2, -1, -1)
keypoints = torch.tensor([[[3.0, 5.0]]], device=device, dtype=dtype).expand(2, -1, -1)
aug = K.AugmentationSequential(
K.RandomAffine(45.0, p=1.0), data_keys=["input", "mask", "bbox_xyxy", "keypoints"]
)
out = aug(image, image, boxes, keypoints)
assert len(calls) == 1
aug.inverse(*out)
assert len(calls) == 2
def test_annotation_replay_after_a_direct_leaf_call(self, device, dtype):
# A child called on its own between the container call and the replay records another draw; the replay
# must still use the params it is given.
with torch.random.fork_rng(devices=[device] if device.type == "cuda" else []):
torch.manual_seed(0)
image = torch.rand(1, 1, 16, 16, device=device, dtype=dtype)
keypoints = torch.tensor([[[3.0, 5.0]]], device=device, dtype=dtype)
aug = K.AugmentationSequential(K.RandomAffine(45.0, p=1.0), data_keys=["input", "keypoints"])
out_image, out_keypoints = aug(image, keypoints)
params = aug._params
aug[0](torch.rand(3, 1, 16, 16, device=device, dtype=dtype))
_, back = aug.inverse(out_image, out_keypoints, params=params)
assert_close(back, keypoints)
def test_annotation_replay_of_an_oversized_padded_crop(self, device, dtype):
# A crop larger than the padded canvas is scaled against that canvas. The replayed matrix must use the
# unpadded input shape (``forward_input_shape`` minus ``padding_size``), or the keypoints shift.
with torch.random.fork_rng(devices=[device] if device.type == "cuda" else []):
torch.manual_seed(0)
image = torch.rand(2, 1, 16, 16, device=device, dtype=dtype)
keypoints = torch.tensor([[[3.0, 5.0], [9.0, 12.0]]], device=device, dtype=dtype).expand(2, -1, -1)
aug = K.AugmentationSequential(K.RandomCrop((20, 22), padding=1, p=1.0), data_keys=["input", "keypoints"])
# A forward call rejects the oversized crop (#4414), so draw its parameters for replay.
def oversized_params(shape):
return [
ParamItem(name, oversized_random_crop_params(crop, shape)) for name, crop in aug.named_children()
]
params = oversized_params(image.shape)
_, expected = aug(image, keypoints, params=params)
other = torch.rand(3, 1, 15, 17, device=device, dtype=dtype)
aug(other, keypoints[:1].expand(3, -1, -1), params=oversized_params(other.shape))
replayed = aug(keypoints, params=params, data_keys=["keypoints"])
assert_close(replayed, expected)
class TestCrop3DShapePropagation(BaseTester):
@pytest.mark.parametrize("batch", [0, 1, 2])
@pytest.mark.parametrize(
"first_type,nested", [(K.CenterCrop3D, False), (K.RandomCrop3D, False), (K.CenterCrop3D, True)]
)
@pytest.mark.parametrize("second_type", [K.CenterCrop3D, K.RandomCrop3D])
@pytest.mark.parametrize("p", [0.0, 1.0])
def test_downstream_canvas_and_replay(self, batch, first_type, nested, second_type, p, device, dtype):
from copy import deepcopy
from testing.base import supports_bilinear_3d_grid_sample
if batch and not supports_bilinear_3d_grid_sample(device, dtype):
pytest.skip("3D grid sampling is unsupported for this device/dtype.")
image = torch.arange(batch * 8 * 10 * 12, device=device, dtype=dtype).reshape(batch, 1, 8, 10, 12)
def make_seq():
first = first_type((4, 5, 6), p=p)
return K.AugmentationSequential(
K.AugmentationSequential(first) if nested else first, second_type((2, 3, 4))
)
torch.manual_seed(17)
expected = second_type((2, 3, 4))(first_type((4, 5, 6), p=p)(image))
seq = make_seq()
seen = []
handle = seq[1].register_forward_pre_hook(lambda module, args: seen.append(args[0].shape))
torch.manual_seed(17)
actual = seq(image)
handle.remove()
self.assert_close(actual, expected)
saved = deepcopy(seq._params)
# A skipped crop keeps its input canvas; an empty batch has no sample to skip (#4429).
canvas = (4, 5, 6) if p == 1.0 or batch == 0 else (8, 10, 12)
assert seen == [torch.Size((batch, 1, *canvas))]
assert tuple(saved[1].data["forward_input_shape"]) == seen[0]
assert saved[1].data["output_size"].shape == (batch, 3)
src = saved[1].data["src"]
assert (src >= 0).all()
assert (src < src.new_tensor(canvas[::-1])).all()
seq(image)
self.assert_close(seq(image, params=deepcopy(saved)), actual)
self.assert_close(make_seq()(image, params=deepcopy(saved)), actual)
@pytest.mark.parametrize("batch", [0, 2])
def test_dynamo(self, batch, device, dtype, torch_optimizer):
from testing.base import supports_bilinear_3d_grid_sample
if batch and not supports_bilinear_3d_grid_sample(device, dtype):
pytest.skip("3D grid sampling is unsupported for this device/dtype.")
image = torch.arange(batch * 8 * 10 * 12, device=device, dtype=dtype).reshape(batch, 1, 8, 10, 12)
first, second = K.CenterCrop3D((4, 5, 6)), K.CenterCrop3D((2, 3, 4))
expected = second(first(image))
seq = K.AugmentationSequential(first, second)
self.assert_close(torch_optimizer(seq)(image), expected)
def test_static_shape_does_not_read_scalars(self, device):
from kornia.augmentation.container.image import _get_new_batch_shape
shape = torch.Size((2, 1, 8, 10, 12))
for op in (K.CenterCrop3D((4, 5, 6)), K.RandomCrop3D((4, 5, 6))):
params = {key: value.to(device) for key, value in op.forward_parameters(shape).items()}
with torch.profiler.profile(activities=[torch.profiler.ProfilerActivity.CPU]) as profiler:
actual = _get_new_batch_shape(ParamItem("crop", params), shape, op)
assert actual == (2, 1, 4, 5, 6)
assert not any(event.key == "aten::_local_scalar_dense" for event in profiler.key_averages())
@pytest.mark.parametrize("p", [0.0, 1.0])
def test_export_shape_respects_static_crop_gate(self, p):
from kornia.augmentation.container.image import _get_new_batch_shape
shape = torch.Size((2, 1, 8, 10, 12))
for crop in (K.CenterCrop3D, K.RandomCrop3D):
op = crop((4, 5, 6), p=p)
params = ParamItem("crop", op.forward_parameters(shape))
with patch("kornia.augmentation.container.image.is_exporting", return_value=True):
actual = _get_new_batch_shape(params, shape, op)
assert actual == (shape if p == 0.0 else torch.Size((2, 1, 4, 5, 6)))
@pytest.mark.parametrize("first_type", [K.CenterCrop3D, K.RandomCrop3D])
@pytest.mark.parametrize("second_type", [K.CenterCrop3D, K.RandomCrop3D])
def test_intermediate_probability_follows_gate(self, first_type, second_type, device, dtype):
from testing.base import supports_bilinear_3d_grid_sample
if not supports_bilinear_3d_grid_sample(device, dtype):
pytest.skip("3D grid sampling is unsupported for this device/dtype.")
image = torch.arange(2 * 8 * 10 * 12, device=device, dtype=dtype).reshape(2, 1, 8, 10, 12)
canvases = set()
for seed in range(6):
torch.manual_seed(seed)
expected = second_type((2, 3, 4))(first_type((4, 5, 6), p=0.5)(image))
seq = K.AugmentationSequential(first_type((4, 5, 6), p=0.5), second_type((2, 3, 4)))
torch.manual_seed(seed)
self.assert_close(seq(image), expected)
canvases.add(tuple(seq._params[1].data["forward_input_shape"].tolist()))
# p < 1 reads the sampled gate; both outcomes must occur and each must set the next crop's canvas.
assert canvases == {(2, 1, 4, 5, 6), (2, 1, 8, 10, 12)}
@pytest.mark.parametrize("batch", [0, 2])
def test_numpy_integer_size(self, batch, device, dtype):
from kornia.augmentation.container.image import _get_new_batch_shape
from testing.base import supports_bilinear_3d_grid_sample
if batch and not supports_bilinear_3d_grid_sample(device, dtype):
pytest.skip("3D grid sampling is unsupported for this device/dtype.")
# The 3D crops accept NumPy integer sizes; the tracked shape must hold Python ints for the next generator.
size = (np.int64(4), np.int64(5), np.int64(6))
image = torch.arange(batch * 8 * 10 * 12, device=device, dtype=dtype).reshape(batch, 1, 8, 10, 12)
for first_type in (K.CenterCrop3D, K.RandomCrop3D):
torch.manual_seed(17)
expected = K.RandomCrop3D((2, 3, 4))(first_type(size)(image))
seq = K.AugmentationSequential(first_type(size), K.RandomCrop3D((2, 3, 4)))
torch.manual_seed(17)
self.assert_close(seq(image), expected)
assert tuple(seq._params[1].data["forward_input_shape"].tolist()) == (batch, 1, 4, 5, 6)
op = first_type(size)
params = ParamItem("crop", op.forward_parameters(image.shape))
with patch("kornia.augmentation.container.image.is_exporting", return_value=True):
actual = _get_new_batch_shape(params, image.shape, op)
assert [type(dim) for dim in actual] == [int] * 5
assert actual == (batch, 1, 4, 5, 6)
class TestPadToMosaicShapePropagation(BaseTester):
@pytest.mark.parametrize(
"make_first",
[
lambda: K.PadTo((40, 48)),
lambda: K.PadTo((16, 48), crop_if_exceeds=False), # keeps the larger input height, pads the width
lambda: K.RandomMosaic(output_size=(20, 30), p=1.0),
],
ids=["PadTo", "PadTo-no-crop", "RandomMosaic"],
)
@pytest.mark.parametrize(
"make_second",
[lambda: K.Resize(16), lambda: K.CenterCrop(8, cropping_mode="resample")],
ids=["Resize-int", "CenterCrop-resample"],
)
def test_next_child_sees_the_new_canvas_6043(self, make_first, make_second, device, dtype):
if not supports_bilinear_2d_grid_sample(device, dtype):
pytest.skip("2D bilinear grid sampling is unsupported for this device/dtype.")
image = torch.rand(2, 3, 24, 40).to(device, dtype)
seq = K.AugmentationSequential(make_first(), make_second())
actual = seq(image)
canvas = seq[0](image, params=seq._params[0].data)
assert tuple(seq._params[1].data["forward_input_shape"].tolist()) == tuple(canvas.shape)
self.assert_close(actual, make_second()(canvas))
@pytest.mark.parametrize("batch", [0, 2])
@pytest.mark.parametrize("exporting", [False, True])
def test_tracked_size_6043(self, batch, exporting):
from kornia.augmentation.container.image import _get_new_batch_shape
shape = torch.Size((batch, 3, 24, 40))
# RandomMosaic resizes to ``output_size`` also when the mosaic is skipped, and keeps the size without one.
cases = [
(K.PadTo((40, 48)), (40, 48)),
(K.PadTo((16, 48)), (16, 48)), # a target below the input height crops it
(K.PadTo((16, 48), crop_if_exceeds=False), (24, 48)), # keeps the larger input height
(K.RandomMosaic(output_size=(20, 30), p=0.0), (20, 30)),
(K.RandomMosaic(p=1.0), (24, 40)),
]
for op, size in cases:
params = ParamItem("op", op.forward_parameters(shape))
with patch("kornia.augmentation.container.image.is_exporting", return_value=exporting):
assert _get_new_batch_shape(params, shape, op) == (batch, 3, *size)