Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

34.2 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import annotations

from copy import deepcopy

import pytest
import torch

import kornia.augmentation as K

from testing.base import BaseTester, supports_bilinear_2d_grid_sample_backward


class TestEmptyGeometricInverse(BaseTester):
    @pytest.mark.parametrize("method", ["inverse_inputs", "inverse_masks"])
    @pytest.mark.parametrize(
        "make_aug",
        [
            lambda: K.Resize((4, 5)),
            lambda: K.Resize(4, side="long"),
            lambda: K.LongestMaxSize(4),
            lambda: K.SmallestMaxSize(4),
            lambda: K.CenterCrop((4, 5), cropping_mode="resample"),
            lambda: K.RandomResizedCrop((4, 5), cropping_mode="resample"),
        ],
        ids=["resize-tuple", "resize-int", "longest", "smallest", "center-crop", "resized-crop"],
    )
    def test_restores_spatial_shape_4429(self, make_aug, method, device, dtype):
        aug = make_aug()
        params = aug.forward_parameters((0, 3, 6, 8))
        # A mask can have fewer channels than the image whose parameters it replays.
        channels = 1 if method == "inverse_masks" else 3
        cropped = torch.empty(0, channels, 4, 5, device=device, dtype=dtype, requires_grad=True)
        restored = getattr(aug, method)(cropped, params, aug.flags)
        assert restored.shape == (0, channels, 6, 8)
        assert restored.device == device
        assert restored.dtype == dtype
        restored.sum().backward()
        self.assert_close(cropped.grad, torch.empty_like(cropped))

    @pytest.mark.parametrize("method", ["inverse_inputs", "inverse_masks"])
    def test_without_shape_metadata_preserves_input(self, method, device, dtype):
        aug = K.RandomHorizontalFlip()
        image = torch.empty(0, 3, 4, 5, device=device, dtype=dtype)
        output = getattr(aug, method)(image, {"batch_prob": torch.empty(0)}, aug.flags)
        self.assert_close(output, image)

    @pytest.mark.parametrize(
        "padding,pad_if_needed,size",
        [
            (None, False, (4, 5)),
            (2, False, (4, 5)),
            ((1, 2, 3, 4), False, (4, 5)),
            ((0, 1, 2, 3), True, (10, 12)),
        ],
    )
    @pytest.mark.parametrize("method", ["inverse_inputs", "inverse_masks", "inverse"])
    @pytest.mark.parametrize("p", [0.0, 1.0])
    def test_crop_replays_padding_history_4429(self, padding, pad_if_needed, size, method, p, device, dtype):
        aug = K.RandomCrop(size, padding=padding, pad_if_needed=pad_if_needed, cropping_mode="resample", p=p)
        params = deepcopy(aug.forward_parameters((0, 3, 6, 8)))
        assert params["padding_size"].shape == (0, 4)
        assert params["forward_input_shape"].tolist() == [0, 3, 6, 8]
        channels = 1 if method == "inverse_masks" else 3
        cropped = torch.empty(0, channels, *size, device=device, dtype=dtype, requires_grad=True)
        # Replay must use the original padding history, including after module flags change.
        aug.flags.update(padding=7, pad_if_needed=False, size=(2, 3))
        if method == "inverse":
            restored = aug.inverse(cropped, params=params)
        else:
            restored = getattr(aug, method)(cropped, params, aug.flags)
        assert restored.shape == (0, channels, 6, 8)
        assert restored.device == device
        assert restored.dtype == dtype
        restored.sum().backward()
        self.assert_close(cropped.grad, torch.empty_like(cropped))

    @pytest.mark.parametrize(
        "make_aug",
        [
            lambda: K.Resize((4, 5), p=0.0),
            lambda: K.RandomCrop((4, 5), padding=(1, 2, 3, 4), p=0.0, cropping_mode="resample"),
        ],
        ids=["resize", "crop"],
    )
    @pytest.mark.parametrize("method", ["inverse_inputs", "inverse_masks"])
    def test_nonempty_skipped_batch_is_unchanged(self, make_aug, method, device, dtype):
        aug = make_aug()
        image = torch.rand(2, 3, 6, 8, device=device, dtype=dtype)
        params = aug.forward_parameters(image.shape)
        output = getattr(aug, method)(image, params, aug.flags)
        self.assert_close(output, image, rtol=0, atol=0)

    @pytest.mark.parametrize("p", [0.0, 0.5, 1.0])
    @pytest.mark.parametrize("height,width", [(6, 8), (9, 13)])
    def test_sequential_round_trip_4429(self, p, height, width, device, dtype):
        if not supports_bilinear_2d_grid_sample_backward(device, dtype):
            pytest.skip("The forward resampling kernel does not support this device/dtype's backward pass.")
        aug = K.AugmentationSequential(
            K.RandomCrop((4, 5), padding=(1, 2, 3, 4), cropping_mode="resample", p=p),
            K.Resize((2, 3)),
            data_keys=["input"],
        )
        image = torch.empty(0, 3, height, width, device=device, dtype=dtype, requires_grad=True)
        output = aug(image)
        mask = torch.empty(0, 1, *output.shape[-2:], device=device, dtype=dtype, requires_grad=True)
        restored_image, restored_mask = aug.inverse(output, mask, data_keys=["input", "mask"])
        assert restored_image.shape == image.shape
        assert restored_mask.shape == (0, 1, height, width)
        for restored, original in ((restored_image, image), (restored_mask, mask)):
            assert restored.device == original.device
            assert restored.dtype == original.dtype
            restored.sum().backward()
            self.assert_close(original.grad, torch.empty_like(original))

    @pytest.mark.parametrize(
        "crop_p,resize_p",
        [
            (0.0, 0.0),
            (0.0, 0.5),
            (0.0, 1.0),
            (0.5, 0.0),
            (0.5, 0.5),
            (0.5, 1.0),
            (1.0, 0.0),
            (1.0, 0.5),
            (1.0, 1.0),
        ],
    )
    @pytest.mark.parametrize("height,width", [(6, 8), (9, 13)])
    def test_empty_stage_shape_history_4429(self, crop_p, resize_p, height, width, device, dtype):
        crop = K.RandomCrop((4, 5), cropping_mode="resample", p=crop_p)
        resize = K.Resize((3, 4), p=resize_p)
        sequence = K.AugmentationSequential(crop, resize, data_keys=["input"])
        image = torch.empty(0, 3, height, width, device=device, dtype=dtype)
        output = sequence(image)
        saved_params = deepcopy(sequence._params)

        # An empty batch has no sample to skip, so every stage takes its configured output size whatever its
        # probability (#4429), as on 0.8.3 for the crops; the tracked shapes must follow that forward.
        crop_size = (4, 5)
        output_size = (3, 4)
        assert saved_params[0].data["forward_input_shape"].tolist() == [0, 3, height, width]
        assert saved_params[1].data["forward_input_shape"].tolist() == [0, 3, *crop_size]
        assert output.shape == (0, 3, *output_size)
        assert output.dtype == image.dtype
        assert output.device == image.device

    @pytest.mark.parametrize(
        "height,width,intermediate_size",
        [(6, 8, (7, 9)), (9, 13, (7, 10))],
    )
    def test_empty_integer_resize_shape_history_4429(self, height, width, intermediate_size, device, dtype):
        first_resize = K.Resize(7, side="short")
        second_resize = K.Resize((3, 4))
        sequence = K.AugmentationSequential(first_resize, second_resize, data_keys=["input"])
        image = torch.empty(0, 3, height, width, device=device, dtype=dtype)
        direct_output = first_resize(image)
        assert direct_output.shape == (0, 3, *intermediate_size)
        assert direct_output.dtype == image.dtype
        assert direct_output.device == image.device

        output = sequence(image)
        saved_params = deepcopy(sequence._params)
        assert output.shape == (0, 3, 3, 4)
        assert output.dtype == image.dtype
        assert output.device == image.device
        assert saved_params[0].data["forward_input_shape"].tolist() == [0, 3, height, width]
        assert saved_params[1].data["forward_input_shape"].tolist() == [0, 3, *intermediate_size]

        for replay_state in ("immediate", "after-forward"):
            if replay_state == "after-forward":
                next_height, next_width = (9, 13) if (height, width) == (6, 8) else (6, 8)
                nonempty = torch.arange(
                    2 * 3 * next_height * next_width,
                    device=device,
                    dtype=dtype,
                ).reshape(2, 3, next_height, next_width)
                sequence(nonempty)

            mask = torch.empty(0, 1, *output.shape[-2:], device=device, dtype=dtype, requires_grad=True)
            intermediate = second_resize.inverse(mask, params=saved_params[1].data)
            assert intermediate.shape == (0, 1, *intermediate_size)
            restored = first_resize.inverse(intermediate, params=saved_params[0].data)
            assert restored.shape == (0, 1, height, width)

            replayed = sequence.inverse(mask, params=saved_params, data_keys=["mask"])
            assert replayed.shape == (0, 1, height, width)
            assert replayed.dtype == mask.dtype
            assert replayed.device == mask.device
            replayed.sum().backward()
            assert mask.grad is not None
            assert mask.grad.shape == mask.shape

    @pytest.mark.parametrize("height,width", [(6, 8), (9, 13)])
    def test_empty_nested_shape_history_4429(self, height, width, device, dtype):
        crop = K.RandomCrop((4, 5), padding=(1, 2, 3, 4), cropping_mode="resample", p=1.0)
        inner_resize = K.Resize((3, 4))
        inner = K.ImageSequential(crop, inner_resize)
        outer_resize = K.Resize((2, 3))
        sequence = K.AugmentationSequential(inner, outer_resize, data_keys=["input"])
        image = torch.empty(0, 3, height, width, device=device, dtype=dtype)
        output = sequence(image)
        saved_params = deepcopy(sequence._params)
        inner_params = saved_params[0].data
        assert output.shape == (0, 3, 2, 3)
        assert output.dtype == image.dtype
        assert output.device == image.device
        assert inner_params[0].data["forward_input_shape"].tolist() == [0, 3, height, width]
        assert inner_params[1].data["forward_input_shape"].tolist() == [0, 3, 4, 5]
        assert saved_params[1].data["forward_input_shape"].tolist() == [0, 3, 3, 4]

        mask = torch.empty(0, 1, *output.shape[-2:], device=device, dtype=dtype, requires_grad=True)
        outer_restored = outer_resize.inverse(mask, params=saved_params[1].data)
        assert outer_restored.shape == (0, 1, 3, 4)
        inner_restored = inner_resize.inverse(outer_restored, params=inner_params[1].data)
        assert inner_restored.shape == (0, 1, 4, 5)
        restored = crop.inverse(inner_restored, params=inner_params[0].data)
        assert restored.shape == (0, 1, height, width)

        replayed = sequence.inverse(mask, params=deepcopy(saved_params), data_keys=["mask"])
        assert replayed.shape == (0, 1, height, width)
        assert replayed.dtype == mask.dtype
        assert replayed.device == mask.device
        replayed.sum().backward()
        assert mask.grad is not None
        assert mask.grad.shape == mask.shape

    @pytest.mark.parametrize("height,width", [(6, 8), (9, 13)])
    @pytest.mark.parametrize("replay_state", ["immediate", "after-forward", "fresh-instance"])
    def test_mask_only_saved_params_replay_4429(self, height, width, replay_state, device, dtype):
        if not supports_bilinear_2d_grid_sample_backward(device, dtype):
            pytest.skip("The forward resampling kernel does not support this device/dtype's backward pass.")

        def make_sequence():
            return K.AugmentationSequential(
                K.RandomCrop(
                    (4, 5),
                    padding=(1, 2, 3, 4),
                    cropping_mode="resample",
                    p=1.0,
                ),
                data_keys=["input"],
            )

        sequence = make_sequence()
        output = sequence(torch.empty(0, 3, height, width, device=device, dtype=dtype))
        saved_params = deepcopy(sequence._params)
        assert output.shape == (0, 3, 4, 5)
        assert saved_params[0].data["forward_input_shape"].tolist() == [0, 3, height, width]

        if replay_state == "after-forward":
            next_height, next_width = (9, 13) if (height, width) == (6, 8) else (6, 8)
            nonempty = torch.arange(
                2 * 3 * next_height * next_width,
                device=device,
                dtype=dtype,
            ).reshape(2, 3, next_height, next_width)
            sequence(nonempty)
        elif replay_state == "fresh-instance":
            sequence = make_sequence()

        mask = torch.empty(0, 1, *output.shape[-2:], device=device, dtype=dtype, requires_grad=True)
        restored = sequence.inverse(mask, params=saved_params, data_keys=["mask"])
        assert restored.shape == (0, 1, height, width)
        assert restored.device == device
        assert restored.dtype == dtype
        restored.sum().backward()
        assert mask.grad is not None
        assert mask.grad.shape == mask.shape

    @pytest.mark.parametrize("height,width", [(6, 8), (9, 13)])
    def test_two_stage_mask_saved_params_replay_4429(self, height, width, device, dtype):
        if not supports_bilinear_2d_grid_sample_backward(device, dtype):
            pytest.skip("The forward resampling kernel does not support this device/dtype's backward pass.")
        crop = K.RandomCrop(
            (4, 5),
            padding=(1, 2, 3, 4),
            cropping_mode="resample",
            p=1.0,
        )
        resize = K.Resize((3, 4))
        sequence = K.AugmentationSequential(crop, resize, data_keys=["input"])
        output = sequence(torch.empty(0, 3, height, width, device=device, dtype=dtype))
        saved_params = deepcopy(sequence._params)
        mask = torch.empty(0, 1, *output.shape[-2:], device=device, dtype=dtype, requires_grad=True)

        intermediate = resize.inverse(mask, params=saved_params[1].data)
        assert intermediate.shape == (0, 1, 4, 5)
        restored = crop.inverse(intermediate, params=saved_params[0].data)
        assert restored.shape == (0, 1, height, width)

        replayed = sequence.inverse(mask, params=deepcopy(saved_params), data_keys=["mask"])
        assert replayed.shape == (0, 1, height, width)
        assert replayed.device == device
        assert replayed.dtype == dtype
        replayed.sum().backward()
        assert mask.grad is not None
        assert mask.grad.shape == mask.shape

    @pytest.mark.parametrize(
        "make_sequence",
        [
            lambda: K.AugmentationSequential(
                K.RandomCrop(
                    (4, 5),
                    padding=(1, 2, 3, 4),
                    cropping_mode="resample",
                    p=1.0,
                ),
                data_keys=["input"],
            ),
            lambda: K.AugmentationSequential(K.Resize((4, 5)), data_keys=["input"]),
        ],
        ids=["random-crop", "resize"],
    )
    def test_dynamo_mask_only_saved_params_replay_4429(
        self,
        make_sequence,
        torch_optimizer,
        device,
        dtype,
    ):
        sequence = make_sequence()
        output = sequence(torch.empty(0, 3, 9, 13, device=device, dtype=dtype))
        saved_params = deepcopy(sequence._params)
        mask = torch.empty(0, 1, *output.shape[-2:], device=device, dtype=dtype)

        def replay(saved_mask):
            return sequence.inverse(saved_mask, params=saved_params, data_keys=["mask"])

        eager = replay(mask)
        compiled = torch_optimizer(replay)
        actual = compiled(mask)
        self.assert_close(actual, eager)
        assert actual.shape == (0, 1, 9, 13)
        assert actual.device == device
        assert actual.dtype == dtype

    @pytest.mark.parametrize(
        "make_aug,output_size",
        [
            (lambda p: K.Resize((4, 5), p=p), (4, 5)),
            (lambda p: K.Resize(4, side="long", p=p), (3, 4)),
            (lambda p: K.LongestMaxSize(4, p=p), (3, 4)),
            (lambda p: K.SmallestMaxSize(4, p=p), (4, 5)),
            (lambda p: K.CenterCrop((4, 5), cropping_mode="resample", p=p), (4, 5)),
            (lambda p: K.RandomResizedCrop((4, 5), cropping_mode="resample", p=p), (4, 5)),
            (lambda p: K.RandomCrop((4, 5), padding=(1, 2, 3, 4), cropping_mode="resample", p=p), (4, 5)),
        ],
        ids=["resize-tuple", "resize-int", "longest", "smallest", "center-crop", "resized-crop", "crop"],
    )
    @pytest.mark.parametrize("p", [0.0, 0.5, 1.0])
    def test_public_round_trip_4429(self, make_aug, output_size, p, device, dtype):
        aug = make_aug(p)
        if not isinstance(aug, K.Resize) and not supports_bilinear_2d_grid_sample_backward(device, dtype):
            pytest.skip("The forward resampling kernel does not support this device/dtype's backward pass.")
        image = torch.empty(0, 3, 6, 8, device=device, dtype=dtype, requires_grad=True)
        output = aug(image)
        # An empty batch takes the configured output size whatever ``p`` is (#4429); the inverse undoes it.
        assert output.shape == (0, 3, *output_size)
        restored = aug.inverse(output)
        assert restored.shape == image.shape
        assert restored.device == image.device
        assert restored.dtype == image.dtype
        restored.sum().backward()
        self.assert_close(image.grad, torch.empty_like(image))

    @pytest.mark.parametrize("batch_size", [0, 2])
    @pytest.mark.parametrize("p", [0.0, 1.0])
    @pytest.mark.parametrize("data_key", ["input", "mask"])
    @pytest.mark.parametrize(
        "make_aug",
        [
            lambda p: K.RandomCrop((4, 5), p=p, cropping_mode="slice"),
            lambda p: K.CenterCrop((4, 5), p=p, cropping_mode="slice"),
            lambda p: K.RandomResizedCrop((4, 5), p=p, cropping_mode="slice"),
        ],
        ids=["random-crop", "center-crop", "resized-crop"],
    )
    def test_slice_crop_inverse_4429(self, make_aug, data_key, p, batch_size, device, dtype):
        # Slice mode inverts like resample mode, at every batch size: an empty or unapplied batch keeps its canvas,
        # and an applied one is warped back with the inverse of the recorded transform matrix.
        aug = make_aug(p)
        channels = 3 if data_key == "input" else 1
        image = torch.rand(batch_size, channels, 6, 8, device=device, dtype=dtype)
        output = aug(image)
        method = aug.inverse_inputs if data_key == "input" else aug.inverse_masks
        restored = method(output, aug._params, aug.flags, aug.compute_inverse_transformation(aug.transform_matrix))
        assert restored.shape == image.shape
        if p == 0.0:
            self.assert_close(restored, image)
        seq = K.AugmentationSequential(make_aug(p), data_keys=["input", "mask"])
        mask = torch.rand(batch_size, 1, 6, 8, device=device, dtype=dtype)
        outputs = dict(zip(["input", "mask"], seq(torch.rand(batch_size, 3, 6, 8, device=device, dtype=dtype), mask)))
        assert seq.inverse(outputs[data_key], data_keys=[data_key]).shape[-2:] == (6, 8)

    @pytest.mark.parametrize("batch_size", [0, 2])
    @pytest.mark.parametrize(
        "make_aug",
        [
            lambda: K.CenterCrop((4, 5), cropping_mode="slice"),
            lambda: K.RandomCrop((4, 5), p=1.0, cropping_mode="slice"),
            lambda: K.RandomCrop((4, 5), padding=1, p=1.0, cropping_mode="slice"),
            lambda: K.RandomResizedCrop((4, 5), p=1.0, cropping_mode="slice"),
        ],
        ids=["center-crop", "random-crop", "random-crop-padding", "resized-crop"],
    )
    def test_slice_crop_inverse_keeps_geometry(self, make_aug, batch_size, device, dtype):
        # Boxes and keypoints invert through the transform matrix, which slice mode records too.
        seq = K.AugmentationSequential(make_aug(), data_keys=["input", "bbox_xyxy", "keypoints"])
        image = torch.rand(batch_size, 3, 6, 8, device=device, dtype=dtype)
        boxes = torch.tensor([[[1.0, 1.0, 3.0, 3.0]]], device=device, dtype=dtype).expand(batch_size, 1, 4)
        keypoints = torch.ones(batch_size, 2, 2, device=device, dtype=dtype)
        _, out_boxes, out_keypoints = seq(image, boxes, keypoints)
        restored_boxes, restored_keypoints = seq.inverse(out_boxes, out_keypoints, data_keys=["bbox_xyxy", "keypoints"])
        self.assert_close(restored_boxes, boxes)
        self.assert_close(restored_keypoints, keypoints)

    @pytest.mark.parametrize(
        "make_first,intermediate_size",
        [
            (lambda: K.LongestMaxSize(12), (9, 12)),
            (lambda: K.Resize(12, side="long"), (9, 12)),
            (lambda: K.SmallestMaxSize(12), (12, 14)),
        ],
        ids=["longest", "resize-long", "smallest"],
    )
    def test_empty_max_size_shape_history_4429(self, make_first, intermediate_size, device, dtype):
        # The container must size an integer resize on the side the module selects, as the module does.
        first = make_first()
        second = K.Resize((3, 4))
        sequence = K.AugmentationSequential(first, second, data_keys=["input"])
        image = torch.empty(0, 3, 9, 11, device=device, dtype=dtype)
        assert first(image).shape == (0, 3, *intermediate_size)
        output = sequence(image)
        assert output.shape == (0, 3, 3, 4)
        params = sequence._params
        assert params[1].data["forward_input_shape"].tolist() == [0, 3, *intermediate_size]
        mask = torch.empty(0, 1, 3, 4, device=device, dtype=dtype)
        assert second.inverse(mask, params=params[1].data).shape == (0, 1, *intermediate_size)
        assert sequence.inverse(mask, params=params, data_keys=["mask"]).shape == (0, 1, 9, 11)

    @pytest.mark.parametrize("same_on_frame", [False, True])
    def test_empty_video_resize_4429(self, same_on_frame, device, dtype):
        # VideoSequential tracks shapes with the same helper; an empty batch used to raise IndexError there.
        sequence = K.VideoSequential(
            K.RandomCrop((6, 7), cropping_mode="resample"),
            K.LongestMaxSize(10),
            data_format="BTCHW",
            same_on_frame=same_on_frame,
        )
        video = torch.empty(0, 2, 3, 9, 11, device=device, dtype=dtype)
        output = sequence(video)
        assert output.shape == (0, 2, 3, 8, 10)
        assert sequence.inverse(output).shape == video.shape


class TestEmptyCenterResizedCropHistory(BaseTester):
    @pytest.fixture(params=[K.CenterCrop, K.RandomResizedCrop], ids=["center", "resized"])
    def crop_type(self, request):
        return request.param

    @pytest.fixture(
        params=[("center", True), ("resized", False), ("resized", True)],
        ids=["center", "resized", "resized-same-batch"],
    )
    def video_crop(self, request):
        name, same_on_batch = request.param
        if name == "center":
            return K.CenterCrop((4, 5), cropping_mode="resample")
        return K.RandomResizedCrop((4, 5), cropping_mode="resample", same_on_batch=same_on_batch)

    def _assert_params_equal(self, actual, expected):
        assert len(actual) == len(expected)
        for left, right in zip(actual, expected):
            assert left.name == right.name
            assert type(left.data) is type(right.data)
            if isinstance(left.data, list):
                self._assert_params_equal(left.data, right.data)
            else:
                assert left.data.keys() == right.data.keys()
                for key in left.data:
                    self.assert_close(left.data[key], right.data[key], rtol=0, atol=0)

    @pytest.mark.parametrize("cropping_mode", ["slice", "resample"])
    @pytest.mark.parametrize("p", [0.0, 0.5, 1.0])
    @pytest.mark.parametrize("input_size,crop_size", [((6, 8), (4, 5)), ((9, 13), (3, 7))])
    def test_new_crop_parameter_shape_history_4429(
        self, crop_type, cropping_mode, p, input_size, crop_size, device, dtype
    ):
        crop = crop_type(crop_size, cropping_mode=cropping_mode, p=p)
        sequence = K.ImageSequential(crop, K.Resize((2, 3)))
        image = torch.empty(0, 3, *input_size, device=device, dtype=dtype)

        torch.manual_seed(17)
        standalone = deepcopy(crop.forward_parameters(image.shape))
        torch.manual_seed(17)
        params = sequence.forward_parameters(image.shape)
        assert params[0].data.keys() == standalone.keys()
        for key in standalone:
            self.assert_close(params[0].data[key], standalone[key], rtol=0, atol=0)
        assert params[0].data["forward_input_shape"].tolist() == [0, 3, *input_size]
        # An empty gate takes the crop's configured size whatever ``p`` is, as the image forward does (#4429).
        assert params[1].data["forward_input_shape"].tolist() == [0, 3, *crop_size]
        if crop_type is K.CenterCrop:
            assert params[0].data["output_size"].shape == (0, 2)
        else:
            assert "output_size" not in params[0].data

    @pytest.mark.parametrize("cropping_mode", ["slice", "resample"])
    def test_center_integer_shape_history_4429(self, cropping_mode, device, dtype):
        sequence = K.ImageSequential(K.CenterCrop(4, cropping_mode=cropping_mode), K.Resize((2, 3)))
        image = torch.empty(0, 3, 9, 13, device=device, dtype=dtype)
        params = sequence.forward_parameters(image.shape)
        assert params[0].data["forward_input_shape"].tolist() == [0, 3, 9, 13]
        assert params[1].data["forward_input_shape"].tolist() == [0, 3, 4, 4]

    @pytest.mark.parametrize("input_size,crop_size", [((6, 8), (4, 5)), ((9, 13), (3, 7))])
    @pytest.mark.parametrize("replay_state", ["immediate", "after-forward", "fresh-instance"])
    @pytest.mark.parametrize("channels", [1, 3], ids=["mask", "image"])
    def test_new_crop_saved_params_replay_4429(
        self, crop_type, input_size, crop_size, replay_state, channels, device, dtype
    ):
        if not supports_bilinear_2d_grid_sample_backward(device, dtype):
            pytest.skip("The resampling kernel does not support this device/dtype's backward pass.")

        def make_sequence():
            return K.AugmentationSequential(
                crop_type(crop_size, cropping_mode="resample"), K.Resize((2, 3)), data_keys=["input"]
            )

        sequence = make_sequence()
        image = torch.empty(0, 3, *input_size, device=device, dtype=dtype, requires_grad=True)
        output = sequence(image)
        params = deepcopy(sequence._params)
        snapshot = deepcopy(params)
        assert output.shape == (0, 3, 2, 3)
        assert output.device == device
        assert output.dtype == dtype
        assert params[0].data["forward_input_shape"].tolist() == [0, 3, *input_size]
        assert params[1].data["forward_input_shape"].tolist() == [0, 3, *crop_size]

        if replay_state == "after-forward":
            later = torch.arange(2 * 3 * 11 * 17, device=device, dtype=dtype).reshape(2, 3, 11, 17)
            sequence(later)
        elif replay_state == "fresh-instance":
            sequence = make_sequence()

        inverse_input = (
            output if channels == 3 else torch.empty(0, 1, 2, 3, device=device, dtype=dtype, requires_grad=True)
        )
        # Replay through the container before direct child inverses can update their saved state.
        replayed = sequence.inverse(inverse_input, params=params, data_keys=["input" if channels == 3 else "mask"])
        assert replayed.shape == (0, channels, *input_size)
        assert replayed.device == device
        assert replayed.dtype == dtype
        assert replayed.numel() == 0
        intermediate = sequence[1].inverse(inverse_input, params=params[1].data)
        assert intermediate.shape == (0, channels, *crop_size)
        restored = sequence[0].inverse(intermediate, params=params[0].data)
        assert restored.shape == (0, channels, *input_size)
        replayed.sum().backward()
        original = image if channels == 3 else inverse_input
        assert original.grad is not None
        assert original.grad.shape == original.shape
        self._assert_params_equal(params, snapshot)

    @pytest.mark.parametrize("input_size,crop_size", [((6, 8), (4, 5)), ((9, 13), (3, 7))])
    def test_new_crop_nested_shape_history_4429(self, crop_type, input_size, crop_size, device, dtype):
        if not supports_bilinear_2d_grid_sample_backward(device, dtype):
            pytest.skip("The resampling kernel does not support this device/dtype's backward pass.")
        crop = crop_type(crop_size, cropping_mode="resample")
        inner_resize = K.Resize((2, 3))
        inner = K.ImageSequential(crop, inner_resize)
        outer_resize = K.Resize((3, 4))
        sequence = K.AugmentationSequential(inner, outer_resize, data_keys=["input"])
        image = torch.empty(0, 3, *input_size, device=device, dtype=dtype)
        output = sequence(image)
        params = deepcopy(sequence._params)
        snapshot = deepcopy(params)
        inner_params = params[0].data
        assert output.shape == (0, 3, 3, 4)
        assert inner_params[0].data["forward_input_shape"].tolist() == [0, 3, *input_size]
        assert inner_params[1].data["forward_input_shape"].tolist() == [0, 3, *crop_size]
        assert params[1].data["forward_input_shape"].tolist() == [0, 3, 2, 3]

        mask = torch.empty(0, 1, 3, 4, device=device, dtype=dtype, requires_grad=True)
        outer_restored = outer_resize.inverse(mask, params=params[1].data)
        assert outer_restored.shape == (0, 1, 2, 3)
        inner_restored = inner_resize.inverse(outer_restored, params=inner_params[1].data)
        assert inner_restored.shape == (0, 1, *crop_size)
        restored = crop.inverse(inner_restored, params=inner_params[0].data)
        assert restored.shape == (0, 1, *input_size)
        replayed = sequence.inverse(mask, params=params, data_keys=["mask"])
        assert replayed.shape == (0, 1, *input_size)
        assert replayed.device == device
        assert replayed.dtype == dtype
        replayed.sum().backward()
        assert mask.grad is not None
        assert mask.grad.shape == mask.shape
        self._assert_params_equal(params, snapshot)

    @pytest.mark.parametrize("data_format", ["BTCHW", "BCTHW"])
    @pytest.mark.parametrize("same_on_frame", [False, True])
    def test_new_crop_video_parameters_4429(self, video_crop, data_format, same_on_frame, device, dtype):
        sequence = K.VideoSequential(video_crop, K.Resize((2, 3)), data_format=data_format, same_on_frame=same_on_frame)
        shape = (0, 2, 3, 9, 13) if data_format == "BTCHW" else (0, 3, 2, 9, 13)
        video = torch.empty(shape, device=device, dtype=dtype)
        params = sequence.forward_parameters(video.shape)
        # Video draws may have a nonzero leading dimension before frame/batch broadcasting.
        assert params[0].data["forward_input_shape"][-2:].tolist() == [9, 13]
        assert params[1].data["forward_input_shape"][-2:].tolist() == [4, 5]

    @pytest.mark.parametrize("data_format", ["BTCHW", "BCTHW"])
    @pytest.mark.parametrize("same_on_frame", [False, True])
    def test_new_crop_video_round_trip_4429(self, video_crop, data_format, same_on_frame, device, dtype):
        if not supports_bilinear_2d_grid_sample_backward(device, dtype):
            pytest.skip("The resampling kernel does not support this device/dtype's backward pass.")
        sequence = K.VideoSequential(video_crop, K.Resize((2, 3)), data_format=data_format, same_on_frame=same_on_frame)
        shape = (0, 2, 3, 9, 13) if data_format == "BTCHW" else (0, 3, 2, 9, 13)
        output_shape = (0, 2, 3, 2, 3) if data_format == "BTCHW" else (0, 3, 2, 2, 3)
        video = torch.empty(shape, device=device, dtype=dtype, requires_grad=True)
        params = deepcopy(sequence.forward_parameters(video.shape))
        snapshot = deepcopy(params)
        output = sequence(video, params=params)
        assert output.shape == output_shape
        assert output.device == device
        assert output.dtype == dtype
        assert params[0].data["forward_input_shape"][-2:].tolist() == [9, 13]
        assert params[1].data["forward_input_shape"][-2:].tolist() == [4, 5]
        intermediate = sequence[1].inverse(output.reshape(0, 3, 2, 3), params=params[1].data)
        assert intermediate.shape == (0, 3, 4, 5)
        assert sequence[0].inverse(intermediate, params=params[0].data).shape == (0, 3, 9, 13)
        restored = sequence.inverse(output, params=params)
        assert restored.shape == video.shape
        assert restored.device == device
        assert restored.dtype == dtype
        restored.sum().backward()
        assert video.grad is not None
        assert video.grad.shape == video.shape
        self._assert_params_equal(params, snapshot)

    def test_resized_crop_downstream_center_validation_4429(self, device, dtype):
        sequence = K.ImageSequential(K.RandomResizedCrop((4, 5)), K.CenterCrop((6, 7)))
        image = torch.empty(0, 3, 9, 13, device=device, dtype=dtype)
        with pytest.raises(AssertionError, match=r"Crop size must be smaller.*\(4, 5\).*\(6, 7\)"):
            sequence.forward_parameters(image.shape)

    @pytest.mark.parametrize("input_size,crop_size", [((6, 8), (4, 5)), ((9, 13), (3, 7))])
    def test_dynamo_new_crop_shape_history_4429(self, crop_type, input_size, crop_size, torch_optimizer, device, dtype):
        sequence = K.AugmentationSequential(
            crop_type(crop_size, cropping_mode="resample"), K.Resize((2, 3)), data_keys=["input"]
        )

        def run(image):
            params = sequence.forward_parameters(image.shape)
            output = sequence(image, params=params)
            return output, params[1].data["forward_input_shape"]

        image = torch.empty(0, 3, *input_size, device=device, dtype=dtype)
        expected_shape = [0, 3, *crop_size]
        eager_output, eager_shape = run(image)
        assert eager_output.shape == (0, 3, 2, 3)
        assert eager_shape.tolist() == expected_shape
        compiled = torch_optimizer(run)
        actual_output, actual_shape = compiled(image)
        assert actual_output.shape == (0, 3, 2, 3)
        assert actual_output.dtype == dtype
        assert actual_output.device == device
        assert actual_shape.tolist() == expected_shape
        self.assert_close(actual_output, eager_output)
        self.assert_close(actual_shape, eager_shape, rtol=0, atol=0)