Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

45.9 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import copy
import inspect
import pickle
from typing import List

import pytest
import torch

import kornia.augmentation as K
from kornia.augmentation.auto.autoaugment import AutoAugment
from kornia.augmentation.auto.operations import OperationBase, PolicySequential, ops
from kornia.augmentation.auto.rand_augment.rand_augment import RandAugment
from kornia.augmentation.auto.rand_augment.rand_augment import default_policy as randaug_config
from kornia.augmentation.auto.trivial_augment import TrivialAugment
from kornia.augmentation.auto.trivial_augment.trivial_augment import default_policy as trivial_config
from kornia.augmentation.container import AugmentationSequential
from kornia.geometry.bbox import bbox_to_mask

from testing.augmentation.utils import reproducibility_test
from testing.base import BaseTester, assert_close


def _find_all_ops() -> List[OperationBase]:
    _ops = [op for _, op in inspect.getmembers(ops, inspect.isclass)]
    return [op() for op in _ops if issubclass(op, OperationBase) and op != OperationBase]


def _test_sequential(augment_method, device, dtype):
    height, width = 50, 100
    inp = torch.rand(1, 3, height, width, device=device, dtype=dtype)
    bbox = torch.tensor([[[35.5, 1], [66, 1], [66, 25], [35.5, 25]]], device=device, dtype=dtype)
    keypoints = torch.tensor([[[46.5, 11.5], [54.5, 11.6]]], device=device, dtype=dtype)
    mask = bbox_to_mask(
        torch.tensor([[[15.5, 0], [90, 0], [90, 40], [15.5, 40]]], device=device, dtype=dtype), size=(height, width)
    )[:, None]
    aug = AugmentationSequential(augment_method, data_keys=["input", "mask", "bbox", "keypoints"])
    out = aug(inp, mask, bbox, keypoints)
    assert out[0].shape == inp.shape
    assert out[1].shape == mask.shape
    assert out[2].shape == bbox.shape
    assert out[3].shape == keypoints.shape
    assert set(out[1].unique().tolist()).issubset(set(mask.unique().tolist()))

    out_inv = aug.inverse(*out)
    assert out_inv[0].shape == inp.shape
    assert out_inv[1].shape == mask.shape
    assert out_inv[2].shape == bbox.shape
    assert out_inv[3].shape == keypoints.shape
    assert set(out_inv[1].unique().tolist()).issubset(set(mask.unique().tolist()))

    reproducibility_test((inp, mask, bbox, keypoints), aug)


class TestAutoAugment(BaseTester):
    @pytest.mark.parametrize("policy", ["imagenet", "cifar10", "svhn", [[("shear_x", 0.9, 4), ("invert", 0.2, None)]]])
    def test_smoke(self, policy, device, dtype):
        aug = AutoAugment(policy)
        in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
        aug(in_tensor)
        aug.is_intensity_only()

    def test_transform_mat(self, device, dtype):
        aug = AutoAugment([[("shear_x", 0.9, 4), ("invert", 0.2, None)]], transformation_matrix_mode="silence")
        in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
        aug(in_tensor)
        trans = aug.get_transformation_matrix(in_tensor, params=aug._params)
        self.assert_close(trans, aug.transform_matrix)

    def test_reproduce(self, device, dtype):
        aug = AutoAugment()
        in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
        out_tensor = aug(in_tensor)
        out_tensor_2 = aug(in_tensor, params=aug._params)
        self.assert_close(out_tensor, out_tensor_2)

    def test_sequential(augment_method, device, dtype):
        _test_sequential(AutoAugment(), device=device, dtype=dtype)

    @pytest.mark.parametrize("magnitude", [10, 11, -1, -2])
    def test_magnitude_bin_out_of_range_names_the_op(self, magnitude):
        """A bin indexes two adjacent points, so 9 is the last usable one.

        Over-large bins raised a raw IndexError naming an internal tensor, and negative
        ones wrapped silently onto a reversed range.
        """
        with pytest.raises(ValueError, match=r"Expect magnitude bin for `rotate` in \[0, 9\]"):
            AutoAugment(policy=[[("rotate", 1.0, magnitude)]])

    @pytest.mark.parametrize("name", ["auto_contrast", "invert", "equalize"])
    def test_ops_that_ignore_magnitude_are_not_gated(self, name):
        """These take `_: int` and never index the scale, so any bin stays legal."""
        AutoAugment(policy=[[(name, 1.0, 10)]])

    def test_magnitude_bin_endpoints_are_accepted(self):
        AutoAugment(policy=[[("rotate", 1.0, 0)]])
        AutoAugment(policy=[[("rotate", 1.0, 9)]])


class TestRandAugment(BaseTester):
    @pytest.mark.parametrize(
        ("factor", "policy"),
        [
            ("translate_x", [[("translate_x", -0.5, 0.5)]]),
            ("translate_y", [[("translate_y", -0.5, 0.5)]]),
        ],
    )
    def test_translation_magnitude_is_scaled_to_pixels(self, factor, policy):
        aug = RandAugment(n=1, m=29, policy=policy)
        batch_shape = torch.Size([4, 3, 24, 40])
        params = aug.forward_parameters(batch_shape)

        magnitude = params[0].data[0].data[factor]
        dimension = 40 if factor == "translate_x" else 24
        expected = torch.full_like(magnitude, (29 / 30) * 0.5 * dimension)

        assert torch.allclose(magnitude.abs(), expected)

    @pytest.mark.parametrize(
        ("m", "expected_bits"),
        [
            (3, 7),
            (7, 7),
            (8, 6),
            (29, 4),
        ],
    )
    def test_posterize_default_range(self, m, expected_bits):
        posterize_policy = next(policy for policy in randaug_config if policy[0][0] == "posterize")
        aug = RandAugment(n=1, m=m, policy=[posterize_policy])
        batch_shape = torch.Size([4, 3, 32, 32])
        params = aug.forward_parameters(batch_shape)

        bits = params[0].data[0].data["bits_factor"]
        expected = torch.full_like(bits, expected_bits)

        assert torch.equal(bits, expected)

    @pytest.mark.parametrize("policy", [None, [[("translate_y", -0.5, 0.5)]]])
    def test_smoke(self, policy):
        n = len(randaug_config) if policy is None else 1
        aug = RandAugment(n=n, m=15, policy=policy)
        in_tensor = torch.rand(10, 3, 50, 50, requires_grad=True)
        aug(in_tensor)

    def test_transform_mat(self, device, dtype):
        aug = RandAugment(n=3, m=15)
        in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
        aug(in_tensor)
        trans = aug.get_transformation_matrix(in_tensor, params=aug._params)
        self.assert_close(trans, aug.transform_matrix)

    def test_transform_mat_repeated_policy(self, device, dtype):
        aug = RandAugment(n=2, m=10, policy=[[("translate_y", -0.5, 0.5)], [("translate_x", -0.5, 0.5)]])
        in_tensor = torch.ones(2, 3, 10, 10, device=device, dtype=dtype)
        for _ in range(3):
            aug(in_tensor)
        expected = aug.get_transformation_matrix(in_tensor, params=aug._params, recompute=True)
        self.assert_close(aug.transform_matrix, expected)

    def test_reproduce(self, device, dtype):
        aug = RandAugment(n=3, m=15)
        in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
        out_tensor = aug(in_tensor)
        out_tensor_2 = aug(in_tensor, params=aug._params)
        self.assert_close(out_tensor, out_tensor_2)

    def test_sequential(augment_method, device, dtype):
        _test_sequential(RandAugment(n=3, m=15), device=device, dtype=dtype)

    @pytest.mark.parametrize("m", [1, 15, 29])
    def test_m_accepts_the_interval_its_message_names(self, m):
        """The guard is exclusive at both ends, and the message says so."""
        RandAugment(n=2, m=m)

    @pytest.mark.parametrize("m", [-1, 0, 30, 31])
    def test_m_outside_the_open_interval_is_rejected(self, m):
        with pytest.raises(ValueError, match=r"Expect `m` in \(0, 30\)"):
            RandAugment(n=2, m=m)

    @pytest.mark.parametrize("n", [0, -1, len(randaug_config) + 1])
    def test_n_outside_the_policy_length_is_rejected(self, n):
        """`n=0` applied nothing and `n > len(policy)` was silently clamped."""
        with pytest.raises(ValueError, match=r"Expect `n` in \[1, 15\]"):
            RandAugment(n=n, m=10)

    def test_n_is_bounded_by_the_supplied_policy(self):
        policy = [[("translate_y", -0.5, 0.5)], [("translate_x", -0.5, 0.5)]]
        RandAugment(n=2, m=10, policy=policy)
        with pytest.raises(ValueError, match=r"Expect `n` in \[1, 2\]"):
            RandAugment(n=3, m=10, policy=policy)


class TestTrivialAugment(BaseTester):
    @pytest.mark.parametrize("policy", [None, [[("translate_y", -0.5, 0.5)]]])
    def test_smoke(self, policy):
        aug = TrivialAugment(policy=policy)
        in_tensor = torch.rand(10, 3, 50, 50, requires_grad=True)
        aug(in_tensor)

    def test_transform_mat(self, device, dtype):
        aug = TrivialAugment()
        in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
        aug(in_tensor)
        aug(in_tensor)
        trans = aug.get_transformation_matrix(in_tensor, params=aug._params)
        self.assert_close(trans, aug.transform_matrix)

    def test_transform_mat_repeated_policy(self, device, dtype):
        aug = TrivialAugment(policy=[[("translate_y", -0.5, 0.5)]])
        in_tensor = torch.ones(2, 3, 10, 10, device=device, dtype=dtype)
        params_1 = aug.forward_parameters(in_tensor.shape)
        params_2 = copy.deepcopy(params_1)
        params_2[0].data[0].data["translate_y"] += 1.0
        aug(in_tensor, params=params_1)
        aug(in_tensor, params=params_2)
        stale = aug.get_transformation_matrix(in_tensor, params=params_1, recompute=True)
        expected = aug.get_transformation_matrix(in_tensor, params=params_2, recompute=True)
        assert not torch.allclose(aug.transform_matrix, stale)
        self.assert_close(aug.transform_matrix, expected)

    def test_transform_mat_fresh_instance(self):
        aug = TrivialAugment(policy=[[("translate_y", -0.5, 0.5)]])
        assert aug.transform_matrix is None

    def test_transform_mat_nested_sequential(self, device, dtype):
        aug = TrivialAugment(policy=[[("translate_y", -0.5, 0.5)]])
        seq = AugmentationSequential(aug, data_keys=["input"])
        in_tensor = torch.ones(2, 3, 10, 10, device=device, dtype=dtype)
        seq(in_tensor)
        seq(in_tensor)
        nested_params = seq._params[0].data
        expected = aug.get_transformation_matrix(in_tensor, params=nested_params, recompute=True)
        self.assert_close(aug.transform_matrix, expected)
        assert aug._params is nested_params

    def test_subpolicy_params_after_forward(self, device, dtype):
        aug = TrivialAugment(policy=[[("translate_y", -0.5, 0.5)]])
        subpolicy = next(iter(aug.children()))
        in_tensor = torch.ones(2, 3, 10, 10, device=device, dtype=dtype)
        aug(in_tensor)
        assert subpolicy._params is aug._params[0].data

    def test_reproduce(self, device, dtype):
        aug = TrivialAugment()
        in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
        out_tensor = aug(in_tensor)
        out_tensor_2 = aug(in_tensor, params=aug._params)
        self.assert_close(out_tensor, out_tensor_2)

    def test_sequential(augment_method, device, dtype):
        _test_sequential(TrivialAugment(), device=device, dtype=dtype)

    @pytest.mark.device_agnostic
    def test_default_posterize_keeps_four_to_eight_bits_6246(self):
        # #6246: the default candidate was (0, 4), so one posterize draw in 8 kept 0 bits, a black image, and one
        # in 4 kept 1 bit. It now keeps 4 to 8 bits, as RandAugment and AutoAugment do.
        candidate = next(policy for policy in trivial_config if policy[0][0] == "posterize")
        aug = TrivialAugment(policy=[candidate])
        with torch.random.fork_rng():
            torch.manual_seed(0)
            bits = torch.cat(
                [aug.forward_parameters(torch.Size([64, 3, 4, 4]))[0].data[0].data["bits_factor"] for _ in range(8)]
            )
        assert bits.min() == 4
        assert bits.max() == 8

    def test_default_posterize_never_blacks_out_an_image_6246(self, device, dtype):
        candidate = next(policy for policy in trivial_config if policy[0][0] == "posterize")
        aug = TrivialAugment(policy=[candidate])
        image = torch.linspace(0.05, 0.95, 64 * 3 * 4 * 4, device=device, dtype=dtype).view(64, 3, 4, 4)
        with torch.random.fork_rng(devices=[device] if device.type == "cuda" else []):
            torch.manual_seed(0)
            for _ in range(8):
                assert (aug(image).flatten(1).amax(1) > 0).all()


@pytest.mark.parametrize("low_dtype", [torch.float16, torch.bfloat16])
def test_operation_preserves_input_dtype(device, low_dtype):
    # OperationBase.forward gated the op output with a float32 ``batch_prob``
    # mask, so ``mask * op(x) + (1 - mask) * x`` promoted a float16 / bfloat16
    # input to float32. float32 / float64 inputs are unaffected (promotion goes
    # up), which is why this only shows below float32. Covers RandAugment /
    # AutoAugment / TrivialAugment / AugMix, which all route through this method.
    # dtype preservation is device-independent; check it on CPU.
    if torch.device(device).type != "cpu":
        pytest.skip("dtype-preservation check runs on CPU")
    checked = 0
    for op in _find_all_ops():
        x = torch.rand(2, 3, 16, 16, dtype=low_dtype)
        try:
            out = op(x)
        except RuntimeError as e:
            # some kernels (e.g. grid_sample on older torch) have no half/bfloat16
            # CPU path -- that is a torch capability gap, not a kornia contract.
            if "not implemented for" in str(e):
                continue
            raise
        assert out.dtype == low_dtype, f"{type(op).__name__}: {out.dtype} != {low_dtype}"
        checked += 1
    assert checked, f"no op could run in {low_dtype} on this build"


@pytest.mark.parametrize(
    ("op", "factor"),
    [
        (ops.Rotate, "degrees"),
        (ops.ShearX, "shear_x"),
        (ops.ShearY, "shear_y"),
        (ops.TranslateX, "translate_x"),
        (ops.TranslateY, "translate_y"),
    ],
)
def test_symmetric_magnitude_negates_rather_than_zeroing(op, factor):
    # ``symmetric_megnitude`` multiplied the magnitude by a bool mask, so half the draws
    # came out as 0 instead of -m: these five ops only ever ran in the positive direction
    # -- only counter-clockwise, only right, only down -- and were a silent no-op the rest
    # of the time. They are the only ops that set the flag, and ``RandAugment`` is the only
    # composer that samples through it. Sampling is device-independent, so this runs on CPU.
    operation = op(None, 1.0)
    torch.manual_seed(42)
    mags = torch.cat([operation.forward_parameters(torch.Size([64, 3, 8, 8]))[factor] for _ in range(8)])
    assert (mags < 0).any(), f"{factor}: no negative magnitude in {mags.numel()} draws"
    assert (mags > 0).any(), f"{factor}: no positive magnitude in {mags.numel()} draws"
    assert not (mags == 0).any(), f"{factor}: {(mags == 0).sum()} of {mags.numel()} draws were zeroed"


@pytest.mark.parametrize(
    ("op", "factor", "dim"), [(ops.TranslateX, "translate_x", -1), (ops.TranslateY, "translate_y", -2)]
)
def test_translate_set_magnitude_is_a_fraction_of_the_image_6215(op, factor, dim):
    # #6215: a set magnitude, learned or passed as ``mag``, replaced the generator's draw after the draw had been
    # converted to pixels, so ``TranslateX(initial_magnitude=0.2)`` translated by 0.2 pixels while the same 0.2
    # sampled from ``magnitude_range`` translated by a fifth of the image. A non-square shape tells the axes apart.
    shape = torch.Size([4, 3, 64, 48])
    expected = torch.full((4,), 0.2 * shape[dim])
    sampled = op(None, 1.0, magnitude_range=(0.2, 0.2)).forward_parameters(shape)[factor]
    learned = op(initial_magnitude=0.2, initial_probability=1.0).forward_parameters(shape)[factor]
    passed = op(None, 1.0).forward_parameters(shape, mag=torch.tensor(0.2))[factor]
    for params in (sampled, learned, passed):
        assert_close(params.abs(), expected)


def test_translate_learned_magnitude_gradient_6215():
    # The conversion to pixels stays differentiable: the learned fraction's gradient matches central differences.
    image = torch.rand(2, 3, 16, 24, generator=torch.Generator().manual_seed(0), dtype=torch.float64)

    def loss(magnitude: float) -> torch.Tensor:
        operation = ops.TranslateX(initial_magnitude=magnitude, initial_probability=1.0, symmetric_megnitude=False)
        return operation, (operation(image) * torch.linspace(0, 1, 24, dtype=torch.float64)).sum()

    operation, value = loss(0.2)
    value.backward()
    eps = 1e-4
    numeric = (loss(0.2 + eps)[1] - loss(0.2 - eps)[1]) / (2 * eps)
    assert_close(operation._magnitude.grad.double()[0], numeric.detach(), rtol=1e-3, atol=1e-3)


# AutoAugment sub-policies are ``(name, probability, magnitude_bin)``; the other two are
# ``(name, min_magnitude, max_magnitude)``. RandAugment and TrivialAugment allow only one
# operation per sub-policy, so a mixed draw has to come from two of them -- and cannot
# happen at all under TrivialAugment, which draws exactly one operation.
_AA_GEOMETRIC, _AA_INTENSITY = ("translate_x", 1.0, 5), ("solarize", 1.0, 5)
_GEOMETRIC, _INTENSITY = ("translate_x", -0.5, 0.5), ("solarize", 0.0, 1.0)


@pytest.mark.parametrize(
    ("intensity_only", "mixed", "geometry_only"),
    [
        (
            lambda: AutoAugment(policy=[[_AA_INTENSITY]]),
            lambda: AutoAugment(policy=[[_AA_GEOMETRIC, _AA_INTENSITY]]),
            lambda: AutoAugment(policy=[[_AA_GEOMETRIC]]),
        ),
        (
            lambda: RandAugment(n=1, m=15, policy=[[_INTENSITY]]),
            lambda: RandAugment(n=2, m=15, policy=[[_GEOMETRIC], [_INTENSITY]]),
            lambda: RandAugment(n=1, m=15, policy=[[_GEOMETRIC]]),
        ),
        (
            lambda: TrivialAugment(policy=[[_INTENSITY]]),
            None,
            lambda: TrivialAugment(policy=[[_GEOMETRIC]]),
        ),
    ],
    ids=["AutoAugment", "RandAugment", "TrivialAugment"],
)
def test_inverse_refuses_a_non_invertible_draw(intensity_only, mixed, geometry_only, device, dtype):
    # ``inverse`` can only undo geometric operations. The sub-policy is redrawn on every
    # forward pass, so a draw holding an intensity operation used to come back through
    # ``inverse`` with that operation still applied -- and on a draw with no geometry at
    # all, as the input unchanged, with no error and no warning. It refuses now, the way
    # ``MixAugmentationBaseV2.inverse`` already does.
    x = torch.rand(2, 3, 8, 6, device=device, dtype=dtype)

    aug = intensity_only()
    with pytest.raises(RuntimeError, match="is not supported"):
        aug.inverse(aug(x))

    if mixed is not None:
        # half-undoing a mixed draw is the same trap, so it is refused too
        aug = mixed()
        with pytest.raises(RuntimeError, match="is not supported"):
            aug.inverse(aug(x))

    # a geometry-only draw still inverts. A ramp is what bilinear resampling reproduces
    # exactly, so replaying the same draw backwards has to return it -- everywhere except
    # the pixels the draw pushed out of frame, which the same round trip over ``ones``
    # marks for us. Half precision cannot hold that tolerance (``1 - 1e-4`` even rounds to
    # ``1.0`` there), so the value check runs in float32 or wider; the refusals above keep
    # the fixture dtype.
    aug = geometry_only()
    value_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
    ramp = 0.5 * (
        torch.linspace(0, 1, 8, device=device, dtype=value_dtype)[:, None]
        + torch.linspace(0, 1, 6, device=device, dtype=value_dtype)
    ).expand(2, 3, 8, 6)
    params = aug.forward_parameters(ramp.shape)
    kept = aug.inverse(aug(torch.ones_like(ramp), params=params), params=params) > 1 - 1e-4
    assert kept.any(), "the drawn geometry pushed the whole image out of frame"
    round_trip = aug.inverse(aug(ramp, params=params), params=params)
    assert_close(round_trip[kept], ramp[kept], rtol=1e-4, atol=1e-4)


def test_inverse_ignores_an_operation_its_gate_skipped(device, dtype):
    # an intensity operation whose probability gate skipped every sample left the input
    # untouched, so the draw is pure geometry and inverts; applied on any row, it refuses
    x = torch.rand(2, 3, 8, 6, device=device, dtype=dtype)
    aug = AutoAugment(policy=[[("translate_x", 1.0, 5), ("solarize", 0.0, 5)]])
    assert aug.inverse(aug(x)).shape == x.shape

    aug = AutoAugment(policy=[[("translate_x", 1.0, 5), ("solarize", 1.0, 5)]])
    with pytest.raises(RuntimeError, match="applied RandomSolarize"):
        aug.inverse(aug(x))


def test_inverse_without_a_forward_pass_still_reports_missing_params(device, dtype):
    # the refusal must not shadow the pre-existing error for an un-run policy
    x = torch.rand(2, 3, 8, 6, device=device, dtype=dtype)
    with pytest.raises(ValueError, match="No parameters available"):
        RandAugment(n=1, m=5).inverse(x)


def test_operation_base_deepcopy_after_use() -> None:
    operation = ops.Brightness()
    state_dict = operation.state_dict()

    assert isinstance(operation._probability, torch.nn.Parameter)
    assert isinstance(operation.probability, torch.Tensor)
    assert operation.probability.item() == operation.op.p
    assert hasattr(operation, "temperature")
    assert "_probability" in state_dict
    assert "temperature" in state_dict

    restored = ops.Brightness()
    restored.load_state_dict(state_dict, strict=True)

    assert restored.probability.item() == operation.probability.item()
    assert torch.equal(restored.temperature, operation.temperature)

    x = torch.rand(2, 3, 32, 32)

    cases = [
        (RandAugment(n=2, m=15), "forward"),
        (AutoAugment(), "train"),
        (AutoAugment(), "eval"),
        (TrivialAugment(), "train"),
        (AugmentationSequential(RandAugment(n=2, m=15)), "forward"),
    ]

    for aug, action in cases:
        if action == "forward":
            aug(x)
        elif action == "train":
            aug.train()
        else:
            aug.eval()

        copy.deepcopy(aug)


# Generated with Pillow 12.3.0:
# from PIL import Image, ImageEnhance
# image = Image.new("RGB", (2, 2))
# image.putdata([(32, 96, 160), (224, 64, 128), (80, 192, 48), (144, 112, 208)])
# for enhancer in (ImageEnhance.Brightness, ImageEnhance.Contrast, ImageEnhance.Color):
#     for factor in (0.3, 1.9):
#         print(list(enhancer(image).enhance(factor).getdata()))
# Source: PIL.ImageEnhance, the reference enhancer formulas used by the policies.
@pytest.mark.parametrize(
    "operation,factor,pixels",
    [
        (ops.Brightness, 0.3, [(9, 28, 48), (67, 19, 38), (24, 57, 14), (43, 33, 62)]),
        (ops.Brightness, 1.9, [(60, 182, 255), (255, 121, 243), (152, 255, 91), (255, 212, 255)]),
        (ops.Contrast, 0.3, [(93, 112, 132), (151, 103, 122), (108, 141, 98), (127, 117, 146)]),
        (ops.Contrast, 1.9, [(0, 74, 196), (255, 13, 135), (44, 255, 0), (165, 104, 255)]),
        (ops.Saturate, 0.3, [(68, 87, 106), (150, 102, 121), (123, 157, 113), (136, 126, 155)]),
        (ops.Saturate, 1.9, [(0, 106, 228), (255, 14, 136), (24, 237, 0), (153, 93, 255)]),
    ],
)
def test_auto_enhancer_matches_pil_6253(operation, factor, pixels, device, dtype):
    """Match reference pixels and deterministic replay on the selected device."""
    image = (
        torch.tensor(
            [(32, 96, 160), (224, 64, 128), (80, 192, 48), (144, 112, 208)],
            device=device,
            dtype=dtype,
        )
        .reshape(2, 2, 3)
        .permute(2, 0, 1)
        .unsqueeze(0)
        / 255
    )
    expected = torch.tensor(pixels, device=device, dtype=dtype).reshape(2, 2, 3).permute(2, 0, 1).unsqueeze(0) / 255
    # Keep learnable magnitudes with the default CPU parameter generator, as
    # policy containers do; the enhancement transfers factors to the input device.
    aug = operation(initial_magnitude=factor, initial_probability=1.0, magnitude_range=(0.1, 1.9)).to(dtype=dtype)
    params = aug.forward_parameters(image.shape)
    output = aug(image, params=params)
    # PIL quantizes intermediate grayscale and the final blend to uint8;
    # Kornia keeps float intermediates. Compare at the same 8-bit precision.
    assert_close((output * 255).round(), (expected * 255).round(), rtol=0, atol=1 + 255 * torch.finfo(dtype).eps)
    assert_close(aug(image, params=params), output, rtol=0, atol=0)


def test_auto_contrast_blends_towards_each_images_own_mean_6253(device, dtype):
    """Blend each image towards its own grayscale mean, as PIL does, not towards the batch mean."""
    # A constant image is its own grayscale mean, so PIL's contrast leaves it unchanged at any factor.
    image = torch.stack([torch.full((3, 4, 5), 0.2), torch.full((3, 4, 5), 0.8)]).to(device=device, dtype=dtype)
    aug = ops.Contrast(initial_magnitude=0.5, initial_probability=1.0).to(dtype=dtype)
    assert_close(aug(image), image)


@pytest.mark.parametrize("operation", [ops.Brightness, ops.Contrast, ops.Saturate])
def test_auto_enhancer_keeps_magnitude_gradient_6253(operation, device, dtype):
    """Keep finite nonzero magnitude gradients for each enhancement operation."""
    image = torch.tensor(
        [[[[0.2, 0.4]], [[0.5, 0.7]], [[0.3, 0.6]]]],
        device=device,
        dtype=dtype,
    )
    aug = operation(initial_magnitude=0.7, initial_probability=1.0).to(dtype=dtype)
    assert aug._magnitude is not None
    (gradient,) = torch.autograd.grad(aug(image).square().sum(), aug._magnitude)
    assert torch.isfinite(gradient).all()
    assert gradient.abs().sum() > 0
    assert aug.op.p == 1.0
    assert aug.op.p_batch == 1.0


@pytest.mark.parametrize("operation", [ops.Brightness, ops.Contrast, ops.Saturate])
def test_auto_enhancer_keeps_the_identity_gradient_at_exact_0_and_1_6296(operation, device, dtype):
    """Return the input at the neutral magnitude with an identity gradient on black and white pixels too."""
    row = torch.tensor([0.0, 0.25, 0.5, 1.0], device=device, dtype=dtype).view(1, 1, 1, 4)
    image = row.expand(1, 3, 4, 4).contiguous().requires_grad_()
    aug = operation(initial_magnitude=1.0, initial_probability=1.0).to(dtype=dtype)
    out = aug(image)
    (gradient,) = torch.autograd.grad(out.sum(), image)
    assert_close(out, image)
    assert_close(gradient, torch.ones_like(image))


@pytest.mark.parametrize("policy", ["auto", "rand", "trivial"])
def test_policy_brightness_uses_multiplicative_factor_6253(policy, device, dtype):
    """Use multiplicative brightness through all three policy containers."""
    image = torch.full((2, 3, 4, 5), 0.4, device=device, dtype=dtype)
    if policy == "auto":
        aug = AutoAugment(policy=[[("brightness", 1.0, 5)]])
    elif policy == "rand":
        aug = RandAugment(n=1, m=12, policy=[[("brightness", 0.1, 1.9)]])
    else:
        aug = TrivialAugment(policy=[[("brightness", 0.1, 1.9)]])
    aug = aug.to(device=device, dtype=dtype)
    params = aug.forward_parameters(image.shape)
    factor = params[0].data[0].data["brightness_factor"].to(image)
    expected = (image * factor[:, None, None, None]).clamp(0, 1)
    assert_close(aug(image, params=params), expected)


# Every wrapper with a magnitude. ``Posterize`` truncates it with ``long`` and ``Solarize`` compares against it, so
# their output carries no gradient for it on any device.
_MAGNITUDE_OPS = [
    op
    for _, op in inspect.getmembers(ops, inspect.isclass)
    if issubclass(op, OperationBase)
    and op is not OperationBase
    and "initial_magnitude" in inspect.signature(op).parameters
]
_NOT_DIFFERENTIABLE_IN_MAGNITUDE = (ops.Posterize, ops.Solarize)
_DIFFERENTIABLE_OPS = [op for op in _MAGNITUDE_OPS if op not in _NOT_DIFFERENTIABLE_IN_MAGNITUDE]
_MAGNITUDE_OP_IDS = [op.__name__ for op in _MAGNITUDE_OPS]
# One wrapper per place a generator keeps its range: a buffer, an attribute ``make_samplers`` rebuilds, a raw tuple.
_RANGE_KINDS = [ops.Brightness, ops.ShearX, ops.Posterize, ops.TranslateX]


def _round_trip_pickle(module):
    return pickle.loads(pickle.dumps(module))  # noqa: S301


def _assert_magnitude_gradient(output, operation):
    (gradient,) = torch.autograd.grad(output.square().sum(), operation._magnitude)
    assert torch.isfinite(gradient).all(), type(operation).__name__
    assert gradient.abs().sum() > 0, type(operation).__name__


class TestOperationAfterTo6297(BaseTester):
    """``.to()`` moves a wrapper's magnitude, range and generator together.

    The range was a plain attribute holding the tensor from before the move, and the generators hand their draws back
    on the CPU after one. A run on the CPU alone cannot fail most of these, so they need ``--device`` of an accelerator.
    """

    @pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
    def test_forward_parameters_puts_the_factor_with_the_magnitude(self, operation, device, dtype):
        moved = operation(initial_probability=1.0).to(device=device, dtype=dtype)
        params = moved.forward_parameters(torch.Size([2, 3, 16, 16]))
        assert params[moved._factor_name].device == moved._magnitude.device == device
        assert params[moved._factor_name].shape == (2,)

    @pytest.mark.parametrize("probability", [1.0, 0.5])
    @pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
    def test_forward_and_replay(self, operation, probability, device, dtype):
        moved = operation(initial_probability=probability).to(device=device, dtype=dtype)
        image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
        assert moved(image).device == device
        params = moved.forward_parameters(image.shape)
        self.assert_close(moved(image, params=params), moved(image, params=params))

    @pytest.mark.parametrize("operation", _DIFFERENTIABLE_OPS, ids=[op.__name__ for op in _DIFFERENTIABLE_OPS])
    def test_magnitude_keeps_its_gradient(self, operation, device, dtype):
        moved = operation(initial_probability=1.0).to(device=device, dtype=dtype)
        image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
        _assert_magnitude_gradient(moved(image), moved)
        params = moved.forward_parameters(image.shape)
        _assert_magnitude_gradient(moved(image, params=params), moved)

    @pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
    def test_magnitude_stays_clamped_to_the_range(self, operation, device, dtype):
        moved = operation(initial_probability=1.0).to(device=device, dtype=dtype)
        low, high = moved.magnitude_range
        with torch.no_grad():
            moved._magnitude.fill_(float(high) + 1.0)
        self.assert_close(moved.magnitude, torch.full((1,), float(high), device=device, dtype=dtype))
        with torch.no_grad():
            moved._magnitude.fill_(float(low) - 1.0)
        self.assert_close(moved.magnitude, torch.full((1,), float(low), device=device, dtype=dtype))

    def test_magnitude_on_the_device_with_the_operation_elsewhere(self, device, dtype):
        # ``mag`` overrides the draw of a wrapper left where it was built, so the factor follows ``mag``.
        operation = ops.Brightness(None, 1.0)
        image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
        mag = torch.tensor(0.5, device=device, dtype=dtype, requires_grad=True)
        params = operation.forward_parameters(image.shape, mag=mag)
        assert params["brightness_factor"].device == device
        (gradient,) = torch.autograd.grad(operation(image, params=params).sum(), mag)
        assert gradient.abs() > 0

    @pytest.mark.parametrize("operation", _RANGE_KINDS, ids=[op.__name__ for op in _RANGE_KINDS])
    def test_generator_moved_on_its_own(self, operation, device, dtype):
        # ``set_rng_device_and_dtype`` moves the draws and, for ``ShearGenerator``, the range they come from, but not
        # the magnitude of the wrapper. The clamp has to work across that.
        wrapper = operation(initial_probability=1.0)
        wrapper.op.set_rng_device_and_dtype(device, dtype)
        assert wrapper.magnitude.device == wrapper._magnitude.device
        wrapper.forward_parameters(torch.Size([2, 3, 16, 16]))

    def test_policy_sequential_keeps_every_magnitude_gradient(self, device, dtype):
        # ``Posterize`` runs first: its output has a gradient of zero, which would cut the graph of what runs before it.
        learnable = (ops.Brightness, ops.ShearX, ops.TranslateY, ops.Rotate)
        policy = PolicySequential(
            ops.Posterize(initial_probability=1.0), *[op(initial_probability=1.0) for op in learnable]
        ).to(device=device, dtype=dtype)
        image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
        params = policy.forward_parameters(image.shape)
        assert len(params) == len(learnable) + 1
        self.assert_close(policy(image, params=params), policy(image, params=params))
        output = policy(image)
        assert output.device == device
        magnitudes = [operation._magnitude for operation in list(policy.children())[1:]]
        gradients = torch.autograd.grad(output.square().sum(), magnitudes)
        for gradient in gradients:
            assert torch.isfinite(gradient).all()
            assert gradient.abs().sum() > 0

    @pytest.mark.parametrize(
        "make_policy",
        [
            pytest.param(lambda: RandAugment(n=len(randaug_config), m=15), id="RandAugment"),
            pytest.param(TrivialAugment, id="TrivialAugment"),
            pytest.param(AutoAugment, id="AutoAugment"),
        ],
    )
    def test_builtin_policy(self, make_policy, device, dtype):
        moved = make_policy().to(device=device, dtype=dtype)
        image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
        output = moved(image)
        assert output.device == device
        self.assert_close(moved(image, params=moved._params), output)
        params = moved.forward_parameters(image.shape)
        self.assert_close(moved(image, params=params), moved(image, params=params))

    @pytest.mark.parametrize(
        "make_policy",
        [
            pytest.param(lambda: RandAugment(n=3, m=15), id="RandAugment"),
            pytest.param(TrivialAugment, id="TrivialAugment"),
            pytest.param(AutoAugment, id="AutoAugment"),
        ],
    )
    def test_builtin_policy_in_a_container(self, make_policy, device, dtype):
        _test_sequential(make_policy().to(device=device, dtype=dtype), device=device, dtype=dtype)

    def test_transformation_matrix_of_a_policy_with_magnitudes(self, device, dtype):
        policy = PolicySequential(
            ops.TranslateX(initial_probability=1.0),
            ops.ShearY(initial_probability=1.0),
            ops.Rotate(initial_probability=1.0),
        ).to(device=device, dtype=dtype)
        image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
        params = policy.forward_parameters(image.shape)
        policy(image, params=params)
        recomputed = policy.get_transformation_matrix(image, params=params, recompute=True)
        self.assert_close(policy.transform_matrix, recomputed)

    @pytest.mark.parametrize("candidate", randaug_config, ids=[subpolicy[0][0] for subpolicy in randaug_config])
    def test_rand_augment_magnitude_matches_the_unmoved_policy(self, candidate, device, dtype):
        # A draw selects ``n`` candidates, so go through each. The range of a moved policy is on the device: reading it
        # has to neither mix devices nor change the magnitude ``m`` selects from it.
        moved = RandAugment(n=1, m=10, policy=[candidate]).to(device=device, dtype=dtype)
        image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
        params = moved.forward_parameters(image.shape)
        self.assert_close(moved(image, params=params), moved(image, params=params))
        factor = moved[0][0]._factor_name
        if factor is None:
            return
        unmoved = RandAugment(n=1, m=10, policy=[candidate]).forward_parameters(image.shape)
        self.assert_close(
            params[0].data[0].data[factor].abs().to(device=device, dtype=dtype),
            unmoved[0].data[0].data[factor].abs().to(device=device, dtype=dtype),
        )

    @pytest.mark.parametrize("copier", [copy.deepcopy, _round_trip_pickle], ids=["deepcopy", "pickle"])
    @pytest.mark.parametrize("operation", _RANGE_KINDS, ids=[op.__name__ for op in _RANGE_KINDS])
    def test_copy_of_a_moved_operation_owns_its_magnitude(self, operation, copier, device, dtype):
        moved = operation(initial_probability=1.0).to(device=device, dtype=dtype)
        image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
        moved(image)
        clone = copier(moved)
        assert clone._magnitude is not moved._magnitude
        assert clone._magnitude.device == device
        assert clone.magnitude_range is getattr(clone.op._param_generator, clone._factor_name)
        output = clone(image)
        if operation in _NOT_DIFFERENTIABLE_IN_MAGNITUDE:
            return
        _assert_magnitude_gradient(output, clone)
        assert moved._magnitude.grad is None

    def test_state_dict_round_trip_after_to(self, device, dtype):
        source = ops.Brightness(initial_probability=1.0, magnitude_range=(0.4, 0.6)).to(device=device, dtype=dtype)
        loaded = ops.Brightness(initial_probability=1.0, magnitude_range=(0.2, 1.8)).to(device=device, dtype=dtype)
        loaded.load_state_dict(source.state_dict(), strict=True)
        self.assert_close(loaded.magnitude_range, source.magnitude_range)
        self.assert_close(loaded.magnitude, source.magnitude)
        loaded(torch.rand(2, 3, 16, 16, device=device, dtype=dtype))

    @pytest.mark.device_agnostic
    @pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
    def test_range_follows_a_dtype_move(self, operation):
        # What the CPU can see of the stale range: ``.to()`` replaces the tensors of the generator, on any device.
        moved = operation(initial_probability=1.0).to(torch.float64)
        assert moved.magnitude_range is getattr(moved.op._param_generator, moved._factor_name)
        assert moved.magnitude.dtype == torch.float64

    @pytest.mark.device_agnostic
    @pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
    def test_factor_follows_a_magnitude_on_another_device(self, operation):
        # What the CPU can see of the draw's placement: a magnitude on ``meta`` meets a draw on the CPU.
        wrapper = operation(None, 1.0)
        params = wrapper.forward_parameters(torch.Size([2, 3, 8, 8]), mag=torch.zeros(1, device="meta"))
        factor = params[wrapper._factor_name]
        assert factor.device.type == "meta"
        assert factor.shape == (2,)

    @pytest.mark.device_agnostic
    @pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
    def test_a_number_still_replaces_the_draw(self, operation):
        # The annotation says tensor, but a plain number has always worked, as it has no device to follow.
        wrapper = operation(None, 1.0, symmetric_megnitude=False)
        factor = wrapper.forward_parameters(torch.Size([2, 3, 8, 8]), mag=0.25)[wrapper._factor_name]
        assert factor.shape == (2,)

    @pytest.mark.device_agnostic
    def test_factor_name_the_generator_lacks_raises_at_construction(self):
        with pytest.raises(AttributeError, match="nonsense"):
            OperationBase(K.RandomRotation(30.0), initial_magnitude=[("nonsense", 1.0)])

    @pytest.mark.device_agnostic
    def test_state_dict_keys_stay_the_same(self):
        # Checkpoints embed these names, so following a range that is not a parameter must not add one. A parameter
        # range did have a key of its own, which the tests below cover.
        common = {"_magnitude", "_probability", "temperature"}
        assert set(ops.Brightness().state_dict()) == common | {"op._param_generator.brightness_factor"}
        assert set(ops.ShearX().state_dict()) == common
        assert set(ops.Posterize().state_dict()) == common
        assert set(ops.TranslateX().state_dict()) == common

    @pytest.mark.device_agnostic
    def test_deepcopy_with_a_learnable_shear_range(self):
        # ``ShearGenerator`` derives the range in ``make_samplers``, so with a parameter it holds autograd history. A
        # copy of the generator rebuilds it (#6541), and the wrapper must not keep a reference of its own.
        wrapper = OperationBase(
            K.RandomShear(torch.nn.Parameter(torch.tensor([0.0, 0.3])), p=1.0, align_corners=True),
            initial_magnitude=[("shear_x", 0.1)],
        )
        clone = copy.deepcopy(wrapper)
        assert clone.magnitude_range is clone.op._param_generator.shear_x
        assert clone.magnitude_range is not wrapper.magnitude_range


def _wrapper_with_learnable_range(low=0.0, high=30.0):
    return OperationBase(
        K.RandomRotation(torch.nn.Parameter(torch.tensor([low, high])), p=1.0), initial_magnitude=[("degrees", 5.0)]
    )


# The ``state_dict`` of ``_wrapper_with_learnable_range`` before #6297. ``magnitude_range`` was an attribute, so
# ``nn.Module`` registered a parameter range on the wrapper too, and checkpoints hold that alias beside the generator's.
_OLD_LEARNABLE_RANGE_KEYS = (
    "magnitude_range",
    "_magnitude",
    "_probability",
    "temperature",
    "op._param_generator.degrees",
)


def _old_learnable_range_state_dict(prefix=""):
    return {
        f"{prefix}magnitude_range": torch.tensor([0.0, 20.0]),
        f"{prefix}_magnitude": torch.tensor([7.0]),
        f"{prefix}_probability": torch.tensor([0.5]),
        f"{prefix}temperature": torch.tensor([0.1]),
        f"{prefix}op._param_generator.degrees": torch.tensor([0.0, 20.0]),
    }


class TestLearnableMagnitudeRangeCheckpoint6297(BaseTester):
    def test_old_state_dict_loads_strictly(self, device, dtype):
        old = _old_learnable_range_state_dict()
        assert tuple(old) == _OLD_LEARNABLE_RANGE_KEYS
        wrapper = _wrapper_with_learnable_range().to(device=device, dtype=dtype)
        wrapper.load_state_dict(old, strict=True)
        expected = torch.tensor([0.0, 20.0], device=device, dtype=dtype)
        self.assert_close(wrapper.magnitude_range, expected)
        self.assert_close(wrapper.op._param_generator.degrees, expected)
        self.assert_close(wrapper._magnitude, torch.tensor([7.0], device=device, dtype=dtype))
        assert "magnitude_range" in old, "the caller's dict stays as it was"

    def test_old_state_dict_loads_strictly_inside_a_policy(self, device, dtype):
        policy = PolicySequential(_wrapper_with_learnable_range()).to(device=device, dtype=dtype)
        name = next(iter(policy._modules))
        policy.load_state_dict(_old_learnable_range_state_dict(f"{name}."), strict=True)
        self.assert_close(policy[0].magnitude_range, torch.tensor([0.0, 20.0], device=device, dtype=dtype))

    @pytest.mark.device_agnostic
    def test_only_the_old_key_is_forgiven(self):
        wrapper = _wrapper_with_learnable_range()
        unexpected = {**_old_learnable_range_state_dict(), "nonsense": torch.zeros(1)}
        with pytest.raises(RuntimeError, match=r'Unexpected key.*"nonsense"') as error:
            wrapper.load_state_dict(unexpected, strict=True)
        assert "magnitude_range" not in str(error.value)
        missing = _old_learnable_range_state_dict()
        del missing["op._param_generator.degrees"]
        with pytest.raises(RuntimeError, match=r"Missing key.*op\._param_generator\.degrees"):
            wrapper.load_state_dict(missing, strict=True)

    def test_state_dict_round_trip(self, device, dtype):
        source = _wrapper_with_learnable_range(0.0, 20.0).to(device=device, dtype=dtype)
        state = source.state_dict()
        assert set(state) == {"_magnitude", "_probability", "temperature", "op._param_generator.degrees"}
        loaded = _wrapper_with_learnable_range().to(device=device, dtype=dtype)
        loaded.load_state_dict(state, strict=True)
        self.assert_close(loaded.magnitude_range, source.magnitude_range)

    def test_optimizer_on_the_moved_wrapper_holds_each_parameter_once(self, device, dtype):
        wrapper = _wrapper_with_learnable_range().to(device=device, dtype=dtype)
        optimizer = torch.optim.SGD(wrapper.parameters(), lr=0.1)
        held = [parameter for group in optimizer.param_groups for parameter in group["params"]]
        assert {id(parameter) for parameter in held} == {
            id(wrapper._magnitude),
            id(wrapper._probability),
            id(wrapper.magnitude_range),
        }
        assert len(held) == 3
        assert all(parameter.device == device for parameter in held)
        wrapper(torch.rand(2, 3, 16, 16, device=device, dtype=dtype)).square().sum().backward()
        before = wrapper._magnitude.detach().clone()
        optimizer.step()
        assert not torch.equal(wrapper._magnitude, before)

    @pytest.mark.device_agnostic
    @pytest.mark.parametrize("learnable", [True, False], ids=["parameter", "buffer"])
    def test_wrapper_pickled_with_the_old_attribute(self, learnable):
        # A whole module saved before #6297 brings the attribute back with it: in ``_parameters`` for a parameter range,
        # where it would demand the old key from every later load, and otherwise as a stale tensor in ``__dict__``.
        wrapper = _wrapper_with_learnable_range() if learnable else ops.Brightness()
        generator_range = wrapper.magnitude_range
        state = dict(wrapper.__dict__)
        if learnable:
            state["_parameters"] = {"magnitude_range": generator_range, **state["_parameters"]}
        else:
            state["magnitude_range"] = generator_range.clone()
        restored = type(wrapper).__new__(type(wrapper))
        restored.__setstate__(state)
        assert "magnitude_range" not in restored.__dict__
        assert restored.magnitude_range is generator_range
        assert "magnitude_range" not in restored.state_dict()
        assert "magnitude_range" not in dict(restored.named_parameters())
        restored.load_state_dict(wrapper.state_dict(), strict=True)