π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import copy
import inspect
import pickle
from typing import List
import pytest
import torch
import kornia.augmentation as K
from kornia.augmentation.auto.autoaugment import AutoAugment
from kornia.augmentation.auto.operations import OperationBase, PolicySequential, ops
from kornia.augmentation.auto.rand_augment.rand_augment import RandAugment
from kornia.augmentation.auto.rand_augment.rand_augment import default_policy as randaug_config
from kornia.augmentation.auto.trivial_augment import TrivialAugment
from kornia.augmentation.auto.trivial_augment.trivial_augment import default_policy as trivial_config
from kornia.augmentation.container import AugmentationSequential
from kornia.geometry.bbox import bbox_to_mask
from testing.augmentation.utils import reproducibility_test
from testing.base import BaseTester, assert_close
def _find_all_ops() -> List[OperationBase]:
_ops = [op for _, op in inspect.getmembers(ops, inspect.isclass)]
return [op() for op in _ops if issubclass(op, OperationBase) and op != OperationBase]
def _test_sequential(augment_method, device, dtype):
height, width = 50, 100
inp = torch.rand(1, 3, height, width, device=device, dtype=dtype)
bbox = torch.tensor([[[35.5, 1], [66, 1], [66, 25], [35.5, 25]]], device=device, dtype=dtype)
keypoints = torch.tensor([[[46.5, 11.5], [54.5, 11.6]]], device=device, dtype=dtype)
mask = bbox_to_mask(
torch.tensor([[[15.5, 0], [90, 0], [90, 40], [15.5, 40]]], device=device, dtype=dtype), size=(height, width)
)[:, None]
aug = AugmentationSequential(augment_method, data_keys=["input", "mask", "bbox", "keypoints"])
out = aug(inp, mask, bbox, keypoints)
assert out[0].shape == inp.shape
assert out[1].shape == mask.shape
assert out[2].shape == bbox.shape
assert out[3].shape == keypoints.shape
assert set(out[1].unique().tolist()).issubset(set(mask.unique().tolist()))
out_inv = aug.inverse(*out)
assert out_inv[0].shape == inp.shape
assert out_inv[1].shape == mask.shape
assert out_inv[2].shape == bbox.shape
assert out_inv[3].shape == keypoints.shape
assert set(out_inv[1].unique().tolist()).issubset(set(mask.unique().tolist()))
reproducibility_test((inp, mask, bbox, keypoints), aug)
class TestAutoAugment(BaseTester):
@pytest.mark.parametrize("policy", ["imagenet", "cifar10", "svhn", [[("shear_x", 0.9, 4), ("invert", 0.2, None)]]])
def test_smoke(self, policy, device, dtype):
aug = AutoAugment(policy)
in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
aug(in_tensor)
aug.is_intensity_only()
def test_transform_mat(self, device, dtype):
aug = AutoAugment([[("shear_x", 0.9, 4), ("invert", 0.2, None)]], transformation_matrix_mode="silence")
in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
aug(in_tensor)
trans = aug.get_transformation_matrix(in_tensor, params=aug._params)
self.assert_close(trans, aug.transform_matrix)
def test_reproduce(self, device, dtype):
aug = AutoAugment()
in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
out_tensor = aug(in_tensor)
out_tensor_2 = aug(in_tensor, params=aug._params)
self.assert_close(out_tensor, out_tensor_2)
def test_sequential(augment_method, device, dtype):
_test_sequential(AutoAugment(), device=device, dtype=dtype)
@pytest.mark.parametrize("magnitude", [10, 11, -1, -2])
def test_magnitude_bin_out_of_range_names_the_op(self, magnitude):
"""A bin indexes two adjacent points, so 9 is the last usable one.
Over-large bins raised a raw IndexError naming an internal tensor, and negative
ones wrapped silently onto a reversed range.
"""
with pytest.raises(ValueError, match=r"Expect magnitude bin for `rotate` in \[0, 9\]"):
AutoAugment(policy=[[("rotate", 1.0, magnitude)]])
@pytest.mark.parametrize("name", ["auto_contrast", "invert", "equalize"])
def test_ops_that_ignore_magnitude_are_not_gated(self, name):
"""These take `_: int` and never index the scale, so any bin stays legal."""
AutoAugment(policy=[[(name, 1.0, 10)]])
def test_magnitude_bin_endpoints_are_accepted(self):
AutoAugment(policy=[[("rotate", 1.0, 0)]])
AutoAugment(policy=[[("rotate", 1.0, 9)]])
class TestRandAugment(BaseTester):
@pytest.mark.parametrize(
("factor", "policy"),
[
("translate_x", [[("translate_x", -0.5, 0.5)]]),
("translate_y", [[("translate_y", -0.5, 0.5)]]),
],
)
def test_translation_magnitude_is_scaled_to_pixels(self, factor, policy):
aug = RandAugment(n=1, m=29, policy=policy)
batch_shape = torch.Size([4, 3, 24, 40])
params = aug.forward_parameters(batch_shape)
magnitude = params[0].data[0].data[factor]
dimension = 40 if factor == "translate_x" else 24
expected = torch.full_like(magnitude, (29 / 30) * 0.5 * dimension)
assert torch.allclose(magnitude.abs(), expected)
@pytest.mark.parametrize(
("m", "expected_bits"),
[
(3, 7),
(7, 7),
(8, 6),
(29, 4),
],
)
def test_posterize_default_range(self, m, expected_bits):
posterize_policy = next(policy for policy in randaug_config if policy[0][0] == "posterize")
aug = RandAugment(n=1, m=m, policy=[posterize_policy])
batch_shape = torch.Size([4, 3, 32, 32])
params = aug.forward_parameters(batch_shape)
bits = params[0].data[0].data["bits_factor"]
expected = torch.full_like(bits, expected_bits)
assert torch.equal(bits, expected)
@pytest.mark.parametrize("policy", [None, [[("translate_y", -0.5, 0.5)]]])
def test_smoke(self, policy):
n = len(randaug_config) if policy is None else 1
aug = RandAugment(n=n, m=15, policy=policy)
in_tensor = torch.rand(10, 3, 50, 50, requires_grad=True)
aug(in_tensor)
def test_transform_mat(self, device, dtype):
aug = RandAugment(n=3, m=15)
in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
aug(in_tensor)
trans = aug.get_transformation_matrix(in_tensor, params=aug._params)
self.assert_close(trans, aug.transform_matrix)
def test_transform_mat_repeated_policy(self, device, dtype):
aug = RandAugment(n=2, m=10, policy=[[("translate_y", -0.5, 0.5)], [("translate_x", -0.5, 0.5)]])
in_tensor = torch.ones(2, 3, 10, 10, device=device, dtype=dtype)
for _ in range(3):
aug(in_tensor)
expected = aug.get_transformation_matrix(in_tensor, params=aug._params, recompute=True)
self.assert_close(aug.transform_matrix, expected)
def test_reproduce(self, device, dtype):
aug = RandAugment(n=3, m=15)
in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
out_tensor = aug(in_tensor)
out_tensor_2 = aug(in_tensor, params=aug._params)
self.assert_close(out_tensor, out_tensor_2)
def test_sequential(augment_method, device, dtype):
_test_sequential(RandAugment(n=3, m=15), device=device, dtype=dtype)
@pytest.mark.parametrize("m", [1, 15, 29])
def test_m_accepts_the_interval_its_message_names(self, m):
"""The guard is exclusive at both ends, and the message says so."""
RandAugment(n=2, m=m)
@pytest.mark.parametrize("m", [-1, 0, 30, 31])
def test_m_outside_the_open_interval_is_rejected(self, m):
with pytest.raises(ValueError, match=r"Expect `m` in \(0, 30\)"):
RandAugment(n=2, m=m)
@pytest.mark.parametrize("n", [0, -1, len(randaug_config) + 1])
def test_n_outside_the_policy_length_is_rejected(self, n):
"""`n=0` applied nothing and `n > len(policy)` was silently clamped."""
with pytest.raises(ValueError, match=r"Expect `n` in \[1, 15\]"):
RandAugment(n=n, m=10)
def test_n_is_bounded_by_the_supplied_policy(self):
policy = [[("translate_y", -0.5, 0.5)], [("translate_x", -0.5, 0.5)]]
RandAugment(n=2, m=10, policy=policy)
with pytest.raises(ValueError, match=r"Expect `n` in \[1, 2\]"):
RandAugment(n=3, m=10, policy=policy)
class TestTrivialAugment(BaseTester):
@pytest.mark.parametrize("policy", [None, [[("translate_y", -0.5, 0.5)]]])
def test_smoke(self, policy):
aug = TrivialAugment(policy=policy)
in_tensor = torch.rand(10, 3, 50, 50, requires_grad=True)
aug(in_tensor)
def test_transform_mat(self, device, dtype):
aug = TrivialAugment()
in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
aug(in_tensor)
aug(in_tensor)
trans = aug.get_transformation_matrix(in_tensor, params=aug._params)
self.assert_close(trans, aug.transform_matrix)
def test_transform_mat_repeated_policy(self, device, dtype):
aug = TrivialAugment(policy=[[("translate_y", -0.5, 0.5)]])
in_tensor = torch.ones(2, 3, 10, 10, device=device, dtype=dtype)
params_1 = aug.forward_parameters(in_tensor.shape)
params_2 = copy.deepcopy(params_1)
params_2[0].data[0].data["translate_y"] += 1.0
aug(in_tensor, params=params_1)
aug(in_tensor, params=params_2)
stale = aug.get_transformation_matrix(in_tensor, params=params_1, recompute=True)
expected = aug.get_transformation_matrix(in_tensor, params=params_2, recompute=True)
assert not torch.allclose(aug.transform_matrix, stale)
self.assert_close(aug.transform_matrix, expected)
def test_transform_mat_fresh_instance(self):
aug = TrivialAugment(policy=[[("translate_y", -0.5, 0.5)]])
assert aug.transform_matrix is None
def test_transform_mat_nested_sequential(self, device, dtype):
aug = TrivialAugment(policy=[[("translate_y", -0.5, 0.5)]])
seq = AugmentationSequential(aug, data_keys=["input"])
in_tensor = torch.ones(2, 3, 10, 10, device=device, dtype=dtype)
seq(in_tensor)
seq(in_tensor)
nested_params = seq._params[0].data
expected = aug.get_transformation_matrix(in_tensor, params=nested_params, recompute=True)
self.assert_close(aug.transform_matrix, expected)
assert aug._params is nested_params
def test_subpolicy_params_after_forward(self, device, dtype):
aug = TrivialAugment(policy=[[("translate_y", -0.5, 0.5)]])
subpolicy = next(iter(aug.children()))
in_tensor = torch.ones(2, 3, 10, 10, device=device, dtype=dtype)
aug(in_tensor)
assert subpolicy._params is aug._params[0].data
def test_reproduce(self, device, dtype):
aug = TrivialAugment()
in_tensor = torch.rand(10, 3, 50, 50, device=device, dtype=dtype, requires_grad=True)
out_tensor = aug(in_tensor)
out_tensor_2 = aug(in_tensor, params=aug._params)
self.assert_close(out_tensor, out_tensor_2)
def test_sequential(augment_method, device, dtype):
_test_sequential(TrivialAugment(), device=device, dtype=dtype)
@pytest.mark.device_agnostic
def test_default_posterize_keeps_four_to_eight_bits_6246(self):
# #6246: the default candidate was (0, 4), so one posterize draw in 8 kept 0 bits, a black image, and one
# in 4 kept 1 bit. It now keeps 4 to 8 bits, as RandAugment and AutoAugment do.
candidate = next(policy for policy in trivial_config if policy[0][0] == "posterize")
aug = TrivialAugment(policy=[candidate])
with torch.random.fork_rng():
torch.manual_seed(0)
bits = torch.cat(
[aug.forward_parameters(torch.Size([64, 3, 4, 4]))[0].data[0].data["bits_factor"] for _ in range(8)]
)
assert bits.min() == 4
assert bits.max() == 8
def test_default_posterize_never_blacks_out_an_image_6246(self, device, dtype):
candidate = next(policy for policy in trivial_config if policy[0][0] == "posterize")
aug = TrivialAugment(policy=[candidate])
image = torch.linspace(0.05, 0.95, 64 * 3 * 4 * 4, device=device, dtype=dtype).view(64, 3, 4, 4)
with torch.random.fork_rng(devices=[device] if device.type == "cuda" else []):
torch.manual_seed(0)
for _ in range(8):
assert (aug(image).flatten(1).amax(1) > 0).all()
@pytest.mark.parametrize("low_dtype", [torch.float16, torch.bfloat16])
def test_operation_preserves_input_dtype(device, low_dtype):
# OperationBase.forward gated the op output with a float32 ``batch_prob``
# mask, so ``mask * op(x) + (1 - mask) * x`` promoted a float16 / bfloat16
# input to float32. float32 / float64 inputs are unaffected (promotion goes
# up), which is why this only shows below float32. Covers RandAugment /
# AutoAugment / TrivialAugment / AugMix, which all route through this method.
# dtype preservation is device-independent; check it on CPU.
if torch.device(device).type != "cpu":
pytest.skip("dtype-preservation check runs on CPU")
checked = 0
for op in _find_all_ops():
x = torch.rand(2, 3, 16, 16, dtype=low_dtype)
try:
out = op(x)
except RuntimeError as e:
# some kernels (e.g. grid_sample on older torch) have no half/bfloat16
# CPU path -- that is a torch capability gap, not a kornia contract.
if "not implemented for" in str(e):
continue
raise
assert out.dtype == low_dtype, f"{type(op).__name__}: {out.dtype} != {low_dtype}"
checked += 1
assert checked, f"no op could run in {low_dtype} on this build"
@pytest.mark.parametrize(
("op", "factor"),
[
(ops.Rotate, "degrees"),
(ops.ShearX, "shear_x"),
(ops.ShearY, "shear_y"),
(ops.TranslateX, "translate_x"),
(ops.TranslateY, "translate_y"),
],
)
def test_symmetric_magnitude_negates_rather_than_zeroing(op, factor):
# ``symmetric_megnitude`` multiplied the magnitude by a bool mask, so half the draws
# came out as 0 instead of -m: these five ops only ever ran in the positive direction
# -- only counter-clockwise, only right, only down -- and were a silent no-op the rest
# of the time. They are the only ops that set the flag, and ``RandAugment`` is the only
# composer that samples through it. Sampling is device-independent, so this runs on CPU.
operation = op(None, 1.0)
torch.manual_seed(42)
mags = torch.cat([operation.forward_parameters(torch.Size([64, 3, 8, 8]))[factor] for _ in range(8)])
assert (mags < 0).any(), f"{factor}: no negative magnitude in {mags.numel()} draws"
assert (mags > 0).any(), f"{factor}: no positive magnitude in {mags.numel()} draws"
assert not (mags == 0).any(), f"{factor}: {(mags == 0).sum()} of {mags.numel()} draws were zeroed"
@pytest.mark.parametrize(
("op", "factor", "dim"), [(ops.TranslateX, "translate_x", -1), (ops.TranslateY, "translate_y", -2)]
)
def test_translate_set_magnitude_is_a_fraction_of_the_image_6215(op, factor, dim):
# #6215: a set magnitude, learned or passed as ``mag``, replaced the generator's draw after the draw had been
# converted to pixels, so ``TranslateX(initial_magnitude=0.2)`` translated by 0.2 pixels while the same 0.2
# sampled from ``magnitude_range`` translated by a fifth of the image. A non-square shape tells the axes apart.
shape = torch.Size([4, 3, 64, 48])
expected = torch.full((4,), 0.2 * shape[dim])
sampled = op(None, 1.0, magnitude_range=(0.2, 0.2)).forward_parameters(shape)[factor]
learned = op(initial_magnitude=0.2, initial_probability=1.0).forward_parameters(shape)[factor]
passed = op(None, 1.0).forward_parameters(shape, mag=torch.tensor(0.2))[factor]
for params in (sampled, learned, passed):
assert_close(params.abs(), expected)
def test_translate_learned_magnitude_gradient_6215():
# The conversion to pixels stays differentiable: the learned fraction's gradient matches central differences.
image = torch.rand(2, 3, 16, 24, generator=torch.Generator().manual_seed(0), dtype=torch.float64)
def loss(magnitude: float) -> torch.Tensor:
operation = ops.TranslateX(initial_magnitude=magnitude, initial_probability=1.0, symmetric_megnitude=False)
return operation, (operation(image) * torch.linspace(0, 1, 24, dtype=torch.float64)).sum()
operation, value = loss(0.2)
value.backward()
eps = 1e-4
numeric = (loss(0.2 + eps)[1] - loss(0.2 - eps)[1]) / (2 * eps)
assert_close(operation._magnitude.grad.double()[0], numeric.detach(), rtol=1e-3, atol=1e-3)
# AutoAugment sub-policies are ``(name, probability, magnitude_bin)``; the other two are
# ``(name, min_magnitude, max_magnitude)``. RandAugment and TrivialAugment allow only one
# operation per sub-policy, so a mixed draw has to come from two of them -- and cannot
# happen at all under TrivialAugment, which draws exactly one operation.
_AA_GEOMETRIC, _AA_INTENSITY = ("translate_x", 1.0, 5), ("solarize", 1.0, 5)
_GEOMETRIC, _INTENSITY = ("translate_x", -0.5, 0.5), ("solarize", 0.0, 1.0)
@pytest.mark.parametrize(
("intensity_only", "mixed", "geometry_only"),
[
(
lambda: AutoAugment(policy=[[_AA_INTENSITY]]),
lambda: AutoAugment(policy=[[_AA_GEOMETRIC, _AA_INTENSITY]]),
lambda: AutoAugment(policy=[[_AA_GEOMETRIC]]),
),
(
lambda: RandAugment(n=1, m=15, policy=[[_INTENSITY]]),
lambda: RandAugment(n=2, m=15, policy=[[_GEOMETRIC], [_INTENSITY]]),
lambda: RandAugment(n=1, m=15, policy=[[_GEOMETRIC]]),
),
(
lambda: TrivialAugment(policy=[[_INTENSITY]]),
None,
lambda: TrivialAugment(policy=[[_GEOMETRIC]]),
),
],
ids=["AutoAugment", "RandAugment", "TrivialAugment"],
)
def test_inverse_refuses_a_non_invertible_draw(intensity_only, mixed, geometry_only, device, dtype):
# ``inverse`` can only undo geometric operations. The sub-policy is redrawn on every
# forward pass, so a draw holding an intensity operation used to come back through
# ``inverse`` with that operation still applied -- and on a draw with no geometry at
# all, as the input unchanged, with no error and no warning. It refuses now, the way
# ``MixAugmentationBaseV2.inverse`` already does.
x = torch.rand(2, 3, 8, 6, device=device, dtype=dtype)
aug = intensity_only()
with pytest.raises(RuntimeError, match="is not supported"):
aug.inverse(aug(x))
if mixed is not None:
# half-undoing a mixed draw is the same trap, so it is refused too
aug = mixed()
with pytest.raises(RuntimeError, match="is not supported"):
aug.inverse(aug(x))
# a geometry-only draw still inverts. A ramp is what bilinear resampling reproduces
# exactly, so replaying the same draw backwards has to return it -- everywhere except
# the pixels the draw pushed out of frame, which the same round trip over ``ones``
# marks for us. Half precision cannot hold that tolerance (``1 - 1e-4`` even rounds to
# ``1.0`` there), so the value check runs in float32 or wider; the refusals above keep
# the fixture dtype.
aug = geometry_only()
value_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
ramp = 0.5 * (
torch.linspace(0, 1, 8, device=device, dtype=value_dtype)[:, None]
+ torch.linspace(0, 1, 6, device=device, dtype=value_dtype)
).expand(2, 3, 8, 6)
params = aug.forward_parameters(ramp.shape)
kept = aug.inverse(aug(torch.ones_like(ramp), params=params), params=params) > 1 - 1e-4
assert kept.any(), "the drawn geometry pushed the whole image out of frame"
round_trip = aug.inverse(aug(ramp, params=params), params=params)
assert_close(round_trip[kept], ramp[kept], rtol=1e-4, atol=1e-4)
def test_inverse_ignores_an_operation_its_gate_skipped(device, dtype):
# an intensity operation whose probability gate skipped every sample left the input
# untouched, so the draw is pure geometry and inverts; applied on any row, it refuses
x = torch.rand(2, 3, 8, 6, device=device, dtype=dtype)
aug = AutoAugment(policy=[[("translate_x", 1.0, 5), ("solarize", 0.0, 5)]])
assert aug.inverse(aug(x)).shape == x.shape
aug = AutoAugment(policy=[[("translate_x", 1.0, 5), ("solarize", 1.0, 5)]])
with pytest.raises(RuntimeError, match="applied RandomSolarize"):
aug.inverse(aug(x))
def test_inverse_without_a_forward_pass_still_reports_missing_params(device, dtype):
# the refusal must not shadow the pre-existing error for an un-run policy
x = torch.rand(2, 3, 8, 6, device=device, dtype=dtype)
with pytest.raises(ValueError, match="No parameters available"):
RandAugment(n=1, m=5).inverse(x)
def test_operation_base_deepcopy_after_use() -> None:
operation = ops.Brightness()
state_dict = operation.state_dict()
assert isinstance(operation._probability, torch.nn.Parameter)
assert isinstance(operation.probability, torch.Tensor)
assert operation.probability.item() == operation.op.p
assert hasattr(operation, "temperature")
assert "_probability" in state_dict
assert "temperature" in state_dict
restored = ops.Brightness()
restored.load_state_dict(state_dict, strict=True)
assert restored.probability.item() == operation.probability.item()
assert torch.equal(restored.temperature, operation.temperature)
x = torch.rand(2, 3, 32, 32)
cases = [
(RandAugment(n=2, m=15), "forward"),
(AutoAugment(), "train"),
(AutoAugment(), "eval"),
(TrivialAugment(), "train"),
(AugmentationSequential(RandAugment(n=2, m=15)), "forward"),
]
for aug, action in cases:
if action == "forward":
aug(x)
elif action == "train":
aug.train()
else:
aug.eval()
copy.deepcopy(aug)
# Generated with Pillow 12.3.0:
# from PIL import Image, ImageEnhance
# image = Image.new("RGB", (2, 2))
# image.putdata([(32, 96, 160), (224, 64, 128), (80, 192, 48), (144, 112, 208)])
# for enhancer in (ImageEnhance.Brightness, ImageEnhance.Contrast, ImageEnhance.Color):
# for factor in (0.3, 1.9):
# print(list(enhancer(image).enhance(factor).getdata()))
# Source: PIL.ImageEnhance, the reference enhancer formulas used by the policies.
@pytest.mark.parametrize(
"operation,factor,pixels",
[
(ops.Brightness, 0.3, [(9, 28, 48), (67, 19, 38), (24, 57, 14), (43, 33, 62)]),
(ops.Brightness, 1.9, [(60, 182, 255), (255, 121, 243), (152, 255, 91), (255, 212, 255)]),
(ops.Contrast, 0.3, [(93, 112, 132), (151, 103, 122), (108, 141, 98), (127, 117, 146)]),
(ops.Contrast, 1.9, [(0, 74, 196), (255, 13, 135), (44, 255, 0), (165, 104, 255)]),
(ops.Saturate, 0.3, [(68, 87, 106), (150, 102, 121), (123, 157, 113), (136, 126, 155)]),
(ops.Saturate, 1.9, [(0, 106, 228), (255, 14, 136), (24, 237, 0), (153, 93, 255)]),
],
)
def test_auto_enhancer_matches_pil_6253(operation, factor, pixels, device, dtype):
"""Match reference pixels and deterministic replay on the selected device."""
image = (
torch.tensor(
[(32, 96, 160), (224, 64, 128), (80, 192, 48), (144, 112, 208)],
device=device,
dtype=dtype,
)
.reshape(2, 2, 3)
.permute(2, 0, 1)
.unsqueeze(0)
/ 255
)
expected = torch.tensor(pixels, device=device, dtype=dtype).reshape(2, 2, 3).permute(2, 0, 1).unsqueeze(0) / 255
# Keep learnable magnitudes with the default CPU parameter generator, as
# policy containers do; the enhancement transfers factors to the input device.
aug = operation(initial_magnitude=factor, initial_probability=1.0, magnitude_range=(0.1, 1.9)).to(dtype=dtype)
params = aug.forward_parameters(image.shape)
output = aug(image, params=params)
# PIL quantizes intermediate grayscale and the final blend to uint8;
# Kornia keeps float intermediates. Compare at the same 8-bit precision.
assert_close((output * 255).round(), (expected * 255).round(), rtol=0, atol=1 + 255 * torch.finfo(dtype).eps)
assert_close(aug(image, params=params), output, rtol=0, atol=0)
def test_auto_contrast_blends_towards_each_images_own_mean_6253(device, dtype):
"""Blend each image towards its own grayscale mean, as PIL does, not towards the batch mean."""
# A constant image is its own grayscale mean, so PIL's contrast leaves it unchanged at any factor.
image = torch.stack([torch.full((3, 4, 5), 0.2), torch.full((3, 4, 5), 0.8)]).to(device=device, dtype=dtype)
aug = ops.Contrast(initial_magnitude=0.5, initial_probability=1.0).to(dtype=dtype)
assert_close(aug(image), image)
@pytest.mark.parametrize("operation", [ops.Brightness, ops.Contrast, ops.Saturate])
def test_auto_enhancer_keeps_magnitude_gradient_6253(operation, device, dtype):
"""Keep finite nonzero magnitude gradients for each enhancement operation."""
image = torch.tensor(
[[[[0.2, 0.4]], [[0.5, 0.7]], [[0.3, 0.6]]]],
device=device,
dtype=dtype,
)
aug = operation(initial_magnitude=0.7, initial_probability=1.0).to(dtype=dtype)
assert aug._magnitude is not None
(gradient,) = torch.autograd.grad(aug(image).square().sum(), aug._magnitude)
assert torch.isfinite(gradient).all()
assert gradient.abs().sum() > 0
assert aug.op.p == 1.0
assert aug.op.p_batch == 1.0
@pytest.mark.parametrize("operation", [ops.Brightness, ops.Contrast, ops.Saturate])
def test_auto_enhancer_keeps_the_identity_gradient_at_exact_0_and_1_6296(operation, device, dtype):
"""Return the input at the neutral magnitude with an identity gradient on black and white pixels too."""
row = torch.tensor([0.0, 0.25, 0.5, 1.0], device=device, dtype=dtype).view(1, 1, 1, 4)
image = row.expand(1, 3, 4, 4).contiguous().requires_grad_()
aug = operation(initial_magnitude=1.0, initial_probability=1.0).to(dtype=dtype)
out = aug(image)
(gradient,) = torch.autograd.grad(out.sum(), image)
assert_close(out, image)
assert_close(gradient, torch.ones_like(image))
@pytest.mark.parametrize("policy", ["auto", "rand", "trivial"])
def test_policy_brightness_uses_multiplicative_factor_6253(policy, device, dtype):
"""Use multiplicative brightness through all three policy containers."""
image = torch.full((2, 3, 4, 5), 0.4, device=device, dtype=dtype)
if policy == "auto":
aug = AutoAugment(policy=[[("brightness", 1.0, 5)]])
elif policy == "rand":
aug = RandAugment(n=1, m=12, policy=[[("brightness", 0.1, 1.9)]])
else:
aug = TrivialAugment(policy=[[("brightness", 0.1, 1.9)]])
aug = aug.to(device=device, dtype=dtype)
params = aug.forward_parameters(image.shape)
factor = params[0].data[0].data["brightness_factor"].to(image)
expected = (image * factor[:, None, None, None]).clamp(0, 1)
assert_close(aug(image, params=params), expected)
# Every wrapper with a magnitude. ``Posterize`` truncates it with ``long`` and ``Solarize`` compares against it, so
# their output carries no gradient for it on any device.
_MAGNITUDE_OPS = [
op
for _, op in inspect.getmembers(ops, inspect.isclass)
if issubclass(op, OperationBase)
and op is not OperationBase
and "initial_magnitude" in inspect.signature(op).parameters
]
_NOT_DIFFERENTIABLE_IN_MAGNITUDE = (ops.Posterize, ops.Solarize)
_DIFFERENTIABLE_OPS = [op for op in _MAGNITUDE_OPS if op not in _NOT_DIFFERENTIABLE_IN_MAGNITUDE]
_MAGNITUDE_OP_IDS = [op.__name__ for op in _MAGNITUDE_OPS]
# One wrapper per place a generator keeps its range: a buffer, an attribute ``make_samplers`` rebuilds, a raw tuple.
_RANGE_KINDS = [ops.Brightness, ops.ShearX, ops.Posterize, ops.TranslateX]
def _round_trip_pickle(module):
return pickle.loads(pickle.dumps(module)) # noqa: S301
def _assert_magnitude_gradient(output, operation):
(gradient,) = torch.autograd.grad(output.square().sum(), operation._magnitude)
assert torch.isfinite(gradient).all(), type(operation).__name__
assert gradient.abs().sum() > 0, type(operation).__name__
class TestOperationAfterTo6297(BaseTester):
"""``.to()`` moves a wrapper's magnitude, range and generator together.
The range was a plain attribute holding the tensor from before the move, and the generators hand their draws back
on the CPU after one. A run on the CPU alone cannot fail most of these, so they need ``--device`` of an accelerator.
"""
@pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
def test_forward_parameters_puts_the_factor_with_the_magnitude(self, operation, device, dtype):
moved = operation(initial_probability=1.0).to(device=device, dtype=dtype)
params = moved.forward_parameters(torch.Size([2, 3, 16, 16]))
assert params[moved._factor_name].device == moved._magnitude.device == device
assert params[moved._factor_name].shape == (2,)
@pytest.mark.parametrize("probability", [1.0, 0.5])
@pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
def test_forward_and_replay(self, operation, probability, device, dtype):
moved = operation(initial_probability=probability).to(device=device, dtype=dtype)
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
assert moved(image).device == device
params = moved.forward_parameters(image.shape)
self.assert_close(moved(image, params=params), moved(image, params=params))
@pytest.mark.parametrize("operation", _DIFFERENTIABLE_OPS, ids=[op.__name__ for op in _DIFFERENTIABLE_OPS])
def test_magnitude_keeps_its_gradient(self, operation, device, dtype):
moved = operation(initial_probability=1.0).to(device=device, dtype=dtype)
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
_assert_magnitude_gradient(moved(image), moved)
params = moved.forward_parameters(image.shape)
_assert_magnitude_gradient(moved(image, params=params), moved)
@pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
def test_magnitude_stays_clamped_to_the_range(self, operation, device, dtype):
moved = operation(initial_probability=1.0).to(device=device, dtype=dtype)
low, high = moved.magnitude_range
with torch.no_grad():
moved._magnitude.fill_(float(high) + 1.0)
self.assert_close(moved.magnitude, torch.full((1,), float(high), device=device, dtype=dtype))
with torch.no_grad():
moved._magnitude.fill_(float(low) - 1.0)
self.assert_close(moved.magnitude, torch.full((1,), float(low), device=device, dtype=dtype))
def test_magnitude_on_the_device_with_the_operation_elsewhere(self, device, dtype):
# ``mag`` overrides the draw of a wrapper left where it was built, so the factor follows ``mag``.
operation = ops.Brightness(None, 1.0)
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
mag = torch.tensor(0.5, device=device, dtype=dtype, requires_grad=True)
params = operation.forward_parameters(image.shape, mag=mag)
assert params["brightness_factor"].device == device
(gradient,) = torch.autograd.grad(operation(image, params=params).sum(), mag)
assert gradient.abs() > 0
@pytest.mark.parametrize("operation", _RANGE_KINDS, ids=[op.__name__ for op in _RANGE_KINDS])
def test_generator_moved_on_its_own(self, operation, device, dtype):
# ``set_rng_device_and_dtype`` moves the draws and, for ``ShearGenerator``, the range they come from, but not
# the magnitude of the wrapper. The clamp has to work across that.
wrapper = operation(initial_probability=1.0)
wrapper.op.set_rng_device_and_dtype(device, dtype)
assert wrapper.magnitude.device == wrapper._magnitude.device
wrapper.forward_parameters(torch.Size([2, 3, 16, 16]))
def test_policy_sequential_keeps_every_magnitude_gradient(self, device, dtype):
# ``Posterize`` runs first: its output has a gradient of zero, which would cut the graph of what runs before it.
learnable = (ops.Brightness, ops.ShearX, ops.TranslateY, ops.Rotate)
policy = PolicySequential(
ops.Posterize(initial_probability=1.0), *[op(initial_probability=1.0) for op in learnable]
).to(device=device, dtype=dtype)
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
params = policy.forward_parameters(image.shape)
assert len(params) == len(learnable) + 1
self.assert_close(policy(image, params=params), policy(image, params=params))
output = policy(image)
assert output.device == device
magnitudes = [operation._magnitude for operation in list(policy.children())[1:]]
gradients = torch.autograd.grad(output.square().sum(), magnitudes)
for gradient in gradients:
assert torch.isfinite(gradient).all()
assert gradient.abs().sum() > 0
@pytest.mark.parametrize(
"make_policy",
[
pytest.param(lambda: RandAugment(n=len(randaug_config), m=15), id="RandAugment"),
pytest.param(TrivialAugment, id="TrivialAugment"),
pytest.param(AutoAugment, id="AutoAugment"),
],
)
def test_builtin_policy(self, make_policy, device, dtype):
moved = make_policy().to(device=device, dtype=dtype)
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
output = moved(image)
assert output.device == device
self.assert_close(moved(image, params=moved._params), output)
params = moved.forward_parameters(image.shape)
self.assert_close(moved(image, params=params), moved(image, params=params))
@pytest.mark.parametrize(
"make_policy",
[
pytest.param(lambda: RandAugment(n=3, m=15), id="RandAugment"),
pytest.param(TrivialAugment, id="TrivialAugment"),
pytest.param(AutoAugment, id="AutoAugment"),
],
)
def test_builtin_policy_in_a_container(self, make_policy, device, dtype):
_test_sequential(make_policy().to(device=device, dtype=dtype), device=device, dtype=dtype)
def test_transformation_matrix_of_a_policy_with_magnitudes(self, device, dtype):
policy = PolicySequential(
ops.TranslateX(initial_probability=1.0),
ops.ShearY(initial_probability=1.0),
ops.Rotate(initial_probability=1.0),
).to(device=device, dtype=dtype)
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
params = policy.forward_parameters(image.shape)
policy(image, params=params)
recomputed = policy.get_transformation_matrix(image, params=params, recompute=True)
self.assert_close(policy.transform_matrix, recomputed)
@pytest.mark.parametrize("candidate", randaug_config, ids=[subpolicy[0][0] for subpolicy in randaug_config])
def test_rand_augment_magnitude_matches_the_unmoved_policy(self, candidate, device, dtype):
# A draw selects ``n`` candidates, so go through each. The range of a moved policy is on the device: reading it
# has to neither mix devices nor change the magnitude ``m`` selects from it.
moved = RandAugment(n=1, m=10, policy=[candidate]).to(device=device, dtype=dtype)
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
params = moved.forward_parameters(image.shape)
self.assert_close(moved(image, params=params), moved(image, params=params))
factor = moved[0][0]._factor_name
if factor is None:
return
unmoved = RandAugment(n=1, m=10, policy=[candidate]).forward_parameters(image.shape)
self.assert_close(
params[0].data[0].data[factor].abs().to(device=device, dtype=dtype),
unmoved[0].data[0].data[factor].abs().to(device=device, dtype=dtype),
)
@pytest.mark.parametrize("copier", [copy.deepcopy, _round_trip_pickle], ids=["deepcopy", "pickle"])
@pytest.mark.parametrize("operation", _RANGE_KINDS, ids=[op.__name__ for op in _RANGE_KINDS])
def test_copy_of_a_moved_operation_owns_its_magnitude(self, operation, copier, device, dtype):
moved = operation(initial_probability=1.0).to(device=device, dtype=dtype)
image = torch.rand(2, 3, 16, 16, device=device, dtype=dtype)
moved(image)
clone = copier(moved)
assert clone._magnitude is not moved._magnitude
assert clone._magnitude.device == device
assert clone.magnitude_range is getattr(clone.op._param_generator, clone._factor_name)
output = clone(image)
if operation in _NOT_DIFFERENTIABLE_IN_MAGNITUDE:
return
_assert_magnitude_gradient(output, clone)
assert moved._magnitude.grad is None
def test_state_dict_round_trip_after_to(self, device, dtype):
source = ops.Brightness(initial_probability=1.0, magnitude_range=(0.4, 0.6)).to(device=device, dtype=dtype)
loaded = ops.Brightness(initial_probability=1.0, magnitude_range=(0.2, 1.8)).to(device=device, dtype=dtype)
loaded.load_state_dict(source.state_dict(), strict=True)
self.assert_close(loaded.magnitude_range, source.magnitude_range)
self.assert_close(loaded.magnitude, source.magnitude)
loaded(torch.rand(2, 3, 16, 16, device=device, dtype=dtype))
@pytest.mark.device_agnostic
@pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
def test_range_follows_a_dtype_move(self, operation):
# What the CPU can see of the stale range: ``.to()`` replaces the tensors of the generator, on any device.
moved = operation(initial_probability=1.0).to(torch.float64)
assert moved.magnitude_range is getattr(moved.op._param_generator, moved._factor_name)
assert moved.magnitude.dtype == torch.float64
@pytest.mark.device_agnostic
@pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
def test_factor_follows_a_magnitude_on_another_device(self, operation):
# What the CPU can see of the draw's placement: a magnitude on ``meta`` meets a draw on the CPU.
wrapper = operation(None, 1.0)
params = wrapper.forward_parameters(torch.Size([2, 3, 8, 8]), mag=torch.zeros(1, device="meta"))
factor = params[wrapper._factor_name]
assert factor.device.type == "meta"
assert factor.shape == (2,)
@pytest.mark.device_agnostic
@pytest.mark.parametrize("operation", _MAGNITUDE_OPS, ids=_MAGNITUDE_OP_IDS)
def test_a_number_still_replaces_the_draw(self, operation):
# The annotation says tensor, but a plain number has always worked, as it has no device to follow.
wrapper = operation(None, 1.0, symmetric_megnitude=False)
factor = wrapper.forward_parameters(torch.Size([2, 3, 8, 8]), mag=0.25)[wrapper._factor_name]
assert factor.shape == (2,)
@pytest.mark.device_agnostic
def test_factor_name_the_generator_lacks_raises_at_construction(self):
with pytest.raises(AttributeError, match="nonsense"):
OperationBase(K.RandomRotation(30.0), initial_magnitude=[("nonsense", 1.0)])
@pytest.mark.device_agnostic
def test_state_dict_keys_stay_the_same(self):
# Checkpoints embed these names, so following a range that is not a parameter must not add one. A parameter
# range did have a key of its own, which the tests below cover.
common = {"_magnitude", "_probability", "temperature"}
assert set(ops.Brightness().state_dict()) == common | {"op._param_generator.brightness_factor"}
assert set(ops.ShearX().state_dict()) == common
assert set(ops.Posterize().state_dict()) == common
assert set(ops.TranslateX().state_dict()) == common
@pytest.mark.device_agnostic
def test_deepcopy_with_a_learnable_shear_range(self):
# ``ShearGenerator`` derives the range in ``make_samplers``, so with a parameter it holds autograd history. A
# copy of the generator rebuilds it (#6541), and the wrapper must not keep a reference of its own.
wrapper = OperationBase(
K.RandomShear(torch.nn.Parameter(torch.tensor([0.0, 0.3])), p=1.0, align_corners=True),
initial_magnitude=[("shear_x", 0.1)],
)
clone = copy.deepcopy(wrapper)
assert clone.magnitude_range is clone.op._param_generator.shear_x
assert clone.magnitude_range is not wrapper.magnitude_range
def _wrapper_with_learnable_range(low=0.0, high=30.0):
return OperationBase(
K.RandomRotation(torch.nn.Parameter(torch.tensor([low, high])), p=1.0), initial_magnitude=[("degrees", 5.0)]
)
# The ``state_dict`` of ``_wrapper_with_learnable_range`` before #6297. ``magnitude_range`` was an attribute, so
# ``nn.Module`` registered a parameter range on the wrapper too, and checkpoints hold that alias beside the generator's.
_OLD_LEARNABLE_RANGE_KEYS = (
"magnitude_range",
"_magnitude",
"_probability",
"temperature",
"op._param_generator.degrees",
)
def _old_learnable_range_state_dict(prefix=""):
return {
f"{prefix}magnitude_range": torch.tensor([0.0, 20.0]),
f"{prefix}_magnitude": torch.tensor([7.0]),
f"{prefix}_probability": torch.tensor([0.5]),
f"{prefix}temperature": torch.tensor([0.1]),
f"{prefix}op._param_generator.degrees": torch.tensor([0.0, 20.0]),
}
class TestLearnableMagnitudeRangeCheckpoint6297(BaseTester):
def test_old_state_dict_loads_strictly(self, device, dtype):
old = _old_learnable_range_state_dict()
assert tuple(old) == _OLD_LEARNABLE_RANGE_KEYS
wrapper = _wrapper_with_learnable_range().to(device=device, dtype=dtype)
wrapper.load_state_dict(old, strict=True)
expected = torch.tensor([0.0, 20.0], device=device, dtype=dtype)
self.assert_close(wrapper.magnitude_range, expected)
self.assert_close(wrapper.op._param_generator.degrees, expected)
self.assert_close(wrapper._magnitude, torch.tensor([7.0], device=device, dtype=dtype))
assert "magnitude_range" in old, "the caller's dict stays as it was"
def test_old_state_dict_loads_strictly_inside_a_policy(self, device, dtype):
policy = PolicySequential(_wrapper_with_learnable_range()).to(device=device, dtype=dtype)
name = next(iter(policy._modules))
policy.load_state_dict(_old_learnable_range_state_dict(f"{name}."), strict=True)
self.assert_close(policy[0].magnitude_range, torch.tensor([0.0, 20.0], device=device, dtype=dtype))
@pytest.mark.device_agnostic
def test_only_the_old_key_is_forgiven(self):
wrapper = _wrapper_with_learnable_range()
unexpected = {**_old_learnable_range_state_dict(), "nonsense": torch.zeros(1)}
with pytest.raises(RuntimeError, match=r'Unexpected key.*"nonsense"') as error:
wrapper.load_state_dict(unexpected, strict=True)
assert "magnitude_range" not in str(error.value)
missing = _old_learnable_range_state_dict()
del missing["op._param_generator.degrees"]
with pytest.raises(RuntimeError, match=r"Missing key.*op\._param_generator\.degrees"):
wrapper.load_state_dict(missing, strict=True)
def test_state_dict_round_trip(self, device, dtype):
source = _wrapper_with_learnable_range(0.0, 20.0).to(device=device, dtype=dtype)
state = source.state_dict()
assert set(state) == {"_magnitude", "_probability", "temperature", "op._param_generator.degrees"}
loaded = _wrapper_with_learnable_range().to(device=device, dtype=dtype)
loaded.load_state_dict(state, strict=True)
self.assert_close(loaded.magnitude_range, source.magnitude_range)
def test_optimizer_on_the_moved_wrapper_holds_each_parameter_once(self, device, dtype):
wrapper = _wrapper_with_learnable_range().to(device=device, dtype=dtype)
optimizer = torch.optim.SGD(wrapper.parameters(), lr=0.1)
held = [parameter for group in optimizer.param_groups for parameter in group["params"]]
assert {id(parameter) for parameter in held} == {
id(wrapper._magnitude),
id(wrapper._probability),
id(wrapper.magnitude_range),
}
assert len(held) == 3
assert all(parameter.device == device for parameter in held)
wrapper(torch.rand(2, 3, 16, 16, device=device, dtype=dtype)).square().sum().backward()
before = wrapper._magnitude.detach().clone()
optimizer.step()
assert not torch.equal(wrapper._magnitude, before)
@pytest.mark.device_agnostic
@pytest.mark.parametrize("learnable", [True, False], ids=["parameter", "buffer"])
def test_wrapper_pickled_with_the_old_attribute(self, learnable):
# A whole module saved before #6297 brings the attribute back with it: in ``_parameters`` for a parameter range,
# where it would demand the old key from every later load, and otherwise as a stale tensor in ``__dict__``.
wrapper = _wrapper_with_learnable_range() if learnable else ops.Brightness()
generator_range = wrapper.magnitude_range
state = dict(wrapper.__dict__)
if learnable:
state["_parameters"] = {"magnitude_range": generator_range, **state["_parameters"]}
else:
state["magnitude_range"] = generator_range.clone()
restored = type(wrapper).__new__(type(wrapper))
restored.__setstate__(state)
assert "magnitude_range" not in restored.__dict__
assert restored.magnitude_range is generator_range
assert "magnitude_range" not in restored.state_dict()
assert "magnitude_range" not in dict(restored.named_parameters())
restored.load_state_dict(wrapper.state_dict(), strict=True)