π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
fix(augmentation): read a set TranslateX/TranslateY magnitude as a fraction of the image (#6216)
Saksham Sharma1ad7dba140b7
diff --git a/changelog.d/6216.fixed.md b/changelog.d/6216.fixed.mdnew file mode 100644index 000000000..f414af31a--- /dev/null+++ b/changelog.d/6216.fixed.md@@ -0,0 +1 @@+`TranslateX` and `TranslateY` in `kornia.augmentation.auto.operations` now read a learned magnitude, or one passed to `forward_parameters`, as a fraction of the image width or height, the same as a magnitude sampled from `magnitude_range`. They used to translate by that value in pixels, so the default `TranslateX()` moved the image by 0.2 pixels. `RandAugment`, `TrivialAugment` and `AutoAugment` already read their translate magnitudes as fractions, so their translations do not change.diff --git a/kornia/augmentation/auto/operations/ops.py b/kornia/augmentation/auto/operations/ops.pyindex 8f49e31a7..95dbbcc68 100644--- a/kornia/augmentation/auto/operations/ops.py+++ b/kornia/augmentation/auto/operations/ops.py@@ -15,13 +15,20 @@# limitations under the License.#-from typing import Optional, Tuple+from typing import Dict, Optional, Tupleimport torchimport kornia.augmentation as Kfrom kornia.augmentation.auto.operations.base import OperationBase++def _fraction_to_pixels(mag: Optional[torch.Tensor], size: int) -> Optional[torch.Tensor]:+ # RandomTranslate's generator samples a fraction of the image and stores it in pixels, so a magnitude that+ # replaces the draw has to be converted the same way.+ return None if mag is None else mag * size++__all__ = ["AutoContrast","Brightness",@@ -524,8 +531,12 @@ class ShearY(OperationBase):class TranslateX(OperationBase):"""Apply translate operation along x-axis.+ The magnitude is a fraction of the image width, whether it is sampled from ``magnitude_range``,+ learned, or passed to :meth:`forward_parameters`; the wrapped augmentation translates by that fraction+ of the width in pixels.+Args:- initial_magnitude: the initial magnitude.+ initial_magnitude: the initial magnitude, a fraction of the image width.initial_probability: the initial probability. If None, the augmentation will be randomlyapplied according to he augmentation sampling range.magnitude_range: the sampling range for random sampling and clamping the optimized magnitude.@@ -554,12 +565,31 @@ class TranslateX(OperationBase):symmetric_megnitude=symmetric_megnitude,)+ def forward_parameters(+ self, batch_shape: torch.Size, mag: Optional[torch.Tensor] = None+ ) -> Dict[str, torch.Tensor]:+ """Sample parameters, converting a set magnitude from a fraction of the width to pixels.++ Args:+ batch_shape: Input batch shape used for sampling.+ mag: Optional magnitude override, a fraction of the image width.++ Returns:+ Parameter dictionary consumed by :meth:`forward`.+ """+ mag = _fraction_to_pixels(self.magnitude if mag is None else mag, batch_shape[-1])+ return super().forward_parameters(batch_shape, mag)+class TranslateY(OperationBase):"""Apply translate operation along y-axis.+ The magnitude is a fraction of the image height, whether it is sampled from ``magnitude_range``,+ learned, or passed to :meth:`forward_parameters`; the wrapped augmentation translates by that fraction+ of the height in pixels.+Args:- initial_magnitude: the initial magnitude.+ initial_magnitude: the initial magnitude, a fraction of the image height.initial_probability: the initial probability. If None, the augmentation will be randomlyapplied according to he augmentation sampling range.magnitude_range: the sampling range for random sampling and clamping the optimized magnitude.@@ -587,3 +617,18 @@ class TranslateY(OperationBase):temperature=temperature,symmetric_megnitude=symmetric_megnitude,)++ def forward_parameters(+ self, batch_shape: torch.Size, mag: Optional[torch.Tensor] = None+ ) -> Dict[str, torch.Tensor]:+ """Sample parameters, converting a set magnitude from a fraction of the height to pixels.++ Args:+ batch_shape: Input batch shape used for sampling.+ mag: Optional magnitude override, a fraction of the image height.++ Returns:+ Parameter dictionary consumed by :meth:`forward`.+ """+ mag = _fraction_to_pixels(self.magnitude if mag is None else mag, batch_shape[-2])+ return super().forward_parameters(batch_shape, mag)diff --git a/kornia/augmentation/auto/rand_augment/rand_augment.py b/kornia/augmentation/auto/rand_augment/rand_augment.pyindex 8996315b3..2c0ee9cb0 100644--- a/kornia/augmentation/auto/rand_augment/rand_augment.py+++ b/kornia/augmentation/auto/rand_augment/rand_augment.py@@ -183,14 +183,6 @@ class RandAugment(PolicyAugmentBase):else:mag = m * float(maxval - minval) + minval- # RandAugment magnitudes for translation are specified as fractions- # of the image dimensions, while RandomTranslate stores pixel- # translations in its sampled parameters.- if op._factor_name == "translate_x":- mag = mag * batch_shape[-1]- elif op._factor_name == "translate_y":- mag = mag * batch_shape[-2]-mod_param = op.forward_parameters(batch_shape, mag=mag)# Compose itparam = ParamItem(name, [ParamItem(next(iter(module.named_children()))[0], mod_param)])diff --git a/tests/augmentation/test_auto_operation.py b/tests/augmentation/test_auto_operation.pyindex 72ca5929d..b4bd52d67 100644--- a/tests/augmentation/test_auto_operation.py+++ b/tests/augmentation/test_auto_operation.py@@ -341,6 +341,37 @@ def test_symmetric_magnitude_negates_rather_than_zeroing(op, factor):assert not (mags == 0).any(), f"{factor}: {(mags == 0).sum()} of {mags.numel()} draws were zeroed"+ ("op", "factor", "dim"), [(ops.TranslateX, "translate_x", -1), (ops.TranslateY, "translate_y", -2)]+)+def test_translate_set_magnitude_is_a_fraction_of_the_image_6215(op, factor, dim):+ # #6215: a set magnitude, learned or passed as ``mag``, replaced the generator's draw after the draw had been+ # converted to pixels, so ``TranslateX(initial_magnitude=0.2)`` translated by 0.2 pixels while the same 0.2+ # sampled from ``magnitude_range`` translated by a fifth of the image. A non-square shape tells the axes apart.+ shape = torch.Size([4, 3, 64, 48])+ expected = torch.full((4,), 0.2 * shape[dim])+ sampled = op(None, 1.0, magnitude_range=(0.2, 0.2)).forward_parameters(shape)[factor]+ learned = op(initial_magnitude=0.2, initial_probability=1.0).forward_parameters(shape)[factor]+ passed = op(None, 1.0).forward_parameters(shape, mag=torch.tensor(0.2))[factor]+ for params in (sampled, learned, passed):+ assert_close(params.abs(), expected)+++def test_translate_learned_magnitude_gradient_6215():+ # The conversion to pixels stays differentiable: the learned fraction's gradient matches central differences.+ image = torch.rand(2, 3, 16, 24, generator=torch.Generator().manual_seed(0), dtype=torch.float64)++ def loss(magnitude: float) -> torch.Tensor:+ operation = ops.TranslateX(initial_magnitude=magnitude, initial_probability=1.0, symmetric_megnitude=False)+ return operation, (operation(image) * torch.linspace(0, 1, 24, dtype=torch.float64)).sum()++ operation, value = loss(0.2)+ value.backward()+ eps = 1e-4+ numeric = (loss(0.2 + eps)[1] - loss(0.2 - eps)[1]) / (2 * eps)+ assert_close(operation._magnitude.grad.double()[0], numeric.detach(), rtol=1e-3, atol=1e-3)++# AutoAugment sub-policies are ``(name, probability, magnitude_bin)``; the other two are# ``(name, min_magnitude, max_magnitude)``. RandAugment and TrivialAugment allow only one# operation per sub-policy, so a mixed draw has to come from two of them -- and cannot