Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(augmentation): read a set TranslateX/TranslateY magnitude as a fraction of the image (#6216)

Saksham Sharma1ad7dba140b7
diff --git a/changelog.d/6216.fixed.md b/changelog.d/6216.fixed.md
new file mode 100644
index 000000000..f414af31a
--- /dev/null
+++ b/changelog.d/6216.fixed.md
@@ -0,0 +1 @@
+`TranslateX` and `TranslateY` in `kornia.augmentation.auto.operations` now read a learned magnitude, or one passed to `forward_parameters`, as a fraction of the image width or height, the same as a magnitude sampled from `magnitude_range`. They used to translate by that value in pixels, so the default `TranslateX()` moved the image by 0.2 pixels. `RandAugment`, `TrivialAugment` and `AutoAugment` already read their translate magnitudes as fractions, so their translations do not change.
diff --git a/kornia/augmentation/auto/operations/ops.py b/kornia/augmentation/auto/operations/ops.py
index 8f49e31a7..95dbbcc68 100644
--- a/kornia/augmentation/auto/operations/ops.py
+++ b/kornia/augmentation/auto/operations/ops.py
@@ -15,13 +15,20 @@
# limitations under the License.
#
-from typing import Optional, Tuple
+from typing import Dict, Optional, Tuple
import torch
import kornia.augmentation as K
from kornia.augmentation.auto.operations.base import OperationBase
+
+def _fraction_to_pixels(mag: Optional[torch.Tensor], size: int) -> Optional[torch.Tensor]:
+ # RandomTranslate's generator samples a fraction of the image and stores it in pixels, so a magnitude that
+ # replaces the draw has to be converted the same way.
+ return None if mag is None else mag * size
+
+
__all__ = [
"AutoContrast",
"Brightness",
@@ -524,8 +531,12 @@ class ShearY(OperationBase):
class TranslateX(OperationBase):
"""Apply translate operation along x-axis.
+ The magnitude is a fraction of the image width, whether it is sampled from ``magnitude_range``,
+ learned, or passed to :meth:`forward_parameters`; the wrapped augmentation translates by that fraction
+ of the width in pixels.
+
Args:
- initial_magnitude: the initial magnitude.
+ initial_magnitude: the initial magnitude, a fraction of the image width.
initial_probability: the initial probability. If None, the augmentation will be randomly
applied according to he augmentation sampling range.
magnitude_range: the sampling range for random sampling and clamping the optimized magnitude.
@@ -554,12 +565,31 @@ class TranslateX(OperationBase):
symmetric_megnitude=symmetric_megnitude,
)
+ def forward_parameters(
+ self, batch_shape: torch.Size, mag: Optional[torch.Tensor] = None
+ ) -> Dict[str, torch.Tensor]:
+ """Sample parameters, converting a set magnitude from a fraction of the width to pixels.
+
+ Args:
+ batch_shape: Input batch shape used for sampling.
+ mag: Optional magnitude override, a fraction of the image width.
+
+ Returns:
+ Parameter dictionary consumed by :meth:`forward`.
+ """
+ mag = _fraction_to_pixels(self.magnitude if mag is None else mag, batch_shape[-1])
+ return super().forward_parameters(batch_shape, mag)
+
class TranslateY(OperationBase):
"""Apply translate operation along y-axis.
+ The magnitude is a fraction of the image height, whether it is sampled from ``magnitude_range``,
+ learned, or passed to :meth:`forward_parameters`; the wrapped augmentation translates by that fraction
+ of the height in pixels.
+
Args:
- initial_magnitude: the initial magnitude.
+ initial_magnitude: the initial magnitude, a fraction of the image height.
initial_probability: the initial probability. If None, the augmentation will be randomly
applied according to he augmentation sampling range.
magnitude_range: the sampling range for random sampling and clamping the optimized magnitude.
@@ -587,3 +617,18 @@ class TranslateY(OperationBase):
temperature=temperature,
symmetric_megnitude=symmetric_megnitude,
)
+
+ def forward_parameters(
+ self, batch_shape: torch.Size, mag: Optional[torch.Tensor] = None
+ ) -> Dict[str, torch.Tensor]:
+ """Sample parameters, converting a set magnitude from a fraction of the height to pixels.
+
+ Args:
+ batch_shape: Input batch shape used for sampling.
+ mag: Optional magnitude override, a fraction of the image height.
+
+ Returns:
+ Parameter dictionary consumed by :meth:`forward`.
+ """
+ mag = _fraction_to_pixels(self.magnitude if mag is None else mag, batch_shape[-2])
+ return super().forward_parameters(batch_shape, mag)
diff --git a/kornia/augmentation/auto/rand_augment/rand_augment.py b/kornia/augmentation/auto/rand_augment/rand_augment.py
index 8996315b3..2c0ee9cb0 100644
--- a/kornia/augmentation/auto/rand_augment/rand_augment.py
+++ b/kornia/augmentation/auto/rand_augment/rand_augment.py
@@ -183,14 +183,6 @@ class RandAugment(PolicyAugmentBase):
else:
mag = m * float(maxval - minval) + minval
- # RandAugment magnitudes for translation are specified as fractions
- # of the image dimensions, while RandomTranslate stores pixel
- # translations in its sampled parameters.
- if op._factor_name == "translate_x":
- mag = mag * batch_shape[-1]
- elif op._factor_name == "translate_y":
- mag = mag * batch_shape[-2]
-
mod_param = op.forward_parameters(batch_shape, mag=mag)
# Compose it
param = ParamItem(name, [ParamItem(next(iter(module.named_children()))[0], mod_param)])
diff --git a/tests/augmentation/test_auto_operation.py b/tests/augmentation/test_auto_operation.py
index 72ca5929d..b4bd52d67 100644
--- a/tests/augmentation/test_auto_operation.py
+++ b/tests/augmentation/test_auto_operation.py
@@ -341,6 +341,37 @@ def test_symmetric_magnitude_negates_rather_than_zeroing(op, factor):
assert not (mags == 0).any(), f"{factor}: {(mags == 0).sum()} of {mags.numel()} draws were zeroed"
+ ("op", "factor", "dim"), [(ops.TranslateX, "translate_x", -1), (ops.TranslateY, "translate_y", -2)]
+)
+def test_translate_set_magnitude_is_a_fraction_of_the_image_6215(op, factor, dim):
+ # #6215: a set magnitude, learned or passed as ``mag``, replaced the generator's draw after the draw had been
+ # converted to pixels, so ``TranslateX(initial_magnitude=0.2)`` translated by 0.2 pixels while the same 0.2
+ # sampled from ``magnitude_range`` translated by a fifth of the image. A non-square shape tells the axes apart.
+ shape = torch.Size([4, 3, 64, 48])
+ expected = torch.full((4,), 0.2 * shape[dim])
+ sampled = op(None, 1.0, magnitude_range=(0.2, 0.2)).forward_parameters(shape)[factor]
+ learned = op(initial_magnitude=0.2, initial_probability=1.0).forward_parameters(shape)[factor]
+ passed = op(None, 1.0).forward_parameters(shape, mag=torch.tensor(0.2))[factor]
+ for params in (sampled, learned, passed):
+ assert_close(params.abs(), expected)
+
+
+def test_translate_learned_magnitude_gradient_6215():
+ # The conversion to pixels stays differentiable: the learned fraction's gradient matches central differences.
+ image = torch.rand(2, 3, 16, 24, generator=torch.Generator().manual_seed(0), dtype=torch.float64)
+
+ def loss(magnitude: float) -> torch.Tensor:
+ operation = ops.TranslateX(initial_magnitude=magnitude, initial_probability=1.0, symmetric_megnitude=False)
+ return operation, (operation(image) * torch.linspace(0, 1, 24, dtype=torch.float64)).sum()
+
+ operation, value = loss(0.2)
+ value.backward()
+ eps = 1e-4
+ numeric = (loss(0.2 + eps)[1] - loss(0.2 - eps)[1]) / (2 * eps)
+ assert_close(operation._magnitude.grad.double()[0], numeric.detach(), rtol=1e-3, atol=1e-3)
+
+
# AutoAugment sub-policies are ``(name, probability, magnitude_bin)``; the other two are
# ``(name, min_magnitude, max_magnitude)``. RandAugment and TrivialAugment allow only one
# operation per sub-policy, so a mixed draw has to come from two of them -- and cannot