Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(augmentation): register dispatcher sequences as child modules (#6449)

Afloatb985896b72bf
diff --git a/changelog.d/6449.fixed.md b/changelog.d/6449.fixed.md
new file mode 100644
index 000000000..254a29a35
--- /dev/null
+++ b/changelog.d/6449.fixed.md
@@ -0,0 +1 @@
+Register augmentation dispatcher children so parameter discovery, checkpoints, training modes, and device or dtype moves include the contained sequences. After `.to(device)`, `.cuda()`, or `.double()`, the children sample on that device and in that dtype, as in `AugmentationSequential`, so random draws after a move can differ from Kornia 0.9.0. `state_dict()` now includes child buffers and parameters. A compatibility fallback preserves initialized and lazy child buffers and parameters, and non-tensor extra state, for legacy checkpoints without augmentation entries. With `strict=True`, missing new-format augmentation state, including a completely absent child subtree, raises a missing-key error when checkpoint metadata is preserved.
diff --git a/kornia/augmentation/container/dispatcher.py b/kornia/augmentation/container/dispatcher.py
index e6877063d..10a11fe58 100644
--- a/kornia/augmentation/container/dispatcher.py
+++ b/kornia/augmentation/container/dispatcher.py
@@ -15,13 +15,35 @@
# limitations under the License.
#
-from typing import List, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union
from torch import Tensor, nn
+from torch.nn.parameter import is_lazy
from .augment import AugmentationSequential
+def _fill_missing_augmentation_state(
+ module: nn.Module,
+ state_dict: Dict[str, Any],
+ prefix: str,
+ local_metadata: Dict[str, Any],
+ *args: Any,
+) -> None:
+ # Children used to live in a plain tuple, so old checkpoints contain no augmentation state.
+ # Version 2 records the registered layout: never hide its missing keys, even if all child state is absent.
+ if local_metadata.get("version", 1) >= 2:
+ return
+ if any(key.startswith(prefix + "augmentations.") for key in state_dict):
+ return
+ for name, value in module.state_dict(keep_vars=False).items():
+ if name.startswith("augmentations."):
+ # Lazy parameters and buffers must reach their load hooks without tensor operations.
+ state_dict[prefix + name] = (
+ value.detach().clone() if isinstance(value, Tensor) and not is_lazy(value) else value
+ )
+
+
class ManyToManyAugmentationDispather(nn.Module):
r"""Dispatches different augmentations to different inputs element-wisely.
@@ -57,10 +79,13 @@ class ManyToManyAugmentationDispather(nn.Module):
"""
+ _version = 2
+
def __init__(self, *augmentations: AugmentationSequential) -> None:
super().__init__()
self._check_consistency(*augmentations)
- self.augmentations = augmentations
+ self.augmentations = nn.ModuleList(augmentations)
+ self._register_load_state_dict_pre_hook(_fill_missing_augmentation_state, with_module=True)
def _check_consistency(self, *augmentations: AugmentationSequential) -> bool:
for i, aug in enumerate(augmentations):
@@ -125,11 +150,14 @@ class ManyToOneAugmentationDispather(nn.Module):
"""
+ _version = 2
+
def __init__(self, *augmentations: AugmentationSequential, strict: bool = True) -> None:
super().__init__()
self.strict = strict
self._check_consistency(*augmentations)
- self.augmentations = augmentations
+ self.augmentations = nn.ModuleList(augmentations)
+ self._register_load_state_dict_pre_hook(_fill_missing_augmentation_state, with_module=True)
def _check_consistency(self, *augmentations: AugmentationSequential) -> bool:
for i, aug in enumerate(augmentations):
diff --git a/tests/augmentation/container/test_dispatcher_module_state.py b/tests/augmentation/container/test_dispatcher_module_state.py
new file mode 100644
index 000000000..d7216ae5b
--- /dev/null
+++ b/tests/augmentation/container/test_dispatcher_module_state.py
@@ -0,0 +1,226 @@
+# LICENSE HEADER MANAGED BY add-license-header
+#
+# Copyright 2018 Kornia Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import copy
+
+import pytest
+import torch
+from torch import nn
+
+import kornia.augmentation as K
+
+from testing.base import BaseTester
+
+
[email protected]("dispatcher_class", [K.ManyToOneAugmentationDispather, K.ManyToManyAugmentationDispather])
+class TestDispatcherModuleState(BaseTester):
+ @staticmethod
+ def call(dispatcher, image):
+ if isinstance(dispatcher, K.ManyToManyAugmentationDispather):
+ return dispatcher(*[(image,)] * len(dispatcher.augmentations))
+ return dispatcher(image)
+
+ def test_to_propagates_to_children(self, device, dtype, dispatcher_class):
+ layers = [nn.Conv2d(2, 2, 1), nn.Conv2d(2, 2, 1)]
+ dispatcher = dispatcher_class(*[K.AugmentationSequential(layer, data_keys=["input"]) for layer in layers])
+ assert dispatcher.to(device=device, dtype=dtype) is dispatcher
+ for layer in layers:
+ assert layer.weight.device == device
+ assert layer.weight.dtype == dtype
+ image = torch.rand(2, 2, 4, 5, device=device, dtype=dtype)
+ for actual, layer in zip(self.call(dispatcher, image), layers):
+ self.assert_close(actual, layer(image), rtol=0, atol=0)
+
+ def test_eval_propagates_without_updating_statistics(self, device, dtype, dispatcher_class):
+ layers = [nn.BatchNorm2d(2).to(device=device, dtype=dtype) for _ in range(2)]
+ dispatcher = dispatcher_class(*[K.AugmentationSequential(layer, data_keys=["input"]) for layer in layers])
+ dispatcher.eval()
+ assert all(not module.training for sequence in dispatcher.augmentations for module in sequence.modules())
+ image = torch.arange(80, device=device, dtype=dtype).reshape(2, 2, 4, 5) / 80
+ before = [layer.running_mean.clone() for layer in layers]
+ actual = self.call(dispatcher, image)
+ for output, layer, mean in zip(actual, layers, before):
+ self.assert_close(layer.running_mean, mean, rtol=0, atol=0)
+ self.assert_close(output, layer(image), rtol=0, atol=0)
+ dispatcher.train()
+ assert all(module.training for sequence in dispatcher.augmentations for module in sequence.modules())
+
+ def test_parameters_and_checkpoint_roundtrip(self, device, dtype, dispatcher_class):
+ layers = [nn.Conv2d(2, 2, 1).to(device=device, dtype=dtype) for _ in range(2)]
+ dispatcher = dispatcher_class(*[K.AugmentationSequential(layer, data_keys=["input"]) for layer in layers])
+ expected_parameters = [parameter for layer in layers for parameter in layer.parameters()]
+ assert {id(p) for p in dispatcher.parameters()} == {id(p) for p in expected_parameters}
+ state = copy.deepcopy(dispatcher.state_dict())
+ assert len(state) == 4
+ image = torch.rand(2, 2, 4, 5, device=device, dtype=dtype)
+ expected = self.call(dispatcher, image)
+ optimizer = torch.optim.SGD(dispatcher.parameters(), lr=0.1)
+ sum(value.sum() for value in expected).backward()
+ before = layers[0].weight.detach().clone()
+ optimizer.step()
+ assert not torch.equal(layers[0].weight, before)
+ restored_layers = [nn.Conv2d(2, 2, 1).to(device=device, dtype=dtype) for _ in range(2)]
+ restored = dispatcher_class(
+ *[K.AugmentationSequential(layer, data_keys=["input"]) for layer in restored_layers]
+ )
+ restored.load_state_dict(state, strict=True)
+ for actual, reference in zip(self.call(restored, image), expected):
+ self.assert_close(actual, reference, rtol=0, atol=0)
+
+ def test_moved_children_sample_like_a_moved_sequence(self, device, dtype, dispatcher_class):
+ # After a move the children draw on that device and in that dtype, as an AugmentationSequential moved the
+ # same way does. A CPU run with a non-default dtype checks the dtype half; a CUDA run checks the device.
+ def sequences():
+ return [
+ K.AugmentationSequential(K.RandomHorizontalFlip(p=0.5), K.RandomBrightness((0.8, 1.2), p=1.0))
+ for _ in range(2)
+ ]
+
+ dispatcher = dispatcher_class(*sequences()).to(device=device, dtype=dtype)
+ reference = [sequence.to(device=device, dtype=dtype) for sequence in sequences()]
+ for sequence in dispatcher.augmentations:
+ for augmentation in sequence.children():
+ assert augmentation.device.type == device.type
+ assert augmentation.dtype == dtype
+ image = torch.rand(4, 3, 4, 5, device=device, dtype=dtype)
+ torch.manual_seed(0)
+ actual = self.call(dispatcher, image)
+ torch.manual_seed(0)
+ expected = [sequence(image) for sequence in reference]
+ for output, target in zip(actual, expected):
+ self.assert_close(output, target, rtol=0, atol=0)
+
+ def test_learnable_range_is_a_parameter(self, device, dtype, dispatcher_class):
+ # Ranges stay on CPU in the default dtype unless the augmentation's RNG is moved.
+ ranges = [nn.Parameter(torch.tensor([0.8, 1.2])) for _ in range(2)]
+ dispatcher = dispatcher_class(
+ *[K.AugmentationSequential(K.RandomBrightness(value, p=1.0), data_keys=["input"]) for value in ranges]
+ )
+ assert {id(parameter) for parameter in dispatcher.parameters()} == {id(value) for value in ranges}
+
+ @pytest.mark.parametrize("nested", [False, True])
+ @pytest.mark.parametrize("learnable", [False, True])
+ def test_state_dict_saved_before_registration_loads_strictly(
+ self, device, dtype, dispatcher_class, nested, learnable
+ ):
+ ranges = [nn.Parameter(torch.tensor([0.8, 1.2])) if learnable else (0.8, 1.2) for _ in range(2)]
+ dispatcher = dispatcher_class(
+ *[K.AugmentationSequential(K.RandomBrightness(value, p=1.0), data_keys=["input"]) for value in ranges]
+ ).to(device=device, dtype=dtype)
+ module = nn.Sequential(nn.Identity(), dispatcher) if nested else dispatcher
+ expected = copy.deepcopy(module.state_dict())
+ assert expected
+
+ # Before registration the dispatcher contributed no child state, including when nested.
+ legacy = {}
+ module.load_state_dict(legacy, strict=True)
+ assert legacy == {}
+ for name, value in module.state_dict().items():
+ self.assert_close(value, expected[name], rtol=0, atol=0)
+
+ # A checkpoint with any child state uses the new layout: do not hide a missing key.
+ partial = dict(list(expected.items())[:1])
+ with pytest.raises(RuntimeError, match="Missing key"):
+ module.load_state_dict(partial, strict=True)
+
+ @pytest.mark.parametrize("nested", [False, True])
+ def test_current_state_dict_missing_all_child_state_is_reported(self, device, dtype, dispatcher_class, nested):
+ dispatcher = dispatcher_class(
+ *[
+ K.AugmentationSequential(K.RandomBrightness(nn.Parameter(torch.tensor([0.8, 1.2])), p=1.0))
+ for _ in range(2)
+ ]
+ ).to(device=device, dtype=dtype)
+ module = nn.Sequential(nn.Identity(), dispatcher) if nested else dispatcher
+ state = copy.deepcopy(module.state_dict())
+ missing_keys = set(state)
+ assert missing_keys
+ # Removing the whole child subtree must not turn a current-format checkpoint into a legacy one.
+ # Keep the state_dict metadata so the checkpoint's format remains identifiable.
+ state.clear()
+
+ with pytest.raises(RuntimeError, match="Missing key"):
+ module.load_state_dict(state, strict=True)
+ result = module.load_state_dict(state, strict=False)
+ assert set(result.missing_keys) == missing_keys
+ assert result.unexpected_keys == []
+
+ @pytest.mark.parametrize("nested", [False, True])
+ @pytest.mark.parametrize("lazy_kind", ["linear", "batch_norm"])
+ def test_legacy_state_keeps_lazy_children_uninitialized(self, device, dtype, dispatcher_class, nested, lazy_kind):
+ layer = nn.LazyLinear(5) if lazy_kind == "linear" else nn.LazyBatchNorm2d()
+ dispatcher = dispatcher_class(K.AugmentationSequential(layer, data_keys=["input"])).to(
+ device=device, dtype=dtype
+ )
+ module = nn.Sequential(nn.Identity(), dispatcher) if nested else dispatcher
+ uninitialized = {
+ name: value for name, value in module.state_dict().items() if torch.nn.parameter.is_lazy(value)
+ }
+ assert uninitialized
+
+ result = module.load_state_dict({}, strict=True)
+ assert result.missing_keys == []
+ assert result.unexpected_keys == []
+ for name, value in module.state_dict().items():
+ if name in uninitialized:
+ assert value is uninitialized[name]
+ assert torch.nn.parameter.is_lazy(value)
+ assert value.device == device
+ assert value.dtype == dtype
+
+ image = torch.rand(2, 3, 4, 5, device=device, dtype=dtype)
+ output = self.call(dispatcher, image)
+ assert len(output) == 1
+ assert output[0].shape == image.shape
+ assert output[0].device == device
+ assert output[0].dtype == dtype
+ assert torch.isfinite(output[0]).all()
+ assert not any(torch.nn.parameter.is_lazy(value) for value in module.state_dict().values())
+
+ @pytest.mark.device_agnostic
+ @pytest.mark.parametrize("nested", [False, True])
+ def test_non_tensor_extra_state_loads_legacy_and_roundtrips(self, dispatcher_class, nested):
+ class ExtraStateModule(nn.Identity):
+ def __init__(self, value):
+ super().__init__()
+ self.value = {"label": value}
+
+ def get_extra_state(self):
+ return self.value
+
+ def set_extra_state(self, state):
+ self.value = state
+
+ layers = [ExtraStateModule("first"), ExtraStateModule("second")]
+ dispatcher = dispatcher_class(*[K.AugmentationSequential(layer, data_keys=["input"]) for layer in layers])
+ module = nn.Sequential(nn.Identity(), dispatcher) if nested else dispatcher
+ expected = copy.deepcopy(module.state_dict())
+ assert len(expected) == 2
+
+ # The unregistered layout saved version-1 metadata without any child state.
+ legacy = copy.deepcopy(expected)
+ legacy.clear()
+ prefixes = ("", "0", "1") if nested else ("",)
+ legacy._metadata = {name: {"version": 1} for name in prefixes}
+ module.load_state_dict(legacy, strict=True)
+ assert legacy == {}
+ assert module.state_dict() == expected
+
+ for layer in layers:
+ layer.value = {"label": "changed"}
+ module.load_state_dict(expected, strict=True)
+ assert module.state_dict() == expected