π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
fix(augmentation): register dispatcher sequences as child modules (#6449)
Afloatb985896b72bf
diff --git a/changelog.d/6449.fixed.md b/changelog.d/6449.fixed.mdnew file mode 100644index 000000000..254a29a35--- /dev/null+++ b/changelog.d/6449.fixed.md@@ -0,0 +1 @@+Register augmentation dispatcher children so parameter discovery, checkpoints, training modes, and device or dtype moves include the contained sequences. After `.to(device)`, `.cuda()`, or `.double()`, the children sample on that device and in that dtype, as in `AugmentationSequential`, so random draws after a move can differ from Kornia 0.9.0. `state_dict()` now includes child buffers and parameters. A compatibility fallback preserves initialized and lazy child buffers and parameters, and non-tensor extra state, for legacy checkpoints without augmentation entries. With `strict=True`, missing new-format augmentation state, including a completely absent child subtree, raises a missing-key error when checkpoint metadata is preserved.diff --git a/kornia/augmentation/container/dispatcher.py b/kornia/augmentation/container/dispatcher.pyindex e6877063d..10a11fe58 100644--- a/kornia/augmentation/container/dispatcher.py+++ b/kornia/augmentation/container/dispatcher.py@@ -15,13 +15,35 @@# limitations under the License.#-from typing import List, Tuple, Union+from typing import Any, Dict, List, Tuple, Unionfrom torch import Tensor, nn+from torch.nn.parameter import is_lazyfrom .augment import AugmentationSequential+def _fill_missing_augmentation_state(+ module: nn.Module,+ state_dict: Dict[str, Any],+ prefix: str,+ local_metadata: Dict[str, Any],+ *args: Any,+) -> None:+ # Children used to live in a plain tuple, so old checkpoints contain no augmentation state.+ # Version 2 records the registered layout: never hide its missing keys, even if all child state is absent.+ if local_metadata.get("version", 1) >= 2:+ return+ if any(key.startswith(prefix + "augmentations.") for key in state_dict):+ return+ for name, value in module.state_dict(keep_vars=False).items():+ if name.startswith("augmentations."):+ # Lazy parameters and buffers must reach their load hooks without tensor operations.+ state_dict[prefix + name] = (+ value.detach().clone() if isinstance(value, Tensor) and not is_lazy(value) else value+ )++class ManyToManyAugmentationDispather(nn.Module):r"""Dispatches different augmentations to different inputs element-wisely.@@ -57,10 +79,13 @@ class ManyToManyAugmentationDispather(nn.Module):"""+ _version = 2+def __init__(self, *augmentations: AugmentationSequential) -> None:super().__init__()self._check_consistency(*augmentations)- self.augmentations = augmentations+ self.augmentations = nn.ModuleList(augmentations)+ self._register_load_state_dict_pre_hook(_fill_missing_augmentation_state, with_module=True)def _check_consistency(self, *augmentations: AugmentationSequential) -> bool:for i, aug in enumerate(augmentations):@@ -125,11 +150,14 @@ class ManyToOneAugmentationDispather(nn.Module):"""+ _version = 2+def __init__(self, *augmentations: AugmentationSequential, strict: bool = True) -> None:super().__init__()self.strict = strictself._check_consistency(*augmentations)- self.augmentations = augmentations+ self.augmentations = nn.ModuleList(augmentations)+ self._register_load_state_dict_pre_hook(_fill_missing_augmentation_state, with_module=True)def _check_consistency(self, *augmentations: AugmentationSequential) -> bool:for i, aug in enumerate(augmentations):diff --git a/tests/augmentation/container/test_dispatcher_module_state.py b/tests/augmentation/container/test_dispatcher_module_state.pynew file mode 100644index 000000000..d7216ae5b--- /dev/null+++ b/tests/augmentation/container/test_dispatcher_module_state.py@@ -0,0 +1,226 @@+# LICENSE HEADER MANAGED BY add-license-header+#+# Copyright 2018 Kornia Team+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import copy++import pytest+import torch+from torch import nn++import kornia.augmentation as K++from testing.base import BaseTester++[email protected]("dispatcher_class", [K.ManyToOneAugmentationDispather, K.ManyToManyAugmentationDispather])+class TestDispatcherModuleState(BaseTester):+ @staticmethod+ def call(dispatcher, image):+ if isinstance(dispatcher, K.ManyToManyAugmentationDispather):+ return dispatcher(*[(image,)] * len(dispatcher.augmentations))+ return dispatcher(image)++ def test_to_propagates_to_children(self, device, dtype, dispatcher_class):+ layers = [nn.Conv2d(2, 2, 1), nn.Conv2d(2, 2, 1)]+ dispatcher = dispatcher_class(*[K.AugmentationSequential(layer, data_keys=["input"]) for layer in layers])+ assert dispatcher.to(device=device, dtype=dtype) is dispatcher+ for layer in layers:+ assert layer.weight.device == device+ assert layer.weight.dtype == dtype+ image = torch.rand(2, 2, 4, 5, device=device, dtype=dtype)+ for actual, layer in zip(self.call(dispatcher, image), layers):+ self.assert_close(actual, layer(image), rtol=0, atol=0)++ def test_eval_propagates_without_updating_statistics(self, device, dtype, dispatcher_class):+ layers = [nn.BatchNorm2d(2).to(device=device, dtype=dtype) for _ in range(2)]+ dispatcher = dispatcher_class(*[K.AugmentationSequential(layer, data_keys=["input"]) for layer in layers])+ dispatcher.eval()+ assert all(not module.training for sequence in dispatcher.augmentations for module in sequence.modules())+ image = torch.arange(80, device=device, dtype=dtype).reshape(2, 2, 4, 5) / 80+ before = [layer.running_mean.clone() for layer in layers]+ actual = self.call(dispatcher, image)+ for output, layer, mean in zip(actual, layers, before):+ self.assert_close(layer.running_mean, mean, rtol=0, atol=0)+ self.assert_close(output, layer(image), rtol=0, atol=0)+ dispatcher.train()+ assert all(module.training for sequence in dispatcher.augmentations for module in sequence.modules())++ def test_parameters_and_checkpoint_roundtrip(self, device, dtype, dispatcher_class):+ layers = [nn.Conv2d(2, 2, 1).to(device=device, dtype=dtype) for _ in range(2)]+ dispatcher = dispatcher_class(*[K.AugmentationSequential(layer, data_keys=["input"]) for layer in layers])+ expected_parameters = [parameter for layer in layers for parameter in layer.parameters()]+ assert {id(p) for p in dispatcher.parameters()} == {id(p) for p in expected_parameters}+ state = copy.deepcopy(dispatcher.state_dict())+ assert len(state) == 4+ image = torch.rand(2, 2, 4, 5, device=device, dtype=dtype)+ expected = self.call(dispatcher, image)+ optimizer = torch.optim.SGD(dispatcher.parameters(), lr=0.1)+ sum(value.sum() for value in expected).backward()+ before = layers[0].weight.detach().clone()+ optimizer.step()+ assert not torch.equal(layers[0].weight, before)+ restored_layers = [nn.Conv2d(2, 2, 1).to(device=device, dtype=dtype) for _ in range(2)]+ restored = dispatcher_class(+ *[K.AugmentationSequential(layer, data_keys=["input"]) for layer in restored_layers]+ )+ restored.load_state_dict(state, strict=True)+ for actual, reference in zip(self.call(restored, image), expected):+ self.assert_close(actual, reference, rtol=0, atol=0)++ def test_moved_children_sample_like_a_moved_sequence(self, device, dtype, dispatcher_class):+ # After a move the children draw on that device and in that dtype, as an AugmentationSequential moved the+ # same way does. A CPU run with a non-default dtype checks the dtype half; a CUDA run checks the device.+ def sequences():+ return [+ K.AugmentationSequential(K.RandomHorizontalFlip(p=0.5), K.RandomBrightness((0.8, 1.2), p=1.0))+ for _ in range(2)+ ]++ dispatcher = dispatcher_class(*sequences()).to(device=device, dtype=dtype)+ reference = [sequence.to(device=device, dtype=dtype) for sequence in sequences()]+ for sequence in dispatcher.augmentations:+ for augmentation in sequence.children():+ assert augmentation.device.type == device.type+ assert augmentation.dtype == dtype+ image = torch.rand(4, 3, 4, 5, device=device, dtype=dtype)+ torch.manual_seed(0)+ actual = self.call(dispatcher, image)+ torch.manual_seed(0)+ expected = [sequence(image) for sequence in reference]+ for output, target in zip(actual, expected):+ self.assert_close(output, target, rtol=0, atol=0)++ def test_learnable_range_is_a_parameter(self, device, dtype, dispatcher_class):+ # Ranges stay on CPU in the default dtype unless the augmentation's RNG is moved.+ ranges = [nn.Parameter(torch.tensor([0.8, 1.2])) for _ in range(2)]+ dispatcher = dispatcher_class(+ *[K.AugmentationSequential(K.RandomBrightness(value, p=1.0), data_keys=["input"]) for value in ranges]+ )+ assert {id(parameter) for parameter in dispatcher.parameters()} == {id(value) for value in ranges}++ @pytest.mark.parametrize("nested", [False, True])+ @pytest.mark.parametrize("learnable", [False, True])+ def test_state_dict_saved_before_registration_loads_strictly(+ self, device, dtype, dispatcher_class, nested, learnable+ ):+ ranges = [nn.Parameter(torch.tensor([0.8, 1.2])) if learnable else (0.8, 1.2) for _ in range(2)]+ dispatcher = dispatcher_class(+ *[K.AugmentationSequential(K.RandomBrightness(value, p=1.0), data_keys=["input"]) for value in ranges]+ ).to(device=device, dtype=dtype)+ module = nn.Sequential(nn.Identity(), dispatcher) if nested else dispatcher+ expected = copy.deepcopy(module.state_dict())+ assert expected++ # Before registration the dispatcher contributed no child state, including when nested.+ legacy = {}+ module.load_state_dict(legacy, strict=True)+ assert legacy == {}+ for name, value in module.state_dict().items():+ self.assert_close(value, expected[name], rtol=0, atol=0)++ # A checkpoint with any child state uses the new layout: do not hide a missing key.+ partial = dict(list(expected.items())[:1])+ with pytest.raises(RuntimeError, match="Missing key"):+ module.load_state_dict(partial, strict=True)++ @pytest.mark.parametrize("nested", [False, True])+ def test_current_state_dict_missing_all_child_state_is_reported(self, device, dtype, dispatcher_class, nested):+ dispatcher = dispatcher_class(+ *[+ K.AugmentationSequential(K.RandomBrightness(nn.Parameter(torch.tensor([0.8, 1.2])), p=1.0))+ for _ in range(2)+ ]+ ).to(device=device, dtype=dtype)+ module = nn.Sequential(nn.Identity(), dispatcher) if nested else dispatcher+ state = copy.deepcopy(module.state_dict())+ missing_keys = set(state)+ assert missing_keys+ # Removing the whole child subtree must not turn a current-format checkpoint into a legacy one.+ # Keep the state_dict metadata so the checkpoint's format remains identifiable.+ state.clear()++ with pytest.raises(RuntimeError, match="Missing key"):+ module.load_state_dict(state, strict=True)+ result = module.load_state_dict(state, strict=False)+ assert set(result.missing_keys) == missing_keys+ assert result.unexpected_keys == []++ @pytest.mark.parametrize("nested", [False, True])+ @pytest.mark.parametrize("lazy_kind", ["linear", "batch_norm"])+ def test_legacy_state_keeps_lazy_children_uninitialized(self, device, dtype, dispatcher_class, nested, lazy_kind):+ layer = nn.LazyLinear(5) if lazy_kind == "linear" else nn.LazyBatchNorm2d()+ dispatcher = dispatcher_class(K.AugmentationSequential(layer, data_keys=["input"])).to(+ device=device, dtype=dtype+ )+ module = nn.Sequential(nn.Identity(), dispatcher) if nested else dispatcher+ uninitialized = {+ name: value for name, value in module.state_dict().items() if torch.nn.parameter.is_lazy(value)+ }+ assert uninitialized++ result = module.load_state_dict({}, strict=True)+ assert result.missing_keys == []+ assert result.unexpected_keys == []+ for name, value in module.state_dict().items():+ if name in uninitialized:+ assert value is uninitialized[name]+ assert torch.nn.parameter.is_lazy(value)+ assert value.device == device+ assert value.dtype == dtype++ image = torch.rand(2, 3, 4, 5, device=device, dtype=dtype)+ output = self.call(dispatcher, image)+ assert len(output) == 1+ assert output[0].shape == image.shape+ assert output[0].device == device+ assert output[0].dtype == dtype+ assert torch.isfinite(output[0]).all()+ assert not any(torch.nn.parameter.is_lazy(value) for value in module.state_dict().values())++ @pytest.mark.device_agnostic+ @pytest.mark.parametrize("nested", [False, True])+ def test_non_tensor_extra_state_loads_legacy_and_roundtrips(self, dispatcher_class, nested):+ class ExtraStateModule(nn.Identity):+ def __init__(self, value):+ super().__init__()+ self.value = {"label": value}++ def get_extra_state(self):+ return self.value++ def set_extra_state(self, state):+ self.value = state++ layers = [ExtraStateModule("first"), ExtraStateModule("second")]+ dispatcher = dispatcher_class(*[K.AugmentationSequential(layer, data_keys=["input"]) for layer in layers])+ module = nn.Sequential(nn.Identity(), dispatcher) if nested else dispatcher+ expected = copy.deepcopy(module.state_dict())+ assert len(expected) == 2++ # The unregistered layout saved version-1 metadata without any child state.+ legacy = copy.deepcopy(expected)+ legacy.clear()+ prefixes = ("", "0", "1") if nested else ("",)+ legacy._metadata = {name: {"version": 1} for name in prefixes}+ module.load_state_dict(legacy, strict=True)+ assert legacy == {}+ assert module.state_dict() == expected++ for layer in layers:+ layer.value = {"label": "changed"}+ module.load_state_dict(expected, strict=True)+ assert module.state_dict() == expected