π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
fix(augmentation): video boxes follow each frame through a VideoSequential mix (#6173) (#6273)
Dmytro Mishkind98c48298a50
diff --git a/changelog.d/6273.fixed.md b/changelog.d/6273.fixed.mdnew file mode 100644index 000000000..bb73e7364--- /dev/null+++ b/changelog.d/6273.fixed.md@@ -0,0 +1,4 @@+Video boxes passed through `VideoSequential(RandomMosaic)` now follow the mosaic of their own frame, as image boxes do,+with `same_on_frame=True` or `False` and with `p < 1`; before, they were returned untransformed. Frames that a partial+gate leaves with fewer boxes are padded to the mosaic box count with rows that export as zero in every mode, also after+a later geometric transform, and `VideoBoxes.to_tensor()` returns them as one `(B, T, N, ...)` tensor.diff --git a/kornia/augmentation/_2d/mix/mosaic.py b/kornia/augmentation/_2d/mix/mosaic.pyindex e25dca9f7..f3f9764f3 100644--- a/kornia/augmentation/_2d/mix/mosaic.py+++ b/kornia/augmentation/_2d/mix/mosaic.py@@ -190,10 +190,12 @@ class RandomMosaic(MixAugmentationBaseV2):if not bool(to_apply.all()):# An unselected sample keeps its own boxes unchanged, followed by padding up to the mosaic box count.- # The padding is recorded in ``_N`` and exported as all-zero (zero-area) rows.+ # The padding is recorded in ``_N`` and marked invalid, so every export gives all-zero rows for it, also+ # after a later geometric transform and in a container such as VideoBoxes that does not keep ``_N``.out_boxes._data[~to_apply, :num_boxes] = input._data[~to_apply]out_boxes._valid[~to_apply, :num_boxes] = input._valid[~to_apply]out_boxes._data[~to_apply, num_boxes:] = 0.0+ out_boxes._valid[~to_apply, num_boxes:] = Falsepadding = out_boxes._data.shape[1] - num_boxesmerged_padding = out_boxes._N if out_boxes._N is not None else [0] * len(to_apply)input_padding = input._N if input._N is not None else [0] * len(to_apply)diff --git a/kornia/augmentation/container/video.py b/kornia/augmentation/container/video.pyindex a5de94ecf..296e6904f 100644--- a/kornia/augmentation/container/video.py+++ b/kornia/augmentation/container/video.py@@ -24,7 +24,8 @@ import kornia.augmentation as Kfrom kornia.augmentation.base import _AugmentationBasefrom kornia.augmentation.container.base import SequentialBasefrom kornia.augmentation.container.image import ImageSequential, _get_new_batch_shape-from kornia.geometry.boxes import Boxes+from kornia.augmentation.utils.helpers import _frame_boxes, _video_boxes+from kornia.geometry.boxes import Boxes, VideoBoxesfrom kornia.geometry.keypoints import Keypointsfrom .params import ParamItem@@ -465,6 +466,9 @@ class VideoSequential(ImageSequential):)input = super().transform_boxes(input, params, extra_args=extra_args)input = input.data.reshape(batchsize, frame_num, input.data.size(1), 4, 2)+ elif isinstance(input, VideoBoxes):+ frames = super().transform_boxes(_frame_boxes(input), params, extra_args=extra_args)+ input = _video_boxes(frames, input.temporal_channel_size)else:input = super().transform_boxes(input, params, extra_args=extra_args)return input@@ -487,6 +491,9 @@ class VideoSequential(ImageSequential):)input = super().inverse_boxes(input, params, extra_args=extra_args)input = input.data.reshape(batchsize, frame_num, input.data.size(1), 4, 2)+ elif isinstance(input, VideoBoxes):+ frames = super().inverse_boxes(_frame_boxes(input), params, extra_args=extra_args)+ input = _video_boxes(frames, input.temporal_channel_size)else:input = super().inverse_boxes(input, params, extra_args=extra_args)return inputdiff --git a/kornia/augmentation/utils/helpers.py b/kornia/augmentation/utils/helpers.pyindex d89434cbf..7d3b43406 100644--- a/kornia/augmentation/utils/helpers.py+++ b/kornia/augmentation/utils/helpers.py@@ -25,7 +25,7 @@ from torch.distributions import Beta, Uniform# Both moved to kornia.core.ops; re-exported so that existing imports from this module keep working.from kornia.core.ops import _constant_tensor, _flatten_constant # noqa: F401from kornia.core.utils import _extract_device_dtype-from kornia.geometry.boxes import Boxes+from kornia.geometry.boxes import Boxes, VideoBoxesfrom kornia.geometry.keypoints import Keypoints@@ -79,6 +79,34 @@ def _boxes_to_padded_tensor(boxes: Boxes, mode: str) -> torch.Tensor:return torch.where(valid, out, torch.zeros((), device=out.device, dtype=out.dtype))+def _frame_boxes(boxes: VideoBoxes) -> Boxes:+ r"""View :class:`VideoBoxes` as the frame-level :class:`Boxes` of its :math:`(B \cdot T, N, 4, 2)` storage.++ The children of :class:`VideoSequential` run on the flattened frames, and their parameters (a mix's+ ``permutation``, a gate) index frames. ``VideoBoxes`` indexes whole videos, so a child given one would select+ videos with frame indices (#6173). The view shares the storage and the validity mask.+ """+ frames = Boxes(boxes._data, False, mode=boxes._mode)+ frames._N = boxes._N+ frames._valid = boxes._valid+ frames._is_batched = boxes._is_batched+ return frames+++def _video_boxes(frames: Boxes, temporal_channel_size: int) -> VideoBoxes:+ """Wrap frame-level :class:`Boxes` back into :class:`VideoBoxes` with ``temporal_channel_size`` frames.++ A ``VideoBoxes`` is a dense container, so the per-frame padding counts ``_N`` are not carried over. Padding a+ child adds, such as the rows a partial ``RandomMosaic`` gate pads a skipped frame with, is marked invalid+ instead; every export zeroes invalid rows and every indexed read or write carries them.+ """+ boxes = VideoBoxes(frames._data, False, mode=frames._mode)+ boxes._valid = frames._valid+ boxes._is_batched = frames._is_batched+ boxes.temporal_channel_size = temporal_channel_size+ return boxes++def _validate_input(f: Callable[..., Any]) -> Callable[..., Any]:r"""Validate the 2D input of the wrapped function.diff --git a/tests/augmentation/container/test_video_sequential.py b/tests/augmentation/container/test_video_sequential.pyindex 5eb88a60e..ebf1d0ad8 100644--- a/tests/augmentation/container/test_video_sequential.py+++ b/tests/augmentation/container/test_video_sequential.py@@ -315,6 +315,140 @@ class TestVideoSequential:params = aug.forward_parameters(torch.Size((batch_size * frame_num, 3, 8, 8)))assert_close(out, aug(video.reshape(-1, 3, 8, 8), params=params).reshape(video.shape))+ @staticmethod+ def _tagged_boxes(batch_size, frame_num, device, dtype):+ """One box per frame, shifted by its video and frame, so a box names the frame it came from."""+ box = torch.tensor([[1.0, 1.0], [4.0, 1.0], [4.0, 4.0], [1.0, 4.0]], device=device, dtype=dtype)+ shift = torch.stack(+ torch.meshgrid(+ torch.arange(batch_size, device=device, dtype=dtype),+ torch.arange(frame_num, device=device, dtype=dtype),+ indexing="ij",+ ),+ dim=-1,+ )+ return (box + 0.5 * shift[:, :, None]).unsqueeze(2).contiguous() # (B, T, 1, 4, 2)++ @staticmethod+ def _dense_boxes(boxes):+ from kornia.augmentation.utils.helpers import _boxes_to_padded_tensor++ return _boxes_to_padded_tensor(boxes, "vertices_plus")++ @pytest.mark.parametrize("p", [1.0, 0.5])+ def test_mosaic_boxes_same_on_frame_follow_each_frame_6173(self, p, device, dtype):+ # Frame t of the boxes is the mosaic of frame t of every video with the video-level parameters, as for the+ # images. VideoBoxes indexes whole videos, so the mosaic's frame permutation raised IndexError (#6173).+ batch_size, frame_num = 3, 2+ mosaic = K.RandomMosaic(p=p)+ aug = K.AugmentationSequential(K.VideoSequential(mosaic, same_on_frame=True), data_keys=["input", "bbox"])+ video = torch.rand(batch_size, frame_num, 3, 8, 8, device=device, dtype=dtype)+ boxes = self._tagged_boxes(batch_size, frame_num, device, dtype)+ partial = False+ for seed in range(6):+ torch.manual_seed(seed)+ out, out_boxes = aug(video, boxes)+ torch.manual_seed(seed)+ params = mosaic.forward_parameters(torch.Size((batch_size, 3, 8, 8)))+ applied = params["batch_prob"] > 0.5+ partial |= bool(applied.any()) and not bool(applied.all())+ for frame in range(frame_num):+ assert_close(out[:, frame], mosaic(video[:, frame], params=params))+ frame_boxes = kornia.geometry.boxes.Boxes.from_tensor(boxes[:, frame], mode="vertices_plus")+ expected = self._dense_boxes(mosaic.transform_boxes(frame_boxes, params, mosaic.flags))+ assert out_boxes.shape == (batch_size, frame_num, *expected.shape[1:])+ assert_close(out_boxes[:, frame], expected)+ assert partial == (p < 1.0)++ @pytest.mark.parametrize("p", [1.0, 0.5])+ def test_mosaic_boxes_not_same_on_frame_match_flattened_frames_6173(self, p, device, dtype):+ # With same_on_frame=False the mosaic runs on the B * T frames; the boxes follow the same frames (#6173).+ batch_size, frame_num = 2, 3+ mosaic = K.RandomMosaic(p=p)+ aug = K.AugmentationSequential(K.VideoSequential(mosaic, same_on_frame=False), data_keys=["input", "bbox"])+ video = torch.rand(batch_size, frame_num, 3, 8, 8, device=device, dtype=dtype)+ boxes = self._tagged_boxes(batch_size, frame_num, device, dtype)+ frames = video.reshape(-1, 3, 8, 8)+ frame_boxes = kornia.geometry.boxes.Boxes.from_tensor(boxes.flatten(0, 1), mode="vertices_plus")+ partial = False+ for seed in range(6):+ torch.manual_seed(seed)+ out, out_boxes = aug(video, boxes)+ torch.manual_seed(seed)+ params = mosaic.forward_parameters(torch.Size((batch_size * frame_num, 3, 8, 8)))+ applied = params["batch_prob"] > 0.5+ partial |= bool(applied.any()) and not bool(applied.all())+ assert_close(out, mosaic(frames, params=params).reshape(video.shape))+ expected = self._dense_boxes(mosaic.transform_boxes(frame_boxes.clone(), params, mosaic.flags))+ assert_close(out_boxes, expected.reshape(batch_size, frame_num, *expected.shape[1:]))+ assert partial == (p < 1.0)++ def test_mosaic_video_boxes_object_input_exports_padded_6173(self, device, dtype):+ # A VideoBoxes input comes back as VideoBoxes. After a partial gate its frames hold 4 or 1 boxes, and+ # to_tensor() returns them as one padded (B, T, N, 4, 2) tensor, equal to the dense tensor input's output.+ from kornia.geometry.boxes import VideoBoxes++ batch_size, frame_num = 3, 2+ mosaic = K.RandomMosaic(p=0.5)+ aug = K.AugmentationSequential(K.VideoSequential(mosaic, same_on_frame=True), data_keys=["input", "bbox"])+ video = torch.rand(batch_size, frame_num, 3, 8, 8, device=device, dtype=dtype)+ boxes = self._tagged_boxes(batch_size, frame_num, device, dtype)+ partial = False+ for seed in range(6):+ torch.manual_seed(seed)+ _, dense = aug(video, boxes)+ torch.manual_seed(seed)+ _, wrapped = aug(video, VideoBoxes.from_tensor(boxes))+ torch.manual_seed(seed)+ applied = mosaic.forward_parameters(torch.Size((batch_size, 3, 8, 8)))["batch_prob"] > 0.5+ partial |= bool(applied.any()) and not bool(applied.all())+ assert isinstance(wrapped, VideoBoxes)+ exported = wrapped.to_tensor()+ assert exported.shape == dense.shape+ assert_close(exported, dense)+ assert wrapped.to_tensor("xyxy").shape == (*dense.shape[:3], 4)+ assert partial++ @pytest.mark.parametrize("then_flip", [False, True])+ def test_mosaic_video_boxes_padding_exports_as_zero_6173(self, then_flip, device, dtype):+ # A video a partial gate skips keeps its one box per frame, followed by padding rows up to the mosaic box+ # count. A stored zero row exports as [0, 0, 1, 1] in the exclusive modes and a later flip moves it (to+ # [7, 0, 8, 1] on 8-pixel frames), so the padding has to export as zero in every mode and after indexing,+ # while the real rows export as their stored coordinates do.+ from kornia.geometry.boxes import Boxes, VideoBoxes++ batch_size, frame_num = 3, 2+ mosaic = K.RandomMosaic(p=0.5)+ children = [mosaic] + ([K.RandomHorizontalFlip(p=1.0)] if then_flip else [])+ aug = K.AugmentationSequential(K.VideoSequential(*children, same_on_frame=True), data_keys=["input", "bbox"])+ video = torch.rand(batch_size, frame_num, 3, 8, 8, device=device, dtype=dtype)+ boxes = self._tagged_boxes(batch_size, frame_num, device, dtype)+ partial = False+ for seed in range(6):+ torch.manual_seed(seed)+ _, dense = aug(video, boxes)+ torch.manual_seed(seed)+ _, wrapped = aug(video, VideoBoxes.from_tensor(boxes))+ torch.manual_seed(seed)+ applied = mosaic.forward_parameters(torch.Size((batch_size, 3, 8, 8)))["batch_prob"] > 0.5+ if bool(applied.all()):+ continue+ partial |= bool(applied.any())+ num_boxes = wrapped.data.shape[1]+ # In a skipped video, the rows after each frame's own box are padding.+ padding = ~applied.to(device)[:, None, None] & (torch.arange(num_boxes, device=device) >= boxes.shape[2])+ padding = padding.expand(batch_size, frame_num, num_boxes)+ assert_close(wrapped.to_tensor(), dense)+ for mode in ("xyxy", "xyxy_plus", "xywh", "vertices", "vertices_plus"):+ exported = wrapped.to_tensor(mode)+ assert (exported[padding] == 0).all(), mode+ frames = Boxes(wrapped.data, False).to_tensor(mode).reshape(exported.shape)+ assert_close(exported[~padding], frames[~padding])+ # Indexing whole videos keeps each frame's padding.+ assert_close(wrapped[1:].to_tensor(mode), exported[1:])+ assert_close(wrapped[0].to_tensor(mode), exported[:1])+ assert partial+@staticmethoddef _tagged_video(batch_size, frame_num, data_format, device, dtype):"""Build a video whose pixel value is ``10 * (video + 1) + frame``.