Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(filters): restore half-precision bilateral sigma gradients (#6284) (#6285)

Vuln3rat0r24cf9fa3f6e8
diff --git a/changelog.d/6285.fixed.md b/changelog.d/6285.fixed.md
new file mode 100644
index 000000000..169b621b1
--- /dev/null
+++ b/changelog.d/6285.fixed.md
@@ -0,0 +1 @@
+`bilateral_blur` and `joint_bilateral_blur` take each window sum in one reduction again on the CPU and CUDA. In 0.9.0 they summed over the kernel's columns and then its rows, which rounded every partial sum to the input dtype and made float16 and bfloat16 gradients with respect to a tensor `sigma_color` or `sigma_space` up to about six times less accurate (float16 `sigma_color`: a relative error of 7.5e-2 instead of 1.3e-2). MPS keeps the two-step sum, which is much faster there, so on MPS those half-precision gradients stay less accurate; pass a float32 input there when they matter, since tensor sigmas take the input's dtype.
diff --git a/kornia/filters/bilateral.py b/kornia/filters/bilateral.py
index f51d7eec9..ed293e4fa 100644
--- a/kornia/filters/bilateral.py
+++ b/kornia/filters/bilateral.py
@@ -270,11 +270,11 @@ def _bilateral_blur(
pad_y, pad_x = _compute_zero_padding(kernel_size)
# Keep both patch axes: flattening them would copy every overlapping window. The windows are viewed in the order of
- # their elements in memory, (B, C, H, Ky, W, Kx), so that the arithmetic allocates every temporary contiguous, and
- # every reduction, here and in the backward, runs over one axis. MPS reduces a permuted 6-D tensor about 150 times
- # slower than a contiguous one (an M1: 238 ms against 1.4 ms for the colour distance of an 8 x 3 x 256 x 256 batch
- # with a 5 x 5 kernel), and two axes that are not adjacent, such as Ky and Kx, about 140 times slower than one at a
- # time, on torch 2.14 and on a 2.16 nightly alike.
+ # their elements in memory, (B, C, H, Ky, W, Kx), so that the arithmetic allocates every temporary contiguous, and,
+ # on MPS, every reduction, here and in the backward, runs over one axis. MPS reduces a permuted 6-D tensor about 150
+ # times slower than a contiguous one (an M1: 238 ms against 1.4 ms for the colour distance of an 8 x 3 x 256 x 256
+ # batch with a 5 x 5 kernel), and two axes that are not adjacent, such as Ky and Kx, about 140 times slower than one
+ # at a time, on torch 2.14 and on a 2.16 nightly alike.
padded_input = F.pad(input, (pad_x, pad_x, pad_y, pad_y), mode=border_type)
unfolded_input = padded_input.unfold(2, ky, 1).unfold(3, kx, 1).permute(0, 1, 2, 4, 3, 5) # (B, C, H, Ky, W, Kx)
@@ -311,8 +311,12 @@ def _bilateral_blur(
# to the system on every free, faults fewer pages back in per call.
kernel = space_kernel * (-0.5 / sigma_color**2 * color_distance_sq).exp() # (B, 1, H, Ky, W, Kx)
del color_distance_sq
- # Over Kx, then over Ky.
- return (unfolded_input * kernel).sum(-1).sum(-2) / kernel.sum(-1).sum(-2)
+ # On MPS over Kx, then over Ky, for the speed above. Elsewhere one reduction over both window axes: the two-step sum
+ # rounds every partial sum over Kx to a half dtype, which costs float16 and bfloat16 gradients with respect to
+ # tensor sigmas their accuracy.
+ if input.device.type == "mps":
+ return (unfolded_input * kernel).sum(-1).sum(-2) / kernel.sum(-1).sum(-2)
+ return (unfolded_input * kernel).sum((-3, -1)) / kernel.sum((-3, -1))
def bilateral_blur(
@@ -351,6 +355,11 @@ def bilateral_blur(
windows for every input size. Under ``torch.compile``, only kernels of up to 25 taps are filtered tap by
tap, since the compiled loop has a block of operations per tap. Both forms compute the same weights and add
up the weighted values in a different order, so they agree to rounding.
+ - On MPS, each window is summed over its columns and then over its rows, because MPS reduces two non-adjacent
+ axes much more slowly than one. In float16 and bfloat16 that rounds every partial sum, so there the
+ gradients with respect to a tensor ``sigma_color`` or ``sigma_space`` are less accurate than on the CPU or
+ CUDA. Tensor sigmas are cast to the input's dtype, so keeping them in float32 does not help; pass a float32
+ input on MPS when those gradients matter.
Arguments:
input: the input torch.Tensor with shape ``(B,C,H,W)``.
diff --git a/tests/filters/test_bilateral.py b/tests/filters/test_bilateral.py
index 8033ac245..5b76d5f06 100644
--- a/tests/filters/test_bilateral.py
+++ b/tests/filters/test_bilateral.py
@@ -302,6 +302,34 @@ class TestBilateralBlur(BaseTester):
expected = torch.tensor([[[[edge * 1j, 1j / (1 + 2 * e**2), edge * 1j]]]], dtype=torch.complex64)
self.assert_close(actual, expected)
+ @pytest.mark.parametrize("half_dtype", [torch.float16])
+ def test_half_precision_sigma_gradients(self, device, half_dtype):
+ # A two-step window sum in half precision truncated intermediate partial sums, raising sigma gradient relative
+ # error to ~7.5e-2 for sigma_color and ~3.8e-2 for sigma_space (#6284). Single-step reduction restores < 2.5e-2.
+ if device.type == "mps":
+ pytest.skip("MPS retains two-step sum for reduction performance")
+ if not supports_reflect_padding(device, half_dtype):
+ pytest.skip("reflection padding is unsupported for this device/dtype")
+ torch.manual_seed(0)
+ x = torch.rand(1, 3, 32, 40)
+ sigma_color = torch.tensor([0.1])
+ sigma_space = torch.tensor([[1.5, 1.5]])
+ w = torch.rand(1, 3, 32, 40)
+
+ def grads(dtype):
+ leaves = [t.to(device=device, dtype=dtype).clone().requires_grad_() for t in (x, sigma_color, sigma_space)]
+ out = bilateral_blur(leaves[0], (5, 5), leaves[1], leaves[2])
+ (out.float() * w.to(device=device)).sum().backward()
+ return [t.grad.float().cpu() for t in leaves]
+
+ ref = grads(torch.float32)
+ actual = grads(half_dtype)
+
+ rel_color = ((actual[1] - ref[1]).abs().max() / ref[1].abs().max()).item()
+ rel_space = ((actual[2] - ref[2]).abs().max() / ref[2].abs().max()).item()
+ assert rel_color < 0.025
+ assert rel_space < 0.025
+
@pytest.mark.parametrize("shape", [(1, 1, 8, 15), (2, 3, 11, 7)])
@pytest.mark.parametrize("kernel_size", [5, (3, 5)])
@pytest.mark.parametrize("sigma_color", [1, 0.1])