Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(enhance): avoid float16 sigmoid gradient overflow (#6508)

Puttala Jeevan Kumara0af49f54b21
diff --git a/changelog.d/6508.fixed.md b/changelog.d/6508.fixed.md
new file mode 100644
index 000000000..fb6802101
--- /dev/null
+++ b/changelog.d/6508.fixed.md
@@ -0,0 +1 @@
+`adjust_sigmoid` and `AdjustSigmoid` no longer return NaN gradients for float16 images in `[0, 1]` once `|gain * (cutoff - x)|` exceeds about 11, for example `gain=30` (default `cutoff=0.5`) or `gain=40` with `cutoff=0.3`; very large gains such as `gain=200` work as well. With Python-number `gain` and `cutoff`, float16 images are now evaluated in float32 and rounded back to float16, so float16 outputs and gradients can differ slightly from earlier releases, also at the default `gain=10` (they are more accurate). Other image dtypes and tensor-valued `gain`/`cutoff` are unchanged.
diff --git a/kornia/enhance/adjust.py b/kornia/enhance/adjust.py
index 828a163ae..f68d521e5 100644
--- a/kornia/enhance/adjust.py
+++ b/kornia/enhance/adjust.py
@@ -777,7 +777,21 @@ def adjust_sigmoid(image: torch.Tensor, cutoff: float = 0.5, gain: float = 10, i
_check_floating_image(image, "adjust_sigmoid")
KORNIA_CHECK_IS_TENSOR(image, "Expected shape (*, H, W)")
- return 1 - 1 / (1 + (gain * (cutoff - image)).exp()) if inv else 1 / (1 + (gain * (cutoff - image)).exp())
+ # Half exp can overflow even for ordinary image values. Tensor parameters keep their own promotion.
+ promote = image.dtype == torch.float16 and isinstance(gain, (int, float)) and isinstance(cutoff, (int, float))
+ if promote:
+ image = image.float()
+ bound = abs(gain) * max(abs(cutoff), abs(1.0 - cutoff))
+ # On [0, 1], a bound of 16 keeps exp and the reciprocal cube in double backward normal in float32.
+ # At larger bounds, evaluate sigmoid on its nonpositive side to avoid positive-side saturation in backward.
+ if bound > 16.0:
+ logits = gain * (image - cutoff)
+ positive = logits >= 0
+ tail = torch.where(positive, -logits, logits).sigmoid()
+ result = torch.where(positive, tail, 1 - tail) if inv else torch.where(positive, 1 - tail, tail)
+ return result.half()
+ result = 1 - 1 / (1 + (gain * (cutoff - image)).exp()) if inv else 1 / (1 + (gain * (cutoff - image)).exp())
+ return result.half() if promote else result
def adjust_log(image: torch.Tensor, gain: float = 1, inv: bool = False, clip_output: bool = True) -> torch.Tensor:
diff --git a/tests/enhance/test_adjust.py b/tests/enhance/test_adjust.py
index 8e15b5efe..4c933700f 100644
--- a/tests/enhance/test_adjust.py
+++ b/tests/enhance/test_adjust.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
-from math import prod
+from math import exp, prod
import pytest
import torch
@@ -931,6 +931,171 @@ class TestAdjustBrightness(BaseTester):
class TestAdjustSigmoid(BaseTester):
+ @pytest.mark.parametrize("half_dtype", [torch.float16])
+ @pytest.mark.parametrize("inv", [False, True])
+ @pytest.mark.parametrize("scripted", [False, True])
+ def test_half_default_mean_loss_derivatives(self, device, half_dtype, inv, scripted):
+ image = torch.zeros(1, 1, 256, 256, device=device, dtype=half_dtype, requires_grad=True)
+ op = kornia.enhance.AdjustSigmoid(gain=10.0, inv=inv)
+ if scripted:
+ op = torch.jit.script(op)
+ # The mean supplies a small backward seed: half reciprocal backward underflows even though exp(5) is finite.
+ for _ in range(2 if scripted else 1):
+ loss = op(image).float().mean()
+ (gradient,) = torch.autograd.grad(loss, image, create_graph=True)
+ (second,) = torch.autograd.grad(gradient, image, torch.ones_like(gradient))
+
+ tail = exp(-5.0)
+ scale = (-1.0 if inv else 1.0) / image.numel()
+ expected_gradient = scale * 10.0 * tail / (1 + tail) ** 2
+ expected_second = scale * 100.0 * tail * (1 - tail) / (1 + tail) ** 3
+ self.assert_close(gradient, torch.full_like(image, expected_gradient), rtol=0, atol=0)
+ self.assert_close(second, torch.full_like(image, expected_second), rtol=0, atol=0)
+
+ @pytest.mark.parametrize("half_dtype", [torch.float16])
+ @pytest.mark.parametrize(
+ "gain,cutoff", [(34.0, 0.5), (40.0, 0.5), (40.0, 0.3), (60.0, 0.5), (100.0, 0.5), (200.0, 0.5)]
+ )
+ @pytest.mark.parametrize("gain_sign", [1.0, -1.0])
+ @pytest.mark.parametrize("inv", [False, True])
+ @pytest.mark.parametrize("scripted", [False, True])
+ def test_half_high_gain_grid_derivatives(self, device, half_dtype, gain, cutoff, gain_sign, inv, scripted):
+ # Use the actual half-rounded inputs in the independent Python-double reference.
+ values = torch.linspace(0, 1, 1001, dtype=half_dtype).tolist()
+ image = torch.tensor(values, device=device, dtype=half_dtype).view(1, 1, 1, -1).requires_grad_()
+ gain *= gain_sign
+ op = kornia.enhance.AdjustSigmoid(cutoff=cutoff, gain=gain, inv=inv)
+ if scripted:
+ op = torch.jit.script(op)
+ # Also exercise TorchScript's optimized backward after its profiling run.
+ for _ in range(2 if scripted else 1):
+ actual = op(image)
+ (gradient,) = torch.autograd.grad(actual, image, torch.ones_like(actual), create_graph=True)
+ (second,) = torch.autograd.grad(gradient, image, torch.ones_like(gradient))
+
+ logits = [gain * (x - cutoff) for x in values]
+ tails = [exp(-abs(z)) for z in logits]
+ sign = -1.0 if inv else 1.0
+ expected_gradient = image.new_tensor([sign * gain * t / (1 + t) ** 2 for t in tails]).view_as(image)
+ expected_second = image.new_tensor(
+ [sign * (-1.0 if z > 0 else 1.0) * gain**2 * t * (1 - t) / (1 + t) ** 3 for z, t in zip(logits, tails)]
+ ).view_as(image)
+ assert actual.dtype == image.dtype
+ assert actual.device == image.device
+ assert actual.shape == image.shape
+ assert torch.isfinite(actual).all()
+ # Allow half rounding, including one subnormal step, but do not lose representable derivatives.
+ for result, expected in [(gradient, expected_gradient), (second, expected_second)]:
+ assert torch.isfinite(result).all()
+ assert (result[expected != 0] != 0).all()
+ self.assert_close(result, expected, rtol=2e-3, atol=2**-24)
+
+ @pytest.mark.parametrize("half_dtype", [torch.float16])
+ @pytest.mark.parametrize("gain", [200.0, -200.0])
+ @pytest.mark.parametrize("inv", [False, True])
+ def test_half_outside_range_derivatives(self, device, half_dtype, gain, inv):
+ # These saturated pixels should have zero derivatives, not exp overflow followed by 0 * inf.
+ image = torch.tensor([-65504.0, -20.0, 21.0, 65504.0], device=device, dtype=half_dtype).requires_grad_()
+ actual = kornia.enhance.adjust_sigmoid(image, gain=gain, inv=inv)
+ (gradient,) = torch.autograd.grad(actual, image, torch.ones_like(actual), create_graph=True)
+ (second,) = torch.autograd.grad(gradient, image, torch.ones_like(gradient))
+ assert torch.isfinite(actual).all()
+ self.assert_close(gradient, torch.zeros_like(image), rtol=0, atol=0)
+ self.assert_close(second, torch.zeros_like(image), rtol=0, atol=0)
+
+ @pytest.mark.parametrize("half_dtype", [torch.float16])
+ @pytest.mark.parametrize("inv", [False, True])
+ @pytest.mark.parametrize("gain", [23.0, -23.0, 30.0, -30.0, 32.0, -32.0])
+ def test_half_high_gain_second_derivative(self, device, half_dtype, gain, inv):
+ values = [0.0, 0.25, 0.5, 0.75, 1.0]
+ image = torch.tensor(values, device=device, dtype=half_dtype).view(1, 1, 1, 5).requires_grad_()
+ actual = kornia.enhance.AdjustSigmoid(gain=gain, inv=inv)(image)
+ (gradient,) = torch.autograd.grad(actual, image, torch.ones_like(actual), create_graph=True)
+ (second,) = torch.autograd.grad(gradient, image, torch.ones_like(gradient))
+
+ # Evaluate the analytic derivatives through the small tail, avoiding cancellation in 1 - sigmoid.
+ logits = [gain * (x - 0.5) for x in values]
+ tails = [exp(-abs(z)) for z in logits]
+ sign = -1.0 if inv else 1.0
+ expected_gradient = [sign * gain * t / (1 + t) ** 2 for t in tails]
+ expected_second = [
+ sign * (-1.0 if z > 0 else 1.0) * gain**2 * t * (1 - t) / (1 + t) ** 3 for z, t in zip(logits, tails)
+ ]
+ self.assert_close(gradient, image.new_tensor(expected_gradient).view_as(image), rtol=0, atol=0)
+ self.assert_close(second, image.new_tensor(expected_second).view_as(image), rtol=0, atol=0)
+
+ @pytest.mark.parametrize("half_dtype", [torch.bfloat16])
+ @pytest.mark.parametrize("inv", [False, True])
+ @pytest.mark.parametrize("value,cutoff,gain", [(-7.90625, 0.75, 10.0), (-3.53125, 0.7578125, 20.0)])
+ def test_bfloat16_outside_range_second_derivative(self, device, half_dtype, value, cutoff, gain, inv):
+ # Promoting bfloat16 here can overflow double-backward intermediates that were finite before.
+ image = torch.tensor([value], device=device, dtype=half_dtype).view(1, 1, 1, 1).requires_grad_()
+ actual = kornia.enhance.adjust_sigmoid(image, cutoff=cutoff, gain=gain, inv=inv)
+ (gradient,) = torch.autograd.grad(actual, image, torch.ones_like(actual), create_graph=True)
+ (second,) = torch.autograd.grad(gradient, image, torch.ones_like(gradient))
+ assert torch.isfinite(actual).all()
+ assert torch.isfinite(gradient).all()
+ assert torch.isfinite(second).all()
+
+ @pytest.mark.parametrize("inv", [False, True])
+ def test_broadcast_tensor_parameters_keep_gradients_and_dtype(self, device, dtype, inv):
+ image = torch.tensor([0.25, 0.5, 0.75], device=device, dtype=dtype).view(1, 1, 1, 3).requires_grad_()
+ cutoff = torch.tensor([0.5, 0.5, 0.5], device=device, dtype=torch.float32, requires_grad=True)
+ gain = torch.tensor([30.0, -30.0, 10.0], device=device, dtype=torch.float32, requires_grad=True)
+ actual = kornia.enhance.adjust_sigmoid(image, cutoff=cutoff, gain=gain, inv=inv)
+ image_grad, cutoff_grad, gain_grad = torch.autograd.grad(actual, (image, cutoff, gain), torch.ones_like(actual))
+ assert actual.dtype == torch.promote_types(dtype, torch.float32)
+ assert actual.shape == image.shape
+ assert actual.device == image.device
+ # All values are exactly representable, so the reference does not inherit image rounding error.
+ tails = [exp(-7.5), 1.0, exp(-2.5)]
+ sign = -1.0 if inv else 1.0
+ slopes = [sign * t / (1 + t) ** 2 for t in tails]
+ expected_image = [g * s for g, s in zip([30.0, -30.0, 10.0], slopes)]
+ self.assert_close(image_grad, image.new_tensor(expected_image).view_as(image), rtol=1e-6, atol=0)
+ self.assert_close(cutoff_grad, cutoff.new_tensor([-v for v in expected_image]), rtol=1e-6, atol=0)
+ self.assert_close(
+ gain_grad, gain.new_tensor([d * s for d, s in zip([-0.25, 0.0, 0.25], slopes)]), rtol=1e-6, atol=0
+ )
+
+ @pytest.mark.parametrize("inv", [False, True])
+ @pytest.mark.parametrize("gain", [0.0, 10.0, 30.0, -30.0])
+ def test_bounded_values_and_gradients(self, device, dtype, gain, inv):
+ # Dark pixels overflow exp(gain * (cutoff - x)) in float16 at gain=30, although both the
+ # sigmoid and its derivative are bounded. Check both tails and the derivative at the cutoff.
+ image = torch.tensor([0.0, 0.25, 0.5, 0.75, 1.0], device=device, dtype=dtype).view(1, 1, 1, 5)
+ image.requires_grad_()
+ actual = kornia.enhance.AdjustSigmoid(gain=gain, inv=inv)(image)
+ (gradient,) = torch.autograd.grad(actual.sum(), image)
+
+ # Independent analytic reference, evaluated in Python double precision, without an optional dependency.
+ values = [1.0 / (1.0 + exp(gain * (0.5 - x))) for x in [0.0, 0.25, 0.5, 0.75, 1.0]]
+ derivatives = [gain * y * (1.0 - y) * (-1.0 if inv else 1.0) for y in values]
+ expected = image.new_tensor([1.0 - y if inv else y for y in values]).view_as(image)
+ expected_gradient = image.new_tensor(derivatives).view_as(image)
+ assert actual.dtype == image.dtype
+ assert actual.device == image.device
+ self.assert_close(actual, expected)
+ # In particular, both tiny tail derivatives must survive; replacing the expression with a
+ # low-precision sigmoid alone rounds the bright tail to 1 and gives it a zero derivative.
+ self.assert_close(gradient, expected_gradient, rtol=0.02, atol=1e-7)
+
+ @pytest.mark.parametrize("inv", [False, True])
+ def test_gradcheck_high_gain(self, device, inv):
+ image = torch.tensor([0.25, 0.5, 0.75], device=device, dtype=torch.float64).view(1, 1, 1, 3)
+ self.gradcheck(kornia.enhance.AdjustSigmoid(gain=30.0, inv=inv), (image,))
+
+ @pytest.mark.parametrize("inv", [False, True])
+ def test_dynamo_high_gain_backward(self, device, dtype, inv, torch_optimizer):
+ image = torch.tensor([0.0, 0.5, 1.0], device=device, dtype=dtype).view(1, 1, 1, 3).requires_grad_()
+ op = kornia.enhance.AdjustSigmoid(gain=30.0, inv=inv)
+ expected = op(image)
+ expected_gradient = torch.autograd.grad(expected.sum(), image)[0]
+ actual = torch_optimizer(op, fullgraph=True)(image)
+ gradient = torch.autograd.grad(actual.sum(), image)[0]
+ self.assert_close(actual, expected)
+ self.assert_close(gradient, expected_gradient, rtol=0.02, atol=1e-7)
+
@pytest.mark.parametrize("shape", [(3, 4, 4), (2, 3, 4, 4)])
def test_shape_sigmoid(self, shape, device):
inputs = torch.ones(*shape, device=device)