Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

perf(enhance): clamp brightness and contrast results in place when no gradient is recorded (#6276)

Dmytro Mishkinc19c6cbbcd55
diff --git a/changelog.d/6276.fixed.md b/changelog.d/6276.fixed.md
new file mode 100644
index 000000000..d5262f58f
--- /dev/null
+++ b/changelog.d/6276.fixed.md
@@ -0,0 +1,6 @@
+`adjust_brightness` and `adjust_contrast`, and with them `RandomBrightness`, `RandomContrast` and `ColorJiggle`,
+clamp their own freshly computed result in place when no gradient is recorded. That saves one image-sized temporary
+per call. At batch 32 of 3Γ—256Γ—256 float32 on CPU, the extra 25 MB buffer made glibc trim and re-fault the heap on
+every call (18,400 page faults per call against 12,256), and `RandomBrightness` took 8.6–9.7 ms against 6.2 ms in
+0.8.3. It now takes 7.5 ms, the remaining cost being the `[0, 1]` input clamp added in #5980. Values, gradients and
+the caller's tensor are unchanged.
diff --git a/kornia/enhance/adjust.py b/kornia/enhance/adjust.py
index 3f1c80b27..fd74a01d6 100644
--- a/kornia/enhance/adjust.py
+++ b/kornia/enhance/adjust.py
@@ -75,6 +75,28 @@ def _lookup_value_check(cond: torch.Tensor, msg: str) -> None:
torch._assert_async(cond, msg)
+def _clip_unit_fresh(image: torch.Tensor) -> torch.Tensor:
+ """Clamp a tensor this module just allocated into ``[0, 1]``, in place when no gradient is recorded.
+
+ The values equal ``image.clamp(0, 1)``. Without a recorded gradient the clamp reuses the fresh buffer instead of
+ allocating another image-sized one: at batch 32 that third 25 MB temporary per call made glibc trim and re-fault
+ the heap on every call. Only call it on a tensor no caller can see.
+
+ An integer tensor, which reaches here only with checks disabled, keeps the out-of-place clamp: the float bounds
+ promote its result to the default float dtype, which an in-place clamp cannot do. So does a tensor under a function
+ transform: vmap has a batching rule for ``clamp`` but not for ``clamp_``, whose fallback loops over the batch and
+ rejects an empty one. TorchScript cannot compile the function-transform query, so a scripted call keeps it too.
+ """
+ if not torch.jit.is_scripting(): # noqa: SIM102 - TorchScript only skips a bare is_scripting() guard
+ if (
+ image.is_floating_point()
+ and not (torch.is_grad_enabled() and image.requires_grad)
+ and not torch._C._are_functorch_transforms_active()
+ ):
+ return image.clamp_(min=0.0, max=1.0)
+ return image.clamp(min=0.0, max=1.0)
+
+
def _check_floating_image(image: torch.Tensor, op: str) -> None:
"""Reject an integer image in an op that expects floating-point values in ``[0, 1]``.
@@ -551,7 +573,7 @@ def adjust_contrast(image: torch.Tensor, factor: Union[float, torch.Tensor], cli
# Truncate between pixel values
if clip_output:
- img_adjust = img_adjust.clamp(min=0.0, max=1.0)
+ img_adjust = _clip_unit_fresh(img_adjust)
return img_adjust
@@ -675,7 +697,7 @@ def adjust_brightness(
# truncate between pixel values
if clip_output:
- img_adjust = img_adjust.clamp(min=0.0, max=1.0)
+ img_adjust = _clip_unit_fresh(img_adjust)
return img_adjust
diff --git a/tests/enhance/test_adjust.py b/tests/enhance/test_adjust.py
index b98c67c6b..994a372b7 100644
--- a/tests/enhance/test_adjust.py
+++ b/tests/enhance/test_adjust.py
@@ -25,6 +25,7 @@ import kornia
from kornia import enhance
from kornia.constants import pi
from kornia.core._compat import torch_version_ge
+from kornia.core.check import are_checks_enabled, disable_checks, enable_checks
from testing.base import DYNAMO_UNAVAILABLE_REASON, BaseTester, dynamo_is_available
@@ -1837,3 +1838,84 @@ class TestEmptyIntensity(BaseTester):
assert output.device == image.device
assert output.dtype == dtype
self.assert_close(output, expected)
+
+
+class TestClipUnitFreshBuffer(BaseTester):
+ # adjust_brightness and adjust_contrast clamp their own fresh result in place when no gradient is recorded, so a
+ # batch-32 call allocates one image-sized temporary fewer. The caller's tensor is never modified, and the values
+ # and gradients equal the out-of-place clamp.
+ _REFERENCE = {
+ "adjust_brightness": lambda image, factor: torch.clamp(image + factor.view(-1, 1, 1, 1), 0.0, 1.0),
+ "adjust_contrast": lambda image, factor: torch.clamp(image * factor.view(-1, 1, 1, 1), 0.0, 1.0),
+ }
+
+ @pytest.mark.parametrize("op", sorted(_REFERENCE))
+ def test_input_unmodified_and_values_match_with_and_without_a_gradient(self, device, dtype, op):
+ fn = getattr(enhance, op)
+ reference = self._REFERENCE[op]
+ image = torch.linspace(-0.5, 1.5, 2 * 3 * 4 * 4, device=device, dtype=dtype).reshape(2, 3, 4, 4)
+ original = image.clone()
+ factor = torch.tensor([0.3, 1.4], device=device, dtype=dtype)
+ with torch.no_grad():
+ out_no_grad = fn(image, factor)
+ out_plain = fn(image, factor)
+ leaf = image.clone().requires_grad_()
+ out_grad = fn(leaf, factor)
+ assert torch.equal(image, original)
+ assert torch.equal(out_no_grad, out_plain)
+ assert torch.equal(out_no_grad, out_grad.detach())
+ self.assert_close(out_no_grad, reference(image, factor))
+
+ ref_leaf = image.clone().requires_grad_()
+ reference(ref_leaf, factor).sum().backward()
+ out_grad.sum().backward()
+ assert leaf.grad is not None
+ self.assert_close(leaf.grad, ref_leaf.grad)
+
+ @pytest.mark.parametrize("op", sorted(_REFERENCE))
+ @pytest.mark.parametrize("batch", [0, 3])
+ def test_vmap_keeps_the_batching_rule(self, device, dtype, op, batch):
+ # vmap has a batching rule for clamp but not for clamp_, whose fallback loops over the batch and rejects an
+ # empty one, so a transformed call keeps the out-of-place clamp. With the fallback disabled, reaching it raises.
+ fn = getattr(enhance, op)
+ image = torch.rand(batch, 3, 4, 4, device=device, dtype=dtype)
+ functorch = torch._C._functorch
+ fallback_enabled = functorch._is_vmap_fallback_enabled()
+ functorch._set_vmap_fallback_enabled(False)
+ try:
+ out = torch.vmap(lambda frame: fn(frame, 0.2))(image)
+ finally:
+ functorch._set_vmap_fallback_enabled(fallback_enabled)
+ self.assert_close(out, fn(image, 0.2))
+
+ @pytest.mark.parametrize("op", sorted(_REFERENCE))
+ def test_jit(self, device, dtype, op):
+ # TorchScript cannot compile the functorch query, so the scripted functions and modules skip the in-place clamp.
+ fn = getattr(enhance, op)
+ module = {"adjust_brightness": enhance.AdjustBrightness, "adjust_contrast": enhance.AdjustContrast}[op]
+ image = torch.linspace(-0.5, 1.5, 2 * 3 * 4 * 4, device=device, dtype=dtype).reshape(2, 3, 4, 4)
+ original = image.clone()
+ factor = torch.tensor([0.3, 1.4], device=device, dtype=dtype)
+ expected = self._REFERENCE[op](image, factor)
+ self.assert_close(torch.jit.script(fn)(image, factor), expected)
+ self.assert_close(torch.jit.script(module(factor))(image), expected)
+ assert torch.equal(image, original)
+
+ @pytest.mark.parametrize("op", sorted(_REFERENCE))
+ def test_integer_image_with_checks_disabled_keeps_the_promoting_clamp(self, device, op):
+ # With checks disabled an integer image reaches the clamp. Its float bounds promote the result to float32,
+ # which an in-place clamp cannot do, so that image keeps the out-of-place clamp and its float32 output.
+ fn = getattr(enhance, op)
+ image = torch.randint(0, 256, (2, 3, 4, 4), device=device, dtype=torch.uint8)
+ original = image.clone()
+ factor = torch.tensor([0.3, 1.4], device=device)
+ was_enabled = are_checks_enabled()
+ disable_checks()
+ try:
+ out = fn(image, factor)
+ finally:
+ if was_enabled:
+ enable_checks()
+ assert torch.equal(image, original)
+ assert out.dtype == torch.float32
+ self.assert_close(out, self._REFERENCE[op](image, factor.to(torch.uint8)))