π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
perf(enhance): clamp brightness and contrast results in place when no gradient is recorded (#6276)
Dmytro Mishkinc19c6cbbcd55
diff --git a/changelog.d/6276.fixed.md b/changelog.d/6276.fixed.mdnew file mode 100644index 000000000..d5262f58f--- /dev/null+++ b/changelog.d/6276.fixed.md@@ -0,0 +1,6 @@+`adjust_brightness` and `adjust_contrast`, and with them `RandomBrightness`, `RandomContrast` and `ColorJiggle`,+clamp their own freshly computed result in place when no gradient is recorded. That saves one image-sized temporary+per call. At batch 32 of 3Γ256Γ256 float32 on CPU, the extra 25 MB buffer made glibc trim and re-fault the heap on+every call (18,400 page faults per call against 12,256), and `RandomBrightness` took 8.6β9.7 ms against 6.2 ms in+0.8.3. It now takes 7.5 ms, the remaining cost being the `[0, 1]` input clamp added in #5980. Values, gradients and+the caller's tensor are unchanged.diff --git a/kornia/enhance/adjust.py b/kornia/enhance/adjust.pyindex 3f1c80b27..fd74a01d6 100644--- a/kornia/enhance/adjust.py+++ b/kornia/enhance/adjust.py@@ -75,6 +75,28 @@ def _lookup_value_check(cond: torch.Tensor, msg: str) -> None:torch._assert_async(cond, msg)+def _clip_unit_fresh(image: torch.Tensor) -> torch.Tensor:+ """Clamp a tensor this module just allocated into ``[0, 1]``, in place when no gradient is recorded.++ The values equal ``image.clamp(0, 1)``. Without a recorded gradient the clamp reuses the fresh buffer instead of+ allocating another image-sized one: at batch 32 that third 25 MB temporary per call made glibc trim and re-fault+ the heap on every call. Only call it on a tensor no caller can see.++ An integer tensor, which reaches here only with checks disabled, keeps the out-of-place clamp: the float bounds+ promote its result to the default float dtype, which an in-place clamp cannot do. So does a tensor under a function+ transform: vmap has a batching rule for ``clamp`` but not for ``clamp_``, whose fallback loops over the batch and+ rejects an empty one. TorchScript cannot compile the function-transform query, so a scripted call keeps it too.+ """+ if not torch.jit.is_scripting(): # noqa: SIM102 - TorchScript only skips a bare is_scripting() guard+ if (+ image.is_floating_point()+ and not (torch.is_grad_enabled() and image.requires_grad)+ and not torch._C._are_functorch_transforms_active()+ ):+ return image.clamp_(min=0.0, max=1.0)+ return image.clamp(min=0.0, max=1.0)++def _check_floating_image(image: torch.Tensor, op: str) -> None:"""Reject an integer image in an op that expects floating-point values in ``[0, 1]``.@@ -551,7 +573,7 @@ def adjust_contrast(image: torch.Tensor, factor: Union[float, torch.Tensor], cli# Truncate between pixel valuesif clip_output:- img_adjust = img_adjust.clamp(min=0.0, max=1.0)+ img_adjust = _clip_unit_fresh(img_adjust)return img_adjust@@ -675,7 +697,7 @@ def adjust_brightness(# truncate between pixel valuesif clip_output:- img_adjust = img_adjust.clamp(min=0.0, max=1.0)+ img_adjust = _clip_unit_fresh(img_adjust)return img_adjustdiff --git a/tests/enhance/test_adjust.py b/tests/enhance/test_adjust.pyindex b98c67c6b..994a372b7 100644--- a/tests/enhance/test_adjust.py+++ b/tests/enhance/test_adjust.py@@ -25,6 +25,7 @@ import korniafrom kornia import enhancefrom kornia.constants import pifrom kornia.core._compat import torch_version_ge+from kornia.core.check import are_checks_enabled, disable_checks, enable_checksfrom testing.base import DYNAMO_UNAVAILABLE_REASON, BaseTester, dynamo_is_available@@ -1837,3 +1838,84 @@ class TestEmptyIntensity(BaseTester):assert output.device == image.deviceassert output.dtype == dtypeself.assert_close(output, expected)+++class TestClipUnitFreshBuffer(BaseTester):+ # adjust_brightness and adjust_contrast clamp their own fresh result in place when no gradient is recorded, so a+ # batch-32 call allocates one image-sized temporary fewer. The caller's tensor is never modified, and the values+ # and gradients equal the out-of-place clamp.+ _REFERENCE = {+ "adjust_brightness": lambda image, factor: torch.clamp(image + factor.view(-1, 1, 1, 1), 0.0, 1.0),+ "adjust_contrast": lambda image, factor: torch.clamp(image * factor.view(-1, 1, 1, 1), 0.0, 1.0),+ }++ @pytest.mark.parametrize("op", sorted(_REFERENCE))+ def test_input_unmodified_and_values_match_with_and_without_a_gradient(self, device, dtype, op):+ fn = getattr(enhance, op)+ reference = self._REFERENCE[op]+ image = torch.linspace(-0.5, 1.5, 2 * 3 * 4 * 4, device=device, dtype=dtype).reshape(2, 3, 4, 4)+ original = image.clone()+ factor = torch.tensor([0.3, 1.4], device=device, dtype=dtype)+ with torch.no_grad():+ out_no_grad = fn(image, factor)+ out_plain = fn(image, factor)+ leaf = image.clone().requires_grad_()+ out_grad = fn(leaf, factor)+ assert torch.equal(image, original)+ assert torch.equal(out_no_grad, out_plain)+ assert torch.equal(out_no_grad, out_grad.detach())+ self.assert_close(out_no_grad, reference(image, factor))++ ref_leaf = image.clone().requires_grad_()+ reference(ref_leaf, factor).sum().backward()+ out_grad.sum().backward()+ assert leaf.grad is not None+ self.assert_close(leaf.grad, ref_leaf.grad)++ @pytest.mark.parametrize("op", sorted(_REFERENCE))+ @pytest.mark.parametrize("batch", [0, 3])+ def test_vmap_keeps_the_batching_rule(self, device, dtype, op, batch):+ # vmap has a batching rule for clamp but not for clamp_, whose fallback loops over the batch and rejects an+ # empty one, so a transformed call keeps the out-of-place clamp. With the fallback disabled, reaching it raises.+ fn = getattr(enhance, op)+ image = torch.rand(batch, 3, 4, 4, device=device, dtype=dtype)+ functorch = torch._C._functorch+ fallback_enabled = functorch._is_vmap_fallback_enabled()+ functorch._set_vmap_fallback_enabled(False)+ try:+ out = torch.vmap(lambda frame: fn(frame, 0.2))(image)+ finally:+ functorch._set_vmap_fallback_enabled(fallback_enabled)+ self.assert_close(out, fn(image, 0.2))++ @pytest.mark.parametrize("op", sorted(_REFERENCE))+ def test_jit(self, device, dtype, op):+ # TorchScript cannot compile the functorch query, so the scripted functions and modules skip the in-place clamp.+ fn = getattr(enhance, op)+ module = {"adjust_brightness": enhance.AdjustBrightness, "adjust_contrast": enhance.AdjustContrast}[op]+ image = torch.linspace(-0.5, 1.5, 2 * 3 * 4 * 4, device=device, dtype=dtype).reshape(2, 3, 4, 4)+ original = image.clone()+ factor = torch.tensor([0.3, 1.4], device=device, dtype=dtype)+ expected = self._REFERENCE[op](image, factor)+ self.assert_close(torch.jit.script(fn)(image, factor), expected)+ self.assert_close(torch.jit.script(module(factor))(image), expected)+ assert torch.equal(image, original)++ @pytest.mark.parametrize("op", sorted(_REFERENCE))+ def test_integer_image_with_checks_disabled_keeps_the_promoting_clamp(self, device, op):+ # With checks disabled an integer image reaches the clamp. Its float bounds promote the result to float32,+ # which an in-place clamp cannot do, so that image keeps the out-of-place clamp and its float32 output.+ fn = getattr(enhance, op)+ image = torch.randint(0, 256, (2, 3, 4, 4), device=device, dtype=torch.uint8)+ original = image.clone()+ factor = torch.tensor([0.3, 1.4], device=device)+ was_enabled = are_checks_enabled()+ disable_checks()+ try:+ out = fn(image, factor)+ finally:+ if was_enabled:+ enable_checks()+ assert torch.equal(image, original)+ assert out.dtype == torch.float32+ self.assert_close(out, self._REFERENCE[op](image, factor.to(torch.uint8)))