π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
fix(camera): keep the Kannala-Brandt undistortion loop rolled under TorchScript (#6258)
Leonardo Vanni64e2ca991808
diff --git a/changelog.d/6258.fixed.md b/changelog.d/6258.fixed.mdnew file mode 100644index 000000000..9ceb1451a--- /dev/null+++ b/changelog.d/6258.fixed.md@@ -0,0 +1 @@+`torch.jit.script(undistort_points_kannala_brandt)` no longer hangs on CUDA from its second call. TorchScript unrolled the ten-step Gauss-Newton loop and its CUDA fuser inlined the steps into an expression that grew exponentially, until the process ran out of host memory. Eager outputs are unchanged.diff --git a/kornia/geometry/camera/distortion_kannala_brandt.py b/kornia/geometry/camera/distortion_kannala_brandt.pyindex 33032228f..99053f1fb 100644--- a/kornia/geometry/camera/distortion_kannala_brandt.py+++ b/kornia/geometry/camera/distortion_kannala_brandt.py@@ -223,8 +223,11 @@ def undistort_points_kannala_brandt(distorted_points_in_camera: torch.Tensor, path = safe_rth.sqrt()th = torch.where(nonzero_radius, th, torch.zeros_like(th))- # gauss-newton- for _ in range(iters):+ # gauss-newton. A while loop, not range(iters): TorchScript unrolls a constant-count loop, and its CUDA fuser+ # then inlines every step into one expression that grows exponentially with the iterations.+ it = 0+ while it < iters:+ it += 1th2 = th * thinner = k0 + th2 * (k1 + th2 * (k2 + th2 * k3))thd = th * (1.0 + th2 * inner)diff --git a/tests/geometry/camera/test_distortion.py b/tests/geometry/camera/test_distortion.pyindex 5cb985517..bd7c210db 100644--- a/tests/geometry/camera/test_distortion.py+++ b/tests/geometry/camera/test_distortion.py@@ -32,6 +32,18 @@ from kornia.geometry.camera.distortion_kannala_brandt import (from testing.base import BaseTester+def _count_nodes(graph, kind: str) -> int:+ """Count the nodes of ``kind`` in a TorchScript graph, inside loop bodies and fusion subgraphs too."""+ count = 0+ for node in graph.nodes():+ count += node.kind() == kind+ for block in node.blocks():+ count += _count_nodes(block, kind)+ if node.hasAttribute("Subgraph"):+ count += _count_nodes(node.g("Subgraph"), kind)+ return count++class TestDistortionAffine(BaseTester):def test_smoke(self, device, dtype):points = torch.tensor([1.0, 2.0], device=device, dtype=dtype)@@ -422,9 +434,17 @@ class TestDistortionKannalaBrandt(BaseTester):points = torch.tensor([919.5000, 1439.5000], device=device, dtype=dtype)params = torch.tensor([600.0, 600.0, 319.5, 239.5, 0.1, 0.2, 0.3, 0.4], device=device, dtype=dtype)op_script = torch.jit.script(undistort_points_kannala_brandt)- actual = op_script(points, params)expected = undistort_points_kannala_brandt(points, params)- self.assert_close(actual, expected)+ # Twice: TorchScript optimizes the graph on the second call. With a constant-count loop it unrolled the ten+ # Gauss-Newton steps there (24 ``aten::sub`` nodes against the 8 of the inlined scripted graph), and on CUDA its+ # fuser then hung (#6257). The scripted function is cached across tests, so compare with its inlined graph,+ # which optimization only shrinks, rather than with the first call.+ scripted_subs = _count_nodes(op_script.inlined_graph, "aten::sub")+ for _ in range(2):+ # On CUDA the optimized call fuses the loop body, and the fused kernels keep bfloat16 intermediates in+ # float32, so that call lands up to 4 bfloat16 ulps from eager (closer to the float64 result).+ self.assert_close(op_script(points, params), expected, low_tolerance=dtype == torch.bfloat16)+ assert _count_nodes(torch.jit.last_executed_optimized_graph(), "aten::sub") <= scripted_subsdef test_jit(self, device, dtype) -> None:self._test_jit_distort(device, dtype)