Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

perf(feature): run patch network convolutions channels-last on MPS and oneDNN CPU (#6502)

Dmytro Mishkin839ca4e8d665
diff --git a/benchmarks/README.md b/benchmarks/README.md
index f3d377961..5f85c362f 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -58,7 +58,7 @@ Deeper, single-topic scripts:
| [`filters/`](filters/) | [`flagship.py`](filters/flagship.py): core filters vs OpenCV/albumentations/torchvision v2/kornia-rs/PIL/scikit-image. [`gaussian_cpu.py`](filters/gaussian_cpu.py): Gaussian blur and scale-pyramid base/branch timing and numerical comparisons; [report](filters/gaussian_cpu.md). |
| [`color/`](color/) | pytest-benchmark microbenchmarks for color conversions (`*_test.py`). |
| [`contrib/`](contrib/) | [`connected_components.py`](contrib/connected_components.py): union-find vs pooling labeling, validated against SciPy; [report](contrib/connected_components.md). |
-| [`feature/`](feature/) | Local-feature detector benchmarks incl. quality (matching) metrics; [`laf_ops.py`](feature/laf_ops.py) microbenchmarks the shared LAF operations and [`ellipse_to_laf.py`](feature/ellipse_to_laf.py) drills into one of them (both base-revision A/B β€” no cross-library baseline exists). [`local_features.py`](feature/local_features.py) measures Oxford graf speed and homography corner error for SIFT, SIFT-AffNet-HardNet and KeyNet-HardNet on CPU, CUDA or MPS (`--device cpu --timing-pairs 2` times the representative 1–2 pair and still scores all five); results in [`graf_benchmark.md`](feature/graf_benchmark.md). [`sift_runtime.py`](feature/sift_runtime.py) and [`plot_sift_runtime.py`](feature/plot_sift_runtime.py) chart scale-space SIFT runtime across releases and batch sizes; results in [`sift_runtime.md`](feature/sift_runtime.md). |
+| [`feature/`](feature/) | Local-feature detector benchmarks incl. quality (matching) metrics; [`laf_ops.py`](feature/laf_ops.py) microbenchmarks the shared LAF operations and [`ellipse_to_laf.py`](feature/ellipse_to_laf.py) drills into one of them (both base-revision A/B β€” no cross-library baseline exists). [`local_features.py`](feature/local_features.py) measures Oxford graf speed and homography corner error for SIFT, SIFT-AffNet-HardNet and KeyNet-HardNet on CPU, CUDA or MPS (`--device cpu --timing-pairs 2` times the representative 1–2 pair and still scores all five); results in [`graf_benchmark.md`](feature/graf_benchmark.md). [`sift_runtime.py`](feature/sift_runtime.py) and [`plot_sift_runtime.py`](feature/plot_sift_runtime.py) chart scale-space SIFT runtime across releases and batch sizes; results in [`sift_runtime.md`](feature/sift_runtime.md). [`patch_network_layout.py`](feature/patch_network_layout.py) times the convolution bodies of the patch descriptor and shape networks (HardNet, HardNet8, SOSNet, TFeat, HyNet, AffNet, OriNet) in the default and the channels-last layout on any device, and their public forwards for a base/branch comparison. |
| [`common.py`](common.py) | Shared methodology utilities β€” use these in every new benchmark. |
[`feature/sift_scale_space.py`](feature/sift_scale_space.py) compares complete SIFT
diff --git a/benchmarks/feature/patch_network_layout.py b/benchmarks/feature/patch_network_layout.py
new file mode 100644
index 000000000..35c391e5f
--- /dev/null
+++ b/benchmarks/feature/patch_network_layout.py
@@ -0,0 +1,198 @@
+# LICENSE HEADER MANAGED BY add-license-header
+#
+# Copyright 2018 Kornia Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Patch descriptor and shape networks: channels-last versus the default memory layout.
+
+Two sections, both on synthetic ``(N, 1, 32, 32)`` patches and untrained weights (``pretrained=False``; the
+weights change neither the layout nor the arithmetic):
+
+- ``layout``: the convolution body of HardNet, HardNet8, SOSNet, TFeat, HyNet, AffNet and OriNet -- the public
+ submodules up to the last convolution -- timed with input and weights in the default layout and again in
+ channels-last. Independent of the layout kornia picks, so it answers which layout a device should use. Each
+ row records the largest absolute difference between the two layouts' outputs.
+- ``forward``: the public forward of each network on ready patches, except AffNet (``LAFAffNetShapeEstimator``),
+ whose forward extracts its own patches from a 480x640 image, plus ``extract_patches_from_pyramid`` followed by
+ ``HardNet`` on that image, as users call them. Run it from the base and the changed checkout to compare the
+ layout kornia picks.
+
+Inputs live on the measured device; no downloads, transfers or file IO are timed.
+"""
+
+from __future__ import annotations
+
+# Imports follow path setup so direct execution always measures this checkout.
+# ruff: noqa: E402
+import argparse
+import copy
+import hashlib
+import sys
+from functools import partial
+from pathlib import Path
+from typing import Callable
+
+REPO = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO))
+sys.path.insert(0, str(REPO / "benchmarks"))
+
+import torch
+from common import kornia_provenance, run_metadata, save_json, setup_run, time_us
+from torch import nn
+
+import kornia
+from kornia import feature
+
+NETWORKS: dict[str, Callable[[], nn.Module]] = {
+ "HardNet": lambda: feature.HardNet(pretrained=False),
+ "HardNet8": lambda: feature.HardNet8(pretrained=False),
+ "SOSNet": lambda: feature.SOSNet(pretrained=False),
+ "TFeat": lambda: feature.TFeat(pretrained=False),
+ "HyNet": lambda: feature.HyNet(pretrained=False),
+ "AffNet": lambda: feature.LAFAffNetShapeEstimator(pretrained=False),
+ "OriNet": lambda: feature.OriNet(pretrained=False),
+}
+
+
+def conv_body(name: str, net: nn.Module) -> nn.Sequential:
+ """The network's layers in call order, up to and including its last convolution, as one plain Sequential.
+
+ Stopping at the last convolution leaves out the trailing BatchNorm that the descriptors keep in float32 for
+ half inputs, and the descriptor normalisation; both act on ``(N, D, 1, 1)`` and cost next to nothing.
+ """
+ if name == "HyNet":
+ layers = [
+ m
+ for block in (net.layer1, net.layer2, net.layer3, net.layer4, net.layer5, net.layer6, net.layer7)
+ for m in block
+ ]
+ else:
+ layers = list(net.layers if name == "SOSNet" else net.features)
+ last = max(i for i, m in enumerate(layers) if isinstance(m, nn.Conv2d))
+ return nn.Sequential(*layers[: last + 1])
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--device", default="cuda")
+ parser.add_argument("--dtype", default="float32", choices=["float32", "float16", "bfloat16", "float64"])
+ parser.add_argument("--threads", type=int, default=4)
+ parser.add_argument("--counts", default="512,2048", help="comma-separated patch counts N")
+ parser.add_argument(
+ "--networks", default=",".join(NETWORKS), help="comma-separated subset of " + ",".join(NETWORKS)
+ )
+ parser.add_argument("--sections", default="layout,forward")
+ parser.add_argument("--cudnn-benchmark", action="store_true", help="set torch.backends.cudnn.benchmark")
+ parser.add_argument("--min-run-time", type=float, default=1.0)
+ parser.add_argument("--json", type=Path)
+ args = parser.parse_args()
+ names = args.networks.split(",")
+ unknown = sorted(set(names) - set(NETWORKS))
+ if unknown:
+ parser.error(f"unknown networks: {unknown}")
+ sections = set(args.sections.split(","))
+ device, dtype, sync = setup_run(args, opencv=False)
+ torch.backends.cudnn.benchmark = args.cudnn_benchmark
+ source, module = kornia_provenance(kornia.__file__)
+ print(f"# kornia source: {source}\n# interpreter: {sys.executable}", flush=True)
+ if module == "outside-checkout":
+ raise RuntimeError("kornia was imported from a different checkout")
+ metadata = {
+ **run_metadata(device),
+ "kornia_module": module,
+ "dtype": args.dtype,
+ "cudnn_benchmark": torch.backends.cudnn.benchmark,
+ "cudnn_allow_tf32": torch.backends.cudnn.allow_tf32,
+ "mkldnn_available": torch.backends.mkldnn.is_available(),
+ "benchmark_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
+ "units": "patches/s",
+ "regime": __doc__,
+ }
+ print(
+ f"# {metadata['platform']} | torch {torch.__version__} | device={device} dtype={args.dtype} "
+ f"threads={args.threads} | oneDNN={metadata['mkldnn_available']} | commit {metadata['git_commit']}"
+ )
+ rows: list[dict[str, object]] = []
+
+ def record(section: str, net: str, count: int, variant: str, fn: Callable[[], object], **extra: object) -> float:
+ median, iqr = time_us(fn, min_run_time=args.min_run_time, sync=sync)
+ rows.append(
+ {
+ "section": section,
+ "network": net,
+ "count": count,
+ "variant": variant,
+ "dtype": args.dtype,
+ "median_us": median,
+ "iqr_us": iqr,
+ "throughput_per_s": count * 1e6 / median,
+ **extra,
+ }
+ )
+ if args.json:
+ save_json(args.json, metadata, rows)
+ return median
+
+ counts = [int(n) for n in args.counts.split(",")]
+ with torch.inference_mode():
+ if "layout" in sections:
+ print("\n# layout: convolution body, ms (median) default | channels-last | speed-up | max|diff|")
+ for name in names:
+ net = NETWORKS[name]().eval().to(device, dtype)
+ body = conv_body(name, net)
+ body_cl = copy.deepcopy(body).to(memory_format=torch.channels_last)
+ for count in counts:
+ x = torch.randn(count, 1, 32, 32, device=device, dtype=dtype)
+ x_cl = x.to(memory_format=torch.channels_last)
+ diff = (body(x).float() - body_cl(x_cl).float()).abs().max().item()
+ default = record("layout", name, count, "default", partial(body, x), max_abs_diff=diff)
+ cl = record("layout", name, count, "channels_last", partial(body_cl, x_cl), max_abs_diff=diff)
+ print(
+ f"{name:9s} N={count:5d} {default / 1e3:9.2f} | {cl / 1e3:9.2f} | x{default / cl:5.2f}"
+ f" | {diff:.1e}",
+ flush=True,
+ )
+ if "forward" in sections:
+ print("\n# forward: public call as kornia runs it, ms (median)")
+ generator = torch.Generator().manual_seed(0)
+ height, width = 480, 640
+ image = torch.rand(1, 1, height, width, generator=generator).to(device, dtype)
+ lafs = {}
+ for count in counts:
+ xy = torch.rand(1, count, 2, generator=generator) * torch.tensor([width - 1.0, height - 1.0])
+ scale = 4.0 + 60.0 * torch.rand(1, count, 1, 1, generator=generator)
+ angle = 360.0 * torch.rand(1, count, 1, generator=generator)
+ lafs[count] = feature.laf_from_center_scale_ori(xy, scale, angle).to(device)
+ for name in names:
+ net = NETWORKS[name]().eval().to(device, dtype)
+ for count in counts:
+ patches = torch.rand(count, 1, 32, 32, device=device, dtype=dtype)
+ call = partial(net, lafs[count], image) if name == "AffNet" else partial(net, patches)
+ median = record("forward", name, count, "kornia", call)
+ print(f"{name:15s} N={count:5d} {median / 1e3:9.2f}", flush=True)
+ hardnet = feature.HardNet(pretrained=False).eval().to(device, dtype)
+ for count in counts:
+
+ def pipeline(laf: torch.Tensor = lafs[count]) -> torch.Tensor:
+ return hardnet(feature.extract_patches_from_pyramid(image, laf, 32).view(-1, 1, 32, 32))
+
+ median = record("forward", "extract+HardNet", count, "kornia", pipeline)
+ print(f"{'extract+HardNet':15s} N={count:5d} {median / 1e3:9.2f}", flush=True)
+ if args.json:
+ save_json(args.json, metadata, rows)
+ print(f"# results written to {args.json}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/changelog.d/6502.added.md b/changelog.d/6502.added.md
new file mode 100644
index 000000000..b1195daf1
--- /dev/null
+++ b/changelog.d/6502.added.md
@@ -0,0 +1,8 @@
+`HardNet`, `HardNet8`, `SOSNet`, `TFeat`, `HyNet`, `LAFAffNetShapeEstimator` and `OriNet` now run their convolutions in
+channels-last memory layout where that is faster: on MPS in float32 and float16, where their convolution bodies are
+1.04-1.38x faster on an Apple M1; on CUDA in float16 and bfloat16, where their forwards on 2048 patches are 1.08-1.44x
+faster on an RTX 4090; and on CPU in float32 when torch is built with oneDNN, where they are 1.2-1.5x faster on an x86
+i7-14700K (`HardNet8` gains only at the default thread count). CUDA float32 keeps the default layout, which is faster
+there. `HardNet` used to convert on every CPU float32 build, including the macOS arm64 wheels, whose NNPACK convolutions
+take only the default layout, so the conversion was a copy that changed nothing. Outputs agree up to rounding: bitwise
+on MPS, and within the dtype's rounding on x86 CPUs and on CUDA.
diff --git a/kornia/feature/_layout.py b/kornia/feature/_layout.py
new file mode 100644
index 000000000..9a8a5cfb5
--- /dev/null
+++ b/kornia/feature/_layout.py
@@ -0,0 +1,57 @@
+# LICENSE HEADER MANAGED BY add-license-header
+#
+# Copyright 2018 Kornia Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import torch
+
+# A defaulted argument rather than a closed-over global: TorchScript cannot read module-level Python values.
+_CPU_HAS_ONEDNN: bool = torch.backends.mkldnn.is_available()
+
+
+def _patch_conv_layout(x: torch.Tensor, cpu_has_onednn: bool = _CPU_HAS_ONEDNN) -> torch.Tensor:
+ r"""Return a patch network's convolution input in the memory format its device convolves fastest in.
+
+ The patch descriptor and shape networks (HardNet, HardNet8, SOSNet, TFeat, HyNet, AffNet, OriNet) call this where
+ their convolutions start. It changes the strides only, never the values; the networks return the same numbers in
+ either layout up to rounding, because a backend may pick other convolution kernels for channels-last (bitwise equal
+ on MPS; within float32 rounding on x86 oneDNN).
+
+ - MPS, float32 and float16: channels-last. The convolution bodies run 1.04-1.38x faster on an Apple M1 with torch
+ 2.14, and extracting 2048 patches and describing them with HardNet 1.07x (float32) and 1.11x (float16).
+ - CUDA, float16 and bfloat16: channels-last. On an RTX 4090 with torch 2.14 the forwards on 2048 patches run
+ 1.08-1.44x faster; on 512 patches the result is mixed (0.83-1.30x across runs, the losses in float16).
+ - CPU float32 with oneDNN: channels-last, which avoids oneDNN's activation reorders: 1.2-1.5x on an x86 i7-14700K,
+ except HardNet8, which gains at the default thread count and is 0.95x on 8 threads. A CPU build without oneDNN,
+ such as the macOS arm64 wheels, convolves with NNPACK, which takes only the default layout, so the input is left
+ alone instead of paying a copy that changes nothing.
+ - Everything else keeps the input's layout. CUDA float32 is among them: under cuDNN's default TF32 convolutions,
+ six of the seven bodies are 0.87-1.00x in channels-last at 2048 patches and more.
+
+ Args:
+ x: convolution input ``(B, C, H, W)``.
+ cpu_has_onednn: whether CPU convolutions run on oneDNN. Leave the default, which asks the torch build.
+
+ Returns:
+ ``x``, as a channels-last copy where that layout is faster.
+ """
+ device = x.device.type
+ if (
+ (device == "cpu" and cpu_has_onednn and x.dtype == torch.float32)
+ or (device == "mps" and x.dtype in (torch.float32, torch.float16))
+ or (device == "cuda" and x.dtype in (torch.float16, torch.bfloat16))
+ ):
+ return x.to(memory_format=torch.channels_last)
+ return x
diff --git a/kornia/feature/_normalization.py b/kornia/feature/_normalization.py
index 77fac236b..78502f5f9 100644
--- a/kornia/feature/_normalization.py
+++ b/kornia/feature/_normalization.py
@@ -21,9 +21,14 @@ from collections.abc import Callable
import torch
from torch import nn
+from kornia.feature._layout import _patch_conv_layout
+
class _DescriptorSequential(nn.Sequential):
- """Keep a descriptor's final BatchNorm statistics and computation in float32 for half inputs."""
+ """Keep a descriptor's final BatchNorm statistics and computation in float32 for half inputs.
+
+ Each convolution also receives its input in the layout ``_patch_conv_layout`` picks for the device.
+ """
def __init__(self, *args: nn.Module | OrderedDict[str, nn.Module]) -> None:
super().__init__(*args)
@@ -61,6 +66,10 @@ class _DescriptorSequential(nn.Sequential):
def forward(self, input: torch.Tensor) -> torch.Tensor:
for module in self:
+ # SOSNet's leading InstanceNorm2d returns the default layout, so the layout is set per convolution; a
+ # convolution's output already has it, which makes the later calls free.
+ if isinstance(module, nn.Conv2d):
+ input = _patch_conv_layout(input)
if getattr(module, "_descriptor_float32_statistics", False) and input.dtype == torch.float16:
# MPS also requires the input and running statistics to have the same dtype.
input = module(input.float()).to(input.dtype)
diff --git a/kornia/feature/affine_shape.py b/kornia/feature/affine_shape.py
index 9a3e59adb..3116f4c61 100644
--- a/kornia/feature/affine_shape.py
+++ b/kornia/feature/affine_shape.py
@@ -26,6 +26,7 @@ from kornia.core.download import hf_url, load_state_dict_from_url
from kornia.filters.kernels import get_gaussian_kernel2d
from kornia.filters.sobel import SpatialGradient
+from ._layout import _patch_conv_layout
from .laf import (
ellipse_to_laf,
extract_patches_from_pyramid,
@@ -348,7 +349,7 @@ class LAFAffNetShapeEstimator(nn.Module):
safe_input, _ = _sanitize_laf(laf)
frame = make_upright(safe_input)
patches: torch.Tensor = extract_patches_from_pyramid(img, frame, PS, True).view(-1, 1, PS, PS)
- xy = self.features(self._normalize_input(patches)).view(-1, 3)
+ xy = self.features(_patch_conv_layout(self._normalize_input(patches))).view(-1, 3)
a1 = torch.cat([1.0 + xy[:, 0].reshape(-1, 1, 1), 0 * xy[:, 0].reshape(-1, 1, 1)], dim=2)
a2 = torch.cat([xy[:, 1].reshape(-1, 1, 1), 1.0 + xy[:, 2].reshape(-1, 1, 1)], dim=2)
new_laf_no_center = torch.cat([a1, a2], dim=1).reshape(B, N, 2, 2)
diff --git a/kornia/feature/hardnet.py b/kornia/feature/hardnet.py
index 697175d4b..9c126d365 100644
--- a/kornia/feature/hardnet.py
+++ b/kornia/feature/hardnet.py
@@ -23,6 +23,7 @@ from torch import nn
from kornia.core.check import KORNIA_CHECK_SHAPE
from kornia.core.download import hf_url, load_state_dict_from_url
from kornia.core.utils import _is_tracing_or_exporting, _l2_normalize, is_mps_tensor_safe
+from kornia.feature._layout import _patch_conv_layout
urls: Dict[str, str | list[str]] = {}
urls["hardnet++"] = [
@@ -141,11 +142,7 @@ class HardNet(nn.Module):
"""
KORNIA_CHECK_SHAPE(input, ["B", "1", "32", "32"])
x_norm: torch.Tensor = self._normalize_input(input)
- # oneDNN's float32 convolutions avoid repeated activation reorders with
- # channels-last input. Keep CUDA's existing layout, which is faster there.
- if input.device.type == "cpu" and input.dtype == torch.float32:
- x_norm = x_norm.to(memory_format=torch.channels_last)
- x_features: torch.Tensor = self.features(x_norm)
+ x_features: torch.Tensor = self.features(_patch_conv_layout(x_norm))
x_out = x_features.flatten(1)
# A constant patch drives the features to zero; the default `eps` is not representable in
# float16, where the normalisation would then return NaN.
@@ -258,7 +255,7 @@ class HardNet8(nn.Module):
"""
KORNIA_CHECK_SHAPE(input, ["B", "1", "32", "32"])
x_norm: torch.Tensor = self._normalize_input(input)
- x_features: torch.Tensor = self.features(x_norm)
+ x_features: torch.Tensor = self.features(_patch_conv_layout(x_norm))
mean: torch.Tensor = torch.jit.annotate(torch.Tensor, self.mean)
components: torch.Tensor = torch.jit.annotate(torch.Tensor, self.components)
x_flat = x_features.flatten(1)
diff --git a/kornia/feature/hynet.py b/kornia/feature/hynet.py
index 4e3dc1041..1bdde8686 100644
--- a/kornia/feature/hynet.py
+++ b/kornia/feature/hynet.py
@@ -22,6 +22,7 @@ from torch import nn
from kornia.core.check import KORNIA_CHECK_SHAPE
from kornia.core.download import hf_url, load_state_dict_from_url
+from kornia.feature._layout import _patch_conv_layout
from kornia.feature._normalization import _DescriptorSequential
urls: Dict[str, str | list[str]] = {}
@@ -328,7 +329,8 @@ class HyNet(nn.Module):
HyNet descriptors with shape ``(B, D)``.
"""
KORNIA_CHECK_SHAPE(x, ["B", "1", "32", "32"])
- x = self.layer1(x)
+ # The leading FilterResponseNorm2d and TLU are pointwise, so the layout reaches the first convolution.
+ x = self.layer1(_patch_conv_layout(x))
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
diff --git a/kornia/feature/orientation.py b/kornia/feature/orientation.py
index 66306be4b..ba966e5ff 100644
--- a/kornia/feature/orientation.py
+++ b/kornia/feature/orientation.py
@@ -27,6 +27,7 @@ from kornia.core.download import hf_url, load_state_dict_from_url
from kornia.filters import SpatialGradient, get_gaussian_discrete_kernel1d, get_gaussian_kernel2d
from kornia.geometry import rad2deg
+from ._layout import _patch_conv_layout
from .laf import extract_patches_from_pyramid, get_laf_orientation, set_laf_orientation
from .siftdesc import _gradient_magnitude_orientation
@@ -284,7 +285,7 @@ class OriNet(nn.Module):
angle in radians: ``(B)``
"""
- xy = self.features(self._normalize_input(patch)).view(-1, 2)
+ xy = self.features(_patch_conv_layout(self._normalize_input(patch))).view(-1, 2)
return torch.atan2(xy[:, 0] + 1e-8, xy[:, 1] + self.eps)
diff --git a/kornia/feature/tfeat.py b/kornia/feature/tfeat.py
index f62d09980..0f927c030 100644
--- a/kornia/feature/tfeat.py
+++ b/kornia/feature/tfeat.py
@@ -22,6 +22,7 @@ from torch import nn
from kornia.core.check import KORNIA_CHECK_SHAPE
from kornia.core.download import hf_url, load_state_dict_from_url
+from kornia.feature._layout import _patch_conv_layout
urls: Dict[str, str | list[str]] = {}
urls["liberty"] = [
@@ -97,6 +98,12 @@ class TFeat(nn.Module):
Descriptor tensor with shape ``(B, 128)``.
"""
KORNIA_CHECK_SHAPE(input, ["B", "1", "32", "32"])
- x = self.features(input)
+ x = input
+ for layer in self.features:
+ # The leading InstanceNorm2d returns the default layout, so the layout is set per convolution; a
+ # convolution's output already has it, which makes the second call free.
+ if isinstance(layer, nn.Conv2d):
+ x = _patch_conv_layout(x)
+ x = layer(x)
x = x.flatten(1)
return self.descr(x)
diff --git a/tests/feature/test_descriptor_normalization.py b/tests/feature/test_descriptor_normalization.py
index 99ddd50d2..4feab317c 100644
--- a/tests/feature/test_descriptor_normalization.py
+++ b/tests/feature/test_descriptor_normalization.py
@@ -20,6 +20,7 @@ import copy
import pytest
import torch
+import kornia.feature._normalization
from kornia.feature import FilterResponseNorm2d, HyNet, LAFDescriptor, SOSNet, get_laf_descriptors
from kornia.feature._normalization import _DescriptorSequential
from kornia.feature.laf import extract_patches_from_pyramid, laf_from_center_scale_ori
@@ -207,7 +208,7 @@ class TestDescriptorNormalization(BaseTester):
@pytest.mark.parametrize("model_cls", [SOSNet, HyNet])
@pytest.mark.parametrize("selection", [slice(None), slice(None, -1), slice(1, None), slice(0, 0), slice(None, 3)])
- def test_sequential_slices(self, device, model_cls, selection):
+ def test_sequential_slices(self, device, model_cls, selection, monkeypatch):
model = model_cls(pretrained=False).to(device).eval()
block = model.layers if model_cls is SOSNet else model.layer7
block[-1].running_var.fill_(1e5)
@@ -220,7 +221,10 @@ class TestDescriptorNormalization(BaseTester):
if isinstance(layer, torch.nn.BatchNorm2d):
assert torch.isfinite(layer.running_var).all()
if model_cls is SOSNet and selection == slice(None, 3):
- # An intermediate BatchNorm keeps ordinary Sequential half-precision behavior.
+ # An intermediate BatchNorm keeps ordinary Sequential half-precision behavior. The plain Sequential leaves
+ # the convolution layout alone, and cuDNN rounds a channels-last float16 convolution differently, so this
+ # bitwise comparison keeps both in the input's layout; test_patch_network_layout covers the layout.
+ monkeypatch.setattr(kornia.feature._normalization, "_patch_conv_layout", lambda x: x)
reference = torch.nn.Sequential(*reference).half()
self.assert_close(sliced(x), reference(x), rtol=0.0, atol=0.0)
else:
diff --git a/tests/feature/test_patch_network_layout.py b/tests/feature/test_patch_network_layout.py
new file mode 100644
index 000000000..bced01264
--- /dev/null
+++ b/tests/feature/test_patch_network_layout.py
@@ -0,0 +1,135 @@
+# LICENSE HEADER MANAGED BY add-license-header
+#
+# Copyright 2018 Kornia Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pytest
+import torch
+from torch import nn
+
+import kornia.feature._normalization
+import kornia.feature.affine_shape
+import kornia.feature.hardnet
+import kornia.feature.hynet
+import kornia.feature.orientation
+import kornia.feature.tfeat
+from kornia.feature import HardNet, HardNet8, HyNet, LAFAffNetShapeEstimator, OriNet, SOSNet, TFeat
+from kornia.feature._layout import _patch_conv_layout
+from kornia.feature.laf import laf_from_center_scale_ori
+
+from testing.base import BaseTester
+
+NETWORKS = {
+ "HardNet": HardNet,
+ "HardNet8": HardNet8,
+ "SOSNet": SOSNet,
+ "TFeat": TFeat,
+ "HyNet": HyNet,
+ "AffNet": LAFAffNetShapeEstimator,
+ "OriNet": OriNet,
+}
+
+# Every module that calls the helper imports it by name.
+CALLERS = [
+ kornia.feature._normalization,
+ kornia.feature.affine_shape,
+ kornia.feature.hardnet,
+ kornia.feature.hynet,
+ kornia.feature.orientation,
+ kornia.feature.tfeat,
+]
+
+
+def _run(name, net, device, dtype):
+ if name == "AffNet":
+ img = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
+ centers = torch.tensor([[[20.0, 24.0], [40.0, 36.0], [31.0, 30.0]]], device=device, dtype=dtype)
+ lafs = laf_from_center_scale_ori(centers, torch.full((1, 3, 1, 1), 6.0, device=device, dtype=dtype))
+ return net(lafs, img)
+ return net(torch.rand(3, 1, 32, 32, device=device, dtype=dtype))
+
+
+class TestPatchConvLayout(BaseTester):
+ def test_cpu_float32_follows_onednn(self):
+ x = torch.rand(2, 4, 3, 3)
+ converted = _patch_conv_layout(x, cpu_has_onednn=True)
+ assert converted.is_contiguous(memory_format=torch.channels_last)
+ assert not converted.is_contiguous()
+ assert _patch_conv_layout(x, cpu_has_onednn=False) is x
+
+ @pytest.mark.parametrize("other", [torch.float64, torch.float16, torch.bfloat16])
+ def test_cpu_other_dtypes_keep_layout(self, other):
+ x = torch.rand(2, 4, 3, 3, dtype=other)
+ assert _patch_conv_layout(x, cpu_has_onednn=True) is x
+
+ def test_values_unchanged(self, device, dtype):
+ x = torch.rand(2, 4, 3, 3, device=device, dtype=dtype)
+ assert torch.equal(_patch_conv_layout(x), x)
+
+ def test_device_decision(self, device, dtype):
+ # The layout measured fastest on each device: MPS float32/float16 (Apple M1), CUDA float16/bfloat16 (RTX 4090;
+ # float32 is slower there under TF32), CPU float32 with oneDNN (x86).
+ expected = {
+ "mps": dtype in (torch.float32, torch.float16),
+ "cuda": dtype in (torch.float16, torch.bfloat16),
+ "cpu": dtype == torch.float32 and torch.backends.mkldnn.is_available(),
+ }.get(device.type, False)
+ x = torch.rand(2, 4, 3, 3, device=device, dtype=dtype)
+ converted = _patch_conv_layout(x)
+ assert (converted is not x) == expected
+ if expected:
+ assert converted.is_contiguous(memory_format=torch.channels_last)
+ assert not converted.is_contiguous()
+
+ @pytest.mark.parametrize("name", NETWORKS)
+ def test_convolutions_receive_device_layout(self, name, device, dtype):
+ torch.manual_seed(0)
+ net = NETWORKS[name](pretrained=False).eval().to(device, dtype)
+ probe = torch.empty(1, 2, 3, 3, device=device, dtype=dtype)
+ channels_last = _patch_conv_layout(probe) is not probe
+ convs = [m for m in net.modules() if isinstance(m, nn.Conv2d)]
+ seen: list[tuple[str, bool, bool]] = []
+
+ def record(module, args):
+ x = args[0]
+ seen.append((repr(module), x.is_contiguous(memory_format=torch.channels_last), x.is_contiguous()))
+
+ hooks = [conv.register_forward_pre_hook(record) for conv in convs]
+ try:
+ with torch.no_grad():
+ _run(name, net, device, dtype)
+ finally:
+ for hook in hooks:
+ hook.remove()
+ assert len(seen) == len(convs)
+ # A single-channel input is contiguous in both layouts, so the first convolution says nothing; every later one
+ # must see the layout the helper picks for the device.
+ for conv, is_channels_last, is_contiguous in seen[1:]:
+ assert (is_channels_last and not is_contiguous) == channels_last, conv
+ assert is_contiguous != channels_last, conv
+
+ @pytest.mark.parametrize("name", NETWORKS)
+ def test_layout_keeps_outputs(self, name, device, dtype, monkeypatch):
+ torch.manual_seed(0)
+ net = NETWORKS[name](pretrained=False).eval().to(device, dtype)
+ torch.manual_seed(1)
+ with torch.no_grad():
+ expected = _run(name, net, device, dtype)
+ for module in CALLERS:
+ monkeypatch.setattr(module, "_patch_conv_layout", lambda x: x)
+ torch.manual_seed(1)
+ with torch.no_grad():
+ actual = _run(name, net, device, dtype)
+ self.assert_close(actual, expected)