π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""End-to-end SIFT descriptor-backend comparison on Oxford graf.
Run from the measured checkout root::
python -m benchmarks.feature.sift_scale_space --seq /data/graf \\
--expected-checkout "$PWD" --device cuda --json graf-sift-scale-space.json
Both public :class:`kornia.feature.SIFTFeatureScaleSpace` backends run their
whole forward pass: DoG detection, orientation, and RootSIFT description. The
patch implementation is constructed without ``descriptor_backend`` so this
harness can also measure a checkout from before that public argument existed.
Timing excludes image I/O, matching, and RANSAC. Matching uses SNN ratio 0.8;
quality is forward ground-truth transfer precision at three pixels and the
existing fixed-seed Graf homography RANSAC corner metric. MPS runs RANSAC on
CPU because MPS batched SVD is unsafe; it remains outside the timed region.
Optional ``--methods opencv --device cpu`` measures native uint8 CPU SIFT with
standard rejection filters plus NumPy RootSIFT; input conversion and conversion
of returned keypoints to Torch tensors are excluded. It shares the grayscale input and matching/RANSAC protocol.
"""
from __future__ import annotations
import argparse
import hashlib
import importlib.util
import json
import math
import sys
import tempfile
import time
from pathlib import Path
import numpy as np
import torch
import kornia
import kornia.feature as KF
from kornia.geometry import RANSAC, transform_points
if importlib.util.find_spec("PIL") is None:
raise SystemExit("SKIP: this benchmark requires the optional Pillow package")
from PIL import Image
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from common import collect_load_metrics, run_metadata, save_json, time_us, versions_line
def homography_quality(
source: torch.Tensor, target: torch.Tensor, ground_truth: torch.Tensor, height: int, width: int
) -> dict:
"""Estimate a homography with the existing graf benchmark RANSAC protocol.
RANSAC consensus is not ground-truth accuracy: a wrong model can have many
inliers, so retain its corner error even when large. Failed estimates have
a null error, never a misleading zero.
"""
result = {"ransac_inliers": 0, "ransac_corner_error_px": None, "ransac_status": "insufficient_matches"}
if source.shape[0] < 4:
return result
ransac = RANSAC("homography", inl_th=2.0, max_iter=10, batch_size=8196, confidence=0.9999, seed=3407)
estimate, mask = ransac(source, target)
inliers = int(mask.sum())
result["ransac_inliers"] = inliers
result["ransac_status"] = "failed_estimate"
if inliers < 4 or not torch.isfinite(estimate).all():
return result
corners = source.new_tensor([[0, 0], [0, height - 1], [width - 1, height - 1], [width - 1, 0]])
# Use explicit homogeneous division to reject a corner at infinity. This
# matches the original graf L1 metric for finite homography projections.
homogeneous = torch.cat([corners, torch.ones_like(corners[:, :1])], dim=-1)
prediction = homogeneous @ estimate.T
reference = homogeneous @ ground_truth.reshape(3, 3).to(source).T
if (prediction[:, 2].abs() <= 1e-8).any() or (reference[:, 2].abs() <= 1e-8).any():
result["ransac_status"] = "nonfinite_projection"
return result
distance = (prediction[:, :2] / prediction[:, 2:] - reference[:, :2] / reference[:, 2:]).abs().sum(-1)
if not torch.isfinite(distance).all():
result["ransac_status"] = "nonfinite_projection"
return result
result["ransac_corner_error_px"] = distance.mean().item()
result["ransac_status"] = "estimated"
return result
def _image_path(seq: Path, index: int) -> Path:
for extension in ("ppm", "png"):
path = seq / f"img{index}.{extension}"
if path.is_file():
return path
raise FileNotFoundError(f"No img{index}.ppm or img{index}.png in {seq}")
def _load_image(path: Path, device: torch.device) -> torch.Tensor:
pixels = np.array(Image.open(path).convert("L"), copy=True)
return torch.from_numpy(pixels).to(device=device, dtype=torch.float32)[None, None].div_(255.0)
def _build(method: str, num_features: int, device: torch.device) -> torch.nn.Module:
# Deliberately leave this keyword off the patch constructor: the same file is
# useful when run from a pre-feature base checkout where the new API is absent.
if method == "patch":
return KF.SIFTFeatureScaleSpace(num_features=num_features, rootsift=True, device=device).eval()
return KF.SIFTFeatureScaleSpace(
num_features=num_features, rootsift=True, device=device, descriptor_backend="pyramid"
).eval()
def _profile_cpu_allocations(run) -> dict[str, int]:
"""Measure tensor allocation churn and peak live bytes in a separate forward.
These are PyTorch CPU allocator events, not RSS or measured DRAM traffic.
Native-library workspaces outside the allocator are not included.
"""
with torch.profiler.profile(activities=[torch.profiler.ProfilerActivity.CPU], profile_memory=True) as profile:
run()
with tempfile.TemporaryDirectory(prefix="kornia-sift-profile-") as directory:
path = Path(directory) / "trace.json"
profile.export_chrome_trace(str(path))
events = json.loads(path.read_text())["traceEvents"]
allocations = sorted(
(event for event in events if event.get("name") == "[memory]" and event["args"]["Device Type"] == 0),
key=lambda event: event["ts"],
)
if not allocations:
raise RuntimeError("CPU profiler did not record allocation events")
live = peak = allocated = 0
for event in allocations:
size = event["args"]["Bytes"]
live += size
peak = max(peak, live)
allocated += max(0, size)
return {"cpu_peak_live_tensor_bytes": peak, "cpu_total_allocated_tensor_bytes": allocated}
@torch.inference_mode()
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--seq", type=Path, required=True)
parser.add_argument("--expected-checkout", type=Path, required=True)
parser.add_argument("--device", choices=("cpu", "mps", "cuda"), default="cpu")
parser.add_argument("--threads", type=int, default=1, help="CPU threads, including inside timed calls")
parser.add_argument("--nf", type=int, default=4096)
parser.add_argument("--methods", nargs="+", choices=("patch", "pyramid", "opencv"), default=["patch", "pyramid"])
parser.add_argument("--min-run-time", type=float, default=1.0)
parser.add_argument("--quality-only", action="store_true")
parser.add_argument("--profile-memory", action="store_true", help="Profile CPU tensor allocations outside timing")
parser.add_argument("--json", type=Path, required=True)
args = parser.parse_args()
if args.threads < 1:
parser.error("--threads must be positive")
if args.profile_memory and (args.device != "cpu" or "opencv" in args.methods):
parser.error("--profile-memory supports PyTorch CPU methods only")
if "opencv" in args.methods:
if args.device != "cpu":
parser.error("OpenCV SIFT is a CPU baseline; run --methods opencv --device cpu")
if importlib.util.find_spec("cv2") is None:
raise SystemExit("SKIP: the OpenCV baseline requires optional opencv-python")
import cv2
cv2.setNumThreads(args.threads)
imported_root = Path(kornia.__file__).resolve().parents[1]
print(f"# interpreter: {sys.executable}\n# kornia: {kornia.__file__}", flush=True)
if imported_root != args.expected_checkout.resolve():
raise RuntimeError(f"Wrong Kornia checkout: {imported_root}")
torch.manual_seed(0)
torch.set_num_threads(args.threads)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
device = torch.device(args.device)
sync = {"cuda": torch.cuda.synchronize, "mps": torch.mps.synchronize}.get(device.type, lambda: None)
ransac_device = torch.device("cpu") if device.type == "mps" else device
paths = [_image_path(args.seq, index) for index in range(1, 7)]
homographies = [args.seq / f"H1to{index}p" for index in range(2, 7)]
images = [_load_image(path, device) for path in paths]
meta = run_metadata(device)
meta.update(
load=collect_load_metrics(),
sequence=args.seq.name,
num_features=args.nf,
seed=0,
rootsift=True,
descriptor_backends=args.methods,
timing=(
"entire SIFTFeatureScaleSpace forward (detection + orientation + description); "
"or native uint8 OpenCV detectAndCompute + RootSIFT; I/O, matching and RANSAC excluded"
),
min_run_time=args.min_run_time,
timer_num_threads=args.threads,
quality_only=args.quality_only,
profile_memory=args.profile_memory,
matching_ratio=0.8,
precision_threshold_px=3.0,
precision_metric="forward GT transfer Euclidean error",
ransac_device=str(ransac_device),
ransac={
"model": "homography",
"seed": 3407,
"inlier_threshold_px": 2.0,
"max_iter": 10,
"batch_size": 8196,
"confidence": 0.9999,
},
ransac_corner_metric="mean L1 corner transfer error against GT in pixels",
image_conversion="Pillow RGB to L",
compile=False,
opencv_regime="uint8 CPU detectAndCompute + NumPy RootSIFT; input/keypoint-to-Torch conversion excluded",
opencv_parameters={"nOctaveLayers": 3, "contrastThreshold": 0.04, "edgeThreshold": 10, "sigma": 1.6},
cudnn_allow_tf32=torch.backends.cudnn.allow_tf32,
cudnn_benchmark=torch.backends.cudnn.benchmark,
input_sha256={path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in paths + homographies},
implementation_sha256={
str(path.relative_to(imported_root)): hashlib.sha256(path.read_bytes()).hexdigest()
for path in (
Path(__file__),
Path(kornia.__file__).parent / "feature" / "integrated.py",
Path(kornia.__file__).parent / "feature" / "scale_space_detector.py",
Path(kornia.__file__).parent / "feature" / "sift" / "scale_space.py",
Path(kornia.__file__).parent / "feature" / "siftdesc.py",
Path(kornia.__file__).parent / "geometry" / "transform" / "pyramid.py",
Path(kornia.__file__).parent / "geometry" / "subpix" / "spatial_soft_argmax.py",
)
if path.is_file()
},
)
print(f"# {meta['git_commit']} | {meta['platform']} | {meta.get('cuda_device', device)}", flush=True)
print(versions_line(meta), flush=True)
rows: list[dict] = []
for method in args.methods:
torch.manual_seed(0)
model = (
cv2.SIFT_create(nfeatures=args.nf, nOctaveLayers=3, contrastThreshold=0.04, edgeThreshold=10, sigma=1.6)
if method == "opencv"
else _build(method, args.nf, device)
)
outputs: list[tuple[torch.Tensor, torch.Tensor]] = []
print(f"# {method}: {model!r}", flush=True)
for index, image in enumerate(images, 1):
if method == "opencv":
pixels = image[0, 0].mul(255).round().to(torch.uint8).numpy()
def run(pixels=pixels, model=model):
keypoints, desc = model.detectAndCompute(pixels, None)
if desc is None:
desc = np.empty((0, 128), dtype=np.float32)
desc = np.sqrt(desc / (np.abs(desc).sum(axis=1, keepdims=True) + 1e-8))
return keypoints, desc
else:
def run(image=image, model=model):
return model(image)
sync()
start = time.perf_counter()
extracted = run()
sync()
warm_seconds = time.perf_counter() - start
if method == "opencv":
keypoints, desc = extracted
points = torch.tensor([kp.pt for kp in keypoints], dtype=torch.float32).reshape(-1, 2)
descriptors = torch.from_numpy(desc)[None]
filled = torch.ones(len(keypoints), dtype=torch.bool)
else:
lafs, _responses, descriptors = extracted
filled = KF.laf_is_filled(lafs)[0]
lafs = lafs[:, filled]
descriptors = descriptors[:, filled]
points = KF.get_laf_center(lafs)[0]
if not torch.isfinite(descriptors).all():
raise RuntimeError(f"Nonfinite descriptors for {method}, image {index}")
median, iqr = float("nan"), float("nan")
if not args.quality_only:
median, iqr = time_us(
run,
min_run_time=max(args.min_run_time, 5.0 * warm_seconds),
sync=sync if device.type == "mps" else None,
num_threads=args.threads,
)
if not math.isfinite(median):
raise RuntimeError(f"Timing failed for {method}, image {index}")
outputs.append((points, descriptors[0]))
row = {
"op": "OpenCVSIFT" if method == "opencv" else "SIFTFeatureScaleSpace",
"backend": method,
"batch": 1,
"image": index,
"height": image.shape[-2],
"width": image.shape[-1],
"dtype": "uint8" if method == "opencv" else "float32",
"features": int(filled.sum()),
"median_us": median,
"iqr_us": iqr,
"throughput_per_s": 1e6 / median if math.isfinite(median) else float("nan"),
}
if args.profile_memory:
row.update(_profile_cpu_allocations(run))
rows.append(row)
if not args.quality_only:
print(
f" img{index}: {median / 1000:.2f} ms +/- {iqr / 1000:.2f}; {int(filled.sum())} features",
flush=True,
)
save_json(args.json, meta, rows)
points1, descriptors1 = outputs[0]
for index, (points2, descriptors2) in enumerate(outputs[1:], 2):
_distance, matches = KF.match_snn(descriptors1, descriptors2, 0.8)
ground_truth = torch.tensor(np.loadtxt(args.seq / f"H1to{index}p"), device=device, dtype=torch.float32)[
None
]
projected = transform_points(ground_truth, points1[None])[0]
error = (projected[matches[:, 0]] - points2[matches[:, 1]]).norm(dim=-1)
correct = int((error <= 3.0).sum())
count = len(matches)
geometry = homography_quality(
points1[matches[:, 0]].to(ransac_device),
points2[matches[:, 1]].to(ransac_device),
ground_truth.to(ransac_device),
*images[0].shape[-2:],
)
rows.append(
{
"op": "matching_quality",
"backend": method,
"batch": 1,
"pair": f"1-{index}",
"median_us": None,
"iqr_us": None,
"throughput_per_s": None,
"matches": count,
"correct_matches": correct,
"precision": correct / count if count else 0.0,
"median_transfer_error_px": error.median().item() if count else None,
**geometry,
}
)
print(
f" 1-{index}: {correct}/{count} correct; RANSAC {geometry['ransac_inliers']} inliers, "
f"corner L1 {geometry['ransac_corner_error_px']}",
flush=True,
)
save_json(args.json, meta, rows)
if __name__ == "__main__":
main()