π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Render the input/output figures shown on the ``docs/source/models/*.rst`` pages.
Unlike ``generate_examples.py`` this script is *not* run by the Sphinx build: it downloads several
pretrained checkpoints (RT-DETR, SAM ViT-B, SOLD2, DeFMO, ...) and runs them on the CPU, which is too
slow and too download-heavy for a docs build. Run it once when a model or its API changes and commit
the resulting figures under ``docs/source/_static/img/models/``::
python docs/generate_model_examples.py # all models
python docs/generate_model_examples.py loftr sold2 # a subset
Each ``figure_<name>`` function runs the model the same way the corresponding page's "Run it" snippet
does (same builder, weights, inputs and prompts) and then draws the result; keep the two in sync when
either changes.
"""
from __future__ import annotations
import sys
from pathlib import Path
from urllib.request import urlopen
import matplotlib as mpl
mpl.use("Agg") # select the headless backend before pyplot is imported
import matplotlib.pyplot as plt
import torch
import kornia as K
from kornia.image import tensor_to_image
from kornia.io import get_sample_images
torch.manual_seed(0)
OUT = Path(__file__).absolute().parent / "source/_static/img/models"
DATA = "https://raw.githubusercontent.com/kornia/data/main/"
KNCHURCH = "https://github.com/kornia/data_test/raw/4ffed08df3d82af85aa9012d3104f19ca4b62604/knchurch_disk.safetensors"
IMAGENET_CLASSES = "https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt"
FIGURES: dict = {}
def figure(fn):
FIGURES[fn.__name__.removeprefix("figure_")] = fn
return fn
def sample(name: str, size: tuple[int, int] | None = None) -> torch.Tensor:
"""One ``(1, 3, H, W)`` float image in [0, 1] from the kornia data repository."""
img = get_sample_images(paths=[DATA + name], resize=size, as_list=True)[0]
return img[None]
def knchurch() -> tuple[torch.Tensor, torch.Tensor]:
d = K.core.load_safetensors(K.core.download_file_from_url(KNCHURCH, validate=K.core.check_safetensors))
return d["img1"], d["img2"]
def imagenet_classes() -> list[str]:
with urlopen(IMAGENET_CLASSES, timeout=60) as resp:
return resp.read().decode().strip().splitlines()
def show(ax, img: torch.Tensor, title: str = "") -> None:
ax.imshow(tensor_to_image(img[0] if img.dim() == 4 else img), cmap="gray" if img.shape[-3] == 1 else None)
ax.set_title(title, fontsize=10)
ax.set_axis_off()
def save(fig, name: str) -> None:
OUT.mkdir(parents=True, exist_ok=True)
fig.savefig(OUT / f"{name}.jpg", dpi=110, bbox_inches="tight", pil_kwargs={"quality": 88})
plt.close(fig)
print(f"wrote {OUT / name}.jpg")
def topk_bars(ax, logits: torch.Tensor, names: list[str], title: str) -> None:
probs = logits.softmax(-1)[0]
top = probs.topk(5)
ax.barh(range(5)[::-1], top.values.tolist(), color="#4c72b0")
ax.set_yticks(range(5)[::-1], [names[i] for i in top.indices.tolist()], fontsize=9)
ax.set_xlim(0, 1)
ax.set_xlabel("softmax probability")
ax.set_title(title, fontsize=10)
# --------------------------------------------------------------------------- object detection
@figure
def figure_rt_detr() -> None:
from kornia.contrib.object_detection import RTDETRDetectorBuilder
image = sample("delorean.png")
detector = RTDETRDetectorBuilder.build("rtdetr_r18vd") # runs at the 640 px the weights were trained for
detections = detector(image) # one (D, 6) tensor per image: class id, score, x, y, w, h
fig, axs = plt.subplots(1, 2, figsize=(9, 3.4))
show(axs[0], image, "input")
show(axs[1], image, f"RT-DETR (r18vd): {detections[0].shape[0]} detections above 0.3")
for cls, score, x, y, w, h in detections[0].tolist():
axs[1].add_patch(mpl.patches.Rectangle((x, y), w, h, fill=False, color="lime", lw=1.5))
axs[1].text(x, y - 3, f"class {int(cls)} Β· {score:.2f}", color="lime", fontsize=8, fontweight="bold")
save(fig, "rt_detr")
@figure
def figure_yunet() -> None:
from kornia.contrib import FaceDetector, FaceDetectorResult, FaceKeypoint
image = sample("crowd.jpg", (587, 900))
detector = FaceDetector() # YuNet expects pixel values in [0, 255]
faces = [FaceDetectorResult(f) for f in detector(image * 255.0)[0]] # one result per detected face
fig, axs = plt.subplots(1, 2, figsize=(11, 3.8))
show(axs[0], image, "input")
show(axs[1], image, f"YuNet: {len(faces)} faces with score > 0.3 (boxes + 5 landmarks)")
for f in faces:
x1, y1, x2, y2 = f.xmin.item(), f.ymin.item(), f.xmax.item(), f.ymax.item()
axs[1].add_patch(mpl.patches.Rectangle((x1, y1), x2 - x1, y2 - y1, fill=False, color="lime", lw=1.0))
for kp in FaceKeypoint:
x, y = f.get_keypoint(kp).tolist()
axs[1].plot(x, y, ".", color="yellow", ms=2.5)
fig.tight_layout()
save(fig, "yunet")
# --------------------------------------------------------------------------- segmentation
def _sam_figure(model_type: str, name: str, title: str) -> None:
from kornia.contrib.visual_prompter import VisualPrompter
from kornia.geometry.boxes import Boxes
from kornia.geometry.keypoints import Keypoints
from kornia.models.sam import SamConfig
image = sample("simba.png")[0] # (3, H, W)
prompter = VisualPrompter(SamConfig(model_type, pretrained=True))
prompter.set_image(image) # encode once, query many times
keypoints = Keypoints(torch.tensor([[[300.0, 90.0]]])) # (K, N, 2): K prompts of N (x, y) points; on the eye
labels = torch.tensor([[1]]) # 1 = foreground, 0 = background
box = Boxes.from_tensor(torch.tensor([[[180.0, 20.0, 380.0, 240.0]]]), mode="xyxy") # around the head
point_pred = prompter.predict(keypoints=keypoints, keypoints_labels=labels) # 3 candidate masks
box_pred = prompter.predict(boxes=box, multimask_output=False) # 1 mask
best = point_pred.scores.argmax() # pick the candidate with the highest predicted IoU
fig, axs = plt.subplots(1, 3, figsize=(12, 3.0))
show(axs[0], image, "input + prompts (star: point, box)")
axs[0].plot(300, 90, "*", color="lime", ms=12, mec="black")
axs[0].add_patch(mpl.patches.Rectangle((180, 20), 200, 220, fill=False, color="cyan", lw=1.5))
for ax, pred, m, t in ((axs[1], point_pred, best, "point"), (axs[2], box_pred, 0, "box")):
show(ax, image, f"{title}, {t} prompt (predicted IoU {pred.scores[0, m]:.2f})")
mask = pred.binary_masks[0, m].float()
ax.imshow(tensor_to_image(torch.stack([mask * 0, mask, mask * 0.6, mask * 0.55])), interpolation="nearest")
fig.tight_layout()
save(fig, name)
@figure
def figure_segment_anything() -> None:
_sam_figure("vit_b", "segment_anything", "SAM ViT-B")
@figure
def figure_mobile_sam() -> None:
_sam_figure("mobile_sam", "mobile_sam", "MobileSAM")
@figure
def figure_efficient_vit() -> None:
from kornia.models.efficient_vit import EfficientViT, EfficientViTConfig
image = sample("panda.jpg", (224, 224))
model = EfficientViT.from_config(EfficientViTConfig.from_pretrained("b1", 224)).eval()
# the backbones were trained on images normalized with the ImageNet mean and standard deviation
x = K.enhance.normalize(image, torch.tensor([0.485, 0.456, 0.406]), torch.tensor([0.229, 0.224, 0.225]))
with torch.no_grad():
feats = model(x) # dict of feature maps: "input", "stage0" ... "stage_final"
stages = [k for k in feats if k.startswith("stage")]
fig, axs = plt.subplots(1, 1 + len(stages), figsize=(2.3 * (1 + len(stages)), 2.6))
show(axs[0], image, "input 224x224")
for ax, k in zip(axs[1:], stages):
f = feats[k][0]
ax.imshow(f.abs().mean(0), cmap="magma")
ax.set_title(f"{k}: {tuple(f.shape)}", fontsize=8)
ax.set_axis_off()
fig.suptitle("EfficientViT-B1 pyramid, mean |activation| per stage", fontsize=10)
save(fig, "efficient_vit")
# --------------------------------------------------------------------------- classification backbones
@figure
def figure_tiny_vit() -> None:
from kornia.models.tiny_vit import TinyViT
image = sample("panda.jpg", (224, 224))
model = TinyViT.from_config("5m", pretrained=True).eval() # ImageNet-1k weights
# the checkpoints were trained on images normalized with the ImageNet mean and standard deviation
x = K.enhance.normalize(image, torch.tensor([0.485, 0.456, 0.406]), torch.tensor([0.229, 0.224, 0.225]))
with torch.no_grad():
logits = model(x) # (1, 1000)
fig, axs = plt.subplots(1, 2, figsize=(9, 3), width_ratios=[1, 1.6])
show(axs[0], image, "input 224x224")
topk_bars(axs[1], logits, imagenet_classes(), "TinyViT-5M top-5 ImageNet classes")
fig.tight_layout()
save(fig, "tiny_vit")
@figure
def figure_vit() -> None:
from kornia.models.vit import VisionTransformer
image = sample("panda.jpg", (224, 224))
vit = VisionTransformer.from_config("vit_b/16", pretrained=True).eval() # AugReg ImageNet-21k weights
# the AugReg weights were trained on images normalized with mean = std = 0.5, i.e. in [-1, 1]
x = K.enhance.normalize(image, torch.tensor([0.5, 0.5, 0.5]), torch.tensor([0.5, 0.5, 0.5]))
with torch.no_grad():
tokens = vit(x) # (1, 197, 768): class token + 14x14 patch tokens
fig, axs = plt.subplots(1, 3, figsize=(11, 3.2), width_ratios=[1, 1, 1.3])
show(axs[0], image, "input 224x224 β 14x14 patches of 16 px")
for p in range(0, 224, 16):
axs[0].axhline(p - 0.5, color="white", lw=0.4, alpha=0.7)
axs[0].axvline(p - 0.5, color="white", lw=0.4, alpha=0.7)
axs[1].imshow(tokens[0, 1:].norm(dim=-1).view(14, 14), cmap="viridis")
axs[1].set_title("βpatch tokenβ on the 14x14 grid", fontsize=10)
axs[1].set_axis_off()
axs[2].imshow(tokens[0, :, :96].T, aspect="auto", cmap="coolwarm")
axs[2].set_title("output (1, 197, 768): first 96 dims of each token", fontsize=10)
axs[2].set_xlabel("token (0 = class token)")
axs[2].set_ylabel("embedding dim")
fig.suptitle("VisionTransformer vit_b/16 with the pretrained AugReg ImageNet-21k weights", fontsize=10)
fig.tight_layout()
save(fig, "vit")
@figure
def figure_vit_mobile() -> None:
from kornia.models.vit_mobile import MobileViT
image = sample("panda.jpg", (256, 256))
mvit = MobileViT(mode="xxs").eval() # random init: no pretrained weights
with torch.no_grad():
feats = mvit(image) # (1, 320, 8, 8) feature map
fig, axs = plt.subplots(1, 2, figsize=(7.5, 3.6))
show(axs[0], image, "input 256x256")
axs[1].imshow(feats[0].abs().mean(0), cmap="magma")
axs[1].set_title(f"output {tuple(feats.shape)}: mean |activation|", fontsize=10)
axs[1].set_axis_off()
fig.suptitle("MobileViT-XXS with random initialisation (stride-32 feature map)", fontsize=10)
fig.tight_layout()
save(fig, "vit_mobile")
# --------------------------------------------------------------------------- local features and matching
def _draw_matches(ax, img1, img2, pts1, pts2, title: str, max_lines: int = 150) -> None:
w = img1.shape[-1]
show(ax, torch.cat([img1, img2], dim=-1), title)
idx = torch.randperm(len(pts1))[:max_lines]
for (x1, y1), (x2, y2) in zip(pts1[idx].tolist(), pts2[idx].tolist()):
ax.plot([x1, x2 + w], [y1, y2], color="lime", lw=0.4, alpha=0.8)
ax.plot(pts1[:, 0], pts1[:, 1], ".", color="yellow", ms=1.2)
ax.plot(pts2[:, 0] + w, pts2[:, 1], ".", color="yellow", ms=1.2)
@figure
def figure_loftr() -> None:
from kornia.feature import LoFTR
img1, img2 = knchurch() # two (1, 3, H, W) views of the same scene
matcher = LoFTR(pretrained="outdoor").eval()
with torch.no_grad():
out = matcher({"image0": K.color.rgb_to_grayscale(img1), "image1": K.color.rgb_to_grayscale(img2)})
pts1, pts2, conf = out["keypoints0"], out["keypoints1"], out["confidence"]
fig, ax = plt.subplots(figsize=(9, 5))
_draw_matches(
ax, img1, img2, pts1, pts2, f"LoFTR (outdoor): {len(pts1)} matches, mean confidence {conf.mean():.2f}"
)
save(fig, "loftr")
@figure
def figure_sold2() -> None:
from kornia.feature import SOLD2
img1, img2 = knchurch()
gray = K.color.rgb_to_grayscale(torch.cat([img1, img2]))
sold2 = SOLD2(pretrained=True).eval()
with torch.no_grad():
out = sold2(gray) # line_segments: list of (N, 2, 2) in (y, x); dense_desc: (B, 128, H/4, W/4)
lines1, lines2 = out["line_segments"]
matches = sold2.match(
lines1, lines2, out["dense_desc"][0:1], out["dense_desc"][1:2]
) # (N1,) index into lines2, -1 = none
valid = matches != -1
m1, m2 = lines1[valid], lines2[matches[valid]]
fig, axs = plt.subplots(1, 3, figsize=(12, 4.2), width_ratios=[1, 1, 2])
for ax, img, lines, t in (
(axs[0], img1, lines1, "SOLD2 lines, image 1"),
(axs[1], img2, lines2, "SOLD2 lines, image 2"),
):
show(ax, img, f"{t} ({len(lines)})")
for (y1, x1), (y2, x2) in lines.tolist():
ax.plot([x1, x2], [y1, y2], color="lime", lw=0.8)
w = img1.shape[-1]
show(axs[2], torch.cat([img1, img2], dim=-1), f"{int(valid.sum())} matched lines")
colors = plt.cm.hsv(torch.linspace(0, 1, len(m1)).numpy())
for c, ((ay, ax_), (by, bx)), ((cy, cx), (dy, dx)) in zip(colors, m1.tolist(), m2.tolist()):
axs[2].plot([ax_, bx], [ay, by], color=c, lw=1.2)
axs[2].plot([cx + w, dx + w], [cy, dy], color=c, lw=1.2)
save(fig, "sold2")
@figure
def figure_hardnet() -> None:
from kornia.feature import HardNet, KeyNetDetector, extract_patches_from_pyramid, match_snn
img1, img2 = knchurch()
gray1, gray2 = K.color.rgb_to_grayscale(img1), K.color.rgb_to_grayscale(img2)
detector = KeyNetDetector(pretrained=True, num_features=1500).eval()
hardnet = HardNet(pretrained=True).eval()
with torch.no_grad():
lafs1, _ = detector(gray1)
lafs2, _ = detector(gray2)
patches1 = extract_patches_from_pyramid(gray1, lafs1)[0] # (N, 1, 32, 32)
patches2 = extract_patches_from_pyramid(gray2, lafs2)[0]
desc1, desc2 = hardnet(patches1), hardnet(patches2) # (N, 128), L2-normalised
_, idx = match_snn(desc1, desc2, th=0.9)
pts1 = K.feature.get_laf_center(lafs1)[0, idx[:, 0]]
pts2 = K.feature.get_laf_center(lafs2)[0, idx[:, 1]]
fig = plt.figure(figsize=(12, 4.2))
gs = fig.add_gridspec(2, 2, width_ratios=[1, 1.9])
ax = fig.add_subplot(gs[0, 0])
ax.imshow(tensor_to_image(torch.cat(list(patches1[:8]), dim=-1)), cmap="gray")
ax.set_title("8 of the 1500 KeyNet patches (32x32)", fontsize=10)
ax.set_axis_off()
ax = fig.add_subplot(gs[1, 0])
ax.imshow(desc1[:8], aspect="auto", cmap="coolwarm")
ax.set_title("their HardNet descriptors (8 x 128)", fontsize=10)
ax.set_yticks([])
ax = fig.add_subplot(gs[:, 1])
_draw_matches(ax, img1, img2, pts1, pts2, f"KeyNet + HardNet + SNN matching: {len(idx)} matches")
fig.tight_layout()
save(fig, "hardnet")
@figure
def figure_affnet() -> None:
from kornia.feature import KeyNetDetector, LAFAffNetShapeEstimator, laf_to_boundary_points
img1, _ = knchurch()
gray = K.color.rgb_to_grayscale(img1)
detector = KeyNetDetector(pretrained=True, num_features=32).eval()
affnet = LAFAffNetShapeEstimator(pretrained=True).eval()
with torch.no_grad():
lafs, _ = detector(gray) # (1, N, 2, 3) circular local affine frames
lafs_affine = affnet(lafs, gray) # (1, N, 2, 3) affine-covariant frames
h, w = gray.shape[-2:]
fig, axs = plt.subplots(1, 2, figsize=(7, 4.4))
panels = ((axs[0], lafs, "KeyNet LAFs (isotropic)"), (axs[1], lafs_affine, "after AffNet (affine shape)"))
for ax, frames, t in panels:
show(ax, img1, t)
for pts in laf_to_boundary_points(frames)[0].tolist():
xs, ys = zip(*pts)
ax.plot(xs, ys, color="lime", lw=1.0)
ax.set_xlim(0, w)
ax.set_ylim(h, 0)
save(fig, "affnet")
# --------------------------------------------------------------------------- edges and enhance
@figure
def figure_dexined() -> None:
from kornia.contrib.edge_detection import EdgeDetectorBuilder
image = sample("girona.png")
detector = EdgeDetectorBuilder.build("dexined", pretrained=True, image_size=352) # resize + normalise + sigmoid
with torch.no_grad():
edges = detector(image)[0] # list of (1, 1, H, W) edge probabilities in [0, 1], one per input image
fig, axs = plt.subplots(1, 2, figsize=(9, 3.2))
show(axs[0], image, "input")
show(axs[1], 1 - edges, "DexiNed edge probability (dark = edge)")
save(fig, "dexined")
@figure
def figure_defmo() -> None:
from kornia.feature import DeFMO
# Synthesise a striped ball flying across a static background: the blurred frame is the
# mean of its 24 positions, exactly the image-formation model DeFMO inverts.
background = sample("girona.png", (240, 320))
h, w = background.shape[-2:]
yy, xx = torch.meshgrid(torch.arange(h), torch.arange(w), indexing="ij")
orange = torch.tensor([1.0, 0.6, 0.1])[None, :, None, None]
navy = torch.tensor([0.05, 0.05, 0.25])[None, :, None, None]
frames = []
for t in torch.linspace(0, 1, 24):
cx, cy = 80 + 160 * t, 150 - 60 * t
alpha = (((xx - cx) ** 2 + (yy - cy) ** 2) < 42**2).float()[None, None]
stripe = (((xx - cx) + (yy - cy)).abs() < 10).float()[None, None]
frames.append(alpha * (orange * (1 - stripe) + navy * stripe) + (1 - alpha) * background)
blurred = torch.stack(frames).mean(0) # (1, 3, H, W)
defmo = DeFMO(pretrained=True).eval()
with torch.no_grad():
subframes = defmo(torch.cat([blurred, background], dim=1)) # (1, 24, 4, H, W) RGBA sub-frames
fig, axs = plt.subplots(2, 4, figsize=(12, 4.8))
show(axs[0, 0], blurred, "input: motion-blurred frame")
show(axs[0, 1], background, "input: background estimate")
show(axs[0, 2], frames[0], "ground truth, first sub-frame")
show(axs[0, 3], frames[-1], "ground truth, last sub-frame")
for ax, i in zip(axs[1], (0, 8, 16, 23)):
rgba = subframes[0, i]
composed = rgba[3:] * rgba[:3] + (1 - rgba[3:]) * background[0] # alpha-blend onto the background
show(ax, composed, f"DeFMO sub-frame {i + 1}/24")
fig.tight_layout()
save(fig, "defmo")
if __name__ == "__main__":
wanted = sys.argv[1:] or list(FIGURES)
unknown = [n for n in wanted if n not in FIGURES]
if unknown:
raise SystemExit(f"unknown figure(s) {unknown}; choose from {sorted(FIGURES)}")
for n in wanted:
print(f"== {n}")
FIGURES[n]()