π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import annotations
import importlib
import io
import math
import os
from pathlib import Path
from typing import Optional
from urllib.request import urlopen
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn.functional as F
from matplotlib.axes import Axes
from PIL import Image
import kornia as K
mpl.use("Agg")
def download_tutorials_examples(download_infos: dict[str, str], directory: Path):
URL_BASE = "https://raw.githubusercontent.com/kornia/tutorials/master/"
for filename, path in download_infos.items():
url = URL_BASE + path
# perform request; S310 is safe here because the URL is the fixed https:// base above
with urlopen(url, timeout=60) as resp: # noqa: S310
response = resp.read()
path = directory / filename
with open(path, "wb") as fp:
fp.write(response)
def read_img_from_url(url: str, resize_to: Optional[tuple[int, int]] = None, **resize_kwargs) -> torch.Tensor:
# perform request; S310 is safe here because every caller passes a fixed https:// constant
with urlopen(url, timeout=60) as resp: # noqa: S310
response = resp.read()
# decode to an HxWx3 RGB array; ``convert`` also drops the alpha of the RGBA sources.
# ``np.array`` rather than ``asarray``: the tensor must not alias PIL's read-only buffer.
img: np.ndarray = np.array(Image.open(io.BytesIO(response)).convert("RGB"))
# convert the image to a tensor
img_t: torch.Tensor = K.image.image_to_tensor(img, keepdim=False) # 1xCxHXW
img_t = img_t.float() / 255.0
if resize_to is None:
img_t = K.geometry.resize(img_t, 184, **resize_kwargs)
else:
img_t = K.geometry.resize(img_t, resize_to, **resize_kwargs)
return img_t
def write_png(path: Path, img: np.ndarray) -> None:
"""Save an ``HxW`` grayscale, ``HxWx3`` RGB or ``HxWx4`` RGBA ``uint8`` array as a PNG."""
Image.fromarray(np.ascontiguousarray(img)).save(path)
def draw_lafs(ax: Axes, lafs: torch.Tensor, color: str) -> None:
"""Outline the local affine frames of the first image of ``lafs`` on an existing axes.
Reproduces ``kornia_moons.viz.visualize_LAF(..., draw_ori=False)``: the regions are drawn at
half the LAF scale, and the leading boundary point -- the LAF centre, which ``draw_ori`` would
join to the outline -- is dropped.
"""
xs, ys = K.feature.laf.get_laf_pts_to_draw(K.feature.scale_laf(lafs, 0.5), 0)
ax.plot(xs[1:], ys[1:], color, linewidth=1)
def transparent_pad(src: torch.Tensor, shape: tuple[int, int]) -> torch.Tensor:
"""Apply a transparent pad to src (centerized) to match with shape (h, w)"""
w_pad = abs(int(src.shape[-1] - shape[-1]) // 2)
h_pad = abs(int(src.shape[-2] - shape[-2]) // 2)
return F.pad(K.color.rgb_to_rgba(src, 1.0), (w_pad, w_pad, h_pad, h_pad), "constant", 0.0)
def draw_bbox_kpts(imgs: torch.Tensor, bboxes: torch.Tensor, keypoints: torch.Tensor) -> torch.Tensor:
rectangle = torch.zeros(imgs.shape[0], imgs.shape[1], 4)
rectangle[..., 0] = bboxes[..., 0] # x1
rectangle[..., 1] = bboxes[..., 1] # y1
rectangle[..., 2] = bboxes[..., 0] + bboxes[..., -2] # x2
rectangle[..., 3] = bboxes[..., 1] + bboxes[..., -1] # y2
color = torch.tensor([1, 0, 0]).repeat(imgs.shape[0], imgs.shape[1], 1)
# keep every box four-sided: a side outside the image is not drawn by draw_rectangle
rectangle[..., 0::2] = rectangle[..., 0::2].clamp(0, imgs.shape[-1] - 1)
rectangle[..., 1::2] = rectangle[..., 1::2].clamp(0, imgs.shape[-2] - 1)
imgs_draw = K.image.draw_rectangle(imgs, rectangle, color=color)
rectangle2 = torch.zeros(imgs.shape[0], imgs.shape[1], 4)
for n in range(keypoints.shape[-2]):
rectangle2[..., n, 0] = keypoints[..., n, 0] - 2
rectangle2[..., n, 1] = keypoints[..., n, 1] - 2
rectangle2[..., n, 2] = keypoints[..., n, 0] + 2
rectangle2[..., n, 3] = keypoints[..., n, 1] + 2
color = torch.tensor([0, 0, 1]).repeat(imgs.shape[0], imgs.shape[1], 1)
rectangle2[..., 0::2] = rectangle2[..., 0::2].clamp(0, imgs.shape[-1] - 1)
rectangle2[..., 1::2] = rectangle2[..., 1::2].clamp(0, imgs.shape[-2] - 1)
return K.image.draw_rectangle(imgs_draw, rectangle2, color=color, fill=True)
def main():
# Download the tutorial examples for the main docs
# Note: Training API examples (image_classifier, object_detection, semantic_segmentation) removed
# as they depend on kornia.x which has been removed
URLS_TUTORIALS_EXAMPLES = {}
OUTPUT_PATH_SCRIPTS = Path(__file__).absolute().parent / "source/_static/scripts/"
os.makedirs(OUTPUT_PATH_SCRIPTS, exist_ok=True)
print(f"Downloading script examples from kornia/tutorials. Saving into the path {OUTPUT_PATH_SCRIPTS}.")
download_tutorials_examples(URLS_TUTORIALS_EXAMPLES, OUTPUT_PATH_SCRIPTS)
# load the images
BASE_IMAGE_URL1: str = "https://raw.githubusercontent.com/kornia/data/main/panda.jpg" # augmentation
BASE_IMAGE_URL2: str = "https://raw.githubusercontent.com/kornia/data/main/simba.png" # color
BASE_IMAGE_URL3: str = "https://raw.githubusercontent.com/kornia/data/main/girona.png" # enhance
BASE_IMAGE_URL4: str = "https://raw.githubusercontent.com/kornia/data/main/baby_giraffe.png" # morphology
BASE_IMAGE_URL5: str = "https://raw.githubusercontent.com/kornia/data/main/persistencia_memoria.jpg" # filters
BASE_IMAGE_URL6: str = "https://raw.githubusercontent.com/kornia/data/main/delorean.png" # geometry
hash1 = "4ffed08df3d82af85aa9012d3104f19ca4b62604"
BASE_IMAGEOUTDOOR_URL7: str = ( # image matching
f"https://github.com/kornia/data_test/raw/{hash1}/knchurch_disk.safetensors"
)
BASE_IMAGEOUTDOOR_URL8: str = ( # Response functions
"https://github.com/kornia/data/raw/main/kornia_banner_pixie.png"
)
MASK_IMAGE_URL2: str = "https://raw.githubusercontent.com/kornia/data/main/simba_mask.png"
BASE_IMAGE_URL_CROWD: str = "https://raw.githubusercontent.com/kornia/data/main/crowd.jpg" # face detection
OUTPUT_PATH = Path(__file__).absolute().parent / "source/_static/img"
os.makedirs(OUTPUT_PATH, exist_ok=True)
print(f"Pointing images to path {OUTPUT_PATH}.")
img1 = read_img_from_url(BASE_IMAGE_URL1)
img2 = read_img_from_url(BASE_IMAGE_URL2, img1.shape[-2:])
img3 = read_img_from_url(BASE_IMAGE_URL3, img1.shape[-2:])
img4 = read_img_from_url(BASE_IMAGE_URL4)
img5 = read_img_from_url(BASE_IMAGE_URL5, (234, 320))
img6 = read_img_from_url(BASE_IMAGE_URL6)
img_kornia = read_img_from_url(BASE_IMAGEOUTDOOR_URL8)
# Read the masks as (B, H, W)
mask2 = read_img_from_url(MASK_IMAGE_URL2, img1.shape[-2:], interpolation="nearest")
mask2 = mask2.median(dim=1)[0]
# TODO: make this more generic for modules out of kornia.augmentation
# Dictionary containing the transforms to generate the sample images:
# Key: Name of the transform class.
# Value: (parameters, num_samples, seed)
mod = importlib.import_module("kornia.augmentation")
augmentations_list: dict = {
"CenterCrop": ((184, 184), 1, 2018),
"ColorJiggle": ((0.3, 0.3, 0.3, 0.3), 2, 2018),
"ColorJitter": ((0.3, 0.3, 0.3, 0.3), 2, 2022),
"PadTo": (((220, 450),), 1, 2022),
"RandomAffine": (((-15.0, 20.0), (0.1, 0.1), (0.7, 1.3), 20), 2, 2019),
"RandomBoxBlur": (((7, 7),), 1, 2020),
"RandomBrightness": (((0.0, 1.0),), 2, 2022),
"RandomContrast": (((0.0, 1.0),), 2, 2022),
"RandomCrop": ((img1.shape[-2:], (50, 50)), 2, 2020),
"RandomChannelDropout": ((), 1, 2020),
"RandomChannelShuffle": ((), 1, 2020),
"RandomElasticTransform": (((63, 63), (32, 32), (2.0, 2.0)), 2, 2018),
"RandomEqualize": ((), 1, 2020),
"RandomErasing": (((0.2, 0.4), (0.3, 1 / 0.3)), 2, 2017),
"RandomFisheye": ((torch.tensor([-0.3, 0.3]), torch.tensor([-0.3, 0.3]), torch.tensor([0.9, 1.0])), 2, 2020),
"RandomGamma": (((0.0, 1.0),), 2, 2022),
"RandomGaussianBlur": (((3, 3), (0.1, 2.0)), 1, 2020),
"RandomGaussianIllumination": (((0.5, 0.5), (0.5, 0.5), (0.5, 0.5), (-1.0, 1.0)), 2, 2021),
"RandomGaussianNoise": ((0.0, 0.05), 1, 2020),
"RandomGrayscale": ((), 1, 2020),
"RandomHue": (((-0.5, 0.5),), 2, 2022),
"RandomHorizontalFlip": ((), 1, 2020),
"RandomInvert": ((), 1, 2020),
"RandomJPEG": (((1.0, 5.0),), 1, 2024),
"RandomLinearCornerIllumination": (((0.5, 0.5), (-1.0, 1.0)), 2, 2021),
"RandomLinearIllumination": (((0.5, 0.5), (-1.0, 1.0)), 2, 2021),
"RandomMedianBlur": (((3, 3),), 1, 2023),
"RandomMotionBlur": ((7, 35.0, 0.5), 2, 2020),
"RandomPerspective": ((0.2,), 2, 2020),
"RandomPlanckianJitter": ((), 2, 2022),
"RandomPlasmaShadow": (((0.2, 0.5),), 2, 2022),
"RandomPlasmaBrightness": ((), 2, 2022),
"RandomPlasmaContrast": ((), 2, 2022),
"RandomPosterize": (((1, 4),), 2, 2016),
"RandomResizedCrop": ((img1.shape[-2:], (1.0, 2.0), (1.0, 2.0)), 2, 2020),
"RandomRotation": ((45.0,), 2, 2019),
"RandomSaltAndPepperNoise": (((0.05, 0.5), (0.1, 0.7)), 2, 2024),
"RandomSaturation": (((0.5, 5.0),), 2, 2022),
"RandomSharpness": ((16.0,), 1, 2019),
"RandomSolarize": ((0.2, 0.2), 2, 2019),
"RandomVerticalFlip": ((), 1, 2020),
"RandomThinPlateSpline": ((), 1, 2020),
"RandomJigsaw": ((), 2, 2020),
}
# ITERATE OVER THE TRANSFORMS
for aug_name, (args, num_samples, seed) in augmentations_list.items():
img_in = img1.repeat(num_samples, 1, 1, 1)
# dynamically create the class instance
cls = getattr(mod, aug_name)
try:
aug = cls(*args, p=1.0)
except TypeError:
aug = cls(*args)
# set seed
torch.manual_seed(seed)
if aug_name == "RandomJigsaw": # make sure the image is dividable
img_in = K.geometry.resize(img_in, (1020, 500))
elif aug_name == "RandomJPEG":
img_in = img_in[..., :176, :]
# apply the augmentation to the image and concat
out = aug(img_in)
# save ori image to concatenate into the out image
ori = img_in[0]
if aug_name == "CenterCrop":
# Convert to RGBA, and center the output image with transparent pad
out = transparent_pad(out, tuple(img1[-2:].shape))
ori = K.color.rgb_to_rgba(ori, 1.0) # To match the dims
elif aug_name == "PadTo":
# Convert to RGBA, and center the original image with transparent pad
ori = transparent_pad(img_in[0], tuple(out.shape[-2:]))
out = K.color.rgb_to_rgba(out, 1.0) # To match the dims
out = torch.cat([ori, *(out[i] for i in range(out.size(0)))], dim=-1)
# save the output image
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{aug_name}.png", out_np)
sig = f"{aug_name}({', '.join([str(a) for a in args])}, p=1.0)"
print(f"Generated image example for {aug_name}. {sig}")
mix_augmentations_list = {"RandomMixUpV2": ((), 2, 20), "RandomCutMixV2": ((), 2, 2019), "PatchMix": ((), 2, 2024)}
# ITERATE OVER THE TRANSFORMS
for aug_name, (args, _, seed) in mix_augmentations_list.items():
img_in = torch.cat([img1, img2])
# dynamically create the class instance
cls = getattr(mod, aug_name)
aug = cls(*args, p=1.0)
# set seed
torch.manual_seed(seed)
# apply the augmentation to the image and concat
# PatchMix returns (B, C, H, W); index [0] to get (C, H, W) for cat
if aug_name == "PatchMix":
img_aug = aug(img_in)[0]
else:
img_aug, _ = aug(img_in, torch.tensor([0, 1]))
output = torch.cat([img_in[0], img_in[1], img_aug], dim=-1)
# save the output image
out_np = K.image.tensor_to_image((output * 255.0).byte())
write_png(OUTPUT_PATH / f"{aug_name}.png", out_np)
sig = f"{aug_name}({', '.join([str(a) for a in args])}, p=1.0)"
print(f"Generated image example for {aug_name}. {sig}")
mask_augmentations_list = {"RandomTransplantation": (([0],), 0)}
# ITERATE OVER THE TRANSFORMS
for aug_name, (args, seed) in mask_augmentations_list.items():
img_in = torch.cat([img1, img2])
mask_in = torch.cat([torch.zeros_like(mask2), mask2])
# dynamically create the class instance
cls = getattr(mod, aug_name)
aug = cls(*args, p=1.0)
# set seed
torch.manual_seed(seed)
# apply the augmentation to the image and concat
img_aug, _ = aug(img_in, mask_in)
output = torch.cat([img_in[0], img_in[1], img_aug[0]], dim=-1)
# save the output image
out_np = K.image.tensor_to_image((output * 255.0).byte())
write_png(OUTPUT_PATH / f"{aug_name}.png", out_np)
sig = f"{aug_name}({', '.join([str(a) for a in args])}, p=1.0)"
print(f"Generated image example for {aug_name}. {sig}")
# Containers
aug_container_list = {
"AugmentationSequential": (
{
"args": (
K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=1.0),
K.augmentation.RandomAffine(360, [0.1, 0.1], [0.7, 1.2], [30.0, 50.0], p=1.0),
K.augmentation.RandomPerspective(0.5, p=1.0),
),
"data_keys": ["input", "bbox_xywh", "keypoints"],
},
(
torch.tensor([[[125, 5, 115, 80]]], dtype=torch.float32), # bbox
torch.tensor([[[166, 42], [197, 42]]], dtype=torch.float32), # keypoints
),
2,
2023,
),
"PatchSequential": (
{
"args": (
K.augmentation.ColorJitter(0.2, 0.1, 0.1, 0.1, p=1),
K.augmentation.RandomAffine(10, [0.1, 0.2], [0.7, 1.2], [0.0, 15.0], p=1),
K.augmentation.RandomPerspective(0.3, p=1),
K.augmentation.RandomSolarize(0.01, 0.05, p=0.6),
),
"grid_size": (2, 2),
"same_on_batch": False,
"patchwise_apply": False,
},
(),
2,
2023,
),
}
for aug_name, (args, labels, num_samples, seed) in aug_container_list.items():
img_in = img1.repeat(num_samples, 1, 1, 1)
cls = getattr(mod, aug_name)
tfms = args.pop("args")
augs = cls(*tfms, **args)
# set seed
torch.manual_seed(seed)
if aug_name == "PatchSequential":
out = augs(img_in)
inp = img_in
else:
labels = (labels[0].expand(num_samples, -1, -1), labels[1].expand(num_samples, -1, -1))
out = augs(img_in, *labels)
out = draw_bbox_kpts(out[0], out[1].int(), out[2].int())
inp = draw_bbox_kpts(img_in, labels[0].int(), labels[1].int())
output = torch.cat([inp[0], *(out[i] for i in range(out.size(0)))], dim=-1)
# save the output image
out_np = K.image.tensor_to_image((output * 255.0).byte())
write_png(OUTPUT_PATH / f"{aug_name}.png", out_np)
sig = f"{aug_name}({', '.join([str(a) for a in args])}, p=1.0)"
print(f"Generated image example for {aug_name}. {sig}")
# ------------------------------------------------------------------------------------
mod = importlib.import_module("kornia.color")
color_transforms_list: dict = {
"grayscale_to_rgb": ((), 3),
"rgb_to_bgr": ((), 1),
"rgb_to_grayscale": ((), 1),
"rgb_to_hsv": ((), 1),
"rgb_to_hls": ((), 1),
"rgb_to_luv": ((), 1),
"rgb_to_lab": ((), 1),
# "rgb_to_rgba": ((1.,), 1),
"rgb_to_xyz": ((), 1),
"rgb_to_ycbcr": ((), 1),
"rgb_to_yuv": ((), 1),
"rgb_to_linear_rgb": ((), 1),
"apply_colormap": ((K.color.ColorMap("autumn", 256),), 1),
"ApplyColorMap": ((K.color.ColorMap("winter", 256),), 1),
}
# ITERATE OVER THE TRANSFORMS
for fn_name, (args, _) in color_transforms_list.items():
# import function and apply
fn = getattr(mod, fn_name)
if fn_name == "grayscale_to_rgb":
out = fn(K.color.rgb_to_grayscale(img2), *args)
elif fn_name == "apply_colormap":
gray_image = (K.color.rgb_to_grayscale(img2) * 255.0).round()
out = fn(gray_image, *args)
elif fn_name == "ApplyColorMap":
gray_image = (K.color.rgb_to_grayscale(img2) * 255.0).round()
out = fn(*args)(gray_image)
else:
out = fn(img2, *args)
# perform normalization to visualize
if fn_name == "rgb_to_lab":
out = out[:, :1] / 100.0
elif fn_name == "rgb_to_hsv":
out[:, :1] = out[:, :1] / 2 * math.pi
elif fn_name == "rgb_to_luv":
out = out[:, :1] / 116.0
# repeat channels for grayscale
if out.shape[1] != 3:
out = out.repeat(1, 3, 1, 1)
# save the output image
if fn_name in ("grayscale_to_rgb", "apply_colormap", "ApplyColorMap"):
out = torch.cat(
[K.color.rgb_to_grayscale(img2[0]).repeat(3, 1, 1), *(out[i] for i in range(out.size(0)))], dim=-1
)
else:
out = torch.cat([img2[0], *(out[i] for i in range(out.size(0)))], dim=-1)
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{fn_name}.png", out_np)
sig = f"{fn_name}({', '.join([str(a) for a in args])})"
print(f"Generated image example for {fn_name}. {sig}")
# korna.color.colormap
colormaps_list = {"AUTUMN": (256,)}
bar_img_gray = torch.arange(0, 256).repeat(1, 40, 1) # 1x1x40x256
bar_img = K.color.grayscale_to_rgb(bar_img_gray)
# ITERATE OVER THE COLORMAPS
for colormap_name, args in colormaps_list.items():
cm = K.color.ColorMap(base=colormap_name, num_colors=args[0])
out = K.color.apply_colormap(bar_img_gray, cm)[0]
out = torch.cat([bar_img, out], dim=-1)
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{colormap_name}.png", out_np)
sig = f"{colormap_name}({', '.join([str(a) for a in args])})"
print(f"Generated image example for {colormap_name}. {sig}")
# Plot for all ColorMaps (ColorMapType.png)
height_image = 40
num_colors = 256
num_columns = 3
# 1 x height_image x num_colors
input_tensor = torch.arange(start=0, end=num_colors, step=1).unsqueeze(0).repeat(1, height_image, 1)
input_tensor = input_tensor.to("cpu").to(torch.float32)
# Get colormap list
colormap_list = K.color.ColorMapType.list()
num_colormaps = len(colormap_list)
# Calculate number of rows needed
num_rows = (num_colormaps + num_columns - 1) // num_columns
# Create figure and axis objects
fig, axes = plt.subplots(num_rows, num_columns, figsize=(12, 8))
for i, ax in enumerate(axes.flat):
if i < num_colormaps:
cmap = K.color.ColorMap(base=colormap_list[i], num_colors=num_colors)
res = K.color.ApplyColorMap(colormap=cmap)(input_tensor)[0]
ax.imshow(res.permute(1, 2, 0).numpy())
ax.set_title(colormap_list[i], fontsize=12)
ax.axis("off")
else:
fig.delaxes(ax)
fig.tight_layout()
fig.savefig(os.path.join(OUTPUT_PATH, "ColorMapType.png"), dpi=300)
# korna.enhance module
mod = importlib.import_module("kornia.enhance")
transforms: dict = {
"adjust_brightness": ((torch.tensor([0.25, 0.5]),), 2),
"adjust_contrast": ((torch.tensor([0.65, 0.5]),), 2),
"adjust_gamma": ((torch.tensor([0.85, 0.75]), 2.0), 2),
"adjust_hue": ((torch.tensor([-math.pi / 4, math.pi / 4]),), 2),
"adjust_saturation": ((torch.tensor([1.0, 2.0]),), 2),
"solarize": ((torch.tensor([0.8, 0.5]), torch.tensor([-0.25, 0.25])), 2),
"posterize": ((torch.tensor([4, 2]),), 2),
"sharpness": ((torch.tensor([1.0, 2.5]),), 2),
"equalize": ((), 1),
"invert": ((), 1),
"equalize_clahe": ((), 1),
"add_weighted": ((0.75, 0.25, 2.0), 1),
"jpeg_codec_differentiable": ((torch.tensor([50]),), 1),
}
# ITERATE OVER THE TRANSFORMS
for fn_name, (args, num_samples) in transforms.items():
img_in = img3.repeat(num_samples, 1, 1, 1)
if fn_name == "jpeg_codec_differentiable":
img_in = img_in[..., :176, :]
args_in = (img_in, args[0], img2, args[1], args[2]) if fn_name == "add_weighted" else (img_in, *args)
# import function and apply
fn = getattr(mod, fn_name)
out = fn(*args_in)
# save the output image
out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1)
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{fn_name}.png", out_np)
sig = f"{fn_name}({', '.join([str(a) for a in args])})"
print(f"Generated image example for {fn_name}. {sig}")
# korna.morphology module
mod = importlib.import_module("kornia.morphology")
kernel = torch.tensor([[0, 1, 0], [1, 1, 1], [0, 1, 0]])
transforms: dict = {
"dilation": ((kernel,), 1),
"erosion": ((kernel,), 1),
"opening": ((kernel,), 1),
"closing": ((kernel,), 1),
"gradient": ((kernel,), 1),
"top_hat": ((kernel,), 1),
"bottom_hat": ((kernel,), 1),
}
# ITERATE OVER THE TRANSFORMS
for fn_name, (args, num_samples) in transforms.items():
img_in = img4.repeat(num_samples, 1, 1, 1)
args_in = (img_in, *args)
# import function and apply
# import pdb;pdb.set_trace()
fn = getattr(mod, fn_name)
out = fn(*args_in)
# save the output image
out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1)
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{fn_name}.png", out_np)
sig = f"{fn_name}({', '.join([str(a) for a in args])})"
print(f"Generated image example for {fn_name}. {sig}")
# korna.filters module
mod = importlib.import_module("kornia.filters")
kernel = torch.tensor([[0, 1, 0], [1, 1, 1], [0, 1, 0]])
transforms: dict = {
"bilateral_blur": (((11, 11), 0.1, (3, 3)), 1),
"joint_bilateral_blur": (((11, 11), 0.1, (3, 3)), 1),
"box_blur": (((5, 5),), 1),
"median_blur": (((5, 5),), 1),
"gaussian_blur2d": (((5, 5), (1.5, 1.5)), 1),
"guided_blur": (((5, 5), 0.01), 1),
"motion_blur": ((5, 90.0, 1.0), 1),
"max_blur_pool2d": ((5,), 1),
"blur_pool2d": ((5,), 1),
"unsharp_mask": (((5, 5), (1.5, 1.5)), 1),
"laplacian": ((5,), 1),
"sobel": ((), 1),
"spatial_gradient": ((), 1),
"canny": ((), 1),
}
# ITERATE OVER THE TRANSFORMS
for fn_name, (args, num_samples) in transforms.items():
img_in = img5.repeat(num_samples, 1, 1, 1)
if fn_name == "joint_bilateral_blur":
guide = K.geometry.resize(img2.repeat(num_samples, 1, 1, 1), img_in.shape[-2:])
args_in = (img_in, guide, *args)
elif fn_name == "guided_blur":
args_in = (img_in, img_in, *args)
else:
args_in = (img_in, *args)
# import function and apply
fn = getattr(mod, fn_name)
out = fn(*args_in)
if fn_name in ("max_blur_pool2d", "blur_pool2d"):
out = K.geometry.resize(out, img_in.shape[-2:])
if fn_name == "canny":
out = out[1].repeat(1, 3, 1, 1)
if isinstance(out, torch.Tensor):
out = out.clamp(min=0.0, max=1.0)
if fn_name in ("laplacian", "sobel", "spatial_gradient", "canny"):
out = K.enhance.normalize_min_max(out)
if fn_name == "spatial_gradient":
out = out.permute(2, 1, 0, 3, 4).squeeze()
if fn_name == "joint_bilateral_blur":
out = torch.cat([args_in[1], out], dim=-1)
# save the output image
out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1)
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{fn_name}.png", out_np)
sig = f"{fn_name}({', '.join([str(a) for a in args])})"
print(f"Generated image example for {fn_name}. {sig}")
# kornia.filters.in_range
mod = importlib.import_module("kornia.filters")
transforms: dict = {
"in_range": (((0.314, 0.2, 0.2), (0.47, 1.0, 1.0), True), 1),
}
# ITERATE OVER THE TRANSFORMS
for fn_name, (args, _) in transforms.items():
img_hsv = K.color.rgb_to_hsv(img1)
h, s, v = torch.split(img_hsv, split_size_or_sections=1, dim=1)
h = h / (2 * torch.pi)
img_hsv = torch.cat((h, s, v), dim=1)
args_in = (img_hsv, *args)
fn = getattr(mod, fn_name)
mask = fn(*args_in)
filtered = img1 * mask
mask = mask.repeat(1, img1.shape[1], 1, 1)
# save the output image
out = torch.cat([img1[0], mask[0], filtered[0]], dim=-1)
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{fn_name}.png", out_np)
sig = f"{fn_name}({', '.join([str(a) for a in args])})"
print(f"Generated image example for {fn_name}. {sig}")
# korna.geometry.transform module
mod = importlib.import_module("kornia.geometry.transform")
h, w = img6.shape[-2:]
def _get_tps_args():
src = torch.tensor([[[-1.0, -1.0], [-1.0, 1.0], [1.0, -1.0], [1.0, -1.0], [0.0, 0.0]]]).repeat(2, 1, 1) # Bx5x2
dst = src + torch.distributions.Uniform(-0.2, 0.2).rsample((2, 5, 2))
kernel, affine = K.geometry.transform.get_tps_transform(dst, src)
return src, kernel, affine
transforms: dict = {
"warp_affine": (
(
K.geometry.transform.get_affine_matrix2d(
translations=torch.zeros(2, 2),
center=(torch.tensor([w, h]) / 2).repeat(2, 1),
scale=torch.distributions.Uniform(0.5, 1.5).rsample((2, 2)),
angle=torch.tensor([-25.0, 25.0]),
)[:, :2, :3],
(h, w),
),
2,
),
"remap": (
(
*(K.geometry.create_meshgrid(h, w, normalized_coordinates=True) - 0.25).unbind(-1),
"bilinear",
"zeros",
True,
True,
),
1,
),
"warp_image_tps": ((_get_tps_args()), 2),
"rotate": ((torch.tensor([-15.0, 25.0]),), 2),
"translate": ((torch.tensor([[10.0, -15], [50.0, -25.0]]),), 2),
"scale": ((torch.tensor([[0.5, 1.25], [1.0, 1.5]]),), 2),
"shear": ((torch.tensor([[0.1, -0.2], [-0.2, 0.1]]),), 2),
"rot180": ((), 1),
"hflip": ((), 1),
"vflip": ((), 1),
"resize": (((120, 220),), 1),
"rescale": ((0.5,), 1),
"elastic_transform2d": ((torch.rand(1, 2, h, w) * 2 - 1, (63, 63), (32, 32), (4.0, 4.0)), 1),
"pyrdown": ((), 1),
"pyrup": ((), 1),
"build_pyramid": ((3,), 1),
"build_laplacian_pyramid": ((3,), 1),
}
# ITERATE OVER THE TRANSFORMS
for fn_name, (args, num_samples) in transforms.items():
img_in = img6.repeat(num_samples, 1, 1, 1)
args_in = (img_in, *args)
# import function and apply
fn = getattr(mod, fn_name)
out = fn(*args_in)
if fn_name in ("resize", "rescale", "pyrdown", "pyrup"):
h_new, w_new = out.shape[-2:]
out = torch.nn.functional.pad(out, (0, (w - w_new), 0, (h - h_new)))
if fn_name == "build_pyramid":
_out = []
for pyr in out[1:]:
h_new, w_new = pyr.shape[-2:]
out_tmp = torch.nn.functional.pad(pyr, (0, (w - w_new), 0, (h - h_new)))
_out.append(out_tmp)
out = torch.cat(_out)
if fn_name == "build_laplacian_pyramid":
h_, w_ = out[0].shape[-2:]
_out = [out[0]]
for pyr in out[1:]:
h_new, w_new = pyr.shape[-2:]
out_tmp = torch.nn.functional.pad(pyr, (0, (w_ - w_new), 0, (h_ - h_new)))
print(out_tmp.size())
_out.append(out_tmp)
out = torch.cat(_out)
# save the output image
if fn_name != "build_laplacian_pyramid":
out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1)
else:
out = torch.cat([*(out[i] for i in range(out.size(0)))], dim=-1)
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{fn_name}.png", out_np)
sig = f"{fn_name}({', '.join([str(a) for a in args])})"
print(f"Generated image example for {fn_name}. {sig}")
# Image Matching and local features
img_matching_path = K.core.download_file_from_url(BASE_IMAGEOUTDOOR_URL7, validate=K.core.check_safetensors)
img_outdoor = K.core.load_safetensors(img_matching_path)["img2"]
print("Generating local feature detections ")
disk = K.feature.DISK.from_pretrained("depth")
with torch.no_grad():
disk_feat = disk(img_outdoor)[0]
xy = disk_feat.keypoints.detach().cpu().numpy()
cur_fname = str(OUTPUT_PATH / "disk_outdoor_depth.jpg")
plt.figure()
plt.imshow(K.tensor_to_image(img_outdoor))
plt.scatter(xy[:, 0], xy[:, 1], 3, color="lime")
plt.title('DISK("depth") keypoints')
plt.savefig(cur_fname)
plt.close()
kah = K.feature.KeyNetAffNetHardNet(512).eval()
with torch.no_grad():
lafs, _resps, _descs = kah(K.color.rgb_to_grayscale(img_outdoor))
fig1, ax = plt.subplots(1, 1)
ax.imshow(K.image.tensor_to_image(img_outdoor[0]))
draw_lafs(ax, lafs, color="lime")
ax.set_title("KeyNetAffNet 512 LAFs")
cur_fname = str(OUTPUT_PATH / "keynet_affnet.jpg")
fig1.savefig(cur_fname)
plt.close()
keynet = K.feature.KeyNetDetector(True, 512).eval()
with torch.no_grad():
lafs, _resps = keynet(K.color.rgb_to_grayscale(img_outdoor))
xy = K.feature.get_laf_center(lafs).detach().cpu().numpy().reshape(-1, 2)
cur_fname = str(OUTPUT_PATH / "keynet.jpg")
plt.figure()
plt.imshow(K.tensor_to_image(img_outdoor))
plt.scatter(xy[:, 0], xy[:, 1], 3, color="lime")
plt.title("KeyNet 512 keypoints")
plt.savefig(cur_fname)
plt.close()
# korna.feature module
mod = importlib.import_module("kornia.feature")
responses: list = [
"harris_response",
"gftt_response",
"hessian_response",
"dog_response_single",
"KeyNet",
"DISK",
"ALIKED",
"XFeat",
"SIFTDescriptorFromPyramid",
"SIFTFeatureScaleSpace",
]
# ITERATE OVER THE TRANSFORMS
for fn_name in responses:
# import function and apply
img_in = K.color.rgb_to_grayscale(img_kornia)
if fn_name == "KeyNet":
fn = K.feature.KeyNet(True)
out = fn(img_in)
elif fn_name == "DISK":
fn = K.feature.DISK.from_pretrained("depth")
h, w = img_outdoor.shape[2:]
pd_h = 32 - h % 32 if h % 32 > 0 else 0
pd_w = 32 - w % 32 if w % 32 > 0 else 0
img_in = torch.nn.functional.pad(img_outdoor, (0, pd_w, 0, pd_h), value=0.0)
out, _ = fn.heatmap_and_dense_descriptors(img_in)
out = K.color.grayscale_to_rgb(out)
elif fn_name == "ALIKED":
fn = K.feature.ALIKED.from_pretrained()
with torch.no_grad():
_, out = fn.extract_dense_map(img_outdoor)
out = K.color.grayscale_to_rgb(out)
img_in = img_outdoor
elif fn_name == "XFeat":
fn = K.feature.XFeat.from_pretrained()
with torch.no_grad():
preprocessed, _, _ = fn._preprocess_tensor(img_outdoor)
_, K1, _ = fn.net(preprocessed)
out = K.feature.XFeat._get_kpts_heatmap(K1)
# upsample heatmap from preprocessed size back to original image size
out = torch.nn.functional.interpolate(out, img_outdoor.shape[-2:], mode="bilinear", align_corners=False)
out = K.color.grayscale_to_rgb(out)
img_in = img_outdoor
elif fn_name == "SIFTFeatureScaleSpace":
fn = K.feature.SIFTFeatureScaleSpace(num_features=2048, descriptor_backend="pyramid").eval()
with torch.no_grad():
lafs, scores, _ = fn(img_in)
h, w = img_in.shape[-2:]
valid = K.feature.laf_is_filled(lafs)[0]
xy = K.feature.get_laf_center(lafs)[0, valid].round().long()
heatmap = img_in.new_zeros(1, 1, h * w)
index = xy[:, 1].clamp(0, h - 1) * w + xy[:, 0].clamp(0, w - 1)
heatmap[0, 0].scatter_add_(0, index, scores[0, valid].abs())
out = K.color.grayscale_to_rgb(F.max_pool2d(heatmap.view(1, 1, h, w), 5, stride=1, padding=2))
img_in = K.color.grayscale_to_rgb(img_in)
elif fn_name == "SIFTDescriptorFromPyramid":
# A regular LAF grid turns the sparse descriptor into a compact score map:
# show distance to the centre descriptor on the same RGB canvas as other examples.
h, w = img_kornia.shape[-2:]
ys = torch.arange(12, h - 12, 12, device=img_kornia.device, dtype=img_kornia.dtype)
xs = torch.arange(12, w - 12, 12, device=img_kornia.device, dtype=img_kornia.dtype)
yy, xx = torch.meshgrid(ys, xs, indexing="ij")
xy = torch.stack([xx, yy], dim=-1).reshape(1, -1, 2)
scale = torch.full((*xy.shape[:2], 1, 1), 6.0, device=xy.device, dtype=xy.dtype)
lafs = K.feature.laf_from_center_scale_ori(xy, scale)
desc = K.feature.SIFTDescriptorFromPyramid().eval()(K.color.rgb_to_grayscale(img_kornia), lafs)
centre = desc[:, desc.shape[1] // 2 : desc.shape[1] // 2 + 1]
out = (desc - centre).square().sum(-1).sqrt().reshape(1, 1, yy.shape[0], yy.shape[1])
out = F.interpolate(out, (h, w), mode="bilinear", align_corners=False)
out = K.color.grayscale_to_rgb(out)
img_in = img_kornia
else:
fn = getattr(mod, fn_name)
out = fn(img_in)
out = out - out.min()
out = out / (1e-8 + out.max())
# save the output image
out = torch.cat([img_in[0], *(out[i] for i in range(out.size(0)))], dim=-1)
out_np = K.image.tensor_to_image((out * 255.0).byte())
write_png(OUTPUT_PATH / f"{fn_name}.png", out_np)
sig = f"{fn_name}({', '.join([str(a) for a in args])})"
print(f"Generated image example for response function {fn_name}")
# Face detection with the pretrained YuNet model, for the landing-page "models" card. The
# checkpoint is ~400 KB, so this stays cheap in the docs build. ``img_crowd`` is RGB like the
# other tensors here, so the detector and the drawn boxes share the same tensor.
img_crowd = read_img_from_url(BASE_IMAGE_URL_CROWD, (480, 736))
face_detector = K.contrib.FaceDetector().eval()
with torch.no_grad():
detections = face_detector(img_crowd * 255.0)[0]
faces = [K.contrib.FaceDetectorResult(d) for d in detections]
confident = [torch.cat([f.top_left, f.bottom_right]) for f in faces if f.score > 0.7]
out = img_crowd.clone()
if confident:
boxes = torch.stack(confident)[None]
for pad in (0, 1, 2): # three passes give a 3 px outline
padded = boxes + torch.tensor([-pad, -pad, pad, pad])
# keep the outline four-sided for a face at the border
padded[..., 0::2] = padded[..., 0::2].clamp(0, out.shape[-1] - 1)
padded[..., 1::2] = padded[..., 1::2].clamp(0, out.shape[-2] - 1)
out = K.image.draw_rectangle(out, padded, color=torch.tensor([0.4, 1.0, 0.2]))
write_png(OUTPUT_PATH / "face_detection.png", K.image.tensor_to_image((out[0] * 255.0).byte()))
print(f"Generated image example for FaceDetector. {len(confident)} faces")
if __name__ == "__main__":
main()