Code-first video & image, rendered natively on WebGPU.
/**
* Lens remap: turns a cube map drawn around the eye (`cubeFaceCameras`) into
* the picture of a lens. Every output pixel becomes a ray (perspective,
* equirectangular or equidistant fisheye), optionally seen by an observer
* moving near the speed of light: the ray is aberrated back to the direction
* it left the scene, and its colour Doppler-shifted (`relativistic-doppler`).
*
* The cube is mipmapped first, so a lens or a fast observer that squeezes
* much of the scene into few pixels filters it instead of aliasing.
*/
import type { LensProjection } from "../math3d/cube-lens.js";
import {
DOPPLER_LUT_MAX,
DOPPLER_LUT_SIZE,
dopplerLuminanceGain,
dopplerLut,
lorentzFactor,
} from "../math3d/relativistic-doppler.js";
import type { Vec3 } from "../math3d/vector3.js";
import { BufferPool } from "../renderer2d/buffer-pool.js";
/** An observer moving at `speed` (a fraction of c) through a scene at rest. */
export interface RelativisticObserver {
/** β = v / c, 0 ≤ β < 1. */
speed: number;
/** Unit direction of motion in world space. */
direction: Vec3;
/** Shift colours along the spectrum (blue ahead, red behind). */
doppler: boolean;
/** Brighten ahead and dim behind (relativistic beaming). */
beaming: boolean;
/**
* Exposure for the shifted light: stops, or "auto" to meter the centre of
* the view, so what the camera looks at keeps its brightness at rest.
*/
exposure: number | "auto";
}
export interface LensRemapOptions {
projection: LensProjection;
/** Degrees; see `CubeLensOptions.fov`. */
fov: number;
zoom: number;
/** The camera's frame: screen right, screen up, forward (world space). */
basis: { right: Vec3; up: Vec3; forward: Vec3 };
observer?: RelativisticObserver | null;
}
const PROJECTION_INDEX: Record<LensProjection, number> = {
perspective: 0,
equirectangular: 1,
fisheye: 2,
};
/** How the observer changes colour: none, spectral table, or brightness only. */
const ColorMode = { None: 0, Spectral: 1, Beaming: 2 } as const;
const mipWgsl = /* wgsl */ `
@group(0) @binding(0) var samp : sampler;
@group(0) @binding(1) var src : texture_2d<f32>;
struct VsOut {
@builtin(position) pos : vec4<f32>,
@location(0) uv : vec2<f32>,
};
@vertex
fn vs(@builtin(vertex_index) i : u32) -> VsOut {
let p = vec2<f32>(f32((i << 1u) & 2u), f32(i & 2u));
var o : VsOut;
o.pos = vec4<f32>(p * 2.0 - 1.0, 0.0, 1.0);
o.uv = vec2<f32>(p.x, 1.0 - p.y);
return o;
}
// One bilinear tap between four texels of the level above: a 2x2 box.
@fragment
fn fs(in : VsOut) -> @location(0) vec4<f32> {
return textureSampleLevel(src, samp, in.uv, 0.0);
}
`;
const remapWgsl = /* wgsl */ `
struct Lens {
right : vec4<f32>, // xyz screen right, w projection (0 persp, 1 equirect, 2 fisheye)
up : vec4<f32>, // xyz screen up, w scale (persp: tan(fov/2)/zoom, fisheye: fov/2/zoom)
forward : vec4<f32>, // xyz forward, w aspect (width / height)
motion : vec4<f32>, // xyz direction of motion, w β
params : vec4<f32>, // x color mode, y exposure gain, z γ, w ln(max D)
};
@group(0) @binding(0) var<uniform> lens : Lens;
@group(0) @binding(1) var samp : sampler;
@group(0) @binding(2) var colorCube : texture_cube<f32>;
@group(0) @binding(3) var hdrCube : texture_cube<f32>;
@group(0) @binding(4) var<storage, read> dopplerLut : array<vec4<f32>>;
const PI = 3.14159265358979;
const LUT_SIZE = ${DOPPLER_LUT_SIZE}.0;
struct VsOut {
@builtin(position) pos : vec4<f32>,
@location(0) ndc : vec2<f32>,
};
@vertex
fn vs(@builtin(vertex_index) i : u32) -> VsOut {
let p = vec2<f32>(f32((i << 1u) & 2u), f32(i & 2u));
var o : VsOut;
o.pos = vec4<f32>(p * 2.0 - 1.0, 0.0, 1.0);
o.ndc = p * 2.0 - 1.0;
return o;
}
struct Ray {
dir : vec3<f32>,
inside : f32,
};
// The ray an output pixel sees, in world space.
fn lensRay(ndc : vec2<f32>) -> Ray {
let r = lens.right.xyz;
let u = lens.up.xyz;
let f = lens.forward.xyz;
let scale = lens.up.w;
let aspect = lens.forward.w;
let projection = lens.right.w;
var out : Ray;
out.inside = 1.0;
if (projection < 0.5) {
out.dir = normalize(f + ndc.x * aspect * scale * r + ndc.y * scale * u);
} else if (projection < 1.5) {
let lon = ndc.x * PI;
let lat = ndc.y * 0.5 * PI;
out.dir = cos(lat) * (sin(lon) * r + cos(lon) * f) + sin(lat) * u;
} else {
let q = vec2<f32>(ndc.x * aspect, ndc.y);
let rho = length(q);
let theta = rho * scale;
let side = select(vec2<f32>(0.0), q / rho, rho > 1e-6);
out.dir = cos(theta) * f + sin(theta) * (side.x * r + side.y * u);
out.inside = select(0.0, 1.0, theta <= PI);
}
return out;
}
struct Seen {
dir : vec3<f32>,
doppler : f32,
};
// Aberration: where a ray seen by the moving observer left the scene.
fn aberrate(observed : vec3<f32>) -> Seen {
let beta = lens.motion.w;
let n = lens.motion.xyz;
let c = clamp(dot(observed, n), -1.0, 1.0);
let cs = (c - beta) / (1.0 - beta * c);
let side = observed - c * n;
let len = length(side);
// Exactly ahead or behind, any perpendicular will do: sin θ is 0.
let fallback = normalize(cross(n, select(vec3<f32>(1.0, 0.0, 0.0), vec3<f32>(0.0, 1.0, 0.0), abs(n.x) > 0.9)));
let perp = select(fallback, side / len, len > 1e-6);
var out : Seen;
out.dir = cs * n + sqrt(max(0.0, 1.0 - cs * cs)) * perp;
out.doppler = lens.params.z * (1.0 + beta * c);
return out;
}
// Linear-sRGB matrix for Doppler factor d, interpolated in ln d.
fn dopplerMatrix(d : f32) -> mat3x3<f32> {
let span = lens.params.w;
let t = clamp((log(d) + span) / (2.0 * span), 0.0, 1.0) * (LUT_SIZE - 1.0);
let i0 = u32(floor(t));
let i1 = min(i0 + 1u, u32(LUT_SIZE) - 1u);
let w = t - f32(i0);
let r0 = mix(dopplerLut[i0 * 3u], dopplerLut[i1 * 3u], w).xyz;
let r1 = mix(dopplerLut[i0 * 3u + 1u], dopplerLut[i1 * 3u + 1u], w).xyz;
let r2 = mix(dopplerLut[i0 * 3u + 2u], dopplerLut[i1 * 3u + 2u], w).xyz;
// Rows in the table; WGSL matrices take columns.
return transpose(mat3x3<f32>(r0, r1, r2));
}
// The observer's light from scene light \`c\` (linear rgb) at Doppler factor d.
fn shiftLight(c : vec3<f32>, d : f32) -> vec3<f32> {
let mode = lens.params.x;
var out = c;
if (mode > 1.5) {
out = c * pow(d, 4.0);
} else if (mode > 0.5) {
out = dopplerMatrix(d) * c;
}
return max(out * lens.params.y, vec3<f32>(0.0));
}
fn srgbToLinear(c : vec3<f32>) -> vec3<f32> {
let lo = c / 12.92;
let hi = pow((max(c, vec3<f32>(0.0)) + 0.055) / 1.055, vec3<f32>(2.4));
return select(hi, lo, c <= vec3<f32>(0.04045));
}
fn linearToSrgb(c : vec3<f32>) -> vec3<f32> {
let x = clamp(c, vec3<f32>(0.0), vec3<f32>(1.0));
let lo = x * 12.92;
let hi = 1.055 * pow(x, vec3<f32>(1.0 / 2.4)) - 0.055;
return select(hi, lo, x <= vec3<f32>(0.0031308));
}
struct Sampled {
color : vec4<f32>,
hdr : vec4<f32>,
};
fn sampleLens(ndc : vec2<f32>) -> Sampled {
let ray = lensRay(ndc);
var seen : Seen;
seen.dir = ray.dir;
seen.doppler = 1.0;
if (lens.motion.w > 0.0) {
seen = aberrate(ray.dir);
}
// The engine's frames are right-handed, cube lookups left-handed.
let lookup = vec3<f32>(seen.dir.xy, -seen.dir.z);
var color = textureSample(colorCube, samp, lookup);
var hdr = textureSample(hdrCube, samp, lookup);
if (lens.params.x > 0.5) {
// Premultiplied display colour: shift its light, then re-encode.
let a = color.a;
let lin = srgbToLinear(color.rgb / max(a, 1e-4));
color = vec4<f32>(linearToSrgb(shiftLight(lin, seen.doppler)) * a, a);
// Radiance is linear and premultiplied: the shift applies directly.
hdr = vec4<f32>(shiftLight(hdr.rgb, seen.doppler), hdr.a);
}
var out : Sampled;
out.color = color * ray.inside;
out.hdr = hdr * ray.inside;
return out;
}
@fragment
fn fs_color(in : VsOut) -> @location(0) vec4<f32> {
return sampleLens(in.ndc).color;
}
struct HdrOut {
@location(0) color : vec4<f32>,
@location(1) hdr : vec4<f32>,
};
@fragment
fn fs_color_hdr(in : VsOut) -> HdrOut {
let s = sampleLens(in.ndc);
var out : HdrOut;
out.color = s.color;
out.hdr = s.hdr;
return out;
}
`;
const HDR_FORMAT: GPUTextureFormat = "rgba16float";
export class LensRemapPipeline {
private device: GPUDevice;
private colorFormat: GPUTextureFormat;
private mipModule: GPUShaderModule;
private mipLayout: GPUBindGroupLayout;
private mipPipes = new Map<GPUTextureFormat, GPURenderPipeline>();
private remapLayout: GPUBindGroupLayout;
private remapModule: GPUShaderModule;
private remapPipe: GPURenderPipeline | null = null;
private remapHdrPipe: GPURenderPipeline | null = null;
private mipSampler: GPUSampler;
private cubeSampler: GPUSampler;
private placeholderCube: GPUTexture;
private luts = new Map<boolean, GPUBuffer>();
private uniformPool = new BufferPool(
80,
GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
);
private uniformData = new Float32Array(20);
constructor(device: GPUDevice, colorFormat: GPUTextureFormat) {
this.device = device;
this.colorFormat = colorFormat;
this.mipModule = device.createShaderModule({
label: "lens-mip.wgsl",
code: mipWgsl,
});
this.remapModule = device.createShaderModule({
label: "lens-remap.wgsl",
code: remapWgsl,
});
this.mipLayout = device.createBindGroupLayout({
label: "LensMipLayout",
entries: [
{
binding: 0,
visibility: GPUShaderStage.FRAGMENT,
sampler: { type: "filtering" },
},
{
binding: 1,
visibility: GPUShaderStage.FRAGMENT,
texture: { sampleType: "float" },
},
],
});
this.remapLayout = device.createBindGroupLayout({
label: "LensRemapLayout",
entries: [
{
binding: 0,
visibility: GPUShaderStage.FRAGMENT,
buffer: { type: "uniform" },
},
{
binding: 1,
visibility: GPUShaderStage.FRAGMENT,
sampler: { type: "filtering" },
},
{
binding: 2,
visibility: GPUShaderStage.FRAGMENT,
texture: { sampleType: "float", viewDimension: "cube" },
},
{
binding: 3,
visibility: GPUShaderStage.FRAGMENT,
texture: { sampleType: "float", viewDimension: "cube" },
},
{
binding: 4,
visibility: GPUShaderStage.FRAGMENT,
buffer: { type: "read-only-storage" },
},
],
});
this.mipSampler = device.createSampler({
label: "LensMipSampler",
magFilter: "linear",
minFilter: "linear",
});
this.cubeSampler = device.createSampler({
label: "LensCubeSampler",
magFilter: "linear",
minFilter: "linear",
mipmapFilter: "linear",
maxAnisotropy: 16,
});
this.placeholderCube = device.createTexture({
label: "LensPlaceholderCube",
size: { width: 1, height: 1, depthOrArrayLayers: 6 },
format: HDR_FORMAT,
usage: GPUTextureUsage.TEXTURE_BINDING,
});
}
/** Mip levels a cube face of `size` gets. */
static mipLevels(size: number): number {
return Math.floor(Math.log2(Math.max(1, size))) + 1;
}
/** Fills every mip level of every layer of `texture` from level 0. */
generateMips(encoder: GPUCommandEncoder, texture: GPUTexture): void {
const pipe = this.mipPipeline(texture.format);
for (let layer = 0; layer < texture.depthOrArrayLayers; layer++) {
for (let level = 1; level < texture.mipLevelCount; level++) {
const view = (mip: number) =>
texture.createView({
dimension: "2d",
baseArrayLayer: layer,
arrayLayerCount: 1,
baseMipLevel: mip,
mipLevelCount: 1,
});
const pass = encoder.beginRenderPass({
label: "LensMip",
colorAttachments: [
{ view: view(level), loadOp: "clear", storeOp: "store" },
],
});
pass.setPipeline(pipe);
pass.setBindGroup(
0,
this.device.createBindGroup({
layout: this.mipLayout,
entries: [
{ binding: 0, resource: this.mipSampler },
{ binding: 1, resource: view(level - 1) },
],
}),
);
pass.draw(3);
pass.end();
}
}
}
/**
* Draws the lens's picture of `color` (and of `hdr`, the scene radiance,
* into `destHdr` when both are given). Generates the cubes' mips first.
*/
remap(
encoder: GPUCommandEncoder,
color: GPUTexture,
hdr: GPUTexture | null,
dest: GPUTextureView,
destHdr: GPUTextureView | null,
width: number,
height: number,
options: LensRemapOptions,
): void {
this.generateMips(encoder, color);
if (hdr) this.generateMips(encoder, hdr);
const withHdr = Boolean(hdr && destHdr);
const pass = encoder.beginRenderPass({
label: "LensRemap",
colorAttachments: [
{ view: dest, loadOp: "clear", storeOp: "store" },
...(withHdr && destHdr
? [
{
view: destHdr,
loadOp: "clear" as const,
storeOp: "store" as const,
},
]
: []),
],
});
pass.setPipeline(this.remapPipeline(withHdr));
pass.setBindGroup(
0,
this.device.createBindGroup({
layout: this.remapLayout,
entries: [
{
binding: 0,
resource: {
buffer: this.uniformPool.getBuffer(
this.device,
this.packUniforms(options, width, height),
),
},
},
{ binding: 1, resource: this.cubeSampler },
{ binding: 2, resource: color.createView({ dimension: "cube" }) },
{
binding: 3,
resource: (hdr ?? this.placeholderCube).createView({
dimension: "cube",
}),
},
{
binding: 4,
resource: { buffer: this.lut(options.observer?.beaming ?? true) },
},
],
}),
);
pass.draw(3);
pass.end();
}
private packUniforms(
options: LensRemapOptions,
width: number,
height: number,
): Float32Array {
const u = this.uniformData;
const { right, up, forward } = options.basis;
const half = (options.fov * Math.PI) / 360;
const scale =
options.projection === "fisheye"
? half / options.zoom
: Math.tan(half) / options.zoom;
u.set([...right, PROJECTION_INDEX[options.projection]], 0);
u.set([...up, scale], 4);
u.set([...forward, width / Math.max(1, height)], 8);
const observer = options.observer;
const beta = observer ? Math.min(Math.max(observer.speed, 0), 0.99999) : 0;
const direction = observer?.direction ?? forward;
u.set([...direction, beta], 12);
u.set(
[
observer && beta > 0 ? colorMode(observer) : ColorMode.None,
observer ? exposureGain(observer, beta, forward) : 1,
lorentzFactor(beta),
Math.log(DOPPLER_LUT_MAX),
],
16,
);
return u;
}
private lut(beaming: boolean): GPUBuffer {
let buffer = this.luts.get(beaming);
if (!buffer) {
const data = dopplerLut(beaming);
buffer = this.device.createBuffer({
label: `DopplerLut${beaming ? "Beaming" : ""}`,
size: data.byteLength,
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
});
this.device.queue.writeBuffer(buffer, 0, data);
this.luts.set(beaming, buffer);
}
return buffer;
}
private mipPipeline(format: GPUTextureFormat): GPURenderPipeline {
let pipe = this.mipPipes.get(format);
if (!pipe) {
pipe = this.device.createRenderPipeline({
label: `LensMip:${format}`,
layout: this.device.createPipelineLayout({
bindGroupLayouts: [this.mipLayout],
}),
vertex: { module: this.mipModule, entryPoint: "vs" },
fragment: {
module: this.mipModule,
entryPoint: "fs",
targets: [{ format }],
},
primitive: { topology: "triangle-list" },
});
this.mipPipes.set(format, pipe);
}
return pipe;
}
private remapPipeline(withHdr: boolean): GPURenderPipeline {
const cached = withHdr ? this.remapHdrPipe : this.remapPipe;
if (cached) return cached;
const pipe = this.device.createRenderPipeline({
label: withHdr ? "LensRemapHdr" : "LensRemap",
layout: this.device.createPipelineLayout({
bindGroupLayouts: [this.remapLayout],
}),
vertex: { module: this.remapModule, entryPoint: "vs" },
fragment: {
module: this.remapModule,
entryPoint: withHdr ? "fs_color_hdr" : "fs_color",
targets: withHdr
? [{ format: this.colorFormat }, { format: HDR_FORMAT }]
: [{ format: this.colorFormat }],
},
primitive: { topology: "triangle-list" },
});
if (withHdr) this.remapHdrPipe = pipe;
else this.remapPipe = pipe;
return pipe;
}
/** Call once per frame: recycles the per-pass uniforms. */
resetPools(): void {
this.uniformPool.reset();
}
destroy(): void {
this.uniformPool.destroy();
this.placeholderCube.destroy();
for (const buffer of this.luts.values()) buffer.destroy();
this.luts.clear();
}
}
/** The gain on the shifted light: metered at the centre of the view, or fixed stops. */
function exposureGain(
observer: RelativisticObserver,
beta: number,
forward: Vec3,
): number {
if (observer.exposure !== "auto") return 2 ** observer.exposure;
const c =
forward[0] * observer.direction[0] +
forward[1] * observer.direction[1] +
forward[2] * observer.direction[2];
const d = lorentzFactor(beta) * (1 + beta * c);
return 1 / dopplerLuminanceGain(d, observer.doppler, observer.beaming);
}
function colorMode(observer: RelativisticObserver): number {
if (observer.doppler) return ColorMode.Spectral;
return observer.beaming ? ColorMode.Beaming : ColorMode.None;
}