Skip to content

Code-first video & image, rendered natively on WebGPU.

6.5 KB
Raw Blame
/**
 * @file temporal-deflicker-pipeline.ts
 * Native WebGPU Temporal De-flickering Pipeline
 * Combines dense optical flow motion estimation with motion-compensated forward warping
 * and adaptive photometric blending to eliminate generative video flicker.
 */

import type { DeflickerOptions } from "@framefields/core";
import {
	temporalWarpBlendRgba8Wgsl,
	temporalWarpBlendRgba16fWgsl,
} from "../shaders/temporal-warp-blend.wgsl.js";
import { OpticalFlowComputePipeline } from "./optical-flow-pipeline.js";

export class TemporalDeflickerPipeline {
	private flowPipeline: OpticalFlowComputePipeline;
	private rgba8Pipeline: GPUComputePipeline;
	private rgba16fPipeline: GPUComputePipeline;
	private uniformBuffer: GPUBuffer;
	private bilinearSampler: GPUSampler;

	private historyTexture?: GPUTexture;
	private cachedOutputTexture?: GPUTexture;

	constructor(private device: GPUDevice) {
		this.flowPipeline = new OpticalFlowComputePipeline(device);

		const rgba8Module = device.createShaderModule({
			label: "temporal_warp_blend_rgba8.wgsl",
			code: temporalWarpBlendRgba8Wgsl,
		});
		this.rgba8Pipeline = device.createComputePipeline({
			label: "TemporalWarpBlendRgba8Pipeline",
			layout: "auto",
			compute: {
				module: rgba8Module,
				entryPoint: "warpBlend",
			},
		});

		const rgba16fModule = device.createShaderModule({
			label: "temporal_warp_blend_rgba16f.wgsl",
			code: temporalWarpBlendRgba16fWgsl,
		});
		this.rgba16fPipeline = device.createComputePipeline({
			label: "TemporalWarpBlendRgba16fPipeline",
			layout: "auto",
			compute: {
				module: rgba16fModule,
				entryPoint: "warpBlend",
			},
		});

		this.uniformBuffer = device.createBuffer({
			size: 32,
			usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
			label: "temporal_deflicker_uniform",
		});

		this.bilinearSampler = device.createSampler({
			label: "temporal_deflicker_sampler",
			magFilter: "linear",
			minFilter: "linear",
			addressModeU: "clamp-to-edge",
			addressModeV: "clamp-to-edge",
		});
	}

	public execute(
		currTexture: GPUTexture,
		options: DeflickerOptions = {},
		customEncoder?: GPUCommandEncoder,
	): GPUTexture {
		const width = currTexture.width;
		const height = currTexture.height;
		const rawBlendWeight = options.blendWeight ?? 0.35;
		const blendWeight =
			typeof rawBlendWeight === "number"
				? rawBlendWeight
				: Number(rawBlendWeight);
		const disocclusionThreshold = options.disocclusionThreshold ?? 0.15;
		const maxMotionPixels = options.maxMotionPixels ?? 64.0;
		const format: GPUTextureFormat = currTexture.format;
		const isRgba16f = format === "rgba16float";

		// 1. Ensure output texture matches size & format
		if (
			!this.cachedOutputTexture ||
			this.cachedOutputTexture.width !== width ||
			this.cachedOutputTexture.height !== height ||
			this.cachedOutputTexture.format !== format
		) {
			this.cachedOutputTexture?.destroy();
			this.cachedOutputTexture = this.device.createTexture({
				label: `deflickered_output_${format}`,
				size: [width, height],
				format,
				usage:
					GPUTextureUsage.STORAGE_BINDING |
					GPUTextureUsage.TEXTURE_BINDING |
					GPUTextureUsage.COPY_SRC |
					GPUTextureUsage.COPY_DST |
					GPUTextureUsage.RENDER_ATTACHMENT,
			});
		}

		// 2. Check if this is the first frame or dimension change
		const isFirstFrame =
			!this.historyTexture ||
			this.historyTexture.width !== width ||
			this.historyTexture.height !== height;

		if (isFirstFrame) {
			this.historyTexture?.destroy();
			this.historyTexture = this.device.createTexture({
				label: "deflicker_history_texture",
				size: [width, height],
				format,
				usage:
					GPUTextureUsage.TEXTURE_BINDING |
					GPUTextureUsage.COPY_DST |
					GPUTextureUsage.COPY_SRC,
			});

			const commandEncoder =
				customEncoder ??
				this.device.createCommandEncoder({
					label: "deflicker_first_frame_encoder",
				});

			// Copy current frame to history buffer
			commandEncoder.copyTextureToTexture(
				{ texture: currTexture },
				{ texture: this.historyTexture },
				[width, height],
			);

			// Copy current frame directly to output
			commandEncoder.copyTextureToTexture(
				{ texture: currTexture },
				{ texture: this.cachedOutputTexture },
				[width, height],
			);

			if (!customEncoder) {
				this.device.queue.submit([commandEncoder.finish()]);
			}
			return this.cachedOutputTexture;
		}

		// 3. For Frame N >= 1: Compute Optical Flow motion vectors between history (N-1) and curr (N)
		const flowTexture = this.flowPipeline.execute(
			this.historyTexture,
			currTexture,
			options.opticalFlow,
			customEncoder,
		);

		// 4. Update uniform buffer
		const uniformData = new ArrayBuffer(32);
		const u32View = new Uint32Array(uniformData);
		const f32View = new Float32Array(uniformData);
		u32View[0] = width;
		u32View[1] = height;
		f32View[2] = blendWeight;
		f32View[3] = disocclusionThreshold;
		f32View[4] = maxMotionPixels;
		f32View[5] = 0.0; // isFirstFrame = false

		this.device.queue.writeBuffer(
			this.uniformBuffer,
			0,
			uniformData as unknown as GPUAllowSharedBufferSource,
		);

		const pipeline = isRgba16f ? this.rgba16fPipeline : this.rgba8Pipeline;

		const commandEncoder =
			customEncoder ??
			this.device.createCommandEncoder({
				label: "deflicker_blend_encoder",
			});

		const bindGroup = this.device.createBindGroup({
			layout: pipeline.getBindGroupLayout(0),
			entries: [
				{ binding: 0, resource: this.historyTexture.createView() },
				{ binding: 1, resource: currTexture.createView() },
				{ binding: 2, resource: flowTexture.createView() },
				{ binding: 3, resource: this.cachedOutputTexture.createView() },
				{ binding: 4, resource: { buffer: this.uniformBuffer } },
				{ binding: 5, resource: this.bilinearSampler },
			],
		});

		const pass = commandEncoder.beginComputePass({
			label: "deflicker_blend_pass",
		});
		pass.setPipeline(pipeline);
		pass.setBindGroup(0, bindGroup);
		pass.dispatchWorkgroups(Math.ceil(width / 16), Math.ceil(height / 16));
		pass.end();

		// Update history texture with the deflickered result for recursive temporal coherence
		commandEncoder.copyTextureToTexture(
			{ texture: this.cachedOutputTexture },
			{ texture: this.historyTexture },
			[width, height],
		);

		if (!customEncoder) {
			this.device.queue.submit([commandEncoder.finish()]);
		}

		return this.cachedOutputTexture;
	}

	public reset(): void {
		this.historyTexture?.destroy();
		this.historyTexture = undefined;
	}

	public destroy(): void {
		this.flowPipeline.destroy();
		this.historyTexture?.destroy();
		this.cachedOutputTexture?.destroy();
		this.uniformBuffer.destroy();
	}
}