Code-first video & image, rendered natively on WebGPU.
/// <reference types="webgpu" />
import type { SignalData, VirtualMediaData } from "@framefields/core";
import {
unwrapFrameParam,
type WebGPUNodeRenderer,
} from "@framefields/node-sdk";
import {
createFullscreenPipeline,
drawChildToTexture,
drawFullscreenPass,
drawTextureToTarget,
fragmentBindGroupLayout,
signalTextureView,
signalTiming,
texturesLayout,
} from "@framefields/node-sdk/gpu";
import {
MAX_FEATHER,
MAX_OFFSET,
MAX_RADIUS,
normalizePatches,
type PatchHealNodeConfig,
} from "../shared/index.js";
const WGSL_FULLSCREEN_VS = `
struct VertexOutput {
@builtin(position) position : vec4<f32>,
@location(0) uv : vec2<f32>,
};
@vertex
fn vs_main(@builtin(vertex_index) vertexIndex : u32) -> VertexOutput {
var pos = array<vec2<f32>, 4>(
vec2<f32>(-1.0, 1.0),
vec2<f32>( 1.0, 1.0),
vec2<f32>(-1.0, -1.0),
vec2<f32>( 1.0, -1.0)
);
var uvs = array<vec2<f32>, 4>(
vec2<f32>(0.0, 0.0),
vec2<f32>(1.0, 0.0),
vec2<f32>(0.0, 1.0),
vec2<f32>(1.0, 1.0)
);
var output : VertexOutput;
output.position = vec4<f32>(pos[vertexIndex], 0.0, 1.0);
output.uv = uvs[vertexIndex];
return output;
}
`;
const WGSL_PATCHHEAL_UNIFORMS = `
struct PatchHealUniforms {
offsetX : f32,
offsetY : f32,
centerX : f32,
centerY : f32,
radius : f32,
sourceRadius : f32,
feather : f32,
opacity : f32,
mode : f32,
hasMask : f32,
hasOffsetXSig : f32,
hasOffsetYSig : f32,
hasCenterXSig : f32,
hasCenterYSig : f32,
hasRadiusSig : f32,
hasSourceRadSig : f32,
hasFeatherSig : f32,
hasOpacitySig : f32,
pad0 : f32,
pad1 : f32,
};
`;
const WGSL_PATCHHEAL_FS = `
${WGSL_PATCHHEAL_UNIFORMS}
@group(0) @binding(0) var<uniform> u : PatchHealUniforms;
@group(1) @binding(0) var srcTex : texture_2d<f32>;
@group(1) @binding(1) var maskTex : texture_2d<f32>;
@group(1) @binding(2) var samp : sampler;
@group(2) @binding(0) var offsetXSigTex : texture_2d<f32>;
@group(2) @binding(1) var offsetYSigTex : texture_2d<f32>;
@group(2) @binding(2) var centerXSigTex : texture_2d<f32>;
@group(2) @binding(3) var centerYSigTex : texture_2d<f32>;
@group(2) @binding(4) var radiusSigTex : texture_2d<f32>;
@group(2) @binding(5) var sourceRadSigTex : texture_2d<f32>;
@group(2) @binding(6) var featherSigTex : texture_2d<f32>;
@group(2) @binding(7) var opacitySigTex : texture_2d<f32>;
@group(2) @binding(8) var signalSamp : sampler;
fn getLuminance(c : vec3<f32>) -> f32 {
return dot(c, vec3<f32>(0.2126, 0.7152, 0.0722));
}
fn sampleBoxBlur(tex : texture_2d<f32>, smp : sampler, centerUv : vec2<f32>, blurPx : f32, dims : vec2<f32>) -> vec4<f32> {
let texel = 1.0 / dims;
let stepSize = max(1.0, blurPx / 3.0);
var accum = vec4<f32>(0.0);
var count = 0.0;
for (var y = -2; y <= 2; y++) {
for (var x = -2; x <= 2; x++) {
let offset = vec2<f32>(f32(x) * stepSize, f32(y) * stepSize) * texel;
let sampleUv = clamp(centerUv + offset, vec2<f32>(0.0), vec2<f32>(1.0));
accum += textureSampleLevel(tex, smp, sampleUv, 0.0);
count += 1.0;
}
}
return accum / count;
}
@fragment
fn fs_main(in : VertexOutput) -> @location(0) vec4<f32> {
let dimensions = vec2<f32>(textureDimensions(srcTex));
var offsetX = u.offsetX;
if (u.hasOffsetXSig > 0.5) {
offsetX = textureSampleLevel(offsetXSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
}
var offsetY = u.offsetY;
if (u.hasOffsetYSig > 0.5) {
offsetY = textureSampleLevel(offsetYSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
}
var centerX = u.centerX;
if (u.hasCenterXSig > 0.5) {
centerX = textureSampleLevel(centerXSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
}
var centerY = u.centerY;
if (u.hasCenterYSig > 0.5) {
centerY = textureSampleLevel(centerYSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
}
var radius = u.radius;
if (u.hasRadiusSig > 0.5) {
radius = textureSampleLevel(radiusSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
}
var sourceRadius = u.sourceRadius;
if (u.hasSourceRadSig > 0.5) {
sourceRadius = textureSampleLevel(sourceRadSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
}
var feather = u.feather;
if (u.hasFeatherSig > 0.5) {
feather = textureSampleLevel(featherSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
}
var opacity = u.opacity;
if (u.hasOpacitySig > 0.5) {
opacity = textureSampleLevel(opacitySigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r;
}
let dstCol = textureSampleLevel(srcTex, samp, in.uv, 0.0);
// Calculate mask weight (either from explicit mask or procedural circular patch)
var maskWeight = 0.0;
let targetCenterUV = vec2<f32>(centerX, centerY);
if (u.hasMask > 0.5) {
let maskCol = textureSampleLevel(maskTex, samp, in.uv, 0.0);
maskWeight = max(maskCol.a, max(maskCol.r, max(maskCol.g, maskCol.b)));
} else {
let distPx = length((in.uv - targetCenterUV) * dimensions);
let innerRadius = radius * max(0.0, 1.0 - feather / 100.0);
maskWeight = 1.0 - smoothstep(innerRadius, max(innerRadius + 0.001, radius), distPx);
}
let blendWeight = clamp(maskWeight * opacity, 0.0, 1.0);
if (blendWeight < 1e-4) {
return dstCol;
}
// Compute source sample coordinate (scaling from sourceRadius to target radius if different)
let sourceCenterUV = targetCenterUV + vec2<f32>(offsetX, offsetY) / dimensions;
let srcScale = max(1.0, sourceRadius) / max(1.0, radius);
let uvFromTargetCenter = in.uv - targetCenterUV;
let srcUV = clamp(sourceCenterUV + uvFromTargetCenter * srcScale, vec2<f32>(0.0), vec2<f32>(1.0));
let srcCol = textureSampleLevel(srcTex, samp, srcUV, 0.0);
var healedCol = srcCol;
if (u.mode > 0.5 && u.mode < 1.5) {
// Mode 1: Seamless Heal (Poisson gradient / local illumination matching)
let blurRadius = max(4.0, radius * 0.4);
let srcBlur = sampleBoxBlur(srcTex, samp, srcUV, blurRadius, dimensions);
let dstBlur = sampleBoxBlur(srcTex, samp, in.uv, blurRadius, dimensions);
let srcDetail = srcCol.rgb - srcBlur.rgb;
let healedRgb = dstBlur.rgb + srcDetail;
let clampedRgb = clamp(healedRgb, vec3<f32>(0.0), vec3<f32>(srcCol.a));
healedCol = vec4<f32>(clampedRgb, srcCol.a);
} else if (u.mode >= 1.5) {
// Mode 2: Texture Transfer (Luminance structure matching)
let blurRadius = max(4.0, radius * 0.4);
let srcBlur = sampleBoxBlur(srcTex, samp, srcUV, blurRadius, dimensions);
let lumaSrc = getLuminance(srcCol.rgb);
let lumaSrcBlur = getLuminance(srcBlur.rgb);
let lumaDelta = lumaSrc - lumaSrcBlur;
let healedRgb = dstCol.rgb + vec3<f32>(lumaDelta);
let clampedRgb = clamp(healedRgb, vec3<f32>(0.0), vec3<f32>(dstCol.a));
healedCol = vec4<f32>(clampedRgb, dstCol.a);
}
return mix(dstCol, healedCol, blendWeight);
}
`;
interface DevicePatchHealResources {
uniformLayout: GPUBindGroupLayout;
textureLayout: GPUBindGroupLayout;
signalTextureLayout: GPUBindGroupLayout;
pipeline: GPURenderPipeline;
}
const deviceResourceCache = new WeakMap<GPUDevice, DevicePatchHealResources>();
function getDeviceResources(
device: GPUDevice,
format: GPUTextureFormat,
): DevicePatchHealResources {
let res = deviceResourceCache.get(device);
if (res) return res;
const uniformLayout = fragmentBindGroupLayout(device, ["uniform"]);
const textureLayout = texturesLayout(device, 2);
const signalTextureLayout = texturesLayout(device, 8);
const shaderModule = device.createShaderModule({
code: `${WGSL_FULLSCREEN_VS}\n${WGSL_PATCHHEAL_FS}`,
});
const pipelineLayout = device.createPipelineLayout({
bindGroupLayouts: [uniformLayout, textureLayout, signalTextureLayout],
});
const pipeline = createFullscreenPipeline(device, {
module: shaderModule,
layouts: pipelineLayout,
format,
blend: {
color: { srcFactor: "one", dstFactor: "zero", operation: "add" },
alpha: { srcFactor: "one", dstFactor: "zero", operation: "add" },
},
vertexEntryPoint: "vs_main",
fragmentEntryPoint: "fs_main",
});
res = {
uniformLayout,
textureLayout,
signalTextureLayout,
pipeline,
};
deviceResourceCache.set(device, res);
return res;
}
type PatchHealSignalData = SignalData & { nodeId?: string };
interface PatchHealOp extends PatchHealNodeConfig {
op: "PatchHeal";
maskMedia?: VirtualMediaData;
inputs?: Record<
string,
{
connectionValid: boolean;
outputItem: { type: string; data: unknown } | null;
}
>;
}
export const PatchHealWebGPURenderer: WebGPUNodeRenderer = async (args) => {
const {
ctx,
encoder,
pass,
targetTexture,
targetWidth,
targetHeight,
props,
} = args;
const op = props.virtualMedia?.operation as unknown as
| PatchHealOp
| undefined;
if (op?.op !== "PatchHeal" || !op) return;
const childMedia = props.virtualMedia?.children?.[0];
if (!childMedia) return;
pass.end();
const width = targetWidth;
const height = targetHeight;
const frame = props.frame ?? 0;
const compositionFrame = props.compositionFrame ?? frame;
const fps = props.fps || 30;
const nodeId = op.inputs ? Object.keys(op.inputs)[0] : "patch_heal_node";
// 1. Draw source child media into tmpSrcTex
const { texture: tmpSrcTex, view: tmpSrcView } = await drawChildToTexture(
args,
childMedia,
{
props: {
...props,
virtualMedia: childMedia,
renderId: `${props.renderId}-src`,
},
},
);
// 2. Draw optional mask child media into maskTex if present
let maskTex: GPUTexture | null = null;
if (op.maskMedia) {
({ texture: maskTex } = await drawChildToTexture(args, op.maskMedia, {
props: {
...props,
virtualMedia: op.maskMedia,
renderId: `${props.renderId}-mask`,
excludeTextures: [...(props.excludeTextures || []), tmpSrcTex],
},
exclude: [tmpSrcTex],
}));
}
// 3. Compile pipeline resources
const {
pipeline,
uniformLayout: uLayout,
textureLayout: tLayout,
signalTextureLayout: sigLayout,
} = getDeviceResources(ctx.device, ctx.renderer.format);
const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);
const { elapsedSeconds, durationSeconds } = signalTiming(props, frame, fps);
const patches = normalizePatches(op);
const resolveBindable = (
configKey:
| "centerX"
| "centerY"
| "offsetX"
| "offsetY"
| "radius"
| "sourceRadius"
| "feather"
| "opacity",
fallbackVal: number,
minVal: number,
maxVal: number,
): {
val: number;
hasStaticSig: boolean;
sd: PatchHealSignalData | null;
} => {
const handleIdKey = `${configKey}HandleId` as keyof PatchHealOp;
const handleId = op[handleIdKey] as string | null | undefined;
const input = handleId ? op.inputs?.[handleId] : null;
const hasSignal = !!(
input?.connectionValid &&
(input.outputItem?.type === "Signal" ||
input.outputItem?.type === "Numeric")
);
const sd =
hasSignal && input?.outputItem?.data
? (input.outputItem.data as PatchHealSignalData)
: null;
let val = fallbackVal;
let hasStaticSig = false;
if (!hasSignal) {
if (input?.connectionValid && input.outputItem?.type === "Number") {
const numVal =
typeof input.outputItem.data === "number"
? input.outputItem.data
: ((input.outputItem.data as { value?: number })?.value ??
fallbackVal);
val = Math.max(minVal, Math.min(maxVal, Number(numVal)));
} else {
const rawVal = unwrapFrameParam(
fallbackVal,
compositionFrame,
fps,
).value;
const numVal = Number(rawVal ?? fallbackVal);
val = Math.max(
minVal,
Math.min(maxVal, isNaN(numVal) ? fallbackVal : numVal),
);
}
} else if (sd) {
val = Math.max(minVal, Math.min(maxVal, Number(sd.offset ?? 0.0)));
hasStaticSig = true;
}
return { val, hasStaticSig, sd };
};
const dummyMaskView = maskTex ? maskTex.createView() : tmpSrcView;
// Execute patches sequentially (ping-pong between intermediate textures if > 1 patch)
let currentInputTex = tmpSrcTex;
const pingTex = ctx.renderer.getTemporaryTexture(width, height, [
tmpSrcTex,
...(maskTex ? [maskTex] : []),
targetTexture,
...(props.excludeTextures || []),
]);
const pongTex =
patches.length > 1
? ctx.renderer.getTemporaryTexture(width, height, [
tmpSrcTex,
pingTex,
...(maskTex ? [maskTex] : []),
targetTexture,
...(props.excludeTextures || []),
])
: pingTex;
for (let i = 0; i < patches.length; i++) {
const patch = patches[i];
const isFirstPatch = i === 0;
const centerXRes = isFirstPatch
? resolveBindable("centerX", patch.centerX, 0.0, 1.0)
: { val: patch.centerX, hasStaticSig: false, sd: null };
const centerYRes = isFirstPatch
? resolveBindable("centerY", patch.centerY, 0.0, 1.0)
: { val: patch.centerY, hasStaticSig: false, sd: null };
const offsetXRes = isFirstPatch
? resolveBindable("offsetX", patch.offsetX, -MAX_OFFSET, MAX_OFFSET)
: { val: patch.offsetX, hasStaticSig: false, sd: null };
const offsetYRes = isFirstPatch
? resolveBindable("offsetY", patch.offsetY, -MAX_OFFSET, MAX_OFFSET)
: { val: patch.offsetY, hasStaticSig: false, sd: null };
const radiusRes = isFirstPatch
? resolveBindable("radius", patch.radius, 1, MAX_RADIUS)
: { val: patch.radius, hasStaticSig: false, sd: null };
const sourceRadiusRes = isFirstPatch
? resolveBindable(
"sourceRadius",
patch.sourceRadius ?? patch.radius,
1,
MAX_RADIUS,
)
: {
val: patch.sourceRadius ?? patch.radius,
hasStaticSig: false,
sd: null,
};
const featherRes = isFirstPatch
? resolveBindable("feather", patch.feather, 0, MAX_FEATHER)
: { val: patch.feather, hasStaticSig: false, sd: null };
const opacityRes = isFirstPatch
? resolveBindable("opacity", patch.opacity, 0.0, 1.0)
: { val: patch.opacity, hasStaticSig: false, sd: null };
const signalContext = {
args,
width,
height,
frame,
fps,
elapsedSeconds,
durationSeconds,
};
const getSignalView = (
res: {
val: number;
hasStaticSig: boolean;
sd: PatchHealSignalData | null;
},
suffix: string,
) => signalTextureView(signalContext, res, `${nodeId}_${suffix}`);
const offsetXView = getSignalView(offsetXRes, "offsetX_sig");
const offsetYView = getSignalView(offsetYRes, "offsetY_sig");
const centerXView = getSignalView(centerXRes, "centerX_sig");
const centerYView = getSignalView(centerYRes, "centerY_sig");
const radiusView = getSignalView(radiusRes, "radius_sig");
const sourceRadiusView = getSignalView(sourceRadiusRes, "sourceRadius_sig");
const featherView = getSignalView(featherRes, "feather_sig");
const opacityView = getSignalView(opacityRes, "opacity_sig");
const modeNum =
patch.mode === "Clone"
? 0.0
: patch.mode === "TextureTransfer"
? 2.0
: 1.0;
const patchData = new Float32Array(20);
patchData[0] = offsetXRes.val;
patchData[1] = offsetYRes.val;
patchData[2] = centerXRes.val;
patchData[3] = centerYRes.val;
patchData[4] = radiusRes.val;
patchData[5] = sourceRadiusRes.val;
patchData[6] = featherRes.val;
patchData[7] = opacityRes.val;
patchData[8] = modeNum;
patchData[9] = maskTex ? 1.0 : 0.0;
patchData[10] = offsetXRes.hasStaticSig ? 1.0 : 0.0;
patchData[11] = offsetYRes.hasStaticSig ? 1.0 : 0.0;
patchData[12] = centerXRes.hasStaticSig ? 1.0 : 0.0;
patchData[13] = centerYRes.hasStaticSig ? 1.0 : 0.0;
patchData[14] = radiusRes.hasStaticSig ? 1.0 : 0.0;
patchData[15] = sourceRadiusRes.hasStaticSig ? 1.0 : 0.0;
patchData[16] = featherRes.hasStaticSig ? 1.0 : 0.0;
patchData[17] = opacityRes.hasStaticSig ? 1.0 : 0.0;
patchData[18] = 0.0;
patchData[19] = 0.0;
const uBuffer = ctx.renderer.getTemporaryBuffer(patchData);
const destinationTex = i % 2 === 0 ? pingTex : pongTex;
const texBindGroup = ctx.device.createBindGroup({
layout: tLayout,
entries: [
{ binding: 0, resource: currentInputTex.createView() },
{ binding: 1, resource: dummyMaskView },
{ binding: 2, resource: sampler },
],
});
const sigBindGroup = ctx.device.createBindGroup({
layout: sigLayout,
entries: [
{ binding: 0, resource: offsetXView },
{ binding: 1, resource: offsetYView },
{ binding: 2, resource: centerXView },
{ binding: 3, resource: centerYView },
{ binding: 4, resource: radiusView },
{ binding: 5, resource: sourceRadiusView },
{ binding: 6, resource: featherView },
{ binding: 7, resource: opacityView },
{ binding: 8, resource: sampler },
],
});
drawFullscreenPass(encoder, destinationTex.createView(), pipeline, [
ctx.device.createBindGroup({
layout: uLayout,
entries: [{ binding: 0, resource: { buffer: uBuffer } }],
}),
texBindGroup,
sigBindGroup,
]);
currentInputTex = destinationTex;
}
// Final blit to targetView
drawTextureToTarget(args, currentInputTex);
};