Code-first video & image, rendered natively on WebGPU.
/// <reference types="webgpu" />
import type { VirtualMediaData } from "@framefields/core";
import {
resolveHandleParam,
type WebGPUNodeRenderer,
} from "@framefields/node-sdk";
import {
createFullscreenPipeline,
drawChildToTexture,
drawFullscreenPass,
drawTextureToTarget,
FULLSCREEN_QUAD_WGSL,
fragmentBindGroupLayout,
signalTextureView,
signalTiming,
texturesLayout,
} from "@framefields/node-sdk/gpu";
import {
MAX_RADIUS,
type MaskChannel,
type MaskMathOp,
type MaskMathSignalData,
type MaskOperation,
type MaskOutputFormat,
} from "../shared/index.js";
const OPERATION_MAP: Record<MaskOperation, number> = {
Union: 0,
Intersect: 1,
Subtract: 2,
Difference: 3,
Invert: 4,
Dilate: 5,
Erode: 6,
Choke: 7,
Feather: 8,
};
const CHANNEL_MAP: Record<MaskChannel, number> = {
Alpha: 0,
Luminance: 1,
Red: 2,
Green: 3,
Blue: 4,
};
const OUTPUT_FORMAT_MAP: Record<MaskOutputFormat, number> = {
WhiteWithAlpha: 0,
GrayscaleRGB: 1,
AlphaOnly: 2,
PassthroughRGB: 3,
};
const WGSL_MASKMATH = `
struct MaskMathUniforms {
dirX : f32,
dirY : f32,
radius : f32,
threshold : f32,
clampMin : f32,
clampMax : f32,
operation : f32,
channelA : f32,
channelB : f32,
binarize : f32,
invertResult : f32,
outputFormat : f32,
hasMaskB : f32,
hasRadiusSig : f32,
hasThreshSig : f32,
hasClampMinSig : f32,
hasClampMaxSig : f32,
isSecondPass : f32,
pad0 : f32,
pad1 : f32,
};
@group(0) @binding(0) var<uniform> u : MaskMathUniforms;
@group(1) @binding(0) var texA : texture_2d<f32>;
@group(1) @binding(1) var texB : texture_2d<f32>;
@group(1) @binding(2) var origTexA : texture_2d<f32>;
@group(1) @binding(3) var samp : sampler;
@group(2) @binding(0) var radiusSigTex : texture_2d<f32>;
@group(2) @binding(1) var threshSigTex : texture_2d<f32>;
@group(2) @binding(2) var clampMinSigTex : texture_2d<f32>;
@group(2) @binding(3) var clampMaxSigTex : texture_2d<f32>;
@group(2) @binding(4) var signalSamp : sampler;
${FULLSCREEN_QUAD_WGSL}
fn extractChannel(color : vec4<f32>, chCode : f32) -> f32 {
if (chCode < 0.5) {
// Alpha
return color.a;
} else if (chCode < 1.5) {
// Luminance
let luma = dot(color.rgb, vec3<f32>(0.2126, 0.7152, 0.0722));
if (color.a > 1e-4) {
return min(1.0, luma / color.a);
}
return luma;
} else if (chCode < 2.5) {
// Red
if (color.a > 1e-4) {
return min(1.0, color.r / color.a);
}
return color.r;
} else if (chCode < 3.5) {
// Green
if (color.a > 1e-4) {
return min(1.0, color.g / color.a);
}
return color.g;
} else {
// Blue
if (color.a > 1e-4) {
return min(1.0, color.b / color.a);
}
return color.b;
}
}
@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
let dimensions = vec2<f32>(textureDimensions(texA));
let texelSize = 1.0 / max(dimensions, vec2<f32>(1.0, 1.0));
var radius = u.radius;
if (u.hasRadiusSig > 0.5) {
radius = clamp(textureSampleLevel(radiusSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, 0.0, 200.0);
}
var threshold = u.threshold;
if (u.hasThreshSig > 0.5) {
threshold = clamp(textureSampleLevel(threshSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, 0.0, 1.0);
}
var clampMin = u.clampMin;
if (u.hasClampMinSig > 0.5) {
clampMin = clamp(textureSampleLevel(clampMinSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, 0.0, 1.0);
}
var clampMax = u.clampMax;
if (u.hasClampMaxSig > 0.5) {
clampMax = clamp(textureSampleLevel(clampMaxSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, 0.0, 1.0);
}
let op = u.operation;
let direction = vec2<f32>(u.dirX, u.dirY);
var val = 0.0;
if (u.isSecondPass <= 0.5) {
// -------------------------------------------------------------
// PASS 1: Horizontal pass / Boolean operation
// -------------------------------------------------------------
let sampleA = textureSampleLevel(texA, samp, in.uv, 0.0);
let sampleB = textureSampleLevel(texB, samp, in.uv, 0.0);
let a = extractChannel(sampleA, u.channelA);
let b = select(0.0, extractChannel(sampleB, u.channelB), u.hasMaskB > 0.5);
if (op < 0.5) {
// Union (A + B)
val = select(a, clamp(a + b, 0.0, 1.0), u.hasMaskB > 0.5);
} else if (op < 1.5) {
// Intersect (min(A, B))
val = select(a, min(a, b), u.hasMaskB > 0.5);
} else if (op < 2.5) {
// Subtract (clamp(A - B, 0, 1))
val = select(a, clamp(a - b, 0.0, 1.0), u.hasMaskB > 0.5);
} else if (op < 3.5) {
// Difference (|A - B|)
val = select(a, abs(a - b), u.hasMaskB > 0.5);
} else if (op < 4.5) {
// Invert (1 - A)
val = 1.0 - a;
} else if (op < 5.5) {
// Dilate (Horizontal Max)
if (radius <= 0.0) {
val = a;
} else {
let stepScale = max(1.0, radius / 20.0);
let sampleRadius = min(i32(ceil(radius / stepScale)), 48);
var maxVal = a;
for (var i = -sampleRadius; i <= sampleRadius; i++) {
let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
let s = textureSampleLevel(texA, samp, in.uv + offset, 0.0);
maxVal = max(maxVal, extractChannel(s, u.channelA));
}
val = maxVal;
}
} else if (op < 7.5) {
// Erode & Choke (Horizontal Min)
if (radius <= 0.0) {
val = a;
} else {
let stepScale = max(1.0, radius / 20.0);
let sampleRadius = min(i32(ceil(radius / stepScale)), 48);
var minVal = a;
for (var i = -sampleRadius; i <= sampleRadius; i++) {
let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
let s = textureSampleLevel(texA, samp, in.uv + offset, 0.0);
minVal = min(minVal, extractChannel(s, u.channelA));
}
val = minVal;
}
} else {
// Feather (Horizontal Gaussian Blur)
if (radius <= 0.0) {
val = a;
} else {
let spatial_sigma = max(0.1, radius / 2.0);
let stepScale = max(1.0, spatial_sigma / 20.0);
let effectiveSigma = spatial_sigma / stepScale;
let sampleRadius = min(i32(ceil(3.0 * effectiveSigma)), 48);
var accum = 0.0;
var totalWeight = 0.0;
for (var i = -sampleRadius; i <= sampleRadius; i++) {
let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
let x = f32(i) * stepScale;
let weight = exp(-0.5 * (x / spatial_sigma) * (x / spatial_sigma));
let s = textureSampleLevel(texA, samp, in.uv + offset, 0.0);
accum += extractChannel(s, u.channelA) * weight;
totalWeight += weight;
}
val = accum / max(totalWeight, 1e-5);
}
}
return vec4<f32>(val, val, val, val);
}
// -------------------------------------------------------------
// PASS 2: Vertical pass & Post-processing (clamp, binarize, format)
// -------------------------------------------------------------
let samplePass1 = textureSampleLevel(texA, samp, in.uv, 0.0);
var matte = samplePass1.r;
// 1. Spatial kernel processing (Dilate, Erode, Choke, Feather) when radius > 0
if (radius > 0.0) {
if (op >= 4.5 && op < 5.5) {
// Dilate (Vertical Max)
let stepScale = max(1.0, radius / 20.0);
let sampleRadius = min(i32(ceil(radius / stepScale)), 48);
var maxVal = matte;
for (var i = -sampleRadius; i <= sampleRadius; i++) {
let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
let s = textureSampleLevel(texA, samp, in.uv + offset, 0.0);
maxVal = max(maxVal, s.r);
}
matte = maxVal;
} else if (op >= 5.5 && op < 7.5) {
// Erode & Choke (Vertical Min)
let stepScale = max(1.0, radius / 20.0);
let sampleRadius = min(i32(ceil(radius / stepScale)), 48);
var minVal = matte;
for (var i = -sampleRadius; i <= sampleRadius; i++) {
let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
let s = textureSampleLevel(texA, samp, in.uv + offset, 0.0);
minVal = min(minVal, s.r);
}
matte = minVal;
} else if (op >= 7.5) {
// Feather (Vertical Gaussian Blur)
let spatial_sigma = max(0.1, radius / 2.0);
let stepScale = max(1.0, spatial_sigma / 20.0);
let effectiveSigma = spatial_sigma / stepScale;
let sampleRadius = min(i32(ceil(3.0 * effectiveSigma)), 48);
var accum = 0.0;
var totalWeight = 0.0;
for (var i = -sampleRadius; i <= sampleRadius; i++) {
let offset = vec2<f32>(f32(i) * stepScale) * direction * texelSize;
let x = f32(i) * stepScale;
let weight = exp(-0.5 * (x / spatial_sigma) * (x / spatial_sigma));
let s = textureSampleLevel(texA, samp, in.uv + offset, 0.0);
accum += s.r * weight;
totalWeight += weight;
}
matte = accum / max(totalWeight, 1e-5);
}
}
// 2. Choke non-linear threshold contraction curve (applies even if radius == 0)
if (op >= 6.5 && op < 7.5) {
let threshRange = max(0.01, 1.0 - threshold);
matte = clamp((matte - threshold) / threshRange, 0.0, 1.0);
}
// Range clamping
let lowerLimit = min(clampMin, clampMax);
let upperLimit = max(clampMin, clampMax);
matte = clamp(matte, lowerLimit, upperLimit);
// Optional Binarization
if (u.binarize > 0.5) {
matte = select(0.0, 1.0, matte >= threshold);
}
// Optional Inversion
if (u.invertResult > 0.5) {
matte = 1.0 - matte;
}
// Output format conversion
let fmt = u.outputFormat;
if (fmt < 0.5) {
// WhiteWithAlpha: vec4(matte, matte, matte, matte) (premultiplied standard mask)
return vec4<f32>(matte, matte, matte, matte);
} else if (fmt < 1.5) {
// GrayscaleRGB: vec4(matte, matte, matte, 1.0)
return vec4<f32>(matte, matte, matte, 1.0);
} else if (fmt < 2.5) {
// AlphaOnly: vec4(0.0, 0.0, 0.0, matte)
return vec4<f32>(0.0, 0.0, 0.0, matte);
} else {
// PassthroughRGB: Mask A color with new alpha
let origCol = textureSampleLevel(origTexA, samp, in.uv, 0.0);
if (origCol.a < 1e-4) {
return vec4<f32>(0.0, 0.0, 0.0, 0.0);
}
let unpremult = origCol.rgb / origCol.a;
return vec4<f32>(unpremult * matte, matte);
}
}
`;
interface DeviceMaskMathResources {
uniformLayout: GPUBindGroupLayout;
textureLayout: GPUBindGroupLayout;
signalTextureLayout: GPUBindGroupLayout;
pipeline: GPURenderPipeline;
}
const deviceResourceCache = new WeakMap<GPUDevice, DeviceMaskMathResources>();
function getMaskMathResources(
device: GPUDevice,
targetFormat: GPUTextureFormat,
): DeviceMaskMathResources {
const cached = deviceResourceCache.get(device);
if (cached) return cached;
const uniformLayout = fragmentBindGroupLayout(device, ["uniform"]);
const textureLayout = texturesLayout(device, 3);
const signalTextureLayout = texturesLayout(device, 4);
const shaderModule = device.createShaderModule({
code: WGSL_MASKMATH,
});
const pipelineLayout = device.createPipelineLayout({
bindGroupLayouts: [uniformLayout, textureLayout, signalTextureLayout],
});
const pipeline = createFullscreenPipeline(device, {
module: shaderModule,
layouts: pipelineLayout,
format: targetFormat,
});
const res: DeviceMaskMathResources = {
uniformLayout,
textureLayout,
signalTextureLayout,
pipeline,
};
deviceResourceCache.set(device, res);
return res;
}
export const WebGPURenderer: WebGPUNodeRenderer = async (args) => {
const {
ctx,
encoder,
pass,
targetTexture,
targetWidth,
targetHeight,
props,
} = args;
const op = props.virtualMedia?.operation as MaskMathOp | undefined;
if (op?.op !== "MaskMath" || !op) return;
const childMedia = props.virtualMedia?.children?.[0];
if (!childMedia) return;
// End incoming pass
pass.end();
const frame = props.frame ?? 0;
const fps = props.fps || 30;
const width = targetWidth;
const height = targetHeight;
const resolveField = (
handleId: string | null | undefined,
fallbackVal: number,
clampMin = 0,
clampMax = 1000,
) =>
resolveHandleParam<MaskMathSignalData>(
op.inputs,
handleId,
fallbackVal,
clampMin,
clampMax,
);
const radiusRes = resolveField(
op.radiusHandleId,
Number(op.radius ?? 0),
0,
MAX_RADIUS,
);
const thresholdRes = resolveField(
op.thresholdHandleId,
Number(op.threshold ?? 0.5),
0.0,
1.0,
);
const clampMinRes = resolveField(
op.clampMinHandleId,
Number(op.clampMin ?? 0.0),
0.0,
1.0,
);
const clampMaxRes = resolveField(
op.clampMaxHandleId,
Number(op.clampMax ?? 1.0),
0.0,
1.0,
);
// 1. Render primary Mask A child media
const { texture: tmpATex, view: tmpAView } = await drawChildToTexture(
args,
childMedia,
{
props: {
...props,
virtualMedia: childMedia,
renderId: `${props.renderId}-maskA`,
},
},
);
// 2. Render optional Mask B child media if connected
let tmpBTex: GPUTexture | null = null;
if (op.maskBMedia) {
({ texture: tmpBTex } = await drawChildToTexture(
args,
op.maskBMedia as VirtualMediaData,
{
props: {
...props,
virtualMedia: op.maskBMedia as VirtualMediaData,
renderId: `${props.renderId}-maskB`,
excludeTextures: [...(props.excludeTextures || []), tmpATex],
},
exclude: [tmpATex],
},
));
}
// 3. Compile pipeline resources
const { pipeline, uniformLayout, textureLayout, signalTextureLayout } =
getMaskMathResources(ctx.device, ctx.renderer.format);
const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);
const { elapsedSeconds, durationSeconds } = signalTiming(props, frame, fps);
const signalContext = {
args,
width,
height,
frame,
fps,
elapsedSeconds,
durationSeconds,
};
const getSignalView = (
res: {
val: number;
hasStaticSig: boolean;
sd: MaskMathSignalData | null;
handleId?: string | null;
},
suffix: string,
) =>
signalTextureView(
signalContext,
res,
res.handleId ?? `${props.renderId ?? "maskmath"}_${suffix}`,
);
const radiusView = getSignalView(radiusRes, "radius_sig");
const threshView = getSignalView(thresholdRes, "threshold_sig");
const clampMinView = getSignalView(clampMinRes, "clampMin_sig");
const clampMaxView = getSignalView(clampMaxRes, "clampMax_sig");
const sigBindGroup = ctx.device.createBindGroup({
layout: signalTextureLayout,
entries: [
{ binding: 0, resource: radiusView },
{ binding: 1, resource: threshView },
{ binding: 2, resource: clampMinView },
{ binding: 3, resource: clampMaxView },
{ binding: 4, resource: sampler },
],
});
const operationCode = OPERATION_MAP[op.operation ?? "Union"] ?? 0;
const channelACode = CHANNEL_MAP[op.channelA ?? "Alpha"] ?? 0;
const channelBCode = CHANNEL_MAP[op.channelB ?? "Alpha"] ?? 0;
const outputFormatCode =
OUTPUT_FORMAT_MAP[op.outputFormat ?? "WhiteWithAlpha"] ?? 0;
const dummyBView = tmpBTex ? tmpBTex.createView() : tmpAView;
// Populate separate uniform data buffers for each pass
const hUniformData = new Float32Array(20);
const vUniformData = new Float32Array(20);
const baseUniforms = [
0.0, // [0] dirX (set per pass)
0.0, // [1] dirY (set per pass)
radiusRes.val, // [2] radius
thresholdRes.val, // [3] threshold
clampMinRes.val, // [4] clampMin
clampMaxRes.val, // [5] clampMax
operationCode, // [6] operation
channelACode, // [7] channelA
channelBCode, // [8] channelB
op.binarize ? 1.0 : 0.0, // [9] binarize
op.invertResult ? 1.0 : 0.0, // [10] invertResult
outputFormatCode, // [11] outputFormat
tmpBTex ? 1.0 : 0.0, // [12] hasMaskB
radiusRes.hasStaticSig ? 1.0 : 0.0, // [13] hasRadiusSig
thresholdRes.hasStaticSig ? 1.0 : 0.0, // [14] hasThreshSig
clampMinRes.hasStaticSig ? 1.0 : 0.0, // [15] hasClampMinSig
clampMaxRes.hasStaticSig ? 1.0 : 0.0, // [16] hasClampMaxSig
0.0, // [17] isSecondPass (set per pass)
0.0, // [18] pad0
0.0, // [19] pad1
];
hUniformData.set(baseUniforms);
hUniformData[0] = 1.0; // dirX
hUniformData[1] = 0.0; // dirY
hUniformData[17] = 0.0; // isSecondPass = false
vUniformData.set(baseUniforms);
vUniformData[0] = 0.0; // dirX
vUniformData[1] = 1.0; // dirY
vUniformData[17] = 1.0; // isSecondPass = true
const outTex1 = ctx.renderer.getTemporaryTexture(width, height, [
tmpATex,
...(tmpBTex ? [tmpBTex] : []),
targetTexture,
...(props.excludeTextures || []),
]);
const outTex2 = ctx.renderer.getTemporaryTexture(width, height, [
tmpATex,
...(tmpBTex ? [tmpBTex] : []),
outTex1,
targetTexture,
...(props.excludeTextures || []),
]);
// Pass 1: Horizontal Pass (or direct combination)
const hBuffer = ctx.renderer.getTemporaryBuffer(hUniformData);
drawFullscreenPass(encoder, outTex1.createView(), pipeline, [
ctx.device.createBindGroup({
layout: uniformLayout,
entries: [{ binding: 0, resource: { buffer: hBuffer } }],
}),
ctx.device.createBindGroup({
layout: textureLayout,
entries: [
{ binding: 0, resource: tmpAView },
{ binding: 1, resource: dummyBView },
{ binding: 2, resource: tmpAView },
{ binding: 3, resource: sampler },
],
}),
sigBindGroup,
]);
// Pass 2: Vertical Pass + post-processing
const vBuffer = ctx.renderer.getTemporaryBuffer(vUniformData);
drawFullscreenPass(encoder, outTex2.createView(), pipeline, [
ctx.device.createBindGroup({
layout: uniformLayout,
entries: [{ binding: 0, resource: { buffer: vBuffer } }],
}),
ctx.device.createBindGroup({
layout: textureLayout,
entries: [
{ binding: 0, resource: outTex1.createView() },
{ binding: 1, resource: dummyBView },
{ binding: 2, resource: tmpAView },
{ binding: 3, resource: sampler },
],
}),
sigBindGroup,
]);
// 4. Final Pass: draw output into targetView
const finalOpacity = props.opacity ?? op.opacity ?? 1.0;
drawTextureToTarget(args, outTex2, finalOpacity);
};