Code-first video & image, rendered natively on WebGPU.
/// <reference types="webgpu" />
import type { SignalData } from "@framefields/core";
import {
resolveHandleParam,
type WebGPUNodeRenderer,
} from "@framefields/node-sdk";
import {
createFullscreenPipeline,
drawChildToTexture,
drawFullscreenPass,
drawTextureToTarget,
FULLSCREEN_QUAD_WGSL,
fragmentBindGroupLayout,
signalTextureView,
signalTiming,
} from "@framefields/node-sdk/gpu";
import type { ColorBalanceOperation } from "../shared/config.js";
type ColorBalanceSignalData = SignalData & { nodeId?: string };
interface ColorBalanceOpWithHandles extends ColorBalanceOperation {
inputs?: Record<
string,
{
connectionValid: boolean;
outputItem: {
type: string;
data: unknown;
} | null;
}
>;
}
interface DeviceColorBalanceResources {
pipeline: GPURenderPipeline;
uniformLayout: GPUBindGroupLayout;
textureLayout: GPUBindGroupLayout;
signalTextureLayout: GPUBindGroupLayout;
}
const deviceResourceCache = new WeakMap<
GPUDevice,
DeviceColorBalanceResources
>();
const WGSL_COLOR_BALANCE_SHADER = `
struct ColorBalanceUniforms {
shadows : vec4<f32>,
midtones : vec4<f32>,
highlights : vec4<f32>,
preserveLuminosity : f32,
hasShadowsCyanRedSig : f32,
hasShadowsMagentaGreenSig : f32,
hasShadowsYellowBlueSig : f32,
hasMidtonesCyanRedSig : f32,
hasMidtonesMagentaGreenSig : f32,
hasMidtonesYellowBlueSig : f32,
hasHighlightsCyanRedSig : f32,
hasHighlightsMagentaGreenSig : f32,
hasHighlightsYellowBlueSig : f32,
_pad0 : f32,
_pad1 : f32,
};
@group(0) @binding(0) var<uniform> u : ColorBalanceUniforms;
@group(1) @binding(0) var tex : texture_2d<f32>;
@group(1) @binding(1) var samp : sampler;
@group(2) @binding(0) var shadowsCyanRedSigTex : texture_2d<f32>;
@group(2) @binding(1) var shadowsMagentaGreenSigTex : texture_2d<f32>;
@group(2) @binding(2) var shadowsYellowBlueSigTex : texture_2d<f32>;
@group(2) @binding(3) var midtonesCyanRedSigTex : texture_2d<f32>;
@group(2) @binding(4) var midtonesMagentaGreenSigTex: texture_2d<f32>;
@group(2) @binding(5) var midtonesYellowBlueSigTex : texture_2d<f32>;
@group(2) @binding(6) var highlightsCyanRedSigTex : texture_2d<f32>;
@group(2) @binding(7) var highlightsMagentaGreenSigTex: texture_2d<f32>;
@group(2) @binding(8) var highlightsYellowBlueSigTex: texture_2d<f32>;
@group(2) @binding(9) var signalSamp : sampler;
${FULLSCREEN_QUAD_WGSL}
fn get_luminance(c: vec3<f32>) -> f32 {
return dot(c, vec3<f32>(0.2126, 0.7152, 0.0722));
}
fn set_luminance(c: vec3<f32>, l: f32) -> vec3<f32> {
let d = l - get_luminance(c);
var res = c + vec3<f32>(d);
let min_c = min(res.r, min(res.g, res.b));
let max_c = max(res.r, max(res.g, res.b));
if (min_c < 0.0) {
res = l + ((res - vec3<f32>(l)) * l) / max(l - min_c, 1e-5);
}
if (max_c > 1.0) {
res = l + ((res - vec3<f32>(l)) * (1.0 - l)) / max(max_c - l, 1e-5);
}
return clamp(res, vec3<f32>(0.0), vec3<f32>(1.0));
}
@fragment fn fs(in : VSOut) -> @location(0) vec4<f32> {
let color = textureSampleLevel(tex, samp, in.uv, 0.0);
if (color.a <= 0.00001) {
return color;
}
let alpha = color.a;
let rgb = color.rgb / max(alpha, 0.0001);
var sShift = u.shadows.rgb;
if (u.hasShadowsCyanRedSig > 0.5) {
sShift.r = clamp(textureSampleLevel(shadowsCyanRedSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
if (u.hasShadowsMagentaGreenSig > 0.5) {
sShift.g = clamp(textureSampleLevel(shadowsMagentaGreenSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
if (u.hasShadowsYellowBlueSig > 0.5) {
sShift.b = clamp(textureSampleLevel(shadowsYellowBlueSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
var mShift = u.midtones.rgb;
if (u.hasMidtonesCyanRedSig > 0.5) {
mShift.r = clamp(textureSampleLevel(midtonesCyanRedSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
if (u.hasMidtonesMagentaGreenSig > 0.5) {
mShift.g = clamp(textureSampleLevel(midtonesMagentaGreenSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
if (u.hasMidtonesYellowBlueSig > 0.5) {
mShift.b = clamp(textureSampleLevel(midtonesYellowBlueSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
var hShift = u.highlights.rgb;
if (u.hasHighlightsCyanRedSig > 0.5) {
hShift.r = clamp(textureSampleLevel(highlightsCyanRedSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
if (u.hasHighlightsMagentaGreenSig > 0.5) {
hShift.g = clamp(textureSampleLevel(highlightsMagentaGreenSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
if (u.hasHighlightsYellowBlueSig > 0.5) {
hShift.b = clamp(textureSampleLevel(highlightsYellowBlueSigTex, signalSamp, vec2<f32>(0.5, 0.5), 0.0).r, -100.0, 100.0) / 100.0;
}
let origLum = get_luminance(rgb);
// Tonal range weighting curves
let wShadows = clamp((1.0 - origLum) * (1.0 - origLum * 1.5), 0.0, 1.0);
let wMidtones = clamp(1.0 - 4.0 * (origLum - 0.5) * (origLum - 0.5), 0.0, 1.0);
let wHighlights = clamp(origLum * (origLum * 1.5 - 0.5), 0.0, 1.0);
let delta = wShadows * sShift + wMidtones * mShift + wHighlights * hShift;
var adjustedRgb = clamp(rgb + delta, vec3<f32>(0.0), vec3<f32>(1.0));
if (u.preserveLuminosity > 0.5) {
adjustedRgb = set_luminance(adjustedRgb, origLum);
}
return vec4<f32>(adjustedRgb * alpha, alpha);
}
`;
function getColorBalanceResources(
device: GPUDevice,
format: GPUTextureFormat,
): DeviceColorBalanceResources {
let res = deviceResourceCache.get(device);
if (res) return res;
const uniformLayout = fragmentBindGroupLayout(
device,
["uniform"],
"ColorBalanceUniformLayout",
);
const textureLayout = fragmentBindGroupLayout(
device,
["texture", "sampler"],
"ColorBalanceTextureLayout",
);
const signalTextureLayout = fragmentBindGroupLayout(
device,
[
"texture",
"texture",
"texture",
"texture",
"texture",
"texture",
"texture",
"texture",
"texture",
"sampler",
],
"ColorBalanceSignalTextureLayout",
);
const module = device.createShaderModule({
label: "ColorBalanceShaderModule",
code: WGSL_COLOR_BALANCE_SHADER,
});
const pipeline = createFullscreenPipeline(device, {
label: "ColorBalancePipeline",
module,
layouts: [uniformLayout, textureLayout, signalTextureLayout],
format,
});
res = {
pipeline,
uniformLayout,
textureLayout,
signalTextureLayout,
};
deviceResourceCache.set(device, res);
return res;
}
const uniformBufferData = new Float32Array(24);
export const ColorBalanceWebGPURenderer: WebGPUNodeRenderer = async (args) => {
const {
ctx,
encoder,
pass,
targetTexture,
targetWidth,
targetHeight,
props,
} = args;
const op = props.virtualMedia?.operation as unknown as
| ColorBalanceOpWithHandles
| undefined;
if (op?.op !== "ColorBalance" || !props.virtualMedia?.children?.[0]) return;
pass.end();
const frame = props.frame ?? 0;
const compositionFrame = props.compositionFrame ?? frame;
const fps = props.fps || 30;
const childMedia = props.virtualMedia.children[0];
const width = targetWidth;
const height = targetHeight;
const nodeId =
(props.virtualMedia?.metadata as { id?: string } | undefined)?.id ||
(op.inputs ? Object.keys(op.inputs)[0] : null) ||
props.renderId ||
"colorbalance_node";
const resolveField = (
handleId: string | null | undefined,
fallbackVal: number,
clampMin = -100,
clampMax = 100,
) =>
resolveHandleParam<ColorBalanceSignalData>(
op.inputs,
handleId,
fallbackVal,
clampMin,
clampMax,
);
const resolveVal = (val: unknown): number => {
if (val !== undefined && val !== null) {
if (typeof val === "object" && val !== null) {
if (
"get" in (val as Record<string, unknown>) &&
typeof (val as { get: unknown }).get === "function"
) {
val = (
val as { get: (ctx: { frame: number; fps: number }) => unknown }
).get({ frame: compositionFrame, fps });
} else if ("value" in (val as Record<string, unknown>)) {
val = (val as { value: unknown }).value;
}
} else if (typeof val === "function") {
val = (val as (f: number, fps: number) => unknown)(
compositionFrame,
fps,
);
}
}
const n = Number(val ?? 0);
return isNaN(n) ? 0 : n;
};
const sCR = resolveField(
op.shadows_cyanRedHandleId,
resolveVal(op.shadows?.cyanRed),
);
const sMG = resolveField(
op.shadows_magentaGreenHandleId,
resolveVal(op.shadows?.magentaGreen),
);
const sYB = resolveField(
op.shadows_yellowBlueHandleId,
resolveVal(op.shadows?.yellowBlue),
);
const mCR = resolveField(
op.midtones_cyanRedHandleId,
resolveVal(op.midtones?.cyanRed),
);
const mMG = resolveField(
op.midtones_magentaGreenHandleId,
resolveVal(op.midtones?.magentaGreen),
);
const mYB = resolveField(
op.midtones_yellowBlueHandleId,
resolveVal(op.midtones?.yellowBlue),
);
const hCR = resolveField(
op.highlights_cyanRedHandleId,
resolveVal(op.highlights?.cyanRed),
);
const hMG = resolveField(
op.highlights_magentaGreenHandleId,
resolveVal(op.highlights?.magentaGreen),
);
const hYB = resolveField(
op.highlights_yellowBlueHandleId,
resolveVal(op.highlights?.yellowBlue),
);
const { texture: tmpTex } = await drawChildToTexture(args, childMedia);
const { pipeline, uniformLayout, textureLayout, signalTextureLayout } =
getColorBalanceResources(ctx.device, ctx.renderer.format);
const sampler = ctx.renderer.samplerCache.getSampler(ctx.device);
const { elapsedSeconds, durationSeconds } = signalTiming(props, frame, fps);
const signalContext = {
args,
width,
height,
frame,
fps,
elapsedSeconds,
durationSeconds,
};
const getSignalView = (
res: {
val: number;
hasStaticSig: boolean;
sd: ColorBalanceSignalData | null;
},
suffix: string,
) => signalTextureView(signalContext, res, `${nodeId}_${suffix}`);
const sCRView = getSignalView(sCR, "shadows_cyanRed_sig");
const sMGView = getSignalView(sMG, "shadows_magentaGreen_sig");
const sYBView = getSignalView(sYB, "shadows_yellowBlue_sig");
const mCRView = getSignalView(mCR, "midtones_cyanRed_sig");
const mMGView = getSignalView(mMG, "midtones_magentaGreen_sig");
const mYBView = getSignalView(mYB, "midtones_yellowBlue_sig");
const hCRView = getSignalView(hCR, "highlights_cyanRed_sig");
const hMGView = getSignalView(hMG, "highlights_magentaGreen_sig");
const hYBView = getSignalView(hYB, "highlights_yellowBlue_sig");
const preserveLum = op.preserveLuminosity === false ? 0.0 : 1.0;
uniformBufferData[0] = sCR.val / 100;
uniformBufferData[1] = sMG.val / 100;
uniformBufferData[2] = sYB.val / 100;
uniformBufferData[3] = 0.0;
uniformBufferData[4] = mCR.val / 100;
uniformBufferData[5] = mMG.val / 100;
uniformBufferData[6] = mYB.val / 100;
uniformBufferData[7] = 0.0;
uniformBufferData[8] = hCR.val / 100;
uniformBufferData[9] = hMG.val / 100;
uniformBufferData[10] = hYB.val / 100;
uniformBufferData[11] = 0.0;
uniformBufferData[12] = preserveLum;
uniformBufferData[13] = sCR.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[14] = sMG.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[15] = sYB.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[16] = mCR.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[17] = mMG.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[18] = mYB.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[19] = hCR.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[20] = hMG.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[21] = hYB.hasStaticSig ? 1.0 : 0.0;
uniformBufferData[22] = 0.0;
uniformBufferData[23] = 0.0;
const buffer = ctx.renderer.getTemporaryBuffer(uniformBufferData);
const outTex = ctx.renderer.getTemporaryTexture(width, height, [
tmpTex,
targetTexture,
...(props.excludeTextures || []),
]);
const sigBindGroup = ctx.device.createBindGroup({
layout: signalTextureLayout,
entries: [
{ binding: 0, resource: sCRView },
{ binding: 1, resource: sMGView },
{ binding: 2, resource: sYBView },
{ binding: 3, resource: mCRView },
{ binding: 4, resource: mMGView },
{ binding: 5, resource: mYBView },
{ binding: 6, resource: hCRView },
{ binding: 7, resource: hMGView },
{ binding: 8, resource: hYBView },
{ binding: 9, resource: sampler },
],
});
drawFullscreenPass(encoder, outTex.createView(), pipeline, [
ctx.device.createBindGroup({
layout: uniformLayout,
entries: [{ binding: 0, resource: { buffer } }],
}),
ctx.renderer.bindGroupCache.getBindGroup(
ctx.device,
textureLayout,
tmpTex,
sampler,
),
sigBindGroup,
]);
drawTextureToTarget(args, outTex, props.opacity ?? 1);
};