Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add Shader Paint functionality and related components

konstantin-paulus81d29d5d8804
diff --git a/apps/cli/src/compile-project.ts b/apps/cli/src/compile-project.ts
index fda7ee2e..d7d8ad6c 100644
--- a/apps/cli/src/compile-project.ts
+++ b/apps/cli/src/compile-project.ts
@@ -103,6 +103,7 @@ const BUILT_IN_TAGS: Record<string, string> = {
solidPaint: "SolidPaint", linearGradientPaint: "LinearGradientPaint",
radialGradientPaint: "RadialGradientPaint", colorStop: "ColorStop",
htmlPaint: "HtmlPaint", html: "Html", surfacePaint: "SurfacePaint", surface: "Surface",
+ shaderPaint: "ShaderPaint",
};
// Tags that exist in both vocabularies. Every collision is SVG-only (HTML has
diff --git a/apps/web/src/components/engine/api/observers.ts b/apps/web/src/components/engine/api/observers.ts
index 9f584a0f..dd8f4036 100644
--- a/apps/web/src/components/engine/api/observers.ts
+++ b/apps/web/src/components/engine/api/observers.ts
@@ -118,6 +118,12 @@ export function observeWorld(world: EngineWorld) {
utils.persistEntity(world, eid);
});
+ observe(world, "set", c.Shader, (eid, { code, uniforms }) => {
+ c.Shader.code[eid] = code ?? c.Shader.code[eid] ?? "";
+ c.Shader.uniforms[eid] = uniforms ?? c.Shader.uniforms[eid] ?? null;
+ utils.persistEntity(world, eid);
+ });
+
observe(world, "set", c.Name, (eid, name) => {
c.Name[eid] = name ?? c.Name[eid] ?? "";
utils.persistEntity(world, eid);
@@ -496,6 +502,7 @@ export function observeWorld(world: EngineWorld) {
utils.disposeDecoders(world, eid);
utils.disposeHtmlHosts(world, eid);
utils.disposeSurfaceHosts(world, eid);
+ utils.disposeShaderHosts(world, eid);
utils.disconnectAudioBus(world, eid);
utils.unpersistEntity(world, eid);
world.rebuildTimelineIndex();
diff --git a/apps/web/src/components/engine/api/serialize.ts b/apps/web/src/components/engine/api/serialize.ts
index 9cb7de03..0aefe3d0 100644
--- a/apps/web/src/components/engine/api/serialize.ts
+++ b/apps/web/src/components/engine/api/serialize.ts
@@ -133,6 +133,12 @@ export function serializeEntity(world: EngineWorld, eid: number): DBEntity {
if (hasComponent(world, eid, c.ScaleMode)) {
record.ScaleMode = c.ScaleMode[eid];
}
+ if (hasComponent(world, eid, c.Shader)) {
+ record.Shader = {
+ code: c.Shader.code[eid],
+ uniforms: c.Shader.uniforms[eid] ?? undefined,
+ };
+ }
if (hasComponent(world, eid, c.Volume)) {
record.Volume = c.Volume[eid];
}
@@ -347,6 +353,9 @@ export function deserializeEntity(world: EngineWorld, eid: number, e: Partial<DB
if (e.ScaleMode !== undefined) {
setComponent(world, eid, c.ScaleMode, e.ScaleMode);
}
+ if (e.Shader !== undefined) {
+ setComponent(world, eid, c.Shader, e.Shader);
+ }
if (e.Volume !== undefined) {
setComponent(world, eid, c.Volume, e.Volume);
}
diff --git a/apps/web/src/components/engine/api/utils.ts b/apps/web/src/components/engine/api/utils.ts
index 25b1fbc0..76ea253e 100644
--- a/apps/web/src/components/engine/api/utils.ts
+++ b/apps/web/src/components/engine/api/utils.ts
@@ -171,6 +171,17 @@ export function disposeSurfaceHosts(world: EngineWorld, eid: number): void {
}
}
+export function disposeShaderHosts(world: EngineWorld, eid: number): void {
+ const ShaderHost = world.components.ShaderHost;
+
+ ShaderHost[eid]?.dispose();
+ ShaderHost[eid] = null;
+
+ for (const child of query(world, [ChildOf(eid)])) {
+ disposeShaderHosts(world, child);
+ }
+}
+
export function disconnectAudioBus(world: EngineWorld, eid: number): void {
const AudioBus = world.components.AudioBus;
AudioBus[eid]?.disconnect();
diff --git a/apps/web/src/components/engine/components/components.ts b/apps/web/src/components/engine/components/components.ts
index e1b6fae4..517ca8d7 100644
--- a/apps/web/src/components/engine/components/components.ts
+++ b/apps/web/src/components/engine/components/components.ts
@@ -8,6 +8,7 @@ import type { Token } from "../utils/text";
import type { AudioBus as AudioBusInstance } from "../services/audio-bus";
import type { HtmlHost as HtmlHostInstance } from "../decoders/html";
import type { SurfaceHost as SurfaceHostInstance } from "../decoders/surface";
+import type { ShaderHost as ShaderHostInstance } from "../decoders/shader";
import type { ImageDecoder as ImageDecoderInstance } from "../decoders/image";
import type { VideoDecoderInstance } from "../decoders/video";
import type { SequenceDecoder as SequenceDecoderInstance } from "../decoders/sequence";
@@ -381,6 +382,13 @@ export const HtmlHost = [] as (HtmlHostInstance | null)[];
export const SurfaceHost = [] as (SurfaceHostInstance | null)[];
+export const Shader = {
+ code: [] as string[],
+ uniforms: [] as (Record<string, number | number[] | string> | null)[],
+};
+
+export const ShaderHost = [] as (ShaderHostInstance | null)[];
+
// Renderer-runtime state on mounted entities (never serialized)
export type MountData = {
props: Record<string, unknown>;
diff --git a/apps/web/src/components/engine/components/constants.ts b/apps/web/src/components/engine/components/constants.ts
index 54147082..659094b3 100644
--- a/apps/web/src/components/engine/components/constants.ts
+++ b/apps/web/src/components/engine/components/constants.ts
@@ -67,6 +67,7 @@ export enum PaintType {
WAVEFORM,
HTML,
SURFACE,
+ SHADER,
}
export enum EffectType {
diff --git a/apps/web/src/components/engine/decoders/index.ts b/apps/web/src/components/engine/decoders/index.ts
index 30d1f985..85b09456 100644
--- a/apps/web/src/components/engine/decoders/index.ts
+++ b/apps/web/src/components/engine/decoders/index.ts
@@ -8,5 +8,6 @@ export * from './caption';
export * from './html';
export * from './image';
export * from './sequence';
+export * from './shader';
export * from './time-stretcher';
export * from './video';
diff --git a/apps/web/src/components/engine/decoders/shader.ts b/apps/web/src/components/engine/decoders/shader.ts
new file mode 100644
index 00000000..8f6afff3
--- /dev/null
+++ b/apps/web/src/components/engine/decoders/shader.ts
@@ -0,0 +1,374 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { addComponent } from "../api/events";
+import { parseColor } from "@/utils/color";
+
+import type { EngineWorld } from "../api/world";
+
+type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+type UniformValue = number | number[] | string;
+
+/**
+ * Runtime pipeline for one shader paint: compiles the paint's fragment-stage
+ * WGSL against the engine prelude, samples the media paint directly below the
+ * shader in the fill stack into a texture, runs the pipeline into an
+ * offscreen WebGPU canvas, and hands the result back to the Canvas2D
+ * compositor via `drawImage`. Created lazily by `resolveShaderHost`, exactly
+ * like the media decoders.
+ *
+ * The authored contract (see reference/jsx/shader-paint.md): the user writes
+ * `@fragment fn main(@location(0) uv: vec2f) -> @location(0) vec4f` with `uv`
+ * in box space, samples the media through `sampleSource(uv)` (which honors
+ * the media paint's `objectFit` placement), and declares parameters as
+ * `@group(1)` uniforms whose values arrive by name from the `uniforms` prop.
+ */
+
+// One device for every shader paint in the page; pipelines are per-host.
+let devicePromise: Promise<GPUDevice | null> | null = null;
+
+function sharedDevice(): Promise<GPUDevice | null> {
+ devicePromise ??= (async () => {
+ const adapter = await navigator.gpu?.requestAdapter();
+ if (!adapter) {
+ console.error("[shader-paint] WebGPU is not available; shader paints render as passthrough");
+ return null;
+ }
+ return await adapter.requestDevice();
+ })();
+ return devicePromise;
+}
+
+const PRELUDE = /* wgsl */ `
+struct Globals {
+ resolution: vec2f,
+ time: f32,
+ _pad: f32,
+ fitOffset: vec2f,
+ fitSize: vec2f,
+}
+
+@group(0) @binding(0) var source: texture_2d<f32>;
+@group(0) @binding(1) var sourceSampler: sampler;
+@group(0) @binding(2) var<uniform> globals: Globals;
+
+struct VSOut {
+ @builtin(position) position: vec4f,
+ @location(0) uv: vec2f,
+}
+
+@vertex
+fn vs(@builtin(vertex_index) i: u32) -> VSOut {
+ var corners = array<vec2f, 3>(vec2f(-1.0, -1.0), vec2f(3.0, -1.0), vec2f(-1.0, 3.0));
+ var out: VSOut;
+ out.position = vec4f(corners[i], 0.0, 1.0);
+ out.uv = vec2f((corners[i].x + 1.0) * 0.5, 1.0 - (corners[i].y + 1.0) * 0.5);
+ return out;
+}
+
+// Samples the media in box space: uv spans the parent's box, the media sits
+// within it per objectFit, and letterbox areas return transparent black.
+fn sampleSource(uv: vec2f) -> vec4f {
+ let suv = (uv - globals.fitOffset) / globals.fitSize;
+ let inside = all(suv >= vec2f(0.0)) && all(suv <= vec2f(1.0));
+ let color = textureSampleLevel(source, sourceSampler, clamp(suv, vec2f(0.0), vec2f(1.0)), 0.0);
+ return select(vec4f(0.0), color, inside);
+}
+`;
+
+const UNIFORM_COMPONENTS: Record<string, number> = {
+ f32: 1,
+ vec2f: 2, "vec2<f32>": 2,
+ vec3f: 3, "vec3<f32>": 3,
+ vec4f: 4, "vec4<f32>": 4,
+};
+
+const UNIFORM_RE = /@group\(1\)\s*@binding\((\d+)\)\s*var\s*<\s*uniform\s*>\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*([A-Za-z0-9_]+(?:<[A-Za-z0-9_]+>)?)/g;
+
+type UniformSlot = {
+ name: string;
+ binding: number;
+ components: number;
+ buffer: GPUBuffer;
+};
+
+type Pipeline = {
+ device: GPUDevice;
+ pipeline: GPURenderPipeline;
+ sampler: GPUSampler;
+ globals: GPUBuffer;
+ slots: UniformSlot[];
+ group1: GPUBindGroup;
+ group0Layout: GPUBindGroupLayout;
+};
+
+export class ShaderHost {
+ private canvas = new OffscreenCanvas(1, 1);
+ private context: GPUCanvasContext | null = null;
+ private gpu: Pipeline | null = null;
+ private sourceTexture: GPUTexture | null = null;
+ private group0: GPUBindGroup | null = null;
+ private code = "";
+ private generation = 0;
+ private compilePromise: Promise<void> | null = null;
+ private colorCache = new Map<string, number | null>();
+ private disposed = false;
+
+ /** Compiles when the code differs from the current pipeline's. */
+ public setCode(code: string): void {
+ if (this.disposed || code === this.code) return;
+ this.code = code;
+ const gen = ++this.generation;
+ this.compilePromise = this.compile(code, gen).catch((error) => {
+ if (gen === this.generation) this.gpu = null;
+ console.error("[shader-paint]", error);
+ });
+ }
+
+ /** Resolves once the current code has compiled (or failed). */
+ public whenReady(): Promise<void> | null {
+ return this.compilePromise;
+ }
+
+ /** True when a pipeline exists — callers fall back to drawing the media directly otherwise. */
+ public get ready(): boolean {
+ return this.gpu !== null && !this.disposed;
+ }
+
+ private async compile(code: string, gen: number): Promise<void> {
+ const device = await sharedDevice();
+ if (!device || this.disposed) return;
+
+ const module = device.createShaderModule({ code: PRELUDE + code });
+ const info = await module.getCompilationInfo();
+ if (gen !== this.generation) return;
+
+ const preludeLines = PRELUDE.split("\n").length;
+ const errors = info.messages.filter((m) => m.type === "error");
+ if (errors.length > 0) {
+ this.gpu = null;
+ const details = errors
+ .map((m) => `${Math.max(1, m.lineNum - preludeLines + 1)}:${m.linePos} ${m.message}`)
+ .join("\n");
+ throw new Error(`WGSL compile failed:\n${details}`);
+ }
+
+ const group0Layout = device.createBindGroupLayout({
+ entries: [
+ { binding: 0, visibility: GPUShaderStage.FRAGMENT, texture: { sampleType: "float" } },
+ { binding: 1, visibility: GPUShaderStage.FRAGMENT, sampler: { type: "filtering" } },
+ { binding: 2, visibility: GPUShaderStage.FRAGMENT, buffer: { type: "uniform" } },
+ ],
+ });
+
+ // Declared @group(1) uniforms get one small buffer each, matched to the
+ // `uniforms` record by name. An explicit layout keeps declared-but-unused
+ // bindings valid (an "auto" layout would strip them).
+ const slots: UniformSlot[] = [];
+ for (const match of code.matchAll(UNIFORM_RE)) {
+ const components = UNIFORM_COMPONENTS[match[3]];
+ if (components === undefined) {
+ console.warn(`[shader-paint] uniform \`${match[2]}\` has unsupported type ${match[3]} (use f32/vec2f/vec3f/vec4f)`);
+ continue;
+ }
+ slots.push({
+ name: match[2],
+ binding: Number(match[1]),
+ components,
+ buffer: device.createBuffer({
+ size: components * 4,
+ usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+ }),
+ });
+ }
+
+ const group1Layout = device.createBindGroupLayout({
+ entries: slots.map((slot) => ({
+ binding: slot.binding,
+ visibility: GPUShaderStage.FRAGMENT,
+ buffer: { type: "uniform" as const },
+ })),
+ });
+
+ const pipeline = device.createRenderPipeline({
+ layout: device.createPipelineLayout({ bindGroupLayouts: [group0Layout, group1Layout] }),
+ vertex: { module, entryPoint: "vs" },
+ fragment: {
+ module,
+ entryPoint: "main",
+ targets: [{ format: navigator.gpu.getPreferredCanvasFormat() }],
+ },
+ });
+ if (gen !== this.generation || this.disposed) return;
+
+ this.gpu = {
+ device,
+ pipeline,
+ sampler: device.createSampler({ magFilter: "linear", minFilter: "linear" }),
+ globals: device.createBuffer({ size: 32, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST }),
+ slots,
+ group1: device.createBindGroup({
+ layout: group1Layout,
+ entries: slots.map((slot) => ({ binding: slot.binding, resource: { buffer: slot.buffer } })),
+ }),
+ group0Layout,
+ };
+ // The source texture and its bind group belong to the old pipeline's layout.
+ this.sourceTexture?.destroy();
+ this.sourceTexture = null;
+ this.group0 = null;
+ }
+
+ private writeUniforms(uniforms: Record<string, UniformValue> | null): void {
+ const gpu = this.gpu!;
+ for (const slot of gpu.slots) {
+ const value = uniforms?.[slot.name];
+ if (value === undefined) continue;
+ const data = new Float32Array(slot.components);
+ if (typeof value === "number") {
+ data[0] = value;
+ } else if (Array.isArray(value)) {
+ for (let i = 0; i < Math.min(slot.components, value.length); i++) data[i] = value[i];
+ } else {
+ if (!this.colorCache.has(value)) this.colorCache.set(value, parseColor(value));
+ const color = this.colorCache.get(value);
+ if (color == null) continue;
+ data[0] = ((color >> 16) & 0xff) / 255;
+ if (slot.components >= 2) data[1] = ((color >> 8) & 0xff) / 255;
+ if (slot.components >= 3) data[2] = (color & 0xff) / 255;
+ if (slot.components >= 4) data[3] = 1;
+ }
+ gpu.device.queue.writeBuffer(slot.buffer, 0, data);
+ }
+ }
+
+ /**
+ * Runs the shader over `source` and draws the result into the parent's
+ * box. `fit` is the media's drawImage placement within the box (from
+ * getScaledImageProps); `time` is the parent's local time in seconds.
+ * A null `source` runs the shader procedurally: the bound texture stays
+ * transparent black, so `sampleSource` returns vec4f(0) everywhere.
+ * Returns false when no pipeline is ready — the caller then draws the
+ * media directly instead.
+ */
+ public draw(
+ ctx: Ctx2D,
+ width: number,
+ height: number,
+ source: GPUCopyExternalImageSource | null,
+ sourceWidth: number,
+ sourceHeight: number,
+ fit: [dx: number, dy: number, sw: number, sh: number],
+ time: number,
+ uniforms: Record<string, UniformValue> | null,
+ ): boolean {
+ const gpu = this.gpu;
+ if (!gpu || this.disposed) return false;
+
+ const w = Math.max(1, Math.round(width));
+ const h = Math.max(1, Math.round(height));
+ if (this.canvas.width !== w || this.canvas.height !== h) {
+ this.canvas.width = w;
+ this.canvas.height = h;
+ }
+ if (this.context === null) {
+ this.context = this.canvas.getContext("webgpu");
+ if (!this.context) return false;
+ this.context.configure({
+ device: gpu.device,
+ format: navigator.gpu.getPreferredCanvasFormat(),
+ alphaMode: "premultiplied",
+ });
+ }
+
+ const sw = source ? Math.max(1, Math.round(sourceWidth)) : 1;
+ const sh = source ? Math.max(1, Math.round(sourceHeight)) : 1;
+ if (!this.sourceTexture || this.sourceTexture.width !== sw || this.sourceTexture.height !== sh || !this.group0) {
+ this.sourceTexture?.destroy();
+ this.sourceTexture = gpu.device.createTexture({
+ size: [sw, sh],
+ format: "rgba8unorm",
+ usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.RENDER_ATTACHMENT,
+ });
+ this.group0 = gpu.device.createBindGroup({
+ layout: gpu.group0Layout,
+ entries: [
+ { binding: 0, resource: this.sourceTexture.createView() },
+ { binding: 1, resource: gpu.sampler },
+ { binding: 2, resource: { buffer: gpu.globals } },
+ ],
+ });
+ }
+
+ if (source) {
+ gpu.device.queue.copyExternalImageToTexture(
+ { source },
+ { texture: this.sourceTexture, premultipliedAlpha: true },
+ [sw, sh],
+ );
+ }
+
+ const [dx, dy, dw, dh] = fit;
+ gpu.device.queue.writeBuffer(gpu.globals, 0, new Float32Array([
+ w, h, time, 0,
+ dx / w, dy / h, dw / w, dh / h,
+ ]));
+ this.writeUniforms(uniforms);
+
+ const encoder = gpu.device.createCommandEncoder();
+ const pass = encoder.beginRenderPass({
+ colorAttachments: [{
+ view: this.context.getCurrentTexture().createView(),
+ clearValue: { r: 0, g: 0, b: 0, a: 0 },
+ loadOp: "clear",
+ storeOp: "store",
+ }],
+ });
+ pass.setPipeline(gpu.pipeline);
+ pass.setBindGroup(0, this.group0);
+ pass.setBindGroup(1, gpu.group1);
+ pass.draw(3);
+ pass.end();
+ gpu.device.queue.submit([encoder.finish()]);
+
+ ctx.drawImage(this.canvas, 0, 0, width, height);
+ return true;
+ }
+
+ public dispose(): void {
+ if (this.disposed) return;
+ this.disposed = true;
+ this.generation++;
+ this.sourceTexture?.destroy();
+ this.sourceTexture = null;
+ if (this.gpu) {
+ this.gpu.globals.destroy();
+ for (const slot of this.gpu.slots) slot.buffer.destroy();
+ this.gpu = null;
+ }
+ this.context?.unconfigure();
+ this.canvas.width = 0;
+ this.canvas.height = 0;
+ }
+}
+
+/**
+ * The shader paint's host, created lazily like the media decoders and kept in
+ * sync with the persisted `Shader.code` component. Returns null until the
+ * paint carries code.
+ */
+export function resolveShaderHost(world: EngineWorld, fid: number): ShaderHost | null {
+ const c = world.components;
+ const code = c.Shader.code[fid];
+ if (!code) return null;
+
+ let host = c.ShaderHost[fid];
+ if (!host) {
+ addComponent(world, fid, c.ShaderHost, false);
+ host = new ShaderHost();
+ c.ShaderHost[fid] = host;
+ }
+ host.setCode(code);
+ return host;
+}
diff --git a/apps/web/src/components/engine/systems/playback.ts b/apps/web/src/components/engine/systems/playback.ts
index 7f106c82..926b09a9 100644
--- a/apps/web/src/components/engine/systems/playback.ts
+++ b/apps/web/src/components/engine/systems/playback.ts
@@ -4,7 +4,7 @@
import { hasComponent, Not, Or, query } from 'bitecs';
import { ChildOf, PaintType } from '../components';
-import { resolveAudioDecoder, resolveCaptionDecoder, resolveImageDecoder, resolveSequenceDecoder, resolveVideoDecoder } from '../decoders';
+import { resolveAudioDecoder, resolveCaptionDecoder, resolveImageDecoder, resolveSequenceDecoder, resolveShaderHost, resolveVideoDecoder } from '../decoders';
import { getParentEntity } from '../api';
import { getTransitionWindow } from '../utils/transition';
import { AudioBus } from '../services/audio-bus';
@@ -249,6 +249,11 @@ function forwardDecoders(world: EngineWorld, sid: number, eid: number): void {
if (c.Paint[fid] === PaintType.IMAGE && visualsEnabled) {
forwardImageDecoder(world, sid, eid, fid);
}
+
+ if (c.Paint[fid] === PaintType.SHADER && visualsEnabled) {
+ const ready = resolveShaderHost(world, fid)?.whenReady();
+ if (ready) world.promises?.push(ready);
+ }
}
}
diff --git a/apps/web/src/components/engine/systems/render.ts b/apps/web/src/components/engine/systems/render.ts
index fff3452c..83f6f92c 100644
--- a/apps/web/src/components/engine/systems/render.ts
+++ b/apps/web/src/components/engine/systems/render.ts
@@ -6,7 +6,7 @@ import { hasComponent, query, Not, Or } from 'bitecs';
import { ChildOf, COMPOSITE_OPERATIONS, EffectType, GeometryType, PaintType, ScaleMode, StrokeCap, StrokeJoin, TransitionType } from '../components';
import { getParentEntity } from '../api/base';
import { cubicBezier } from 'animejs';
-import { resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder, resolveCaptionDecoder } from '../decoders';
+import { resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder, resolveCaptionDecoder, resolveShaderHost } from '../decoders';
import { getAudioPeaks } from '../decoders/audio-peaks';
import { renderText } from '../utils/text';
import { getTransitionWindow } from '../utils/transition';
@@ -285,9 +285,11 @@ function buildEffects(world: EngineWorld, eid: number): string | null {
export function renderFills(world: EngineWorld, eid: number): void {
const c = world.components;
const ctx = world.ctx!;
+ const fills = c.Cache.fills[eid];
- for (const fid of c.Cache.fills[eid]) {
- if (hasComponent(world, fid, c.Hidden)) continue;
+ for (let index = 0; index < fills.length; index++) {
+ const fid = fills[index];
+ if (hasComponent(world, fid, c.Hidden) || shaderConsumesFill(world, eid, fills, index)) continue;
const savedAlpha = ctx.globalAlpha;
const savedCO = ctx.globalCompositeOperation;
const bi = c.Appearance.blendMode[fid] ?? 0;
@@ -390,6 +392,8 @@ export function renderFills(world: EngineWorld, eid: number): void {
ctx.fill();
} else if (c.Paint[fid] === PaintType.WAVEFORM) {
renderWaveform(world, eid, fid);
+ } else if (c.Paint[fid] === PaintType.SHADER) {
+ renderShaderFill(world, eid, fills, index);
}
ctx.globalCompositeOperation = savedCO;
@@ -397,6 +401,90 @@ export function renderFills(world: EngineWorld, eid: number): void {
}
}
+/** The current frame of a video/image paint, as a GPU-uploadable source. */
+function shaderSourceBitmap(
+ world: EngineWorld,
+ fid: number,
+ w: number,
+ h: number,
+): { source: GPUCopyExternalImageSource; width: number; height: number } | null {
+ const c = world.components;
+
+ if (c.Paint[fid] === PaintType.IMAGE) {
+ const bitmap = resolveImageDecoder(world, fid)?.decoder?.getBitmap(w, h);
+ return bitmap ? { source: bitmap, width: bitmap.width, height: bitmap.height } : null;
+ }
+ if (c.Paint[fid] === PaintType.VIDEO) {
+ const frame = resolveVideoDecoder(world, fid)?.toBitmap();
+ return frame ? { source: frame, width: frame.width, height: frame.height } : null;
+ }
+ return null;
+}
+
+/**
+ * Whether the fill at `index` is the input of a ready shader paint directly
+ * above it. Only the immediate neighbor counts (a hidden paint in between
+ * decouples the pair). Must mirror `renderShaderFill`'s input selection
+ * exactly — a consumed media paint that the shader then fails to draw would
+ * blank the element, so both sides check pipeline readiness and frame
+ * availability.
+ */
+function shaderConsumesFill(world: EngineWorld, eid: number, fills: number[], index: number): boolean {
+ const c = world.components;
+ const fid = fills[index];
+ if (c.Paint[fid] !== PaintType.IMAGE && c.Paint[fid] !== PaintType.VIDEO) return false;
+
+ const next = fills[index + 1];
+ if (next === undefined || c.Paint[next] !== PaintType.SHADER) return false;
+ if (hasComponent(world, next, c.Hidden)) return false;
+ if (!resolveShaderHost(world, next)?.ready) return false;
+
+ const w = c.Computed.width[eid];
+ const h = c.Computed.height[eid];
+ return shaderSourceBitmap(world, fid, w, h) !== null;
+}
+
+/**
+ * Draws a shader paint: the media paint directly below it in the fill stack
+ * is sampled as the shader's `source` texture and its output lands in the
+ * parent's box in the media paint's place. Without a media paint below the
+ * shader runs procedurally over a transparent source; before the pipeline is
+ * ready it draws nothing and the media, if any, draws normally.
+ */
+function renderShaderFill(world: EngineWorld, eid: number, fills: number[], index: number): void {
+ const c = world.components;
+ const ctx = world.ctx!;
+
+ const host = resolveShaderHost(world, fills[index]);
+ if (!host?.ready) return;
+
+ const w = c.Computed.width[eid];
+ const h = c.Computed.height[eid];
+
+ // A visible media paint directly below is the shader's input. Anything
+ // else (no fill below, a hidden one, a solid/gradient) runs the shader
+ // procedurally over a transparent source, stacking like a normal paint.
+ const mid = fills[index - 1];
+ const isMedia = mid !== undefined
+ && !hasComponent(world, mid, c.Hidden)
+ && (c.Paint[mid] === PaintType.IMAGE || c.Paint[mid] === PaintType.VIDEO);
+ let input: ReturnType<typeof shaderSourceBitmap> = null;
+ if (isMedia) {
+ input = shaderSourceBitmap(world, mid, w, h);
+ if (!input) return;
+ }
+
+ const fit = input
+ ? getScaledImageProps(c.ScaleMode[mid] ?? 0, input.width, input.height, w, h)
+ : [0, 0, w, h] as [number, number, number, number];
+ const time = (c.Computed.localTime[eid] ?? 0) / world.frameRate;
+
+ ctx.save();
+ ctx.clip();
+ host.draw(ctx, w, h, input?.source ?? null, input?.width ?? 1, input?.height ?? 1, fit, time, c.Shader.uniforms[fills[index]] ?? null);
+ ctx.restore();
+}
+
function renderShadows(world: EngineWorld, eid: number): void {
const ctx = world.ctx!;
const c = world.components;
diff --git a/apps/web/src/components/engine/types/db.ts b/apps/web/src/components/engine/types/db.ts
index 21e5d06a..20681265 100644
--- a/apps/web/src/components/engine/types/db.ts
+++ b/apps/web/src/components/engine/types/db.ts
@@ -82,6 +82,10 @@ export interface DBEntity {
Blur?: number;
AssetId?: string;
ScaleMode?: number;
+ Shader?: {
+ code?: string;
+ uniforms?: Record<string, number | number[] | string>;
+ };
Volume?: number;
Muted?: {};
ColorStop?: {
diff --git a/apps/web/src/components/sidebar-right/inspector/fill-row.tsx b/apps/web/src/components/sidebar-right/inspector/fill-row.tsx
index 58bcc426..5c740f56 100644
--- a/apps/web/src/components/sidebar-right/inspector/fill-row.tsx
+++ b/apps/web/src/components/sidebar-right/inspector/fill-row.tsx
@@ -35,6 +35,7 @@ const FILL_TYPE_LABELS: Record<PaintType, string> = {
[PaintType.WAVEFORM]: 'Waveform',
[PaintType.HTML]: 'Html',
[PaintType.SURFACE]: 'Surface',
+ [PaintType.SHADER]: 'Shader',
};
export function FillRow(props: FillRowProps) {
diff --git a/apps/web/src/components/timeline/layers/sub-item.tsx b/apps/web/src/components/timeline/layers/sub-item.tsx
index 2d9f685e..e37b7fec 100644
--- a/apps/web/src/components/timeline/layers/sub-item.tsx
+++ b/apps/web/src/components/timeline/layers/sub-item.tsx
@@ -45,6 +45,7 @@ export function SubItemLayer(props: SubItemLayerProps) {
if (c.Paint[eid()] === PaintType.WAVEFORM) return 'Waveform';
if (c.Paint[eid()] === PaintType.HTML) return 'HTML';
if (c.Paint[eid()] === PaintType.SURFACE) return 'Surface';
+ if (c.Paint[eid()] === PaintType.SHADER) return 'Shader';
return 'Fill';
}
if (hasComponent(world, eid(), c.ColorStop)) return 'Stop';
diff --git a/apps/web/src/components/ui/fill-item.tsx b/apps/web/src/components/ui/fill-item.tsx
index cc5e6168..65c66986 100644
--- a/apps/web/src/components/ui/fill-item.tsx
+++ b/apps/web/src/components/ui/fill-item.tsx
@@ -407,7 +407,7 @@ function PaintItemIcon(props: PaintItemIconProps) {
<OpacitySwatch color={color()!} opacity={opacity()} />
</Show>
- <Show when={type() === PaintType.HTML || type() === PaintType.SURFACE}>
+ <Show when={type() === PaintType.HTML || type() === PaintType.SURFACE || type() === PaintType.SHADER}>
<div class="flex size-full items-center justify-center bg-muted text-muted-foreground">
<Icon name="html-small" />
</div>
@@ -439,6 +439,8 @@ function FillLabel(props: FillLabelProps) {
return "HTML";
} else if (type() === PaintType.SURFACE) {
return "Surface";
+ } else if (type() === PaintType.SHADER) {
+ return "Shader";
} else {
const asset = assets.get(assetId() ?? "");
return asset?.name ?? "Unknown";
diff --git a/apps/web/src/context/editor-api/selection.ts b/apps/web/src/context/editor-api/selection.ts
index 50f92fe8..f71ab3e1 100644
--- a/apps/web/src/context/editor-api/selection.ts
+++ b/apps/web/src/context/editor-api/selection.ts
@@ -56,6 +56,7 @@ export function entityType(world: EngineWorld, eid: number): string {
case PaintType.VIDEO: return "videoPaint";
case PaintType.HTML: return "htmlPaint";
case PaintType.SURFACE: return "surfacePaint";
+ case PaintType.SHADER: return "shaderPaint";
default: return "paint";
}
}
diff --git a/apps/web/src/utils/jsx.ts b/apps/web/src/utils/jsx.ts
index 7ec2bf1f..112fa029 100644
--- a/apps/web/src/utils/jsx.ts
+++ b/apps/web/src/utils/jsx.ts
@@ -762,6 +762,9 @@ export class WorldDocument implements ProjectDocument<HostNode> {
appendChild(world, fid, eid);
this.registerHtmlHost(eid, host);
break;
+ } case "ShaderPaint": {
+ setComponent(world, eid, c.Paint, PaintType.SHADER);
+ break;
} case "SurfacePaint": {
setComponent(world, eid, c.Paint, PaintType.SURFACE);
const host = this.trackHost(new SurfaceHost());
@@ -1195,6 +1198,35 @@ export class WorldDocument implements ProjectDocument<HostNode> {
setComponent(world, eid, c.Key, value.trim());
break;
}
+ case "wgsl": {
+ assert(c.Paint[eid] === PaintType.SHADER, "`wgsl` only applies to <ShaderPaint>");
+ assert(typeof value === "string", "`wgsl` must be a string" + `, value: ${value}`);
+ assert(value.trim().length > 0, "`wgsl` must be a non-empty string");
+ setComponent(world, eid, c.Shader, { code: value });
+ break;
+ }
+ case "uniforms": {
+ assert(c.Paint[eid] === PaintType.SHADER, "`uniforms` only applies to <ShaderPaint>");
+ assert(
+ typeof value === "object" && value !== null && !Array.isArray(value),
+ "`uniforms` must be an object of numbers, number arrays, or CSS colors" + `, value: ${value}`,
+ );
+ for (const [uniform, entry] of Object.entries(value)) {
+ if (typeof entry === "number") {
+ assert(Number.isFinite(entry), `uniform \`${uniform}\` must be finite, value: ${entry}`);
+ } else if (Array.isArray(entry)) {
+ assert(
+ entry.length >= 2 && entry.length <= 4 && entry.every((v) => typeof v === "number" && Number.isFinite(v)),
+ `uniform \`${uniform}\` must be an array of 2-4 finite numbers, value: ${entry}`,
+ );
+ } else {
+ assert(typeof entry === "string", `uniform \`${uniform}\` must be a number, number array, or CSS color, value: ${entry}`);
+ assert(parseColor(entry) !== null, `uniform \`${uniform}\` is not a valid CSS color: "${entry}"`);
+ }
+ }
+ setComponent(world, eid, c.Shader, { uniforms: { ...value } });
+ break;
+ }
case "fontFamily":
assert(typeof value === "string", "`fontFamily` must be a string" + `, value: ${value}`);
assert(value.trim().length > 0, "`fontFamily` must be a non-empty string");
@@ -1706,6 +1738,13 @@ export class WorldDocument implements ProjectDocument<HostNode> {
const objectFit = this.props(eid)?.objectFit as keyof typeof SCALE_MODE_MAP | undefined;
setComponent(world, fid, c.ScaleMode, objectFit ? SCALE_MODE_MAP[objectFit] : ScaleMode.COVER);
setComponent(world, fid, c.AssetId, asset.id);
+ // The asset resolves after the element's JSX paint children have
+ // attached, but the media paint belongs at the bottom of the stack:
+ // paint children draw over it, and a shader paint reads it.
+ const fills = query(world, [c.Paint, ChildOf(eid), Not(c.Deleted)]);
+ if (fills.length > 0) {
+ setComponent(world, fid, c.ItemIndex, Math.min(...fills.map((f) => c.ItemIndex[f] ?? 0)) - 1);
+ }
appendChild(world, fid, eid);
if (!hasComponent(world, eid, c.Size)) {
diff --git a/examples/08-shader-paint.tsx b/examples/08-shader-paint.tsx
new file mode 100644
index 00000000..8e96fb20
--- /dev/null
+++ b/examples/08-shader-paint.tsx
@@ -0,0 +1,46 @@
+/* @jsxImportSource @diffusionstudio/jsx */
+/* A shader paint post-processing a video: chromatic aberration pulsing with
+ * the playhead plus a vignette, driven by two uniforms.
+ *
+ * dapi mount examples/08-shader-paint.tsx
+ *
+ * <shaderPaint> applies its WGSL to the media paint directly below it in the
+ * paint stack (here: the paint created by the video's `src`). The engine
+ * provides `sampleSource(uv)` (box-space sampling that honors objectFit),
+ * `globals.time` (the clip's local time in seconds, playhead-driven, so
+ * scrubbing and exports stay frame-accurate), and binds the `uniforms` prop
+ * to the shader's @group(1) declarations by name. Patch them live:
+ *
+ * dapi node patch '[{ "id": <paintId>, "uniforms": { "strength": 3, "vignette": 0.8 } }]'
+ */
+
+const VIDEOS = "https://mdn.github.io/shared-assets/videos";
+
+const ABERRATION = /* wgsl */ `
+ @group(1) @binding(0) var<uniform> strength: f32;
+ @group(1) @binding(1) var<uniform> vignette: f32;
+
+ @fragment
+ fn main(@location(0) uv: vec2f) -> @location(0) vec4f {
+ let center = uv - vec2f(0.5);
+ let pulse = strength * (0.6 + 0.4 * sin(globals.time * 2.0));
+ let shift = center * pulse * 0.01;
+
+ let r = sampleSource(uv + shift).r;
+ let ga = sampleSource(uv);
+ let b = sampleSource(uv - shift).b;
+
+ let falloff = 1.0 - vignette * smoothstep(0.3, 0.75, length(center));
+ return vec4f(vec3f(r, ga.g, b) * falloff, ga.a);
+ }
+`;
+
+export default function ShaderPaintExample() {
+ return (
+ <scene key="example-shader-paint" name="Shader paint" width={1920} height={1080} fill="#000000">
+ <video src={`${VIDEOS}/sintel-short.mp4`} width={1920} height={1080}>
+ <shaderPaint wgsl={ABERRATION} uniforms={{ strength: 1.5, vignette: 0.6 }} />
+ </video>
+ </scene>
+ );
+}
diff --git a/examples/README.md b/examples/README.md
index 9fcff965..1ae3916c 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -13,10 +13,11 @@ key, so re-mounting an example replaces its scene in place.
| [05-anime-timeline.tsx](05-anime-timeline.tsx) | `dapi mount examples/05-anime-timeline.tsx` | anime.js timeline seeked from `useTicker`, driving an ECS node and `<html>` content in lockstep |
| [06-three.tsx](06-three.tsx) | `dapi mount examples/06-three.tsx` | three.js WebGL renderer owning a `<surface>`, glTF model loaded over the network |
| [07-webgpu.tsx](07-webgpu.tsx) | `dapi mount examples/07-webgpu.tsx` | raw WebGPU on a `<surface>`: a triangle whose colors cycle with composition time |
+| [08-shader-paint.tsx](08-shader-paint.tsx) | `dapi mount examples/08-shader-paint.tsx` | `<shaderPaint>` post-processing a `<video>`: WGSL chromatic aberration + vignette, uniforms patchable live |
Requirements: `02-genai.tsx` consumes generation credits (results are cached per session);
`04-html-in-canvas.tsx` and `05-anime-timeline.tsx` need Chromium's html-in-canvas API
-(`chrome://flags/#canvas-draw-element`); `07-webgpu.tsx` needs WebGPU; `01-basics.tsx` and
-`06-three.tsx` fetch remote media.
+(`chrome://flags/#canvas-draw-element`); `07-webgpu.tsx` and `08-shader-paint.tsx` need WebGPU;
+`01-basics.tsx`, `06-three.tsx`, and `08-shader-paint.tsx` fetch remote media.
Typecheck with `tsc -p examples --noEmit` (part of `npm run check`).
diff --git a/packages/jsx/src/elements.ts b/packages/jsx/src/elements.ts
index db6c8008..5c6cbe9c 100644
--- a/packages/jsx/src/elements.ts
+++ b/packages/jsx/src/elements.ts
@@ -29,6 +29,7 @@ import type {
RectProps,
SceneProps,
SequenceProps,
+ ShaderPaintProps,
SolidPaintProps,
SurfacePaintProps,
SurfaceProps,
@@ -67,5 +68,6 @@ export const RadialGradientPaint = hostElement<GradientPaintProps>("RadialGradie
export const ColorStop = hostElement<ColorStopProps>("ColorStop");
export const HtmlPaint = hostElement<HtmlPaintProps>("HtmlPaint");
export const Html = hostElement<HtmlProps>("Html");
+export const ShaderPaint = hostElement<ShaderPaintProps>("ShaderPaint");
export const SurfacePaint = hostElement<SurfacePaintProps>("SurfacePaint");
export const Surface = hostElement<SurfaceProps>("Surface");
diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.ts
index 8c7906f5..8630889a 100644
--- a/packages/jsx/src/index.ts
+++ b/packages/jsx/src/index.ts
@@ -28,6 +28,7 @@ export {
Rect,
Scene,
Sequence,
+ ShaderPaint,
SolidPaint,
Surface,
SurfacePaint,
@@ -56,6 +57,7 @@ export type {
RectProps,
SceneProps,
SequenceProps,
+ ShaderPaintProps,
SolidPaintProps,
SurfacePaintProps,
SurfaceProps,
diff --git a/packages/jsx/src/jsx-runtime.ts b/packages/jsx/src/jsx-runtime.ts
index 62a9462f..634812fa 100644
--- a/packages/jsx/src/jsx-runtime.ts
+++ b/packages/jsx/src/jsx-runtime.ts
@@ -31,6 +31,7 @@ import type {
RectProps,
SceneProps,
SequenceProps,
+ ShaderPaintProps,
SolidPaintProps,
SurfacePaintProps,
SurfaceProps,
@@ -73,6 +74,7 @@ export declare namespace JSX {
colorStop: ColorStopProps;
htmlPaint: HtmlPaintProps;
html: HtmlProps;
+ shaderPaint: ShaderPaintProps;
surfacePaint: SurfacePaintProps;
surface: SurfaceProps;
}
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index 8778d298..362ec1bc 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -178,6 +178,20 @@ export type PatchProps = {
src?: string | AssetRef;
/** How the source maps into the box. Default "cover" on `<Video>`, "contain" on `<Image>`. */
objectFit?: Fit;
+ /**
+ * Fragment-stage WGSL of a `<ShaderPaint>`, applied to the video/image
+ * paint directly below it in the paint stack, or run procedurally (over a
+ * transparent source) when there is none. Entry point
+ * `@fragment fn main(@location(0) uv: vec2f) -> @location(0) vec4f`;
+ * sample the media with `sampleSource(uv)`.
+ */
+ wgsl?: string;
+ /**
+ * Values for the shader's `@group(1)` uniform declarations, matched by
+ * name: numbers bind to `f32`, arrays of 2-4 to `vec2f`-`vec4f`, CSS
+ * color strings to `vec3f`/`vec4f`.
+ */
+ uniforms?: Record<string, number | number[] | string>;
/** Decibels: 0 = unity gain, negative attenuates (-6 ≈ half as loud), -Infinity = silence. Use `muted` to silence. Animatable. */
volume?: Animatable<number>;
/** Excludes the node's audio from the mix; independent of `volume`. */
@@ -232,6 +246,8 @@ export const PATCH_PROP_KEYS = Object.keys({
fill: true,
src: true,
objectFit: true,
+ wgsl: true,
+ uniforms: true,
volume: true,
muted: true,
fontFamily: true,
@@ -296,11 +312,17 @@ export type ColorStopProps = Required<Pick<PatchProps, "offset" | "color">> &
export type VideoProps = CommonProps &
Required<Pick<PatchProps, "src">> &
- Pick<PatchProps, "objectFit" | "volume" | "muted" | "syncTo">;
+ Pick<PatchProps, "objectFit" | "volume" | "muted" | "syncTo"> & {
+ /** Paint children, stacked over the media paint created by `src`. */
+ children?: SolidJSX.Element;
+ };
export type ImageProps = CommonProps &
Required<Pick<PatchProps, "src">> &
- Pick<PatchProps, "objectFit">;
+ Pick<PatchProps, "objectFit"> & {
+ /** Paint children, stacked over the media paint created by `src`. */
+ children?: SolidJSX.Element;
+ };
export type HtmlPaintProps = Pick<PatchProps, "opacity"> & {
/**
@@ -320,6 +342,10 @@ export type HtmlProps = CommonProps & Pick<HtmlPaintProps, "children">;
type HostCanvas = typeof globalThis extends { HTMLCanvasElement: new () => infer T } ? T
: { width: number; height: number; getContext(contextId: string, options?: unknown): unknown };
+/** `<ShaderPaint>` — transforms the media paint directly below it. Takes no children. */
+export type ShaderPaintProps = Required<Pick<PatchProps, "wgsl">> &
+ Pick<PatchProps, "uniforms" | "opacity">;
+
export type SurfacePaintProps = Pick<PatchProps, "opacity"> & {
/**
* Receives the surface's backing canvas once the paint materializes. Draw to
diff --git a/reference/jsx/README.md b/reference/jsx/README.md
index c0401de0..ac277a0c 100644
--- a/reference/jsx/README.md
+++ b/reference/jsx/README.md
@@ -19,6 +19,7 @@ The pipeline is driven by two commands: [`dapi mount`](../mount.md) renders the
| [paints.md](./paints.md) | `<solidPaint>`, gradients, `<colorStop>` |
| [html-paint.md](./html-paint.md) | `<html>`: reactive HTML children drawn into the box |
| [surface-paint.md](./surface-paint.md) | `<surface>`: a ref-provided canvas you draw into, sampled every frame |
+| [shader-paint.md](./shader-paint.md) | `<shaderPaint>`: a WGSL fragment shader over the media paint below it |
| [media.md](./media.md) | `src` resolution (paths, URLs, asset ids, `AssetRef`) |
| [timing.md](./timing.md) | `start` / `end` / `sourceIn` / `sourceOut`, time formats |
| [keyframes.md](./keyframes.md) | Keyframe animation and easing |
diff --git a/reference/jsx/image.md b/reference/jsx/image.md
index 8d36b50c..245e0f6e 100644
--- a/reference/jsx/image.md
+++ b/reference/jsx/image.md
@@ -15,4 +15,4 @@ All [common props](./elements.md#common-props), plus:
| `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). |
| `objectFit` | `"cover" \| "contain" \| "fill"` | `"cover"` | How the source maps into the box. |
-A paint child draws over the media paint created by `src` (see [paints.md](./paints.md)).
+A paint child draws over the media paint created by `src` (see [paints.md](./paints.md)); a [`<shaderPaint>`](./shader-paint.md) child instead post-processes it, so the image renders through the shader.
diff --git a/reference/jsx/paints.md b/reference/jsx/paints.md
index 7a8ded57..04db242c 100644
--- a/reference/jsx/paints.md
+++ b/reference/jsx/paints.md
@@ -21,6 +21,7 @@ Paint elements are valid inside any filled visual element (`<rect>`, `<scene>`,
| `<colorStop>` | `offset` (**required**, `0`-`1`), `color` (**required**), `opacity` | Gradient color stop. Valid only inside gradient paints, which take no other children. |
| [`<htmlPaint>`](./html-paint.md) | `opacity`, HTML children | Reactive HTML laid out and drawn into the parent's box (flagged Chromium API). `<html>` is shorthand for a `<rect>` carrying one. |
| [`<surfacePaint>`](./surface-paint.md) | `opacity`, `ref` (**required**) | A canvas your `ref` callback draws into (any context type), sampled into the parent's box every frame. `<surface>` is shorthand for a `<rect>` carrying one. |
+| [`<shaderPaint>`](./shader-paint.md) | `opacity`, `wgsl` (**required**), `uniforms` | A WGSL fragment shader applied to the video/image paint directly below it (that media renders only through the shader's output), or run procedurally when there is none. |
Colors accept any CSS color; alpha is ignored (use `opacity`). `color`, `opacity`, and `offset` are animatable (see [keyframes.md](./keyframes.md)), so gradients can animate. Paints have no spatial or timing props and cannot be document roots.
diff --git a/reference/jsx/shader-paint.md b/reference/jsx/shader-paint.md
new file mode 100644
index 00000000..8155254d
--- /dev/null
+++ b/reference/jsx/shader-paint.md
@@ -0,0 +1,104 @@
+# `<shaderPaint>`
+
+A paint that runs a **WGSL fragment shader over the media paint directly below it** in the parent's paint stack. On a `<video>` or `<image>` the media paint created by `src` sits first, so a `<shaderPaint>` child post-processes the frame: the media renders only through the shader's output, letting shaders remap colors, distort, or cut alpha (chroma key). Rendering uses WebGPU; where WebGPU is unavailable the shader is skipped and the media draws unshaded.
+
+Without a media paint directly below, the shader runs **procedurally**: `sampleSource` returns transparent black everywhere and the output is generated purely from `uv`, `globals.time`, and uniforms, stacking over any paints below like a normal paint. That makes `<shaderPaint>` in a `<rect>` a compact alternative to a WebGPU-driven [`<surface>`](./surface-paint.md) for generative fills.
+
+```tsx
+const INVERT = /* wgsl */ `
+ @fragment
+ fn main(@location(0) uv: vec2f) -> @location(0) vec4f {
+ let color = sampleSource(uv);
+ return vec4f(1.0 - color.rgb, color.a);
+ }
+`;
+
+<video src="/Movies/clip.mp4" width={1920} height={1080}>
+ <shaderPaint wgsl={INVERT} />
+</video>
+```
+
+```tsx
+<rect width={960} height={540}>
+ <shaderPaint wgsl={/* wgsl */ `
+ @fragment
+ fn main(@location(0) uv: vec2f) -> @location(0) vec4f {
+ let rgb = 0.5 + 0.5 * cos(globals.time + uv.xyx * 4.0 + vec3f(0.0, 2.0, 4.0));
+ return vec4f(rgb, 1.0);
+ }
+ `} />
+</rect>
+```
+
+Before the pipeline has compiled a `<shaderPaint>` draws nothing and the stack renders as if it were absent. A media paint below it whose frame has not decoded yet also keeps the shader silent, so the media's own loading state stays visible.
+
+## The shader contract
+
+You write only the fragment stage; the engine owns the vertex stage and prepends a prelude. The entry point is:
+
+```wgsl
+@fragment
+fn main(@location(0) uv: vec2f) -> @location(0) vec4f
+```
+
+`uv` spans the parent's box, `(0,0)` top-left to `(1,1)` bottom-right. The output is premultiplied-alpha and clips to the parent's box (including `cornerRadius`), like every paint. The prelude provides:
+
+| Binding | Meaning |
+| ------- | ------- |
+| `sampleSource(uv: vec2f) -> vec4f` | Samples the media in box space, honoring its `objectFit` placement; letterbox areas return transparent black. Premultiplied. |
+| `source: texture_2d<f32>`, `sourceSampler: sampler` | The raw frame texture, for sampling outside the fit mapping. |
+| `globals.time: f32` | The parent element's local time in seconds. Playhead-driven, so scrubbing and exports are frame-accurate. |
+| `globals.resolution: vec2f` | The parent's box size in composition pixels. |
+| `globals.fitOffset`, `globals.fitSize: vec2f` | The media's `objectFit` placement within the box, normalized (what `sampleSource` applies). |
+
+## Uniforms
+
+Parameters are declared as individual `@group(1)` uniforms and their values arrive through the `uniforms` prop, **matched by declaration name**:
+
+```tsx
+const TINT = /* wgsl */ `
+ @group(1) @binding(0) var<uniform> strength: f32;
+ @group(1) @binding(1) var<uniform> tint: vec3f;
+
+ @fragment
+ fn main(@location(0) uv: vec2f) -> @location(0) vec4f {
+ let color = sampleSource(uv);
+ return vec4f(mix(color.rgb, tint * color.a, strength), color.a);
+ }
+`;
+
+<shaderPaint wgsl={TINT} uniforms={{ strength: 0.5, tint: "#FF0055" }} />
+```
+
+Value mapping: a `number` binds to `f32`, an array of 2-4 numbers to `vec2f`/`vec3f`/`vec4f`, a CSS color string to `vec3f` (rgb, 0-1) or `vec4f` (alpha 1). Supported uniform types are the `f32` family only. Undeclared keys are ignored; declared uniforms without a value read as zeros.
+
+## Props
+
+| Prop | Type | Default | Meaning |
+| ---- | ---- | ------- | ------- |
+| `wgsl` | `string` | **required** | Fragment-stage WGSL source (entry point `main`). |
+| `uniforms` | `Record<string, number \| number[] \| string>` | `{}` | Values for the shader's `@group(1)` uniforms, by name. |
+| `opacity` | `Animatable<number>` | `1` | Paint opacity, `0`-`1`. |
+
+Like all paints it stacks with siblings in document order; `<shaderPaint>` takes no children.
+
+## Patching
+
+Shader paints are live entities: find the paint id under the element with [`dapi node tree`](../node/tree.md), then patch [`dapi node patch`](../node/patch.md):
+
+```bash
+# tweak parameters (hot update, no recompile)
+dapi node patch '[{ "id": 42, "uniforms": { "strength": 2 } }]'
+
+# swap the shader (async recompile; the media shows unshaded until it lands)
+dapi node patch '[{ "id": 42, "wgsl": "..." }]'
+```
+
+Attach one to an existing video with [`dapi node insert`](../node/insert.md) (`node insert <videoId> '<shaderPaint wgsl={...} />'`).
+
+## Errors and limitations
+
+- WGSL compile errors are reported on the app console (`dapi logs`) with line numbers relative to your source; the paint renders as passthrough until a fix lands.
+- Pipeline compilation is asynchronous. Live playback may show the first frames unshaded; exports wait for compilation, so rendered output is always shaded.
+- One shader reads one media paint; shaders do not chain and do not read solid/gradient/html/surface paints or the composited stack (over those the shader runs procedurally and stacks on top).
+- For full custom pipelines (own vertex stage, WebGL, three.js) use [`<surface>`](./surface-paint.md), which owns its canvas outright.
diff --git a/reference/jsx/video.md b/reference/jsx/video.md
index aecee581..4c2484f8 100644
--- a/reference/jsx/video.md
+++ b/reference/jsx/video.md
@@ -18,4 +18,4 @@ All [common props](./elements.md#common-props), plus:
| `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. |
| `syncTo` | `string` | none | Key of another element carrying audio; derives `start` by audio alignment (see [audio-sync.md](./audio-sync.md)). Mutually exclusive with `start`. |
-A paint child draws over the media paint created by `src` (see [paints.md](./paints.md)).
+A paint child draws over the media paint created by `src` (see [paints.md](./paints.md)); a [`<shaderPaint>`](./shader-paint.md) child instead post-processes it, so the frame renders through the shader.