The VS Code of video editing, built for agents.
TypeScript
Add blend mode and surface handling in JSX and runtime
konstantin-paulus1760028e9656
diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.tsindex 0870e119..21a0fa17 100644--- a/packages/jsx/src/index.ts+++ b/packages/jsx/src/index.ts@@ -41,6 +41,7 @@ export type {AnimationProps,AnimationType,AudioProps,+ BlendMode,CameraMatrix,CaptionPreset,CaptionsProps,@@ -67,6 +68,7 @@ export type {StrokeCap,StrokeJoin,StrokeProps,+ SurfaceHandle,SurfacePaintProps,SurfaceProps,TextProps,diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex be6b8b63..b4f46983 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -20,6 +20,28 @@ export type StrokeJoin = "miter" | "round" | "bevel";/** How a stroke ends an open path: the canvas `lineCap` values. */export type StrokeCap = "butt" | "round" | "square";+/**+ * How an element composites over what is below it: the canvas+ * `globalCompositeOperation` blend modes, camelCase. Default "sourceOver".+ */+export type BlendMode =+ | "sourceOver"+ | "multiply"+ | "screen"+ | "overlay"+ | "darken"+ | "lighten"+ | "colorDodge"+ | "colorBurn"+ | "hardLight"+ | "softLight"+ | "difference"+ | "exclusion"+ | "hue"+ | "saturation"+ | "color"+ | "luminosity";+/*** An `<effect>`'s filter — the CSS filter functions, applied to the parent's* rendered pixels. `blur` takes a radius in px, `hueRotate` degrees, the@@ -68,6 +90,9 @@ export type AnimatableProperty =| "width"| "height"| "rotation"+ | "scale"+ | "scaleX"+ | "scaleY"| "opacity"| "cornerRadius"| "volume"@@ -170,12 +195,29 @@ type SizeProps = {type TransformProps = PositionProps & OffsetProps & SizeProps & {/** Rotation in degrees. */rotation?: number;+ /** Uniform scale about the box origin, 1 = natural size. Overrides `scaleX`/`scaleY` while set. */+ scale?: number;+ /** Per-axis scale, 1 = natural size. */+ scaleX?: number;+ scaleY?: number;/** Opacity, 0–1 (out-of-range values clamp, like CSS). */opacity?: number;/** Uniform corner radius, px. */cornerRadius?: number;};+/** How an element composites, and whether it does at all. */+type CompositeProps = {+ /** Blend mode over what is below. Default "sourceOver". */+ blendMode?: BlendMode;+ /**+ * Excludes the element from rendering (and its audio from the mix) without+ * removing it: it keeps its place in the timeline and its children. Absent+ * means shown.+ */+ hidden?: boolean;+};+type TimingProps = {/** Parent-timeline time at which the node begins. Default 0. */start?: Time;@@ -185,6 +227,11 @@ type TimingProps = {sourceIn?: Time;/** Source out point: where playback ends within the source. Defaults to the natural end. Alternative to `end`. */sourceOut?: Time;+ /**+ * Speed multiplier for the node's local time, 1 = normal: at 2, twice the+ * source plays in the same stretch of timeline. Default 1.+ */+ playbackRate?: number;};type SequenceItemProps = {@@ -239,7 +286,10 @@ type ColorProps = {};/** What every visual node accepts on top of its own props. */-type CommonProps = IdentityProps & TransformProps & TimingProps & SequenceItemProps;+type CommonProps = IdentityProps & TransformProps & CompositeProps & TimingProps & SequenceItemProps;++/** What every paint accepts on top of its own props. */+type PaintProps = OpacityProps & CompositeProps;/** Sub-entity children (`<KeyframeTrack>`) an element that is itself a style takes. */type TrackChildren = {@@ -324,7 +374,7 @@ export type RectProps = CommonProps & FillProps & {* paint: `color`/`opacity` are its paint, `width`/`join`/`cap`/`miterLimit`* its line style. Several stack in document order, later ones on top.*/-export type StrokeProps = ColorProps & OpacityProps & TrackChildren & {+export type StrokeProps = ColorProps & PaintProps & TrackChildren & {/** Line width, px. Default 1. */width?: number;/** How the stroke turns corners. Default "miter". */@@ -339,7 +389,7 @@ export type StrokeProps = ColorProps & OpacityProps & TrackChildren & {* `<shadow>` — a drop shadow beneath the parent's box (or glyphs): a blurred,* offset copy of its silhouette in `color`. Several stack in document order.*/-export type ShadowProps = ColorProps & OpacityProps & TrackChildren & {+export type ShadowProps = ColorProps & OpacityProps & Pick<CompositeProps, "hidden"> & TrackChildren & {/** Blur radius, px. Default 0. */blur?: number;/** Where the shadow sits relative to the silhouette, px. Default 0. */@@ -403,9 +453,9 @@ export type KeyframeProps = {easing?: Easing;};-export type SolidPaintProps = ColorProps & OpacityProps & TrackChildren;+export type SolidPaintProps = ColorProps & PaintProps & TrackChildren;-export type GradientPaintProps = OpacityProps & {+export type GradientPaintProps = PaintProps & {/** Gradient rotation in degrees. Defaults to 0 (left to right). */rotation?: number;/** `<ColorStop>` children — the gradient's color stops. */@@ -427,7 +477,7 @@ export type ImageProps = CommonProps & MediaProps & FitProps & {children?: SolidJSX.Element;};-export type HtmlPaintProps = OpacityProps & {+export type HtmlPaintProps = PaintProps & {/*** HTML children — real DOM elements laid out by the browser at the parent* geometry's box size and drawn into it (html-in-canvas). Fully reactive:@@ -446,7 +496,7 @@ type HostCanvas = typeof globalThis extends { HTMLCanvasElement: new () => infer: { width: number; height: number; getContext(contextId: string, options?: unknown): unknown };/** `<ShaderPaint>` — transforms the media paint directly below it. Takes no children. */-export type ShaderPaintProps = OpacityProps & {+export type ShaderPaintProps = PaintProps & {/*** Fragment-stage WGSL, applied to the video/image paint directly below it* in the paint stack, or run procedurally (over a transparent source) when@@ -463,14 +513,21 @@ export type ShaderPaintProps = OpacityProps & {uniforms?: Record<string, number | number[] | string>;};-export type SurfacePaintProps = OpacityProps & {- /**- * Receives the surface's backing canvas once the paint materializes. Draw to- * any context type (2d, webgl, webgpu); the engine samples the bitmap every- * frame and stretches it into the parent geometry's box. Callback form- * only — the renderer has no element to assign to a variable ref.- */- ref: (canvas: HostCanvas) => void;+/**+ * What a `<surface>` / `<surfacePaint>` ref receives: the element's node, as+ * every ref does, with the paint's backing canvas on it. Draw to `canvas` with+ * any context type (2d, webgl, webgpu); the engine samples the bitmap every+ * frame and stretches it into the holder's box. The canvas is allocated with+ * the element and sized to the holder's `width`/`height` (a same-size set is a+ * no-op, so `renderer.setSize` from your own code is not clobbered); null+ * where the host has no DOM. Both ref forms work: `ref={(s) => ...}` and+ * `let s: SurfaceHandle; <surface ref={s} />`.+ */+export type SurfaceHandle = { readonly canvas: HostCanvas | null };++export type SurfacePaintProps = PaintProps & {+ /** Callback or variable ref; receives a `SurfaceHandle`. */+ ref?: SurfaceHandle | ((surface: SurfaceHandle) => void);};/** `<Surface>` — a rectangle carrying a `<SurfacePaint>` with the given ref. */diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 47fd9ebe..7095c485 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -13,9 +13,12 @@ import {Assets,Audio,Background,+ BlendMode,+ BlendModeType,Blur,Chars,ClipsContent,+ CornerRadius,createEntity,DEFAULT_BACKGROUND,Color,@@ -31,6 +34,7 @@ import {getEntityTree,getParentEntity,getParentNode,+ Hidden,isText,ItemIndex,Keyframe,@@ -43,15 +47,19 @@ import {PaintType,parseColor,Playback,+ PlaybackRate,Position,removeChild,resizeEntity,+ Scale,ScaleMode,ScaleModeType,secondsToFrames,getEntityChildren,Group,Sequential,+ Shader,+ Size,Stage,Root,Rotation,@@ -67,9 +75,14 @@ import {StrokeCap,StrokeJoin,StrokeStyle,+ SurfaceHost,+ SurfaceHostHandle,TextAlign,TextBaseline,TextStyle,+ Transition,+ TransitionType,+ UniformScale,Volume,} from '@diffusionstudio/runtime';import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';@@ -83,6 +96,17 @@ export interface SceneNode {readonly entity: Entity;}+/**+ * The node a `<surface>` or `<surfacePaint>` element creates: what its `ref`+ * receives, like any other element's ref receives its node. `canvas` is the+ * backing canvas (on the geometry for a `<surface>`, whose surface is its+ * intrinsic paint; on the paint sub-entity for a `<surfacePaint>`), allocated+ * with the element and sized to the holder's box; null without a DOM.+ */+export interface SurfaceNode extends SceneNode {+ readonly canvas: HTMLCanvasElement | null;+}+export interface TextNode {text: string;parent: SceneNode | null;@@ -208,6 +232,36 @@ const PAINT_TYPES: Record<string, PaintType> = {solidPaint: PaintType.SOLID,linearGradientPaint: PaintType.LINEAR_GRADIENT,radialGradientPaint: PaintType.RADIAL_GRADIENT,+ shaderPaint: PaintType.SHADER,+ surfacePaint: PaintType.SURFACE,+};++const TRANSITION_TYPES: Record<string, TransitionType> = {+ dissolve: TransitionType.DISSOLVE,+ slideFromRight: TransitionType.SLIDE_FROM_RIGHT,+ slideFromLeft: TransitionType.SLIDE_FROM_LEFT,+ fadeToBlack: TransitionType.FADE_TO_BLACK,+ fadeToWhite: TransitionType.FADE_TO_WHITE,+};++/** The canvas composite operations, spelled camelCase like the other enums. */+const BLEND_MODES: Record<string, BlendModeType> = {+ sourceOver: BlendModeType.SOURCE_OVER,+ multiply: BlendModeType.MULTIPLY,+ screen: BlendModeType.SCREEN,+ overlay: BlendModeType.OVERLAY,+ darken: BlendModeType.DARKEN,+ lighten: BlendModeType.LIGHTEN,+ colorDodge: BlendModeType.COLOR_DODGE,+ colorBurn: BlendModeType.COLOR_BURN,+ hardLight: BlendModeType.HARD_LIGHT,+ softLight: BlendModeType.SOFT_LIGHT,+ difference: BlendModeType.DIFFERENCE,+ exclusion: BlendModeType.EXCLUSION,+ hue: BlendModeType.HUE,+ saturation: BlendModeType.SATURATION,+ color: BlendModeType.COLOR,+ luminosity: BlendModeType.LUMINOSITY,};/**@@ -222,6 +276,9 @@ const TRACK_PROPERTIES: Record<string, PropertyPath> = {width: 'width',height: 'height',rotation: 'rotation',+ scale: 'scale',+ scaleX: 'scale.x',+ scaleY: 'scale.y',opacity: 'opacity',cornerRadius: 'vertexRadius',volume: 'volume',@@ -316,6 +373,30 @@ const FONT_WEIGHTS: Record<string, string> = {bold: '700',};+/**+ * Allocates a surface paint's host with the element (a canvas needs a DOM;+ * a host without one leaves the paint blank and `canvas` null).+ */+function attachSurfaceHost(paint: Entity): void {+ if (typeof document === 'undefined') return;+ paint.add(SurfaceHostHandle);+ paint.set(SurfaceHostHandle, new SurfaceHost());+}++/**+ * Sizes the surface canvases of `holder` (its own, for a `<surface>`, and+ * those of its `<surfacePaint>` children) to its authored box, so they draw+ * at the box's resolution. Same-size calls are no-ops (SurfaceHost).+ */+function syncSurfaceSize(world: World, holder: Entity): void {+ const size = holder.get(Size);+ if (size === undefined) return;+ holder.get(SurfaceHostHandle)?.setSize(size.width, size.height);+ for (const child of getEntityChildren(world, holder)) {+ child.get(SurfaceHostHandle)?.setSize(size.width, size.height);+ }+}+function toNumber(value: unknown) {if (value === undefined || value === null) {return undefined;@@ -342,7 +423,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {this.stage = { entity: world.get(Root)! };}- public createElement(tag: string): SceneNode {+ public createElement(tag: string): SceneNode | SurfaceNode {// Composition elements arrive in either spelling: the camelCase// intrinsics a project authors, or the PascalCase components the compile// step (and an app inserting elements) uses.@@ -436,11 +517,30 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {}case 'solidPaint':case 'linearGradientPaint':- case 'radialGradientPaint': {+ case 'radialGradientPaint':+ case 'shaderPaint':+ case 'surfacePaint': {entity = createEntity(this.world);entity.add(Paint);entity.set(Paint, { value: PAINT_TYPES[name]! });if (name === 'solidPaint') entity.add(Color);+ if (name === 'shaderPaint') entity.add(Shader);+ if (name === 'surfacePaint') attachSurfaceHost(entity);+ break;+ }+ case 'surface': {+ // A rect whose intrinsic paint is the surface, like <video>'s is+ // its media: the host lives on the geometry itself.+ entity = createEntity(this.world);+ entity.add(Geometry);+ entity.set(Geometry, { value: GeometryType.RECT });+ entity.add(Position);+ entity.set(Position, { x: 0, y: 0 });+ entity.add(Paint);+ entity.set(Paint, { value: PaintType.SURFACE });+ attachSurfaceHost(entity);+ resizeEntity(this.world, entity, { width: 100, height: 100 });+ syncSurfaceSize(this.world, entity);break;}case 'colorStop': {@@ -490,7 +590,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {}default:throw new Error(- `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect>, <text>, <video>, <image>, <audio>, <solidPaint>, <linearGradientPaint>, <radialGradientPaint>, <colorStop>, <stroke>, <shadow>, <effect>, <animation>, <keyframeTrack> and <keyframe>).`,+ `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect>, <text>, <video>, <image>, <audio>, <surface>, <solidPaint>, <linearGradientPaint>, <radialGradientPaint>, <shaderPaint>, <surfacePaint>, <colorStop>, <stroke>, <shadow>, <effect>, <animation>, <keyframeTrack> and <keyframe>).`,);}@@ -499,6 +599,17 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.set(Authored, { tag: name, props: {} });}+ if (name === 'surface' || name === 'surfacePaint') {+ // A getter, not a copy: the node holds no state of its own.+ const node: SurfaceNode = {+ entity,+ get canvas() {+ return entity.get(SurfaceHostHandle)?.canvas ?? null;+ },+ };+ return node;+ }+return { entity };}@@ -598,6 +709,101 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.set(Rotation, { value: toNumber(value) ?? 0 });return;}+ case 'scale': {+ // Uniform; wins over scaleX/scaleY while present (motion system).+ if (entity.has(Sequential)) return;+ const scale = toNumber(value);+ if (scale === undefined) {+ entity.remove(UniformScale);+ return;+ }++ entity.add(UniformScale);+ entity.set(UniformScale, { value: scale });+ return;+ }+ case 'scaleX':+ case 'scaleY': {+ if (entity.has(Sequential)) return;+ entity.add(Scale);+ entity.set(Scale, { [name === 'scaleX' ? 'x' : 'y']: toNumber(value) ?? 1 });+ return;+ }+ case 'cornerRadius': {+ const radius = toNumber(value);+ if (radius === undefined) {+ entity.remove(CornerRadius);+ return;+ }++ entity.add(CornerRadius);+ entity.set(CornerRadius, { value: radius });+ return;+ }+ case 'blendMode': {+ const mode = typeof value === 'string' ? BLEND_MODES[value] : undefined;+ if (mode === undefined || mode === BlendModeType.SOURCE_OVER) {+ entity.remove(BlendMode);+ return;+ }++ entity.add(BlendMode);+ entity.set(BlendMode, { value: mode });+ return;+ }+ case 'hidden': {+ if (value === true && entity !== this.stage.entity) {+ entity.add(Hidden);+ } else {+ entity.remove(Hidden);+ }+ return;+ }+ case 'playbackRate': {+ const rate = toNumber(value);+ if (rate === undefined || rate === 0) {+ entity.remove(PlaybackRate);+ return;+ }++ entity.add(PlaybackRate);+ entity.set(PlaybackRate, { value: rate });+ return;+ }+ case 'transition': {+ if (typeof value !== 'object' || value === null) {+ entity.remove(Transition);+ return;+ }++ if (!entity.has(Transition)) {+ entity.add(Transition);+ entity.set(Transition, { type: TransitionType.DISSOLVE, duration: this.toFrames(1) });+ }++ const spec = value as { type?: unknown; duration?: unknown };+ if ('type' in spec) {+ const type = typeof spec.type === 'string' ? TRANSITION_TYPES[spec.type] : undefined;+ entity.set(Transition, { type: type ?? TransitionType.DISSOLVE });+ }+ if ('duration' in spec) {+ entity.set(Transition, { duration: this.toFrames(toSeconds(spec.duration) ?? 1) });+ }+ return;+ }+ case 'wgsl': {+ if (!entity.has(Shader)) return;+ entity.set(Shader, { code: typeof value === 'string' ? value : '' });+ return;+ }+ case 'uniforms': {+ if (!entity.has(Shader)) return;+ const uniforms = typeof value === 'object' && value !== null && !Array.isArray(value)+ ? { ...(value as Record<string, number | number[] | string>) }+ : null;+ entity.set(Shader, { uniforms });+ return;+ }case 'width':case 'height': {const size = toNumber(value);@@ -608,6 +814,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {}if (size === undefined) return;resizeEntity(this.world, entity, { [name]: size });+ syncSurfaceSize(this.world, entity);return;}case 'join': {@@ -897,6 +1104,9 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {if (node.entity.has(KeyframeTrack)) {this.resolveTrackProperty(node.entity, node.entity.get(Authored)?.props.property);}+ if (node.entity.has(SurfaceHostHandle)) {+ syncSurfaceSize(this.world, parent.entity);+ }}const siblings = getEntityChildren(this.world, parent.entity).filter((sibling) => sibling !== node.entity);diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.mdindex b6c2bf77..0f633226 100644--- a/packages/runtime/MIGRATION.md+++ b/packages/runtime/MIGRATION.md@@ -155,6 +155,8 @@ apps/web now depends on `@diffusionstudio/runtime` and `@diffusionstudio/koota-s**`PatchProps` and `PATCH_PROP_KEYS` are gone.** They were the property table of the old `dapi node patch` command (any subset of props per patch entry, allowlisted at runtime); with the source as the document nothing patches by prop name (edits go through `editProperty`/`insertElement` and the desktop writer), and `PATCH_PROP_KEYS` had no consumer left. `packages/jsx/src/types.ts` now defines small internal prop groups (`IdentityProps`, `PositionProps`, `OffsetProps`, `SizeProps`, `TransformProps`, `TimingProps`, `SequenceItemProps`, `FillProps`, `MediaProps`, `FitProps`, `AudioTrackProps`, `OpacityProps`, `ColorProps`, `CommonProps`, `TrackChildren`) that the element prop types compose, and props one element owns (`type`, `value`, `property`, `time`, `easing`, `phase`, `duration`, `delay`, `join`, `cap`, `miterLimit`, `blur`, the text style, the caption props, `wgsl`/`uniforms`) live on that element's type, so the shared-name unions the table forced (`type?: EffectType | AnimationType`, `value?: Animatable<number> | string`) are gone. Gone with it: the inline `Animatable<T>`/`Keyframe<T>` keyframe arrays and the `animations?: AnimationSpec[]` list (`<keyframeTrack>`/`<animation>` are the only spelling; every prop is a plain value), and `packages/reconciler/src/patch.ts`. `reference/jsx/{keyframes,animations}.md` were rewritten for the tags and the `Animatable`/`animations` rows dropped from `elements.md`/`audio.md`/`captions.md`/`html.md`/`paints.md`; the rest of `reference/jsx` stays stale as noted below.+ **Static props the runtime already played but the host dropped, plus `<shaderPaint>`/`<surfacePaint>`/`<surface>`.** `cornerRadius` (`CornerRadius`; the `vertexRadius` keyframe path was already mapped, so a keyframed radius worked while a static one was ignored), `transition` on sequence items (`Transition` on the outgoing clip: `{}` is dissolve/1s, a partial value merges, `null`/unset removes; playback still drops it when the parent is not `Sequential`), and four new props: `scale` (`UniformScale`, wins over per-axis while present, as the motion system already resolves it), `scaleX`/`scaleY` (`Scale`), `blendMode` (`BlendMode`, camelCase composite operations, `sourceOver` removes the trait), `hidden` (`Hidden`), `playbackRate` (`PlaybackRate`, 0/unset removes). `scale`/`scaleX`/`scaleY` joined `AnimatableProperty` (motion paths `scale`/`scale.x`/`scale.y` existed). Types: `CompositeProps` (`blendMode`, `hidden`) is in `CommonProps` and in the new `PaintProps` (`OpacityProps` + composite) that all paints and `<stroke>` use; `<shadow>` takes `hidden` only. `<shaderPaint>` is `Paint(SHADER)` + `Shader` (`wgsl` -> `code`, `uniforms` copied); the host resolves lazily from `Shader.code` as before. `<surfacePaint>` is `Paint(SURFACE)`; `<surface>` a rect with an intrinsic (unauthored) surface paint child. Refs are plain Solid refs everywhere: `RuntimeDocument` implements no `applyRef`, so the renderer's `use` falls through to Solid's default and both forms (`ref={fn}`, `ref={variable}`) receive the `HostNode` synchronously. `<surface>` is a rect whose surface is its intrinsic paint, like `<video>`'s media (`Paint(SURFACE)` + `SurfaceHostHandle` on the geometry itself, no child paint; `renderIntrinsicFill` gained the SURFACE branch), `<surfacePaint>` the sub-entity form. Both nodes are `SurfaceNode` (`{ entity, canvas }`, `canvas` a getter over the entity's `SurfaceHostHandle`, typed for authors as `SurfaceHandle` in `@diffusionstudio/jsx`); the `SurfaceHost` is allocated with the element (`attachSurfaceHost`, skipped without a DOM) and sized by `syncSurfaceSize` when a paint meets its holder and whenever the holder's `width`/`height` props are set. `SurfaceHost.setSize` became a no-op for an unchanged size, since assigning a canvas dimension clears the bitmap even at the same value and user code (three.js `setSize`) may already have applied it. Because Solid applies refs before props on the spread path, the canvas seen *inside* the ref callback is not yet box-sized; it is by the time the render returns, and drawing happens in tickers/effects anyway. An earlier draft deferred a canvas-valued ref through `onMount`; dropped for the uniform, synchronous behaviour. A duplicated `<surface>` is a static empty one (its ref is not a `PropValue`), matching the old docs.+**Left stale on purpose** (deferred to a docs/examples pass): all eight `examples/*.tsx` still open with `<rect scene="...">` and no `<stage>`, so `tsc -p examples --noEmit` fails on them; `reference/jsx/{scene,roots}.md` still describes the pre-`<stage>` model where a mount grafts roots into an existing document; and `reference/jsx/{audio-sync,video,audio,elements}.md` still document `key` as the `syncTo` target. Deliberately not half-migrated: `reference/jsx` is stale as a whole, and patching the `key` rows alone would hide that.## Remaining work (suggested order)diff --git a/packages/runtime/src/media/surface.ts b/packages/runtime/src/media/surface.tsindex c4d5a803..bd31a790 100644--- a/packages/runtime/src/media/surface.ts+++ b/packages/runtime/src/media/surface.ts@@ -8,19 +8,24 @@ type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;* Backing store for one surface paint: a detached canvas handed to the* mount's `ref` callback, which draws into it with any context type (2d,* webgl, webgpu). The engine samples the bitmap every frame and stretches it- * into the parent geometry's box. After the initial box-sized allocation the- * bitmap belongs to user code — external renderers (e.g. three.js via- * `renderer.setSize`) resize it through their own APIs, so the engine never- * touches it again.+ * into the parent geometry's box. The host allocates the bitmap at the box+ * size and follows the box's authored size; user code (e.g. three.js via+ * `renderer.setSize`) may resize it through its own APIs as well.*/export class SurfaceHost {public readonly canvas = document.createElement('canvas');private disposed = false;- /** Initial bitmap allocation — called once at materialization, before the ref runs. */+ /**+ * Bitmap allocation at the box size. A no-op when the size is unchanged:+ * assigning a canvas dimension clears the bitmap even when the value is+ * the same, and user code may already have applied this size itself.+ */public setSize(width: number, height: number): void {- this.canvas.width = Math.max(1, Math.round(width));- this.canvas.height = Math.max(1, Math.round(height));+ const w = Math.max(1, Math.round(width));+ const h = Math.max(1, Math.round(height));+ if (this.canvas.width !== w) this.canvas.width = w;+ if (this.canvas.height !== h) this.canvas.height = h;}public draw(ctx: Ctx2D, width: number, height: number): void {diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex 8df5235f..9c675ad2 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -199,8 +199,9 @@ function buildEffects(world: World, entity: Entity): string | null {* before the Paint sub-entities so it always sits at the bottom of the fill* stack. A shader paint first in the stack takes an intrinsic image/video as* its input instead (see `renderShaderFill`), in which case the media is not- * drawn here. Only media paints are intrinsic so far; a waveform (an audio- * clip's) has no picture on the canvas.+ * drawn here. Media paints and the surface paint (a `<surface>`, whose host+ * lives on the geometry) are intrinsic; a waveform (an audio clip's) has no+ * picture on the canvas.*/export function renderIntrinsicFill(world: World, entity: Entity): void {if (entity.has(Color)) {@@ -211,7 +212,22 @@ export function renderIntrinsicFill(world: World, entity: Entity): void {ctx.fill();}- const kind = mediaKind(getIntrinsicPaint(entity));+ const intrinsic = getIntrinsicPaint(entity);+ if (intrinsic === PaintType.SURFACE) {+ const host = entity.has(SurfaceHostHandle) ? entity.get(SurfaceHostHandle) : null;+ if (host) {+ const ctx = getCtx(world);+ const computed = store(world, Computed);+ const eid = entity.id();+ ctx.save();+ ctx.clip();+ host.draw(ctx, computed.width[eid]!, computed.height[eid]!);+ ctx.restore();+ }+ return;+ }++ const kind = mediaKind(intrinsic);if (kind === null) return;if (shaderInput(world, entity, store(world, Cache).fills[entity.id()] ?? [], 0) === entity) return;renderMedia(world, entity, entity, kind);