Code-first video & image, rendered natively on WebGPU.
fix(webgpu-renderers): premultiply straight-alpha textures on upload
okanasldc0a97875e60
diff --git a/packages/webgpu-renderers/src/color.test.ts b/packages/webgpu-renderers/src/color.test.tsindex bb208715..60b46d9f 100644--- a/packages/webgpu-renderers/src/color.test.ts+++ b/packages/webgpu-renderers/src/color.test.ts@@ -1,5 +1,5 @@import { describe, expect, it } from "vitest";-import { parseColor, premultiplyColor } from "./color.js";+import { parseColor, premultiplyColor, premultiplyPixels } from "./color.js";describe("parseColor", () => {it("returns transparent black for null/undefined", () => {@@ -103,3 +103,47 @@ describe("premultiplyColor", () => {});});});++describe("premultiplyPixels", () => {+ it("zeroes RGB under fully transparent pixels", () => {+ const px = new Uint8Array([255, 200, 100, 0]);+ expect(Array.from(premultiplyPixels(px))).toEqual([0, 0, 0, 0]);+ });++ it("leaves opaque pixels untouched and scales partial ones", () => {+ const px = new Uint8Array([10, 20, 30, 255, 255, 255, 255, 128]);+ expect(Array.from(premultiplyPixels(px))).toEqual([+ 10, 20, 30, 255, 128, 128, 128, 128,+ ]);+ });++ it("rounds like Math.round(c * a / 255) for every 8-bit pair", () => {+ const px = new Uint8Array(256 * 256 * 4);+ for (let c = 0; c < 256; c++) {+ for (let a = 0; a < 256; a++) {+ const i = (c * 256 + a) * 4;+ px[i] = c;+ px[i + 3] = a;+ }+ }+ premultiplyPixels(px);+ for (let c = 0; c < 256; c++) {+ for (let a = 0; a < 256; a++) {+ const got = px[(c * 256 + a) * 4];+ if (got !== Math.round((c * a) / 255)) {+ expect({ c, a, got }).toEqual({+ c,+ a,+ got: Math.round((c * a) / 255),+ });+ }+ }+ }+ });++ it("works on Uint8ClampedArray in place", () => {+ const px = new Uint8ClampedArray([200, 0, 0, 51]);+ expect(premultiplyPixels(px)).toBe(px);+ expect(Array.from(px)).toEqual([40, 0, 0, 51]);+ });+});diff --git a/packages/webgpu-renderers/src/color.ts b/packages/webgpu-renderers/src/color.tsindex 9b4ceef5..076e749a 100644--- a/packages/webgpu-renderers/src/color.ts+++ b/packages/webgpu-renderers/src/color.ts@@ -49,3 +49,25 @@ export function premultiplyColor(color: Color): Color {a: color.a,};}++/**+ * Premultiplies straight-alpha RGBA8 pixels in place and returns them.+ *+ * Every 2D pipeline blends with premultiplied "over", so straight pixels+ * (decoded PNGs, `getImageData`, WebCodecs RGBA) must pass through here+ * before `writeTexture`. Otherwise RGB under transparent pixels is added+ * onto the destination.+ */+export function premultiplyPixels<T extends Uint8Array | Uint8ClampedArray>(+ pixels: T,+): T {+ for (let i = 0; i < pixels.length; i += 4) {+ const a = pixels[i + 3];+ if (a === 255) continue;+ // (c·a + 127) / 255, floored, equals round(c·a / 255) for 8-bit ints.+ pixels[i] = ((pixels[i] * a + 127) / 255) | 0;+ pixels[i + 1] = ((pixels[i + 1] * a + 127) / 255) | 0;+ pixels[i + 2] = ((pixels[i + 2] * a + 127) / 255) | 0;+ }+ return pixels;+}diff --git a/packages/webgpu-renderers/src/nodes/image-alpha.test.ts b/packages/webgpu-renderers/src/nodes/image-alpha.test.tsnew file mode 100644index 00000000..974f23b8--- /dev/null+++ b/packages/webgpu-renderers/src/nodes/image-alpha.test.ts@@ -0,0 +1,41 @@+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import sharp from "sharp";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { imageLoader } from "./image.js";++// A straight-alpha PNG with non-zero RGB under transparent pixels, as AI+// cutouts and many editors write: the compositor's premultiplied "over"+// would add that RGB onto the background unless it is premultiplied on load.+describe("image alpha", () => {+ let dir: string;+ let pngPath: string;++ beforeAll(async () => {+ dir = await mkdtemp(join(tmpdir(), "image-alpha-"));+ pngPath = join(dir, "straight.png");+ const raw = Buffer.from(+ [+ [255, 255, 255, 0], // white under alpha 0+ [255, 0, 0, 128], // red at half alpha+ [0, 255, 0, 255], // opaque green+ ].flat(),+ );+ const png = await sharp(raw, { raw: { width: 3, height: 1, channels: 4 } })+ .png()+ .toBuffer();+ await writeFile(pngPath, png);+ });++ afterAll(async () => {+ await rm(dir, { recursive: true, force: true });+ });++ it("premultiplies straight-alpha PNGs in headless mode", async () => {+ const data = await imageLoader.load(pngPath, true);+ expect(Array.from(data.buffer as Uint8Array)).toEqual([+ 0, 0, 0, 0, 128, 0, 0, 128, 0, 255, 0, 255,+ ]);+ });+});diff --git a/packages/webgpu-renderers/src/nodes/image-texture.ts b/packages/webgpu-renderers/src/nodes/image-texture.tsindex 2c647eae..8c8d5507 100644--- a/packages/webgpu-renderers/src/nodes/image-texture.ts+++ b/packages/webgpu-renderers/src/nodes/image-texture.ts@@ -1,4 +1,7 @@-/** A decoded image: a browser bitmap or, headless, raw RGBA8 pixels. */+/**+ * A decoded image: a browser bitmap or, headless, raw RGBA8 pixels.+ * `buffer` must already be premultiplied (see `premultiplyPixels`).+ */export interface DecodedImageData {width: number;height: number;@@ -6,7 +9,10 @@ export interface DecodedImageData {buffer?: Parameters<GPUQueue["writeTexture"]>[1] | null;}-/** An rgba8unorm texture holding `data` (left empty when it has neither form). */+/**+ * A premultiplied-alpha rgba8unorm texture holding `data` (left empty when+ * it has neither form).+ */export function createImageTexture(device: GPUDevice,data: DecodedImageData,@@ -23,7 +29,7 @@ export function createImageTexture(if (data.bitmap) {device.queue.copyExternalImageToTexture({ source: data.bitmap },- { texture: tex },+ { texture: tex, premultipliedAlpha: true },[data.width, data.height],);} else if (data.buffer) {diff --git a/packages/webgpu-renderers/src/nodes/image.test.ts b/packages/webgpu-renderers/src/nodes/image.test.tsindex a9ee435c..25b50927 100644--- a/packages/webgpu-renderers/src/nodes/image.test.ts+++ b/packages/webgpu-renderers/src/nodes/image.test.ts@@ -74,7 +74,8 @@ describe("Image Node", () => {expect(globalThis.fetch).toHaveBeenCalledWith("http://test.com/img.png");expect(data.width).toBe(1);expect(data.height).toBe(1);- expect(data.buffer).toBeDefined();+ // sharp's straight [0,1,2,a=3] comes back premultiplied.+ expect(Array.from(data.buffer as Uint8Array)).toEqual([0, 0, 0, 3]);expect(data.bitmap).toBeUndefined();});@@ -133,7 +134,11 @@ describe("Image Node", () => {await drawImageNode(mockCtx, mockPass, props);expect(mockDevice.createTexture).toHaveBeenCalled();- expect(mockDevice.queue.copyExternalImageToTexture).toHaveBeenCalled();+ expect(mockDevice.queue.copyExternalImageToTexture).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({ premultipliedAlpha: true }),+ expect.anything(),+ );expect(mockCtx.renderer.drawTexture).toHaveBeenCalled();});diff --git a/packages/webgpu-renderers/src/nodes/image.ts b/packages/webgpu-renderers/src/nodes/image.tsindex 9fcb84e5..4e32ad39 100644--- a/packages/webgpu-renderers/src/nodes/image.ts+++ b/packages/webgpu-renderers/src/nodes/image.ts@@ -1,3 +1,4 @@+import { premultiplyPixels } from "../color.js";import type { RenderContextValue } from "../render-context.js";import { textureCache } from "../texture-cache.js";import { createImageTexture } from "./image-texture.js";@@ -55,10 +56,11 @@ class ImageLoader {.ensureAlpha().raw().toBuffer({ resolveWithObject: true });+ // Decoded PNG/WebP is straight alpha; the compositor expects premultiplied.return {width: info.width,height: info.height,- buffer: new Uint8Array(data),+ buffer: premultiplyPixels(new Uint8Array(data)),};}diff --git a/packages/webgpu-renderers/src/nodes/lottie.ts b/packages/webgpu-renderers/src/nodes/lottie.tsindex b04ee628..8cc125eb 100644--- a/packages/webgpu-renderers/src/nodes/lottie.ts+++ b/packages/webgpu-renderers/src/nodes/lottie.ts@@ -4,6 +4,7 @@ import {localFontManager,} from "@framefields/client-utils";import { DotLottie, type Fit } from "@lottiefiles/dotlottie-web";+import { premultiplyPixels } from "../color.js";import {getHeadlessFontData,registerHeadlessFont,@@ -554,10 +555,13 @@ export async function drawLottieNode(const imgData = (c2d as unknown as OffscreenCanvasRenderingContext2D).getImageData(0, 0, instance.width, instance.height);- const pixelBuffer = new Uint8Array(- imgData.data.buffer,- imgData.data.byteOffset,- imgData.data.byteLength,+ // getImageData returns straight alpha.+ const pixelBuffer = premultiplyPixels(+ new Uint8Array(+ imgData.data.buffer,+ imgData.data.byteOffset,+ imgData.data.byteLength,+ ),);ctx.device.queue.writeTexture({ texture: tex },diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.ts b/packages/webgpu-renderers/src/nodes/media-decoder.tsindex 696a1e61..619bacdd 100644--- a/packages/webgpu-renderers/src/nodes/media-decoder.ts+++ b/packages/webgpu-renderers/src/nodes/media-decoder.ts@@ -7,6 +7,7 @@ import {type WrappedAudioBuffer,type WrappedCanvas,} from "mediabunny";+import { premultiplyPixels } from "../color.js";import { inputStore } from "../input-store.js";/**@@ -319,6 +320,8 @@ export class VideoDecoder extends BaseMediaDecoder<await currentSample.copyTo(buffer, { format: "RGBA" });if (this.currentFetchId !== fetchId || this.destroyed || !this.input)return null;+ // WebCodecs RGBA is straight alpha; the compositor expects premultiplied.+ premultiplyPixels(buffer);return { width, height, buffer };}diff --git a/packages/webgpu-renderers/src/nodes/svg.ts b/packages/webgpu-renderers/src/nodes/svg.tsindex 16654988..feafda65 100644--- a/packages/webgpu-renderers/src/nodes/svg.ts+++ b/packages/webgpu-renderers/src/nodes/svg.ts@@ -61,6 +61,7 @@ class SvgLoader {const rendered = await renderAsync(text, {fitTo: { mode: "width", value: resolvedWidth },});+ // resvg's pixels are already premultiplied, as createImageTexture expects.return {width: rendered.width,height: rendered.height,diff --git a/packages/webgpu-renderers/src/renderer2d/text-atlas.ts b/packages/webgpu-renderers/src/renderer2d/text-atlas.tsindex 82484d68..c0256ed1 100644--- a/packages/webgpu-renderers/src/renderer2d/text-atlas.ts+++ b/packages/webgpu-renderers/src/renderer2d/text-atlas.ts@@ -1,3 +1,4 @@+import { premultiplyPixels } from "../color.js";import type { Rect } from "./index.js";export interface AtlasRegion {@@ -86,7 +87,8 @@ export class TextAtlas {const imgData = c2d.getImageData(0, 0, width, height);this.device.queue.writeTexture({ texture: this.texture, origin: [region.x, region.y, 0] },- imgData.data,+ // getImageData returns straight alpha.+ premultiplyPixels(imgData.data),{ bytesPerRow: width * 4 },[width, height],);diff --git a/packages/webgpu-renderers/src/slug/slug-font-cache.ts b/packages/webgpu-renderers/src/slug/slug-font-cache.tsindex ae5c677d..39215244 100644--- a/packages/webgpu-renderers/src/slug/slug-font-cache.ts+++ b/packages/webgpu-renderers/src/slug/slug-font-cache.ts@@ -1,4 +1,5 @@import * as fontkit from "fontkit";+import { premultiplyPixels } from "../color.js";import { debugLog } from "../debug-log.js";import type { FontkitFont } from "./slug-generator.js";import { SlugGenerator } from "./slug-generator.js";@@ -437,17 +438,7 @@ export class SlugFontCache {.raw().toBuffer({ resolveWithObject: true });- const pixels = new Uint8Array(data.length);- pixels.set(data);- for (let i = 0; i < pixels.length; i += 4) {- const alpha = pixels[i + 3];- if (alpha < 255) {- const f = alpha / 255;- pixels[i] = Math.round(pixels[i] * f);- pixels[i + 1] = Math.round(pixels[i + 1] * f);- pixels[i + 2] = Math.round(pixels[i + 2] * f);- }- }+ const pixels = premultiplyPixels(new Uint8Array(data));tex = device.createTexture({label: `Emoji-${char}`,