Skip to content

Code-first video & image, rendered natively on WebGPU.

fix(webgpu-renderers): premultiply straight-alpha textures on upload

okanasldc0a97875e60
diff --git a/packages/webgpu-renderers/src/color.test.ts b/packages/webgpu-renderers/src/color.test.ts
index bb208715..60b46d9f 100644
--- a/packages/webgpu-renderers/src/color.test.ts
+++ b/packages/webgpu-renderers/src/color.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { parseColor, premultiplyColor } from "./color.js";
+import { parseColor, premultiplyColor, premultiplyPixels } from "./color.js";
describe("parseColor", () => {
it("returns transparent black for null/undefined", () => {
@@ -103,3 +103,47 @@ describe("premultiplyColor", () => {
});
});
});
+
+describe("premultiplyPixels", () => {
+ it("zeroes RGB under fully transparent pixels", () => {
+ const px = new Uint8Array([255, 200, 100, 0]);
+ expect(Array.from(premultiplyPixels(px))).toEqual([0, 0, 0, 0]);
+ });
+
+ it("leaves opaque pixels untouched and scales partial ones", () => {
+ const px = new Uint8Array([10, 20, 30, 255, 255, 255, 255, 128]);
+ expect(Array.from(premultiplyPixels(px))).toEqual([
+ 10, 20, 30, 255, 128, 128, 128, 128,
+ ]);
+ });
+
+ it("rounds like Math.round(c * a / 255) for every 8-bit pair", () => {
+ const px = new Uint8Array(256 * 256 * 4);
+ for (let c = 0; c < 256; c++) {
+ for (let a = 0; a < 256; a++) {
+ const i = (c * 256 + a) * 4;
+ px[i] = c;
+ px[i + 3] = a;
+ }
+ }
+ premultiplyPixels(px);
+ for (let c = 0; c < 256; c++) {
+ for (let a = 0; a < 256; a++) {
+ const got = px[(c * 256 + a) * 4];
+ if (got !== Math.round((c * a) / 255)) {
+ expect({ c, a, got }).toEqual({
+ c,
+ a,
+ got: Math.round((c * a) / 255),
+ });
+ }
+ }
+ }
+ });
+
+ it("works on Uint8ClampedArray in place", () => {
+ const px = new Uint8ClampedArray([200, 0, 0, 51]);
+ expect(premultiplyPixels(px)).toBe(px);
+ expect(Array.from(px)).toEqual([40, 0, 0, 51]);
+ });
+});
diff --git a/packages/webgpu-renderers/src/color.ts b/packages/webgpu-renderers/src/color.ts
index 9b4ceef5..076e749a 100644
--- a/packages/webgpu-renderers/src/color.ts
+++ b/packages/webgpu-renderers/src/color.ts
@@ -49,3 +49,25 @@ export function premultiplyColor(color: Color): Color {
a: color.a,
};
}
+
+/**
+ * Premultiplies straight-alpha RGBA8 pixels in place and returns them.
+ *
+ * Every 2D pipeline blends with premultiplied "over", so straight pixels
+ * (decoded PNGs, `getImageData`, WebCodecs RGBA) must pass through here
+ * before `writeTexture`. Otherwise RGB under transparent pixels is added
+ * onto the destination.
+ */
+export function premultiplyPixels<T extends Uint8Array | Uint8ClampedArray>(
+ pixels: T,
+): T {
+ for (let i = 0; i < pixels.length; i += 4) {
+ const a = pixels[i + 3];
+ if (a === 255) continue;
+ // (c·a + 127) / 255, floored, equals round(c·a / 255) for 8-bit ints.
+ pixels[i] = ((pixels[i] * a + 127) / 255) | 0;
+ pixels[i + 1] = ((pixels[i + 1] * a + 127) / 255) | 0;
+ pixels[i + 2] = ((pixels[i + 2] * a + 127) / 255) | 0;
+ }
+ return pixels;
+}
diff --git a/packages/webgpu-renderers/src/nodes/image-alpha.test.ts b/packages/webgpu-renderers/src/nodes/image-alpha.test.ts
new file mode 100644
index 00000000..974f23b8
--- /dev/null
+++ b/packages/webgpu-renderers/src/nodes/image-alpha.test.ts
@@ -0,0 +1,41 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import sharp from "sharp";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { imageLoader } from "./image.js";
+
+// A straight-alpha PNG with non-zero RGB under transparent pixels, as AI
+// cutouts and many editors write: the compositor's premultiplied "over"
+// would add that RGB onto the background unless it is premultiplied on load.
+describe("image alpha", () => {
+ let dir: string;
+ let pngPath: string;
+
+ beforeAll(async () => {
+ dir = await mkdtemp(join(tmpdir(), "image-alpha-"));
+ pngPath = join(dir, "straight.png");
+ const raw = Buffer.from(
+ [
+ [255, 255, 255, 0], // white under alpha 0
+ [255, 0, 0, 128], // red at half alpha
+ [0, 255, 0, 255], // opaque green
+ ].flat(),
+ );
+ const png = await sharp(raw, { raw: { width: 3, height: 1, channels: 4 } })
+ .png()
+ .toBuffer();
+ await writeFile(pngPath, png);
+ });
+
+ afterAll(async () => {
+ await rm(dir, { recursive: true, force: true });
+ });
+
+ it("premultiplies straight-alpha PNGs in headless mode", async () => {
+ const data = await imageLoader.load(pngPath, true);
+ expect(Array.from(data.buffer as Uint8Array)).toEqual([
+ 0, 0, 0, 0, 128, 0, 0, 128, 0, 255, 0, 255,
+ ]);
+ });
+});
diff --git a/packages/webgpu-renderers/src/nodes/image-texture.ts b/packages/webgpu-renderers/src/nodes/image-texture.ts
index 2c647eae..8c8d5507 100644
--- a/packages/webgpu-renderers/src/nodes/image-texture.ts
+++ b/packages/webgpu-renderers/src/nodes/image-texture.ts
@@ -1,4 +1,7 @@
-/** A decoded image: a browser bitmap or, headless, raw RGBA8 pixels. */
+/**
+ * A decoded image: a browser bitmap or, headless, raw RGBA8 pixels.
+ * `buffer` must already be premultiplied (see `premultiplyPixels`).
+ */
export interface DecodedImageData {
width: number;
height: number;
@@ -6,7 +9,10 @@ export interface DecodedImageData {
buffer?: Parameters<GPUQueue["writeTexture"]>[1] | null;
}
-/** An rgba8unorm texture holding `data` (left empty when it has neither form). */
+/**
+ * A premultiplied-alpha rgba8unorm texture holding `data` (left empty when
+ * it has neither form).
+ */
export function createImageTexture(
device: GPUDevice,
data: DecodedImageData,
@@ -23,7 +29,7 @@ export function createImageTexture(
if (data.bitmap) {
device.queue.copyExternalImageToTexture(
{ source: data.bitmap },
- { texture: tex },
+ { texture: tex, premultipliedAlpha: true },
[data.width, data.height],
);
} else if (data.buffer) {
diff --git a/packages/webgpu-renderers/src/nodes/image.test.ts b/packages/webgpu-renderers/src/nodes/image.test.ts
index a9ee435c..25b50927 100644
--- a/packages/webgpu-renderers/src/nodes/image.test.ts
+++ b/packages/webgpu-renderers/src/nodes/image.test.ts
@@ -74,7 +74,8 @@ describe("Image Node", () => {
expect(globalThis.fetch).toHaveBeenCalledWith("http://test.com/img.png");
expect(data.width).toBe(1);
expect(data.height).toBe(1);
- expect(data.buffer).toBeDefined();
+ // sharp's straight [0,1,2,a=3] comes back premultiplied.
+ expect(Array.from(data.buffer as Uint8Array)).toEqual([0, 0, 0, 3]);
expect(data.bitmap).toBeUndefined();
});
@@ -133,7 +134,11 @@ describe("Image Node", () => {
await drawImageNode(mockCtx, mockPass, props);
expect(mockDevice.createTexture).toHaveBeenCalled();
- expect(mockDevice.queue.copyExternalImageToTexture).toHaveBeenCalled();
+ expect(mockDevice.queue.copyExternalImageToTexture).toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({ premultipliedAlpha: true }),
+ expect.anything(),
+ );
expect(mockCtx.renderer.drawTexture).toHaveBeenCalled();
});
diff --git a/packages/webgpu-renderers/src/nodes/image.ts b/packages/webgpu-renderers/src/nodes/image.ts
index 9fcb84e5..4e32ad39 100644
--- a/packages/webgpu-renderers/src/nodes/image.ts
+++ b/packages/webgpu-renderers/src/nodes/image.ts
@@ -1,3 +1,4 @@
+import { premultiplyPixels } from "../color.js";
import type { RenderContextValue } from "../render-context.js";
import { textureCache } from "../texture-cache.js";
import { createImageTexture } from "./image-texture.js";
@@ -55,10 +56,11 @@ class ImageLoader {
.ensureAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
+ // Decoded PNG/WebP is straight alpha; the compositor expects premultiplied.
return {
width: info.width,
height: info.height,
- buffer: new Uint8Array(data),
+ buffer: premultiplyPixels(new Uint8Array(data)),
};
}
diff --git a/packages/webgpu-renderers/src/nodes/lottie.ts b/packages/webgpu-renderers/src/nodes/lottie.ts
index b04ee628..8cc125eb 100644
--- a/packages/webgpu-renderers/src/nodes/lottie.ts
+++ b/packages/webgpu-renderers/src/nodes/lottie.ts
@@ -4,6 +4,7 @@ import {
localFontManager,
} from "@framefields/client-utils";
import { DotLottie, type Fit } from "@lottiefiles/dotlottie-web";
+import { premultiplyPixels } from "../color.js";
import {
getHeadlessFontData,
registerHeadlessFont,
@@ -554,10 +555,13 @@ export async function drawLottieNode(
const imgData = (
c2d as unknown as OffscreenCanvasRenderingContext2D
).getImageData(0, 0, instance.width, instance.height);
- const pixelBuffer = new Uint8Array(
- imgData.data.buffer,
- imgData.data.byteOffset,
- imgData.data.byteLength,
+ // getImageData returns straight alpha.
+ const pixelBuffer = premultiplyPixels(
+ new Uint8Array(
+ imgData.data.buffer,
+ imgData.data.byteOffset,
+ imgData.data.byteLength,
+ ),
);
ctx.device.queue.writeTexture(
{ texture: tex },
diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.ts b/packages/webgpu-renderers/src/nodes/media-decoder.ts
index 696a1e61..619bacdd 100644
--- a/packages/webgpu-renderers/src/nodes/media-decoder.ts
+++ b/packages/webgpu-renderers/src/nodes/media-decoder.ts
@@ -7,6 +7,7 @@ import {
type WrappedAudioBuffer,
type WrappedCanvas,
} from "mediabunny";
+import { premultiplyPixels } from "../color.js";
import { inputStore } from "../input-store.js";
/**
@@ -319,6 +320,8 @@ export class VideoDecoder extends BaseMediaDecoder<
await currentSample.copyTo(buffer, { format: "RGBA" });
if (this.currentFetchId !== fetchId || this.destroyed || !this.input)
return null;
+ // WebCodecs RGBA is straight alpha; the compositor expects premultiplied.
+ premultiplyPixels(buffer);
return { width, height, buffer };
}
diff --git a/packages/webgpu-renderers/src/nodes/svg.ts b/packages/webgpu-renderers/src/nodes/svg.ts
index 16654988..feafda65 100644
--- a/packages/webgpu-renderers/src/nodes/svg.ts
+++ b/packages/webgpu-renderers/src/nodes/svg.ts
@@ -61,6 +61,7 @@ class SvgLoader {
const rendered = await renderAsync(text, {
fitTo: { mode: "width", value: resolvedWidth },
});
+ // resvg's pixels are already premultiplied, as createImageTexture expects.
return {
width: rendered.width,
height: rendered.height,
diff --git a/packages/webgpu-renderers/src/renderer2d/text-atlas.ts b/packages/webgpu-renderers/src/renderer2d/text-atlas.ts
index 82484d68..c0256ed1 100644
--- a/packages/webgpu-renderers/src/renderer2d/text-atlas.ts
+++ b/packages/webgpu-renderers/src/renderer2d/text-atlas.ts
@@ -1,3 +1,4 @@
+import { premultiplyPixels } from "../color.js";
import type { Rect } from "./index.js";
export interface AtlasRegion {
@@ -86,7 +87,8 @@ export class TextAtlas {
const imgData = c2d.getImageData(0, 0, width, height);
this.device.queue.writeTexture(
{ texture: this.texture, origin: [region.x, region.y, 0] },
- imgData.data,
+ // getImageData returns straight alpha.
+ premultiplyPixels(imgData.data),
{ bytesPerRow: width * 4 },
[width, height],
);
diff --git a/packages/webgpu-renderers/src/slug/slug-font-cache.ts b/packages/webgpu-renderers/src/slug/slug-font-cache.ts
index ae5c677d..39215244 100644
--- a/packages/webgpu-renderers/src/slug/slug-font-cache.ts
+++ b/packages/webgpu-renderers/src/slug/slug-font-cache.ts
@@ -1,4 +1,5 @@
import * as fontkit from "fontkit";
+import { premultiplyPixels } from "../color.js";
import { debugLog } from "../debug-log.js";
import type { FontkitFont } from "./slug-generator.js";
import { SlugGenerator } from "./slug-generator.js";
@@ -437,17 +438,7 @@ export class SlugFontCache {
.raw()
.toBuffer({ resolveWithObject: true });
- const pixels = new Uint8Array(data.length);
- pixels.set(data);
- for (let i = 0; i < pixels.length; i += 4) {
- const alpha = pixels[i + 3];
- if (alpha < 255) {
- const f = alpha / 255;
- pixels[i] = Math.round(pixels[i] * f);
- pixels[i + 1] = Math.round(pixels[i + 1] * f);
- pixels[i + 2] = Math.round(pixels[i + 2] * f);
- }
- }
+ const pixels = premultiplyPixels(new Uint8Array(data));
tex = device.createTexture({
label: `Emoji-${char}`,