Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): warn when a video input declares alpha but decodes opaque (#3771)
Santhi Prakashcad7dbed0dc2
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex 3b8879f87d..d5de062553 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -41,6 +41,7 @@ import {extractMediaMetadata,type VideoMetadata,} from "../utils/ffprobe.js";+import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";import {analyzeCompositionHdr,isHdrColorSpace as isHdrColorSpaceUtil,@@ -1844,6 +1845,23 @@ export async function extractAllVideoFrames(const sdrToHdrTransfers: Array<HdrTransfer | undefined> = resolvedVideos.map(() => undefined);breakdown.hdrProbeMs = Date.now() - phase2ProbeStart;+ // Warning only: an opaque video used as a full-frame background is legitimate.+ const alphaWarnedSrcs = new Set<string>();+ if (resolvedVideos.length > 0) {+ await Promise.all(+ resolvedVideos.map(async ({ video, videoPath }, index) => {+ if (signal?.aborted) return;+ const metadata = videoMetadata[index];+ if (!metadata?.hasAlpha || !codecMayHaveAlpha(metadata.videoCodec)) return;+ if (alphaWarnedSrcs.has(video.src)) return;+ alphaWarnedSrcs.add(video.src);+ const decoder = decoderForCodec(metadata.videoCodec);+ if ((await probeInputAlphaPlane(videoPath, decoder, signal)) !== true) return;+ process.stderr.write(inputAlphaOpaqueWarning(video.src));+ }),+ );+ }+const hdrPreflightStart = Date.now();const hdrInfo = analyzeCompositionHdr(videoColorSpaces);// Track entries the HDR preflight validated as non-extractable so they candiff --git a/packages/engine/src/utils/alphaPlaneProbe.test.ts b/packages/engine/src/utils/alphaPlaneProbe.test.tsnew file mode 100644index 0000000000..85d8ff117e--- /dev/null+++ b/packages/engine/src/utils/alphaPlaneProbe.test.ts@@ -0,0 +1,115 @@+import { spawnSync } from "node:child_process";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { decoderForCodec } from "../services/videoFrameExtractor.js";+import {+ inputAlphaOpaqueWarning,+ probeInputAlphaPlane,+ sampledRgbaAlphaIsFullyOpaque,+} from "./alphaPlaneProbe.js";+import { getFfmpegBinary } from "./ffmpegBinaries.js";++const BYTES_PER_FRAME = 8 * 8 * 4; // 256 — one 8x8 rgba frame++/** Build a raw rgba sample where the alpha byte of every pixel is `alpha`. */+function sample(alpha: number, frameCount: number): Buffer {+ const buf = Buffer.alloc(BYTES_PER_FRAME * frameCount, 0);+ for (let i = 3; i < buf.length; i += 4) {+ buf[i] = alpha;+ }+ return buf;+}++describe("sampledRgbaAlphaIsFullyOpaque", () => {+ it("returns true when every alpha byte in a single frame is 255", () => {+ expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 1))).toBe(true);+ });++ it("returns true across a multi-frame sample (2 and 3 frames)", () => {+ expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 2))).toBe(true);+ expect(sampledRgbaAlphaIsFullyOpaque(sample(255, 3))).toBe(true);+ });++ it("returns false when any pixel shows full transparency", () => {+ const buf = sample(255, 1);+ buf[3] = 0; // first pixel's alpha+ expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false);+ });++ it("returns false when any pixel shows partial transparency", () => {+ const buf = sample(255, 2);+ buf[3 + 4 * 10] = 254; // partial alpha on the 11th pixel+ expect(sampledRgbaAlphaIsFullyOpaque(buf)).toBe(false);+ });++ it("returns undefined for an empty sample (inconclusive, not a warning)", () => {+ expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(0))).toBeUndefined();+ });++ it("returns undefined for a byte count that is not a whole-frame multiple", () => {+ expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME - 1))).toBeUndefined();+ });++ it("returns undefined for an oversized sample beyond the 3-frame ceiling", () => {+ expect(sampledRgbaAlphaIsFullyOpaque(Buffer.alloc(BYTES_PER_FRAME * 4))).toBeUndefined();+ });+});++describe("inputAlphaOpaqueWarning", () => {+ it("names the offending file and carries the re-export remedy", () => {+ const line = inputAlphaOpaqueWarning("avatar.webm");+ expect(line).toContain('src="avatar.webm"');+ expect(line).toContain("declares an alpha channel");+ expect(line).toContain("first frames decode fully opaque");+ expect(line).toContain("alpha pixel format");+ expect(line).not.toContain("yuva420p");+ expect(line.endsWith("\n")).toBe(true);+ });+});++const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;++describe.skipIf(!HAS_FFMPEG)("probeInputAlphaPlane on real files", () => {+ let dir: string;+ beforeAll(() => {+ dir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-test-"));+ });+ afterAll(() => {+ rmSync(dir, { recursive: true, force: true });+ });++ const encode = (name: string, color: string, codecArgs: string[]): string => {+ const out = join(dir, name);+ const args = [+ "-v",+ "error",+ "-f",+ "lavfi",+ "-i",+ `color=c=${color}:s=32x32:d=0.2,format=rgba`,+ ...codecArgs,+ ];+ const res = spawnSync(getFfmpegBinary(), [...args, "-y", out]);+ expect(res.status).toBe(0);+ return out;+ };+ const vp9 = ["-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p"];+ const prores = ["-c:v", "prores_ks", "-profile:v", "4", "-pix_fmt", "yuva444p10le"];++ it("reports a VP9 input whose alpha is all opaque", async () => {+ const file = encode("opaque.webm", "red", vp9);+ expect(await probeInputAlphaPlane(file, decoderForCodec("vp9"))).toBe(true);+ });++ it("does not flag a VP9 input that is really transparent", async () => {+ const file = encode("clear.webm", "[email protected]", vp9);+ expect(await probeInputAlphaPlane(file, decoderForCodec("vp9"))).toBe(false);+ });++ it("probes ProRes 4444 with its own decoder, not the VP9 one", async () => {+ const file = encode("opaque.mov", "red", prores);+ expect(await probeInputAlphaPlane(file, decoderForCodec("prores"))).toBe(true);+ });+});diff --git a/packages/engine/src/utils/alphaPlaneProbe.ts b/packages/engine/src/utils/alphaPlaneProbe.tsnew file mode 100644index 0000000000..b963ca1156--- /dev/null+++ b/packages/engine/src/utils/alphaPlaneProbe.ts@@ -0,0 +1,74 @@+import { mkdtempSync, readFileSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { runFfmpeg } from "./runFfmpeg.js";++// Up to 3 frames at 8x8 rgba; a 1- or 2-frame clip is still a whole sample.+const BYTES_PER_SAMPLE_FRAME = 8 * 8 * 4;+const MAX_SAMPLE_BYTES = BYTES_PER_SAMPLE_FRAME * 3;++/** true: every sampled alpha byte is 255; false: some transparency; undefined: malformed sample. */+export function sampledRgbaAlphaIsFullyOpaque(buf: Buffer): boolean | undefined {+ if (+ buf.length === 0 ||+ buf.length > MAX_SAMPLE_BYTES ||+ buf.length % BYTES_PER_SAMPLE_FRAME !== 0+ ) {+ return undefined;+ }+ for (let i = 3; i < buf.length; i += 4) {+ if (buf[i] !== 255) return false;+ }+ return true;+}++/**+ * Whether a video input that declares alpha decodes fully opaque (a remux can keep `alpha_mode=1`+ * while dropping the alpha sidecar). `decoder` must keep alpha. Any failure returns undefined.+ */+export async function probeInputAlphaPlane(+ videoPath: string,+ decoder: string,+ signal?: AbortSignal,+): Promise<boolean | undefined> {+ let probeDir: string | undefined;+ try {+ probeDir = mkdtempSync(join(tmpdir(), "hf-alpha-probe-"));+ const samplePath = join(probeDir, "alpha.raw");+ const result = await runFfmpeg(+ [+ "-v",+ "error",+ "-c:v",+ decoder,+ "-i",+ videoPath,+ "-frames:v",+ "3",+ "-vf",+ "scale=8:8",+ "-pix_fmt",+ "rgba",+ "-f",+ "rawvideo",+ samplePath,+ ],+ { timeout: 30_000, signal },+ );+ if (!result.success) return undefined;+ return sampledRgbaAlphaIsFullyOpaque(readFileSync(samplePath));+ } catch {+ return undefined;+ } finally {+ if (probeDir) rmSync(probeDir, { recursive: true, force: true });+ }+}++export function inputAlphaOpaqueWarning(src: string): string {+ return (+ `[hyperframes:render] WARNING: video src="${src}" declares an alpha channel ` ++ "but its first frames decode fully opaque. If it should be transparent, re-export " ++ "it with an alpha pixel format and avoid remuxing afterward, which can drop the " ++ "alpha while keeping the tag.\n"+ );+}