The open source AI video agent: turns any coding agent into a video agent, with an editor to change anything it makes.
Render: a WebM with an alpha whose key frames are far apart keeps its alpha (#293)
riverisbuilding005f61d76bbe
diff --git a/.changeset/alpha-webm-kept.md b/.changeset/alpha-webm-kept.mdnew file mode 100644index 0000000..0763f03--- /dev/null+++ b/.changeset/alpha-webm-kept.md@@ -0,0 +1,5 @@+---+"openfilm": patch+---++Render: a video with an alpha keeps it. A WebM (VP9) or Apple HEVC with an alpha whose key frames are far apart was given a copy without its alpha for a render, which then showed what lay under the alpha (often a solid color) in the export, and in Studio's picture once the copy was made. An HEVC with an alpha that the browser does not decode now gets a copy with its alpha.diff --git a/packages/openfilm/src/seekable.mjs b/packages/openfilm/src/seekable.mjsindex 8916340..134ecd5 100644--- a/packages/openfilm/src/seekable.mjs+++ b/packages/openfilm/src/seekable.mjs@@ -21,8 +21,8 @@ const MAX_SECONDS = 10 * 60;/** Bigger than this is not copied at its size (H.264 stops at 4096 on a side in most decoders): left as it is when itplays, else copied scaled down to fit. */const MAX_SIDE = 4096;-/** Changing how copies are made makes new ones. */-const VERSION = 2;+/** Changing how copies are made makes new ones (3: a VP9's or an HEVC's alpha was not seen, and its copy was without it). */+const VERSION = 3;/** A copy on the graphics chip that writes nothing for this long has hung (VideoToolbox does on some virtual Macs):stopped, and made on the processor. */const STALL_MS = 15_000;@@ -32,6 +32,22 @@ const probe = (file, args) => new Promise((done, fail) => {execFile('ffprobe', ['-v', 'error', ...args, file], { maxBuffer: 16 << 20, windowsHide: true }, (error, stdout) => (error ? fail(error) : done(String(stdout))));});+/** What ffprobe is asked of a video stream to tell whether it has an alpha (hasAlpha). */+const ALPHA_ENTRIES = 'pix_fmt,view_ids_available:stream_tags=alpha_mode';++/**+ * Whether a video stream (as ffprobe tells it, ALPHA_ENTRIES) has an alpha: in its pixel format (ProRes 4444, PNG,+ * QuickTime Animation…), beside its picture as a VP9's is (its `alpha_mode` tag), or as a second layer, as Apple's HEVC+ * with alpha has it, its pixel format plain yuv420p (ffprobe 7.1 and later list the layers; a spatial video's second+ * view counts too, and is left alone the same way). An older ffprobe ignores the entry it does not know.+ * @param {{ pix_fmt?: string, view_ids_available?: string, tags?: { alpha_mode?: string } }} stream+ */+export function hasAlpha(stream) {+ return /^(yuva|rgba|bgra|argb|abgr|gbrap|ya)/.test(String(stream.pix_fmt))+ || stream.tags?.alpha_mode === '1'+ || String(stream.view_ids_available ?? '').includes(',');+}+/*** What a copy of `file` would be made from: its first video stream, when it is one worth copying (8-bit 4:2:0, no* alpha, not too big or long, key frames far apart). Null when it is better left alone.@@ -40,11 +56,11 @@ const probe = (file, args) => new Promise((done, fail) => {export async function slowSeeking(file) {let info;try {- info = JSON.parse(await probe(file, ['-select_streams', 'v:0', '-show_entries', 'stream=width,height,pix_fmt,avg_frame_rate:format=duration', '-of', 'json']));+ info = JSON.parse(await probe(file, ['-select_streams', 'v:0', '-show_entries', `stream=width,height,avg_frame_rate,${ALPHA_ENTRIES}:format=duration`, '-of', 'json']));} catch { return null; }const stream = info.streams?.[0];const seconds = Number(info.format?.duration);- if (!stream || !['yuv420p', 'yuvj420p'].includes(stream.pix_fmt)) return null;+ if (!stream || !['yuv420p', 'yuvj420p'].includes(String(stream.pix_fmt)) || hasAlpha(stream)) return null;if (!(seconds > 0) || seconds > MAX_SECONDS || stream.width > MAX_SIDE || stream.height > MAX_SIDE) return null;const [n, d] = String(stream.avg_frame_rate).split('/').map(Number);const fps = n > 0 && d > 0 ? n / d : 30;@@ -82,14 +98,14 @@ const decodedAnywhere = (/** @type {{ codec_name?: string, pix_fmt?: string }} *export async function copyNeed(file, { plays, slow = true } = {}) {let info;try {- info = JSON.parse(await probe(file, ['-select_streams', 'v:0', '-show_entries', 'stream=codec_name,width,height,pix_fmt,avg_frame_rate:stream_tags=alpha_mode', '-of', 'json']));+ info = JSON.parse(await probe(file, ['-select_streams', 'v:0', '-show_entries', `stream=codec_name,width,height,avg_frame_rate,${ALPHA_ENTRIES}`, '-of', 'json']));} catch { return null; }const stream = info.streams?.[0];if (!stream) return null;const w = Number(stream.width), h = Number(stream.height);if (plays && !decodedAnywhere(stream) && !(await plays(file))) {const [n, d] = String(stream.avg_frame_rate).split('/').map(Number);- const alpha = /^(yuva|rgba|bgra|argb|abgr|gbrap|ya)/.test(String(stream.pix_fmt)) || stream.tags?.alpha_mode === '1';+ const alpha = hasAlpha(stream);const need = { why: /** @type {const} */ ('unplayable'), fps: n > 0 && d > 0 ? n / d : 30, w, h, alpha };/* too big to copy at its size: scaled down to fit, its sides even. The film draws a video scaled to its clip's box,which the original's size sets, so it is placed the same; left out, it would draw nothing at all */@@ -294,7 +310,10 @@ async function makeCopy(file, out, need, signal) {the render that asked for the copy */const drop = () => { try { rmSync(part, { force: true }); } catch { /* left */ } };try {- await onTheChip(need.alpha ? null : await hardwareEncoder('h264', await encoders()), make, signal);+ /* one with an alpha all on the processor: the chip's decoding (VideoToolbox) drops an HEVC's alpha layer, and says+ nothing */+ if (need.alpha) await make(null, []);+ else await onTheChip(await hardwareEncoder('h264', await encoders()), make, signal);} catch {drop();return signal?.aborted ? 'cancelled' : 'failed';diff --git a/packages/openfilm/src/seekable.test.mjs b/packages/openfilm/src/seekable.test.mjsindex 927391a..0bae182 100644--- a/packages/openfilm/src/seekable.test.mjs+++ b/packages/openfilm/src/seekable.test.mjs@@ -7,7 +7,7 @@ import { join, relative, sep } from 'node:path';import { filmHtml } from './film-doc.mjs';import { launch, serve } from './host.mjs';import { render } from './render.mjs';-import { copyNeed, firstFrame, madeCopy, seekableCopies, slowSeeking } from './seekable.mjs';+import { copyNeed, firstFrame, hasAlpha, madeCopy, seekableCopies, slowSeeking } from './seekable.mjs';import { look } from './look.mjs';const hasFfmpeg = (() => { try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); return true; } catch { return false; } })();@@ -41,6 +41,35 @@ test('a video with key frames far apart gets a copy with the same frames at the}});+test('an alpha is seen in a pixel format, beside a VP9\'s picture, and as an HEVC\'s second layer', () => {+ /* as ffprobe 8.1 tells them */+ assert.equal(hasAlpha({ pix_fmt: 'yuva444p12le' }), true, 'ProRes 4444');+ assert.equal(hasAlpha({ pix_fmt: 'argb' }), true, 'QuickTime Animation');+ assert.equal(hasAlpha({ pix_fmt: 'yuv420p', tags: { alpha_mode: '1' } }), true, 'VP9 with its alpha');+ assert.equal(hasAlpha({ pix_fmt: 'yuv420p', view_ids_available: '0,0', tags: {} }), true, "Apple's HEVC with alpha");+ assert.equal(hasAlpha({ pix_fmt: 'yuv420p', view_ids_available: '', tags: {} }), false, 'HEVC without');+ assert.equal(hasAlpha({ pix_fmt: 'yuv420p' }), false, 'an older ffprobe, H.264');+});++test('a VP9 with an alpha and key frames far apart keeps its alpha: it gets no copy, which would be H.264, without it', ffmpeg, async () => {+ const dir = mkdtempSync(join(tmpdir(), 'of-seekable-alpha-'));+ try {+ /* 4 s, one key frame: as an animation exported with its alpha (WebM, libvpx-vp9) usually is */+ const webm = (name, format) => execFileSync('ffmpeg', ['-v', 'error', '-y', '-f', 'lavfi', '-i', `[email protected]:size=320x180:rate=30,format=${format}`, '-t', '4',+ '-c:v', 'libvpx-vp9', '-pix_fmt', format === 'rgba' ? 'yuva420p' : 'yuv420p', '-g', '120', '-deadline', 'realtime', '-cpu-used', '8', join(dir, name)]);+ webm('alpha.webm', 'rgba');+ webm('opaque.webm', 'yuv420p');+ assert.equal(await slowSeeking(join(dir, 'alpha.webm')), null);+ assert.ok(await slowSeeking(join(dir, 'opaque.webm')), 'the same without an alpha seeks slowly');+ assert.equal(await copyNeed(join(dir, 'alpha.webm'), { plays: async () => true }), null);+ const copies = await seekableCopies([join(dir, 'alpha.webm'), join(dir, 'opaque.webm')], dir, { plays: async () => true });+ assert.deepEqual([...copies.keys()], [join(dir, 'opaque.webm')]);+ assert.equal(madeCopy(join(dir, 'alpha.webm'), dir), null, 'a page asking for it is served it as it is');+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});+test('a render serves the film\'s slow-seeking videos as their copies', ffmpeg, async () => {const dir = mkdtempSync(join(tmpdir(), 'of-seekable-render-'));try {