Skip to content

The open source AI video agent: turns any coding agent into a video agent, with an editor to change anything it makes.

Render: a WebM with an alpha whose key frames are far apart keeps its alpha (#293)

riverisbuilding005f61d76bbe
diff --git a/.changeset/alpha-webm-kept.md b/.changeset/alpha-webm-kept.md
new file mode 100644
index 0000000..0763f03
--- /dev/null
+++ b/.changeset/alpha-webm-kept.md
@@ -0,0 +1,5 @@
+---
+"openfilm": patch
+---
+
+Render: a video with an alpha keeps it. A WebM (VP9) or Apple HEVC with an alpha whose key frames are far apart was given a copy without its alpha for a render, which then showed what lay under the alpha (often a solid color) in the export, and in Studio's picture once the copy was made. An HEVC with an alpha that the browser does not decode now gets a copy with its alpha.
diff --git a/packages/openfilm/src/seekable.mjs b/packages/openfilm/src/seekable.mjs
index 8916340..134ecd5 100644
--- a/packages/openfilm/src/seekable.mjs
+++ b/packages/openfilm/src/seekable.mjs
@@ -21,8 +21,8 @@ const MAX_SECONDS = 10 * 60;
/** Bigger than this is not copied at its size (H.264 stops at 4096 on a side in most decoders): left as it is when it
plays, else copied scaled down to fit. */
const MAX_SIDE = 4096;
-/** Changing how copies are made makes new ones. */
-const VERSION = 2;
+/** Changing how copies are made makes new ones (3: a VP9's or an HEVC's alpha was not seen, and its copy was without it). */
+const VERSION = 3;
/** A copy on the graphics chip that writes nothing for this long has hung (VideoToolbox does on some virtual Macs):
stopped, and made on the processor. */
const STALL_MS = 15_000;
@@ -32,6 +32,22 @@ const probe = (file, args) => new Promise((done, fail) => {
execFile('ffprobe', ['-v', 'error', ...args, file], { maxBuffer: 16 << 20, windowsHide: true }, (error, stdout) => (error ? fail(error) : done(String(stdout))));
});
+/** What ffprobe is asked of a video stream to tell whether it has an alpha (hasAlpha). */
+const ALPHA_ENTRIES = 'pix_fmt,view_ids_available:stream_tags=alpha_mode';
+
+/**
+ * Whether a video stream (as ffprobe tells it, ALPHA_ENTRIES) has an alpha: in its pixel format (ProRes 4444, PNG,
+ * QuickTime Animation…), beside its picture as a VP9's is (its `alpha_mode` tag), or as a second layer, as Apple's HEVC
+ * with alpha has it, its pixel format plain yuv420p (ffprobe 7.1 and later list the layers; a spatial video's second
+ * view counts too, and is left alone the same way). An older ffprobe ignores the entry it does not know.
+ * @param {{ pix_fmt?: string, view_ids_available?: string, tags?: { alpha_mode?: string } }} stream
+ */
+export function hasAlpha(stream) {
+ return /^(yuva|rgba|bgra|argb|abgr|gbrap|ya)/.test(String(stream.pix_fmt))
+ || stream.tags?.alpha_mode === '1'
+ || String(stream.view_ids_available ?? '').includes(',');
+}
+
/**
* What a copy of `file` would be made from: its first video stream, when it is one worth copying (8-bit 4:2:0, no
* alpha, not too big or long, key frames far apart). Null when it is better left alone.
@@ -40,11 +56,11 @@ const probe = (file, args) => new Promise((done, fail) => {
export async function slowSeeking(file) {
let info;
try {
- info = JSON.parse(await probe(file, ['-select_streams', 'v:0', '-show_entries', 'stream=width,height,pix_fmt,avg_frame_rate:format=duration', '-of', 'json']));
+ info = JSON.parse(await probe(file, ['-select_streams', 'v:0', '-show_entries', `stream=width,height,avg_frame_rate,${ALPHA_ENTRIES}:format=duration`, '-of', 'json']));
} catch { return null; }
const stream = info.streams?.[0];
const seconds = Number(info.format?.duration);
- if (!stream || !['yuv420p', 'yuvj420p'].includes(stream.pix_fmt)) return null;
+ if (!stream || !['yuv420p', 'yuvj420p'].includes(String(stream.pix_fmt)) || hasAlpha(stream)) return null;
if (!(seconds > 0) || seconds > MAX_SECONDS || stream.width > MAX_SIDE || stream.height > MAX_SIDE) return null;
const [n, d] = String(stream.avg_frame_rate).split('/').map(Number);
const fps = n > 0 && d > 0 ? n / d : 30;
@@ -82,14 +98,14 @@ const decodedAnywhere = (/** @type {{ codec_name?: string, pix_fmt?: string }} *
export async function copyNeed(file, { plays, slow = true } = {}) {
let info;
try {
- info = JSON.parse(await probe(file, ['-select_streams', 'v:0', '-show_entries', 'stream=codec_name,width,height,pix_fmt,avg_frame_rate:stream_tags=alpha_mode', '-of', 'json']));
+ info = JSON.parse(await probe(file, ['-select_streams', 'v:0', '-show_entries', `stream=codec_name,width,height,avg_frame_rate,${ALPHA_ENTRIES}`, '-of', 'json']));
} catch { return null; }
const stream = info.streams?.[0];
if (!stream) return null;
const w = Number(stream.width), h = Number(stream.height);
if (plays && !decodedAnywhere(stream) && !(await plays(file))) {
const [n, d] = String(stream.avg_frame_rate).split('/').map(Number);
- const alpha = /^(yuva|rgba|bgra|argb|abgr|gbrap|ya)/.test(String(stream.pix_fmt)) || stream.tags?.alpha_mode === '1';
+ const alpha = hasAlpha(stream);
const need = { why: /** @type {const} */ ('unplayable'), fps: n > 0 && d > 0 ? n / d : 30, w, h, alpha };
/* too big to copy at its size: scaled down to fit, its sides even. The film draws a video scaled to its clip's box,
which the original's size sets, so it is placed the same; left out, it would draw nothing at all */
@@ -294,7 +310,10 @@ async function makeCopy(file, out, need, signal) {
the render that asked for the copy */
const drop = () => { try { rmSync(part, { force: true }); } catch { /* left */ } };
try {
- await onTheChip(need.alpha ? null : await hardwareEncoder('h264', await encoders()), make, signal);
+ /* one with an alpha all on the processor: the chip's decoding (VideoToolbox) drops an HEVC's alpha layer, and says
+ nothing */
+ if (need.alpha) await make(null, []);
+ else await onTheChip(await hardwareEncoder('h264', await encoders()), make, signal);
} catch {
drop();
return signal?.aborted ? 'cancelled' : 'failed';
diff --git a/packages/openfilm/src/seekable.test.mjs b/packages/openfilm/src/seekable.test.mjs
index 927391a..0bae182 100644
--- a/packages/openfilm/src/seekable.test.mjs
+++ b/packages/openfilm/src/seekable.test.mjs
@@ -7,7 +7,7 @@ import { join, relative, sep } from 'node:path';
import { filmHtml } from './film-doc.mjs';
import { launch, serve } from './host.mjs';
import { render } from './render.mjs';
-import { copyNeed, firstFrame, madeCopy, seekableCopies, slowSeeking } from './seekable.mjs';
+import { copyNeed, firstFrame, hasAlpha, madeCopy, seekableCopies, slowSeeking } from './seekable.mjs';
import { look } from './look.mjs';
const hasFfmpeg = (() => { try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); return true; } catch { return false; } })();
@@ -41,6 +41,35 @@ test('a video with key frames far apart gets a copy with the same frames at the
}
});
+test('an alpha is seen in a pixel format, beside a VP9\'s picture, and as an HEVC\'s second layer', () => {
+ /* as ffprobe 8.1 tells them */
+ assert.equal(hasAlpha({ pix_fmt: 'yuva444p12le' }), true, 'ProRes 4444');
+ assert.equal(hasAlpha({ pix_fmt: 'argb' }), true, 'QuickTime Animation');
+ assert.equal(hasAlpha({ pix_fmt: 'yuv420p', tags: { alpha_mode: '1' } }), true, 'VP9 with its alpha');
+ assert.equal(hasAlpha({ pix_fmt: 'yuv420p', view_ids_available: '0,0', tags: {} }), true, "Apple's HEVC with alpha");
+ assert.equal(hasAlpha({ pix_fmt: 'yuv420p', view_ids_available: '', tags: {} }), false, 'HEVC without');
+ assert.equal(hasAlpha({ pix_fmt: 'yuv420p' }), false, 'an older ffprobe, H.264');
+});
+
+test('a VP9 with an alpha and key frames far apart keeps its alpha: it gets no copy, which would be H.264, without it', ffmpeg, async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'of-seekable-alpha-'));
+ try {
+ /* 4 s, one key frame: as an animation exported with its alpha (WebM, libvpx-vp9) usually is */
+ const webm = (name, format) => execFileSync('ffmpeg', ['-v', 'error', '-y', '-f', 'lavfi', '-i', `[email protected]:size=320x180:rate=30,format=${format}`, '-t', '4',
+ '-c:v', 'libvpx-vp9', '-pix_fmt', format === 'rgba' ? 'yuva420p' : 'yuv420p', '-g', '120', '-deadline', 'realtime', '-cpu-used', '8', join(dir, name)]);
+ webm('alpha.webm', 'rgba');
+ webm('opaque.webm', 'yuv420p');
+ assert.equal(await slowSeeking(join(dir, 'alpha.webm')), null);
+ assert.ok(await slowSeeking(join(dir, 'opaque.webm')), 'the same without an alpha seeks slowly');
+ assert.equal(await copyNeed(join(dir, 'alpha.webm'), { plays: async () => true }), null);
+ const copies = await seekableCopies([join(dir, 'alpha.webm'), join(dir, 'opaque.webm')], dir, { plays: async () => true });
+ assert.deepEqual([...copies.keys()], [join(dir, 'opaque.webm')]);
+ assert.equal(madeCopy(join(dir, 'alpha.webm'), dir), null, 'a page asking for it is served it as it is');
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
test('a render serves the film\'s slow-seeking videos as their copies', ffmpeg, async () => {
const dir = mkdtempSync(join(tmpdir(), 'of-seekable-render-'));
try {