The open source AI video agent: turns any coding agent into a video agent, with an editor to change anything it makes.
Render: on Windows, a browser that doesn't show the video frame sought (H.264 or HEVC) is not used (#211)
riverisbuilding27432f0095eb
diff --git a/.changeset/windows-video-shown.md b/.changeset/windows-video-shown.mdnew file mode 100644index 0000000..372ee1f--- /dev/null+++ b/.changeset/windows-video-shown.md@@ -0,0 +1,5 @@+---+"openfilm": patch+---++Exports on Windows: on some computers with an NVIDIA GPU, Vulkan never showed the video frame a seek asked for. Video clips came out black, or an export drew one frame every five seconds, as with HEVC footage such as DJI cameras record. OpenFilm now checks once that an H.264 and an HEVC frame show as sought. If one doesn't, it uses the same GPU through Direct3D, or keeps Vulkan and decodes video on the CPU.diff --git a/packages/openfilm/src/host.d.mts b/packages/openfilm/src/host.d.mtsindex 008f50a..33779ce 100644--- a/packages/openfilm/src/host.d.mts+++ b/packages/openfilm/src/host.d.mts@@ -8,8 +8,10 @@ export interface LaunchOptions { headed?: boolean; channel?: string | null; argsexport declare function launch(opts?: LaunchOptions): Promise<Browser>;/** A renderer string that is not a GPU: WebGL drawn on the CPU. */export declare const SOFTWARE_GL: RegExp;-/** Windows: open with Vulkan once, and keep it only when its renderer is a GPU; otherwise Chromium's default. */-export declare function vulkanFirst(renderer?: (browser: Browser) => Promise<string>): (open: (angle: string[]) => Promise<Browser>) => Promise<Browser>;+/** Windows: open with Vulkan once, and keep it only when its renderer is a GPU and it shows videos; otherwise the GPU another way, or Chromium's default. */+export declare function vulkanFirst(renderer?: (browser: Browser) => Promise<string>, presents?: (browser: Browser) => Promise<boolean>): (open: (angle: string[]) => Promise<Browser>) => Promise<Browser>;+/** Whether the browser shows the frame a paused video is sought to, for H.264 and HEVC (some Windows computers with Vulkan do not). */+export declare function presentsVideo(browser: Browser): Promise<boolean>;/** Close every browser launch() opened that is still open: what waits on one fails at once instead of never answering. */export declare function closeBrowsers(): Promise<void>;/** With `offline`: the origins still open to a page, and optionally who answers them (a cache). */diff --git a/packages/openfilm/src/host.mjs b/packages/openfilm/src/host.mjsindex 4fa337a..e5a1d47 100644--- a/packages/openfilm/src/host.mjs+++ b/packages/openfilm/src/host.mjs@@ -291,14 +291,77 @@ async function rendererOf(browser) {try { return await page.evaluate(readRenderer); } finally { await page.close(); }}+/* two seconds of video, red then blue, 640 × 360 at 4 fps, large enough to be decoded on the chip: H.264, and HEVC+ Main 10 (what phones and cameras such as DJI's record) */+const PRESENT_CHECKS = [+ { type: 'video/mp4; codecs="avc1.4D401E"', src: 'data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAANkbW9vdgAAAGxtdmhkAAAAAAAAAAAAAAAAAAAD6AAAB9AAAQAAAQAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAArN0cmFrAAAAXHRraGQAAAADAAAAAAAAAAAAAAABAAAAAAAAB9AAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAoAAAAFoAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAAfQAAAgAAABAAAAAAIrbWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAAABAAAAAkABVxAAAAAAALWhkbHIAAAAAAAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAAB1m1pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAAACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAAZZzdGJsAAAAunN0c2QAAAAAAAAAAQAAAKphdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAoABaABIAAAASAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGP//AAAAMGF2Y0MBTUAW/+EAGGdNQBbsoFAX/LgIgAAAAwCAAAAEB4sWywEABWjr48sgAAAAEHBhc3AAAAABAAAAAQAAABRidHJ0AAAAAAAAD8QAAAUMAAAAGHN0dHMAAAAAAAAAAQAAAAgAABAAAAAAGHN0c3MAAAAAAAAAAgAAAAEAAAAFAAAAQGN0dHMAAAAAAAAABgAAAAEAACAAAAAAAQAAQAAAAAACAAAQAAAAAAEAACAAAAAAAQAAQAAAAAACAAAQAAAAABxzdHNjAAAAAAAAAAEAAAABAAAACAAAAAEAAAA0c3RzegAAAAAAAAAAAAAACAAAAGYAAAAWAAAAEwAAABMAAABlAAAAFgAAABMAAAATAAAAFHN0Y28AAAAAAAAAAQAAA5QAAAA9dWR0YQAAADVtZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGlyYXBwbAAAAAAAAAAAAAAAAAhpbHN0AAAACGZyZWUAAAFLbWRhdAAAAGJliIQAEv/+6Mn8yysv49Q3s0Yps0mnHMI9HTDq9Ryj5DaMReURDAAAAwAAAwAAAwDAA6AsaXWRAN14AAAGSADXhLQmodQXAagd4oI6hTCAEAAAAwAAAwAAAwAAAwAAAwAW8QAAABJBmiNsQQ/+qlUAAAMAAAMAOqAAAAAPQZ5BeIIfAAADAAADACThAAAADwGeYmpBDwAAAwAAAwAk4AAAAGFliIIABX/+7M9+BTcgPIv1lAh0tW5145jTpoZ4rq/03qFxGMOPJbgAAAMAAAMAAAXj8myN92IA8GNgAAAakALcEDCABQhFRBxGhnCVELHSMkAAAAMAAAMAAAMAAAMAAEBBAAAAEkGaI2xBD/6qVQAAAwAAAwA6oQAAAA9BnkF4gj8AAAMAAAMAE3EAAAAPAZ5iakEPAAADAAADACTh' },+ { type: 'video/mp4; codecs="hvc1.2.4.L90.90"', src: 'data:video/mp4;base64,AAAAHGZ0eXBpc29tAAACAGlzb21pc28ybXA0MQAAA9ptb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAH0AABAAABAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAADKXRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAAH0AAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAACgAAAAWgAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAB9AAACAAAAEAAAAAAqFtZGlhAAAAIG1kaGQAAAAAAAAAAAAAAAAAAEAAAACAAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVvSGFuZGxlcgAAAAJMbWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAACDHN0YmwAAAEMc3RzZAAAAAAAAAABAAAA/Gh2YzEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAACgAFoAEgAAABIAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY//8AAAB4aHZjQwECIAAAAJAAAAAAAD/wAPz9+voAAA8DoAABABhAAQwB//8CIAAAAwCQAAADAAADAD+VmAmhAAEAK0IBAQIgAAADAJAAAAMAAAMAP6AFAgFpNllZpJMrwFoCAAADAAIAAAMACBCiAAEAB0QBwXK0YkAAAAAKZmllbAEAAAAAEHBhc3AAAAABAAAAAQAAABRidHJ0AAAAAAAAB6wAAAesAAAAGHN0dHMAAAAAAAAAAQAAAAgAABAAAAAAGHN0c3MAAAAAAAAAAgAAAAEAAAAFAAAAFHNkdHAAAAAAIBAQGCAQEBgAAABQY3R0cwAAAAAAAAAIAAAAAQAAIAAAAAABAABAAAAAAAEAACAAAAAAAQAAAAAAAAABAAAgAAAAAAEAAEAAAAAAAQAAIAAAAAABAAAAAAAAABxzdHNjAAAAAAAAAAEAAAABAAAACAAAAAEAAAA0c3RzegAAAAAAAAAAAAAACAAAAIAAAAAqAAAAJgAAACYAAAB/AAAAKgAAACYAAAAmAAAAFHN0Y28AAAAAAAAAAQAABAYAAAA9dWR0YQAAADVtZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGlyYXBwbAAAAAAAAAAAAAAAAAhpbHN0AAAACGZyZWUAAAHzbWRhdAAAAHwoAa9TGgPNMLj5ZpY//2P/v+ktUPOsHZWT3Zj9dvGaMANAGeMIAAADAh6lxUSQF4CqFkAAAAMAAFtAkwRAlwAAAwAAAwAAMWCHAAADAAADAAADAAWsAAADAAADAAADAAOSAAADAAADAAADAAADAAADAAADAAADAAADAAgoAAAAJgIB0BlfhDGhnLG491jjOcDvabTKBRyXNQB/gIcAD4gAAKOAAAOqAAAAIgIB4ESX4ZBm62qSgPSAim8AADjgAAAEtAAAfUAAA5YAENAAAAAiAAHgJP/6LNM1W1CSAZOL/0AAPCBcgAXsAAB9QAADbgAUUAAAAHsqAawQTSfMXzmsXPkYpn//uOf3fn2+/v/wtzfaQosb38AHUFsIYAAABpymb38AKEFmMIAAAAMAAROTBRi9AAADAAADAAAt4IcAAAMAAAMAAAMABRQAAAMAAAMAAAMAA5oAAAMAAAMAAAMAAAMAAAMAAAMAAAMAAAMABawAAAAmAgHQOV+EMaGcsbj3WOM5wO9ptMoFHJc1AH+AhwAPiAAAo4AAA6oAAAAiAgHgxJfhkmbrapKA9ICKbwAAOOAAAAS0AAB9QAADlgAQ0AAAACIAAeCk//oswzVbUJIBk4v/QAA8IFyABewAAH1AAANuABRQ' },+];++/**+ * Whether this browser shows a paused video's frame once sought, for each kind of video it plays. On some Windows+ * computers with Vulkan the frame sought is never the one shown (every seek waits out its tries in showVideoAt,+ * timeline.mjs: five seconds a frame), or nothing is shown at all (a black video), for some only with HEVC. A video+ * this browser does not play or load says nothing either way: such footage gets a copy it plays (seekable.mjs).+ * @param {import('playwright-core').Browser} browser+ */+export async function presentsVideo(browser) {+ const page = await browser.newPage();+ try {+ return await page.evaluate(async (checks) => {+ for (const { type, src } of checks) {+ const v = document.createElement('video');+ if (!v.canPlayType(type) || typeof v.requestVideoFrameCallback !== 'function') continue;+ v.muted = true;+ v.style.cssText = 'width:320px;height:180px';+ document.body.append(v);+ const loaded = await new Promise((done) => {+ v.addEventListener('loadeddata', () => done(true), { once: true });+ v.addEventListener('error', () => done(false), { once: true });+ setTimeout(() => done(false), 5000);+ v.src = src;+ });+ /* the frame shown must be the one sought (each is a quarter second); a seek just after the file loaded may+ never present it (see showVideoAt), so it is tried again, elsewhere */+ let shown = !loaded;+ for (const t of loaded ? [1.5, 0.5, 1.25] : []) {+ shown = await new Promise((done) => {+ const next = () => v.requestVideoFrameCallback((now, meta) => (Math.abs(meta.mediaTime - t) < 0.125 ? done(true) : next()));+ next();+ setTimeout(() => done(false), 1500);+ v.currentTime = t;+ });+ if (shown) break;+ }+ v.remove();+ if (!shown) return false;+ }+ return true;+ }, PRESENT_CHECKS);+ } finally {+ await page.close();+ }+}++/** The device id in an ANGLE renderer string (`(0x00002489)`), the same GPU's through Vulkan and D3D11. */+const deviceOf = (/** @type {string} */ renderer) => {+ const id = /\(0x([0-9a-f]+)\)/i.exec(renderer);+ return id ? parseInt(id[1], 16) : null;+};+/*** Windows: headless Chromium picks no ANGLE backend by itself and draws WebGL with SwiftShader, on the CPU. Vulkan* reaches the GPU (on a laptop with two, the discrete one: d3d11 and gl take the integrated one). A virtual machine, a* CI runner or a computer without a Vulkan driver has none, so the first browser reads its renderer once: when Vulkan* did not start, or is software too, it and every later browser take Chromium's default.+ *+ * Vulkan on a GPU may still never show a video's frames (presentsVideo). Then the default is taken when it reaches the+ * same GPU (a computer with one), else Vulkan with videos decoded on the CPU; each only when it shows them. When+ * neither does, the default.* @param {(browser: import('playwright-core').Browser) => Promise<string>} [renderer]+ * @param {(browser: import('playwright-core').Browser) => Promise<boolean>} [presents]*/-export function vulkanFirst(renderer = rendererOf) {+export function vulkanFirst(renderer = rendererOf, presents = presentsVideo) {/** @type {Promise<string[]> | null} what every browser after the first is opened with */let chosen = null;/** @param {(angle: string[]) => Promise<import('playwright-core').Browser>} open */@@ -307,16 +370,36 @@ export function vulkanFirst(renderer = rendererOf) {/** @type {(angle: string[]) => void} */let choose = () => {};chosen = new Promise((r) => { choose = r; });+ const read = (/** @type {import('playwright-core').Browser} */ b) => inTime(renderer(b), 30_000, 'no renderer').catch(() => 'none');+ const shows = (/** @type {import('playwright-core').Browser} */ b) => inTime(presents(b), 30_000, '').catch(() => true);const vulkan = ['--use-angle=vulkan'];const browser = await open(vulkan).catch(() => null);- const gpu = browser ? await inTime(renderer(browser), 30_000, 'no renderer').catch(() => 'none') : 'none';- if (browser && gpu !== 'none' && !SOFTWARE_GL.test(gpu)) {+ const gpu = browser ? await read(browser) : 'none';+ if (!browser || gpu === 'none' || SOFTWARE_GL.test(gpu)) {+ await browser?.close().catch(() => {});+ choose([]);+ return open([]);+ }+ if (await shows(browser)) {choose(vulkan);return browser;}- await browser?.close().catch(() => {});+ await browser.close().catch(() => {});+ const fallback = await open([]);+ if (deviceOf(gpu) != null && deviceOf(await read(fallback)) === deviceOf(gpu) && await shows(fallback)) {+ choose([]);+ return fallback;+ }+ const decoded = [...vulkan, '--disable-accelerated-video-decode'];+ const cpu = await open(decoded).catch(() => null);+ if (cpu && await shows(cpu)) {+ await fallback.close().catch(() => {});+ choose(decoded);+ return cpu;+ }+ await cpu?.close().catch(() => {});choose([]);- return open([]);+ return fallback;};}const onWindows = vulkanFirst();diff --git a/packages/openfilm/src/host.test.mjs b/packages/openfilm/src/host.test.mjsindex 4a40407..d03ee5c 100644--- a/packages/openfilm/src/host.test.mjs+++ b/packages/openfilm/src/host.test.mjs@@ -7,7 +7,7 @@ import { get } from 'node:http';import { syncBuiltinESMExports } from 'node:module';import { tmpdir } from 'node:os';import { join } from 'node:path';-import { serve, vulkanFirst } from './host.mjs';+import { launch, presentsVideo, serve, vulkanFirst } from './host.mjs';import { locate } from './shared.mjs';test('cancelled media requests close their files so the project can be renamed', { timeout: 10000 }, async (t) => {@@ -147,7 +147,8 @@ test('Windows: Vulkan when its renderer is a GPU, read once; otherwise the defauconst vulkan = ['--use-angle=vulkan'];const gpu = reads(async () => 'ANGLE (NVIDIA, Vulkan 1.4.329 (NVIDIA GeForce RTX 5070 Laptop GPU (0x00002D58)), NVIDIA)');- let l = launcher(), pick = vulkanFirst(gpu.renderer);+ const always = async () => true;+ let l = launcher(), pick = vulkanFirst(gpu.renderer, always);const browsers = await Promise.all([pick(l.open), pick(l.open), pick(l.open)]);assert.deepEqual(browsers.map((b) => b.angle), [vulkan, vulkan, vulkan]);assert.equal(gpu.count, 1);@@ -161,7 +162,7 @@ test('Windows: Vulkan when its renderer is a GPU, read once; otherwise the defauasync () => { throw new Error('page crashed'); },]) {const software = reads(said);- l = launcher(); pick = vulkanFirst(software.renderer);+ l = launcher(); pick = vulkanFirst(software.renderer, always);const [first, second] = await Promise.all([pick(l.open), pick(l.open)]);assert.deepEqual([first.angle, second.angle], [[], []]);assert.deepEqual(l.opened, [vulkan, [], []]);@@ -170,9 +171,76 @@ test('Windows: Vulkan when its renderer is a GPU, read once; otherwise the defau}const never = reads(async () => 'unused');- l = launcher(true); pick = vulkanFirst(never.renderer);+ l = launcher(true); pick = vulkanFirst(never.renderer, always);assert.deepEqual((await pick(l.open)).angle, []);assert.deepEqual((await pick(l.open)).angle, []);assert.deepEqual(l.opened, [vulkan, [], []]);assert.equal(never.count, 0);});++test('Windows: when Vulkan never shows a video frame, the same GPU by default, else Vulkan decoding on the CPU, else the default', async () => {+ const vulkan = ['--use-angle=vulkan'];+ const cpu = [...vulkan, '--disable-accelerated-video-decode'];+ const nvidia = 'ANGLE (NVIDIA, Vulkan 1.4.329 (NVIDIA GeForce RTX 4050 Laptop GPU (0x000028A1)), NVIDIA)';+ const nvidiaD3d = 'ANGLE (NVIDIA, NVIDIA GeForce RTX 4050 Laptop GPU (0x000028A1) Direct3D11 vs_5_0 ps_5_0, D3D11)';+ const intelD3d = 'ANGLE (Intel, Intel(R) UHD Graphics (0x0000A7A8) Direct3D11 vs_5_0 ps_5_0, D3D11)';+ /** Browsers by what they are opened with: the default's renderer, and which ones show a video's frames. */+ const machine = (/** @type {string} */ fallbackGpu, /** @type {string[][]} */ showing) => {+ const opened = /** @type {string[][]} */ ([]), closed = /** @type {string[][]} */ ([]);+ const open = async (/** @type {string[]} */ angle) => {+ opened.push(angle);+ return /** @type {any} */ ({ angle, close: async () => { closed.push(angle); } });+ };+ const renderer = async (/** @type {any} */ b) => (b.angle.length ? nvidia : fallbackGpu);+ const presents = async (/** @type {any} */ b) => showing.some((a) => a.join() === b.angle.join());+ return { pick: vulkanFirst(renderer, presents), open, opened, closed };+ };++ /* one GPU: D3D11 reaches it and shows the frames */+ let m = machine(nvidiaD3d, [[]]);+ assert.deepEqual((await m.pick(m.open)).angle, []);+ assert.deepEqual((await m.pick(m.open)).angle, []);+ assert.deepEqual(m.opened, [vulkan, [], []]);+ assert.deepEqual(m.closed, [vulkan]);++ /* two GPUs (#64): the default is the integrated one, so Vulkan stays, its videos decoded on the CPU */+ m = machine(intelD3d, [[], cpu]);+ assert.deepEqual((await m.pick(m.open)).angle, cpu);+ assert.deepEqual((await m.pick(m.open)).angle, cpu);+ assert.deepEqual(m.opened, [vulkan, [], cpu, cpu]);+ assert.deepEqual(m.closed, [vulkan, []]);++ /* nothing shows them: the default, as without a GPU */+ m = machine(intelD3d, []);+ assert.deepEqual((await m.pick(m.open)).angle, []);+ assert.deepEqual(m.opened, [vulkan, [], cpu]);+ assert.deepEqual(m.closed, [vulkan, cpu]);+});++test('a browser shows a paused video\'s frame once sought; one that never presents it is told', { timeout: 60_000 }, async () => {+ const browser = await launch();+ try {+ assert.equal(await presentsVideo(browser), true);+ /* a compositor that never presents: the frame callback never comes */+ const never = { newPage: async () => {+ const page = await browser.newPage();+ await page.evaluate(() => { HTMLVideoElement.prototype.requestVideoFrameCallback = () => 0; });+ return page;+ } };+ assert.equal(await presentsVideo(/** @type {any} */ (never)), false);+ /* one that shows a frame, never the one sought: as slow as none at all */+ const wrong = { newPage: async () => {+ const page = await browser.newPage();+ await page.evaluate(() => {+ HTMLVideoElement.prototype.requestVideoFrameCallback = function (callback) {+ setTimeout(() => callback(performance.now(), /** @type {any} */ ({ mediaTime: this.currentTime + 1 })), 10);+ return 0;+ };+ });+ return page;+ } };+ assert.equal(await presentsVideo(/** @type {any} */ (wrong)), false);+ } finally {+ await browser.close();+ }+});