The open source AI video agent: turns any coding agent into a video agent, with an editor to change anything it makes.
CI: no more 45-minute timeouts (apt packages cached, hung chip copies stopped) (#265)
riverisbuildinge052d6e3a3bf
diff --git a/.changeset/hung-chip-copies.md b/.changeset/hung-chip-copies.mdnew file mode 100644index 0000000..abe32df--- /dev/null+++ b/.changeset/hung-chip-copies.md@@ -0,0 +1,5 @@+---+"openfilm": patch+---++A render no longer waits forever on a graphics chip that hangs while it copies footage the browser can't decode or seek well (VideoToolbox does on some virtual Macs). A copy on the chip that writes nothing for 15 seconds is stopped, with no ffmpeg left running, and made on the processor; the chip's encoder is left out for the rest of the launch.diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 3be983f..5a7fb58 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -31,9 +31,45 @@ jobs:cache: pnpm- run: pnpm install --frozen-lockfile- run: pnpm typecheck- - run: pnpm --filter openfilm exec playwright-core install --with-deps chromium-headless-shell- - run: sudo apt-get update && sudo apt-get install -y ffmpeg+ # Ubuntu's packages (Playwright's system libraries and fonts, ffmpeg) come through apt, whose mirror at times took+ # 20 to 40 minutes for them. The .deb files are kept in the Actions cache, by runner image: apt fetches only what+ # changed since. A stalled mirror is left for the next one after 30 s (not 120), a failed download is tried+ # again, and a step that is still slow ends the job there, not at its limit with the tests unrun+ - id: apt+ if: runner.os == 'Linux'+ run: |+ printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nAPT::Keep-Downloaded-Packages "true";\n' | sudo tee /etc/apt/apt.conf.d/80-ci+ echo "prefix=apt-$ImageOS-" >> "$GITHUB_OUTPUT"+ echo "key=apt-$ImageOS-$ImageVersion" >> "$GITHUB_OUTPUT"+ - id: apt-cacheif: runner.os == 'Linux'+ uses: actions/cache/restore@v4+ with:+ path: ~/apt-archives+ key: ${{ steps.apt.outputs.key }}+ restore-keys: ${{ steps.apt.outputs.prefix }}+ - if: runner.os == 'Linux'+ run: |+ mkdir -p ~/apt-archives+ sudo cp ~/apt-archives/*.deb /var/cache/apt/archives/ 2>/dev/null || true+ echo "$(ls ~/apt-archives | wc -l) packages from the cache"+ # on Ubuntu its apt-get update is the only one: ffmpeg's install below uses it+ - run: pnpm --filter openfilm exec playwright-core install --with-deps chromium-headless-shell+ timeout-minutes: 15+ - if: runner.os == 'Linux'+ timeout-minutes: 10+ run: |+ sudo apt-get install -y --no-install-recommends ffmpeg+ ffmpeg -hide_banner -version | head -n 1+ # what the cache keeps: the packages as they are now, none an update replaced+ sudo apt-get autoclean -y > /dev/null+ rm -f ~/apt-archives/*.deb+ cp /var/cache/apt/archives/*.deb ~/apt-archives/+ - if: runner.os == 'Linux' && steps.apt-cache.outputs.cache-hit != 'true'+ uses: actions/cache/save@v4+ with:+ path: ~/apt-archives+ key: ${{ steps.apt.outputs.key }}- run: brew install ffmpegif: runner.os == 'macOS'# Chocolatey reports success when its feed times out and nothing is installed: tried three times, then the stepdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex a6e3b5d..fb88f3e 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -43,8 +43,25 @@ jobs:- run: npm install -g npm@^11.5.1- run: pnpm install --frozen-lockfile- run: pnpm typecheck+ # Ubuntu's packages through apt as ci.yml gets them: from the cache its runs keep, a stalled mirror left after 30 s,+ # a slow step ending the release there rather than hours on+ - id: apt+ run: |+ printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nAPT::Keep-Downloaded-Packages "true";\n' | sudo tee /etc/apt/apt.conf.d/80-ci+ echo "prefix=apt-$ImageOS-" >> "$GITHUB_OUTPUT"+ echo "key=apt-$ImageOS-$ImageVersion" >> "$GITHUB_OUTPUT"+ - uses: actions/cache/restore@v4+ with:+ path: ~/apt-archives+ key: ${{ steps.apt.outputs.key }}+ restore-keys: ${{ steps.apt.outputs.prefix }}+ - run: |+ mkdir -p ~/apt-archives+ sudo cp ~/apt-archives/*.deb /var/cache/apt/archives/ 2>/dev/null || true- run: pnpm --filter openfilm exec playwright-core install --with-deps chromium-headless-shell- - run: sudo apt-get update && sudo apt-get install -y ffmpeg+ timeout-minutes: 15+ - run: sudo apt-get install -y --no-install-recommends ffmpeg+ timeout-minutes: 10- run: pnpm test- name: What the package holds (the official product.json)run: pnpm check:packdiff --git a/packages/openfilm/src/encoders.mjs b/packages/openfilm/src/encoders.mjsindex fbd273f..6a43a2a 100644--- a/packages/openfilm/src/encoders.mjs+++ b/packages/openfilm/src/encoders.mjs@@ -114,8 +114,9 @@ function works(encoder) {const prores = encoder.startsWith('prores_') ? proresOptions(encoder, true) : null;const options = prores ? ['-vf', prores.pix, ...prores.video] : ['-vf', 'format=yuv420p', ...hardwareOptions(encoder, 2e6)];tried.set(encoder, new Promise((done) => {+ /* one that hangs in the chip's driver never sees a SIGTERM, and its answer would never come: killed */execFile('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-f', 'lavfi', '-i', 'testsrc2=size=640x360:rate=30', '-frames:v', '10',- ...options, '-f', 'null', '-'], { timeout: 20_000, windowsHide: true }, (error) => done(!error));+ ...options, '-f', 'null', '-'], { timeout: 20_000, killSignal: 'SIGKILL', windowsHide: true }, (error) => done(!error));}));}return /** @type {Promise<boolean>} */ (tried.get(encoder));diff --git a/packages/openfilm/src/seekable.mjs b/packages/openfilm/src/seekable.mjsindex 9a6d49d..c57254b 100644--- a/packages/openfilm/src/seekable.mjs+++ b/packages/openfilm/src/seekable.mjs@@ -23,6 +23,9 @@ const MAX_SECONDS = 10 * 60;const MAX_SIDE = 4096;/** Changing how copies are made makes new ones. */const VERSION = 2;+/** A copy on the graphics chip that writes nothing for this long has hung (VideoToolbox does on some virtual Macs):+ stopped, and made on the processor. */+const STALL_MS = 15_000;/** @param {string} file @returns {Promise<string>} */const probe = (file, args) => new Promise((done, fail) => {@@ -248,7 +251,11 @@ async function makeCopy(file, out, need, signal) {when told them */const bits = Math.min(120e6, Math.max(4e6, ((need.w * need.h) || 1920 * 1080) * need.fps * 0.3));/* decoded and written on the graphics chip when it can (encoders.mjs), else on the processor */+ let stalled = false;const make = async (/** @type {string | null} */ encoder, /** @type {string[]} */ decoding) => {+ const chip = Boolean(encoder) || decoding.length > 0;+ /* a chip that hung is not tried again for this copy: straight to the processor */+ if (chip && stalled) throw new Error('the graphics chip hung');/* the same frames at the same times (passthrough: no frame is added or dropped), the picture only */const run = ffmpeg([...decoding, '-i', file, '-map', '0:v:0', '-an', '-sn', '-dn', '-fps_mode', 'passthrough',...(need.scaled ? ['-vf', `scale=${need.w}:${need.h}`] : []),@@ -259,7 +266,25 @@ async function makeCopy(file, out, need, signal) {/* its whole tree: an ffmpeg that is a shim (Windows' package managers) runs the real one as its child */const stop = () => kill(run.proc);signal?.addEventListener('abort', stop, { once: true });- try { await run.done; } finally { signal?.removeEventListener('abort', stop); }+ /* on the chip, the part must keep growing: one that does not for STALL_MS is stopped */+ let watch;+ const hung = new Promise((_, fail) => {+ if (!chip) return;+ let size = -1, since = Date.now();+ watch = setInterval(() => {+ let now = 0;+ try { now = statSync(part).size; } catch { /* not there yet */ }+ if (now !== size) { size = now; since = Date.now(); return; }+ if (Date.now() - since < STALL_MS) return;+ stalled = true;+ clearInterval(watch);+ stop();+ /* given a moment to end, so the next try never writes the part with it */+ const ended = run.done.catch(() => {});+ Promise.race([ended, new Promise((r) => setTimeout(r, 5000).unref())]).then(() => fail(new Error(`the graphics chip wrote nothing for ${STALL_MS / 1000} s`)));+ }, 1000);+ });+ try { await Promise.race([run.done, hung]); } finally { clearInterval(watch); signal?.removeEventListener('abort', stop); }};/* a part that cannot be removed (Windows, a moment after its encoder ends) stays in the cache folder: it never failsthe render that asked for the copy */diff --git a/packages/openfilm/src/seekable.test.mjs b/packages/openfilm/src/seekable.test.mjsindex 5a8121d..6ece16c 100644--- a/packages/openfilm/src/seekable.test.mjs+++ b/packages/openfilm/src/seekable.test.mjs@@ -1,7 +1,7 @@import { test } from 'node:test';import assert from 'node:assert/strict';import { execFile, execFileSync } from 'node:child_process';-import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs';+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { filmHtml } from './film-doc.mjs';@@ -200,6 +200,44 @@ console.log(copies.get(${JSON.stringify(file)}) ?? '');`;}});+test('a copy whose graphics chip hangs is stopped and made on the processor, and leaves no ffmpeg running', {+ ...ffmpeg, skip: ffmpeg.skip || (process.platform === 'win32' && 'a shell script stands in for ffmpeg'),+}, async () => {+ const dir = mkdtempSync(join(tmpdir(), 'of-seekable-hung-'));+ try {+ const file = join(dir, 'prores.mov');+ execFileSync('ffmpeg', ['-v', 'error', '-y', '-f', 'lavfi', '-i', 'testsrc2=size=320x180:rate=30', '-t', '1', '-c:v', 'prores_ks', '-profile:v', '3', file]);+ /* ffmpeg, with a chip encoder that passes its test and then hangs on the copy: it writes its pid and never ends */+ const bin = join(dir, 'bin');+ mkdirSync(bin);+ const encoder = process.platform === 'darwin' ? 'h264_videotoolbox' : 'h264_nvenc';+ const pidFile = join(dir, 'hung.pid');+ writeFileSync(join(bin, 'ffmpeg'), `#!/bin/sh+case " $* " in+ *" -encoders "*) "$REAL_FFMPEG" "$@"; echo " V....D ${encoder} the chip's"; exit 0;;+ *" ${encoder} "*" -f null - "*) exit 0;;+ *" ${encoder} "*) echo $$ > '${pidFile}'; exec sleep 600;;+esac+exec "$REAL_FFMPEG" "$@"+`);+ chmodSync(join(bin, 'ffmpeg'), 0o755);+ const real = execFileSync('sh', ['-c', 'command -v ffmpeg']).toString().trim();+ const code = `const { seekableCopies } = await import(${JSON.stringify(new URL('./seekable.mjs', import.meta.url).href)});+const copies = await seekableCopies([${JSON.stringify(file)}], ${JSON.stringify(dir)}, { plays: async () => false, slow: false });+console.log(copies.get(${JSON.stringify(file)}) ?? '');`;+ const env = { ...process.env, PATH: `${bin}:${process.env.PATH}`, REAL_FFMPEG: real };+ const copy = await new Promise((done, fail) => execFile(process.execPath, ['--input-type=module', '-e', code], { env }, (e, out) => (e ? fail(e) : done(String(out).trim()))));+ assert.ok(existsSync(pidFile), 'the chip was tried');+ assert.ok(copy, 'made all the same');+ assert.equal(times(copy).length, 30, 'whole');+ assert.equal(readdirSync(join(dir, '.film', 'cache', 'seek')).length, 1, 'no part left');+ const pid = Number(readFileSync(pidFile, 'utf8'));+ assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }, 'the hung ffmpeg was stopped');+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});+test('look shows ProRes footage, which no browser decodes, through its copy', ffmpeg, async () => {const dir = mkdtempSync(join(tmpdir(), 'of-seekable-prores-look-'));try {