Skip to content

The open source AI video agent: turns any coding agent into a video agent, with an editor to change anything it makes.

CI: no more 45-minute timeouts (apt packages cached, hung chip copies stopped) (#265)

riverisbuildinge052d6e3a3bf
diff --git a/.changeset/hung-chip-copies.md b/.changeset/hung-chip-copies.md
new file mode 100644
index 0000000..abe32df
--- /dev/null
+++ b/.changeset/hung-chip-copies.md
@@ -0,0 +1,5 @@
+---
+"openfilm": patch
+---
+
+A render no longer waits forever on a graphics chip that hangs while it copies footage the browser can't decode or seek well (VideoToolbox does on some virtual Macs). A copy on the chip that writes nothing for 15 seconds is stopped, with no ffmpeg left running, and made on the processor; the chip's encoder is left out for the rest of the launch.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3be983f..5a7fb58 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,9 +31,45 @@ jobs:
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm typecheck
- - run: pnpm --filter openfilm exec playwright-core install --with-deps chromium-headless-shell
- - run: sudo apt-get update && sudo apt-get install -y ffmpeg
+ # Ubuntu's packages (Playwright's system libraries and fonts, ffmpeg) come through apt, whose mirror at times took
+ # 20 to 40 minutes for them. The .deb files are kept in the Actions cache, by runner image: apt fetches only what
+ # changed since. A stalled mirror is left for the next one after 30 s (not 120), a failed download is tried
+ # again, and a step that is still slow ends the job there, not at its limit with the tests unrun
+ - id: apt
+ if: runner.os == 'Linux'
+ run: |
+ printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nAPT::Keep-Downloaded-Packages "true";\n' | sudo tee /etc/apt/apt.conf.d/80-ci
+ echo "prefix=apt-$ImageOS-" >> "$GITHUB_OUTPUT"
+ echo "key=apt-$ImageOS-$ImageVersion" >> "$GITHUB_OUTPUT"
+ - id: apt-cache
if: runner.os == 'Linux'
+ uses: actions/cache/restore@v4
+ with:
+ path: ~/apt-archives
+ key: ${{ steps.apt.outputs.key }}
+ restore-keys: ${{ steps.apt.outputs.prefix }}
+ - if: runner.os == 'Linux'
+ run: |
+ mkdir -p ~/apt-archives
+ sudo cp ~/apt-archives/*.deb /var/cache/apt/archives/ 2>/dev/null || true
+ echo "$(ls ~/apt-archives | wc -l) packages from the cache"
+ # on Ubuntu its apt-get update is the only one: ffmpeg's install below uses it
+ - run: pnpm --filter openfilm exec playwright-core install --with-deps chromium-headless-shell
+ timeout-minutes: 15
+ - if: runner.os == 'Linux'
+ timeout-minutes: 10
+ run: |
+ sudo apt-get install -y --no-install-recommends ffmpeg
+ ffmpeg -hide_banner -version | head -n 1
+ # what the cache keeps: the packages as they are now, none an update replaced
+ sudo apt-get autoclean -y > /dev/null
+ rm -f ~/apt-archives/*.deb
+ cp /var/cache/apt/archives/*.deb ~/apt-archives/
+ - if: runner.os == 'Linux' && steps.apt-cache.outputs.cache-hit != 'true'
+ uses: actions/cache/save@v4
+ with:
+ path: ~/apt-archives
+ key: ${{ steps.apt.outputs.key }}
- run: brew install ffmpeg
if: runner.os == 'macOS'
# Chocolatey reports success when its feed times out and nothing is installed: tried three times, then the step
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index a6e3b5d..fb88f3e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -43,8 +43,25 @@ jobs:
- run: npm install -g npm@^11.5.1
- run: pnpm install --frozen-lockfile
- run: pnpm typecheck
+ # Ubuntu's packages through apt as ci.yml gets them: from the cache its runs keep, a stalled mirror left after 30 s,
+ # a slow step ending the release there rather than hours on
+ - id: apt
+ run: |
+ printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nAPT::Keep-Downloaded-Packages "true";\n' | sudo tee /etc/apt/apt.conf.d/80-ci
+ echo "prefix=apt-$ImageOS-" >> "$GITHUB_OUTPUT"
+ echo "key=apt-$ImageOS-$ImageVersion" >> "$GITHUB_OUTPUT"
+ - uses: actions/cache/restore@v4
+ with:
+ path: ~/apt-archives
+ key: ${{ steps.apt.outputs.key }}
+ restore-keys: ${{ steps.apt.outputs.prefix }}
+ - run: |
+ mkdir -p ~/apt-archives
+ sudo cp ~/apt-archives/*.deb /var/cache/apt/archives/ 2>/dev/null || true
- run: pnpm --filter openfilm exec playwright-core install --with-deps chromium-headless-shell
- - run: sudo apt-get update && sudo apt-get install -y ffmpeg
+ timeout-minutes: 15
+ - run: sudo apt-get install -y --no-install-recommends ffmpeg
+ timeout-minutes: 10
- run: pnpm test
- name: What the package holds (the official product.json)
run: pnpm check:pack
diff --git a/packages/openfilm/src/encoders.mjs b/packages/openfilm/src/encoders.mjs
index fbd273f..6a43a2a 100644
--- a/packages/openfilm/src/encoders.mjs
+++ b/packages/openfilm/src/encoders.mjs
@@ -114,8 +114,9 @@ function works(encoder) {
const prores = encoder.startsWith('prores_') ? proresOptions(encoder, true) : null;
const options = prores ? ['-vf', prores.pix, ...prores.video] : ['-vf', 'format=yuv420p', ...hardwareOptions(encoder, 2e6)];
tried.set(encoder, new Promise((done) => {
+ /* one that hangs in the chip's driver never sees a SIGTERM, and its answer would never come: killed */
execFile('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-f', 'lavfi', '-i', 'testsrc2=size=640x360:rate=30', '-frames:v', '10',
- ...options, '-f', 'null', '-'], { timeout: 20_000, windowsHide: true }, (error) => done(!error));
+ ...options, '-f', 'null', '-'], { timeout: 20_000, killSignal: 'SIGKILL', windowsHide: true }, (error) => done(!error));
}));
}
return /** @type {Promise<boolean>} */ (tried.get(encoder));
diff --git a/packages/openfilm/src/seekable.mjs b/packages/openfilm/src/seekable.mjs
index 9a6d49d..c57254b 100644
--- a/packages/openfilm/src/seekable.mjs
+++ b/packages/openfilm/src/seekable.mjs
@@ -23,6 +23,9 @@ const MAX_SECONDS = 10 * 60;
const MAX_SIDE = 4096;
/** Changing how copies are made makes new ones. */
const VERSION = 2;
+/** A copy on the graphics chip that writes nothing for this long has hung (VideoToolbox does on some virtual Macs):
+ stopped, and made on the processor. */
+const STALL_MS = 15_000;
/** @param {string} file @returns {Promise<string>} */
const probe = (file, args) => new Promise((done, fail) => {
@@ -248,7 +251,11 @@ async function makeCopy(file, out, need, signal) {
when told them */
const bits = Math.min(120e6, Math.max(4e6, ((need.w * need.h) || 1920 * 1080) * need.fps * 0.3));
/* decoded and written on the graphics chip when it can (encoders.mjs), else on the processor */
+ let stalled = false;
const make = async (/** @type {string | null} */ encoder, /** @type {string[]} */ decoding) => {
+ const chip = Boolean(encoder) || decoding.length > 0;
+ /* a chip that hung is not tried again for this copy: straight to the processor */
+ if (chip && stalled) throw new Error('the graphics chip hung');
/* the same frames at the same times (passthrough: no frame is added or dropped), the picture only */
const run = ffmpeg([...decoding, '-i', file, '-map', '0:v:0', '-an', '-sn', '-dn', '-fps_mode', 'passthrough',
...(need.scaled ? ['-vf', `scale=${need.w}:${need.h}`] : []),
@@ -259,7 +266,25 @@ async function makeCopy(file, out, need, signal) {
/* its whole tree: an ffmpeg that is a shim (Windows' package managers) runs the real one as its child */
const stop = () => kill(run.proc);
signal?.addEventListener('abort', stop, { once: true });
- try { await run.done; } finally { signal?.removeEventListener('abort', stop); }
+ /* on the chip, the part must keep growing: one that does not for STALL_MS is stopped */
+ let watch;
+ const hung = new Promise((_, fail) => {
+ if (!chip) return;
+ let size = -1, since = Date.now();
+ watch = setInterval(() => {
+ let now = 0;
+ try { now = statSync(part).size; } catch { /* not there yet */ }
+ if (now !== size) { size = now; since = Date.now(); return; }
+ if (Date.now() - since < STALL_MS) return;
+ stalled = true;
+ clearInterval(watch);
+ stop();
+ /* given a moment to end, so the next try never writes the part with it */
+ const ended = run.done.catch(() => {});
+ Promise.race([ended, new Promise((r) => setTimeout(r, 5000).unref())]).then(() => fail(new Error(`the graphics chip wrote nothing for ${STALL_MS / 1000} s`)));
+ }, 1000);
+ });
+ try { await Promise.race([run.done, hung]); } finally { clearInterval(watch); signal?.removeEventListener('abort', stop); }
};
/* a part that cannot be removed (Windows, a moment after its encoder ends) stays in the cache folder: it never fails
the render that asked for the copy */
diff --git a/packages/openfilm/src/seekable.test.mjs b/packages/openfilm/src/seekable.test.mjs
index 5a8121d..6ece16c 100644
--- a/packages/openfilm/src/seekable.test.mjs
+++ b/packages/openfilm/src/seekable.test.mjs
@@ -1,7 +1,7 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { execFile, execFileSync } from 'node:child_process';
-import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { filmHtml } from './film-doc.mjs';
@@ -200,6 +200,44 @@ console.log(copies.get(${JSON.stringify(file)}) ?? '');`;
}
});
+test('a copy whose graphics chip hangs is stopped and made on the processor, and leaves no ffmpeg running', {
+ ...ffmpeg, skip: ffmpeg.skip || (process.platform === 'win32' && 'a shell script stands in for ffmpeg'),
+}, async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'of-seekable-hung-'));
+ try {
+ const file = join(dir, 'prores.mov');
+ execFileSync('ffmpeg', ['-v', 'error', '-y', '-f', 'lavfi', '-i', 'testsrc2=size=320x180:rate=30', '-t', '1', '-c:v', 'prores_ks', '-profile:v', '3', file]);
+ /* ffmpeg, with a chip encoder that passes its test and then hangs on the copy: it writes its pid and never ends */
+ const bin = join(dir, 'bin');
+ mkdirSync(bin);
+ const encoder = process.platform === 'darwin' ? 'h264_videotoolbox' : 'h264_nvenc';
+ const pidFile = join(dir, 'hung.pid');
+ writeFileSync(join(bin, 'ffmpeg'), `#!/bin/sh
+case " $* " in
+ *" -encoders "*) "$REAL_FFMPEG" "$@"; echo " V....D ${encoder} the chip's"; exit 0;;
+ *" ${encoder} "*" -f null - "*) exit 0;;
+ *" ${encoder} "*) echo $$ > '${pidFile}'; exec sleep 600;;
+esac
+exec "$REAL_FFMPEG" "$@"
+`);
+ chmodSync(join(bin, 'ffmpeg'), 0o755);
+ const real = execFileSync('sh', ['-c', 'command -v ffmpeg']).toString().trim();
+ const code = `const { seekableCopies } = await import(${JSON.stringify(new URL('./seekable.mjs', import.meta.url).href)});
+const copies = await seekableCopies([${JSON.stringify(file)}], ${JSON.stringify(dir)}, { plays: async () => false, slow: false });
+console.log(copies.get(${JSON.stringify(file)}) ?? '');`;
+ const env = { ...process.env, PATH: `${bin}:${process.env.PATH}`, REAL_FFMPEG: real };
+ const copy = await new Promise((done, fail) => execFile(process.execPath, ['--input-type=module', '-e', code], { env }, (e, out) => (e ? fail(e) : done(String(out).trim()))));
+ assert.ok(existsSync(pidFile), 'the chip was tried');
+ assert.ok(copy, 'made all the same');
+ assert.equal(times(copy).length, 30, 'whole');
+ assert.equal(readdirSync(join(dir, '.film', 'cache', 'seek')).length, 1, 'no part left');
+ const pid = Number(readFileSync(pidFile, 'utf8'));
+ assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }, 'the hung ffmpeg was stopped');
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
+
test('look shows ProRes footage, which no browser decodes, through its copy', ffmpeg, async () => {
const dir = mkdtempSync(join(tmpdir(), 'of-seekable-prores-look-'));
try {