Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Survive a stalled render browser and stop leaked test Chromes

Claudeaa2c0a0f7eba
diff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.js
index c48ad0bd6d..cdbcc48849 100644
--- a/scripts/lib/vitestStaleRunRoots.js
+++ b/scripts/lib/vitestStaleRunRoots.js
@@ -11,7 +11,8 @@
*/
import { execFileSync } from 'node:child_process';
import { lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
+import { isAbsolute, join, relative, sep } from 'node:path';
+import { killWithEscalation } from '../../server/lib/killWithEscalation.js';
export const OWNER_FILE = '.owner.pid';
export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;
@@ -68,11 +69,104 @@ function readOwner(path) {
}
}
-export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive } = {}) {
+/**
+ * `[{ pid, profile }]` for every running process started with a
+ * `--user-data-dir` (a test Chrome and its helpers). Empty on win32 or when
+ * `ps` is unavailable: the sweep is best-effort and never fails a run.
+ */
+export function listProfileProcesses() {
+ if (process.platform === 'win32') return [];
+ let out;
+ try {
+ out = execFileSync('ps', ['-A', '-ww', '-o', 'pid=,args='], {
+ encoding: 'utf8', windowsHide: true, maxBuffer: 64 * 1024 * 1024,
+ stdio: ['ignore', 'pipe', 'ignore'], env: { ...process.env, LC_ALL: 'C' },
+ });
+ } catch {
+ return [];
+ }
+ const found = [];
+ for (const line of out.split('\n')) {
+ const match = /^\s*(\d+)\s+(.*)$/.exec(line);
+ const profile = match && /(?:^|\s)--user-data-dir=(\S+)/.exec(match[2])?.[1];
+ if (profile && Number(match[1]) !== process.pid) found.push({ pid: Number(match[1]), profile });
+ }
+ return found;
+}
+
+const isPidAlive = pid => {
+ try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; }
+};
+
+// SIGTERM now, SIGKILL after a grace window if the pid is still there.
+export function terminatePid(pid) {
+ killWithEscalation({ exitCode: null, signalCode: null, kill: signal => { try { return process.kill(pid, signal); } catch { return false; } } },
+ { label: 'Orphaned test browser', stillRunning: () => isPidAlive(pid), delayMs: 2000 });
+}
+
+// First path segment of `path` below `root`, or null when it is not inside.
+function segmentUnder(root, path) {
+ const rel = relative(root, path);
+ if (!rel || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;
+ return rel.split(sep)[0];
+}
+
+/**
+ * Kill every process whose browser profile lives under `root` (a run's own
+ * temp root, at run exit). Synchronous SIGKILL by default, so it is safe in
+ * an `exit` handler. Returns the number of processes signalled.
+ */
+export function killTestBrowsersUnder(root, { listProcesses = listProfileProcesses, kill = pid => process.kill(pid, 'SIGKILL') } = {}) {
+ let killed = 0;
+ for (const { pid, profile } of listProcesses()) {
+ if (segmentUnder(root, profile) === null) continue;
+ try { kill(pid); killed++; } catch { /* already gone */ }
+ }
+ return killed;
+}
+
+/**
+ * A test Chrome outlives an interrupted run (#10840): its profile sits in a
+ * run root whose owner is gone (or was removed with it), and the browser keeps
+ * burning a core. Kill every process whose `--user-data-dir` is under such a
+ * `<prefix>` root in `tmpRoot`. A live owner keeps its browsers; an unknown
+ * owner keeps them until the root is old enough to sweep.
+ */
+export function sweepOrphanedTestBrowsers(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive, listProcesses = listProfileProcesses, terminate = terminatePid } = {}) {
+ const verdicts = new Map();
+ const orphaned = name => {
+ if (verdicts.has(name)) return verdicts.get(name);
+ const path = join(tmpRoot, name);
+ let verdict;
+ try {
+ const stat = lstatSync(path);
+ const owner = readOwner(path);
+ const alive = owner ? probe(owner) : null;
+ verdict = alive === false || (alive === null && now - stat.mtimeMs > STALE_ROOT_AGE_MS);
+ } catch (err) {
+ // The root was removed while its browser kept running.
+ verdict = err?.code === 'ENOENT';
+ }
+ verdicts.set(name, verdict);
+ return verdict;
+ };
+ let killed = 0;
+ for (const { pid, profile } of listProcesses()) {
+ const name = segmentUnder(tmpRoot, profile);
+ if (!name?.startsWith(prefix) || !orphaned(name)) continue;
+ try { terminate(pid); killed++; } catch { /* already gone */ }
+ }
+ if (killed) console.warn(`🧹 Stopped ${killed} test browser process${killed === 1 ? '' : 'es'} left running by an interrupted test run`);
+ return killed;
+}
+
+export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive, listProcesses = listProfileProcesses, terminate = terminatePid } = {}) {
// Require an exact directory-name prefix, never a path or an empty match.
if (typeof prefix !== 'string' || !/^[a-zA-Z0-9-]+-$/.test(prefix)) {
throw new Error('Owned temp prefix must be a nonempty name ending in a hyphen');
}
+ // Browsers first: a root swept below no longer names its owner.
+ sweepOrphanedTestBrowsers(tmpRoot, { prefix, now, probe, listProcesses, terminate });
let names;
try {
names = readdirSync(tmpRoot);
diff --git a/scripts/lib/vitestTempRoot.js b/scripts/lib/vitestTempRoot.js
index 1aa4eb96b6..6fc016452a 100644
--- a/scripts/lib/vitestTempRoot.js
+++ b/scripts/lib/vitestTempRoot.js
@@ -1,7 +1,7 @@
import { existsSync, mkdirSync, mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { basename, join } from 'node:path';
-import { OWNER_FILE, sweepStaleRunRoots, writeOwnerFile } from './vitestStaleRunRoots.js';
+import { OWNER_FILE, killTestBrowsersUnder, sweepStaleRunRoots, writeOwnerFile } from './vitestStaleRunRoots.js';
// Vitest sets these in every pool worker; the enclosing main process owns the root.
const inVitestWorker = () => Boolean(process.env.VITEST_POOL_ID || process.env.VITEST_WORKER_ID);
@@ -22,6 +22,26 @@ function restoreInheritedRootOwnership(root) {
if (!existsSync(join(root, OWNER_FILE))) writeOwnerFile(root);
}
+/**
+ * The owner of a run root stops every test browser still running under it
+ * when it exits or is interrupted (#10840), so an aborted run cannot strand a
+ * Chrome whose worker died. A signal is re-raised once the sweep runs unless
+ * another listener (Vitest's own) is there to handle it.
+ */
+function killRootBrowsersOnExit(root) {
+ if (process.platform === 'win32') return;
+ const sweep = () => {
+ try { killTestBrowsersUnder(root); } catch { /* best effort at exit */ }
+ };
+ process.once('exit', sweep);
+ for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
+ process.once(signal, () => {
+ sweep();
+ if (process.listenerCount(signal) === 0) process.kill(process.pid, signal);
+ });
+ }
+}
+
/** Establish containment at config-load time, before Vitest creates its cache or workers. */
export function bootstrapVitestTempRoot() {
const hostRoot = tmpdir();
@@ -32,6 +52,7 @@ export function bootstrapVitestTempRoot() {
root = mkdtempSync(join(hostRoot, 'pvt-'));
writeOwnerFile(root);
process.env.PORTOS_TEST_TEMP_ROOT = root;
+ killRootBrowsersOnExit(root);
} else {
restoreInheritedRootOwnership(root);
}
diff --git a/scripts/vitestTempRoot.test.js b/scripts/vitestTempRoot.test.js
index 8f19805477..768f2c0e77 100644
--- a/scripts/vitestTempRoot.test.js
+++ b/scripts/vitestTempRoot.test.js
@@ -4,7 +4,8 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, utimesSync, wr
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createVitestTempFixture, runWatchConfigRestart } from './lib/vitestTempRootFixture.js';
-import { STALE_ROOT_AGE_MS, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';
+import { STALE_ROOT_AGE_MS, killTestBrowsersUnder, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';
+import { isProcessAlive } from '../server/test/processAlive.js';
describe('real workspace runner temp lifecycle', () => {
const workspace = 'server';
@@ -94,3 +95,66 @@ describe('real workspace runner temp lifecycle', () => {
}
}, 90000);
});
+
+// #10840: a capture browser whose test process died without cleanup.
+describe.skipIf(process.platform === 'win32')('orphaned test browser sweep', () => {
+ // Stand-in for a test Chrome: a detached process carrying a test profile flag,
+ // started by a parent that exits without cleanup. Returns both pids.
+ const orphanBrowser = profile => {
+ const parent = spawnSync(process.execPath, ['-e', `
+ const { spawn } = require('node:child_process');
+ const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)', '--', '--headless=new', '--user-data-dir=' + ${JSON.stringify(profile)}], { detached: true, stdio: 'ignore' });
+ child.unref();
+ process.stdout.write(String(child.pid));`], { encoding: 'utf8' });
+ expect(parent.status, parent.stderr).toBe(0);
+ return { parentPid: parent.pid, pid: Number(parent.stdout) };
+ };
+ const gone = async pid => {
+ for (let i = 0; i < 100 && isProcessAlive(pid); i++) await new Promise(resolve => setTimeout(resolve, 50));
+ return !isProcessAlive(pid);
+ };
+
+ it('stops a browser left by a dead run and keeps one a live run still owns', async () => {
+ const host = mkdtempSync(join(tmpdir(), 'vrt-'));
+ const pids = [];
+ try {
+ const dead = join(host, 'pvt-dead');
+ mkdirSync(join(dead, 'profile'), { recursive: true });
+ const orphan = orphanBrowser(join(dead, 'profile'));
+ pids.push(orphan.pid);
+ // The run that owned the root is the parent that already exited.
+ writeFileSync(join(dead, '.owner.pid'), `${orphan.parentPid} 0\n`);
+ const live = join(host, 'pvt-live');
+ mkdirSync(join(live, 'profile'), { recursive: true });
+ writeOwnerFile(live);
+ const owned = orphanBrowser(join(live, 'profile'));
+ pids.push(owned.pid);
+ expect(isProcessAlive(orphan.pid)).toBe(true);
+
+ sweepStaleRunRoots(host);
+ expect(await gone(orphan.pid)).toBe(true);
+ expect(existsSync(dead)).toBe(false);
+ expect(isProcessAlive(owned.pid)).toBe(true);
+
+ // The owning run stops its own browsers as it exits.
+ expect(killTestBrowsersUnder(live)).toBe(1);
+ expect(await gone(owned.pid)).toBe(true);
+ } finally {
+ for (const pid of pids) { try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } }
+ rmSync(host, { recursive: true, force: true });
+ }
+ }, 20000);
+
+ it('stops a browser whose run root was already removed', async () => {
+ const host = mkdtempSync(join(tmpdir(), 'vrt-'));
+ let pid;
+ try {
+ ({ pid } = orphanBrowser(join(host, 'pvt-removed', 'profile')));
+ sweepStaleRunRoots(host);
+ expect(await gone(pid)).toBe(true);
+ } finally {
+ try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ }
+ rmSync(host, { recursive: true, force: true });
+ }
+ }, 20000);
+});
diff --git a/scripts/vitestTempRootSetup.js b/scripts/vitestTempRootSetup.js
index 09dd10b81e..75a0f7bc34 100644
--- a/scripts/vitestTempRootSetup.js
+++ b/scripts/vitestTempRootSetup.js
@@ -5,7 +5,7 @@
*/
import { readFileSync, readdirSync, rmSync, statSync } from 'node:fs';
import { join } from 'node:path';
-import { OWNER_FILE } from './lib/vitestStaleRunRoots.js';
+import { OWNER_FILE, killTestBrowsersUnder } from './lib/vitestStaleRunRoots.js';
export function setup() {}
@@ -78,6 +78,9 @@ export function isEffectivelyEmpty(path) {
export function teardown() {
const root = process.env.PORTOS_TEST_TEMP_ROOT;
if (!root) return;
+ // A browser a suite failed to tear down would outlive the run (#10840).
+ const browsers = process.platform === 'win32' ? 0 : killTestBrowsersUnder(root);
+ if (browsers) console.warn(`⚠️ test browser leak: stopped ${browsers} process${browsers === 1 ? '' : 'es'} still running under the run temp root`);
let entries;
try {
entries = readdirSync(root);
diff --git a/server/services/htmlComposition/encode.js b/server/services/htmlComposition/encode.js
index 2a9c67bba4..a5da1f23f2 100644
--- a/server/services/htmlComposition/encode.js
+++ b/server/services/htmlComposition/encode.js
@@ -43,7 +43,18 @@ const SCREENSHOT = Object.freeze({ format: 'png', optimizeForSpeed: true, fromSu
// that a later window continues (parallel segments): the auto-shutter
// comparison at each seam sees the real neighbouring frame instead of treating
// the window edge as an edge of the song.
-export async function encodeComposition(page, contract, outputPath, { musicPath, audio, signal, onProgress, offsetSec = 0, continuous = false, followed = false, videoFilter = null, master = false, runFfmpeg = runFfmpegProcess, spawnProcess = spawn, locateFfmpeg = findFfmpeg, tagFilter = bt709TagFilter } = {}) {
+// A stalled or lost browser (a CDP command past its deadline on an overloaded
+// host, a crashed target) is not the document's fault, and seeks are
+// deterministic, so a fresh browser can redraw the same frame (#10839).
+// Anything else (a script error, a refused asset) fails as before.
+const RECOVERABLE_PAGE_FAILURE = /Browser command timed out|Composition browser target crashed|Composition browser target closed|Managed browser disconnected/;
+const MAX_FRAME_RETRIES = 2;
+
+// `reopenPage(failedPage)` resolves a fresh page when a capture hits a
+// recoverable browser failure; the frame is redrawn there and the encoder
+// keeps running, so frames already piped are kept. `onRetry({ frame, attempt,
+// error })` reports each restart, `frame` counted on song time.
+export async function encodeComposition(page, contract, outputPath, { musicPath, audio, signal, onProgress, offsetSec = 0, continuous = false, followed = false, videoFilter = null, master = false, reopenPage = null, onRetry = null, runFfmpeg = runFfmpegProcess, spawnProcess = spawn, locateFfmpeg = findFfmpeg, tagFilter = bt709TagFilter } = {}) {
const ffmpeg = await locateFfmpeg();
if (!ffmpeg) throw new Error('ffmpeg not found on PATH');
const tag = await tagFilter();
@@ -118,7 +129,7 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,
const abort = () => stop('abort requested');
signal?.addEventListener('abort', abort, { once: true });
if (!Number.isFinite(offsetSec) || offsetSec < 0) throw new Error('offsetSec must be a non-negative number');
- const capture = async t => {
+ const captureOnce = async t => {
page.check();
// awaitPromise in evaluate is essential: each seek owns its paint.
const at = offsetSec ? Math.round((offsetSec + t) * 1e6) / 1e6 : t;
@@ -131,6 +142,25 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,
page.check();
return Buffer.from(data, 'base64');
};
+ const capture = async t => {
+ let stalled = false;
+ for (let attempt = 1; ; attempt++) {
+ try {
+ if (stalled) {
+ page = await reopenPage(page);
+ stalled = false;
+ await frameFormat(page, contract);
+ }
+ return await captureOnce(t);
+ } catch (error) {
+ if (!reopenPage || signal?.aborted || !RECOVERABLE_PAGE_FAILURE.test(error?.message ?? '')) throw error;
+ const frame = Math.round((offsetSec + t) * fps);
+ if (attempt > MAX_FRAME_RETRIES) throw new Error(`${error.message} (song frame ${frame} still failing after ${MAX_FRAME_RETRIES} browser restarts)`, { cause: error });
+ onRetry?.({ frame, attempt, error });
+ stalled = true;
+ }
+ }
+ };
const write = bytes => Promise.race([
new Promise((resolve, reject) => proc.stdin.write(bytes, error => error ? reject(error) : resolve())),
encoderStopped,
@@ -218,12 +248,36 @@ function planSegments(numFrames, workers) {
// `page` draws the first segment; `openPage(signal)` opens each further one
// and the segment closes it. `videoFilterAt(offsetSec)` builds a segment's
// filter on song time. One segment is exactly `encodeComposition`.
-export async function encodeCompositionSegments(page, contract, outputPath, { openPage, workers = compositionRenderWorkers(), offsetSec = 0, videoFilterAt = () => null, signal, onProgress, encode = encodeComposition, runFfmpeg = runFfmpegProcess, locateFfmpeg = findFfmpeg } = {}) {
+// A worker whose browser stalls closes it and continues on a fresh one from
+// the frame it was on (#10839). When that replaces `page` itself, `page` is
+// closed and the result carries `pageReplaced: true`.
+export async function encodeCompositionSegments(page, contract, outputPath, { openPage, workers = compositionRenderWorkers(), offsetSec = 0, videoFilterAt = () => null, signal, onProgress, onRetry = null, encode = encodeComposition, runFfmpeg = runFfmpegProcess, locateFfmpeg = findFfmpeg } = {}) {
const { fps, durationSec } = contract;
const numFrames = Math.round(durationSec * fps);
const segments = planSegments(numFrames, workers);
+ let pageReplaced = false;
+ // `slot.page` is the segment's current page; `reopen` closes the stalled one
+ // first so the two browsers never compete for the same overloaded host.
+ const replaceable = (first, pageSignal) => {
+ const slot = { page: first, reopen: null };
+ if (openPage) slot.reopen = async failed => {
+ await failed.close().catch(() => {});
+ if (failed === page) pageReplaced = true;
+ slot.page = null;
+ slot.page = await openPage(pageSignal);
+ return slot.page;
+ };
+ return slot;
+ };
+ const released = slot => slot.page && slot.page !== page ? slot.page.close().catch(() => {}) : null;
if (segments.length === 1) {
- return encode(page, contract, outputPath, { signal, onProgress, offsetSec, videoFilter: videoFilterAt(offsetSec) });
+ const slot = replaceable(page, signal);
+ try {
+ const result = await encode(page, contract, outputPath, { signal, onProgress, offsetSec, videoFilter: videoFilterAt(offsetSec), reopenPage: slot.reopen, onRetry });
+ return pageReplaced ? { ...result, pageReplaced } : result;
+ } finally {
+ await released(slot);
+ }
}
const ffmpeg = await locateFfmpeg();
if (!ffmpeg) throw new Error('ffmpeg not found on PATH');
@@ -238,12 +292,13 @@ export async function encodeCompositionSegments(page, contract, outputPath, { op
try {
signal?.throwIfAborted();
const results = await Promise.allSettled(segments.map(async ({ start, frames }, i) => {
- let segmentPage = i === 0 ? page : null;
+ const slot = replaceable(i === 0 ? page : null, controller.signal);
try {
- segmentPage ??= await openPage(controller.signal);
+ slot.page ??= await openPage(controller.signal);
const at = Math.round((offsetSec + start / fps) * 1e6) / 1e6;
- const result = await encode(segmentPage, { ...contract, durationSec: frames / fps }, parts[i], {
+ const result = await encode(slot.page, { ...contract, durationSec: frames / fps }, parts[i], {
signal: controller.signal, offsetSec: at, continuous: start > 0, followed: i < segments.length - 1, videoFilter: videoFilterAt(at),
+ reopenPage: slot.reopen, onRetry: onRetry && (detail => onRetry({ ...detail, segment: i })),
onProgress: (_fraction, detail) => {
done[i] = detail?.frame ?? 0;
const frame = done.reduce((sum, value) => sum + value, 0);
@@ -255,7 +310,7 @@ export async function encodeCompositionSegments(page, contract, outputPath, { op
controller.abort(error);
throw error;
} finally {
- if (i > 0 && segmentPage) await segmentPage.close().catch(() => {});
+ await released(slot);
}
}));
signal?.throwIfAborted();
@@ -267,7 +322,7 @@ export async function encodeCompositionSegments(page, contract, outputPath, { op
const joined = await runFfmpeg({ bin: ffmpeg, signal, args: ['-hide_banner', '-loglevel', 'error', '-f', 'concat', '-safe', '0', '-i', listPath,
'-c', 'copy', '-movflags', '+faststart', '-y', outputPath] });
if (!joined.ok) throw new Error(`Joining render segments failed: ${joined.reason}`);
- return Object.keys(histogram).length ? { sampleHistogram: histogram } : {};
+ return { ...(Object.keys(histogram).length ? { sampleHistogram: histogram } : {}), ...(pageReplaced ? { pageReplaced } : {}) };
} finally {
signal?.removeEventListener('abort', relay);
for (const path of [...parts, listPath]) await rm(path, { force: true }).catch(() => {});
diff --git a/server/services/htmlComposition/encode.segments.test.js b/server/services/htmlComposition/encode.segments.test.js
index 367df272fb..2219fa772b 100644
--- a/server/services/htmlComposition/encode.segments.test.js
+++ b/server/services/htmlComposition/encode.segments.test.js
@@ -112,6 +112,97 @@ describe('encodeComposition at segment seams', () => {
});
});
+// #10839: one stalled screenshot restarts that worker's browser, which redraws
+// the frame it was on; a frame that keeps stalling still fails the render.
+describe('encodeCompositionSegments when a worker browser stalls', () => {
+ const stalled = () => new Error('Browser command timed out: Page.captureScreenshot');
+ // `stallsAt(frame, captureCount)` decides whether this page's capture stalls.
+ const capturePage = (name, stallsAt = () => false) => {
+ const state = { name, closed: 0, captures: 0, failure: null, seekAt: null };
+ return Object.assign(state, {
+ check() { if (state.failure) throw state.failure; },
+ async evaluate(expression) { const match = /seek\(([^)]+)\)/.exec(expression); if (match) state.seekAt = Number(match[1]); },
+ async send(method) {
+ if (state.failure) throw state.failure;
+ if (method !== 'Page.captureScreenshot') return {};
+ state.captures += 1;
+ // A CDP deadline poisons the whole page connection, as in browser.js.
+ if (stallsAt(Math.round(state.seekAt * 10), state.captures)) throw (state.failure = stalled());
+ return { data: PNG };
+ },
+ async close() { state.closed += 1; },
+ });
+ };
+ const run = (first, openPage, options = {}) => {
+ const written = [];
+ const encode = (p, c, path, encodeOptions) => encodeComposition(p, c, path, {
+ ...encodeOptions, locateFfmpeg: async () => 'ffmpeg', tagFilter: async () => 'format=yuv420p',
+ spawnProcess: () => {
+ const proc = new EventEmitter();
+ proc.stderr = new EventEmitter();
+ proc.stdin = new EventEmitter();
+ proc.stdin.write = (_bytes, callback) => { written.push(path); callback?.(); return true; };
+ proc.stdin.end = () => proc.emit('close', 0);
+ // A stop (another worker failed) ends the stub encoder as ffmpeg would.
+ proc.kill = signal => { setImmediate(() => proc.emit('close', null, signal)); return true; };
+ return proc;
+ },
+ });
+ const result = encodeCompositionSegments(first, { fps: 10, durationSec: 48, width: 4, height: 4, motionBlur: 1 }, '/tmp/stall.mp4', {
+ workers: 2, openPage, encode, locateFfmpeg: async () => 'ffmpeg', runFfmpeg: async () => ({ ok: true }), ...options,
+ });
+ return { result, written };
+ };
+
+ it('restarts the stalled worker and still writes every frame', async () => {
+ const opened = [];
+ const retries = [];
+ // The second worker's first browser stalls once, on song frame 300.
+ const openPage = async () => {
+ const p = capturePage(`extra${opened.length}`, opened.length === 0 ? frame => frame === 300 : undefined);
+ opened.push(p);
+ return p;
+ };
+ const first = capturePage('first');
+ const { result, written } = run(first, openPage, { onRetry: detail => retries.push(detail) });
+ expect(await result).toEqual({});
+ expect(written.filter(path => path.endsWith('.part0.mp4'))).toHaveLength(240);
+ expect(written.filter(path => path.endsWith('.part1.mp4'))).toHaveLength(240);
+ expect(retries.map(({ frame, attempt, segment }) => ({ frame, attempt, segment }))).toEqual([{ frame: 300, attempt: 1, segment: 1 }]);
+ // The stalled browser closed before its replacement opened and redrew frame 300.
+ expect(opened).toHaveLength(2);
+ expect(opened.map(p => p.closed)).toEqual([1, 1]);
+ expect(opened[1].captures).toBe(180);
+ expect(first.closed).toBe(0);
+ });
+
+ it('replaces the caller\'s page when the first worker stalls and says so', async () => {
+ const first = capturePage('first', frame => frame === 10);
+ const { result } = run(first, async () => capturePage('extra'));
+ expect(await result).toEqual({ pageReplaced: true });
+ expect(first.closed).toBe(1);
+ });
+
+ it('fails the render, naming the frame, when the same frame keeps stalling', async () => {
+ const opened = [];
+ const openPage = async () => { const p = capturePage('extra', frame => frame === 300); opened.push(p); return p; };
+ const { result } = run(capturePage('first'), openPage);
+ await expect(result).rejects.toThrow(/song frame 300 still failing after 2 browser restarts/);
+ // The first worker's page, plus the second worker's original and two restarts.
+ expect(opened).toHaveLength(3);
+ expect(opened.every(p => p.closed === 1)).toBe(true);
+ });
+
+ it('does not restart for a composition script failure', async () => {
+ const opened = [];
+ const broken = capturePage('first');
+ broken.evaluate = async () => { throw new Error('Composition script failed: boom'); };
+ const { result } = run(broken, async () => { const p = capturePage('extra'); opened.push(p); return p; });
+ await expect(result).rejects.toThrow(/Composition script failed: boom/);
+ expect(opened).toHaveLength(1);
+ });
+});
+
const ffmpeg = await findFfmpeg();
describe.skipIf(!ffmpeg)('encodeCompositionSegments with real ffmpeg', () => {
it('joins encoded segments into one stream with every frame and no leftovers', async () => {
diff --git a/server/services/htmlComposition/testBrowserCleanup.js b/server/services/htmlComposition/testBrowserCleanup.js
index f5f9b89760..f8033928ff 100644
--- a/server/services/htmlComposition/testBrowserCleanup.js
+++ b/server/services/htmlComposition/testBrowserCleanup.js
@@ -230,6 +230,25 @@ export async function _describeTestChromeStartup(proc, { source = 'unknown', exe
+ ` profile=${profileState} devToolsActivePort=${port}`;
}
+// A test process that exits without its suite's cleanup (a crash, a worker
+// torn down mid-test) takes its browser with it instead of orphaning it (#10840).
+// One exit hook for the module; the browsers themselves get no extra listeners.
+const startedBrowsers = new Set();
+let exitHooked = false;
+const settled = child => child.exitCode !== null || child.signalCode !== null;
+function killWithTestProcess(proc) {
+ if (!exitHooked) {
+ exitHooked = true;
+ process.once('exit', () => {
+ for (const child of startedBrowsers) {
+ if (!settled(child)) try { child.kill('SIGKILL'); } catch { /* already gone */ }
+ }
+ });
+ }
+ for (const child of startedBrowsers) if (settled(child)) startedBrowsers.delete(child);
+ startedBrowsers.add(proc);
+}
+
// Chrome writes its CDP address to stderr. Keep only a bounded tail and report
// known failure categories: raw stderr can contain the user's profile path.
export function _waitForTestChrome(proc, timeoutMs = 20000, startup, { observeProcess = _testChromeProcessFacts, spawnVersion = spawn } = {}) {
@@ -272,6 +291,7 @@ export function _waitForTestChrome(proc, timeoutMs = 20000, startup, { observePr
diagnostic().then(rejectWithFacts, () => rejectWithFacts('; diagnostic unavailable'));
} else {
proc.removeListener('error', onError);
+ killWithTestProcess(proc);
resolve(value);
}
};
diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.js
index 70735007ab..ade4b19587 100644
--- a/server/services/musicVideo/documentRender.js
+++ b/server/services/musicVideo/documentRender.js
@@ -399,13 +399,18 @@ export async function encodeDocumentComposition({
}
}
// Long windows split across several browsers (encodeCompositionSegments).
- await encodeCompositionSegments(page, { ...target, durationSec: window.durationSec }, silent, {
+ // A stalled worker browser restarts and redraws its frame (#10839).
+ const encoded = await encodeCompositionSegments(page, { ...target, durationSec: window.durationSec }, silent, {
openPage, encode: encodeComposition,
videoFilterAt: (offsetSec) => musicVideoGradeFilter(project.composition?.grade, data.scenes, { fps: target.fps, offsetSec }),
signal, offsetSec: window.startSec, onProgress: (fraction) => onProgress?.(fraction * 0.95),
+ onRetry: ({ frame, attempt, segment, error }) => console.warn(`⚠️ Music-video composition render [${String(jobId).slice(0, 8)}] ${segment == null ? '' : `worker ${segment + 1} `}restarting its browser at song frame ${frame} (retry ${attempt}): ${error.message}`),
});
- page.check();
- await page.close({ verify: true });
+ // A replaced first page was already closed after its failure.
+ if (!encoded?.pageReplaced) {
+ page.check();
+ await page.close({ verify: true });
+ }
page = null;
signal?.throwIfAborted();
const mux = await runFfmpegProcess({