Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Survive a stalled render browser and stop leaked test Chromes
Claudeaa2c0a0f7eba
diff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.jsindex c48ad0bd6d..cdbcc48849 100644--- a/scripts/lib/vitestStaleRunRoots.js+++ b/scripts/lib/vitestStaleRunRoots.js@@ -11,7 +11,8 @@*/import { execFileSync } from 'node:child_process';import { lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';-import { join } from 'node:path';+import { isAbsolute, join, relative, sep } from 'node:path';+import { killWithEscalation } from '../../server/lib/killWithEscalation.js';export const OWNER_FILE = '.owner.pid';export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;@@ -68,11 +69,104 @@ function readOwner(path) {}}-export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive } = {}) {+/**+ * `[{ pid, profile }]` for every running process started with a+ * `--user-data-dir` (a test Chrome and its helpers). Empty on win32 or when+ * `ps` is unavailable: the sweep is best-effort and never fails a run.+ */+export function listProfileProcesses() {+ if (process.platform === 'win32') return [];+ let out;+ try {+ out = execFileSync('ps', ['-A', '-ww', '-o', 'pid=,args='], {+ encoding: 'utf8', windowsHide: true, maxBuffer: 64 * 1024 * 1024,+ stdio: ['ignore', 'pipe', 'ignore'], env: { ...process.env, LC_ALL: 'C' },+ });+ } catch {+ return [];+ }+ const found = [];+ for (const line of out.split('\n')) {+ const match = /^\s*(\d+)\s+(.*)$/.exec(line);+ const profile = match && /(?:^|\s)--user-data-dir=(\S+)/.exec(match[2])?.[1];+ if (profile && Number(match[1]) !== process.pid) found.push({ pid: Number(match[1]), profile });+ }+ return found;+}++const isPidAlive = pid => {+ try { process.kill(pid, 0); return true; } catch (err) { return err.code === 'EPERM'; }+};++// SIGTERM now, SIGKILL after a grace window if the pid is still there.+export function terminatePid(pid) {+ killWithEscalation({ exitCode: null, signalCode: null, kill: signal => { try { return process.kill(pid, signal); } catch { return false; } } },+ { label: 'Orphaned test browser', stillRunning: () => isPidAlive(pid), delayMs: 2000 });+}++// First path segment of `path` below `root`, or null when it is not inside.+function segmentUnder(root, path) {+ const rel = relative(root, path);+ if (!rel || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;+ return rel.split(sep)[0];+}++/**+ * Kill every process whose browser profile lives under `root` (a run's own+ * temp root, at run exit). Synchronous SIGKILL by default, so it is safe in+ * an `exit` handler. Returns the number of processes signalled.+ */+export function killTestBrowsersUnder(root, { listProcesses = listProfileProcesses, kill = pid => process.kill(pid, 'SIGKILL') } = {}) {+ let killed = 0;+ for (const { pid, profile } of listProcesses()) {+ if (segmentUnder(root, profile) === null) continue;+ try { kill(pid); killed++; } catch { /* already gone */ }+ }+ return killed;+}++/**+ * A test Chrome outlives an interrupted run (#10840): its profile sits in a+ * run root whose owner is gone (or was removed with it), and the browser keeps+ * burning a core. Kill every process whose `--user-data-dir` is under such a+ * `<prefix>` root in `tmpRoot`. A live owner keeps its browsers; an unknown+ * owner keeps them until the root is old enough to sweep.+ */+export function sweepOrphanedTestBrowsers(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive, listProcesses = listProfileProcesses, terminate = terminatePid } = {}) {+ const verdicts = new Map();+ const orphaned = name => {+ if (verdicts.has(name)) return verdicts.get(name);+ const path = join(tmpRoot, name);+ let verdict;+ try {+ const stat = lstatSync(path);+ const owner = readOwner(path);+ const alive = owner ? probe(owner) : null;+ verdict = alive === false || (alive === null && now - stat.mtimeMs > STALE_ROOT_AGE_MS);+ } catch (err) {+ // The root was removed while its browser kept running.+ verdict = err?.code === 'ENOENT';+ }+ verdicts.set(name, verdict);+ return verdict;+ };+ let killed = 0;+ for (const { pid, profile } of listProcesses()) {+ const name = segmentUnder(tmpRoot, profile);+ if (!name?.startsWith(prefix) || !orphaned(name)) continue;+ try { terminate(pid); killed++; } catch { /* already gone */ }+ }+ if (killed) console.warn(`🧹 Stopped ${killed} test browser process${killed === 1 ? '' : 'es'} left running by an interrupted test run`);+ return killed;+}++export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive, listProcesses = listProfileProcesses, terminate = terminatePid } = {}) {// Require an exact directory-name prefix, never a path or an empty match.if (typeof prefix !== 'string' || !/^[a-zA-Z0-9-]+-$/.test(prefix)) {throw new Error('Owned temp prefix must be a nonempty name ending in a hyphen');}+ // Browsers first: a root swept below no longer names its owner.+ sweepOrphanedTestBrowsers(tmpRoot, { prefix, now, probe, listProcesses, terminate });let names;try {names = readdirSync(tmpRoot);diff --git a/scripts/lib/vitestTempRoot.js b/scripts/lib/vitestTempRoot.jsindex 1aa4eb96b6..6fc016452a 100644--- a/scripts/lib/vitestTempRoot.js+++ b/scripts/lib/vitestTempRoot.js@@ -1,7 +1,7 @@import { existsSync, mkdirSync, mkdtempSync } from 'node:fs';import { tmpdir } from 'node:os';import { basename, join } from 'node:path';-import { OWNER_FILE, sweepStaleRunRoots, writeOwnerFile } from './vitestStaleRunRoots.js';+import { OWNER_FILE, killTestBrowsersUnder, sweepStaleRunRoots, writeOwnerFile } from './vitestStaleRunRoots.js';// Vitest sets these in every pool worker; the enclosing main process owns the root.const inVitestWorker = () => Boolean(process.env.VITEST_POOL_ID || process.env.VITEST_WORKER_ID);@@ -22,6 +22,26 @@ function restoreInheritedRootOwnership(root) {if (!existsSync(join(root, OWNER_FILE))) writeOwnerFile(root);}+/**+ * The owner of a run root stops every test browser still running under it+ * when it exits or is interrupted (#10840), so an aborted run cannot strand a+ * Chrome whose worker died. A signal is re-raised once the sweep runs unless+ * another listener (Vitest's own) is there to handle it.+ */+function killRootBrowsersOnExit(root) {+ if (process.platform === 'win32') return;+ const sweep = () => {+ try { killTestBrowsersUnder(root); } catch { /* best effort at exit */ }+ };+ process.once('exit', sweep);+ for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) {+ process.once(signal, () => {+ sweep();+ if (process.listenerCount(signal) === 0) process.kill(process.pid, signal);+ });+ }+}+/** Establish containment at config-load time, before Vitest creates its cache or workers. */export function bootstrapVitestTempRoot() {const hostRoot = tmpdir();@@ -32,6 +52,7 @@ export function bootstrapVitestTempRoot() {root = mkdtempSync(join(hostRoot, 'pvt-'));writeOwnerFile(root);process.env.PORTOS_TEST_TEMP_ROOT = root;+ killRootBrowsersOnExit(root);} else {restoreInheritedRootOwnership(root);}diff --git a/scripts/vitestTempRoot.test.js b/scripts/vitestTempRoot.test.jsindex 8f19805477..768f2c0e77 100644--- a/scripts/vitestTempRoot.test.js+++ b/scripts/vitestTempRoot.test.js@@ -4,7 +4,8 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, utimesSync, wrimport { tmpdir } from 'node:os';import { join } from 'node:path';import { createVitestTempFixture, runWatchConfigRestart } from './lib/vitestTempRootFixture.js';-import { STALE_ROOT_AGE_MS, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';+import { STALE_ROOT_AGE_MS, killTestBrowsersUnder, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';+import { isProcessAlive } from '../server/test/processAlive.js';describe('real workspace runner temp lifecycle', () => {const workspace = 'server';@@ -94,3 +95,66 @@ describe('real workspace runner temp lifecycle', () => {}}, 90000);});++// #10840: a capture browser whose test process died without cleanup.+describe.skipIf(process.platform === 'win32')('orphaned test browser sweep', () => {+ // Stand-in for a test Chrome: a detached process carrying a test profile flag,+ // started by a parent that exits without cleanup. Returns both pids.+ const orphanBrowser = profile => {+ const parent = spawnSync(process.execPath, ['-e', `+ const { spawn } = require('node:child_process');+ const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)', '--', '--headless=new', '--user-data-dir=' + ${JSON.stringify(profile)}], { detached: true, stdio: 'ignore' });+ child.unref();+ process.stdout.write(String(child.pid));`], { encoding: 'utf8' });+ expect(parent.status, parent.stderr).toBe(0);+ return { parentPid: parent.pid, pid: Number(parent.stdout) };+ };+ const gone = async pid => {+ for (let i = 0; i < 100 && isProcessAlive(pid); i++) await new Promise(resolve => setTimeout(resolve, 50));+ return !isProcessAlive(pid);+ };++ it('stops a browser left by a dead run and keeps one a live run still owns', async () => {+ const host = mkdtempSync(join(tmpdir(), 'vrt-'));+ const pids = [];+ try {+ const dead = join(host, 'pvt-dead');+ mkdirSync(join(dead, 'profile'), { recursive: true });+ const orphan = orphanBrowser(join(dead, 'profile'));+ pids.push(orphan.pid);+ // The run that owned the root is the parent that already exited.+ writeFileSync(join(dead, '.owner.pid'), `${orphan.parentPid} 0\n`);+ const live = join(host, 'pvt-live');+ mkdirSync(join(live, 'profile'), { recursive: true });+ writeOwnerFile(live);+ const owned = orphanBrowser(join(live, 'profile'));+ pids.push(owned.pid);+ expect(isProcessAlive(orphan.pid)).toBe(true);++ sweepStaleRunRoots(host);+ expect(await gone(orphan.pid)).toBe(true);+ expect(existsSync(dead)).toBe(false);+ expect(isProcessAlive(owned.pid)).toBe(true);++ // The owning run stops its own browsers as it exits.+ expect(killTestBrowsersUnder(live)).toBe(1);+ expect(await gone(owned.pid)).toBe(true);+ } finally {+ for (const pid of pids) { try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } }+ rmSync(host, { recursive: true, force: true });+ }+ }, 20000);++ it('stops a browser whose run root was already removed', async () => {+ const host = mkdtempSync(join(tmpdir(), 'vrt-'));+ let pid;+ try {+ ({ pid } = orphanBrowser(join(host, 'pvt-removed', 'profile')));+ sweepStaleRunRoots(host);+ expect(await gone(pid)).toBe(true);+ } finally {+ try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ }+ rmSync(host, { recursive: true, force: true });+ }+ }, 20000);+});diff --git a/scripts/vitestTempRootSetup.js b/scripts/vitestTempRootSetup.jsindex 09dd10b81e..75a0f7bc34 100644--- a/scripts/vitestTempRootSetup.js+++ b/scripts/vitestTempRootSetup.js@@ -5,7 +5,7 @@*/import { readFileSync, readdirSync, rmSync, statSync } from 'node:fs';import { join } from 'node:path';-import { OWNER_FILE } from './lib/vitestStaleRunRoots.js';+import { OWNER_FILE, killTestBrowsersUnder } from './lib/vitestStaleRunRoots.js';export function setup() {}@@ -78,6 +78,9 @@ export function isEffectivelyEmpty(path) {export function teardown() {const root = process.env.PORTOS_TEST_TEMP_ROOT;if (!root) return;+ // A browser a suite failed to tear down would outlive the run (#10840).+ const browsers = process.platform === 'win32' ? 0 : killTestBrowsersUnder(root);+ if (browsers) console.warn(`⚠️ test browser leak: stopped ${browsers} process${browsers === 1 ? '' : 'es'} still running under the run temp root`);let entries;try {entries = readdirSync(root);diff --git a/server/services/htmlComposition/encode.js b/server/services/htmlComposition/encode.jsindex 2a9c67bba4..a5da1f23f2 100644--- a/server/services/htmlComposition/encode.js+++ b/server/services/htmlComposition/encode.js@@ -43,7 +43,18 @@ const SCREENSHOT = Object.freeze({ format: 'png', optimizeForSpeed: true, fromSu// that a later window continues (parallel segments): the auto-shutter// comparison at each seam sees the real neighbouring frame instead of treating// the window edge as an edge of the song.-export async function encodeComposition(page, contract, outputPath, { musicPath, audio, signal, onProgress, offsetSec = 0, continuous = false, followed = false, videoFilter = null, master = false, runFfmpeg = runFfmpegProcess, spawnProcess = spawn, locateFfmpeg = findFfmpeg, tagFilter = bt709TagFilter } = {}) {+// A stalled or lost browser (a CDP command past its deadline on an overloaded+// host, a crashed target) is not the document's fault, and seeks are+// deterministic, so a fresh browser can redraw the same frame (#10839).+// Anything else (a script error, a refused asset) fails as before.+const RECOVERABLE_PAGE_FAILURE = /Browser command timed out|Composition browser target crashed|Composition browser target closed|Managed browser disconnected/;+const MAX_FRAME_RETRIES = 2;++// `reopenPage(failedPage)` resolves a fresh page when a capture hits a+// recoverable browser failure; the frame is redrawn there and the encoder+// keeps running, so frames already piped are kept. `onRetry({ frame, attempt,+// error })` reports each restart, `frame` counted on song time.+export async function encodeComposition(page, contract, outputPath, { musicPath, audio, signal, onProgress, offsetSec = 0, continuous = false, followed = false, videoFilter = null, master = false, reopenPage = null, onRetry = null, runFfmpeg = runFfmpegProcess, spawnProcess = spawn, locateFfmpeg = findFfmpeg, tagFilter = bt709TagFilter } = {}) {const ffmpeg = await locateFfmpeg();if (!ffmpeg) throw new Error('ffmpeg not found on PATH');const tag = await tagFilter();@@ -118,7 +129,7 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,const abort = () => stop('abort requested');signal?.addEventListener('abort', abort, { once: true });if (!Number.isFinite(offsetSec) || offsetSec < 0) throw new Error('offsetSec must be a non-negative number');- const capture = async t => {+ const captureOnce = async t => {page.check();// awaitPromise in evaluate is essential: each seek owns its paint.const at = offsetSec ? Math.round((offsetSec + t) * 1e6) / 1e6 : t;@@ -131,6 +142,25 @@ export async function encodeComposition(page, contract, outputPath, { musicPath,page.check();return Buffer.from(data, 'base64');};+ const capture = async t => {+ let stalled = false;+ for (let attempt = 1; ; attempt++) {+ try {+ if (stalled) {+ page = await reopenPage(page);+ stalled = false;+ await frameFormat(page, contract);+ }+ return await captureOnce(t);+ } catch (error) {+ if (!reopenPage || signal?.aborted || !RECOVERABLE_PAGE_FAILURE.test(error?.message ?? '')) throw error;+ const frame = Math.round((offsetSec + t) * fps);+ if (attempt > MAX_FRAME_RETRIES) throw new Error(`${error.message} (song frame ${frame} still failing after ${MAX_FRAME_RETRIES} browser restarts)`, { cause: error });+ onRetry?.({ frame, attempt, error });+ stalled = true;+ }+ }+ };const write = bytes => Promise.race([new Promise((resolve, reject) => proc.stdin.write(bytes, error => error ? reject(error) : resolve())),encoderStopped,@@ -218,12 +248,36 @@ function planSegments(numFrames, workers) {// `page` draws the first segment; `openPage(signal)` opens each further one// and the segment closes it. `videoFilterAt(offsetSec)` builds a segment's// filter on song time. One segment is exactly `encodeComposition`.-export async function encodeCompositionSegments(page, contract, outputPath, { openPage, workers = compositionRenderWorkers(), offsetSec = 0, videoFilterAt = () => null, signal, onProgress, encode = encodeComposition, runFfmpeg = runFfmpegProcess, locateFfmpeg = findFfmpeg } = {}) {+// A worker whose browser stalls closes it and continues on a fresh one from+// the frame it was on (#10839). When that replaces `page` itself, `page` is+// closed and the result carries `pageReplaced: true`.+export async function encodeCompositionSegments(page, contract, outputPath, { openPage, workers = compositionRenderWorkers(), offsetSec = 0, videoFilterAt = () => null, signal, onProgress, onRetry = null, encode = encodeComposition, runFfmpeg = runFfmpegProcess, locateFfmpeg = findFfmpeg } = {}) {const { fps, durationSec } = contract;const numFrames = Math.round(durationSec * fps);const segments = planSegments(numFrames, workers);+ let pageReplaced = false;+ // `slot.page` is the segment's current page; `reopen` closes the stalled one+ // first so the two browsers never compete for the same overloaded host.+ const replaceable = (first, pageSignal) => {+ const slot = { page: first, reopen: null };+ if (openPage) slot.reopen = async failed => {+ await failed.close().catch(() => {});+ if (failed === page) pageReplaced = true;+ slot.page = null;+ slot.page = await openPage(pageSignal);+ return slot.page;+ };+ return slot;+ };+ const released = slot => slot.page && slot.page !== page ? slot.page.close().catch(() => {}) : null;if (segments.length === 1) {- return encode(page, contract, outputPath, { signal, onProgress, offsetSec, videoFilter: videoFilterAt(offsetSec) });+ const slot = replaceable(page, signal);+ try {+ const result = await encode(page, contract, outputPath, { signal, onProgress, offsetSec, videoFilter: videoFilterAt(offsetSec), reopenPage: slot.reopen, onRetry });+ return pageReplaced ? { ...result, pageReplaced } : result;+ } finally {+ await released(slot);+ }}const ffmpeg = await locateFfmpeg();if (!ffmpeg) throw new Error('ffmpeg not found on PATH');@@ -238,12 +292,13 @@ export async function encodeCompositionSegments(page, contract, outputPath, { optry {signal?.throwIfAborted();const results = await Promise.allSettled(segments.map(async ({ start, frames }, i) => {- let segmentPage = i === 0 ? page : null;+ const slot = replaceable(i === 0 ? page : null, controller.signal);try {- segmentPage ??= await openPage(controller.signal);+ slot.page ??= await openPage(controller.signal);const at = Math.round((offsetSec + start / fps) * 1e6) / 1e6;- const result = await encode(segmentPage, { ...contract, durationSec: frames / fps }, parts[i], {+ const result = await encode(slot.page, { ...contract, durationSec: frames / fps }, parts[i], {signal: controller.signal, offsetSec: at, continuous: start > 0, followed: i < segments.length - 1, videoFilter: videoFilterAt(at),+ reopenPage: slot.reopen, onRetry: onRetry && (detail => onRetry({ ...detail, segment: i })),onProgress: (_fraction, detail) => {done[i] = detail?.frame ?? 0;const frame = done.reduce((sum, value) => sum + value, 0);@@ -255,7 +310,7 @@ export async function encodeCompositionSegments(page, contract, outputPath, { opcontroller.abort(error);throw error;} finally {- if (i > 0 && segmentPage) await segmentPage.close().catch(() => {});+ await released(slot);}}));signal?.throwIfAborted();@@ -267,7 +322,7 @@ export async function encodeCompositionSegments(page, contract, outputPath, { opconst joined = await runFfmpeg({ bin: ffmpeg, signal, args: ['-hide_banner', '-loglevel', 'error', '-f', 'concat', '-safe', '0', '-i', listPath,'-c', 'copy', '-movflags', '+faststart', '-y', outputPath] });if (!joined.ok) throw new Error(`Joining render segments failed: ${joined.reason}`);- return Object.keys(histogram).length ? { sampleHistogram: histogram } : {};+ return { ...(Object.keys(histogram).length ? { sampleHistogram: histogram } : {}), ...(pageReplaced ? { pageReplaced } : {}) };} finally {signal?.removeEventListener('abort', relay);for (const path of [...parts, listPath]) await rm(path, { force: true }).catch(() => {});diff --git a/server/services/htmlComposition/encode.segments.test.js b/server/services/htmlComposition/encode.segments.test.jsindex 367df272fb..2219fa772b 100644--- a/server/services/htmlComposition/encode.segments.test.js+++ b/server/services/htmlComposition/encode.segments.test.js@@ -112,6 +112,97 @@ describe('encodeComposition at segment seams', () => {});});+// #10839: one stalled screenshot restarts that worker's browser, which redraws+// the frame it was on; a frame that keeps stalling still fails the render.+describe('encodeCompositionSegments when a worker browser stalls', () => {+ const stalled = () => new Error('Browser command timed out: Page.captureScreenshot');+ // `stallsAt(frame, captureCount)` decides whether this page's capture stalls.+ const capturePage = (name, stallsAt = () => false) => {+ const state = { name, closed: 0, captures: 0, failure: null, seekAt: null };+ return Object.assign(state, {+ check() { if (state.failure) throw state.failure; },+ async evaluate(expression) { const match = /seek\(([^)]+)\)/.exec(expression); if (match) state.seekAt = Number(match[1]); },+ async send(method) {+ if (state.failure) throw state.failure;+ if (method !== 'Page.captureScreenshot') return {};+ state.captures += 1;+ // A CDP deadline poisons the whole page connection, as in browser.js.+ if (stallsAt(Math.round(state.seekAt * 10), state.captures)) throw (state.failure = stalled());+ return { data: PNG };+ },+ async close() { state.closed += 1; },+ });+ };+ const run = (first, openPage, options = {}) => {+ const written = [];+ const encode = (p, c, path, encodeOptions) => encodeComposition(p, c, path, {+ ...encodeOptions, locateFfmpeg: async () => 'ffmpeg', tagFilter: async () => 'format=yuv420p',+ spawnProcess: () => {+ const proc = new EventEmitter();+ proc.stderr = new EventEmitter();+ proc.stdin = new EventEmitter();+ proc.stdin.write = (_bytes, callback) => { written.push(path); callback?.(); return true; };+ proc.stdin.end = () => proc.emit('close', 0);+ // A stop (another worker failed) ends the stub encoder as ffmpeg would.+ proc.kill = signal => { setImmediate(() => proc.emit('close', null, signal)); return true; };+ return proc;+ },+ });+ const result = encodeCompositionSegments(first, { fps: 10, durationSec: 48, width: 4, height: 4, motionBlur: 1 }, '/tmp/stall.mp4', {+ workers: 2, openPage, encode, locateFfmpeg: async () => 'ffmpeg', runFfmpeg: async () => ({ ok: true }), ...options,+ });+ return { result, written };+ };++ it('restarts the stalled worker and still writes every frame', async () => {+ const opened = [];+ const retries = [];+ // The second worker's first browser stalls once, on song frame 300.+ const openPage = async () => {+ const p = capturePage(`extra${opened.length}`, opened.length === 0 ? frame => frame === 300 : undefined);+ opened.push(p);+ return p;+ };+ const first = capturePage('first');+ const { result, written } = run(first, openPage, { onRetry: detail => retries.push(detail) });+ expect(await result).toEqual({});+ expect(written.filter(path => path.endsWith('.part0.mp4'))).toHaveLength(240);+ expect(written.filter(path => path.endsWith('.part1.mp4'))).toHaveLength(240);+ expect(retries.map(({ frame, attempt, segment }) => ({ frame, attempt, segment }))).toEqual([{ frame: 300, attempt: 1, segment: 1 }]);+ // The stalled browser closed before its replacement opened and redrew frame 300.+ expect(opened).toHaveLength(2);+ expect(opened.map(p => p.closed)).toEqual([1, 1]);+ expect(opened[1].captures).toBe(180);+ expect(first.closed).toBe(0);+ });++ it('replaces the caller\'s page when the first worker stalls and says so', async () => {+ const first = capturePage('first', frame => frame === 10);+ const { result } = run(first, async () => capturePage('extra'));+ expect(await result).toEqual({ pageReplaced: true });+ expect(first.closed).toBe(1);+ });++ it('fails the render, naming the frame, when the same frame keeps stalling', async () => {+ const opened = [];+ const openPage = async () => { const p = capturePage('extra', frame => frame === 300); opened.push(p); return p; };+ const { result } = run(capturePage('first'), openPage);+ await expect(result).rejects.toThrow(/song frame 300 still failing after 2 browser restarts/);+ // The first worker's page, plus the second worker's original and two restarts.+ expect(opened).toHaveLength(3);+ expect(opened.every(p => p.closed === 1)).toBe(true);+ });++ it('does not restart for a composition script failure', async () => {+ const opened = [];+ const broken = capturePage('first');+ broken.evaluate = async () => { throw new Error('Composition script failed: boom'); };+ const { result } = run(broken, async () => { const p = capturePage('extra'); opened.push(p); return p; });+ await expect(result).rejects.toThrow(/Composition script failed: boom/);+ expect(opened).toHaveLength(1);+ });+});+const ffmpeg = await findFfmpeg();describe.skipIf(!ffmpeg)('encodeCompositionSegments with real ffmpeg', () => {it('joins encoded segments into one stream with every frame and no leftovers', async () => {diff --git a/server/services/htmlComposition/testBrowserCleanup.js b/server/services/htmlComposition/testBrowserCleanup.jsindex f5f9b89760..f8033928ff 100644--- a/server/services/htmlComposition/testBrowserCleanup.js+++ b/server/services/htmlComposition/testBrowserCleanup.js@@ -230,6 +230,25 @@ export async function _describeTestChromeStartup(proc, { source = 'unknown', exe+ ` profile=${profileState} devToolsActivePort=${port}`;}+// A test process that exits without its suite's cleanup (a crash, a worker+// torn down mid-test) takes its browser with it instead of orphaning it (#10840).+// One exit hook for the module; the browsers themselves get no extra listeners.+const startedBrowsers = new Set();+let exitHooked = false;+const settled = child => child.exitCode !== null || child.signalCode !== null;+function killWithTestProcess(proc) {+ if (!exitHooked) {+ exitHooked = true;+ process.once('exit', () => {+ for (const child of startedBrowsers) {+ if (!settled(child)) try { child.kill('SIGKILL'); } catch { /* already gone */ }+ }+ });+ }+ for (const child of startedBrowsers) if (settled(child)) startedBrowsers.delete(child);+ startedBrowsers.add(proc);+}+// Chrome writes its CDP address to stderr. Keep only a bounded tail and report// known failure categories: raw stderr can contain the user's profile path.export function _waitForTestChrome(proc, timeoutMs = 20000, startup, { observeProcess = _testChromeProcessFacts, spawnVersion = spawn } = {}) {@@ -272,6 +291,7 @@ export function _waitForTestChrome(proc, timeoutMs = 20000, startup, { observePrdiagnostic().then(rejectWithFacts, () => rejectWithFacts('; diagnostic unavailable'));} else {proc.removeListener('error', onError);+ killWithTestProcess(proc);resolve(value);}};diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.jsindex 70735007ab..ade4b19587 100644--- a/server/services/musicVideo/documentRender.js+++ b/server/services/musicVideo/documentRender.js@@ -399,13 +399,18 @@ export async function encodeDocumentComposition({}}// Long windows split across several browsers (encodeCompositionSegments).- await encodeCompositionSegments(page, { ...target, durationSec: window.durationSec }, silent, {+ // A stalled worker browser restarts and redraws its frame (#10839).+ const encoded = await encodeCompositionSegments(page, { ...target, durationSec: window.durationSec }, silent, {openPage, encode: encodeComposition,videoFilterAt: (offsetSec) => musicVideoGradeFilter(project.composition?.grade, data.scenes, { fps: target.fps, offsetSec }),signal, offsetSec: window.startSec, onProgress: (fraction) => onProgress?.(fraction * 0.95),+ onRetry: ({ frame, attempt, segment, error }) => console.warn(`⚠️ Music-video composition render [${String(jobId).slice(0, 8)}] ${segment == null ? '' : `worker ${segment + 1} `}restarting its browser at song frame ${frame} (retry ${attempt}): ${error.message}`),});- page.check();- await page.close({ verify: true });+ // A replaced first page was already closed after its failure.+ if (!encoded?.pageReplaced) {+ page.check();+ await page.close({ verify: true });+ }page = null;signal?.throwIfAborted();const mux = await runFfmpegProcess({