Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retry test Chrome startup once on a loaded runner (#12475)

[._.]/ Adam Eivy01996bdbfb1c
diff --git a/server/services/codeAnimation/stages.realBrowser.db.test.js b/server/services/codeAnimation/stages.realBrowser.db.test.js
index 374536dd43..5158808386 100644
--- a/server/services/codeAnimation/stages.realBrowser.db.test.js
+++ b/server/services/codeAnimation/stages.realBrowser.db.test.js
@@ -11,7 +11,7 @@ import { join } from 'node:path';
import { chromium } from 'playwright-core';
import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
import { findFfmpeg, findFfprobe } from '../../lib/ffmpeg.js';
-import { _cleanupTestBrowser, _createTestArtifactsRoot, _waitForTestChrome, _testChromeCaptureArgs } from '../htmlComposition/testBrowserCleanup.js';
+import { _cleanupTestBrowser, _createTestArtifactsRoot, _startTestChromeWithRetry, _testChromeCaptureArgs, _waitForTestChrome } from '../htmlComposition/testBrowserCleanup.js';
import { resolveTestChrome } from '../../lib/browserSuiteGate.js';
let endpoint;
@@ -65,16 +65,25 @@ let proc;
let browser;
let artifacts;
let projectId;
-const detachedChromeProcessGroupId = () => process.platform === 'win32' ? undefined : proc?.pid;
+let startup;
+const detachedChromeProcessGroupId = (child = proc) => process.platform === 'win32' ? undefined : child?.pid;
describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages with real Chrome and ffmpeg', () => {
beforeAll(async () => {
const root = lazyTempDataRoot('portos-code-animation-stages-');
- const profile = join(root, 'chrome-test-profile');
- proc = spawn(chrome, _testChromeCaptureArgs(profile), { stdio: ['ignore', 'ignore', 'pipe'], detached: process.platform !== 'win32' });
- channel('portos.composition-browser.lifecycle').publish({ event: 'child', profile: root, child: proc });
try {
- const ws = await _waitForTestChrome(proc, 20000, { source: 'test-chrome', executable: chrome, profile });
+ const ws = await _startTestChromeWithRetry({
+ launch: attempt => {
+ const profile = join(root, `chrome-test-profile-${attempt}`);
+ const child = spawn(chrome, _testChromeCaptureArgs(profile), { stdio: ['ignore', 'ignore', 'pipe'], detached: process.platform !== 'win32' });
+ channel('portos.composition-browser.lifecycle').publish({ event: 'child', profile: root, child });
+ startup = { source: 'test-chrome', executable: chrome, profile };
+ return child;
+ },
+ onProc: child => { proc = child; },
+ wait: child => _waitForTestChrome(child, 20000, startup),
+ discard: child => _cleanupTestBrowser({ proc: child, cleanup: async () => {}, detachedProcessGroupId: detachedChromeProcessGroupId(child) }),
+ });
endpoint = new URL(ws).origin.replace('ws:', 'http:');
artifacts = _createTestArtifactsRoot();
browser = await chromium.connectOverCDP(endpoint, { artifactsDir: artifacts.path });
@@ -82,9 +91,9 @@ describe.skipIf(!dbReady || !chrome || !ffmpeg || !ffprobe)('Production stages w
// Retain the owned handle for afterAll if termination itself fails.
await _cleanupTestBrowser({ browser, proc, artifacts, cleanup: cleanupTempDataRoots, startupError: error, detachedProcessGroupId: detachedChromeProcessGroupId() });
}
- // Enclose startup (20s), version probe (up to 3s), disconnect (5s), direct
+ // Enclose two startup attempts (2 x 20s), a discard (~20s), version probe (up to 3s), disconnect (5s), direct
// child termination (10s), and detached-group drain (5s) budgets.
- }, 50000);
+ }, 90000);
afterAll(async () => {
if (projectId) await query('DELETE FROM code_animation_projects WHERE id = $1', [projectId]);
diff --git a/server/services/htmlComposition/testBrowserCleanup.js b/server/services/htmlComposition/testBrowserCleanup.js
index 5eb9f88e38..184a1daa75 100644
--- a/server/services/htmlComposition/testBrowserCleanup.js
+++ b/server/services/htmlComposition/testBrowserCleanup.js
@@ -319,6 +319,24 @@ export function _waitForTestChrome(proc, timeoutMs = 20000, startup, { observePr
});
}
+// A loaded runner (CI IO pressure) can leave a healthy Chrome without a DevTools
+// address past the budget. Retry only that timeout, once, on a fresh child; any
+// other failure (spawn error, early exit) is deterministic and surfaces at once.
+// `onProc` hands every spawned child to the caller so its own cleanup owns it;
+// `discard` must terminate a child that is being abandoned for a retry.
+export async function _startTestChromeWithRetry({ launch, discard, onProc = () => {}, wait = _waitForTestChrome, attempts = 2 }) {
+ for (let attempt = 0; ; attempt++) {
+ const proc = launch(attempt);
+ onProc(proc);
+ try {
+ return await wait(proc, attempt);
+ } catch (error) {
+ if (attempt + 1 >= attempts || !/^Test Chrome did not start within/.test(error?.message)) throw error;
+ await discard(proc);
+ }
+ }
+}
+
async function withinDeadline(action, timeoutMs, stage, facts = () => '') {
let timer;
try {
diff --git a/server/services/htmlComposition/testBrowserCleanup.test.js b/server/services/htmlComposition/testBrowserCleanup.test.js
index 10a852caf8..6b073e89db 100644
--- a/server/services/htmlComposition/testBrowserCleanup.test.js
+++ b/server/services/htmlComposition/testBrowserCleanup.test.js
@@ -6,7 +6,7 @@ import { join } from 'node:path';
import { EventEmitter } from 'node:events';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { ownTestBodies } from '../../lib/mockPathsDataRoot.js';
-import { _cleanupTestBrowser, _createTestArtifactsRoot, _describeTestChromeStartup, _testBrowserCleanupSettled, _testChromeProcessFacts, _waitForTestChrome, _withTestCaptureDiagnostics } from './testBrowserCleanup.js';
+import { _cleanupTestBrowser, _createTestArtifactsRoot, _describeTestChromeStartup, _startTestChromeWithRetry, _testBrowserCleanupSettled, _testChromeProcessFacts, _waitForTestChrome, _withTestCaptureDiagnostics } from './testBrowserCleanup.js';
async function waitForFile(path, timeoutMs = 2000) {
const deadline = Date.now() + timeoutMs;
@@ -896,3 +896,27 @@ describe('fixture-owned CDP artifacts root', () => {
expect(files.filter(file => unowned(readFileSync(join(server, file), 'utf8')))).toEqual([]);
});
});
+
+describe('test Chrome startup retry', () => {
+ const timeout = () => new Error('Test Chrome did not start within 20000ms; no stderr');
+
+ it('retries a startup timeout once on a fresh child and discards the abandoned one', async () => {
+ const procs = [];
+ const discard = vi.fn(async () => {});
+ const wait = vi.fn(async (proc, attempt) => { if (attempt === 0) throw timeout(); return 'ws://127.0.0.1:1/devtools/browser/example'; });
+ const ws = await _startTestChromeWithRetry({ launch: attempt => ({ attempt }), onProc: proc => procs.push(proc), wait, discard });
+ expect(ws).toBe('ws://127.0.0.1:1/devtools/browser/example');
+ expect(procs).toEqual([{ attempt: 0 }, { attempt: 1 }]);
+ expect(discard).toHaveBeenCalledExactlyOnceWith(procs[0]);
+ });
+
+ it('gives up after the attempt budget and does not retry deterministic failures', async () => {
+ const discard = vi.fn(async () => {});
+ await expect(_startTestChromeWithRetry({ launch: () => ({}), wait: async () => { throw timeout(); }, discard })).rejects.toThrow(/did not start/);
+ expect(discard).toHaveBeenCalledTimes(1);
+ const exited = vi.fn(async () => { throw new Error('Test Chrome exited before startup (code 1)'); });
+ await expect(_startTestChromeWithRetry({ launch: () => ({}), wait: exited, discard })).rejects.toThrow(/exited/);
+ expect(exited).toHaveBeenCalledTimes(1);
+ expect(discard).toHaveBeenCalledTimes(1);
+ });
+});