Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: distinguish H.264 probe teardown failures from missing codecs in browser fidelity (#11803)
Adam Eivy2124e58d6991
diff --git a/scripts/browser-fidelity.js b/scripts/browser-fidelity.jsindex d3368fc183..7638825bfd 100644--- a/scripts/browser-fidelity.js+++ b/scripts/browser-fidelity.js@@ -49,6 +49,11 @@ async function fidelityToolchain(env) {'/usr/bin/chromium', '/usr/bin/chromium-browser'].find(path => path && existsSync(path));const ffmpegPath = await findFfmpeg();if (!chromePath || !ffmpegPath) throw Error('Fidelity Chrome/FFmpeg toolchain unavailable.');+ // Surface the structured probe cause here: the suite's own prerequisite gate+ // reports through vitest output, which this command's refusal never relays.+ const { probeChromeH264, describeH264Probe } = await import('../server/lib/browserSuiteGate.js');+ const probe = await probeChromeH264(chromePath);+ if (!probe.ok) throw Error(describeH264Probe(probe));const version = (path, args) => {const text = execFileSync(path, args, { encoding: 'utf8', timeout: 5000, maxBuffer: 65536,stdio: ['ignore', 'pipe', 'ignore'], env: fidelityChildEnv(env, '') });diff --git a/server/lib/README.md b/server/lib/README.mdindex fac6defa07..1f2cc2a629 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -770,7 +770,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| Module | Purpose ||---|---|-| `browserSuiteGate.js` | `chromeWithH264(path)` returns the path only when that Chrome decodes H.264 (null on codec-less Chromium); `browserSuiteCanRun(label, prerequisites)` is the `canRun` guard for the cross-workspace `*.browser.test.js` suites (Chrome + ffmpeg + the client bundler). Pass `{ name: discoveredValue }`; a falsy value is missing. Returns `false` with a visible skip log locally, but throws naming the missing prerequisites when `PORTOS_REQUIRE_BROWSER_SUITES` is set — the flag the CI `database` job sets for the browser suites the planner selected (#10312). |+| `browserSuiteGate.js` | `probeChromeH264(path)` runs the H.264 probe as an owned, deadline-bound child (own process group, SIGTERM→SIGKILL ladder awaiting `close`) and resolves `{ ok }` or `{ ok:false, cause }` — `missing-executable`, `codec-unavailable`, `probe-timeout`, `teardown-failed` (answered, then exited nonzero: never trusted, never a missing codec), `no-answer`; `describeH264Probe` is its path/PID-free one-liner; `chromeWithH264(path)` (async) returns the path only when that Chrome decodes H.264, null when the executable or codec is genuinely absent, and `{ probeFailure }` when the probe itself failed (reported by `browserSuiteCanRun` as a lifecycle failure, not a missing prerequisite); `browserSuiteCanRun(label, prerequisites)` is the `canRun` guard for the cross-workspace `*.browser.test.js` suites (Chrome + ffmpeg + the client bundler). Pass `{ name: discoveredValue }`; a falsy value is missing. Returns `false` with a visible skip log locally, but throws naming the missing prerequisites when `PORTOS_REQUIRE_BROWSER_SUITES` is set — the flag the CI `database` job sets for the browser suites the planner selected (#10312). || `dbTestGate.js` | `requireDbOrSkip(label, dbReady, reason)` keeps a missing local test database as a visible skipped suite, but throws when `PORTOS_REQUIRE_DB` is set so CI cannot pass after DB-backed suites disappear. || `gitTestRepo.js` | Shared real-git sandbox for integration tests (#4394): one initialized template (working tree + bare origin) per worker, then `fs.cp` into a fresh temp dir. `makeGitSandbox({ origin })`, `attachBareOrigin(scratch, repo)`, `materializeGitRepo(dest)`, `destroyGitSandbox`, plus `SKIP_HEAVY_INTEGRATION` (`VITEST_FAST=1`). `resetGitSandbox({ scratch, repo, initialHead })` / `resetGitWorktreeSandbox(repo, initialHead)` restore a sandbox in place (branches, worktrees, remote) so a `describe` can build one sandbox in `beforeAll` and reset between tests instead of paying the fs.cp/rm cycle per test (#5902). Every entry point runs `assertTempPath` first, so a path outside `os.tmpdir()` throws instead of `git init`-ing or `rm -rf`-ing a real checkout (#4554). Fixture-local maintenance/GC detachment is disabled so awaited Git operations retain their writers through teardown. Still real git — just not rebuilt from `init`+`commit`+`push` in every `beforeEach`. || `mirrorParity.js` | Source-extraction and comparison primitives for declaration-level contract tests: the vendored `aiToolkit/` copy of `TOOL_USE_RE` (`localModelHeuristics.mirror.test.js`), and the bootstrap sequence's body assertions (`bootstrap*.test.js`). A pure module the browser needs is imported by the client rather than mirrored, so no client↔server copy test should need these: `stripCommentsAndNormalize` (so per-side commentary may diverge but logic may not), `extractDeclaration(src, name)` (balanced `{}`/`()`/`[]` walk over `function` / `async function` / `const`), and `compareDeclaration(serverSrc, clientSrc, name)`. Use these instead of hand-rolling a brace-walker per mirror. Pure — no `vitest` import — so callers own the assertions. |diff --git a/server/lib/browserSuiteGate.js b/server/lib/browserSuiteGate.jsindex 9bf99aab7b..08e93bf54b 100644--- a/server/lib/browserSuiteGate.js+++ b/server/lib/browserSuiteGate.js@@ -1,4 +1,5 @@-import { execFileSync } from './childProcess.js';+import { spawn } from './childProcess.js';+import { IS_WIN32, MAX_OUTPUT_BYTES, killProcessTree } from './bufferedSpawn.js';import { mkdtempSync, rmSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';@@ -9,16 +10,23 @@ import { join } from 'node:path';* provisions them fail fast instead of passing green on a skip.** `prerequisites` maps a human label to whatever was discovered (a path, a- * resolved bundler list, or undefined/null when absent). `onUnavailable` runs+ * resolved bundler list, or undefined/null when absent; `chromeWithH264` may+ * also report `{ probeFailure }` for a probe that ran and failed). `onUnavailable` runs* before the skip or the throw: prerequisite discovery can initialize mocked* path fixtures, and Vitest runs no cleanup hooks for a skipped (or failed) file.*/export function browserSuiteCanRun(label, prerequisites, { onUnavailable } = {}) {- const missing = Object.entries(prerequisites).filter(([, found]) => !found).map(([name]) => name);- if (missing.length === 0) return true;+ const entries = Object.entries(prerequisites);+ // A probe that ran but failed is a distinct verdict from an absent prerequisite.+ const failed = entries.filter(([, found]) => found?.probeFailure);+ const missing = entries.filter(([, found]) => !found).map(([name]) => name);+ if (missing.length === 0 && failed.length === 0) return true;onUnavailable?.();- const reason = `missing ${missing.join(', ')}`;+ const reason = [+ ...(missing.length ? [`missing ${missing.join(', ')}`] : []),+ ...failed.map(([name, found]) => `${name} probe failed (${found.probeFailure})`),+ ].join('; ');if (process.env.PORTOS_REQUIRE_BROWSER_SUITES) {throw new Error(`${label}: browser suite skipped but PORTOS_REQUIRE_BROWSER_SUITES is set — ${reason}`);}@@ -29,23 +37,107 @@ export function browserSuiteCanRun(label, prerequisites, { onUnavailable } = {})const H264_PROBE_PAGE = `data:text/html,<body><script>document.body.textContent='H264:'+document.createElement('video').canPlayType('video/mp4; codecs="avc1.640028"')</script></body>`;+const PROBE_TIMEOUT_MS = 20000;+const PROBE_KILL_GRACE_MS = 5000;++// Runs the probe as an owned child: its own process group on POSIX, a bounded+// deadline, and a SIGTERM -> SIGKILL ladder that waits for `close` so the+// outcome says whether cleanup was actually observed. stderr is discarded, so+// no raw browser output (paths, PIDs) can reach a caller.+function runOwnedProbe(file, args, { timeoutMs, graceMs }) {+ return new Promise((resolve) => {+ const processGroup = !IS_WIN32;+ const child = spawn(file, args, { stdio: ['ignore', 'pipe', 'ignore'], detached: processGroup });+ const timers = [];+ let stdout = '';+ let timedOut = false;+ let settled = false;++ const finish = (outcome) => {+ if (settled) return;+ settled = true;+ timers.forEach(clearTimeout);+ // Reap renderer/GPU helpers a slow-exiting Chrome may have left behind.+ killProcessTree(child, 'SIGKILL', { processGroup });+ child.stdout?.destroy();+ resolve({ stdout, timedOut, ...outcome });+ };++ child.stdout.on('data', (chunk) => {+ stdout = (stdout + chunk).slice(-MAX_OUTPUT_BYTES);+ });+ child.on('error', (error) => finish({ spawnErrorCode: error.code ?? 'UNKNOWN', cleanupVerified: true }));+ child.on('close', (code, signal) => finish({ code, signal, cleanupVerified: true }));++ timers.push(setTimeout(() => {+ timedOut = true;+ killProcessTree(child, 'SIGTERM', { processGroup });+ timers.push(setTimeout(() => {+ killProcessTree(child, 'SIGKILL', { processGroup });+ timers.push(setTimeout(() => finish({ cleanupVerified: false }), graceMs));+ }, graceMs));+ }, timeoutMs));+ });+}+/**- * Whether a Chrome binary can decode H.264 (Playwright's bundled Chromium has- * no proprietary codecs). Returns the binary path when it can, else null, so it- * drops straight into a `browserSuiteCanRun` prerequisite map. A binary that- * fails to launch or answer counts as unable.+ * Probes whether a Chrome binary can decode H.264 and says why when it cannot.+ * Resolves `{ ok: true, chromePath }` or `{ ok: false, cause, ... }` with cause:+ * - `missing-executable` — no path, or the file does not exist+ * - `codec-unavailable` — Chrome answered cleanly and reported no H.264+ * - `probe-timeout` — no exit within the deadline (`cleanupVerified` says whether+ * the child was observed to close after the kill ladder)+ * - `teardown-failed` — Chrome answered but then exited nonzero or by signal;+ * the codec answer is NOT trusted from a probe that did not shut down cleanly+ * - `no-answer` — Chrome did not report a codec verdict at all+ * Never rejects; the profile directory is removed once the child is settled.*/-export function chromeWithH264(chromePath) {- if (!chromePath) return null;+export async function probeChromeH264(chromePath, { timeoutMs = PROBE_TIMEOUT_MS, graceMs = PROBE_KILL_GRACE_MS } = {}) {+ if (!chromePath) return { ok: false, cause: 'missing-executable' };const profile = mkdtempSync(join(tmpdir(), 'portos-h264-probe-'));try {- const dom = execFileSync(chromePath, ['--headless=new', '--disable-gpu', '--no-sandbox',- `--user-data-dir=${profile}`, '--virtual-time-budget=2000', '--dump-dom', H264_PROBE_PAGE],- { encoding: 'utf8', timeout: 20000, stdio: ['ignore', 'pipe', 'ignore'] });- return /H264:\w+/.test(dom) ? chromePath : null;- } catch {- return null;+ const run = await runOwnedProbe(chromePath, ['--headless=new', '--disable-gpu', '--no-sandbox',+ `--user-data-dir=${profile}`, '--virtual-time-budget=2000', '--dump-dom', H264_PROBE_PAGE], { timeoutMs, graceMs });+ if (run.spawnErrorCode) {+ return { ok: false, cause: ['ENOENT', 'ENOTDIR'].includes(run.spawnErrorCode) ? 'missing-executable' : 'no-answer' };+ }+ if (run.timedOut) return { ok: false, cause: 'probe-timeout', timeoutMs, cleanupVerified: run.cleanupVerified };+ const answer = /H264:(\w*)/.exec(run.stdout);+ if (run.code !== 0) {+ return { ok: false, cause: answer ? 'teardown-failed' : 'no-answer', exit: run.code ?? run.signal };+ }+ if (!answer) return { ok: false, cause: 'no-answer', exit: 0 };+ return answer[1] ? { ok: true, chromePath } : { ok: false, cause: 'codec-unavailable' };} finally {rmSync(profile, { recursive: true, force: true });}}++/** One-line, path/PID-free description of a failed `probeChromeH264` result. */+export function describeH264Probe({ cause, exit, timeoutMs, cleanupVerified }) {+ switch (cause) {+ case 'missing-executable': return 'Chrome executable not found';+ case 'codec-unavailable': return 'Chrome reports no H.264 decode support';+ case 'probe-timeout':+ return `Chrome H.264 probe timed out after ${Math.round(timeoutMs / 1000)}s; child cleanup ${cleanupVerified ? 'verified' : 'unverified'}`;+ case 'teardown-failed':+ return `Chrome answered the H.264 probe but then failed to shut down cleanly (exit ${exit}); a browser lifecycle failure, not a missing codec`;+ default: return `Chrome H.264 probe produced no codec answer${exit === undefined ? '' : ` (exit ${exit})`}`;+ }+}++/**+ * Whether a Chrome binary can decode H.264 (Playwright's bundled Chromium has+ * no proprietary codecs). Resolves the binary path when it can, null when the+ * executable or the codec is genuinely absent — so it drops straight into a+ * `browserSuiteCanRun` prerequisite map — and `{ probeFailure }` when the probe+ * itself failed, which `browserSuiteCanRun` reports as a lifecycle failure+ * rather than a missing prerequisite.+ */+export async function chromeWithH264(chromePath, options) {+ const result = await probeChromeH264(chromePath, options);+ if (result.ok) return result.chromePath;+ if (result.cause === 'missing-executable' || result.cause === 'codec-unavailable') return null;+ return { probeFailure: result.cause === 'probe-timeout' && !result.cleanupVerified+ ? 'probe-timeout, cleanup unverified' : result.cause };+}diff --git a/server/lib/browserSuiteGate.test.js b/server/lib/browserSuiteGate.test.jsindex dc2e59dbba..7f8e2bca9d 100644--- a/server/lib/browserSuiteGate.test.js+++ b/server/lib/browserSuiteGate.test.js@@ -1,5 +1,8 @@import { afterEach, describe, expect, it, vi } from 'vitest';-import { browserSuiteCanRun } from './browserSuiteGate.js';+import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { browserSuiteCanRun, chromeWithH264, describeH264Probe, probeChromeH264 } from './browserSuiteGate.js';const original = process.env.PORTOS_REQUIRE_BROWSER_SUITES;@@ -44,3 +47,86 @@ describe('browserSuiteCanRun', () => {expect(log).not.toHaveBeenCalled();});});++describe('H.264 probe outcomes', () => {+ const dirs = [];+ const fakeChrome = (body) => {+ const dir = mkdtempSync(join(tmpdir(), 'portos-fake-chrome-'));+ dirs.push(dir);+ const path = join(dir, 'chrome');+ writeFileSync(path, `#!${process.execPath}\n${body}\n`);+ chmodSync(path, 0o755);+ return path;+ };+ const pidAlive = (pid) => { try { process.kill(pid, 0); return true; } catch { return false; } };+ afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); });++ // The fake binaries are shebang scripts, which Windows cannot execute.+ const posix = it.skipIf(process.platform === 'win32');++ posix('accepts a clean exit that reports H.264', async () => {+ const chrome = fakeChrome("process.stdout.write('<body>H264:probably</body>');");+ expect(await probeChromeH264(chrome)).toEqual({ ok: true, chromePath: chrome });+ expect(await chromeWithH264(chrome)).toBe(chrome);+ });++ posix('treats a positive codec answer followed by a nonzero exit as a lifecycle failure, never a missing codec', async () => {+ const chrome = fakeChrome("process.stdout.write('<body>H264:probably</body>'); process.exitCode = 2;");+ const result = await probeChromeH264(chrome);+ expect(result).toMatchObject({ ok: false, cause: 'teardown-failed', exit: 2 });+ expect(describeH264Probe(result)).toMatch(/lifecycle failure, not a missing codec/);++ const failure = await chromeWithH264(chrome);+ expect(failure).toEqual({ probeFailure: 'teardown-failed' });++ process.env.PORTOS_REQUIRE_BROWSER_SUITES = '1';+ expect(() => browserSuiteCanRun('suite', { 'Chrome with H.264': failure, ffmpeg: '/usr/bin/ffmpeg' }))+ .toThrow('suite: browser suite skipped but PORTOS_REQUIRE_BROWSER_SUITES is set — Chrome with H.264 probe failed (teardown-failed)');+ });++ posix('reports a codec-less Chrome and a missing executable as absent prerequisites', async () => {+ const codecless = fakeChrome("process.stdout.write('<body>H264:</body>');");+ expect(await probeChromeH264(codecless)).toEqual({ ok: false, cause: 'codec-unavailable' });+ expect(await chromeWithH264(codecless)).toBeNull();+ expect(await chromeWithH264(join(tmpdir(), 'portos-no-such-chrome'))).toBeNull();+ expect(await probeChromeH264(join(tmpdir(), 'portos-no-such-chrome'))).toEqual({ ok: false, cause: 'missing-executable' });+ expect(await probeChromeH264(undefined)).toEqual({ ok: false, cause: 'missing-executable' });+ });++ posix('distinguishes a probe that never answered from a codec-less one', async () => {+ const silent = fakeChrome('process.exitCode = 0;');+ expect(await probeChromeH264(silent)).toMatchObject({ ok: false, cause: 'no-answer' });+ const crashed = fakeChrome('process.exitCode = 9;');+ expect(await probeChromeH264(crashed)).toMatchObject({ ok: false, cause: 'no-answer', exit: 9 });+ });++ posix('kills a SIGTERM-ignoring hang and its group, and reports the timeout with verified cleanup', async () => {+ const pids = join(mkdtempSync(join(tmpdir(), 'portos-fake-pids-')), 'pids');+ dirs.push(join(pids, '..'));+ const chrome = fakeChrome(`+ const { spawn } = require('node:child_process');+ const helper = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });+ require('node:fs').writeFileSync(${JSON.stringify(pids)}, process.pid + ' ' + helper.pid);+ process.stdout.write('H264:probably');+ process.on('SIGTERM', () => {});+ setInterval(() => {}, 1000);+ `);+ const result = await probeChromeH264(chrome, { timeoutMs: 800, graceMs: 300 });+ expect(result).toMatchObject({ ok: false, cause: 'probe-timeout', cleanupVerified: true });+ expect(describeH264Probe(result)).toBe('Chrome H.264 probe timed out after 1s; child cleanup verified');+ const [parent, helper] = readFileSync(pids, 'utf8').split(' ').map(Number);+ await vi.waitFor(() => {+ expect(pidAlive(parent)).toBe(false);+ expect(pidAlive(helper)).toBe(false);+ }, { timeout: 3000 });+ expect(await chromeWithH264(chrome, { timeoutMs: 800, graceMs: 300 })).toEqual({ probeFailure: 'probe-timeout' });+ });++ it('names a probe failure beside missing prerequisites when skipping locally', () => {+ delete process.env.PORTOS_REQUIRE_BROWSER_SUITES;+ const log = vi.spyOn(console, 'log').mockImplementation(() => {});++ expect(browserSuiteCanRun('suite', { 'Chrome with H.264': { probeFailure: 'teardown-failed' }, ffmpeg: null })).toBe(false);+ expect(log).toHaveBeenCalledWith('⏭️ suite: skipping suite — missing ffmpeg; Chrome with H.264 probe failed (teardown-failed)');+ });+});diff --git a/server/services/musicVideo/documentRender.browser.test.js b/server/services/musicVideo/documentRender.browser.test.jsindex 14dc05ba26..d143b18e7b 100644--- a/server/services/musicVideo/documentRender.browser.test.js+++ b/server/services/musicVideo/documentRender.browser.test.js@@ -81,7 +81,7 @@ const chrome = [process.env.CHROME_PATH, chromium.executablePath(),].find((path) => path && existsSync(path));const ffmpeg = await findFfmpeg();// Imported discovery helpers can initialize mocked paths even on a skip.-const canRun = browserSuiteCanRun('documentRender browser suite', { 'Chrome with H.264': chromeWithH264(chrome), ffmpeg }, { onUnavailable: cleanupTempDataRoots });+const canRun = browserSuiteCanRun('documentRender browser suite', { 'Chrome with H.264': await chromeWithH264(chrome), ffmpeg }, { onUnavailable: cleanupTempDataRoots });describe.skipIf(!canRun)('layered template with real Chrome and ffmpeg', () => {let proc;