Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

ci: publish aggregate regression report

ldm02e139811c5b5
diff --git a/.github/scripts/render-ci-regression-comment.cjs b/.github/scripts/render-ci-regression-comment.cjs
new file mode 100644
index 000000000..c6a12d7b4
--- /dev/null
+++ b/.github/scripts/render-ci-regression-comment.cjs
@@ -0,0 +1,729 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const COMMENT_MARKER = '<!-- moli-ci-regression-report -->';
+const MAX_JSON_BYTES = 8 * 1024 * 1024;
+const MAX_COMMENT_BYTES = 32 * 1024;
+const MAX_RESULT_ROWS = 2_000;
+const MAX_DETAIL_ROWS = 10;
+const MAX_MATRIX_ROWS = 5_000;
+const MAX_CDP_GROUPS = 100;
+const MIB = 1024 * 1024;
+
+function isObject(value) {
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function finiteNumber(value) {
+ return typeof value === 'number' && Number.isFinite(value) && Math.abs(value) <= Number.MAX_SAFE_INTEGER
+ ? value
+ : null;
+}
+
+function nonNegativeNumber(value) {
+ const number = finiteNumber(value);
+ return number !== null && number >= 0 ? number : null;
+}
+
+function count(value) {
+ const number = nonNegativeNumber(value);
+ return number !== null && Number.isInteger(number) ? number : null;
+}
+
+function objectAt(value, key) {
+ return isObject(value) && isObject(value[key]) ? value[key] : {};
+}
+
+function safeText(value, maximumLength = 120) {
+ if (typeof value !== 'string') {
+ return 'unknown';
+ }
+ return value
+ .replace(/[\u0000-\u001f\u007f]/g, ' ')
+ .replace(/\\/g, '\\\\')
+ .replace(/\|/g, '\\|')
+ .replace(/`/g, "'")
+ .replace(/&/g, '&amp;')
+ .replace(/</g, '&lt;')
+ .replace(/>/g, '&gt;')
+ .slice(0, maximumLength) || 'unknown';
+}
+
+function code(value, maximumLength = 120) {
+ return `\`${safeText(value, maximumLength)}\``;
+}
+
+function readJson(root, relativePath) {
+ if (typeof root !== 'string' || root.length === 0) {
+ throw new Error('artifact root is unavailable');
+ }
+ const filePath = path.join(root, relativePath);
+ const metadata = fs.lstatSync(filePath);
+ if (!metadata.isFile() || metadata.isSymbolicLink()) {
+ throw new Error('artifact JSON is not a regular file');
+ }
+ if (metadata.size > MAX_JSON_BYTES) {
+ throw new Error('artifact JSON exceeds the trusted renderer limit');
+ }
+ return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function loadSection(loader) {
+ try {
+ return { available: true, data: loader() };
+ } catch {
+ return { available: false, data: null };
+ }
+}
+
+function median(values) {
+ const numbers = values.filter((value) => finiteNumber(value) !== null).sort((left, right) => left - right);
+ if (numbers.length === 0) {
+ return null;
+ }
+ const middle = Math.floor(numbers.length / 2);
+ return numbers.length % 2 === 0
+ ? (numbers[middle - 1] + numbers[middle]) / 2
+ : numbers[middle];
+}
+
+function sum(values) {
+ const numbers = values.map(finiteNumber).filter((value) => value !== null);
+ return numbers.length === 0 ? null : numbers.reduce((total, value) => total + value, 0);
+}
+
+function formatInteger(value) {
+ const number = count(value);
+ return number === null ? '—' : number.toLocaleString('en-US');
+}
+
+function formatDecimal(value, digits = 2) {
+ const number = finiteNumber(value);
+ return number === null ? '—' : number.toFixed(digits);
+}
+
+function formatMilliseconds(value) {
+ const number = finiteNumber(value);
+ return number === null ? '—' : `${number.toFixed(2)} ms`;
+}
+
+function formatDurationMilliseconds(value) {
+ const number = nonNegativeNumber(value);
+ if (number === null) {
+ return '—';
+ }
+ if (number < 1_000) {
+ return formatMilliseconds(number);
+ }
+ const totalSeconds = number / 1_000;
+ if (totalSeconds < 60) {
+ return `${totalSeconds.toFixed(2)} s`;
+ }
+ const minutes = Math.floor(totalSeconds / 60);
+ return `${minutes}m ${(totalSeconds - minutes * 60).toFixed(2)}s`;
+}
+
+function formatSeconds(value) {
+ const number = finiteNumber(value);
+ return number === null ? '—' : `${number.toFixed(2)} s`;
+}
+
+function formatPercentValue(value) {
+ const number = finiteNumber(value);
+ return number === null ? '—' : `${number.toFixed(2)}%`;
+}
+
+function formatBytes(value) {
+ const number = finiteNumber(value);
+ if (number === null) {
+ return '—';
+ }
+ const absolute = Math.abs(number);
+ if (absolute >= MIB) {
+ return `${(absolute / MIB).toFixed(2)} MiB`;
+ }
+ if (absolute >= 1024) {
+ return `${(absolute / 1024).toFixed(2)} KiB`;
+ }
+ return `${Math.round(absolute)} B`;
+}
+
+function signed(value, formatter) {
+ const number = finiteNumber(value);
+ if (number === null) {
+ return '—';
+ }
+ if (number === 0) {
+ return formatter(0);
+ }
+ return `${number > 0 ? '+' : '−'}${formatter(Math.abs(number))}`;
+}
+
+function percentDelta(base, head, digits = 2) {
+ const baseline = finiteNumber(base);
+ const current = finiteNumber(head);
+ if (baseline === null || current === null || baseline === 0) {
+ return '—';
+ }
+ const value = ((current - baseline) / Math.abs(baseline)) * 100;
+ if (value === 0) {
+ return '0.00%';
+ }
+ return `${value > 0 ? '+' : '−'}${Math.abs(value).toFixed(digits)}%`;
+}
+
+function comparisonCells(base, head, formatter, percentDigits = 2) {
+ const baseline = finiteNumber(base);
+ const current = finiteNumber(head);
+ return [
+ formatter(baseline),
+ formatter(current),
+ baseline === null || current === null ? '—' : signed(current - baseline, formatter),
+ percentDelta(baseline, current, percentDigits),
+ ];
+}
+
+function metric(summary, caseName, ...pathParts) {
+ let value = objectAt(objectAt(summary, 'cases'), caseName);
+ for (const part of pathParts) {
+ value = isObject(value) ? value[part] : null;
+ }
+ return finiteNumber(value);
+}
+
+function failures(summary) {
+ return count(summary?.gate_failures) ?? count(summary?.total_failures) ?? null;
+}
+
+function statusIcon(failureCount) {
+ if (failureCount === null) {
+ return '⚪';
+ }
+ return failureCount === 0 ? '✅' : '❌';
+}
+
+function loadRelease(root) {
+ const baseStartup = readJson(root, 'base-startup/startup/summary.json');
+ const headStartup = readJson(root, 'head-startup/startup/summary.json');
+ const baseMatrix = readJson(root, 'base-matrix/synthetic-matrix/summary.json');
+ const headMatrix = readJson(root, 'head-matrix/synthetic-matrix/summary.json');
+ const baseMatrixRows = readJson(root, 'base-matrix/synthetic-matrix/matrix.json');
+ const headMatrixRows = readJson(root, 'head-matrix/synthetic-matrix/matrix.json');
+ if (
+ !isObject(baseStartup) ||
+ !isObject(headStartup) ||
+ !isObject(baseMatrix) ||
+ !isObject(headMatrix) ||
+ !Array.isArray(baseMatrixRows) ||
+ !Array.isArray(headMatrixRows) ||
+ baseMatrixRows.length > MAX_MATRIX_ROWS ||
+ headMatrixRows.length > MAX_MATRIX_ROWS
+ ) {
+ throw new Error('invalid release regression artifact');
+ }
+ return { baseStartup, headStartup, baseMatrix, headMatrix, baseMatrixRows, headMatrixRows };
+}
+
+function matrixAggregate(summary, rows, concurrency) {
+ const selected = rows.filter(
+ (row) => isObject(row) && finiteNumber(row.concurrency) === concurrency
+ );
+ const cases = objectAt(summary, 'cases');
+ let unstable = 0;
+ for (const caseValue of Object.values(cases)) {
+ const cell = isObject(caseValue) ? caseValue[String(concurrency)] : null;
+ if (isObject(cell) && cell.stable === false) {
+ unstable += 1;
+ }
+ }
+ return {
+ latency: median(selected.map((row) => row.elapsed_p50_ms)),
+ pss: median(selected.map((row) => row.peak_pss_p50_bytes)),
+ failures: sum(selected.map((row) => row.failures)) ?? 0,
+ unstable,
+ };
+}
+
+function releaseOverview(section) {
+ if (!section.available) {
+ return { status: '⚪', signal: 'artifact unavailable or invalid' };
+ }
+ const { baseStartup, headStartup, baseMatrix, headMatrix } = section.data;
+ const baseFailures = (failures(baseStartup) ?? 0) + (failures(baseMatrix) ?? 0);
+ const headFailures = (failures(headStartup) ?? 0) + (failures(headMatrix) ?? 0);
+ const rawBase = metric(section.data.baseStartup, 'binary-size', 'binary_bytes');
+ const rawHead = metric(headStartup, 'binary-size', 'binary_bytes');
+ return {
+ status: headFailures > 0 ? '❌' : baseFailures > 0 ? '⚠️' : '✅',
+ signal: `HEAD/base failures ${formatInteger(headFailures)}/${formatInteger(baseFailures)}; raw binary ${percentDelta(rawBase, rawHead, 6)}`,
+ };
+}
+
+function renderRelease(section) {
+ const overview = releaseOverview(section);
+ const lines = [`<details open><summary><strong>Release regression</strong> — ${overview.status} ${overview.signal}</summary>`, ''];
+ if (!section.available) {
+ lines.push('Artifact unavailable or invalid. See the source CI run for infrastructure details.', '', '</details>');
+ return lines;
+ }
+
+ const { baseStartup, headStartup, baseMatrix, headMatrix, baseMatrixRows, headMatrixRows } = section.data;
+ const sizeRows = [
+ ['Raw binary', metric(baseStartup, 'binary-size', 'binary_bytes'), metric(headStartup, 'binary-size', 'binary_bytes')],
+ ['Stripped binary', metric(baseStartup, 'binary-size', 'stripped_binary_bytes'), metric(headStartup, 'binary-size', 'stripped_binary_bytes')],
+ ['gzip binary', metric(baseStartup, 'binary-size', 'tar_gz_bytes'), metric(headStartup, 'binary-size', 'tar_gz_bytes')],
+ ['Rootfs', metric(baseStartup, 'image-size', 'image_uncompressed_bytes'), metric(headStartup, 'image-size', 'image_uncompressed_bytes')],
+ ['gzip rootfs', metric(baseStartup, 'image-size', 'image_compressed_bytes'), metric(headStartup, 'image-size', 'image_compressed_bytes')],
+ ];
+ lines.push(
+ '#### Package and image size',
+ '',
+ '| Metric | Base | HEAD | Delta | Delta % |',
+ '| --- | ---: | ---: | ---: | ---: |'
+ );
+ for (const [label, base, head] of sizeRows) {
+ lines.push(`| ${label} | ${comparisonCells(base, head, formatBytes, 6).join(' | ')} |`);
+ }
+
+ const startupRows = [
+ ['Serve ready', 'serve-ready', ['elapsed_ms', 'p50']],
+ ['CDP first page', 'cdp-first-page', ['elapsed_ms', 'p50']],
+ ['CDP warm page', 'cdp-warm-pages', ['cdp_page_elapsed_p50_ms']],
+ ['CLI about:blank', 'cli-fetch-aboutblank', ['elapsed_ms', 'p50']],
+ ['CLI local JS', 'cli-fetch-local-js', ['elapsed_ms', 'p50']],
+ ];
+ lines.push(
+ '',
+ '#### Startup latency and PSS',
+ '',
+ '| Case | Base p50 | HEAD p50 | Delta | Base PSS | HEAD PSS | Delta |',
+ '| --- | ---: | ---: | ---: | ---: | ---: | ---: |'
+ );
+ for (const [label, caseName, latencyPath] of startupRows) {
+ const baseLatency = metric(baseStartup, caseName, ...latencyPath);
+ const headLatency = metric(headStartup, caseName, ...latencyPath);
+ const basePss = metric(baseStartup, caseName, 'pss_bytes', 'p50');
+ const headPss = metric(headStartup, caseName, 'pss_bytes', 'p50');
+ const latencyCells = comparisonCells(baseLatency, headLatency, formatMilliseconds).slice(0, 3);
+ const pssCells = comparisonCells(basePss, headPss, formatBytes).slice(0, 3);
+ lines.push(`| ${label} | ${[...latencyCells, ...pssCells].join(' | ')} |`);
+ }
+
+ const levels = [...new Set([
+ ...(Array.isArray(baseMatrix.concurrency_levels) ? baseMatrix.concurrency_levels : []),
+ ...(Array.isArray(headMatrix.concurrency_levels) ? headMatrix.concurrency_levels : []),
+ ])]
+ .map(finiteNumber)
+ .filter((value) => value !== null && Number.isInteger(value) && value > 0)
+ .sort((left, right) => left - right)
+ .slice(0, 10);
+ lines.push(
+ '',
+ '#### Concurrency matrix',
+ '',
+ '| Concurrency | Base p50 | HEAD p50 | Delta | Base PSS | HEAD PSS | Delta | HEAD failures | Unstable B/H |',
+ '| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |'
+ );
+ for (const level of levels) {
+ const base = matrixAggregate(baseMatrix, baseMatrixRows, level);
+ const head = matrixAggregate(headMatrix, headMatrixRows, level);
+ const latency = comparisonCells(base.latency, head.latency, formatMilliseconds).slice(0, 3);
+ const pss = comparisonCells(base.pss, head.pss, formatBytes).slice(0, 3);
+ lines.push(
+ `| ${level} | ${[...latency, ...pss, formatInteger(head.failures), `${base.unstable}/${head.unstable}`].join(' | ')} |`
+ );
+ }
+ lines.push('', '</details>');
+ return lines;
+}
+
+function loadFrontend(root) {
+ const summary = readJson(root, 'summary.json');
+ if (!isObject(summary) || !isObject(summary.counts) || !Array.isArray(summary.results) || summary.results.length > MAX_RESULT_ROWS) {
+ throw new Error('invalid frontend differential artifact');
+ }
+ return summary;
+}
+
+function frontendOverview(section) {
+ if (!section.available) {
+ return { status: '⚪', signal: 'artifact unavailable or invalid' };
+ }
+ const summary = section.data;
+ const total = summary.results.length;
+ const matches = (count(summary.counts.match) ?? 0) + (count(summary.counts.reference_ok) ?? 0);
+ const problems = Math.max(0, total - matches);
+ return {
+ status: summary.ok === true && problems === 0 ? '✅' : '❌',
+ signal: `${formatInteger(matches)}/${formatInteger(total)} cases matched; ${formatInteger(problems)} issues`,
+ };
+}
+
+function renderFrontend(section) {
+ const overview = frontendOverview(section);
+ const lines = [`<details><summary><strong>Frontend differential</strong> — ${overview.status} ${overview.signal}</summary>`, ''];
+ if (!section.available) {
+ lines.push('Artifact unavailable or invalid. See the source CI run for infrastructure details.', '', '</details>');
+ return lines;
+ }
+ const summary = section.data;
+ const timeline = objectAt(summary, 'timeline');
+ lines.push(
+ '| Match | DOM mismatch | Diagnostic mismatch | Moli error | Reference error | Infrastructure error | Mismatched frames | Duration |',
+ '| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
+ `| ${formatInteger(count(summary.counts.match) ?? 0)} | ${formatInteger(count(summary.counts.dom_mismatch) ?? 0)} | ${formatInteger(count(summary.counts.diagnostic_mismatch) ?? 0)} | ${formatInteger(count(summary.counts.moli_error) ?? 0)} | ${formatInteger(count(summary.counts.reference_error) ?? 0)} | ${formatInteger(count(summary.counts.infrastructure_error) ?? 0)} | ${formatInteger(timeline.mismatchedFrames)} | ${formatDurationMilliseconds(summary.durationMs)} |`
+ );
+ const issues = summary.results
+ .filter((result) => isObject(result) && !['match', 'reference_ok'].includes(result.status))
+ .slice(0, MAX_DETAIL_ROWS);
+ if (issues.length !== 0) {
+ lines.push('', '**First failing cases**', '');
+ for (const issue of issues) {
+ const details = [];
+ if (typeof issue.firstDifference === 'string' && issue.firstDifference.length !== 0) {
+ details.push(`first difference ${code(issue.firstDifference, 160)}`);
+ }
+ if (Array.isArray(issue.mismatchedFrames) && issue.mismatchedFrames.length !== 0) {
+ const frames = issue.mismatchedFrames.slice(0, 3).map((frame) => code(frame, 80));
+ const remainder = issue.mismatchedFrames.length - frames.length;
+ details.push(`frames ${frames.join(', ')}${remainder > 0 ? ` +${remainder} more` : ''}`);
+ }
+ const moli = objectAt(issue, 'moli');
+ const chromium = objectAt(issue, 'chromium');
+ const error = typeof moli.error === 'string'
+ ? `Moli error ${code(moli.error, 160)}`
+ : typeof chromium.error === 'string'
+ ? `Chromium error ${code(chromium.error, 160)}`
+ : null;
+ if (error !== null) {
+ details.push(error);
+ }
+ lines.push(`- ${code(issue.id)} — ${code(issue.status)}${details.length ? ` · ${details.join(' · ')}` : ''}`);
+ }
+ }
+ lines.push('', '</details>');
+ return lines;
+}
+
+function loadAgent(root) {
+ const fast = readJson(root, 'agent-fast/agent-episode/summary.json');
+ const resource = readJson(root, 'agent-resource/agent-episode/summary.json');
+ if (!isObject(fast) || !isObject(resource) || !isObject(fast.targets) || !isObject(resource.targets)) {
+ throw new Error('invalid agent episode artifact');
+ }
+ return { fast, resource };
+}
+
+function target(summary, name) {
+ return objectAt(objectAt(summary, 'targets'), name);
+}
+
+function agentOverview(section) {
+ if (!section.available) {
+ return { status: '⚪', signal: 'artifact unavailable or invalid' };
+ }
+ const totalFailures = (failures(section.data.fast) ?? 0) + (failures(section.data.resource) ?? 0);
+ const moli = target(section.data.fast, 'moli-cdp');
+ return {
+ status: statusIcon(totalFailures),
+ signal: `${formatInteger(moli.passed)}/${formatInteger(moli.episodes)} Moli episodes passed; ${formatInteger(totalFailures)} failures`,
+ };
+}
+
+function renderAgent(section) {
+ const overview = agentOverview(section);
+ const lines = [`<details><summary><strong>Agent episodes · Moli vs Chromium</strong> — ${overview.status} ${overview.signal}</summary>`, ''];
+ if (!section.available) {
+ lines.push('Artifact unavailable or invalid. See the source CI run for infrastructure details.', '', '</details>');
+ return lines;
+ }
+ const { fast, resource } = section.data;
+ lines.push(
+ '| Fast contract | Episodes | Passed | Assertions | Ready p50 | Episode p50 |',
+ '| --- | ---: | ---: | ---: | ---: | ---: |'
+ );
+ for (const name of ['moli-cdp', 'chrome-cdp']) {
+ const item = target(fast, name);
+ lines.push(
+ `| ${name === 'moli-cdp' ? 'Moli' : 'Chromium'} | ${formatInteger(item.episodes)} | ${formatInteger(item.passed)} | ${formatInteger(item.assertions_passed)}/${formatInteger(item.assertions_total)} | ${formatMilliseconds(objectAt(item, 'ready_ms').p50)} | ${formatMilliseconds(objectAt(item, 'elapsed_ms').p50)} |`
+ );
+ }
+ lines.push(
+ '',
+ '| Operation p50 | Chromium | Moli | Delta | Delta % |',
+ '| --- | ---: | ---: | ---: | ---: |'
+ );
+ for (const operation of ['navigate', 'observe', 'fill', 'click']) {
+ const chromeValue = finiteNumber(objectAt(objectAt(target(fast, 'chrome-cdp'), 'operations'), operation).p50);
+ const moliValue = finiteNumber(objectAt(objectAt(target(fast, 'moli-cdp'), 'operations'), operation).p50);
+ lines.push(`| ${operation} | ${comparisonCells(chromeValue, moliValue, formatMilliseconds).join(' | ')} |`);
+ }
+ lines.push(
+ '',
+ '| Idle-resource episode | Peak RSS | Peak PSS | Average CPU | Peak CPU | Peak processes | Sampler |',
+ '| --- | ---: | ---: | ---: | ---: | ---: | --- |'
+ );
+ for (const name of ['moli-cdp', 'chrome-cdp']) {
+ const resources = objectAt(target(resource, name), 'resources');
+ const sampler = objectAt(resources, 'sampler_health');
+ const samplerStatus = sampler.healthy !== true
+ ? '❌'
+ : sampler.pss_complete === false
+ ? '⚠️ PSS partial'
+ : '✅';
+ lines.push(
+ `| ${name === 'moli-cdp' ? 'Moli' : 'Chromium'} | ${formatBytes(resources.peak_rss_bytes)} | ${formatBytes(resources.peak_pss_bytes)} | ${formatPercentValue(resources.average_cpu_percent)} | ${formatPercentValue(resources.peak_cpu_percent)} | ${formatDecimal(resources.peak_process_count, 0)} | ${samplerStatus} |`
+ );
+ }
+ const failingCases = [];
+ for (const [profile, summary] of [['fast', fast], ['resource', resource]]) {
+ for (const name of ['moli-cdp', 'chrome-cdp']) {
+ for (const [caseName, caseValue] of Object.entries(objectAt(target(summary, name), 'cases'))) {
+ if ((count(isObject(caseValue) ? caseValue.failures : null) ?? 0) > 0) {
+ failingCases.push({ profile, target: name, caseName });
+ }
+ }
+ }
+ }
+ if (failingCases.length !== 0) {
+ lines.push('', '**First failing episodes**', '');
+ for (const failure of failingCases.slice(0, MAX_DETAIL_ROWS)) {
+ lines.push(`- ${code(failure.caseName)} — ${failure.profile} / ${failure.target}`);
+ }
+ }
+ lines.push('', '</details>');
+ return lines;
+}
+
+function loadRuntime(root) {
+ const synthetic = readJson(root, 'runtime-synthetic/synthetic/summary.json');
+ const cdpSession = readJson(root, 'cdp-session/cdp-session/summary.json');
+ if (!isObject(synthetic) || !isObject(cdpSession)) {
+ throw new Error('invalid runtime contract artifact');
+ }
+ return { synthetic, cdpSession };
+}
+
+function summarizeCases(cases, includePss) {
+ const entries = Object.entries(isObject(cases) ? cases : {}).slice(0, MAX_RESULT_ROWS);
+ return {
+ cases: entries.length,
+ failures: sum(entries.map(([, value]) => (isObject(value) ? value.failures : null))) ?? 0,
+ latencyP50: median(entries.map(([, value]) => objectAt(value, 'elapsed_ms').p50)),
+ latencyP95: median(entries.map(([, value]) => objectAt(value, 'elapsed_ms').p95)),
+ pssP50: includePss
+ ? median(entries.map(([, value]) => objectAt(value, 'peak_pss_bytes').p50))
+ : null,
+ failing: entries
+ .filter(([, value]) => (count(isObject(value) ? value.failures : null) ?? 0) > 0)
+ .map(([name]) => name)
+ .slice(0, MAX_DETAIL_ROWS),
+ };
+}
+
+function runtimeOverview(section) {
+ if (!section.available) {
+ return { status: '⚪', signal: 'artifact unavailable or invalid' };
+ }
+ const synthetic = summarizeCases(section.data.synthetic.cases, true);
+ const cdp = summarizeCases(target(section.data.cdpSession, 'moli-cdp').cases, false);
+ const caseFailures = synthetic.failures + cdp.failures;
+ const reportedFailures = (failures(section.data.synthetic) ?? 0) + (failures(section.data.cdpSession) ?? 0);
+ const totalFailures = Math.max(caseFailures, reportedFailures);
+ return {
+ status: statusIcon(totalFailures),
+ signal: `${formatInteger(synthetic.cases + cdp.cases)} contract cases; ${formatInteger(totalFailures)} failures`,
+ };
+}
+
+function renderRuntime(section) {
+ const overview = runtimeOverview(section);
+ const lines = [`<details><summary><strong>Runtime and CDP session contracts</strong> — ${overview.status} ${overview.signal}</summary>`, ''];
+ if (!section.available) {
+ lines.push('Artifact unavailable or invalid. See the source CI run for infrastructure details.', '', '</details>');
+ return lines;
+ }
+ const synthetic = summarizeCases(section.data.synthetic.cases, true);
+ const cdp = summarizeCases(target(section.data.cdpSession, 'moli-cdp').cases, false);
+ lines.push(
+ '| Suite | Cases | Failures | Median case p50 | Median case p95 | Median peak PSS p50 |',
+ '| --- | ---: | ---: | ---: | ---: | ---: |',
+ `| Synthetic fetch | ${formatInteger(synthetic.cases)} | ${formatInteger(synthetic.failures)} | ${formatMilliseconds(synthetic.latencyP50)} | ${formatMilliseconds(synthetic.latencyP95)} | ${formatBytes(synthetic.pssP50)} |`,
+ `| Long-lived CDP session | ${formatInteger(cdp.cases)} | ${formatInteger(cdp.failures)} | ${formatMilliseconds(cdp.latencyP50)} | ${formatMilliseconds(cdp.latencyP95)} | — |`
+ );
+ const failing = [...synthetic.failing, ...cdp.failing].slice(0, MAX_DETAIL_ROWS);
+ if (failing.length !== 0) {
+ lines.push('', `**Failing cases:** ${failing.map((name) => code(name)).join(', ')}`);
+ }
+ lines.push('', '</details>');
+ return lines;
+}
+
+function loadCdp(root) {
+ const summary = readJson(root, 'summary.json');
+ if (!isObject(summary) || !Array.isArray(summary.groups) || summary.groups.length > MAX_CDP_GROUPS) {
+ throw new Error('invalid CDP smoke artifact');
+ }
+ return summary;
+}
+
+function cdpStats(summary) {
+ const groups = summary.groups.filter(isObject);
+ const passed = groups.filter((group) => group.status === 'passed').length;
+ const scenarios = sum(groups.map((group) => group.scenarioCount)) ?? 0;
+ const cumulativeSeconds = sum(groups.map((group) => group.durationSeconds));
+ const failures = groups.filter((group) => group.status !== 'passed');
+ const slowest = [...groups]
+ .filter((group) => nonNegativeNumber(group.durationSeconds) !== null)
+ .sort((left, right) => right.durationSeconds - left.durationSeconds)
+ .slice(0, 5);
+ return { groups, passed, scenarios, cumulativeSeconds, failures, slowest };
+}
+
+function cdpOverview(section) {
+ if (!section.available) {
+ return { status: '⚪', signal: 'artifact unavailable or invalid' };
+ }
+ const stats = cdpStats(section.data);
+ return {
+ status: section.data.ok === true && stats.failures.length === 0 ? '✅' : '❌',
+ signal: `${formatInteger(stats.passed)}/${formatInteger(stats.groups.length)} groups passed; ${formatInteger(stats.scenarios)} scenarios`,
+ };
+}
+
+function renderCdp(section) {
+ const overview = cdpOverview(section);
+ const lines = [`<details><summary><strong>CDP smoke</strong> — ${overview.status} ${overview.signal}</summary>`, ''];
+ if (!section.available) {
+ lines.push('Artifact unavailable or invalid. See the source CI run for infrastructure details.', '', '</details>');
+ return lines;
+ }
+ const stats = cdpStats(section.data);
+ lines.push(
+ `Workers: **${formatInteger(section.data.jobs)}** · cumulative group time: **${formatSeconds(stats.cumulativeSeconds)}** · failed groups: **${formatInteger(stats.failures.length)}**`,
+ '',
+ '| Slowest group | Scenarios | Duration | Status |',
+ '| --- | ---: | ---: | --- |'
+ );
+ for (const group of stats.slowest) {
+ lines.push(`| ${code(group.group)} | ${formatInteger(group.scenarioCount)} | ${formatSeconds(group.durationSeconds)} | ${group.status === 'passed' ? '✅' : '❌'} |`);
+ }
+ if (stats.failures.length !== 0) {
+ lines.push('', '**Failed groups**', '');
+ for (const group of stats.failures.slice(0, MAX_DETAIL_ROWS)) {
+ lines.push(`- ${code(group.group)} — ${code(group.status)}${typeof group.error === 'string' ? ` · ${code(group.error, 160)}` : ''}`);
+ }
+ }
+ lines.push('', '</details>');
+ return lines;
+}
+
+function trustedRunUrl(value) {
+ try {
+ const url = new URL(value);
+ if (url.protocol === 'https:' && url.hostname === 'github.com') {
+ return url.toString();
+ }
+ } catch {
+ // Fall through to a non-link label.
+ }
+ return null;
+}
+
+function renderReport({ releaseRoot, frontendRoot, agentRoot, runtimeRoot, cdpRoot, runUrl, conclusion }) {
+ const sections = {
+ release: loadSection(() => loadRelease(releaseRoot)),
+ frontend: loadSection(() => loadFrontend(frontendRoot)),
+ agent: loadSection(() => loadAgent(agentRoot)),
+ runtime: loadSection(() => loadRuntime(runtimeRoot)),
+ cdp: loadSection(() => loadCdp(cdpRoot)),
+ };
+ const overviews = [
+ ['Release regression', releaseOverview(sections.release)],
+ ['Frontend differential', frontendOverview(sections.frontend)],
+ ['Agent episodes', agentOverview(sections.agent)],
+ ['Runtime/CDP contracts', runtimeOverview(sections.runtime)],
+ ['CDP smoke', cdpOverview(sections.cdp)],
+ ];
+ const available = Object.values(sections).filter((section) => section.available).length;
+ const sourceConclusion = ['success', 'failure', 'cancelled', 'timed_out', 'in_progress'].includes(conclusion)
+ ? conclusion
+ : 'unknown';
+ const link = trustedRunUrl(runUrl);
+ const lines = [
+ COMMENT_MARKER,
+ '## CI Regression Report',
+ '',
+ `${link ? `[Source CI run](${link})` : 'Source CI run'} · source state at render: \`${sourceConclusion}\` · artifacts: \`${available}/5\``,
+ '',
+ '| Check | Status | Signal |',
+ '| --- | :---: | --- |',
+ ];
+ for (const [label, overview] of overviews) {
+ lines.push(`| ${label} | ${overview.status} | ${overview.signal} |`);
+ }
+ lines.push(
+ '',
+ ...renderRelease(sections.release),
+ '',
+ ...renderFrontend(sections.frontend),
+ '',
+ ...renderAgent(sections.agent),
+ '',
+ ...renderRuntime(sections.runtime),
+ '',
+ ...renderCdp(sections.cdp),
+ '',
+ '_All artifact fields are parsed by the trusted default-branch renderer; missing or invalid inputs remain visible as unavailable._',
+ ''
+ );
+ const rendered = lines.join('\n');
+ if (Buffer.byteLength(rendered, 'utf8') > MAX_COMMENT_BYTES) {
+ throw new Error('rendered CI regression report exceeds 32 KiB');
+ }
+ return rendered;
+}
+
+function parseArgs(argv) {
+ const values = {};
+ for (let index = 0; index < argv.length; index += 2) {
+ const key = argv[index];
+ const value = argv[index + 1];
+ if (!key?.startsWith('--') || value === undefined) {
+ throw new Error('arguments must be --name value pairs');
+ }
+ values[key.slice(2)] = value;
+ }
+ if (!values.output) {
+ throw new Error('--output is required');
+ }
+ return values;
+}
+
+function main(argv = process.argv.slice(2)) {
+ const args = parseArgs(argv);
+ const rendered = renderReport({
+ releaseRoot: args.release,
+ frontendRoot: args.frontend,
+ agentRoot: args.agent,
+ runtimeRoot: args.runtime,
+ cdpRoot: args.cdp,
+ runUrl: args['run-url'],
+ conclusion: args.conclusion,
+ });
+ fs.writeFileSync(args.output, rendered, 'utf8');
+}
+
+if (require.main === module) {
+ main();
+}
+
+module.exports = {
+ COMMENT_MARKER,
+ loadCdp,
+ loadFrontend,
+ loadRelease,
+ renderReport,
+};
diff --git a/.github/scripts/render-ci-regression-comment.test.cjs b/.github/scripts/render-ci-regression-comment.test.cjs
new file mode 100644
index 000000000..a24adc11b
--- /dev/null
+++ b/.github/scripts/render-ci-regression-comment.test.cjs
@@ -0,0 +1,237 @@
+'use strict';
+
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const test = require('node:test');
+
+const {
+ COMMENT_MARKER,
+ renderReport,
+} = require('./render-ci-regression-comment.cjs');
+
+const MAX_RESULTS_FOR_TEST = 2_001;
+
+function writeJson(root, relativePath, value) {
+ const output = path.join(root, relativePath);
+ fs.mkdirSync(path.dirname(output), { recursive: true });
+ fs.writeFileSync(output, `${JSON.stringify(value)}\n`, 'utf8');
+}
+
+function stats(p50, p95 = p50) {
+ return { count: 1, min: p50, max: p95, median: p50, p50, p90: p95, p95 };
+}
+
+function startupSummary(offset) {
+ return {
+ total_failures: 0,
+ gate_failures: 0,
+ cases: {
+ 'binary-size': {
+ binary_bytes: 100 + offset,
+ stripped_binary_bytes: 80 + offset,
+ tar_gz_bytes: 50 + offset,
+ },
+ 'image-size': {
+ image_uncompressed_bytes: 200 + offset,
+ image_compressed_bytes: 90 + offset,
+ },
+ 'serve-ready': { elapsed_ms: stats(20 + offset), pss_bytes: stats(1_000 + offset) },
+ 'cdp-first-page': { elapsed_ms: stats(30 + offset), pss_bytes: stats(2_000 + offset) },
+ 'cdp-warm-pages': {
+ cdp_page_elapsed_p50_ms: 10 + offset,
+ elapsed_ms: stats(100 + offset),
+ pss_bytes: stats(3_000 + offset),
+ },
+ 'cli-fetch-aboutblank': { elapsed_ms: stats(15 + offset), pss_bytes: stats(1_500 + offset) },
+ 'cli-fetch-local-js': { elapsed_ms: stats(18 + offset), pss_bytes: stats(1_800 + offset) },
+ },
+ };
+}
+
+function matrixSummary(offset) {
+ return {
+ total_failures: 0,
+ gate_failures: 0,
+ stability_failures: 0,
+ concurrency_levels: [1],
+ cases: {
+ 'static-html': {
+ '1': { elapsed_p50_ms: stats(20 + offset), failures: 0, stable: true },
+ },
+ },
+ };
+}
+
+function matrixRows(offset) {
+ return [{
+ concurrency: 1,
+ case: 'static-html',
+ elapsed_p50_ms: 20 + offset,
+ peak_pss_p50_bytes: 4_000 + offset,
+ failures: 0,
+ }];
+}
+
+function agentTarget({ chrome = false, failure = false }) {
+ return {
+ episodes: 2,
+ passed: failure ? 1 : 2,
+ failures: failure ? 1 : 0,
+ assertions_total: 4,
+ assertions_passed: failure ? 3 : 4,
+ ready_ms: stats(chrome ? 200 : 20),
+ elapsed_ms: stats(chrome ? 80 : 30),
+ operations: {
+ navigate: stats(chrome ? 40 : 10),
+ observe: stats(chrome ? 4 : 2),
+ fill: stats(chrome ? 6 : 3),
+ click: stats(chrome ? 8 : 4),
+ },
+ resources: {
+ peak_rss_bytes: chrome ? 500 * 1024 : 100 * 1024,
+ peak_pss_bytes: chrome ? 400 * 1024 : 80 * 1024,
+ average_cpu_percent: chrome ? 4 : 2,
+ peak_cpu_percent: chrome ? 20 : 10,
+ peak_process_count: chrome ? 10 : 1,
+ sampler_health: { healthy: true, pss_complete: !chrome },
+ },
+ cases: {
+ 'episode-safe': { failures: 0 },
+ ...(failure ? { 'episode|unsafe<': { failures: 1 } } : {}),
+ },
+ };
+}
+
+function createArtifacts(root) {
+ const releaseRoot = path.join(root, 'release');
+ writeJson(releaseRoot, 'base-startup/startup/summary.json', startupSummary(0));
+ writeJson(releaseRoot, 'head-startup/startup/summary.json', startupSummary(10));
+ writeJson(releaseRoot, 'base-matrix/synthetic-matrix/summary.json', matrixSummary(0));
+ writeJson(releaseRoot, 'head-matrix/synthetic-matrix/summary.json', matrixSummary(2));
+ writeJson(releaseRoot, 'base-matrix/synthetic-matrix/matrix.json', matrixRows(0));
+ writeJson(releaseRoot, 'head-matrix/synthetic-matrix/matrix.json', matrixRows(2));
+
+ const frontendRoot = path.join(root, 'frontend');
+ writeJson(frontendRoot, 'summary.json', {
+ ok: false,
+ counts: { match: 1, dom_mismatch: 1 },
+ durationMs: 1_250,
+ timeline: { chromiumFrames: 5, moliFrames: 5, mismatchedFrames: 1 },
+ results: [
+ { id: 'safe', status: 'match', mismatchedFrames: [] },
+ {
+ id: 'bad|case<',
+ status: 'dom_mismatch',
+ firstDifference: '$.body|<',
+ mismatchedFrames: ['ready|<'],
+ },
+ ],
+ });
+
+ const agentRoot = path.join(root, 'agent');
+ const fastAgent = {
+ total_failures: 1,
+ gate_failures: 1,
+ targets: {
+ 'moli-cdp': agentTarget({ failure: true }),
+ 'chrome-cdp': agentTarget({ chrome: true }),
+ },
+ };
+ const resourceAgent = {
+ total_failures: 0,
+ gate_failures: 0,
+ targets: {
+ 'moli-cdp': agentTarget({}),
+ 'chrome-cdp': agentTarget({ chrome: true }),
+ },
+ };
+ writeJson(agentRoot, 'agent-fast/agent-episode/summary.json', fastAgent);
+ writeJson(agentRoot, 'agent-resource/agent-episode/summary.json', resourceAgent);
+
+ const runtimeRoot = path.join(root, 'runtime');
+ writeJson(runtimeRoot, 'runtime-synthetic/synthetic/summary.json', {
+ total_failures: 0,
+ cases: {
+ static: { failures: 0, elapsed_ms: stats(20, 25), peak_pss_bytes: stats(2_048) },
+ dynamic: { failures: 0, elapsed_ms: stats(30, 35), peak_pss_bytes: stats(4_096) },
+ },
+ });
+ writeJson(runtimeRoot, 'cdp-session/cdp-session/summary.json', {
+ total_failures: 0,
+ gate_failures: 0,
+ targets: {
+ 'moli-cdp': {
+ cases: {
+ session: { failures: 0, elapsed_ms: stats(15, 18) },
+ },
+ },
+ },
+ });
+
+ const cdpRoot = path.join(root, 'cdp');
+ writeJson(cdpRoot, 'summary.json', {
+ ok: false,
+ jobs: 4,
+ groups: [
+ { group: 'core', status: 'passed', scenarioCount: 10, durationSeconds: 2 },
+ { group: 'bad|group<', status: 'failed', scenarioCount: 3, durationSeconds: 4, error: 'boom|<tag>' },
+ ],
+ });
+
+ return { releaseRoot, frontendRoot, agentRoot, runtimeRoot, cdpRoot };
+}
+
+test('renders all five trusted artifact sections into one bounded report', (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'moli-ci-report-'));
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+ const roots = createArtifacts(root);
+ const report = renderReport({
+ ...roots,
+ runUrl: 'https://github.com/lexmount/moli/actions/runs/123',
+ conclusion: 'failure',
+ });
+
+ assert.ok(report.startsWith(COMMENT_MARKER));
+ assert.match(report, /artifacts: `5\/5`/);
+ assert.match(report, /Raw binary \| 100 B \| 110 B \| \+10 B \| \+10\.000000%/);
+ assert.match(report, /Frontend differential/);
+ assert.match(report, /`bad\\\|case&lt;`/);
+ assert.match(report, /first difference `\$\.body\\\|&lt;`/);
+ assert.match(report, /frames `ready\\\|&lt;`/);
+ assert.match(report, /First failing episodes/);
+ assert.match(report, /`episode\\\|unsafe&lt;`/);
+ assert.match(report, /PSS partial/);
+ assert.match(report, /Runtime and CDP session contracts/);
+ assert.match(report, /`bad\\\|group&lt;`/);
+ assert.ok(Buffer.byteLength(report, 'utf8') < 32 * 1024);
+});
+
+test('renders missing artifacts as unavailable without throwing', () => {
+ const report = renderReport({
+ runUrl: 'not-a-trusted-url',
+ conclusion: 'timed_out',
+ });
+
+ assert.match(report, /artifacts: `0\/5`/);
+ assert.equal((report.match(/Artifact unavailable or invalid\./g) || []).length, 5);
+ assert.doesNotMatch(report, /\]\(not-a-trusted-url\)/);
+});
+
+test('rejects an unbounded frontend result list as unavailable', (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'moli-ci-report-limit-'));
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+ const frontendRoot = path.join(root, 'frontend');
+ writeJson(frontendRoot, 'summary.json', {
+ ok: false,
+ counts: { infrastructure_error: MAX_RESULTS_FOR_TEST },
+ results: Array.from({ length: MAX_RESULTS_FOR_TEST }, (_, index) => ({
+ id: `case-${index}`,
+ status: 'infrastructure_error',
+ })),
+ });
+
+ const report = renderReport({ frontendRoot, conclusion: 'failure' });
+ assert.match(report, /artifacts: `0\/5`/);
+});
diff --git a/.github/scripts/wait-for-workflow-artifact.cjs b/.github/scripts/wait-for-workflow-artifact.cjs
index 6704a53d1..1d0a0109a 100644
--- a/.github/scripts/wait-for-workflow-artifact.cjs
+++ b/.github/scripts/wait-for-workflow-artifact.cjs
@@ -2,9 +2,30 @@
const DEFAULT_POLL_INTERVAL_MS = 30_000;
const DEFAULT_TIMEOUT_MS = 110 * 60_000;
+const MAX_ARTIFACT_NAMES = 20;
-async function pollWorkflowArtifact({
- artifactName,
+function normalizeArtifactNames(artifactNames) {
+ if (!Array.isArray(artifactNames)) {
+ throw new Error('artifactNames must be an array');
+ }
+ const names = [...new Set(artifactNames)];
+ if (
+ names.length === 0 ||
+ names.length > MAX_ARTIFACT_NAMES ||
+ names.some(
+ (name) =>
+ typeof name !== 'string' ||
+ name.length === 0 ||
+ name.length > 128
+ )
+ ) {
+ throw new Error(`artifactNames must contain 1-${MAX_ARTIFACT_NAMES} bounded names`);
+ }
+ return names;
+}
+
+async function pollWorkflowArtifacts({
+ artifactNames,
listArtifacts,
getWorkflowRun,
now = Date.now,
@@ -12,16 +33,22 @@ async function pollWorkflowArtifact({
pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
timeoutMs = DEFAULT_TIMEOUT_MS,
}) {
+ const names = normalizeArtifactNames(artifactNames);
const deadline = now() + timeoutMs;
while (true) {
const artifacts = await listArtifacts();
- const artifact = artifacts.find(
- (candidate) => candidate.name === artifactName && candidate.expired !== true
+ const availableNames = new Set(
+ artifacts
+ .filter((artifact) => artifact.expired !== true)
+ .map((artifact) => artifact.name)
);
- if (artifact) {
+ const availableArtifacts = names.filter((name) => availableNames.has(name));
+ const missingArtifacts = names.filter((name) => !availableNames.has(name));
+ if (missingArtifacts.length === 0) {
return {
- artifactAvailable: true,
+ availableArtifacts,
+ missingArtifacts,
conclusion: '',
};
}
@@ -29,7 +56,8 @@ async function pollWorkflowArtifact({
const workflowRun = await getWorkflowRun();
if (workflowRun.status === 'completed') {
return {
- artifactAvailable: false,
+ availableArtifacts,
+ missingArtifacts,
conclusion: workflowRun.conclusion || 'failure',
};
}
@@ -37,7 +65,8 @@ async function pollWorkflowArtifact({
const remainingMs = deadline - now();
if (remainingMs <= 0) {
return {
- artifactAvailable: false,
+ availableArtifacts,
+ missingArtifacts,
conclusion: 'timed_out',
};
}
@@ -45,6 +74,30 @@ async function pollWorkflowArtifact({
}
}
+async function pollWorkflowArtifact({
+ artifactName,
+ listArtifacts,
+ getWorkflowRun,
+ now = Date.now,
+ sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)),
+ pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
+ timeoutMs = DEFAULT_TIMEOUT_MS,
+}) {
+ const result = await pollWorkflowArtifacts({
+ artifactNames: [artifactName],
+ listArtifacts,
+ getWorkflowRun,
+ now,
+ sleep,
+ pollIntervalMs,
+ timeoutMs,
+ });
+ return {
+ artifactAvailable: result.availableArtifacts.includes(artifactName),
+ conclusion: result.conclusion,
+ };
+}
+
async function waitForWorkflowArtifact({
github,
context,
@@ -97,5 +150,58 @@ async function waitForWorkflowArtifact({
}
}
+async function waitForWorkflowArtifacts({
+ github,
+ context,
+ core,
+ artifactNames,
+ pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
+ timeoutMs = DEFAULT_TIMEOUT_MS,
+}) {
+ const run = context.payload.workflow_run;
+ if (!run?.id) {
+ throw new Error('workflow_run.id is required');
+ }
+ const names = normalizeArtifactNames(artifactNames);
+ const owner = context.repo.owner;
+ const repo = context.repo.repo;
+ const runId = run.id;
+ const result = await pollWorkflowArtifacts({
+ artifactNames: names,
+ pollIntervalMs,
+ timeoutMs,
+ listArtifacts: async () => {
+ const response = await github.rest.actions.listWorkflowRunArtifacts({
+ owner,
+ repo,
+ run_id: runId,
+ per_page: 100,
+ });
+ return response.data.artifacts;
+ },
+ getWorkflowRun: async () => {
+ const response = await github.rest.actions.getWorkflowRun({
+ owner,
+ repo,
+ run_id: runId,
+ });
+ return response.data;
+ },
+ });
+
+ core.setOutput('available_artifacts', JSON.stringify(result.availableArtifacts));
+ core.setOutput('missing_artifacts', JSON.stringify(result.missingArtifacts));
+ core.setOutput('all_available', String(result.missingArtifacts.length === 0));
+ core.setOutput('conclusion', result.conclusion);
+
+ if (result.missingArtifacts.length !== 0) {
+ core.notice(
+ `Workflow run ${runId} finished or the wait timed out before these artifacts were uploaded: ${result.missingArtifacts.join(', ')}.`
+ );
+ }
+}
+
module.exports = waitForWorkflowArtifact;
module.exports.pollWorkflowArtifact = pollWorkflowArtifact;
+module.exports.pollWorkflowArtifacts = pollWorkflowArtifacts;
+module.exports.waitForWorkflowArtifacts = waitForWorkflowArtifacts;
diff --git a/.github/scripts/wait-for-workflow-artifact.test.cjs b/.github/scripts/wait-for-workflow-artifact.test.cjs
index ce640ee51..408095f7b 100644
--- a/.github/scripts/wait-for-workflow-artifact.test.cjs
+++ b/.github/scripts/wait-for-workflow-artifact.test.cjs
@@ -5,6 +5,8 @@ const test = require('node:test');
const {
pollWorkflowArtifact,
+ pollWorkflowArtifacts,
+ waitForWorkflowArtifacts,
} = require('./wait-for-workflow-artifact.cjs');
test('returns as soon as the requested artifact is available', async () => {
@@ -73,3 +75,84 @@ test('ignores expired artifacts and eventually times out', async () => {
assert.deepEqual(result, { artifactAvailable: false, conclusion: 'timed_out' });
});
+
+test('waits until every requested artifact is available', async () => {
+ let artifactLookups = 0;
+ let sleeps = 0;
+ const result = await pollWorkflowArtifacts({
+ artifactNames: ['release-results', 'frontend-results'],
+ listArtifacts: async () => {
+ artifactLookups += 1;
+ return artifactLookups === 1
+ ? [{ name: 'release-results', expired: false }]
+ : [
+ { name: 'release-results', expired: false },
+ { name: 'frontend-results', expired: false },
+ ];
+ },
+ getWorkflowRun: async () => ({ status: 'in_progress', conclusion: null }),
+ now: () => 0,
+ sleep: async () => {
+ sleeps += 1;
+ },
+ });
+
+ assert.deepEqual(result, {
+ availableArtifacts: ['release-results', 'frontend-results'],
+ missingArtifacts: [],
+ conclusion: '',
+ });
+ assert.equal(sleeps, 1);
+});
+
+test('retains partial artifact availability when the workflow completes', async () => {
+ const result = await pollWorkflowArtifacts({
+ artifactNames: ['release-results', 'frontend-results', 'cdp-results'],
+ listArtifacts: async () => [
+ { name: 'release-results', expired: false },
+ { name: 'frontend-results', expired: true },
+ ],
+ getWorkflowRun: async () => ({ status: 'completed', conclusion: 'failure' }),
+ });
+
+ assert.deepEqual(result, {
+ availableArtifacts: ['release-results'],
+ missingArtifacts: ['frontend-results', 'cdp-results'],
+ conclusion: 'failure',
+ });
+});
+
+test('publishes bounded aggregate artifact outputs', async () => {
+ const outputs = new Map();
+ const notices = [];
+ await waitForWorkflowArtifacts({
+ github: {
+ rest: {
+ actions: {
+ listWorkflowRunArtifacts: async () => ({
+ data: { artifacts: [{ name: 'release-results', expired: false }] },
+ }),
+ getWorkflowRun: async () => ({
+ data: { status: 'completed', conclusion: 'success' },
+ }),
+ },
+ },
+ },
+ context: {
+ payload: { workflow_run: { id: 123 } },
+ repo: { owner: 'lexmount', repo: 'moli' },
+ },
+ core: {
+ setOutput: (name, value) => outputs.set(name, value),
+ notice: (message) => notices.push(message),
+ },
+ artifactNames: ['release-results', 'frontend-results'],
+ });
+
+ assert.equal(outputs.get('available_artifacts'), '["release-results"]');
+ assert.equal(outputs.get('missing_artifacts'), '["frontend-results"]');
+ assert.equal(outputs.get('all_available'), 'false');
+ assert.equal(outputs.get('conclusion'), 'success');
+ assert.equal(notices.length, 1);
+ assert.match(notices[0], /frontend-results/);
+});
diff --git a/.github/workflows/ci-regression-comment.yml b/.github/workflows/ci-regression-comment.yml
new file mode 100644
index 000000000..6f96690b6
--- /dev/null
+++ b/.github/workflows/ci-regression-comment.yml
@@ -0,0 +1,184 @@
+name: CI Regression Report
+
+on:
+ workflow_run:
+ workflows: [CI]
+ types: [in_progress]
+
+concurrency:
+ group: ci-regression-report-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.id }}
+ cancel-in-progress: true
+
+# This trusted default-branch workflow never executes code from the PR. It waits
+# for immutable v4 artifacts, parses only bounded fields, and upserts one report.
+permissions:
+ actions: read
+ contents: read
+ pull-requests: write
+
+jobs:
+ comment:
+ name: Publish aggregate regression report
+ if: >-
+ github.event.workflow_run.event == 'pull_request' &&
+ github.event.workflow_run.pull_requests[0].head.repo.id == github.event.repository.id &&
+ github.event.workflow_run.actor.login != 'dependabot[bot]'
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ steps:
+ - name: Checkout trusted report renderer
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ ref: ${{ github.event.repository.default_branch }}
+
+ - name: Wait for aggregate evidence
+ id: wait
+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+ env:
+ ARTIFACT_NAMES: >-
+ ["release-regression-results","frontend-differential-results","agent-episode-results","runtime-contract-results","cdp-smoke-diagnostics"]
+ with:
+ retries: 3
+ script: |
+ const helper = require('./.github/scripts/wait-for-workflow-artifact.cjs');
+ await helper.waitForWorkflowArtifacts({
+ github,
+ context,
+ core,
+ artifactNames: JSON.parse(process.env.ARTIFACT_NAMES),
+ });
+
+ - name: Download release regression evidence
+ if: contains(fromJSON(steps.wait.outputs.available_artifacts), 'release-regression-results')
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: release-regression-results
+ path: ci-regression-artifacts/release
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ run-id: ${{ github.event.workflow_run.id }}
+
+ - name: Download frontend differential evidence
+ if: contains(fromJSON(steps.wait.outputs.available_artifacts), 'frontend-differential-results')
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: frontend-differential-results
+ path: ci-regression-artifacts/frontend
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ run-id: ${{ github.event.workflow_run.id }}
+
+ - name: Download agent episode evidence
+ if: contains(fromJSON(steps.wait.outputs.available_artifacts), 'agent-episode-results')
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: agent-episode-results
+ path: ci-regression-artifacts/agent
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ run-id: ${{ github.event.workflow_run.id }}
+
+ - name: Download runtime contract evidence
+ if: contains(fromJSON(steps.wait.outputs.available_artifacts), 'runtime-contract-results')
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: runtime-contract-results
+ path: ci-regression-artifacts/runtime
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ run-id: ${{ github.event.workflow_run.id }}
+
+ - name: Download CDP smoke evidence
+ if: contains(fromJSON(steps.wait.outputs.available_artifacts), 'cdp-smoke-diagnostics')
+ continue-on-error: true
+ uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: cdp-smoke-diagnostics
+ path: ci-regression-artifacts/cdp
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ run-id: ${{ github.event.workflow_run.id }}
+
+ - name: Install Node.js
+ if: steps.wait.outputs.conclusion != 'cancelled'
+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+ with:
+ node-version: 24.15.0
+
+ - name: Render trusted aggregate report
+ if: steps.wait.outputs.conclusion != 'cancelled'
+ env:
+ RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}
+ CONCLUSION: ${{ steps.wait.outputs.conclusion || 'in_progress' }}
+ run: >-
+ node .github/scripts/render-ci-regression-comment.cjs
+ --release ci-regression-artifacts/release
+ --frontend ci-regression-artifacts/frontend
+ --agent ci-regression-artifacts/agent
+ --runtime ci-regression-artifacts/runtime
+ --cdp ci-regression-artifacts/cdp
+ --run-url "$RUN_URL"
+ --conclusion "$CONCLUSION"
+ --output ci-regression-comment.md
+
+ - name: Publish Actions report
+ if: steps.wait.outputs.conclusion != 'cancelled'
+ run: cat ci-regression-comment.md >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Post PR regression comment
+ if: steps.wait.outputs.conclusion != 'cancelled'
+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+ env:
+ COMMENT_PATH: ci-regression-comment.md
+ with:
+ script: |
+ const fs = require('fs');
+ const run = context.payload.workflow_run;
+ const association = run.pull_requests?.[0];
+ if (!association) {
+ core.notice('No pull request is associated with this workflow run.');
+ return;
+ }
+
+ const pull = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: association.number,
+ });
+ if (pull.data.head.sha !== association.head.sha) {
+ core.notice('Skipping a stale CI regression report for an older PR head.');
+ return;
+ }
+
+ const body = fs.readFileSync(process.env.COMMENT_PATH, 'utf8');
+ const marker = '<!-- moli-ci-regression-report -->';
+ if (!body.includes(marker) || Buffer.byteLength(body, 'utf8') > 32 * 1024) {
+ core.setFailed('Rendered CI regression report is missing its marker or exceeds 32 KiB.');
+ return;
+ }
+ const comments = await github.paginate(github.rest.issues.listComments, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: association.number,
+ per_page: 100,
+ });
+ const existing = comments.find(
+ (comment) =>
+ comment.user?.login === 'github-actions[bot]' &&
+ comment.body?.includes(marker)
+ );
+ if (existing) {
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: existing.id,
+ body,
+ });
+ } else {
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: association.number,
+ body,
+ });
+ }
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a8a812451..5bd7cfc65 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -222,7 +222,7 @@ jobs:
working-directory: moli-benchmark/browser-spider-local
run: npm test
- name: Test trusted report helpers
- run: node --test .github/scripts/wait-for-workflow-artifact.test.cjs
+ run: node --test .github/scripts/*.test.cjs
build-release-head:
name: Build release · HEAD