Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

ci: fail spider bench on browser service errors

ldm0179e363b8b85
diff --git a/moli-benchmark/browser-spider-local/bench.mjs b/moli-benchmark/browser-spider-local/bench.mjs
index 41485d963..6a7c4f7f9 100644
--- a/moli-benchmark/browser-spider-local/bench.mjs
+++ b/moli-benchmark/browser-spider-local/bench.mjs
@@ -528,8 +528,9 @@ Options:
--moli-bin PATH moli binary override.
--chrome-bin PATH Chromium/Chrome binary override.
--assert-no-stale-page-leakage
- Exit non-zero when a service run fails or timed-out navigations
- / mismatched item links indicate stale previous-page scraping.
+ Exit non-zero when timed-out navigations / mismatched item links
+ indicate stale previous-page scraping. Service failures always
+ exit non-zero after writing benchmark evidence.
--sample-interval-ms N Process-tree CPU/RSS/PSS sample interval (minimum 100ms).
--no-resource-sampling Disable resource sampling; the HTML report is still generated.`);
}
@@ -1705,11 +1706,11 @@ async function main() {
writeJson(path.join(runDir, 'summary.json'), summary);
console.log(JSON.stringify(summary, null, 2));
+ const serviceFailures = serviceRunFailures(allResults);
+ if (serviceFailures.length > 0) {
+ throw new Error(`service run failed: ${JSON.stringify(serviceFailures)}`);
+ }
if (args.assertNoStalePageLeakage) {
- const serviceFailures = serviceRunFailures(allResults);
- if (serviceFailures.length > 0) {
- throw new Error(`service run failed before stale-page assertion: ${JSON.stringify(serviceFailures)}`);
- }
const failures = stalePageLeakageFailures(allResults);
if (failures.length > 0) {
throw new Error(`stale page leakage detected: ${JSON.stringify(failures)}`);
diff --git a/moli-benchmark/browser-spider-local/compare.mjs b/moli-benchmark/browser-spider-local/compare.mjs
index 5788769e8..14f351a08 100644
--- a/moli-benchmark/browser-spider-local/compare.mjs
+++ b/moli-benchmark/browser-spider-local/compare.mjs
@@ -21,7 +21,7 @@ const SUITE_CONFIGS = {
commandTimeout: '12m',
gotoTimeoutMs: 1500,
sampleIntervalMs: 100,
- extraArgs: ['--assert-no-stale-page-leakage', '--server-hang-ms', '3000']
+ extraArgs: ['--server-hang-ms', '3000']
},
real48: {
classification: 'informational',
@@ -348,6 +348,52 @@ export function buildSpiderComparison({ baseSha, headSha, executionOrder, runs }
};
}
+export function spiderBenchmarkExecutionFailures(comparison) {
+ const failures = [];
+ for (const suiteName of Object.keys(SUITE_CONFIGS)) {
+ const suite = comparison?.suites?.[suiteName];
+ for (const side of ['base', 'head']) {
+ const run = suite?.[side];
+ if (run?.availability !== 'available') {
+ failures.push({ suite: suiteName, side, reason: 'report-unavailable' });
+ continue;
+ }
+ const serviceRuns = run.metrics?.service_runs;
+ const successfulRuns = run.metrics?.successful_runs;
+ const failedRuns = run.metrics?.failed_runs;
+ if (
+ !Number.isInteger(serviceRuns)
+ || serviceRuns < 1
+ || !Number.isInteger(successfulRuns)
+ || !Number.isInteger(failedRuns)
+ || successfulRuns + failedRuns !== serviceRuns
+ ) {
+ failures.push({ suite: suiteName, side, reason: 'invalid-service-summary' });
+ continue;
+ }
+ if (failedRuns > 0) {
+ failures.push({
+ suite: suiteName,
+ side,
+ reason: 'service-run-failed',
+ failed_runs: failedRuns,
+ service_runs: serviceRuns
+ });
+ continue;
+ }
+ if (run.exit_code !== 0) {
+ failures.push({
+ suite: suiteName,
+ side,
+ reason: 'benchmark-command-failed',
+ exit_code: run.exit_code
+ });
+ }
+ }
+ }
+ return failures;
+}
+
function safeSha(value) {
return /^[0-9a-f]{7,40}$/i.test(value ?? '') ? value.toLowerCase() : 'unknown';
}
@@ -508,6 +554,18 @@ function assessmentSummary(assessment) {
return `⚠️ Deterministic fixture recorded diagnostic issues (non-blocking):\n${reasons}`;
}
+function executionSummary(comparison) {
+ const failures = spiderBenchmarkExecutionFailures(comparison);
+ if (failures.length === 0) {
+ return '✅ All benchmark browser service runs completed.';
+ }
+ const labels = failures
+ .slice(0, 8)
+ .map((failure) => `${failure.suite}/${failure.side}: ${failure.reason}`)
+ .join(', ');
+ return `❌ Benchmark execution was incomplete (${labels}). The workflow fails while preserving the collected evidence.`;
+}
+
function safeRunUrl(value) {
try {
const url = new URL(value);
@@ -561,6 +619,8 @@ export function renderSpiderComparison(comparison, { runUrl, conclusion }) {
SPIDER_CI_COMMENT_MARKER,
'## Spider Bench A/B',
'',
+ executionSummary(comparison),
+ '',
publicHeadline(real48),
'',
assessmentSummary(fixture.assessment),
@@ -584,7 +644,7 @@ export function renderSpiderComparison(comparison, { runUrl, conclusion }) {
'',
`Full HTML, JSON, CSV, logs, and page snapshots: ${workflowLink}.`,
'',
- '_Spider Bench is observational: fixture diagnostics and performance deltas never fail the PR. Investigate changes using the complete artifact and controlled repeated runs._'
+ '_Spider Bench gates execution integrity: missing reports and failed browser service runs fail the workflow. Site content, fixture diagnostics, timing, and performance deltas remain observational._'
].join('\n');
}
@@ -683,11 +743,17 @@ async function runComparisonCommand(args) {
writeJson(path.join(outputDir, 'comparison.json'), comparison);
fs.writeFileSync(path.join(outputDir, 'comment.md'), `${renderSpiderComparison(comparison, {
runUrl: args['run-url'],
- conclusion: args['run-url'] ? 'success' : 'local'
+ conclusion: args['run-url']
+ ? (spiderBenchmarkExecutionFailures(comparison).length === 0 ? 'success' : 'failure')
+ : 'local'
})}\n`, 'utf8');
if (!comparison.suites.fixture.assessment.clean) {
console.warn(`deterministic fixture recorded diagnostic issues: ${comparison.suites.fixture.assessment.reasons.join(', ')}`);
}
+ const executionFailures = spiderBenchmarkExecutionFailures(comparison);
+ if (executionFailures.length > 0) {
+ throw new Error(`Spider Bench execution failed: ${JSON.stringify(executionFailures)}`);
+ }
}
function renderCommand(args) {
diff --git a/moli-benchmark/browser-spider-local/tests/compare.test.mjs b/moli-benchmark/browser-spider-local/tests/compare.test.mjs
index 7a6230123..2d6a2bbf5 100644
--- a/moli-benchmark/browser-spider-local/tests/compare.test.mjs
+++ b/moli-benchmark/browser-spider-local/tests/compare.test.mjs
@@ -11,6 +11,7 @@ import {
readSpiderBenchmark,
renderSpiderComparison,
renderSpiderInfrastructureFailure,
+ spiderBenchmarkExecutionFailures,
SPIDER_CI_COMMENT_MARKER,
SPIDER_COMPARISON_SCHEMA
} from '../compare.mjs';
@@ -290,7 +291,7 @@ test('missing output becomes unavailable instead of throwing', () => {
assert.match(missing.error, /found 0/);
});
-test('comment labels every result as non-blocking evidence', () => {
+test('comment separates execution integrity from observational evidence', () => {
const comparison = buildComparison(
makeFixtureRun({}),
makeFixtureRun({ duration_ms: 82_000, peak_pss_bytes: 448 * 1024 * 1024 }),
@@ -303,6 +304,7 @@ test('comment labels every result as non-blocking evidence', () => {
});
assert.ok(comment.startsWith(SPIDER_CI_COMMENT_MARKER));
+ assert.match(comment, /All benchmark browser service runs completed/);
assert.match(comment, /Deterministic fixture diagnostic is clean/);
assert.match(comment, /Deterministic fixture · diagnostic/);
assert.match(comment, /Public 48-site run · informational/);
@@ -321,7 +323,8 @@ test('comment labels every result as non-blocking evidence', () => {
assert.match(comment, /\| sports \| 27 \/ 40 \| 27 \/ 40 \| 0 \|/);
assert.match(comment, /\+3\.00 MiB \(\+0\.67%\)/);
assert.match(comment, /actions\/runs\/123/);
- assert.match(comment, /never fail the PR/);
+ assert.match(comment, /missing reports and failed browser service runs fail the workflow/);
+ assert.match(comment, /performance deltas remain observational/);
assert.match(comment, /Common ancestor `111111111111` → HEAD `222222222222`/);
assert.match(comment, /### CPU and memory timelines/);
assert.match(comment, /bounded 40-point views of the same complete process-tree samples/);
@@ -456,7 +459,54 @@ test('Moli endpoint discovery accepts ANSI-formatted tracing output', () => {
assert.equal(endpointFromMoliLogs([`stderr: ${log}`]), 'http://127.0.0.1:45445');
});
-test('run command records failed browser runs without failing the observer', () => {
+test('execution gate ignores content diagnostics but rejects failed browser services', () => {
+ const diagnosticOnly = buildComparison(
+ makeFixtureRun({}),
+ makeFixtureRun({
+ actual_rows: 39,
+ sites_meeting_row_contract: 14,
+ mismatched_item_sites: 1
+ }),
+ makeRun({}),
+ makeRun({})
+ );
+ assert.equal(diagnosticOnly.suites.fixture.assessment.clean, false);
+ assert.deepEqual(spiderBenchmarkExecutionFailures(diagnosticOnly), []);
+
+ const failedService = buildComparison(
+ makeFixtureRun({}),
+ makeFixtureRun({}),
+ makeRun({}),
+ makeRun({ successful_runs: 0, failed_runs: 1 })
+ );
+ assert.deepEqual(spiderBenchmarkExecutionFailures(failedService), [{
+ suite: 'real48',
+ side: 'head',
+ reason: 'service-run-failed',
+ failed_runs: 1,
+ service_runs: 1
+ }]);
+
+ const missingReport = buildComparison(
+ makeFixtureRun({}),
+ makeFixtureRun({}),
+ makeRun({}),
+ {
+ availability: 'unavailable',
+ exit_code: 125,
+ error: 'report missing',
+ metrics: null,
+ resource_timeline: null
+ }
+ );
+ assert.deepEqual(spiderBenchmarkExecutionFailures(missingReport), [{
+ suite: 'real48',
+ side: 'head',
+ reason: 'report-unavailable'
+ }]);
+});
+
+test('run command preserves failed browser evidence and exits non-zero', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-observer-'));
try {
const result = spawnSync(process.execPath, [
@@ -470,11 +520,13 @@ test('run command records failed browser runs without failing the observer', ()
'--output-dir', tempDir
], { encoding: 'utf8' });
- assert.equal(result.status, 0, result.stderr);
+ assert.equal(result.status, 1, result.stderr);
const comparison = JSON.parse(fs.readFileSync(path.join(tempDir, 'comparison.json'), 'utf8'));
assert.equal(comparison.suites.fixture.assessment.clean, false);
assert.ok(comparison.suites.fixture.assessment.reasons.includes('head-benchmark-failed'));
+ assert.ok(fs.existsSync(path.join(tempDir, 'comment.md')));
assert.match(result.stderr, /recorded diagnostic issues/);
+ assert.match(result.stderr, /Spider Bench execution failed/);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}