Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
ci: fail spider bench on browser service errors
ldm0179e363b8b85
diff --git a/moli-benchmark/browser-spider-local/bench.mjs b/moli-benchmark/browser-spider-local/bench.mjsindex 41485d963..6a7c4f7f9 100644--- a/moli-benchmark/browser-spider-local/bench.mjs+++ b/moli-benchmark/browser-spider-local/bench.mjs@@ -528,8 +528,9 @@ Options:--moli-bin PATH moli binary override.--chrome-bin PATH Chromium/Chrome binary override.--assert-no-stale-page-leakage- Exit non-zero when a service run fails or timed-out navigations- / mismatched item links indicate stale previous-page scraping.+ Exit non-zero when timed-out navigations / mismatched item links+ indicate stale previous-page scraping. Service failures always+ exit non-zero after writing benchmark evidence.--sample-interval-ms N Process-tree CPU/RSS/PSS sample interval (minimum 100ms).--no-resource-sampling Disable resource sampling; the HTML report is still generated.`);}@@ -1705,11 +1706,11 @@ async function main() {writeJson(path.join(runDir, 'summary.json'), summary);console.log(JSON.stringify(summary, null, 2));+ const serviceFailures = serviceRunFailures(allResults);+ if (serviceFailures.length > 0) {+ throw new Error(`service run failed: ${JSON.stringify(serviceFailures)}`);+ }if (args.assertNoStalePageLeakage) {- const serviceFailures = serviceRunFailures(allResults);- if (serviceFailures.length > 0) {- throw new Error(`service run failed before stale-page assertion: ${JSON.stringify(serviceFailures)}`);- }const failures = stalePageLeakageFailures(allResults);if (failures.length > 0) {throw new Error(`stale page leakage detected: ${JSON.stringify(failures)}`);diff --git a/moli-benchmark/browser-spider-local/compare.mjs b/moli-benchmark/browser-spider-local/compare.mjsindex 5788769e8..14f351a08 100644--- a/moli-benchmark/browser-spider-local/compare.mjs+++ b/moli-benchmark/browser-spider-local/compare.mjs@@ -21,7 +21,7 @@ const SUITE_CONFIGS = {commandTimeout: '12m',gotoTimeoutMs: 1500,sampleIntervalMs: 100,- extraArgs: ['--assert-no-stale-page-leakage', '--server-hang-ms', '3000']+ extraArgs: ['--server-hang-ms', '3000']},real48: {classification: 'informational',@@ -348,6 +348,52 @@ export function buildSpiderComparison({ baseSha, headSha, executionOrder, runs }};}+export function spiderBenchmarkExecutionFailures(comparison) {+ const failures = [];+ for (const suiteName of Object.keys(SUITE_CONFIGS)) {+ const suite = comparison?.suites?.[suiteName];+ for (const side of ['base', 'head']) {+ const run = suite?.[side];+ if (run?.availability !== 'available') {+ failures.push({ suite: suiteName, side, reason: 'report-unavailable' });+ continue;+ }+ const serviceRuns = run.metrics?.service_runs;+ const successfulRuns = run.metrics?.successful_runs;+ const failedRuns = run.metrics?.failed_runs;+ if (+ !Number.isInteger(serviceRuns)+ || serviceRuns < 1+ || !Number.isInteger(successfulRuns)+ || !Number.isInteger(failedRuns)+ || successfulRuns + failedRuns !== serviceRuns+ ) {+ failures.push({ suite: suiteName, side, reason: 'invalid-service-summary' });+ continue;+ }+ if (failedRuns > 0) {+ failures.push({+ suite: suiteName,+ side,+ reason: 'service-run-failed',+ failed_runs: failedRuns,+ service_runs: serviceRuns+ });+ continue;+ }+ if (run.exit_code !== 0) {+ failures.push({+ suite: suiteName,+ side,+ reason: 'benchmark-command-failed',+ exit_code: run.exit_code+ });+ }+ }+ }+ return failures;+}+function safeSha(value) {return /^[0-9a-f]{7,40}$/i.test(value ?? '') ? value.toLowerCase() : 'unknown';}@@ -508,6 +554,18 @@ function assessmentSummary(assessment) {return `⚠️ Deterministic fixture recorded diagnostic issues (non-blocking):\n${reasons}`;}+function executionSummary(comparison) {+ const failures = spiderBenchmarkExecutionFailures(comparison);+ if (failures.length === 0) {+ return '✅ All benchmark browser service runs completed.';+ }+ const labels = failures+ .slice(0, 8)+ .map((failure) => `${failure.suite}/${failure.side}: ${failure.reason}`)+ .join(', ');+ return `❌ Benchmark execution was incomplete (${labels}). The workflow fails while preserving the collected evidence.`;+}+function safeRunUrl(value) {try {const url = new URL(value);@@ -561,6 +619,8 @@ export function renderSpiderComparison(comparison, { runUrl, conclusion }) {SPIDER_CI_COMMENT_MARKER,'## Spider Bench A/B','',+ executionSummary(comparison),+ '',publicHeadline(real48),'',assessmentSummary(fixture.assessment),@@ -584,7 +644,7 @@ export function renderSpiderComparison(comparison, { runUrl, conclusion }) {'',`Full HTML, JSON, CSV, logs, and page snapshots: ${workflowLink}.`,'',- '_Spider Bench is observational: fixture diagnostics and performance deltas never fail the PR. Investigate changes using the complete artifact and controlled repeated runs._'+ '_Spider Bench gates execution integrity: missing reports and failed browser service runs fail the workflow. Site content, fixture diagnostics, timing, and performance deltas remain observational._'].join('\n');}@@ -683,11 +743,17 @@ async function runComparisonCommand(args) {writeJson(path.join(outputDir, 'comparison.json'), comparison);fs.writeFileSync(path.join(outputDir, 'comment.md'), `${renderSpiderComparison(comparison, {runUrl: args['run-url'],- conclusion: args['run-url'] ? 'success' : 'local'+ conclusion: args['run-url']+ ? (spiderBenchmarkExecutionFailures(comparison).length === 0 ? 'success' : 'failure')+ : 'local'})}\n`, 'utf8');if (!comparison.suites.fixture.assessment.clean) {console.warn(`deterministic fixture recorded diagnostic issues: ${comparison.suites.fixture.assessment.reasons.join(', ')}`);}+ const executionFailures = spiderBenchmarkExecutionFailures(comparison);+ if (executionFailures.length > 0) {+ throw new Error(`Spider Bench execution failed: ${JSON.stringify(executionFailures)}`);+ }}function renderCommand(args) {diff --git a/moli-benchmark/browser-spider-local/tests/compare.test.mjs b/moli-benchmark/browser-spider-local/tests/compare.test.mjsindex 7a6230123..2d6a2bbf5 100644--- a/moli-benchmark/browser-spider-local/tests/compare.test.mjs+++ b/moli-benchmark/browser-spider-local/tests/compare.test.mjs@@ -11,6 +11,7 @@ import {readSpiderBenchmark,renderSpiderComparison,renderSpiderInfrastructureFailure,+ spiderBenchmarkExecutionFailures,SPIDER_CI_COMMENT_MARKER,SPIDER_COMPARISON_SCHEMA} from '../compare.mjs';@@ -290,7 +291,7 @@ test('missing output becomes unavailable instead of throwing', () => {assert.match(missing.error, /found 0/);});-test('comment labels every result as non-blocking evidence', () => {+test('comment separates execution integrity from observational evidence', () => {const comparison = buildComparison(makeFixtureRun({}),makeFixtureRun({ duration_ms: 82_000, peak_pss_bytes: 448 * 1024 * 1024 }),@@ -303,6 +304,7 @@ test('comment labels every result as non-blocking evidence', () => {});assert.ok(comment.startsWith(SPIDER_CI_COMMENT_MARKER));+ assert.match(comment, /All benchmark browser service runs completed/);assert.match(comment, /Deterministic fixture diagnostic is clean/);assert.match(comment, /Deterministic fixture · diagnostic/);assert.match(comment, /Public 48-site run · informational/);@@ -321,7 +323,8 @@ test('comment labels every result as non-blocking evidence', () => {assert.match(comment, /\| sports \| 27 \/ 40 \| 27 \/ 40 \| 0 \|/);assert.match(comment, /\+3\.00 MiB \(\+0\.67%\)/);assert.match(comment, /actions\/runs\/123/);- assert.match(comment, /never fail the PR/);+ assert.match(comment, /missing reports and failed browser service runs fail the workflow/);+ assert.match(comment, /performance deltas remain observational/);assert.match(comment, /Common ancestor `111111111111` → HEAD `222222222222`/);assert.match(comment, /### CPU and memory timelines/);assert.match(comment, /bounded 40-point views of the same complete process-tree samples/);@@ -456,7 +459,54 @@ test('Moli endpoint discovery accepts ANSI-formatted tracing output', () => {assert.equal(endpointFromMoliLogs([`stderr: ${log}`]), 'http://127.0.0.1:45445');});-test('run command records failed browser runs without failing the observer', () => {+test('execution gate ignores content diagnostics but rejects failed browser services', () => {+ const diagnosticOnly = buildComparison(+ makeFixtureRun({}),+ makeFixtureRun({+ actual_rows: 39,+ sites_meeting_row_contract: 14,+ mismatched_item_sites: 1+ }),+ makeRun({}),+ makeRun({})+ );+ assert.equal(diagnosticOnly.suites.fixture.assessment.clean, false);+ assert.deepEqual(spiderBenchmarkExecutionFailures(diagnosticOnly), []);++ const failedService = buildComparison(+ makeFixtureRun({}),+ makeFixtureRun({}),+ makeRun({}),+ makeRun({ successful_runs: 0, failed_runs: 1 })+ );+ assert.deepEqual(spiderBenchmarkExecutionFailures(failedService), [{+ suite: 'real48',+ side: 'head',+ reason: 'service-run-failed',+ failed_runs: 1,+ service_runs: 1+ }]);++ const missingReport = buildComparison(+ makeFixtureRun({}),+ makeFixtureRun({}),+ makeRun({}),+ {+ availability: 'unavailable',+ exit_code: 125,+ error: 'report missing',+ metrics: null,+ resource_timeline: null+ }+ );+ assert.deepEqual(spiderBenchmarkExecutionFailures(missingReport), [{+ suite: 'real48',+ side: 'head',+ reason: 'report-unavailable'+ }]);+});++test('run command preserves failed browser evidence and exits non-zero', () => {const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-observer-'));try {const result = spawnSync(process.execPath, [@@ -470,11 +520,13 @@ test('run command records failed browser runs without failing the observer', ()'--output-dir', tempDir], { encoding: 'utf8' });- assert.equal(result.status, 0, result.stderr);+ assert.equal(result.status, 1, result.stderr);const comparison = JSON.parse(fs.readFileSync(path.join(tempDir, 'comparison.json'), 'utf8'));assert.equal(comparison.suites.fixture.assessment.clean, false);assert.ok(comparison.suites.fixture.assessment.reasons.includes('head-benchmark-failed'));+ assert.ok(fs.existsSync(path.join(tempDir, 'comment.md')));assert.match(result.stderr, /recorded diagnostic issues/);+ assert.match(result.stderr, /Spider Bench execution failed/);} finally {fs.rmSync(tempDir, { recursive: true, force: true });}