Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
ci: keep spider bench informational
ldm0fa26a909da09
diff --git a/moli-benchmark/browser-spider-local/bench.mjs b/moli-benchmark/browser-spider-local/bench.mjsindex 6a7c4f7f9..d5b8c186c 100644--- a/moli-benchmark/browser-spider-local/bench.mjs+++ b/moli-benchmark/browser-spider-local/bench.mjs@@ -528,9 +528,10 @@ Options:--moli-bin PATH moli binary override.--chrome-bin PATH Chromium/Chrome binary override.--assert-no-stale-page-leakage- Exit non-zero when timed-out navigations / mismatched item links- indicate stale previous-page scraping. Service failures always- exit non-zero after writing benchmark evidence.+ Opt in to a local correctness assertion: exit non-zero when a+ service run fails or timed-out navigations / mismatched item links+ indicate stale previous-page scraping. CI does not enable this;+ Spider Bench results are informational.--sample-interval-ms N Process-tree CPU/RSS/PSS sample interval (minimum 100ms).--no-resource-sampling Disable resource sampling; the HTML report is still generated.`);}@@ -1706,11 +1707,11 @@ async function main() {writeJson(path.join(runDir, 'summary.json'), summary);console.log(JSON.stringify(summary, null, 2));- const serviceFailures = serviceRunFailures(allResults);- if (serviceFailures.length > 0) {- throw new Error(`service run failed: ${JSON.stringify(serviceFailures)}`);- }if (args.assertNoStalePageLeakage) {+ const serviceFailures = serviceRunFailures(allResults);+ if (serviceFailures.length > 0) {+ throw new Error(`service run failed before stale-page assertion: ${JSON.stringify(serviceFailures)}`);+ }const failures = stalePageLeakageFailures(allResults);if (failures.length > 0) {throw new Error(`stale page leakage detected: ${JSON.stringify(failures)}`);diff --git a/moli-benchmark/browser-spider-local/compare.mjs b/moli-benchmark/browser-spider-local/compare.mjsindex 14f351a08..99ecdedae 100644--- a/moli-benchmark/browser-spider-local/compare.mjs+++ b/moli-benchmark/browser-spider-local/compare.mjs@@ -348,14 +348,14 @@ export function buildSpiderComparison({ baseSha, headSha, executionOrder, runs }};}-export function spiderBenchmarkExecutionFailures(comparison) {- const failures = [];+export function spiderBenchmarkObservationIssues(comparison) {+ const issues = [];for (const suiteName of Object.keys(SUITE_CONFIGS)) {const suite = comparison?.suites?.[suiteName];for (const side of ['base', 'head']) {const run = suite?.[side];if (run?.availability !== 'available') {- failures.push({ suite: suiteName, side, reason: 'report-unavailable' });+ issues.push({ suite: suiteName, side, reason: 'report-unavailable' });continue;}const serviceRuns = run.metrics?.service_runs;@@ -368,11 +368,11 @@ export function spiderBenchmarkExecutionFailures(comparison) {|| !Number.isInteger(failedRuns)|| successfulRuns + failedRuns !== serviceRuns) {- failures.push({ suite: suiteName, side, reason: 'invalid-service-summary' });+ issues.push({ suite: suiteName, side, reason: 'invalid-service-summary' });continue;}if (failedRuns > 0) {- failures.push({+ issues.push({suite: suiteName,side,reason: 'service-run-failed',@@ -382,7 +382,7 @@ export function spiderBenchmarkExecutionFailures(comparison) {continue;}if (run.exit_code !== 0) {- failures.push({+ issues.push({suite: suiteName,side,reason: 'benchmark-command-failed',@@ -391,7 +391,7 @@ export function spiderBenchmarkExecutionFailures(comparison) {}}}- return failures;+ return issues;}function safeSha(value) {@@ -554,16 +554,16 @@ function assessmentSummary(assessment) {return `⚠️ Deterministic fixture recorded diagnostic issues (non-blocking):\n${reasons}`;}-function executionSummary(comparison) {- const failures = spiderBenchmarkExecutionFailures(comparison);- if (failures.length === 0) {- return '✅ All benchmark browser service runs completed.';+function observationSummary(comparison) {+ const issues = spiderBenchmarkObservationIssues(comparison);+ if (issues.length === 0) {+ return '✅ All benchmark browser service runs completed; results are informational.';}- const labels = failures+ const labels = issues.slice(0, 8)- .map((failure) => `${failure.suite}/${failure.side}: ${failure.reason}`)+ .map((issue) => `${issue.suite}/${issue.side}: ${issue.reason}`).join(', ');- return `❌ Benchmark execution was incomplete (${labels}). The workflow fails while preserving the collected evidence.`;+ return `⚠️ Benchmark observation recorded non-blocking run issues: ${labels}.`;}function safeRunUrl(value) {@@ -619,7 +619,7 @@ export function renderSpiderComparison(comparison, { runUrl, conclusion }) {SPIDER_CI_COMMENT_MARKER,'## Spider Bench A/B','',- executionSummary(comparison),+ observationSummary(comparison),'',publicHeadline(real48),'',@@ -644,7 +644,7 @@ export function renderSpiderComparison(comparison, { runUrl, conclusion }) {'',`Full HTML, JSON, CSV, logs, and page snapshots: ${workflowLink}.`,'',- '_Spider Bench gates execution integrity: missing reports and failed browser service runs fail the workflow. Site content, fixture diagnostics, timing, and performance deltas remain observational._'+ '_Spider Bench is informational: site failures, missing benchmark reports, failed browser service runs, fixture diagnostics, timing, and performance deltas do not fail the workflow. Build, runner, and artifact infrastructure errors can still fail it._'].join('\n');}@@ -743,17 +743,11 @@ async function runComparisonCommand(args) {writeJson(path.join(outputDir, 'comparison.json'), comparison);fs.writeFileSync(path.join(outputDir, 'comment.md'), `${renderSpiderComparison(comparison, {runUrl: args['run-url'],- conclusion: args['run-url']- ? (spiderBenchmarkExecutionFailures(comparison).length === 0 ? 'success' : 'failure')- : 'local'+ conclusion: args['run-url'] ? 'success' : 'local'})}\n`, 'utf8');if (!comparison.suites.fixture.assessment.clean) {console.warn(`deterministic fixture recorded diagnostic issues: ${comparison.suites.fixture.assessment.reasons.join(', ')}`);}- const executionFailures = spiderBenchmarkExecutionFailures(comparison);- if (executionFailures.length > 0) {- throw new Error(`Spider Bench execution failed: ${JSON.stringify(executionFailures)}`);- }}function renderCommand(args) {diff --git a/moli-benchmark/browser-spider-local/tests/compare.test.mjs b/moli-benchmark/browser-spider-local/tests/compare.test.mjsindex 2d6a2bbf5..50283a5c1 100644--- a/moli-benchmark/browser-spider-local/tests/compare.test.mjs+++ b/moli-benchmark/browser-spider-local/tests/compare.test.mjs@@ -11,7 +11,7 @@ import {readSpiderBenchmark,renderSpiderComparison,renderSpiderInfrastructureFailure,- spiderBenchmarkExecutionFailures,+ spiderBenchmarkObservationIssues,SPIDER_CI_COMMENT_MARKER,SPIDER_COMPARISON_SCHEMA} from '../compare.mjs';@@ -291,7 +291,7 @@ test('missing output becomes unavailable instead of throwing', () => {assert.match(missing.error, /found 0/);});-test('comment separates execution integrity from observational evidence', () => {+test('comment labels browser execution and public content as informational evidence', () => {const comparison = buildComparison(makeFixtureRun({}),makeFixtureRun({ duration_ms: 82_000, peak_pss_bytes: 448 * 1024 * 1024 }),@@ -304,7 +304,7 @@ test('comment separates execution integrity from observational evidence', () =>});assert.ok(comment.startsWith(SPIDER_CI_COMMENT_MARKER));- assert.match(comment, /All benchmark browser service runs completed/);+ assert.match(comment, /All benchmark browser service runs completed; results are informational/);assert.match(comment, /Deterministic fixture diagnostic is clean/);assert.match(comment, /Deterministic fixture · diagnostic/);assert.match(comment, /Public 48-site run · informational/);@@ -323,8 +323,8 @@ test('comment separates execution integrity from observational evidence', () =>assert.match(comment, /\| sports \| 27 \/ 40 \| 27 \/ 40 \| 0 \|/);assert.match(comment, /\+3\.00 MiB \(\+0\.67%\)/);assert.match(comment, /actions\/runs\/123/);- assert.match(comment, /missing reports and failed browser service runs fail the workflow/);- assert.match(comment, /performance deltas remain observational/);+ assert.match(comment, /missing benchmark reports, failed browser service runs/);+ assert.match(comment, /do not fail the workflow/);assert.match(comment, /Common ancestor `111111111111` → HEAD `222222222222`/);assert.match(comment, /### CPU and memory timelines/);assert.match(comment, /bounded 40-point views of the same complete process-tree samples/);@@ -459,7 +459,7 @@ test('Moli endpoint discovery accepts ANSI-formatted tracing output', () => {assert.equal(endpointFromMoliLogs([`stderr: ${log}`]), 'http://127.0.0.1:45445');});-test('execution gate ignores content diagnostics but rejects failed browser services', () => {+test('observation issues retain failed browser services and missing reports without becoming a gate', () => {const diagnosticOnly = buildComparison(makeFixtureRun({}),makeFixtureRun({@@ -471,7 +471,7 @@ test('execution gate ignores content diagnostics but rejects failed browser servmakeRun({}));assert.equal(diagnosticOnly.suites.fixture.assessment.clean, false);- assert.deepEqual(spiderBenchmarkExecutionFailures(diagnosticOnly), []);+ assert.deepEqual(spiderBenchmarkObservationIssues(diagnosticOnly), []);const failedService = buildComparison(makeFixtureRun({}),@@ -479,7 +479,7 @@ test('execution gate ignores content diagnostics but rejects failed browser servmakeRun({}),makeRun({ successful_runs: 0, failed_runs: 1 }));- assert.deepEqual(spiderBenchmarkExecutionFailures(failedService), [{+ assert.deepEqual(spiderBenchmarkObservationIssues(failedService), [{suite: 'real48',side: 'head',reason: 'service-run-failed',@@ -499,14 +499,14 @@ test('execution gate ignores content diagnostics but rejects failed browser servresource_timeline: null});- assert.deepEqual(spiderBenchmarkExecutionFailures(missingReport), [{+ assert.deepEqual(spiderBenchmarkObservationIssues(missingReport), [{suite: 'real48',side: 'head',reason: 'report-unavailable'}]);});-test('run command preserves failed browser evidence and exits non-zero', () => {+test('run command preserves failed browser evidence and exits zero', () => {const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-observer-'));try {const result = spawnSync(process.execPath, [@@ -520,13 +520,13 @@ test('run command preserves failed browser evidence and exits non-zero', () => {'--output-dir', tempDir], { encoding: 'utf8' });- assert.equal(result.status, 1, result.stderr);+ assert.equal(result.status, 0, result.stderr);const comparison = JSON.parse(fs.readFileSync(path.join(tempDir, 'comparison.json'), 'utf8'));assert.equal(comparison.suites.fixture.assessment.clean, false);- assert.ok(comparison.suites.fixture.assessment.reasons.includes('head-benchmark-failed'));+ assert.ok(comparison.suites.fixture.assessment.reasons.includes('head-service-run-failed'));assert.ok(fs.existsSync(path.join(tempDir, 'comment.md')));assert.match(result.stderr, /recorded diagnostic issues/);- assert.match(result.stderr, /Spider Bench execution failed/);+ assert.doesNotMatch(result.stderr, /Spider Bench execution failed/);} finally {fs.rmSync(tempDir, { recursive: true, force: true });}