Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

ci: keep spider bench informational

ldm0fa26a909da09
diff --git a/moli-benchmark/browser-spider-local/bench.mjs b/moli-benchmark/browser-spider-local/bench.mjs
index 6a7c4f7f9..d5b8c186c 100644
--- a/moli-benchmark/browser-spider-local/bench.mjs
+++ b/moli-benchmark/browser-spider-local/bench.mjs
@@ -528,9 +528,10 @@ Options:
--moli-bin PATH moli binary override.
--chrome-bin PATH Chromium/Chrome binary override.
--assert-no-stale-page-leakage
- Exit non-zero when timed-out navigations / mismatched item links
- indicate stale previous-page scraping. Service failures always
- exit non-zero after writing benchmark evidence.
+ Opt in to a local correctness assertion: exit non-zero when a
+ service run fails or timed-out navigations / mismatched item links
+ indicate stale previous-page scraping. CI does not enable this;
+ Spider Bench results are informational.
--sample-interval-ms N Process-tree CPU/RSS/PSS sample interval (minimum 100ms).
--no-resource-sampling Disable resource sampling; the HTML report is still generated.`);
}
@@ -1706,11 +1707,11 @@ async function main() {
writeJson(path.join(runDir, 'summary.json'), summary);
console.log(JSON.stringify(summary, null, 2));
- const serviceFailures = serviceRunFailures(allResults);
- if (serviceFailures.length > 0) {
- throw new Error(`service run failed: ${JSON.stringify(serviceFailures)}`);
- }
if (args.assertNoStalePageLeakage) {
+ const serviceFailures = serviceRunFailures(allResults);
+ if (serviceFailures.length > 0) {
+ throw new Error(`service run failed before stale-page assertion: ${JSON.stringify(serviceFailures)}`);
+ }
const failures = stalePageLeakageFailures(allResults);
if (failures.length > 0) {
throw new Error(`stale page leakage detected: ${JSON.stringify(failures)}`);
diff --git a/moli-benchmark/browser-spider-local/compare.mjs b/moli-benchmark/browser-spider-local/compare.mjs
index 14f351a08..99ecdedae 100644
--- a/moli-benchmark/browser-spider-local/compare.mjs
+++ b/moli-benchmark/browser-spider-local/compare.mjs
@@ -348,14 +348,14 @@ export function buildSpiderComparison({ baseSha, headSha, executionOrder, runs }
};
}
-export function spiderBenchmarkExecutionFailures(comparison) {
- const failures = [];
+export function spiderBenchmarkObservationIssues(comparison) {
+ const issues = [];
for (const suiteName of Object.keys(SUITE_CONFIGS)) {
const suite = comparison?.suites?.[suiteName];
for (const side of ['base', 'head']) {
const run = suite?.[side];
if (run?.availability !== 'available') {
- failures.push({ suite: suiteName, side, reason: 'report-unavailable' });
+ issues.push({ suite: suiteName, side, reason: 'report-unavailable' });
continue;
}
const serviceRuns = run.metrics?.service_runs;
@@ -368,11 +368,11 @@ export function spiderBenchmarkExecutionFailures(comparison) {
|| !Number.isInteger(failedRuns)
|| successfulRuns + failedRuns !== serviceRuns
) {
- failures.push({ suite: suiteName, side, reason: 'invalid-service-summary' });
+ issues.push({ suite: suiteName, side, reason: 'invalid-service-summary' });
continue;
}
if (failedRuns > 0) {
- failures.push({
+ issues.push({
suite: suiteName,
side,
reason: 'service-run-failed',
@@ -382,7 +382,7 @@ export function spiderBenchmarkExecutionFailures(comparison) {
continue;
}
if (run.exit_code !== 0) {
- failures.push({
+ issues.push({
suite: suiteName,
side,
reason: 'benchmark-command-failed',
@@ -391,7 +391,7 @@ export function spiderBenchmarkExecutionFailures(comparison) {
}
}
}
- return failures;
+ return issues;
}
function safeSha(value) {
@@ -554,16 +554,16 @@ function assessmentSummary(assessment) {
return `⚠️ Deterministic fixture recorded diagnostic issues (non-blocking):\n${reasons}`;
}
-function executionSummary(comparison) {
- const failures = spiderBenchmarkExecutionFailures(comparison);
- if (failures.length === 0) {
- return '✅ All benchmark browser service runs completed.';
+function observationSummary(comparison) {
+ const issues = spiderBenchmarkObservationIssues(comparison);
+ if (issues.length === 0) {
+ return '✅ All benchmark browser service runs completed; results are informational.';
}
- const labels = failures
+ const labels = issues
.slice(0, 8)
- .map((failure) => `${failure.suite}/${failure.side}: ${failure.reason}`)
+ .map((issue) => `${issue.suite}/${issue.side}: ${issue.reason}`)
.join(', ');
- return `❌ Benchmark execution was incomplete (${labels}). The workflow fails while preserving the collected evidence.`;
+ return `⚠️ Benchmark observation recorded non-blocking run issues: ${labels}.`;
}
function safeRunUrl(value) {
@@ -619,7 +619,7 @@ export function renderSpiderComparison(comparison, { runUrl, conclusion }) {
SPIDER_CI_COMMENT_MARKER,
'## Spider Bench A/B',
'',
- executionSummary(comparison),
+ observationSummary(comparison),
'',
publicHeadline(real48),
'',
@@ -644,7 +644,7 @@ export function renderSpiderComparison(comparison, { runUrl, conclusion }) {
'',
`Full HTML, JSON, CSV, logs, and page snapshots: ${workflowLink}.`,
'',
- '_Spider Bench gates execution integrity: missing reports and failed browser service runs fail the workflow. Site content, fixture diagnostics, timing, and performance deltas remain observational._'
+ '_Spider Bench is informational: site failures, missing benchmark reports, failed browser service runs, fixture diagnostics, timing, and performance deltas do not fail the workflow. Build, runner, and artifact infrastructure errors can still fail it._'
].join('\n');
}
@@ -743,17 +743,11 @@ async function runComparisonCommand(args) {
writeJson(path.join(outputDir, 'comparison.json'), comparison);
fs.writeFileSync(path.join(outputDir, 'comment.md'), `${renderSpiderComparison(comparison, {
runUrl: args['run-url'],
- conclusion: args['run-url']
- ? (spiderBenchmarkExecutionFailures(comparison).length === 0 ? 'success' : 'failure')
- : 'local'
+ conclusion: args['run-url'] ? 'success' : 'local'
})}\n`, 'utf8');
if (!comparison.suites.fixture.assessment.clean) {
console.warn(`deterministic fixture recorded diagnostic issues: ${comparison.suites.fixture.assessment.reasons.join(', ')}`);
}
- const executionFailures = spiderBenchmarkExecutionFailures(comparison);
- if (executionFailures.length > 0) {
- throw new Error(`Spider Bench execution failed: ${JSON.stringify(executionFailures)}`);
- }
}
function renderCommand(args) {
diff --git a/moli-benchmark/browser-spider-local/tests/compare.test.mjs b/moli-benchmark/browser-spider-local/tests/compare.test.mjs
index 2d6a2bbf5..50283a5c1 100644
--- a/moli-benchmark/browser-spider-local/tests/compare.test.mjs
+++ b/moli-benchmark/browser-spider-local/tests/compare.test.mjs
@@ -11,7 +11,7 @@ import {
readSpiderBenchmark,
renderSpiderComparison,
renderSpiderInfrastructureFailure,
- spiderBenchmarkExecutionFailures,
+ spiderBenchmarkObservationIssues,
SPIDER_CI_COMMENT_MARKER,
SPIDER_COMPARISON_SCHEMA
} from '../compare.mjs';
@@ -291,7 +291,7 @@ test('missing output becomes unavailable instead of throwing', () => {
assert.match(missing.error, /found 0/);
});
-test('comment separates execution integrity from observational evidence', () => {
+test('comment labels browser execution and public content as informational evidence', () => {
const comparison = buildComparison(
makeFixtureRun({}),
makeFixtureRun({ duration_ms: 82_000, peak_pss_bytes: 448 * 1024 * 1024 }),
@@ -304,7 +304,7 @@ test('comment separates execution integrity from observational evidence', () =>
});
assert.ok(comment.startsWith(SPIDER_CI_COMMENT_MARKER));
- assert.match(comment, /All benchmark browser service runs completed/);
+ assert.match(comment, /All benchmark browser service runs completed; results are informational/);
assert.match(comment, /Deterministic fixture diagnostic is clean/);
assert.match(comment, /Deterministic fixture · diagnostic/);
assert.match(comment, /Public 48-site run · informational/);
@@ -323,8 +323,8 @@ test('comment separates execution integrity from observational evidence', () =>
assert.match(comment, /\| sports \| 27 \/ 40 \| 27 \/ 40 \| 0 \|/);
assert.match(comment, /\+3\.00 MiB \(\+0\.67%\)/);
assert.match(comment, /actions\/runs\/123/);
- assert.match(comment, /missing reports and failed browser service runs fail the workflow/);
- assert.match(comment, /performance deltas remain observational/);
+ assert.match(comment, /missing benchmark reports, failed browser service runs/);
+ assert.match(comment, /do not fail the workflow/);
assert.match(comment, /Common ancestor `111111111111` → HEAD `222222222222`/);
assert.match(comment, /### CPU and memory timelines/);
assert.match(comment, /bounded 40-point views of the same complete process-tree samples/);
@@ -459,7 +459,7 @@ test('Moli endpoint discovery accepts ANSI-formatted tracing output', () => {
assert.equal(endpointFromMoliLogs([`stderr: ${log}`]), 'http://127.0.0.1:45445');
});
-test('execution gate ignores content diagnostics but rejects failed browser services', () => {
+test('observation issues retain failed browser services and missing reports without becoming a gate', () => {
const diagnosticOnly = buildComparison(
makeFixtureRun({}),
makeFixtureRun({
@@ -471,7 +471,7 @@ test('execution gate ignores content diagnostics but rejects failed browser serv
makeRun({})
);
assert.equal(diagnosticOnly.suites.fixture.assessment.clean, false);
- assert.deepEqual(spiderBenchmarkExecutionFailures(diagnosticOnly), []);
+ assert.deepEqual(spiderBenchmarkObservationIssues(diagnosticOnly), []);
const failedService = buildComparison(
makeFixtureRun({}),
@@ -479,7 +479,7 @@ test('execution gate ignores content diagnostics but rejects failed browser serv
makeRun({}),
makeRun({ successful_runs: 0, failed_runs: 1 })
);
- assert.deepEqual(spiderBenchmarkExecutionFailures(failedService), [{
+ assert.deepEqual(spiderBenchmarkObservationIssues(failedService), [{
suite: 'real48',
side: 'head',
reason: 'service-run-failed',
@@ -499,14 +499,14 @@ test('execution gate ignores content diagnostics but rejects failed browser serv
resource_timeline: null
}
);
- assert.deepEqual(spiderBenchmarkExecutionFailures(missingReport), [{
+ assert.deepEqual(spiderBenchmarkObservationIssues(missingReport), [{
suite: 'real48',
side: 'head',
reason: 'report-unavailable'
}]);
});
-test('run command preserves failed browser evidence and exits non-zero', () => {
+test('run command preserves failed browser evidence and exits zero', () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'spider-observer-'));
try {
const result = spawnSync(process.execPath, [
@@ -520,13 +520,13 @@ test('run command preserves failed browser evidence and exits non-zero', () => {
'--output-dir', tempDir
], { encoding: 'utf8' });
- assert.equal(result.status, 1, result.stderr);
+ assert.equal(result.status, 0, result.stderr);
const comparison = JSON.parse(fs.readFileSync(path.join(tempDir, 'comparison.json'), 'utf8'));
assert.equal(comparison.suites.fixture.assessment.clean, false);
- assert.ok(comparison.suites.fixture.assessment.reasons.includes('head-benchmark-failed'));
+ assert.ok(comparison.suites.fixture.assessment.reasons.includes('head-service-run-failed'));
assert.ok(fs.existsSync(path.join(tempDir, 'comment.md')));
assert.match(result.stderr, /recorded diagnostic issues/);
- assert.match(result.stderr, /Spider Bench execution failed/);
+ assert.doesNotMatch(result.stderr, /Spider Bench execution failed/);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}