Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

test(spider): enforce exact fixture row contracts

ldm059edc88e02ef
diff --git a/moli-benchmark/browser-spider-local/bench.mjs b/moli-benchmark/browser-spider-local/bench.mjs
index d5b8c186c..44ca10e9c 100644
--- a/moli-benchmark/browser-spider-local/bench.mjs
+++ b/moli-benchmark/browser-spider-local/bench.mjs
@@ -431,6 +431,7 @@ function parseArgs(argv) {
moliBin: process.env.MOLI_BIN || '',
chromeBin: process.env.CHROME_BIN || '',
assertNoStalePageLeakage: false,
+ assertFixtureContract: false,
sampleResources: true,
sampleIntervalMs: 500
};
@@ -466,6 +467,8 @@ function parseArgs(argv) {
args.chromeBin = next();
} else if (value === '--assert-no-stale-page-leakage') {
args.assertNoStalePageLeakage = true;
+ } else if (value === '--assert-fixture-contract') {
+ args.assertFixtureContract = true;
} else if (value === '--sample-interval-ms') {
args.sampleIntervalMs = Number.parseInt(next(), 10);
} else if (value === '--no-resource-sampling') {
@@ -490,6 +493,9 @@ function parseArgs(argv) {
if (!['fixture', 'real'].includes(args.siteSource)) {
throw new Error(`unknown site source: ${args.siteSource}`);
}
+ if (args.assertFixtureContract && args.siteSource !== 'fixture') {
+ throw new Error('--assert-fixture-contract requires --site-source fixture');
+ }
args.cases = [...new Set(args.cases)];
for (const caseName of args.cases) {
if (!CASE_ORDER.includes(caseName)) {
@@ -530,8 +536,9 @@ Options:
--assert-no-stale-page-leakage
Opt in to a local correctness assertion: exit non-zero when a
service run fails or timed-out navigations / mismatched item links
- indicate stale previous-page scraping. CI does not enable this;
- Spider Bench results are informational.
+ indicate stale previous-page scraping.
+ --assert-fixture-contract Require every deterministic fixture route to return its exact
+ expected row count, with no service failure or stale-page output.
--sample-interval-ms N Process-tree CPU/RSS/PSS sample interval (minimum 100ms).
--no-resource-sampling Disable resource sampling; the HTML report is still generated.`);
}
@@ -1632,6 +1639,54 @@ function serviceRunFailures(results) {
}));
}
+export function fixtureContractFailures(results) {
+ const failures = [];
+ for (const result of results) {
+ if (!result.success) {
+ continue;
+ }
+ const cases = result.metadata?.cases;
+ const summary = result.report?.summary;
+ if (!Array.isArray(cases) || !summary) {
+ failures.push({
+ service: result.service,
+ reason: 'fixture evidence is incomplete'
+ });
+ continue;
+ }
+
+ const sites = cases.flatMap((caseResult) => (
+ Array.isArray(caseResult.siteMeta) ? caseResult.siteMeta : []
+ ));
+ if (sites.length !== summary.totalSites) {
+ failures.push({
+ service: result.service,
+ reason: 'fixture site count mismatch',
+ actualSites: sites.length,
+ expectedSites: summary.totalSites
+ });
+ }
+ for (const site of sites) {
+ if (
+ !Number.isInteger(site.expectedItemCount)
+ || !Number.isInteger(site.itemCount)
+ || site.itemCount !== site.expectedItemCount
+ ) {
+ failures.push({
+ service: result.service,
+ caseName: site.caseName ?? null,
+ site: site.site ?? null,
+ actualItems: Number.isInteger(site.itemCount) ? site.itemCount : null,
+ expectedItems: Number.isInteger(site.expectedItemCount)
+ ? site.expectedItemCount
+ : null
+ });
+ }
+ }
+ }
+ return failures;
+}
+
async function main() {
const args = parseArgs(process.argv.slice(2));
const taskId = randomUUID();
@@ -1649,6 +1704,7 @@ async function main() {
gotoTimeoutMs: args.gotoTimeoutMs,
serverHangMs: args.serverHangMs,
assertNoStalePageLeakage: args.assertNoStalePageLeakage,
+ assertFixtureContract: args.assertFixtureContract,
sampleResources: args.sampleResources,
sampleIntervalMs: args.sampleIntervalMs
});
@@ -1717,6 +1773,16 @@ async function main() {
throw new Error(`stale page leakage detected: ${JSON.stringify(failures)}`);
}
}
+ if (args.assertFixtureContract) {
+ const failures = [
+ ...serviceRunFailures(allResults),
+ ...stalePageLeakageFailures(allResults),
+ ...fixtureContractFailures(allResults)
+ ];
+ if (failures.length > 0) {
+ throw new Error(`fixture contract failed: ${JSON.stringify(failures)}`);
+ }
+ }
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
diff --git a/moli-benchmark/browser-spider-local/tests/bench.test.mjs b/moli-benchmark/browser-spider-local/tests/bench.test.mjs
index 10a6dda57..f8743d27e 100644
--- a/moli-benchmark/browser-spider-local/tests/bench.test.mjs
+++ b/moli-benchmark/browser-spider-local/tests/bench.test.mjs
@@ -1,7 +1,7 @@
import assert from 'node:assert/strict';
import test from 'node:test';
-import { extractItems } from '../bench.mjs';
+import { extractItems, fixtureContractFailures } from '../bench.mjs';
function candidate({ text, href = null, closestHref = null, childHref = null }) {
const anchor = (value) => ({
@@ -89,3 +89,64 @@ test('item extraction snapshots the complete selector set in one page-realm job'
{ title: 'Sports story', link: 'https://example.test/sports' }
]);
});
+
+function fixtureResult(siteMeta, overrides = {}) {
+ return {
+ service: 'moli',
+ success: true,
+ metadata: {
+ cases: [{ caseName: 'news', siteMeta }]
+ },
+ report: {
+ summary: { totalSites: siteMeta.length }
+ },
+ ...overrides
+ };
+}
+
+test('fixture contract accepts exact per-site row counts', () => {
+ const failures = fixtureContractFailures([
+ fixtureResult([
+ { caseName: 'news', site: 'rows', itemCount: 5, expectedItemCount: 5 },
+ { caseName: 'news', site: 'timeout', itemCount: 0, expectedItemCount: 0 }
+ ])
+ ]);
+
+ assert.deepEqual(failures, []);
+});
+
+test('fixture contract catches same-total per-site cancellation and missing evidence', () => {
+ const failures = fixtureContractFailures([
+ fixtureResult([
+ { caseName: 'news', site: 'too-few', itemCount: 4, expectedItemCount: 5 },
+ { caseName: 'news', site: 'too-many', itemCount: 1, expectedItemCount: 0 }
+ ]),
+ {
+ service: 'moli-run-2',
+ success: true,
+ metadata: {},
+ report: null
+ }
+ ]);
+
+ assert.deepEqual(failures, [
+ {
+ service: 'moli',
+ caseName: 'news',
+ site: 'too-few',
+ actualItems: 4,
+ expectedItems: 5
+ },
+ {
+ service: 'moli',
+ caseName: 'news',
+ site: 'too-many',
+ actualItems: 1,
+ expectedItems: 0
+ },
+ {
+ service: 'moli-run-2',
+ reason: 'fixture evidence is incomplete'
+ }
+ ]);
+});