High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
perf(e2e): cut the audit family from five full-site crawls to two
Luigi Agosti8e4563e13f52
diff --git a/tests/e2e/fixtures/audit-site/golden-report.json b/tests/e2e/fixtures/audit-site/golden-report.jsonindex 29358d3a..44026d8e 100644--- a/tests/e2e/fixtures/audit-site/golden-report.json+++ b/tests/e2e/fixtures/audit-site/golden-report.json@@ -2,7 +2,7 @@"generatedAt": "0000-00-00T00:00:00Z","input": {},"options": {- "concurrency": 1,+ "concurrency": 0,"networkMonitor": true},"pages": [diff --git a/tests/e2e/fixtures/audit-site/normalize-report.jq b/tests/e2e/fixtures/audit-site/normalize-report.jqindex 824a47af..3ac8f96a 100644--- a/tests/e2e/fixtures/audit-site/normalize-report.jq+++ b/tests/e2e/fixtures/audit-site/normalize-report.jq@@ -17,6 +17,10 @@# Console-domain fallback never sees uncaught page errors, which ride only# Runtime.exceptionThrown), so its count is structurally always 0; the# audit-repro-extended golden check excludes jsErrors for the cloak provider.+# The run concurrency collapses to 0 so a serial and a concurrent run of the+# same input compare equal: it is the one report field that legitimately+# differs between them, and audit-repro-extended asserts each raw report echoes+# the concurrency it was asked for before normalizing.## Regenerate the golden report (from a runner shell against the e2e stack):# curl -s -H "Authorization: Bearer $E2E_SERVER_TOKEN" -X POST "$E2E_SERVER/audit" \@@ -25,6 +29,7 @@# > tests/e2e/fixtures/audit-site/golden-report.json.generatedAt = "0000-00-00T00:00:00Z"| .input = {}+| .options.concurrency = 0| .pages |= map(del(.screenshot)| .browser.screenshotPath = ""diff --git a/tests/e2e/scenarios/api/audit-basic.sh b/tests/e2e/scenarios/api/audit-basic.shindex 1004c94a..024baa3f 100755--- a/tests/e2e/scenarios/api/audit-basic.sh+++ b/tests/e2e/scenarios/api/audit-basic.sh@@ -33,27 +33,6 @@ fiend_test-# ─────────────────────────────────────────────────────────────────-start_test "concurrency 3 and 1 audit the same URL set"--BODY_URLS="[\"${FIXTURES_URL}/audit-site/index.html\",\"${FIXTURES_URL}/audit-site/clean.html\",\"${FIXTURES_URL}/audit-site/products/p1.html\",\"${FIXTURES_URL}/audit-site/products/p2.html\",\"${FIXTURES_URL}/audit-site/products/p3.html\"]"--pt_post /audit -d "{\"urls\":${BODY_URLS},\"options\":{\"screenshot\":false},\"concurrency\":1}"-assert_ok "audit with concurrency 1"-SET_ONE=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')--pt_post /audit -d "{\"urls\":${BODY_URLS},\"options\":{\"screenshot\":false},\"concurrency\":3}"-assert_ok "audit with concurrency 3"-SET_THREE=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')--if [ "$SET_ONE" = "$SET_THREE" ]; then- pass_assert "same page URL set for concurrency 1 and 3"-else- fail_assert "URL sets differ: $SET_ONE vs $SET_THREE"-fi--end_test-# ─────────────────────────────────────────────────────────────────start_test "run with one unreachable URL completes with error entry"diff --git a/tests/e2e/scenarios/api/audit-repro-extended.sh b/tests/e2e/scenarios/api/audit-repro-extended.shindex 7597254b..2576814a 100755--- a/tests/e2e/scenarios/api/audit-repro-extended.sh+++ b/tests/e2e/scenarios/api/audit-repro-extended.sh@@ -1,30 +1,41 @@#!/bin/bash-# audit-repro-extended.sh — the audit reproducibility guarantee: same input,-# same report. Uses the checked-in fixtures/audit-site/normalize-report.jq-# to strip volatile fields; the golden report is-# fixtures/audit-site/golden-report.json (regeneration instructions live in-# the jq filter's header comment).+# audit-repro-extended.sh — the audit reproducibility guarantee: the same+# input produces the same report, whatever concurrency it ran at. Two crawls,+# one serial and one concurrent, carry every assertion here. Volatile fields+# are stripped by fixtures/audit-site/normalize-report.jq, whose header owns+# the rationale for each one and the golden-report.json regeneration recipe.GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"source "${GROUP_DIR}/../../helpers/api.sh"NORMALIZE="${GROUP_DIR}/../../fixtures/audit-site/normalize-report.jq"GOLDEN="${GROUP_DIR}/../../fixtures/audit-site/golden-report.json"-RUN_BODY="{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":1}"+SITEMAP="${FIXTURES_URL}/audit-site/sitemap.xml"ARTIFACT_DIR="${E2E_ARTIFACT_DIR:-/results}"ARTIFACT_PREFIX="${ARTIFACT_DIR}/audit-repro-${PINCHTAB_E2E_BROWSER:-chrome}"+audit_body() {+ printf '{"sitemapUrl":"%s","options":{"screenshot":false},"concurrency":%s}' "$SITEMAP" "$1"+}+# ─────────────────────────────────────────────────────────────────-start_test "two identical audit runs normalize byte-identical"+start_test "two audit runs normalize byte-identical"-pt_post /audit -d "$RUN_BODY"-assert_ok "first audit run"+pt_post /audit -d "$(audit_body 1)"+assert_ok "serial audit run"+SERIAL_RAW="$RESULT"RUN_ONE=$(echo "$RESULT" | jq -S -f "$NORMALIZE")-pt_post /audit -d "$RUN_BODY"-assert_ok "second audit run"+pt_post /audit -d "$(audit_body 4)"+assert_ok "concurrent audit run"+CONCURRENT_RAW="$RESULT"RUN_TWO=$(echo "$RESULT" | jq -S -f "$NORMALIZE")+# The one field the normalizer collapses because it legitimately differs here,+# so each run still has to prove it echoes the concurrency it was asked for.+assert_json_eq "$SERIAL_RAW" '.options.concurrency' "1" "serial run reports concurrency 1"+assert_json_eq "$CONCURRENT_RAW" '.options.concurrency' "4" "concurrent run reports concurrency 4"+# Matrix runs previously overwrote the next provider's report, losing the one# normalized payload needed to diagnose a provider-only golden mismatch.mkdir -p "$ARTIFACT_DIR"@@ -45,13 +56,8 @@ start_test "normalized report matches the checked-in golden"if [ ! -f "$GOLDEN" ]; thenfail_assert "golden report exists at $GOLDEN"else- # Cloak omits CapRuntimeConsoleEvents: to avoid the Runtime.enable- # bot-detection vector it never enables the Runtime domain, and the- # Console-domain fallback it uses instead cannot observe uncaught page errors- # (those are delivered only via Runtime.exceptionThrown). So on cloak- # browser.jsErrors is structurally always 0 and legitimately diverges from the- # chrome-generated golden. Drop that field from both sides for cloak only;- # every other provider still asserts the count exactly.+ # Cloak's jsErrors count is structurally 0 (see the normalize filter header),+ # so it is dropped from both sides for that provider only.GOLDEN_CMP=$(cat "$GOLDEN")LIVE_CMP="$RUN_ONE"if [ "${PINCHTAB_E2E_BROWSER:-chrome}" = "cloak" ]; then@@ -73,10 +79,8 @@ end_test# ─────────────────────────────────────────────────────────────────start_test "concurrency 4 audits the same page set as serial"-pt_post /audit -d "{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":4}"-assert_ok "concurrent audit run"-SET_FOUR=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')SET_ONE=$(echo "$RUN_ONE" | jq -S '[.pages[].url] | sort')+SET_FOUR=$(echo "$RUN_TWO" | jq -S '[.pages[].url] | sort')if [ "$SET_ONE" = "$SET_FOUR" ]; thenpass_assert "identical page URL set at concurrency 1 and 4"diff --git a/tests/e2e/scenarios/api/audit-sampling-extended.sh b/tests/e2e/scenarios/api/audit-sampling-extended.shindex faf93484..9145c11a 100755--- a/tests/e2e/scenarios/api/audit-sampling-extended.sh+++ b/tests/e2e/scenarios/api/audit-sampling-extended.sh@@ -34,18 +34,21 @@ assert_json_contains "$RESULT" '.pages[0].url' "index.html" "homepage first in pend_test# ─────────────────────────────────────────────────────────────────-start_test "sampling is reproducible across runs"--FIRST_SET=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')--pt_post /audit -d "$SAMPLING_BODY"-assert_ok "second sampled audit"-SECOND_SET=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')--if [ "$FIRST_SET" = "$SECOND_SET" ]; then- pass_assert "identical page URL set across runs"+# Stability is pinned against the checked-in expectation rather than against a+# second crawl: the plan is a pure function of the sitemap (SamplePages keeps+# every ungrouped page in sitemap order, then the lexically-first sampleSize+# members of each template group), so naming the pages the sampler must choose+# also catches a selection that is stable but wrong — which comparing two runs+# of the same build never can.+start_test "sampling picks the documented pages on every run"++SAMPLED=$(echo "$RESULT" | jq -c '[.pages[].url | split("/") | last] | sort')+EXPECTED='["a11y-issues.html","broken-assets.html","clean.html","console-errors.html","cookie-echo.html","forms.html","index.html","p1.html","p2.html"]'++if [ "$SAMPLED" = "$EXPECTED" ]; then+ pass_assert "sampled page set is the 7 non-group pages plus the lexically-first 2 products"else- fail_assert "URL sets differ: $FIRST_SET vs $SECOND_SET"+ fail_assert "sampled page set drifted: $SAMPLED, want $EXPECTED"fiend_test