Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

perf(e2e): cut the audit family from five full-site crawls to two

Luigi Agosti8e4563e13f52
diff --git a/tests/e2e/fixtures/audit-site/golden-report.json b/tests/e2e/fixtures/audit-site/golden-report.json
index 29358d3a..44026d8e 100644
--- a/tests/e2e/fixtures/audit-site/golden-report.json
+++ b/tests/e2e/fixtures/audit-site/golden-report.json
@@ -2,7 +2,7 @@
"generatedAt": "0000-00-00T00:00:00Z",
"input": {},
"options": {
- "concurrency": 1,
+ "concurrency": 0,
"networkMonitor": true
},
"pages": [
diff --git a/tests/e2e/fixtures/audit-site/normalize-report.jq b/tests/e2e/fixtures/audit-site/normalize-report.jq
index 824a47af..3ac8f96a 100644
--- a/tests/e2e/fixtures/audit-site/normalize-report.jq
+++ b/tests/e2e/fixtures/audit-site/normalize-report.jq
@@ -17,6 +17,10 @@
# Console-domain fallback never sees uncaught page errors, which ride only
# Runtime.exceptionThrown), so its count is structurally always 0; the
# audit-repro-extended golden check excludes jsErrors for the cloak provider.
+# The run concurrency collapses to 0 so a serial and a concurrent run of the
+# same input compare equal: it is the one report field that legitimately
+# differs between them, and audit-repro-extended asserts each raw report echoes
+# the concurrency it was asked for before normalizing.
#
# Regenerate the golden report (from a runner shell against the e2e stack):
# curl -s -H "Authorization: Bearer $E2E_SERVER_TOKEN" -X POST "$E2E_SERVER/audit" \
@@ -25,6 +29,7 @@
# > tests/e2e/fixtures/audit-site/golden-report.json
.generatedAt = "0000-00-00T00:00:00Z"
| .input = {}
+| .options.concurrency = 0
| .pages |= map(
del(.screenshot)
| .browser.screenshotPath = ""
diff --git a/tests/e2e/scenarios/api/audit-basic.sh b/tests/e2e/scenarios/api/audit-basic.sh
index 1004c94a..024baa3f 100755
--- a/tests/e2e/scenarios/api/audit-basic.sh
+++ b/tests/e2e/scenarios/api/audit-basic.sh
@@ -33,27 +33,6 @@ fi
end_test
-# ─────────────────────────────────────────────────────────────────
-start_test "concurrency 3 and 1 audit the same URL set"
-
-BODY_URLS="[\"${FIXTURES_URL}/audit-site/index.html\",\"${FIXTURES_URL}/audit-site/clean.html\",\"${FIXTURES_URL}/audit-site/products/p1.html\",\"${FIXTURES_URL}/audit-site/products/p2.html\",\"${FIXTURES_URL}/audit-site/products/p3.html\"]"
-
-pt_post /audit -d "{\"urls\":${BODY_URLS},\"options\":{\"screenshot\":false},\"concurrency\":1}"
-assert_ok "audit with concurrency 1"
-SET_ONE=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')
-
-pt_post /audit -d "{\"urls\":${BODY_URLS},\"options\":{\"screenshot\":false},\"concurrency\":3}"
-assert_ok "audit with concurrency 3"
-SET_THREE=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')
-
-if [ "$SET_ONE" = "$SET_THREE" ]; then
- pass_assert "same page URL set for concurrency 1 and 3"
-else
- fail_assert "URL sets differ: $SET_ONE vs $SET_THREE"
-fi
-
-end_test
-
# ─────────────────────────────────────────────────────────────────
start_test "run with one unreachable URL completes with error entry"
diff --git a/tests/e2e/scenarios/api/audit-repro-extended.sh b/tests/e2e/scenarios/api/audit-repro-extended.sh
index 7597254b..2576814a 100755
--- a/tests/e2e/scenarios/api/audit-repro-extended.sh
+++ b/tests/e2e/scenarios/api/audit-repro-extended.sh
@@ -1,30 +1,41 @@
#!/bin/bash
-# audit-repro-extended.sh — the audit reproducibility guarantee: same input,
-# same report. Uses the checked-in fixtures/audit-site/normalize-report.jq
-# to strip volatile fields; the golden report is
-# fixtures/audit-site/golden-report.json (regeneration instructions live in
-# the jq filter's header comment).
+# audit-repro-extended.sh — the audit reproducibility guarantee: the same
+# input produces the same report, whatever concurrency it ran at. Two crawls,
+# one serial and one concurrent, carry every assertion here. Volatile fields
+# are stripped by fixtures/audit-site/normalize-report.jq, whose header owns
+# the rationale for each one and the golden-report.json regeneration recipe.
GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${GROUP_DIR}/../../helpers/api.sh"
NORMALIZE="${GROUP_DIR}/../../fixtures/audit-site/normalize-report.jq"
GOLDEN="${GROUP_DIR}/../../fixtures/audit-site/golden-report.json"
-RUN_BODY="{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":1}"
+SITEMAP="${FIXTURES_URL}/audit-site/sitemap.xml"
ARTIFACT_DIR="${E2E_ARTIFACT_DIR:-/results}"
ARTIFACT_PREFIX="${ARTIFACT_DIR}/audit-repro-${PINCHTAB_E2E_BROWSER:-chrome}"
+audit_body() {
+ printf '{"sitemapUrl":"%s","options":{"screenshot":false},"concurrency":%s}' "$SITEMAP" "$1"
+}
+
# ─────────────────────────────────────────────────────────────────
-start_test "two identical audit runs normalize byte-identical"
+start_test "two audit runs normalize byte-identical"
-pt_post /audit -d "$RUN_BODY"
-assert_ok "first audit run"
+pt_post /audit -d "$(audit_body 1)"
+assert_ok "serial audit run"
+SERIAL_RAW="$RESULT"
RUN_ONE=$(echo "$RESULT" | jq -S -f "$NORMALIZE")
-pt_post /audit -d "$RUN_BODY"
-assert_ok "second audit run"
+pt_post /audit -d "$(audit_body 4)"
+assert_ok "concurrent audit run"
+CONCURRENT_RAW="$RESULT"
RUN_TWO=$(echo "$RESULT" | jq -S -f "$NORMALIZE")
+# The one field the normalizer collapses because it legitimately differs here,
+# so each run still has to prove it echoes the concurrency it was asked for.
+assert_json_eq "$SERIAL_RAW" '.options.concurrency' "1" "serial run reports concurrency 1"
+assert_json_eq "$CONCURRENT_RAW" '.options.concurrency' "4" "concurrent run reports concurrency 4"
+
# Matrix runs previously overwrote the next provider's report, losing the one
# normalized payload needed to diagnose a provider-only golden mismatch.
mkdir -p "$ARTIFACT_DIR"
@@ -45,13 +56,8 @@ start_test "normalized report matches the checked-in golden"
if [ ! -f "$GOLDEN" ]; then
fail_assert "golden report exists at $GOLDEN"
else
- # Cloak omits CapRuntimeConsoleEvents: to avoid the Runtime.enable
- # bot-detection vector it never enables the Runtime domain, and the
- # Console-domain fallback it uses instead cannot observe uncaught page errors
- # (those are delivered only via Runtime.exceptionThrown). So on cloak
- # browser.jsErrors is structurally always 0 and legitimately diverges from the
- # chrome-generated golden. Drop that field from both sides for cloak only;
- # every other provider still asserts the count exactly.
+ # Cloak's jsErrors count is structurally 0 (see the normalize filter header),
+ # so it is dropped from both sides for that provider only.
GOLDEN_CMP=$(cat "$GOLDEN")
LIVE_CMP="$RUN_ONE"
if [ "${PINCHTAB_E2E_BROWSER:-chrome}" = "cloak" ]; then
@@ -73,10 +79,8 @@ end_test
# ─────────────────────────────────────────────────────────────────
start_test "concurrency 4 audits the same page set as serial"
-pt_post /audit -d "{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":4}"
-assert_ok "concurrent audit run"
-SET_FOUR=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')
SET_ONE=$(echo "$RUN_ONE" | jq -S '[.pages[].url] | sort')
+SET_FOUR=$(echo "$RUN_TWO" | jq -S '[.pages[].url] | sort')
if [ "$SET_ONE" = "$SET_FOUR" ]; then
pass_assert "identical page URL set at concurrency 1 and 4"
diff --git a/tests/e2e/scenarios/api/audit-sampling-extended.sh b/tests/e2e/scenarios/api/audit-sampling-extended.sh
index faf93484..9145c11a 100755
--- a/tests/e2e/scenarios/api/audit-sampling-extended.sh
+++ b/tests/e2e/scenarios/api/audit-sampling-extended.sh
@@ -34,18 +34,21 @@ assert_json_contains "$RESULT" '.pages[0].url' "index.html" "homepage first in p
end_test
# ─────────────────────────────────────────────────────────────────
-start_test "sampling is reproducible across runs"
-
-FIRST_SET=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')
-
-pt_post /audit -d "$SAMPLING_BODY"
-assert_ok "second sampled audit"
-SECOND_SET=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')
-
-if [ "$FIRST_SET" = "$SECOND_SET" ]; then
- pass_assert "identical page URL set across runs"
+# Stability is pinned against the checked-in expectation rather than against a
+# second crawl: the plan is a pure function of the sitemap (SamplePages keeps
+# every ungrouped page in sitemap order, then the lexically-first sampleSize
+# members of each template group), so naming the pages the sampler must choose
+# also catches a selection that is stable but wrong — which comparing two runs
+# of the same build never can.
+start_test "sampling picks the documented pages on every run"
+
+SAMPLED=$(echo "$RESULT" | jq -c '[.pages[].url | split("/") | last] | sort')
+EXPECTED='["a11y-issues.html","broken-assets.html","clean.html","console-errors.html","cookie-echo.html","forms.html","index.html","p1.html","p2.html"]'
+
+if [ "$SAMPLED" = "$EXPECTED" ]; then
+ pass_assert "sampled page set is the 7 non-group pages plus the lexically-first 2 products"
else
- fail_assert "URL sets differ: $FIRST_SET vs $SECOND_SET"
+ fail_assert "sampled page set drifted: $SAMPLED, want $EXPECTED"
fi
end_test