High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
test: skip the jsErrors golden assertion on cloak, which can't capture uncaught page errors
Luigi Agostid343fe4d4d4e
diff --git a/tests/e2e/fixtures/audit-site/normalize-report.jq b/tests/e2e/fixtures/audit-site/normalize-report.jqindex a5538800..824a47af 100644--- a/tests/e2e/fixtures/audit-site/normalize-report.jq+++ b/tests/e2e/fixtures/audit-site/normalize-report.jq@@ -12,7 +12,11 @@# Uncaught JS errors collapse to their COUNT: their message text is assembled# differently by the two capture paths (Runtime.exceptionThrown prefixes the# exception description, the Console-domain fallback does not) and carries the-# script URL, so only the count is comparable across providers.+# script URL, so only the count is comparable — and only among providers that+# capture uncaught errors at all. Cloak omits CapRuntimeConsoleEvents (its+# Console-domain fallback never sees uncaught page errors, which ride only+# Runtime.exceptionThrown), so its count is structurally always 0; the+# audit-repro-extended golden check excludes jsErrors for the cloak provider.## Regenerate the golden report (from a runner shell against the e2e stack):# curl -s -H "Authorization: Bearer $E2E_SERVER_TOKEN" -X POST "$E2E_SERVER/audit" \diff --git a/tests/e2e/scenarios/api/audit-repro-extended.sh b/tests/e2e/scenarios/api/audit-repro-extended.shindex e087f4cf..7597254b 100755--- a/tests/e2e/scenarios/api/audit-repro-extended.sh+++ b/tests/e2e/scenarios/api/audit-repro-extended.sh@@ -11,6 +11,8 @@ source "${GROUP_DIR}/../../helpers/api.sh"NORMALIZE="${GROUP_DIR}/../../fixtures/audit-site/normalize-report.jq"GOLDEN="${GROUP_DIR}/../../fixtures/audit-site/golden-report.json"RUN_BODY="{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":1}"+ARTIFACT_DIR="${E2E_ARTIFACT_DIR:-/results}"+ARTIFACT_PREFIX="${ARTIFACT_DIR}/audit-repro-${PINCHTAB_E2E_BROWSER:-chrome}"# ─────────────────────────────────────────────────────────────────start_test "two identical audit runs normalize byte-identical"@@ -23,6 +25,11 @@ pt_post /audit -d "$RUN_BODY"assert_ok "second audit run"RUN_TWO=$(echo "$RESULT" | jq -S -f "$NORMALIZE")+# Matrix runs previously overwrote the next provider's report, losing the one+# normalized payload needed to diagnose a provider-only golden mismatch.+mkdir -p "$ARTIFACT_DIR"+printf '%s\n' "$RUN_ONE" > "${ARTIFACT_PREFIX}-live.json"+if [ -n "$RUN_ONE" ] && [ "$RUN_ONE" = "$RUN_TWO" ]; thenpass_assert "normalized reports are byte-identical"else@@ -37,11 +44,28 @@ start_test "normalized report matches the checked-in golden"if [ ! -f "$GOLDEN" ]; thenfail_assert "golden report exists at $GOLDEN"-elif diff -q <(echo "$RUN_ONE") "$GOLDEN" >/dev/null 2>&1; then- pass_assert "live report matches golden-report.json"else- fail_assert "live report diverges from golden-report.json (schema/content drift; see normalize-report.jq header for how to regenerate)"- diff <(echo "$RUN_ONE") "$GOLDEN" | head -30+ # Cloak omits CapRuntimeConsoleEvents: to avoid the Runtime.enable+ # bot-detection vector it never enables the Runtime domain, and the+ # Console-domain fallback it uses instead cannot observe uncaught page errors+ # (those are delivered only via Runtime.exceptionThrown). So on cloak+ # browser.jsErrors is structurally always 0 and legitimately diverges from the+ # chrome-generated golden. Drop that field from both sides for cloak only;+ # every other provider still asserts the count exactly.+ GOLDEN_CMP=$(cat "$GOLDEN")+ LIVE_CMP="$RUN_ONE"+ if [ "${PINCHTAB_E2E_BROWSER:-chrome}" = "cloak" ]; then+ LIVE_CMP=$(echo "$RUN_ONE" | jq -S 'del(.pages[].browser.jsErrors?)')+ GOLDEN_CMP=$(echo "$GOLDEN_CMP" | jq -S 'del(.pages[].browser.jsErrors?)')+ fi++ if diff -q <(echo "$LIVE_CMP") <(echo "$GOLDEN_CMP") >/dev/null 2>&1; then+ pass_assert "live report matches golden-report.json"+ else+ fail_assert "live report diverges from golden-report.json (schema/content drift; see normalize-report.jq header for how to regenerate)"+ diff -u <(echo "$GOLDEN_CMP") <(echo "$LIVE_CMP") > "${ARTIFACT_PREFIX}-golden.diff" || true+ head -30 "${ARTIFACT_PREFIX}-golden.diff"+ fifiend_test