High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
test: add audit reproducibility golden and error-path e2e coverage
Luigi Agosti82a3e60f735c
diff --git a/tests/e2e/fixtures/audit-site/empty-sitemap.xml b/tests/e2e/fixtures/audit-site/empty-sitemap.xmlnew file mode 100644index 00000000..9a8842f2--- /dev/null+++ b/tests/e2e/fixtures/audit-site/empty-sitemap.xml@@ -0,0 +1,5 @@+<?xml version="1.0" encoding="UTF-8"?>+<!-- Deliberately empty sitemap: audit-errors-extended asserts that a+ sitemap with zero URLs yields a clean "no URLs to audit" error. -->+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+</urlset>diff --git a/tests/e2e/fixtures/audit-site/golden-report.json b/tests/e2e/fixtures/audit-site/golden-report.jsonnew file mode 100644index 00000000..29fc422f--- /dev/null+++ b/tests/e2e/fixtures/audit-site/golden-report.json@@ -0,0 +1,523 @@+{+ "generatedAt": "0000-00-00T00:00:00Z",+ "input": {},+ "options": {+ "concurrency": 1,+ "networkMonitor": true+ },+ "pages": [+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e7",+ "role": "link",+ "visible": true+ },+ {+ "name": "Broken assets",+ "ref": "e11",+ "role": "link",+ "visible": true+ },+ {+ "name": "Console errors",+ "ref": "e15",+ "role": "link",+ "visible": true+ },+ {+ "name": "Accessibility issues",+ "ref": "e19",+ "role": "link",+ "visible": true+ },+ {+ "name": "Clean page",+ "ref": "e23",+ "role": "link",+ "visible": true+ },+ {+ "name": "Forms",+ "ref": "e27",+ "role": "link",+ "visible": true+ },+ {+ "name": "Cookie echo",+ "ref": "e31",+ "role": "link",+ "visible": true+ },+ {+ "name": "Product 1",+ "ref": "e35",+ "role": "link",+ "visible": true+ },+ {+ "name": "Product 2",+ "ref": "e39",+ "role": "link",+ "visible": true+ },+ {+ "name": "Product 3",+ "ref": "e43",+ "role": "link",+ "visible": true+ },+ {+ "name": "Product 4",+ "ref": "e47",+ "role": "link",+ "visible": true+ },+ {+ "name": "Product 5",+ "ref": "e51",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/index.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Audit Fixture Site",+ "url": "http://fixtures/audit-site/index.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "brokenAssets": [+ {+ "error": "net::ERR_ABORTED",+ "resourceType": "stylesheet",+ "status": 404,+ "url": "http://fixtures/audit-site/assets/missing.css"+ },+ {+ "error": "net::ERR_ABORTED",+ "resourceType": "script",+ "status": 404,+ "url": "http://fixtures/audit-site/assets/missing.js"+ },+ {+ "resourceType": "image",+ "status": 404,+ "url": "http://fixtures/audit-site/assets/missing.png"+ },+ {+ "resourceType": "fetch",+ "status": 404,+ "url": "http://fixtures/audit-site/assets/missing.json"+ }+ ],+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e7",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Stylesheet",+ "status": 404,+ "url": "http://fixtures/audit-site/assets/missing.css"+ },+ {+ "method": "GET",+ "resourceType": "Script",+ "status": 404,+ "url": "http://fixtures/audit-site/assets/missing.js"+ },+ {+ "method": "GET",+ "resourceType": "Fetch",+ "status": 404,+ "url": "http://fixtures/audit-site/assets/missing.json"+ },+ {+ "method": "GET",+ "resourceType": "Image",+ "status": 404,+ "url": "http://fixtures/audit-site/assets/missing.png"+ },+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/broken-assets.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Broken Assets — Audit Fixture Site",+ "url": "http://fixtures/audit-site/broken-assets.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [+ {+ "level": "log",+ "message": "audit-fixture log message "+ },+ {+ "level": "warning",+ "message": "audit-fixture warn message "+ },+ {+ "level": "error",+ "message": "audit-fixture error message "+ }+ ],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e5",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/console-errors.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Console Errors — Audit Fixture Site",+ "url": "http://fixtures/audit-site/console-errors.html"+ },+ {+ "browser": {+ "accessibilityScore": 70,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "ref": "e6",+ "role": "textbox",+ "visible": true+ },+ {+ "ref": "e7",+ "role": "link",+ "visible": true+ },+ {+ "ref": "e8",+ "role": "button",+ "visible": true+ },+ {+ "name": "Home",+ "ref": "e10",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Image",+ "status": 200,+ "url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"+ },+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/a11y-issues.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Accessibility Issues — Audit Fixture Site",+ "url": "http://fixtures/audit-site/a11y-issues.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Search",+ "ref": "e8",+ "role": "textbox",+ "visible": true+ },+ {+ "name": "Search",+ "ref": "e9",+ "role": "button",+ "visible": true+ },+ {+ "name": "Home",+ "ref": "e12",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/clean.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Clean Page — Audit Fixture Site",+ "url": "http://fixtures/audit-site/clean.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Username",+ "ref": "e8",+ "role": "textbox",+ "visible": true+ },+ {+ "name": "Password",+ "ref": "e11",+ "role": "textbox",+ "visible": true+ },+ {+ "name": "Sign in",+ "ref": "e12",+ "role": "button",+ "visible": true+ },+ {+ "name": "Home",+ "ref": "e15",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/forms.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "securityFindings": [+ {+ "detail": "password form submits over plain http to http://fixtures/audit-site/login",+ "ruleId": "insecure-password-form",+ "severity": "high",+ "url": "http://fixtures/audit-site/forms.html"+ }+ ],+ "title": "Forms — Audit Fixture Site",+ "url": "http://fixtures/audit-site/forms.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [+ {+ "level": "log",+ "message": "cookies: "+ }+ ],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e6",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/cookie-echo.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Cookie Echo — Audit Fixture Site",+ "url": "http://fixtures/audit-site/cookie-echo.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e6",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/products/p1.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Product 1 — Audit Fixture Site",+ "url": "http://fixtures/audit-site/products/p1.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e6",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/products/p2.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Product 2 — Audit Fixture Site",+ "url": "http://fixtures/audit-site/products/p2.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e6",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/products/p3.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Product 3 — Audit Fixture Site",+ "url": "http://fixtures/audit-site/products/p3.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e6",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/products/p4.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Product 4 — Audit Fixture Site",+ "url": "http://fixtures/audit-site/products/p4.html"+ },+ {+ "browser": {+ "accessibilityScore": 100,+ "consoleLogs": [],+ "interactiveElements": [+ {+ "name": "Home",+ "ref": "e6",+ "role": "link",+ "visible": true+ }+ ],+ "networkRequests": [+ {+ "method": "GET",+ "resourceType": "Document",+ "status": 200,+ "url": "http://fixtures/audit-site/products/p5.html"+ }+ ],+ "screenshotPath": "",+ "timingMetrics": {}+ },+ "title": "Product 5 — Audit Fixture Site",+ "url": "http://fixtures/audit-site/products/p5.html"+ }+ ],+ "schemaVersion": "1.0",+ "securityFindings": [+ {+ "detail": "password form submits over plain http to http://fixtures/audit-site/login",+ "ruleId": "insecure-password-form",+ "severity": "high",+ "url": "http://fixtures/audit-site/forms.html"+ }+ ],+ "summaryScore": 97+}diff --git a/tests/e2e/fixtures/audit-site/normalize-report.jq b/tests/e2e/fixtures/audit-site/normalize-report.jqnew file mode 100644index 00000000..0be066c5--- /dev/null+++ b/tests/e2e/fixtures/audit-site/normalize-report.jq@@ -0,0 +1,25 @@+# normalize-report.jq — strips the volatile fields from an AuditReport so+# two runs of the same audit compare byte-identical (use with `jq -S`).+#+# Normalized away: the run timestamp, the run input (carries the caller's+# FIXTURES_URL host), all timing metrics, screenshot payloads/paths, console+# timestamps/sources, and network entry fields that vary per run (timings,+# sizes, request ids); network entries are sorted by URL because subresource+# completion order is nondeterministic.+#+# Regenerate the golden report (from a runner shell against the e2e stack):+# curl -s -H "Authorization: Bearer $E2E_SERVER_TOKEN" -X POST "$E2E_SERVER/audit" \+# -d '{"sitemapUrl":"'"$FIXTURES_URL"'/audit-site/sitemap.xml","options":{"screenshot":false},"concurrency":1}' \+# | jq -S -f tests/e2e/fixtures/audit-site/normalize-report.jq \+# > tests/e2e/fixtures/audit-site/golden-report.json+.generatedAt = "0000-00-00T00:00:00Z"+| .input = {}+| .pages |= map(+ del(.screenshot)+ | .browser.screenshotPath = ""+ | .browser.timingMetrics = {}+ | .browser.consoleLogs = ((.browser.consoleLogs // []) | map({level: .level, message: .message}))+ | .browser.networkRequests = ((.browser.networkRequests // [])+ | map({url: .url, method: .method, status: .status, resourceType: .resourceType})+ | sort_by(.url))+ )diff --git a/tests/e2e/scenarios/api/audit-errors-extended.sh b/tests/e2e/scenarios/api/audit-errors-extended.shnew file mode 100755index 00000000..0701322a--- /dev/null+++ b/tests/e2e/scenarios/api/audit-errors-extended.sh@@ -0,0 +1,79 @@+#!/bin/bash+# audit-errors-extended.sh — audit error-path contract:+# - unreachable host → HTTP 200, page entry carries `error`, run continues+# - 404 entry page → HTTP 200, page audited normally (Chrome renders the+# error document; the 404 shows in networkRequests)+# - empty sitemap → HTTP 400 "no URLs to audit"+# After every case the server stays healthy and the next audit succeeds.++GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+source "${GROUP_DIR}/../../helpers/api.sh"++assert_healthy_and_recovers() {+ pt_get /health+ assert_ok "/health ok after the case"+ pt_post /audit -d "{\"urls\":[\"${FIXTURES_URL}/audit-site/clean.html\"],\"options\":{\"screenshot\":false}}"+ assert_ok "follow-up audit succeeds"+ assert_json_eq "$RESULT" '.pages[0].error // "none"' "none" "follow-up page has no error"+}++# ─────────────────────────────────────────────────────────────────+start_test "unreachable host produces a structured page error"++pt_post /audit -d "{\"urls\":[\"http://fixtures:9/down.html\"],\"options\":{\"screenshot\":false}}"+assert_ok "audit run returns 200"+ERR=$(echo "$RESULT" | jq -r '.pages[0].error // empty')+if [ -n "$ERR" ]; then+ pass_assert "page entry carries error ($ERR)"+else+ fail_assert "page entry carries error"+fi++assert_healthy_and_recovers++end_test++# ─────────────────────────────────────────────────────────────────+start_test "404 entry page is audited as a page, not a failure"++pt_post /audit -d "{\"urls\":[\"${FIXTURES_URL}/audit-site/no-such-page.html\"],\"options\":{\"screenshot\":false}}"+assert_ok "audit run returns 200"+assert_json_length "$RESULT" '.pages' 1 "one page entry"++ERR=$(echo "$RESULT" | jq -r '.pages[0].error // empty')+if [ -z "$ERR" ]; then+ pass_assert "no error field (error page rendered and audited)"+else+ fail_assert "no error field (got: $ERR)"+fi++if echo "$RESULT" | jq -e '.pages[0].browser.networkRequests[]? | select(.url | endswith("no-such-page.html")) | select(.status == 404)' >/dev/null 2>&1; then+ pass_assert "document 404 visible in networkRequests"+else+ fail_assert "document 404 visible in networkRequests (got: $(echo "$RESULT" | jq -c '[.pages[0].browser.networkRequests[]? | {url, status}]'))"+fi++assert_healthy_and_recovers++end_test++# ─────────────────────────────────────────────────────────────────+start_test "empty sitemap fails cleanly with no URLs to audit"++STATUS=$(e2e_curl -s -o /tmp/audit-empty-sitemap.json -w '%{http_code}' -X POST "${E2E_SERVER}/audit" \+ -d "{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/empty-sitemap.xml\"}")+if [ "$STATUS" = "400" ]; then+ pass_assert "empty sitemap returns HTTP 400"+else+ fail_assert "empty sitemap returns HTTP 400 (got: $STATUS)"+fi++if grep -q "no URLs to audit" /tmp/audit-empty-sitemap.json; then+ pass_assert "error explains there is nothing to audit"+else+ fail_assert "error explains there is nothing to audit (got: $(cat /tmp/audit-empty-sitemap.json))"+fi++assert_healthy_and_recovers++end_testdiff --git a/tests/e2e/scenarios/api/audit-repro-extended.sh b/tests/e2e/scenarios/api/audit-repro-extended.shnew file mode 100755index 00000000..e087f4cf--- /dev/null+++ b/tests/e2e/scenarios/api/audit-repro-extended.sh@@ -0,0 +1,63 @@+#!/bin/bash+# audit-repro-extended.sh — the audit reproducibility guarantee: same input,+# same report. Uses the checked-in fixtures/audit-site/normalize-report.jq+# to strip volatile fields; the golden report is+# fixtures/audit-site/golden-report.json (regeneration instructions live in+# the jq filter's header comment).++GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+source "${GROUP_DIR}/../../helpers/api.sh"++NORMALIZE="${GROUP_DIR}/../../fixtures/audit-site/normalize-report.jq"+GOLDEN="${GROUP_DIR}/../../fixtures/audit-site/golden-report.json"+RUN_BODY="{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":1}"++# ─────────────────────────────────────────────────────────────────+start_test "two identical audit runs normalize byte-identical"++pt_post /audit -d "$RUN_BODY"+assert_ok "first audit run"+RUN_ONE=$(echo "$RESULT" | jq -S -f "$NORMALIZE")++pt_post /audit -d "$RUN_BODY"+assert_ok "second audit run"+RUN_TWO=$(echo "$RESULT" | jq -S -f "$NORMALIZE")++if [ -n "$RUN_ONE" ] && [ "$RUN_ONE" = "$RUN_TWO" ]; then+ pass_assert "normalized reports are byte-identical"+else+ fail_assert "normalized reports differ"+ diff <(echo "$RUN_ONE") <(echo "$RUN_TWO") | head -20+fi++end_test++# ─────────────────────────────────────────────────────────────────+start_test "normalized report matches the checked-in golden"++if [ ! -f "$GOLDEN" ]; then+ fail_assert "golden report exists at $GOLDEN"+elif diff -q <(echo "$RUN_ONE") "$GOLDEN" >/dev/null 2>&1; then+ pass_assert "live report matches golden-report.json"+else+ fail_assert "live report diverges from golden-report.json (schema/content drift; see normalize-report.jq header for how to regenerate)"+ diff <(echo "$RUN_ONE") "$GOLDEN" | head -30+fi++end_test++# ─────────────────────────────────────────────────────────────────+start_test "concurrency 4 audits the same page set as serial"++pt_post /audit -d "{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":4}"+assert_ok "concurrent audit run"+SET_FOUR=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')+SET_ONE=$(echo "$RUN_ONE" | jq -S '[.pages[].url] | sort')++if [ "$SET_ONE" = "$SET_FOUR" ]; then+ pass_assert "identical page URL set at concurrency 1 and 4"+else+ fail_assert "page sets differ: $SET_ONE vs $SET_FOUR"+fi++end_testdiff --git a/tests/e2e/scenarios/manifest.json b/tests/e2e/scenarios/manifest.jsonindex 86bc8a3e..20359b49 100644--- a/tests/e2e/scenarios/manifest.json+++ b/tests/e2e/scenarios/manifest.json@@ -451,6 +451,26 @@"library","smoke"]+ },+ "api/audit-repro-extended.sh": {+ "tags": [+ "audit",+ "repro",+ "golden"+ ]+ },+ "api/audit-errors-extended.sh": {+ "tags": [+ "audit",+ "errors"+ ]+ },+ "cli/compare-basic.sh": {+ "tags": [+ "compare",+ "audit",+ "visualdiff"+ ]}}}