Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

test: add audit reproducibility golden and error-path e2e coverage

Luigi Agosti82a3e60f735c
diff --git a/tests/e2e/fixtures/audit-site/empty-sitemap.xml b/tests/e2e/fixtures/audit-site/empty-sitemap.xml
new file mode 100644
index 00000000..9a8842f2
--- /dev/null
+++ b/tests/e2e/fixtures/audit-site/empty-sitemap.xml
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Deliberately empty sitemap: audit-errors-extended asserts that a
+ sitemap with zero URLs yields a clean "no URLs to audit" error. -->
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+</urlset>
diff --git a/tests/e2e/fixtures/audit-site/golden-report.json b/tests/e2e/fixtures/audit-site/golden-report.json
new file mode 100644
index 00000000..29fc422f
--- /dev/null
+++ b/tests/e2e/fixtures/audit-site/golden-report.json
@@ -0,0 +1,523 @@
+{
+ "generatedAt": "0000-00-00T00:00:00Z",
+ "input": {},
+ "options": {
+ "concurrency": 1,
+ "networkMonitor": true
+ },
+ "pages": [
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e7",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Broken assets",
+ "ref": "e11",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Console errors",
+ "ref": "e15",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Accessibility issues",
+ "ref": "e19",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Clean page",
+ "ref": "e23",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Forms",
+ "ref": "e27",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Cookie echo",
+ "ref": "e31",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Product 1",
+ "ref": "e35",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Product 2",
+ "ref": "e39",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Product 3",
+ "ref": "e43",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Product 4",
+ "ref": "e47",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "name": "Product 5",
+ "ref": "e51",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/index.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Audit Fixture Site",
+ "url": "http://fixtures/audit-site/index.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "brokenAssets": [
+ {
+ "error": "net::ERR_ABORTED",
+ "resourceType": "stylesheet",
+ "status": 404,
+ "url": "http://fixtures/audit-site/assets/missing.css"
+ },
+ {
+ "error": "net::ERR_ABORTED",
+ "resourceType": "script",
+ "status": 404,
+ "url": "http://fixtures/audit-site/assets/missing.js"
+ },
+ {
+ "resourceType": "image",
+ "status": 404,
+ "url": "http://fixtures/audit-site/assets/missing.png"
+ },
+ {
+ "resourceType": "fetch",
+ "status": 404,
+ "url": "http://fixtures/audit-site/assets/missing.json"
+ }
+ ],
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e7",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Stylesheet",
+ "status": 404,
+ "url": "http://fixtures/audit-site/assets/missing.css"
+ },
+ {
+ "method": "GET",
+ "resourceType": "Script",
+ "status": 404,
+ "url": "http://fixtures/audit-site/assets/missing.js"
+ },
+ {
+ "method": "GET",
+ "resourceType": "Fetch",
+ "status": 404,
+ "url": "http://fixtures/audit-site/assets/missing.json"
+ },
+ {
+ "method": "GET",
+ "resourceType": "Image",
+ "status": 404,
+ "url": "http://fixtures/audit-site/assets/missing.png"
+ },
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/broken-assets.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Broken Assets — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/broken-assets.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [
+ {
+ "level": "log",
+ "message": "audit-fixture log message "
+ },
+ {
+ "level": "warning",
+ "message": "audit-fixture warn message "
+ },
+ {
+ "level": "error",
+ "message": "audit-fixture error message "
+ }
+ ],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e5",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/console-errors.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Console Errors — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/console-errors.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 70,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "ref": "e6",
+ "role": "textbox",
+ "visible": true
+ },
+ {
+ "ref": "e7",
+ "role": "link",
+ "visible": true
+ },
+ {
+ "ref": "e8",
+ "role": "button",
+ "visible": true
+ },
+ {
+ "name": "Home",
+ "ref": "e10",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Image",
+ "status": 200,
+ "url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"
+ },
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/a11y-issues.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Accessibility Issues — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/a11y-issues.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Search",
+ "ref": "e8",
+ "role": "textbox",
+ "visible": true
+ },
+ {
+ "name": "Search",
+ "ref": "e9",
+ "role": "button",
+ "visible": true
+ },
+ {
+ "name": "Home",
+ "ref": "e12",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/clean.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Clean Page — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/clean.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Username",
+ "ref": "e8",
+ "role": "textbox",
+ "visible": true
+ },
+ {
+ "name": "Password",
+ "ref": "e11",
+ "role": "textbox",
+ "visible": true
+ },
+ {
+ "name": "Sign in",
+ "ref": "e12",
+ "role": "button",
+ "visible": true
+ },
+ {
+ "name": "Home",
+ "ref": "e15",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/forms.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "securityFindings": [
+ {
+ "detail": "password form submits over plain http to http://fixtures/audit-site/login",
+ "ruleId": "insecure-password-form",
+ "severity": "high",
+ "url": "http://fixtures/audit-site/forms.html"
+ }
+ ],
+ "title": "Forms — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/forms.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [
+ {
+ "level": "log",
+ "message": "cookies: "
+ }
+ ],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e6",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/cookie-echo.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Cookie Echo — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/cookie-echo.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e6",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/products/p1.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Product 1 — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/products/p1.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e6",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/products/p2.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Product 2 — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/products/p2.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e6",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/products/p3.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Product 3 — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/products/p3.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e6",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/products/p4.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Product 4 — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/products/p4.html"
+ },
+ {
+ "browser": {
+ "accessibilityScore": 100,
+ "consoleLogs": [],
+ "interactiveElements": [
+ {
+ "name": "Home",
+ "ref": "e6",
+ "role": "link",
+ "visible": true
+ }
+ ],
+ "networkRequests": [
+ {
+ "method": "GET",
+ "resourceType": "Document",
+ "status": 200,
+ "url": "http://fixtures/audit-site/products/p5.html"
+ }
+ ],
+ "screenshotPath": "",
+ "timingMetrics": {}
+ },
+ "title": "Product 5 — Audit Fixture Site",
+ "url": "http://fixtures/audit-site/products/p5.html"
+ }
+ ],
+ "schemaVersion": "1.0",
+ "securityFindings": [
+ {
+ "detail": "password form submits over plain http to http://fixtures/audit-site/login",
+ "ruleId": "insecure-password-form",
+ "severity": "high",
+ "url": "http://fixtures/audit-site/forms.html"
+ }
+ ],
+ "summaryScore": 97
+}
diff --git a/tests/e2e/fixtures/audit-site/normalize-report.jq b/tests/e2e/fixtures/audit-site/normalize-report.jq
new file mode 100644
index 00000000..0be066c5
--- /dev/null
+++ b/tests/e2e/fixtures/audit-site/normalize-report.jq
@@ -0,0 +1,25 @@
+# normalize-report.jq — strips the volatile fields from an AuditReport so
+# two runs of the same audit compare byte-identical (use with `jq -S`).
+#
+# Normalized away: the run timestamp, the run input (carries the caller's
+# FIXTURES_URL host), all timing metrics, screenshot payloads/paths, console
+# timestamps/sources, and network entry fields that vary per run (timings,
+# sizes, request ids); network entries are sorted by URL because subresource
+# completion order is nondeterministic.
+#
+# Regenerate the golden report (from a runner shell against the e2e stack):
+# curl -s -H "Authorization: Bearer $E2E_SERVER_TOKEN" -X POST "$E2E_SERVER/audit" \
+# -d '{"sitemapUrl":"'"$FIXTURES_URL"'/audit-site/sitemap.xml","options":{"screenshot":false},"concurrency":1}' \
+# | jq -S -f tests/e2e/fixtures/audit-site/normalize-report.jq \
+# > tests/e2e/fixtures/audit-site/golden-report.json
+.generatedAt = "0000-00-00T00:00:00Z"
+| .input = {}
+| .pages |= map(
+ del(.screenshot)
+ | .browser.screenshotPath = ""
+ | .browser.timingMetrics = {}
+ | .browser.consoleLogs = ((.browser.consoleLogs // []) | map({level: .level, message: .message}))
+ | .browser.networkRequests = ((.browser.networkRequests // [])
+ | map({url: .url, method: .method, status: .status, resourceType: .resourceType})
+ | sort_by(.url))
+ )
diff --git a/tests/e2e/scenarios/api/audit-errors-extended.sh b/tests/e2e/scenarios/api/audit-errors-extended.sh
new file mode 100755
index 00000000..0701322a
--- /dev/null
+++ b/tests/e2e/scenarios/api/audit-errors-extended.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+# audit-errors-extended.sh — audit error-path contract:
+# - unreachable host → HTTP 200, page entry carries `error`, run continues
+# - 404 entry page → HTTP 200, page audited normally (Chrome renders the
+# error document; the 404 shows in networkRequests)
+# - empty sitemap → HTTP 400 "no URLs to audit"
+# After every case the server stays healthy and the next audit succeeds.
+
+GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${GROUP_DIR}/../../helpers/api.sh"
+
+assert_healthy_and_recovers() {
+ pt_get /health
+ assert_ok "/health ok after the case"
+ pt_post /audit -d "{\"urls\":[\"${FIXTURES_URL}/audit-site/clean.html\"],\"options\":{\"screenshot\":false}}"
+ assert_ok "follow-up audit succeeds"
+ assert_json_eq "$RESULT" '.pages[0].error // "none"' "none" "follow-up page has no error"
+}
+
+# ─────────────────────────────────────────────────────────────────
+start_test "unreachable host produces a structured page error"
+
+pt_post /audit -d "{\"urls\":[\"http://fixtures:9/down.html\"],\"options\":{\"screenshot\":false}}"
+assert_ok "audit run returns 200"
+ERR=$(echo "$RESULT" | jq -r '.pages[0].error // empty')
+if [ -n "$ERR" ]; then
+ pass_assert "page entry carries error ($ERR)"
+else
+ fail_assert "page entry carries error"
+fi
+
+assert_healthy_and_recovers
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "404 entry page is audited as a page, not a failure"
+
+pt_post /audit -d "{\"urls\":[\"${FIXTURES_URL}/audit-site/no-such-page.html\"],\"options\":{\"screenshot\":false}}"
+assert_ok "audit run returns 200"
+assert_json_length "$RESULT" '.pages' 1 "one page entry"
+
+ERR=$(echo "$RESULT" | jq -r '.pages[0].error // empty')
+if [ -z "$ERR" ]; then
+ pass_assert "no error field (error page rendered and audited)"
+else
+ fail_assert "no error field (got: $ERR)"
+fi
+
+if echo "$RESULT" | jq -e '.pages[0].browser.networkRequests[]? | select(.url | endswith("no-such-page.html")) | select(.status == 404)' >/dev/null 2>&1; then
+ pass_assert "document 404 visible in networkRequests"
+else
+ fail_assert "document 404 visible in networkRequests (got: $(echo "$RESULT" | jq -c '[.pages[0].browser.networkRequests[]? | {url, status}]'))"
+fi
+
+assert_healthy_and_recovers
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "empty sitemap fails cleanly with no URLs to audit"
+
+STATUS=$(e2e_curl -s -o /tmp/audit-empty-sitemap.json -w '%{http_code}' -X POST "${E2E_SERVER}/audit" \
+ -d "{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/empty-sitemap.xml\"}")
+if [ "$STATUS" = "400" ]; then
+ pass_assert "empty sitemap returns HTTP 400"
+else
+ fail_assert "empty sitemap returns HTTP 400 (got: $STATUS)"
+fi
+
+if grep -q "no URLs to audit" /tmp/audit-empty-sitemap.json; then
+ pass_assert "error explains there is nothing to audit"
+else
+ fail_assert "error explains there is nothing to audit (got: $(cat /tmp/audit-empty-sitemap.json))"
+fi
+
+assert_healthy_and_recovers
+
+end_test
diff --git a/tests/e2e/scenarios/api/audit-repro-extended.sh b/tests/e2e/scenarios/api/audit-repro-extended.sh
new file mode 100755
index 00000000..e087f4cf
--- /dev/null
+++ b/tests/e2e/scenarios/api/audit-repro-extended.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# audit-repro-extended.sh — the audit reproducibility guarantee: same input,
+# same report. Uses the checked-in fixtures/audit-site/normalize-report.jq
+# to strip volatile fields; the golden report is
+# fixtures/audit-site/golden-report.json (regeneration instructions live in
+# the jq filter's header comment).
+
+GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${GROUP_DIR}/../../helpers/api.sh"
+
+NORMALIZE="${GROUP_DIR}/../../fixtures/audit-site/normalize-report.jq"
+GOLDEN="${GROUP_DIR}/../../fixtures/audit-site/golden-report.json"
+RUN_BODY="{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":1}"
+
+# ─────────────────────────────────────────────────────────────────
+start_test "two identical audit runs normalize byte-identical"
+
+pt_post /audit -d "$RUN_BODY"
+assert_ok "first audit run"
+RUN_ONE=$(echo "$RESULT" | jq -S -f "$NORMALIZE")
+
+pt_post /audit -d "$RUN_BODY"
+assert_ok "second audit run"
+RUN_TWO=$(echo "$RESULT" | jq -S -f "$NORMALIZE")
+
+if [ -n "$RUN_ONE" ] && [ "$RUN_ONE" = "$RUN_TWO" ]; then
+ pass_assert "normalized reports are byte-identical"
+else
+ fail_assert "normalized reports differ"
+ diff <(echo "$RUN_ONE") <(echo "$RUN_TWO") | head -20
+fi
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "normalized report matches the checked-in golden"
+
+if [ ! -f "$GOLDEN" ]; then
+ fail_assert "golden report exists at $GOLDEN"
+elif diff -q <(echo "$RUN_ONE") "$GOLDEN" >/dev/null 2>&1; then
+ pass_assert "live report matches golden-report.json"
+else
+ fail_assert "live report diverges from golden-report.json (schema/content drift; see normalize-report.jq header for how to regenerate)"
+ diff <(echo "$RUN_ONE") "$GOLDEN" | head -30
+fi
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "concurrency 4 audits the same page set as serial"
+
+pt_post /audit -d "{\"sitemapUrl\":\"${FIXTURES_URL}/audit-site/sitemap.xml\",\"options\":{\"screenshot\":false},\"concurrency\":4}"
+assert_ok "concurrent audit run"
+SET_FOUR=$(echo "$RESULT" | jq -S '[.pages[].url] | sort')
+SET_ONE=$(echo "$RUN_ONE" | jq -S '[.pages[].url] | sort')
+
+if [ "$SET_ONE" = "$SET_FOUR" ]; then
+ pass_assert "identical page URL set at concurrency 1 and 4"
+else
+ fail_assert "page sets differ: $SET_ONE vs $SET_FOUR"
+fi
+
+end_test
diff --git a/tests/e2e/scenarios/manifest.json b/tests/e2e/scenarios/manifest.json
index 86bc8a3e..20359b49 100644
--- a/tests/e2e/scenarios/manifest.json
+++ b/tests/e2e/scenarios/manifest.json
@@ -451,6 +451,26 @@
"library",
"smoke"
]
+ },
+ "api/audit-repro-extended.sh": {
+ "tags": [
+ "audit",
+ "repro",
+ "golden"
+ ]
+ },
+ "api/audit-errors-extended.sh": {
+ "tags": [
+ "audit",
+ "errors"
+ ]
+ },
+ "cli/compare-basic.sh": {
+ "tags": [
+ "compare",
+ "audit",
+ "visualdiff"
+ ]
}
}
}