Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(a11y): publish the axe audit's vocabulary token so a returned ref is actionable, and pin the native report byte-for-byte (PIN-397)

Luigi Agostibdb7165451eb
diff --git a/internal/audit/a11y_test.go b/internal/audit/a11y_test.go
index 310ed7e9..b278b4de 100644
--- a/internal/audit/a11y_test.go
+++ b/internal/audit/a11y_test.go
@@ -1,12 +1,18 @@
package audit
import (
+ "encoding/json"
+ "flag"
+ "os"
+ "path/filepath"
"reflect"
"testing"
"github.com/pinchtab/pinchtab/internal/bridge/observe"
)
+var updateGolden = flag.Bool("update", false, "rewrite the pinned native a11y report golden")
+
var cleanFacts = PageFacts{Title: "Page", Lang: "en", HeadingLevels: []int{1, 2}}
func findingByRule(t *testing.T, report A11yReport, rule string) A11yFinding {
@@ -20,6 +26,45 @@ func findingByRule(t *testing.T, report A11yReport, rule string) A11yFinding {
return A11yFinding{}
}
+// The AC requires that omitting ?engine= keeps the native report byte-for-byte.
+// The handler's native branch writes exactly what EvaluateA11y returns, so pinning
+// its JSON for a fixed node list is the regression the axe work must not disturb:
+// any drift in a native rule, its severity, its score weight or the field spelling
+// reds here. Regenerate deliberately with `go test ./internal/audit -update`.
+func TestNativeReportGoldenByteForByte(t *testing.T) {
+ nodes := []observe.A11yNode{
+ {Role: "image", Ref: "e1"},
+ {Role: "image", Name: "logo"},
+ {Role: "textbox", Ref: "e2"},
+ {Role: "link"},
+ {Role: "button"},
+ {Role: "button", Name: "Go"},
+ }
+ facts := PageFacts{HeadingLevels: []int{1, 3, 4, 6}}
+
+ got, err := json.MarshalIndent(EvaluateA11y(nodes, facts), "", " ")
+ if err != nil {
+ t.Fatalf("marshal report: %v", err)
+ }
+ got = append(got, '\n')
+
+ path := filepath.Join("testdata", "a11y-native-report.golden.json")
+ if *updateGolden {
+ if err := os.WriteFile(path, got, 0644); err != nil {
+ t.Fatalf("write golden: %v", err)
+ }
+ return
+ }
+ want, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read golden: %v\nRun `go test ./internal/audit -update` and review the diff.", err)
+ }
+ if string(got) != string(want) {
+ t.Fatalf("native a11y report drifted from the golden (omitting engine must stay byte-for-byte).\n"+
+ "If the change is intended, run `go test ./internal/audit -update`, review the diff, and commit it.\n\n got:\n%s\nwant:\n%s", got, want)
+ }
+}
+
func TestCleanTreeScoresPerfect(t *testing.T) {
nodes := []observe.A11yNode{
{Role: "image", Name: "logo"},
diff --git a/internal/audit/testdata/a11y-native-report.golden.json b/internal/audit/testdata/a11y-native-report.golden.json
new file mode 100644
index 00000000..89d29516
--- /dev/null
+++ b/internal/audit/testdata/a11y-native-report.golden.json
@@ -0,0 +1,62 @@
+{
+ "score": 56,
+ "findings": [
+ {
+ "rule": "empty-button",
+ "severity": "moderate",
+ "count": 1,
+ "samples": [
+ "button"
+ ]
+ },
+ {
+ "rule": "empty-link",
+ "severity": "moderate",
+ "count": 1,
+ "samples": [
+ "link"
+ ]
+ },
+ {
+ "rule": "heading-skip",
+ "severity": "minor",
+ "count": 2,
+ "samples": [
+ "h3 follows h1",
+ "h6 follows h4"
+ ]
+ },
+ {
+ "rule": "missing-alt",
+ "severity": "serious",
+ "count": 1,
+ "samples": [
+ "image e1"
+ ]
+ },
+ {
+ "rule": "missing-label",
+ "severity": "serious",
+ "count": 1,
+ "samples": [
+ "textbox e2"
+ ]
+ },
+ {
+ "rule": "missing-lang",
+ "severity": "moderate",
+ "count": 1,
+ "samples": [
+ "html element has no lang attribute"
+ ]
+ },
+ {
+ "rule": "missing-title",
+ "severity": "moderate",
+ "count": 1,
+ "samples": [
+ "document.title is empty"
+ ]
+ }
+ ]
+}
diff --git a/internal/bridge/isolated_eval.go b/internal/bridge/isolated_eval.go
index 5cad504f..b0b06e91 100644
--- a/internal/bridge/isolated_eval.go
+++ b/internal/bridge/isolated_eval.go
@@ -29,10 +29,15 @@ func BackendNodeIDForSelector(ctx context.Context, frameID, selector string) (in
return 0, err
}
+ encodedSelector, err := json.Marshal(selector)
+ if err != nil {
+ return 0, err
+ }
+
var raw json.RawMessage
if err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.evaluate", map[string]any{
- "expression": fmt.Sprintf(`document.querySelector(%q)`, selector),
+ "expression": fmt.Sprintf(`document.querySelector(%s)`, encodedSelector),
"contextId": execID,
}, &raw)
})); err != nil {
diff --git a/internal/cli/actions/actions_a11y.go b/internal/cli/actions/actions_a11y.go
index 4011d59b..471461b9 100644
--- a/internal/cli/actions/actions_a11y.go
+++ b/internal/cli/actions/actions_a11y.go
@@ -39,11 +39,16 @@ func A11yAudit(client *http.Client, base, token string, cmd *cobra.Command) {
params.Set("selector", v)
}
+ // The axe engine re-epochs the tab's ref cache and publishes a fresh
+ // vocabulary token; capture it the way snapshot does (keyed by the resolved
+ // tab, implicit when no --tab) so a later action on a returned ref echoes the
+ // token the audit minted instead of a stale one the server would refuse 409.
+ implicit := params.Get("tabId") == ""
if jsonOutput, _ := cmd.Flags().GetBool("json"); jsonOutput {
- apiclient.DoGet(client, base, token, "/a11y/audit", params)
+ apiclient.DoGetCapturingVocab(client, base, token, "/a11y/audit", params, implicit)
return
}
- printA11ySummary(apiclient.DoGetRaw(client, base, token, "/a11y/audit", params))
+ printA11ySummary(apiclient.DoGetRawCapturingVocab(client, base, token, "/a11y/audit", params, implicit))
}
// printA11ySummary renders a compact human summary for either engine, falling
diff --git a/internal/cli/apiclient/client.go b/internal/cli/apiclient/client.go
index 3a392a58..41379181 100644
--- a/internal/cli/apiclient/client.go
+++ b/internal/cli/apiclient/client.go
@@ -171,6 +171,21 @@ func DoGetRaw(client *http.Client, base, token, path string, params url.Values)
return body
}
+// DoGetRawCapturingVocab is DoGetRaw plus the vocabulary capture of
+// DoGetCapturingVocab: it returns the raw body unprinted (for a caller that
+// renders its own summary) and persists the response's vocabulary token so a
+// later action echoes it. The a11y axe audit needs this: it re-epochs the tab's
+// ref cache, so an action on a returned ref would be refused 409 against a stale
+// token if the CLI never captured the fresh one.
+func DoGetRawCapturingVocab(client *http.Client, base, token, path string, params url.Values, implicit bool) []byte {
+ var headers http.Header
+ r := request{method: "GET", url: buildURL(base, path, params), respHeaders: &headers}
+ status, body := mustRequest(client, token, r)
+ exitOnAPIError(r, status, body)
+ storeVocabToken(base, headers.Get(vocabTabIDHeader), headers.Get(vocabHeader), implicit)
+ return body
+}
+
// DoGetRawAndPrint fetches and prints the raw response body (for --snap flag).
// Best-effort: it reports errors to stderr but does not exit.
func DoGetRawAndPrint(client *http.Client, base, token, pathWithQuery string) {
diff --git a/internal/handlers/a11y_axe.go b/internal/handlers/a11y_axe.go
index 8c53f983..687e81fd 100644
--- a/internal/handlers/a11y_axe.go
+++ b/internal/handlers/a11y_axe.go
@@ -2,6 +2,7 @@ package handlers
import (
"context"
+ "encoding/json"
"fmt"
"net/http"
"strings"
@@ -42,12 +43,14 @@ func (h *Handlers) runAxeAudit(w http.ResponseWriter, r *http.Request, tCtx cont
}
report := audit.BuildAxeReport(raw, assets.AxeVersion, includeIncomplete)
- h.fillAxeRefs(tCtx, resolvedTabID, &report)
+ vocab := h.fillAxeRefs(tCtx, resolvedTabID, &report)
+ w.Header().Set(vocabHeader, vocab)
httpx.JSON(w, 200, struct {
- TabID string `json:"tabId"`
+ TabID string `json:"tabId"`
+ VocabularyToken string `json:"vocabularyToken"`
audit.AxeReport
- }{resolvedTabID, report})
+ }{resolvedTabID, vocab, report})
}
// injectAxeIntoAllFrames loads the axe source into every reachable frame's
@@ -74,7 +77,8 @@ func injectAxeIntoAllFrames(tCtx context.Context) {
func axeRunSnippet(config, selector string) string {
target := "document"
if selector != "" {
- target = fmt.Sprintf("(document.querySelector(%q) || document)", selector)
+ encoded, _ := json.Marshal(selector)
+ target = fmt.Sprintf("(document.querySelector(%s) || document)", encoded)
}
return fmt.Sprintf(`(function(){
return axe.run(%s, %s).then(function(r){
@@ -94,10 +98,15 @@ func axeRunSnippet(config, selector string) string {
// an agent can actuate a failing element directly. A node whose target does not
// resolve to a snapshot ref (a cross-origin iframe, a node not in the tree) is
// left without one rather than guessed.
-func (h *Handlers) fillAxeRefs(tCtx context.Context, tabID string, report *audit.AxeReport) {
- backendToRef := h.snapshotBackendRefs(tCtx, tabID)
+// fillAxeRefs returns the vocabulary token of the ref cache the audit published,
+// so the handler can set the X-PinchTab-Vocab header and body field as /snapshot
+// and /capture do — the audit mints a fresh token whenever it re-epochs the tab
+// (first read, or any read after a navigation), and a client that never learns it
+// would echo a stale one and be refused 409 on the ref the audit just handed out.
+func (h *Handlers) fillAxeRefs(tCtx context.Context, tabID string, report *audit.AxeReport) string {
+ backendToRef, vocab := h.snapshotBackendRefs(tCtx, tabID)
if len(backendToRef) == 0 {
- return
+ return vocab
}
fill := func(violations []audit.AxeViolation) {
for i := range violations {
@@ -122,6 +131,7 @@ func (h *Handlers) fillAxeRefs(tCtx context.Context, tabID string, report *audit
}
fill(report.Violations)
fill(report.IncompleteViolations)
+ return vocab
}
// snapshotBackendRefs builds a full-tree snapshot, stores it as the tab's ref
@@ -131,10 +141,10 @@ func (h *Handlers) fillAxeRefs(tCtx context.Context, tabID string, report *audit
// computed it without publishing the snapshot it came from. A full tree (not the
// interactive filter) is used so non-interactive violation targets — images,
// low-contrast text — still map to a ref.
-func (h *Handlers) snapshotBackendRefs(tCtx context.Context, tabID string) map[int64]string {
+func (h *Handlers) snapshotBackendRefs(tCtx context.Context, tabID string) (map[int64]string, string) {
rawNodes, err := bridge.FetchAXTree(tCtx)
if err != nil {
- return nil
+ return nil, ""
}
flat, _ := bridge.BuildSnapshot(rawNodes, "", -1)
cache := bridge.EpochRefs(h.Bridge.GetRefCache(tabID), flat)
@@ -146,7 +156,7 @@ func (h *Handlers) snapshotBackendRefs(tCtx context.Context, tabID string) map[i
refs[n.NodeID] = n.Ref
}
}
- return refs
+ return refs, cache.DomEpoch
}
// splitCSVParam splits a comma list query value into trimmed, non-empty items.
diff --git a/internal/mcp/tools_a11y.go b/internal/mcp/tools_a11y.go
index f4617391..3949fc07 100644
--- a/internal/mcp/tools_a11y.go
+++ b/internal/mcp/tools_a11y.go
@@ -40,6 +40,11 @@ func handleA11yAudit(c *Client) func(context.Context, mcp.CallToolRequest) (*mcp
if v, ok := optBool(r, "includeIncomplete"); ok && v {
q.Set("includeIncomplete", "true")
}
- return toolResult(c.Get(ctx, "/a11y/audit", routedQuery(r, q)))
+ // The axe engine re-epochs the tab's ref cache and publishes a fresh
+ // vocabulary token. Capture it under the request's tab key (as snapshot
+ // does) so a later pinchtab_click/fill on a returned ref echoes that token
+ // rather than a stale one the server refuses 409. The native engine sends
+ // no token, so this is a no-op there.
+ return toolResult(c.GetCapturingVocab(ctx, "/a11y/audit", routedQuery(r, q), optString(r, "tabId")))
}
}
diff --git a/internal/mcp/tools_a11y_test.go b/internal/mcp/tools_a11y_test.go
new file mode 100644
index 00000000..5091cc44
--- /dev/null
+++ b/internal/mcp/tools_a11y_test.go
@@ -0,0 +1,100 @@
+package mcp
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+// axeAuditMockServer answers /a11y/audit?engine=axe with one ref-carrying
+// violation and stamps the vocabulary token the audit "minted", and echoes each
+// /action body so the test can read the token the follow-up interaction sent.
+func axeAuditMockServer(token string) *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/a11y/audit" {
+ w.Header().Set(vocabHeader, token)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(200)
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "engine": "axe",
+ "version": "4.13.0",
+ "vocabularyToken": token,
+ "violations": []map[string]any{{
+ "id": "image-alt",
+ "impact": "critical",
+ "tags": []string{"wcag2a", "wcag111"},
+ "helpUrl": "https://dequeuniversity.com/rules/axe/4.13/image-alt",
+ "nodes": []map[string]any{{"target": []string{"img"}, "html": "<img src=x>", "ref": "e7"}},
+ }},
+ "score": 90,
+ })
+ return
+ }
+ resp := map[string]any{"path": r.URL.Path}
+ if r.Method == http.MethodPost {
+ body, _ := io.ReadAll(r.Body)
+ var parsed map[string]any
+ if json.Unmarshal(body, &parsed) == nil {
+ resp["body"] = parsed
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(200)
+ _ = json.NewEncoder(w).Encode(resp)
+ }))
+}
+
+// The in-process tool-result test the harness calls for: drive pinchtab_a11y_audit
+// with engine=axe against a stub endpoint and assert the tool hands back the axe
+// violations with a ref an agent can act on.
+func TestA11yAuditAxeToolReturnsViolationsWithRefs(t *testing.T) {
+ srv := axeAuditMockServer("vocab-axe-1")
+ defer srv.Close()
+ c := NewClient(srv.URL, "")
+
+ result := callSharedTool(t, c, "pinchtab_a11y_audit", map[string]any{"engine": "axe", "tabId": "t1"})
+ body := resultJSON(t, result)
+
+ if body["engine"] != "axe" {
+ t.Fatalf("engine = %v, want axe", body["engine"])
+ }
+ violations, _ := body["violations"].([]any)
+ if len(violations) == 0 {
+ t.Fatal("tool result carried no violations")
+ }
+ first, _ := violations[0].(map[string]any)
+ if first["id"] != "image-alt" || first["impact"] != "critical" {
+ t.Fatalf("violation = %v, want image-alt/critical", first)
+ }
+ nodes, _ := first["nodes"].([]any)
+ if len(nodes) == 0 {
+ t.Fatal("violation carried no nodes")
+ }
+ node0, _ := nodes[0].(map[string]any)
+ if node0["ref"] != "e7" {
+ t.Fatalf("node ref = %v, want e7 so the agent can act on it", node0["ref"])
+ }
+}
+
+// Item 1 at the MCP surface: the audit re-epochs the tab and mints a fresh
+// vocabulary token, so the tool must capture it and the next interaction must
+// echo it. Without this a nav -> a11y audit --axe -> click flow echoes a stale
+// token and the server refuses the very ref the audit just handed out with 409.
+func TestA11yAuditAxeTokenIsEchoedOnTheNextAction(t *testing.T) {
+ srv := axeAuditMockServer("vocab-axe-1")
+ defer srv.Close()
+ c := NewClient(srv.URL, "")
+
+ callSharedTool(t, c, "pinchtab_a11y_audit", map[string]any{"engine": "axe", "tabId": "t1"})
+ clickResult := callSharedTool(t, c, "pinchtab_click", map[string]any{"ref": "e7", "tabId": "t1"})
+
+ got, ok := actionVocabSent(t, clickResult)
+ if !ok {
+ t.Fatal("the click carried no vocab token, so the ref the audit minted would be refused 409")
+ }
+ if got != "vocab-axe-1" {
+ t.Errorf("click echoed vocab %q, want the token the axe audit returned", got)
+ }
+}
diff --git a/tests/e2e/scenarios/api/a11y-axe-basic.sh b/tests/e2e/scenarios/api/a11y-axe-basic.sh
index 32012b99..a36e0ff0 100755
--- a/tests/e2e/scenarios/api/a11y-axe-basic.sh
+++ b/tests/e2e/scenarios/api/a11y-axe-basic.sh
@@ -51,7 +51,11 @@ fi
end_test
# ─────────────────────────────────────────────────────────────────
-start_test "a11y axe: a ref-mapped node is actionable via /action"
+# The audit re-epochs the tab's ref cache and mints a fresh vocabulary token, so
+# acting on a returned ref must echo THAT token. An agent that kept a pre-audit
+# token (the nav -> audit -> click flow the card sells) would be refused 409, so
+# the scenario captures the audit's token and proves both directions.
+start_test "a11y axe: a ref-mapped node is actionable with the audit's vocab token"
REF=$(echo "$RESULT" | jq -r 'first(.violations[] | select(.id=="label") | .nodes[] | select(.ref!=null and .ref!="") | .ref)')
if [ -n "$REF" ] && [ "$REF" != "null" ]; then
@@ -62,8 +66,20 @@ else
((ASSERTIONS_FAILED++)) || true
fi
-pt_post /action -d "{\"kind\":\"focus\",\"ref\":\"${REF}\"}"
-assert_ok "focus the failing element by its axe ref"
+VOCAB=$(echo "$RESULT" | jq -r '.vocabularyToken')
+if [ -n "$VOCAB" ] && [ "$VOCAB" != "null" ]; then
+ echo -e " ${GREEN}✓${NC} audit published a vocabulary token $VOCAB"
+ ((ASSERTIONS_PASSED++)) || true
+else
+ echo -e " ${RED}✗${NC} audit published no vocabulary token, so an agent cannot act on its refs"
+ ((ASSERTIONS_FAILED++)) || true
+fi
+
+pt_post /action -d "{\"kind\":\"focus\",\"ref\":\"${REF}\",\"vocab\":\"${VOCAB}\"}"
+assert_ok "focus the failing element echoing the audit's own token"
+
+pt_post /action -d "{\"kind\":\"focus\",\"ref\":\"${REF}\",\"vocab\":\"stale-pre-audit-token\"}"
+assert_http_status 409 "a pre-audit token is refused vocab_superseded"
end_test