High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(a11y): publish the axe audit's vocabulary token so a returned ref is actionable, and pin the native report byte-for-byte (PIN-397)
Luigi Agostibdb7165451eb
diff --git a/internal/audit/a11y_test.go b/internal/audit/a11y_test.goindex 310ed7e9..b278b4de 100644--- a/internal/audit/a11y_test.go+++ b/internal/audit/a11y_test.go@@ -1,12 +1,18 @@package auditimport (+ "encoding/json"+ "flag"+ "os"+ "path/filepath""reflect""testing""github.com/pinchtab/pinchtab/internal/bridge/observe")+var updateGolden = flag.Bool("update", false, "rewrite the pinned native a11y report golden")+var cleanFacts = PageFacts{Title: "Page", Lang: "en", HeadingLevels: []int{1, 2}}func findingByRule(t *testing.T, report A11yReport, rule string) A11yFinding {@@ -20,6 +26,45 @@ func findingByRule(t *testing.T, report A11yReport, rule string) A11yFinding {return A11yFinding{}}+// The AC requires that omitting ?engine= keeps the native report byte-for-byte.+// The handler's native branch writes exactly what EvaluateA11y returns, so pinning+// its JSON for a fixed node list is the regression the axe work must not disturb:+// any drift in a native rule, its severity, its score weight or the field spelling+// reds here. Regenerate deliberately with `go test ./internal/audit -update`.+func TestNativeReportGoldenByteForByte(t *testing.T) {+ nodes := []observe.A11yNode{+ {Role: "image", Ref: "e1"},+ {Role: "image", Name: "logo"},+ {Role: "textbox", Ref: "e2"},+ {Role: "link"},+ {Role: "button"},+ {Role: "button", Name: "Go"},+ }+ facts := PageFacts{HeadingLevels: []int{1, 3, 4, 6}}++ got, err := json.MarshalIndent(EvaluateA11y(nodes, facts), "", " ")+ if err != nil {+ t.Fatalf("marshal report: %v", err)+ }+ got = append(got, '\n')++ path := filepath.Join("testdata", "a11y-native-report.golden.json")+ if *updateGolden {+ if err := os.WriteFile(path, got, 0644); err != nil {+ t.Fatalf("write golden: %v", err)+ }+ return+ }+ want, err := os.ReadFile(path)+ if err != nil {+ t.Fatalf("read golden: %v\nRun `go test ./internal/audit -update` and review the diff.", err)+ }+ if string(got) != string(want) {+ t.Fatalf("native a11y report drifted from the golden (omitting engine must stay byte-for-byte).\n"++ "If the change is intended, run `go test ./internal/audit -update`, review the diff, and commit it.\n\n got:\n%s\nwant:\n%s", got, want)+ }+}+func TestCleanTreeScoresPerfect(t *testing.T) {nodes := []observe.A11yNode{{Role: "image", Name: "logo"},diff --git a/internal/audit/testdata/a11y-native-report.golden.json b/internal/audit/testdata/a11y-native-report.golden.jsonnew file mode 100644index 00000000..89d29516--- /dev/null+++ b/internal/audit/testdata/a11y-native-report.golden.json@@ -0,0 +1,62 @@+{+ "score": 56,+ "findings": [+ {+ "rule": "empty-button",+ "severity": "moderate",+ "count": 1,+ "samples": [+ "button"+ ]+ },+ {+ "rule": "empty-link",+ "severity": "moderate",+ "count": 1,+ "samples": [+ "link"+ ]+ },+ {+ "rule": "heading-skip",+ "severity": "minor",+ "count": 2,+ "samples": [+ "h3 follows h1",+ "h6 follows h4"+ ]+ },+ {+ "rule": "missing-alt",+ "severity": "serious",+ "count": 1,+ "samples": [+ "image e1"+ ]+ },+ {+ "rule": "missing-label",+ "severity": "serious",+ "count": 1,+ "samples": [+ "textbox e2"+ ]+ },+ {+ "rule": "missing-lang",+ "severity": "moderate",+ "count": 1,+ "samples": [+ "html element has no lang attribute"+ ]+ },+ {+ "rule": "missing-title",+ "severity": "moderate",+ "count": 1,+ "samples": [+ "document.title is empty"+ ]+ }+ ]+}diff --git a/internal/bridge/isolated_eval.go b/internal/bridge/isolated_eval.goindex 5cad504f..b0b06e91 100644--- a/internal/bridge/isolated_eval.go+++ b/internal/bridge/isolated_eval.go@@ -29,10 +29,15 @@ func BackendNodeIDForSelector(ctx context.Context, frameID, selector string) (inreturn 0, err}+ encodedSelector, err := json.Marshal(selector)+ if err != nil {+ return 0, err+ }+var raw json.RawMessageif err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {return chromedp.FromContext(ctx).Target.Execute(ctx, "Runtime.evaluate", map[string]any{- "expression": fmt.Sprintf(`document.querySelector(%q)`, selector),+ "expression": fmt.Sprintf(`document.querySelector(%s)`, encodedSelector),"contextId": execID,}, &raw)})); err != nil {diff --git a/internal/cli/actions/actions_a11y.go b/internal/cli/actions/actions_a11y.goindex 4011d59b..471461b9 100644--- a/internal/cli/actions/actions_a11y.go+++ b/internal/cli/actions/actions_a11y.go@@ -39,11 +39,16 @@ func A11yAudit(client *http.Client, base, token string, cmd *cobra.Command) {params.Set("selector", v)}+ // The axe engine re-epochs the tab's ref cache and publishes a fresh+ // vocabulary token; capture it the way snapshot does (keyed by the resolved+ // tab, implicit when no --tab) so a later action on a returned ref echoes the+ // token the audit minted instead of a stale one the server would refuse 409.+ implicit := params.Get("tabId") == ""if jsonOutput, _ := cmd.Flags().GetBool("json"); jsonOutput {- apiclient.DoGet(client, base, token, "/a11y/audit", params)+ apiclient.DoGetCapturingVocab(client, base, token, "/a11y/audit", params, implicit)return}- printA11ySummary(apiclient.DoGetRaw(client, base, token, "/a11y/audit", params))+ printA11ySummary(apiclient.DoGetRawCapturingVocab(client, base, token, "/a11y/audit", params, implicit))}// printA11ySummary renders a compact human summary for either engine, fallingdiff --git a/internal/cli/apiclient/client.go b/internal/cli/apiclient/client.goindex 3a392a58..41379181 100644--- a/internal/cli/apiclient/client.go+++ b/internal/cli/apiclient/client.go@@ -171,6 +171,21 @@ func DoGetRaw(client *http.Client, base, token, path string, params url.Values)return body}+// DoGetRawCapturingVocab is DoGetRaw plus the vocabulary capture of+// DoGetCapturingVocab: it returns the raw body unprinted (for a caller that+// renders its own summary) and persists the response's vocabulary token so a+// later action echoes it. The a11y axe audit needs this: it re-epochs the tab's+// ref cache, so an action on a returned ref would be refused 409 against a stale+// token if the CLI never captured the fresh one.+func DoGetRawCapturingVocab(client *http.Client, base, token, path string, params url.Values, implicit bool) []byte {+ var headers http.Header+ r := request{method: "GET", url: buildURL(base, path, params), respHeaders: &headers}+ status, body := mustRequest(client, token, r)+ exitOnAPIError(r, status, body)+ storeVocabToken(base, headers.Get(vocabTabIDHeader), headers.Get(vocabHeader), implicit)+ return body+}+// DoGetRawAndPrint fetches and prints the raw response body (for --snap flag).// Best-effort: it reports errors to stderr but does not exit.func DoGetRawAndPrint(client *http.Client, base, token, pathWithQuery string) {diff --git a/internal/handlers/a11y_axe.go b/internal/handlers/a11y_axe.goindex 8c53f983..687e81fd 100644--- a/internal/handlers/a11y_axe.go+++ b/internal/handlers/a11y_axe.go@@ -2,6 +2,7 @@ package handlersimport ("context"+ "encoding/json""fmt""net/http""strings"@@ -42,12 +43,14 @@ func (h *Handlers) runAxeAudit(w http.ResponseWriter, r *http.Request, tCtx cont}report := audit.BuildAxeReport(raw, assets.AxeVersion, includeIncomplete)- h.fillAxeRefs(tCtx, resolvedTabID, &report)+ vocab := h.fillAxeRefs(tCtx, resolvedTabID, &report)+ w.Header().Set(vocabHeader, vocab)httpx.JSON(w, 200, struct {- TabID string `json:"tabId"`+ TabID string `json:"tabId"`+ VocabularyToken string `json:"vocabularyToken"`audit.AxeReport- }{resolvedTabID, report})+ }{resolvedTabID, vocab, report})}// injectAxeIntoAllFrames loads the axe source into every reachable frame's@@ -74,7 +77,8 @@ func injectAxeIntoAllFrames(tCtx context.Context) {func axeRunSnippet(config, selector string) string {target := "document"if selector != "" {- target = fmt.Sprintf("(document.querySelector(%q) || document)", selector)+ encoded, _ := json.Marshal(selector)+ target = fmt.Sprintf("(document.querySelector(%s) || document)", encoded)}return fmt.Sprintf(`(function(){return axe.run(%s, %s).then(function(r){@@ -94,10 +98,15 @@ func axeRunSnippet(config, selector string) string {// an agent can actuate a failing element directly. A node whose target does not// resolve to a snapshot ref (a cross-origin iframe, a node not in the tree) is// left without one rather than guessed.-func (h *Handlers) fillAxeRefs(tCtx context.Context, tabID string, report *audit.AxeReport) {- backendToRef := h.snapshotBackendRefs(tCtx, tabID)+// fillAxeRefs returns the vocabulary token of the ref cache the audit published,+// so the handler can set the X-PinchTab-Vocab header and body field as /snapshot+// and /capture do — the audit mints a fresh token whenever it re-epochs the tab+// (first read, or any read after a navigation), and a client that never learns it+// would echo a stale one and be refused 409 on the ref the audit just handed out.+func (h *Handlers) fillAxeRefs(tCtx context.Context, tabID string, report *audit.AxeReport) string {+ backendToRef, vocab := h.snapshotBackendRefs(tCtx, tabID)if len(backendToRef) == 0 {- return+ return vocab}fill := func(violations []audit.AxeViolation) {for i := range violations {@@ -122,6 +131,7 @@ func (h *Handlers) fillAxeRefs(tCtx context.Context, tabID string, report *audit}fill(report.Violations)fill(report.IncompleteViolations)+ return vocab}// snapshotBackendRefs builds a full-tree snapshot, stores it as the tab's ref@@ -131,10 +141,10 @@ func (h *Handlers) fillAxeRefs(tCtx context.Context, tabID string, report *audit// computed it without publishing the snapshot it came from. A full tree (not the// interactive filter) is used so non-interactive violation targets — images,// low-contrast text — still map to a ref.-func (h *Handlers) snapshotBackendRefs(tCtx context.Context, tabID string) map[int64]string {+func (h *Handlers) snapshotBackendRefs(tCtx context.Context, tabID string) (map[int64]string, string) {rawNodes, err := bridge.FetchAXTree(tCtx)if err != nil {- return nil+ return nil, ""}flat, _ := bridge.BuildSnapshot(rawNodes, "", -1)cache := bridge.EpochRefs(h.Bridge.GetRefCache(tabID), flat)@@ -146,7 +156,7 @@ func (h *Handlers) snapshotBackendRefs(tCtx context.Context, tabID string) map[irefs[n.NodeID] = n.Ref}}- return refs+ return refs, cache.DomEpoch}// splitCSVParam splits a comma list query value into trimmed, non-empty items.diff --git a/internal/mcp/tools_a11y.go b/internal/mcp/tools_a11y.goindex f4617391..3949fc07 100644--- a/internal/mcp/tools_a11y.go+++ b/internal/mcp/tools_a11y.go@@ -40,6 +40,11 @@ func handleA11yAudit(c *Client) func(context.Context, mcp.CallToolRequest) (*mcpif v, ok := optBool(r, "includeIncomplete"); ok && v {q.Set("includeIncomplete", "true")}- return toolResult(c.Get(ctx, "/a11y/audit", routedQuery(r, q)))+ // The axe engine re-epochs the tab's ref cache and publishes a fresh+ // vocabulary token. Capture it under the request's tab key (as snapshot+ // does) so a later pinchtab_click/fill on a returned ref echoes that token+ // rather than a stale one the server refuses 409. The native engine sends+ // no token, so this is a no-op there.+ return toolResult(c.GetCapturingVocab(ctx, "/a11y/audit", routedQuery(r, q), optString(r, "tabId")))}}diff --git a/internal/mcp/tools_a11y_test.go b/internal/mcp/tools_a11y_test.gonew file mode 100644index 00000000..5091cc44--- /dev/null+++ b/internal/mcp/tools_a11y_test.go@@ -0,0 +1,100 @@+package mcp++import (+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "testing"+)++// axeAuditMockServer answers /a11y/audit?engine=axe with one ref-carrying+// violation and stamps the vocabulary token the audit "minted", and echoes each+// /action body so the test can read the token the follow-up interaction sent.+func axeAuditMockServer(token string) *httptest.Server {+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/a11y/audit" {+ w.Header().Set(vocabHeader, token)+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(200)+ _ = json.NewEncoder(w).Encode(map[string]any{+ "engine": "axe",+ "version": "4.13.0",+ "vocabularyToken": token,+ "violations": []map[string]any{{+ "id": "image-alt",+ "impact": "critical",+ "tags": []string{"wcag2a", "wcag111"},+ "helpUrl": "https://dequeuniversity.com/rules/axe/4.13/image-alt",+ "nodes": []map[string]any{{"target": []string{"img"}, "html": "<img src=x>", "ref": "e7"}},+ }},+ "score": 90,+ })+ return+ }+ resp := map[string]any{"path": r.URL.Path}+ if r.Method == http.MethodPost {+ body, _ := io.ReadAll(r.Body)+ var parsed map[string]any+ if json.Unmarshal(body, &parsed) == nil {+ resp["body"] = parsed+ }+ }+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(200)+ _ = json.NewEncoder(w).Encode(resp)+ }))+}++// The in-process tool-result test the harness calls for: drive pinchtab_a11y_audit+// with engine=axe against a stub endpoint and assert the tool hands back the axe+// violations with a ref an agent can act on.+func TestA11yAuditAxeToolReturnsViolationsWithRefs(t *testing.T) {+ srv := axeAuditMockServer("vocab-axe-1")+ defer srv.Close()+ c := NewClient(srv.URL, "")++ result := callSharedTool(t, c, "pinchtab_a11y_audit", map[string]any{"engine": "axe", "tabId": "t1"})+ body := resultJSON(t, result)++ if body["engine"] != "axe" {+ t.Fatalf("engine = %v, want axe", body["engine"])+ }+ violations, _ := body["violations"].([]any)+ if len(violations) == 0 {+ t.Fatal("tool result carried no violations")+ }+ first, _ := violations[0].(map[string]any)+ if first["id"] != "image-alt" || first["impact"] != "critical" {+ t.Fatalf("violation = %v, want image-alt/critical", first)+ }+ nodes, _ := first["nodes"].([]any)+ if len(nodes) == 0 {+ t.Fatal("violation carried no nodes")+ }+ node0, _ := nodes[0].(map[string]any)+ if node0["ref"] != "e7" {+ t.Fatalf("node ref = %v, want e7 so the agent can act on it", node0["ref"])+ }+}++// Item 1 at the MCP surface: the audit re-epochs the tab and mints a fresh+// vocabulary token, so the tool must capture it and the next interaction must+// echo it. Without this a nav -> a11y audit --axe -> click flow echoes a stale+// token and the server refuses the very ref the audit just handed out with 409.+func TestA11yAuditAxeTokenIsEchoedOnTheNextAction(t *testing.T) {+ srv := axeAuditMockServer("vocab-axe-1")+ defer srv.Close()+ c := NewClient(srv.URL, "")++ callSharedTool(t, c, "pinchtab_a11y_audit", map[string]any{"engine": "axe", "tabId": "t1"})+ clickResult := callSharedTool(t, c, "pinchtab_click", map[string]any{"ref": "e7", "tabId": "t1"})++ got, ok := actionVocabSent(t, clickResult)+ if !ok {+ t.Fatal("the click carried no vocab token, so the ref the audit minted would be refused 409")+ }+ if got != "vocab-axe-1" {+ t.Errorf("click echoed vocab %q, want the token the axe audit returned", got)+ }+}diff --git a/tests/e2e/scenarios/api/a11y-axe-basic.sh b/tests/e2e/scenarios/api/a11y-axe-basic.shindex 32012b99..a36e0ff0 100755--- a/tests/e2e/scenarios/api/a11y-axe-basic.sh+++ b/tests/e2e/scenarios/api/a11y-axe-basic.sh@@ -51,7 +51,11 @@ fiend_test# ─────────────────────────────────────────────────────────────────-start_test "a11y axe: a ref-mapped node is actionable via /action"+# The audit re-epochs the tab's ref cache and mints a fresh vocabulary token, so+# acting on a returned ref must echo THAT token. An agent that kept a pre-audit+# token (the nav -> audit -> click flow the card sells) would be refused 409, so+# the scenario captures the audit's token and proves both directions.+start_test "a11y axe: a ref-mapped node is actionable with the audit's vocab token"REF=$(echo "$RESULT" | jq -r 'first(.violations[] | select(.id=="label") | .nodes[] | select(.ref!=null and .ref!="") | .ref)')if [ -n "$REF" ] && [ "$REF" != "null" ]; then@@ -62,8 +66,20 @@ else((ASSERTIONS_FAILED++)) || truefi-pt_post /action -d "{\"kind\":\"focus\",\"ref\":\"${REF}\"}"-assert_ok "focus the failing element by its axe ref"+VOCAB=$(echo "$RESULT" | jq -r '.vocabularyToken')+if [ -n "$VOCAB" ] && [ "$VOCAB" != "null" ]; then+ echo -e " ${GREEN}✓${NC} audit published a vocabulary token $VOCAB"+ ((ASSERTIONS_PASSED++)) || true+else+ echo -e " ${RED}✗${NC} audit published no vocabulary token, so an agent cannot act on its refs"+ ((ASSERTIONS_FAILED++)) || true+fi++pt_post /action -d "{\"kind\":\"focus\",\"ref\":\"${REF}\",\"vocab\":\"${VOCAB}\"}"+assert_ok "focus the failing element echoing the audit's own token"++pt_post /action -d "{\"kind\":\"focus\",\"ref\":\"${REF}\",\"vocab\":\"stale-pre-audit-token\"}"+assert_http_status 409 "a pre-audit token is refused vocab_superseded"end_test