Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(browser): compare accessibility only with observed coverage
morluto0123d5976828
diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.tsindex de24a0fc3..ee0709ca1 100644--- a/src/domain/webCaptureDiff.ts+++ b/src/domain/webCaptureDiff.ts@@ -114,29 +114,30 @@ const accessibilityDimension = (const afterAccess = accessibilityComparable(after);const textComparable = beforeAccess.text && afterAccess.text;const nodesComparable = beforeAccess.nodes && afterAccess.nodes;+ const reason =+ "Accessibility tree or text capture was incomplete in at least one observation.";+ if (!nodesComparable)+ return {+ status: "unknown",+ total_changes: 0,+ changes: [],+ reason,+ };return compareDimension(singleton("accessibility_tree",digestCanonicalValue(- accessibilityProjection(- before.accessibility,- textComparable,- nodesComparable,- ),+ accessibilityProjection(before.accessibility, textComparable),),),singleton("accessibility_tree",digestCanonicalValue(- accessibilityProjection(- after.accessibility,- textComparable,- nodesComparable,- ),+ accessibilityProjection(after.accessibility, textComparable),),),beforeAccess.complete && afterAccess.complete,- "Accessibility tree or text capture was incomplete in at least one observation.",+ reason,);};@@ -850,20 +851,32 @@ const accessibilityComparable = (const text =inspection.accessibility.text_capture.status === "included" &&inspection.accessibility.text_capture.excluded_fields === 0;+ const { completeness } = inspection;+ const exclusions = completeness.excluded.filter(+ ({ section }) => section === "accessibility",+ );+ const textOnlyPolicyFilter =+ inspection.accessibility.text_capture.status === "not_approved" &&+ exclusions.length > 0 &&+ exclusions.every(({ reason }) => reason === "not_approved");const nodes =+ [+ completeness.truncated_sections,+ completeness.unavailable_sections,+ completeness.attach_limited_sections,+ ].every((sections) => !sections.includes("accessibility")) &&+ (!completeness.policy_filtered_sections.includes("accessibility") ||+ textOnlyPolicyFilter) &&inspection.accessibility.total_nodes ===- inspection.accessibility.nodes.length;- const complete =- text && nodes && sectionsComplete(inspection, ["accessibility"]);+ inspection.accessibility.nodes.length;+ const complete = text && nodes;return { text, nodes, complete };};const accessibilityProjection = (accessibility: WebPageInspection["accessibility"],includeText: boolean,- includeNodes: boolean,): unknown => {- if (!includeNodes) return { total_nodes: accessibility.total_nodes };const nodeIndexes = new Map(accessibility.nodes.map((node, index) => [node.node_id, index]),);diff --git a/tests/boundary/cli/accessibilityCaptureComparison.test.ts b/tests/boundary/cli/accessibilityCaptureComparison.test.tsnew file mode 100644index 000000000..61dc571ac--- /dev/null+++ b/tests/boundary/cli/accessibilityCaptureComparison.test.ts@@ -0,0 +1,218 @@+import { expect, it } from "vitest";++import { classifyBrowserCompleteness } from "../../../src/domain/browserCompleteness.js";+import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js";+import { runCliJson } from "../../fixtures/cliJsonProcess.js";+import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js";++type Capture = WebPageInspection;+type Coverage =+ | "truncated"+ | "unavailable"+ | "attach_limited"+ | "policy_filtered";++const accessibleCapture = (): Capture => {+ const capture = passiveNetworkCapture();+ capture.accessibility = {+ total_nodes: 2,+ text_capture: {+ status: "included",+ retained_bytes: 10,+ excluded_fields: 0,+ },+ nodes: [+ {+ node_id: "root",+ parent_id: null,+ role: "RootWebArea",+ name: "Page",+ description: null,+ ignored: false,+ states: [],+ },+ {+ node_id: "button",+ parent_id: "root",+ role: "button",+ name: "Submit",+ description: null,+ ignored: false,+ states: [],+ },+ ],+ };+ return capture;+};++const setCoverage = (capture: Capture, coverage: Coverage): void => {+ capture.completeness = classifyBrowserCompleteness({+ policyFilteredSections: new Set(+ coverage === "policy_filtered" ? ["accessibility"] : [],+ ),+ attachLimitedSections: new Set(+ coverage === "attach_limited" ? ["accessibility"] : [],+ ),+ truncatedSections: new Set(+ coverage === "truncated" ? ["accessibility"] : [],+ ),+ unavailableSections: new Set(+ coverage === "unavailable" ? ["accessibility"] : [],+ ),+ excluded:+ coverage === "policy_filtered"+ ? [{ section: "accessibility", reason: "disallowed_origin", count: 1 }]+ : coverage === "unavailable"+ ? [+ {+ section: "accessibility",+ reason: "invalid_protocol_value",+ count: null,+ },+ ]+ : [],+ droppedEvents: {+ scripts: 0,+ network_requests: 0,+ console_events: 0,+ websocket_connections: 0,+ websocket_frames: 0,+ webmcp_tools: 0,+ timeline_events: 0,+ },+ });+};++const compare = async (before: Capture, after: Capture) =>+ await runCliJson(+ [+ "compare-web-captures",+ JSON.stringify({ inspection: before }),+ JSON.stringify({ inspection: after }),+ "--json",+ ],+ process.env,+ );++it.each([+ "truncated",+ "unavailable",+ "attach_limited",+ "policy_filtered",+] as const)(+ "does not infer an accessibility change from a %s tree",+ async (coverage) => {+ const before = accessibleCapture();+ const after = structuredClone(before);+ after.accessibility.nodes.pop();+ after.accessibility.total_nodes = after.accessibility.nodes.length;+ after.accessibility.text_capture.retained_bytes = 4;+ setCoverage(after, coverage);++ const result = await compare(before, after);++ expect(result).toMatchObject({+ operation: "compare_web_captures",+ normalized_result: {+ dimensions: {+ accessibility: {+ status: "unknown",+ total_changes: 0,+ changes: [],+ reason: expect.stringContaining("incomplete"),+ },+ },+ },+ });+ },+ 60_000,+);++it("keeps a count-mismatched tree unknown without a completeness marker", async () => {+ const before = accessibleCapture();+ const after = structuredClone(before);+ after.accessibility.total_nodes += 1;++ const result = await compare(before, after);++ expect(result).toMatchObject({+ normalized_result: {+ dimensions: {+ accessibility: { status: "unknown", total_changes: 0, changes: [] },+ },+ },+ });+});++it("compares known structure when only text was not approved", async () => {+ const before = accessibleCapture();+ const after = structuredClone(before);+ after.accessibility.text_capture = {+ status: "not_approved",+ retained_bytes: 0,+ excluded_fields: 2,+ };+ for (const node of after.accessibility.nodes) {+ node.name = null;+ node.description = null;+ }+ after.completeness = classifyBrowserCompleteness({+ policyFilteredSections: new Set(["accessibility"]),+ attachLimitedSections: new Set(),+ truncatedSections: new Set(),+ unavailableSections: new Set(),+ excluded: [{ section: "accessibility", reason: "not_approved", count: 2 }],+ droppedEvents: {+ scripts: 0,+ network_requests: 0,+ console_events: 0,+ websocket_connections: 0,+ websocket_frames: 0,+ webmcp_tools: 0,+ timeline_events: 0,+ },+ });+ after.accessibility.nodes[1]!.role = "menuitem";++ const result = await compare(before, after);++ expect(result).toMatchObject({+ normalized_result: {+ dimensions: {+ accessibility: {+ status: "changed",+ total_changes: 1,+ changes: [{ identity: "accessibility_tree", change: "modified" }],+ },+ },+ },+ });+});++it("keeps independent resource changes visible beside an unknown tree", async () => {+ const before = accessibleCapture();+ const after = structuredClone(before);+ after.accessibility.nodes.pop();+ after.accessibility.total_nodes = after.accessibility.nodes.length;+ setCoverage(after, "truncated");+ after.resources.push({+ resource_key: `res_${"a".repeat(64)}`,+ url: "https://app.example.test/new.css",+ origin: "https://app.example.test",+ type: "Stylesheet",+ mime_type: "text/css",+ content_size: 10,+ });++ const result = await compare(before, after);++ expect(result).toMatchObject({+ normalized_result: {+ overall_status: "changed",+ dimensions: {+ accessibility: { status: "unknown", total_changes: 0 },+ resources: { status: "changed", total_changes: 1 },+ },+ },+ });+});