Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(browser): compare accessibility only with observed coverage

morluto0123d5976828
diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts
index de24a0fc3..ee0709ca1 100644
--- a/src/domain/webCaptureDiff.ts
+++ b/src/domain/webCaptureDiff.ts
@@ -114,29 +114,30 @@ const accessibilityDimension = (
const afterAccess = accessibilityComparable(after);
const textComparable = beforeAccess.text && afterAccess.text;
const nodesComparable = beforeAccess.nodes && afterAccess.nodes;
+ const reason =
+ "Accessibility tree or text capture was incomplete in at least one observation.";
+ if (!nodesComparable)
+ return {
+ status: "unknown",
+ total_changes: 0,
+ changes: [],
+ reason,
+ };
return compareDimension(
singleton(
"accessibility_tree",
digestCanonicalValue(
- accessibilityProjection(
- before.accessibility,
- textComparable,
- nodesComparable,
- ),
+ accessibilityProjection(before.accessibility, textComparable),
),
),
singleton(
"accessibility_tree",
digestCanonicalValue(
- accessibilityProjection(
- after.accessibility,
- textComparable,
- nodesComparable,
- ),
+ accessibilityProjection(after.accessibility, textComparable),
),
),
beforeAccess.complete && afterAccess.complete,
- "Accessibility tree or text capture was incomplete in at least one observation.",
+ reason,
);
};
@@ -850,20 +851,32 @@ const accessibilityComparable = (
const text =
inspection.accessibility.text_capture.status === "included" &&
inspection.accessibility.text_capture.excluded_fields === 0;
+ const { completeness } = inspection;
+ const exclusions = completeness.excluded.filter(
+ ({ section }) => section === "accessibility",
+ );
+ const textOnlyPolicyFilter =
+ inspection.accessibility.text_capture.status === "not_approved" &&
+ exclusions.length > 0 &&
+ exclusions.every(({ reason }) => reason === "not_approved");
const nodes =
+ [
+ completeness.truncated_sections,
+ completeness.unavailable_sections,
+ completeness.attach_limited_sections,
+ ].every((sections) => !sections.includes("accessibility")) &&
+ (!completeness.policy_filtered_sections.includes("accessibility") ||
+ textOnlyPolicyFilter) &&
inspection.accessibility.total_nodes ===
- inspection.accessibility.nodes.length;
- const complete =
- text && nodes && sectionsComplete(inspection, ["accessibility"]);
+ inspection.accessibility.nodes.length;
+ const complete = text && nodes;
return { text, nodes, complete };
};
const accessibilityProjection = (
accessibility: WebPageInspection["accessibility"],
includeText: boolean,
- includeNodes: boolean,
): unknown => {
- if (!includeNodes) return { total_nodes: accessibility.total_nodes };
const nodeIndexes = new Map(
accessibility.nodes.map((node, index) => [node.node_id, index]),
);
diff --git a/tests/boundary/cli/accessibilityCaptureComparison.test.ts b/tests/boundary/cli/accessibilityCaptureComparison.test.ts
new file mode 100644
index 000000000..61dc571ac
--- /dev/null
+++ b/tests/boundary/cli/accessibilityCaptureComparison.test.ts
@@ -0,0 +1,218 @@
+import { expect, it } from "vitest";
+
+import { classifyBrowserCompleteness } from "../../../src/domain/browserCompleteness.js";
+import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js";
+import { runCliJson } from "../../fixtures/cliJsonProcess.js";
+import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js";
+
+type Capture = WebPageInspection;
+type Coverage =
+ | "truncated"
+ | "unavailable"
+ | "attach_limited"
+ | "policy_filtered";
+
+const accessibleCapture = (): Capture => {
+ const capture = passiveNetworkCapture();
+ capture.accessibility = {
+ total_nodes: 2,
+ text_capture: {
+ status: "included",
+ retained_bytes: 10,
+ excluded_fields: 0,
+ },
+ nodes: [
+ {
+ node_id: "root",
+ parent_id: null,
+ role: "RootWebArea",
+ name: "Page",
+ description: null,
+ ignored: false,
+ states: [],
+ },
+ {
+ node_id: "button",
+ parent_id: "root",
+ role: "button",
+ name: "Submit",
+ description: null,
+ ignored: false,
+ states: [],
+ },
+ ],
+ };
+ return capture;
+};
+
+const setCoverage = (capture: Capture, coverage: Coverage): void => {
+ capture.completeness = classifyBrowserCompleteness({
+ policyFilteredSections: new Set(
+ coverage === "policy_filtered" ? ["accessibility"] : [],
+ ),
+ attachLimitedSections: new Set(
+ coverage === "attach_limited" ? ["accessibility"] : [],
+ ),
+ truncatedSections: new Set(
+ coverage === "truncated" ? ["accessibility"] : [],
+ ),
+ unavailableSections: new Set(
+ coverage === "unavailable" ? ["accessibility"] : [],
+ ),
+ excluded:
+ coverage === "policy_filtered"
+ ? [{ section: "accessibility", reason: "disallowed_origin", count: 1 }]
+ : coverage === "unavailable"
+ ? [
+ {
+ section: "accessibility",
+ reason: "invalid_protocol_value",
+ count: null,
+ },
+ ]
+ : [],
+ droppedEvents: {
+ scripts: 0,
+ network_requests: 0,
+ console_events: 0,
+ websocket_connections: 0,
+ websocket_frames: 0,
+ webmcp_tools: 0,
+ timeline_events: 0,
+ },
+ });
+};
+
+const compare = async (before: Capture, after: Capture) =>
+ await runCliJson(
+ [
+ "compare-web-captures",
+ JSON.stringify({ inspection: before }),
+ JSON.stringify({ inspection: after }),
+ "--json",
+ ],
+ process.env,
+ );
+
+it.each([
+ "truncated",
+ "unavailable",
+ "attach_limited",
+ "policy_filtered",
+] as const)(
+ "does not infer an accessibility change from a %s tree",
+ async (coverage) => {
+ const before = accessibleCapture();
+ const after = structuredClone(before);
+ after.accessibility.nodes.pop();
+ after.accessibility.total_nodes = after.accessibility.nodes.length;
+ after.accessibility.text_capture.retained_bytes = 4;
+ setCoverage(after, coverage);
+
+ const result = await compare(before, after);
+
+ expect(result).toMatchObject({
+ operation: "compare_web_captures",
+ normalized_result: {
+ dimensions: {
+ accessibility: {
+ status: "unknown",
+ total_changes: 0,
+ changes: [],
+ reason: expect.stringContaining("incomplete"),
+ },
+ },
+ },
+ });
+ },
+ 60_000,
+);
+
+it("keeps a count-mismatched tree unknown without a completeness marker", async () => {
+ const before = accessibleCapture();
+ const after = structuredClone(before);
+ after.accessibility.total_nodes += 1;
+
+ const result = await compare(before, after);
+
+ expect(result).toMatchObject({
+ normalized_result: {
+ dimensions: {
+ accessibility: { status: "unknown", total_changes: 0, changes: [] },
+ },
+ },
+ });
+});
+
+it("compares known structure when only text was not approved", async () => {
+ const before = accessibleCapture();
+ const after = structuredClone(before);
+ after.accessibility.text_capture = {
+ status: "not_approved",
+ retained_bytes: 0,
+ excluded_fields: 2,
+ };
+ for (const node of after.accessibility.nodes) {
+ node.name = null;
+ node.description = null;
+ }
+ after.completeness = classifyBrowserCompleteness({
+ policyFilteredSections: new Set(["accessibility"]),
+ attachLimitedSections: new Set(),
+ truncatedSections: new Set(),
+ unavailableSections: new Set(),
+ excluded: [{ section: "accessibility", reason: "not_approved", count: 2 }],
+ droppedEvents: {
+ scripts: 0,
+ network_requests: 0,
+ console_events: 0,
+ websocket_connections: 0,
+ websocket_frames: 0,
+ webmcp_tools: 0,
+ timeline_events: 0,
+ },
+ });
+ after.accessibility.nodes[1]!.role = "menuitem";
+
+ const result = await compare(before, after);
+
+ expect(result).toMatchObject({
+ normalized_result: {
+ dimensions: {
+ accessibility: {
+ status: "changed",
+ total_changes: 1,
+ changes: [{ identity: "accessibility_tree", change: "modified" }],
+ },
+ },
+ },
+ });
+});
+
+it("keeps independent resource changes visible beside an unknown tree", async () => {
+ const before = accessibleCapture();
+ const after = structuredClone(before);
+ after.accessibility.nodes.pop();
+ after.accessibility.total_nodes = after.accessibility.nodes.length;
+ setCoverage(after, "truncated");
+ after.resources.push({
+ resource_key: `res_${"a".repeat(64)}`,
+ url: "https://app.example.test/new.css",
+ origin: "https://app.example.test",
+ type: "Stylesheet",
+ mime_type: "text/css",
+ content_size: 10,
+ });
+
+ const result = await compare(before, after);
+
+ expect(result).toMatchObject({
+ normalized_result: {
+ overall_status: "changed",
+ dimensions: {
+ accessibility: { status: "unknown", total_changes: 0 },
+ resources: { status: "changed", total_changes: 1 },
+ },
+ },
+ });
+});