Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(browser): report mismatched event family selection as unknown in capture comparison (#2239)
LouisChen0043d9b134c0
diff --git a/docs/browser-scenario-contract.md b/docs/browser-scenario-contract.mdindex b994d2936..403865be8 100644--- a/docs/browser-scenario-contract.md+++ b/docs/browser-scenario-contract.md@@ -312,3 +312,9 @@ artifacts carry their normalized before/after digests and capture states.Every changed or unknown artifact record is returned inline for each alignedstep. Missing, truncated, or mismatched capture coverage remains unknowninstead of being treated as equality.++`scenario.event_families` records the sorted, de-duplicated `capture.events`+selection. Captures with different selections report event evidence as unknown+with a recovery instruction, even when no selected family emitted an event.+Captures written before this field existed omit it, so their event evidence is+also unknown.diff --git a/src/browser/BrowserScenarioPartialObservation.ts b/src/browser/BrowserScenarioPartialObservation.tsindex 1fd395a64..ca0df4273 100644--- a/src/browser/BrowserScenarioPartialObservation.ts+++ b/src/browser/BrowserScenarioPartialObservation.ts@@ -89,6 +89,7 @@ export const browserScenarioCaptureData = (input: {.map(({ secret_id: id }) => id).sort(),network_content: input.scenario.capture.network,+ event_families: [...new Set(input.scenario.capture.events)].sort(),},duration_ms: Date.now() - input.startedAt,steps: [...input.steps],diff --git a/src/browser/PlaywrightBrowserScenarioProvider.test.ts b/src/browser/PlaywrightBrowserScenarioProvider.test.tsindex 6d4e284db..aacf766ad 100644--- a/src/browser/PlaywrightBrowserScenarioProvider.test.ts+++ b/src/browser/PlaywrightBrowserScenarioProvider.test.ts@@ -14,10 +14,13 @@ import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";import {browserStepArtifactsSchema,type BrowserScenarioEvent,+ type BrowserScenarioEventFamily,type BrowserStepArtifacts,} from "../domain/browserScenarioCaptureValues.js";import { PLAYWRIGHT_RESPONSE_BODY_PROVENANCE_LIMITATION } from "../domain/browserNetworkEvidence.js";import { projectAnalysisError } from "../domain/analysisErrorProjection.js";+import { compareBrowserScenarios } from "../domain/browserScenarioDiff.js";+import { compareBrowserScenariosInputSchema } from "../domain/browserScenarioDiffValues.js";import {AnalysisCancelledError,AnalysisOutputError,@@ -373,6 +376,56 @@ describe("PlaywrightBrowserScenarioProvider", () => {});});+describe("PlaywrightBrowserScenarioProvider comparison coverage", () => {+ it("keeps quiet captures with different event families from comparing unchanged", async () => {+ const capture = async (events: readonly BrowserScenarioEventFamily[]) => {+ const provider = new PlaywrightBrowserScenarioProvider(() =>+ Promise.resolve(new FakeSession("launch")),+ );+ const result = await provider.captureScenario(scenario({ events }));+ if (!result.ok) throw result.error;+ return result.value;+ };+ const compare = async (+ before: readonly BrowserScenarioEventFamily[],+ after: readonly BrowserScenarioEventFamily[],+ ) =>+ compareBrowserScenarios(+ compareBrowserScenariosInputSchema.parse({+ before_scenario: await capture(before),+ after_scenario: await capture(after),+ }),+ );++ const recorded = await capture(["page-errors", "console", "console"]);+ expect(recorded.completeness.equality_eligible).toBe(true);+ expect(recorded.scenario.event_families).toEqual([+ "console",+ "page-errors",+ ]);+ for (const [before, after] of [+ [[], ["console"]],+ [["console"], ["page-errors"]],+ ] as const) {+ const compared = await compare(before, after);+ expect(compared.overall_status).toBe("unknown");+ expect(+ compared.steps.flatMap((step) => step.artifact_diffs),+ ).toContainEqual(+ expect.objectContaining({+ artifact: "events",+ status: "unknown",+ reason: expect.stringContaining("Event family selections differ"),+ }),+ );+ }+ expect(+ (await compare(["console", "page-errors"], ["page-errors", "console"]))+ .overall_status,+ ).toBe("unchanged");+ });+});+describe("PlaywrightBrowserScenarioProvider cleanup", () => {it("returns a cleanup failure with completed observations when close fails", async () => {const session = new FakeSession("launch", {diff --git a/src/contracts/browserToolContracts.ts b/src/contracts/browserToolContracts.tsindex 7d5a23313..70111640a 100644--- a/src/contracts/browserToolContracts.ts+++ b/src/contracts/browserToolContracts.ts@@ -84,6 +84,7 @@ const exampleScenarioCapture = () => ({response_body: false,header_values: false,},+ event_families: [],},duration_ms: 10,steps: [diff --git a/src/domain/browserScenarioCapture.ts b/src/domain/browserScenarioCapture.tsindex 8ad389588..61c86b431 100644--- a/src/domain/browserScenarioCapture.ts+++ b/src/domain/browserScenarioCapture.ts@@ -3,6 +3,7 @@ import { browserNetworkContentSelectionValuesSchema } from "./browserNetworkEvidimport {browserScenarioCompletenessSchema,+ browserScenarioEventFamilySchema,browserScenarioEventSchema,browserScenarioStepSchema,} from "./browserScenarioCaptureValues.js";@@ -22,6 +23,7 @@ export const browserScenarioCaptureSchema = zaction_count: z.number().int().min(1),secret_references: z.array(z.string().min(1)),network_content: browserNetworkContentSelectionValuesSchema,+ event_families: z.array(browserScenarioEventFamilySchema).optional(),}),duration_ms: z.number().int().min(0),steps: z.array(browserScenarioStepSchema).min(1),diff --git a/src/domain/browserScenarioCaptureValues.ts b/src/domain/browserScenarioCaptureValues.tsindex 81fef0c15..26c1559bb 100644--- a/src/domain/browserScenarioCaptureValues.ts+++ b/src/domain/browserScenarioCaptureValues.ts@@ -119,6 +119,21 @@ export const browserStepArtifactsSchema = z.strictObject({});export type BrowserStepArtifacts = z.infer<typeof browserStepArtifactsSchema>;+/** Event families a scenario can select for capture. */+export const browserScenarioEventFamilySchema = z.enum([+ "console",+ "page-errors",+ "network",+ "websockets",+ "frames",+ "workers",+ "popups",+ "downloads",+]);+export type BrowserScenarioEventFamily = z.infer<+ typeof browserScenarioEventFamilySchema+>;+const eventBase = {sequence: z.number().int().min(1),step_index: z.number().int().min(0),diff --git a/src/domain/browserScenarioDiff.test.ts b/src/domain/browserScenarioDiff.test.tsindex 4c5272b9c..f15bd9da0 100644--- a/src/domain/browserScenarioDiff.test.ts+++ b/src/domain/browserScenarioDiff.test.ts@@ -62,6 +62,7 @@ const networkCapture = (sourceSequence: number, receiptStep: number) => {response_body: false,header_values: false,},+ event_families: ["network"],},events: { items: events, retained: 2, dropped: 0 },});@@ -146,6 +147,94 @@ it("reports content-selection mismatch as unknown instead of a website change",).toBe(false);});+it("reports event-family mismatch and legacy selection as unknown", () => {+ const consoleOnly = scenarioCapture({});+ const withFamilies = (+ capture: BrowserScenarioCapture,+ families: BrowserScenarioCapture["scenario"]["event_families"],+ ) =>+ browserScenarioCaptureSchema.parse({+ ...capture,+ scenario: { ...capture.scenario, event_families: families },+ });+ const compare = (+ before: BrowserScenarioCapture,+ after: BrowserScenarioCapture,+ ) =>+ compareBrowserScenarios(+ compareBrowserScenariosInputSchema.parse({+ before_scenario: before,+ after_scenario: after,+ }),+ );+ const eventDiffs = (compared: ReturnType<typeof compare>) =>+ compared.steps+ .flatMap((step) => step.artifact_diffs)+ .filter(({ artifact }) => artifact === "events");++ const widened = withFamilies(consoleOnly, ["page-errors", "console"]);+ const mismatched = compare(consoleOnly, widened);+ expect(mismatched.overall_status).toBe("unknown");+ expect(eventDiffs(mismatched)).toContainEqual(+ expect.objectContaining({+ status: "unknown",+ before_state: "captured",+ after_state: "captured",+ reason: expect.stringContaining(+ "Event family selections differ (console before, console, page-errors after)",+ ),+ }),+ );+ expect(+ compare(+ withFamilies(consoleOnly, ["console", "page-errors"]),+ withFamilies(consoleOnly, ["page-errors", "console"]),+ ).overall_status,+ ).toBe("unchanged");++ const legacy = withFamilies(consoleOnly, undefined);+ expect(legacy.scenario.event_families).toBeUndefined();+ for (const [before, after] of [+ [legacy, consoleOnly],+ [legacy, legacy],+ ] as const)+ expect(eventDiffs(compare(before, after))).toContainEqual(+ expect.objectContaining({+ status: "unknown",+ reason: expect.stringContaining("Event family selection is unknown"),+ }),+ );+ expect(+ compareBrowserScenariosInputSchema.safeParse({+ before_scenario: {+ ...consoleOnly,+ scenario: { ...consoleOnly.scenario, event_families: ["dialogs"] },+ },+ after_scenario: consoleOnly,+ }).success,+ ).toBe(false);++ const quiet = browserScenarioCaptureSchema.parse({+ ...withFamilies(consoleOnly, []),+ events: { retained: 0, dropped: 0, items: [] },+ });+ const differentEvents = compare(quiet, widened);+ expect(differentEvents.overall_status).toBe("unknown");+ expect(eventDiffs(differentEvents).map(({ status }) => status)).toEqual([+ "unknown",+ "unknown",+ ]);++ const both = eventDiffs(compare(networkCapture(1, 0), consoleOnly));+ expect(both).toContainEqual(+ expect.objectContaining({+ reason: expect.stringMatching(+ /Event family selections differ.*content selections differ/u,+ ),+ }),+ );+});+describe("browser scenario comparison", () => {it("records canonical literal normalization and ignores declared volatile fields", () => {const before = scenarioCapture({@@ -485,6 +574,7 @@ const scenarioCapture = (response_body: false,header_values: false,},+ event_families: ["console"],},duration_ms: options.elapsedMs ?? 10,steps: [diff --git a/src/domain/browserScenarioDiff.ts b/src/domain/browserScenarioDiff.tsindex acfe3dd63..2cb761a76 100644--- a/src/domain/browserScenarioDiff.ts+++ b/src/domain/browserScenarioDiff.ts@@ -5,6 +5,7 @@ import {artifactEvidenceState,captured,eventEvidenceState,+ eventFamilyCoverageReason,eventsFor,networkContentCoverageReason,screenshotProjection,@@ -332,7 +333,7 @@ const compareStepArtifacts = (),beforeState: eventEvidenceState(input.beforeCapture, input.beforeStep),afterState: eventEvidenceState(input.afterCapture, input.afterStep),- incomparableReason: networkContentCoverageReason(+ incomparableReason: eventCoverageReason(input.beforeCapture,input.afterCapture,),@@ -340,6 +341,17 @@ const compareStepArtifacts = (];};+const eventCoverageReason = (+ before: BrowserScenarioCapture,+ after: BrowserScenarioCapture,+): string | undefined => {+ const reasons = [+ eventFamilyCoverageReason(before, after),+ networkContentCoverageReason(before, after),+ ].filter((reason) => reason !== undefined);+ return reasons.length === 0 ? undefined : reasons.join(" ");+};+interface CompareCapturedValuesInput<Value> {readonly artifact: BrowserScenarioArtifactKind;readonly before: CaptureState<Value>;diff --git a/src/domain/browserScenarioDiffEvidence.ts b/src/domain/browserScenarioDiffEvidence.tsindex 015da4bff..f7f4b45e3 100644--- a/src/domain/browserScenarioDiffEvidence.ts+++ b/src/domain/browserScenarioDiffEvidence.ts@@ -130,6 +130,26 @@ export const networkContentCoverageReason = (: undefined;};+/** Different or unrecorded event-family selection is missing comparability, not an observed website change. */+export const eventFamilyCoverageReason = (+ before: BrowserScenarioCapture,+ after: BrowserScenarioCapture,+): string | undefined => {+ const left = before.scenario.event_families;+ const right = after.scenario.event_families;+ if (left === undefined || right === undefined)+ return "Event family selection is unknown in a legacy capture; capture both sides with matching event families before comparing event evidence.";+ const leftFamilies = new Set(left);+ const rightFamilies = new Set(right);+ return leftFamilies.size === rightFamilies.size &&+ [...leftFamilies].every((family) => rightFamilies.has(family))+ ? undefined+ : `Event family selections differ (${familyList(leftFamilies)} before, ${familyList(rightFamilies)} after); capture both sides with matching event families before comparing event evidence.`;+};++const familyList = (families: ReadonlySet<string>): string =>+ families.size === 0 ? "none" : [...families].sort().join(", ");+/** Wrap one value as captured comparison evidence. */export const captured = <Value>(value: Value): CaptureState<Value> => ({state: "captured",diff --git a/src/domain/browserScenarioValues.ts b/src/domain/browserScenarioValues.tsindex 544d4c222..b0344ae0e 100644--- a/src/domain/browserScenarioValues.ts+++ b/src/domain/browserScenarioValues.ts@@ -5,6 +5,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js";import { localPathStringSchema } from "./localPath.js";import { browserNetworkContentSelectionSchema } from "./browserNetworkEvidence.js";+import { browserScenarioEventFamilySchema } from "./browserScenarioCaptureValues.js";import {browserEndpointSchema,@@ -289,20 +290,7 @@ export const browserScenarioCaptureSchema = z.strictObject({after_each_step: z.array(snapshotKindSchema).default([]),at_end: z.array(snapshotKindSchema).default(["url"]),- events: z- .array(- z.enum([- "console",- "page-errors",- "network",- "websockets",- "frames",- "workers",- "popups",- "downloads",- ]),- )- .default([]),+ events: z.array(browserScenarioEventFamilySchema).default([]),network: browserNetworkContentSelectionSchema,}).default({diff --git a/tests/boundary/cli/browserCli.test.ts b/tests/boundary/cli/browserCli.test.tsindex 9084e87a9..7b2cb93ba 100644--- a/tests/boundary/cli/browserCli.test.ts+++ b/tests/boundary/cli/browserCli.test.ts@@ -352,6 +352,7 @@ const completeScenarioCapture = () => {response_body: false,header_values: false,},+ event_families: [],},duration_ms: 1,steps: [diff --git a/tests/boundary/mcp/browserScenarioMcp.test.ts b/tests/boundary/mcp/browserScenarioMcp.test.tsindex c6bfdd6e1..8e086ed06 100644--- a/tests/boundary/mcp/browserScenarioMcp.test.ts+++ b/tests/boundary/mcp/browserScenarioMcp.test.ts@@ -85,6 +85,7 @@ const captureFor = (scenario: BrowserScenario): BrowserScenarioCapture => {action_count: scenario.actions.length,secret_references: scenario.secrets.map(({ secret_id: id }) => id),network_content: scenario.capture.network,+ event_families: scenario.capture.events,},duration_ms: 2,steps: [diff --git a/tests/fixtures/webScriptCapture.ts b/tests/fixtures/webScriptCapture.tsindex 4d3a2ff64..0c6318531 100644--- a/tests/fixtures/webScriptCapture.ts+++ b/tests/fixtures/webScriptCapture.ts@@ -105,6 +105,7 @@ export const scriptScenarioFixture = (response_body: true,header_values: false,},+ event_families: ["network"],},duration_ms: 2,steps: [step(0), step(1)],