Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(browser): retain scenario observations across failures

morluto614e0c2e7481
diff --git a/scripts/fixtures/browser-verifier-site.mjs b/scripts/fixtures/browser-verifier-site.mjs
index d5ccb7923..7a5095e52 100644
--- a/scripts/fixtures/browser-verifier-site.mjs
+++ b/scripts/fixtures/browser-verifier-site.mjs
@@ -7,6 +7,10 @@ export async function startBrowserVerifierSite() {
let port = 0;
let sessionGeneration = 0;
const server = createServer((request, response) => {
+ if (request.url === "/failed-navigation") {
+ response.destroy();
+ return;
+ }
if (request.url === "/slow-json") {
response.setHeader("content-type", "application/json");
response.write('{"pending":');
diff --git a/scripts/lib/browser-scenario-verifier.mjs b/scripts/lib/browser-scenario-verifier.mjs
index a41e7cec5..7a56f990c 100644
--- a/scripts/lib/browser-scenario-verifier.mjs
+++ b/scripts/lib/browser-scenario-verifier.mjs
@@ -1,9 +1,14 @@
+import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { readdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { promisify } from "node:util";
+import { fileURLToPath } from "node:url";
+import { Client } from "@modelcontextprotocol/client";
+import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { browserScenarioSchema } from "../../dist/domain/browserScenario.js";
+import { requireMcpToolError } from "./mcp-verifier-results.mjs";
const execute = promisify(execFile);
@@ -142,6 +147,73 @@ export async function runScenarioCli(scenario) {
return JSON.parse(stdout);
}
+/** Verify failed real navigation retains observations through both public adapters. */
+export async function verifyScenarioFailureEvidence(executable, origin) {
+ const scenario = browserScenarioSchema.parse({
+ browser: { mode: "launch", executable_path: executable },
+ start_url: { url: `${origin}/failed-navigation` },
+ actions: [
+ { step_id: "unused", action: "wait_for_timeout", duration_ms: 1 },
+ ],
+ capture: { events: ["network"] },
+ });
+ const profilesBefore = await scenarioProfiles();
+ let cliFailure;
+ try {
+ await runScenarioCli(scenario);
+ assert.fail("The fixture must fail before initial navigation completes");
+ } catch (cause) {
+ assert.equal(typeof cause.stdout, "string");
+ cliFailure = JSON.parse(cause.stdout);
+ }
+ assertFailedNavigation(cliFailure, scenario.start_url.url);
+ const transport = new StdioClientTransport({
+ command: process.execPath,
+ args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],
+ env: process.env,
+ stderr: "pipe",
+ });
+ const client = new Client({
+ name: "browser-scenario-failure-e2e",
+ version: "1",
+ });
+ try {
+ await client.connect(transport);
+ const result = await client.callTool({
+ name: "capture_browser_scenario",
+ arguments: scenario,
+ });
+ assert.equal(result.isError, true);
+ assertFailedNavigation(requireMcpToolError(result), scenario.start_url.url);
+ } finally {
+ await client.close();
+ await transport.close();
+ }
+ const profilesAfter = await scenarioProfiles();
+ assert.ok([...profilesAfter].every((profile) => profilesBefore.has(profile)));
+ return {
+ mocked: false,
+ cli: true,
+ stdio_mcp: true,
+ failed_navigation_events_retained: true,
+ profile_cleanup: true,
+ };
+}
+
+const assertFailedNavigation = (error, requestedUrl) => {
+ assert.equal(error?.code, "execution_failure");
+ const observation = error.details?.partial_observation;
+ assert.equal(observation?.kind, "browser-scenario-observation");
+ assert.equal(observation.capture.browser.cleanup, "terminated-owned-process");
+ assert.deepEqual(observation.capture.steps, []);
+ assert.ok(
+ observation.capture.events.items.some(
+ (event) =>
+ event.kind === "request-failed" && event.url.url === requestedUrl,
+ ),
+ );
+};
+
/** Snapshot provider-owned profile names for cleanup comparison. */
export async function scenarioProfiles() {
return new Set(
diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs
index 58ae384ee..5f05981d6 100644
--- a/scripts/verify-real-browser.mjs
+++ b/scripts/verify-real-browser.mjs
@@ -16,7 +16,7 @@ import {
listBrowserTargetsInputSchema,
} from "../dist/domain/browserObservation.js";
import { observeWebSessionInputSchema } from "../dist/domain/browserSession.js";
-import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiff.js";
+import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiffSchemas.js";
import {
captureWebScreenshotInputSchema,
compareWebScreenshotsInputSchema,
@@ -33,6 +33,7 @@ import {
browserScenario,
runScenarioCli,
scenarioProfiles,
+ verifyScenarioFailureEvidence,
} from "./lib/browser-scenario-verifier.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import { verifyLargeScreenshotE2e } from "./lib/browser-screenshot-e2e.mjs";
@@ -294,6 +295,10 @@ try {
if ([...profilesAfter].some((entry) => !profilesBefore.has(entry)))
throw new Error("Scenario launch retained a temporary browser profile");
assertScenarioCapture(launchedScenario.value);
+ const scenarioFailure = await verifyScenarioFailureEvidence(
+ executable,
+ site.origin,
+ );
const scenarioResults = JSON.stringify([
attachedScenario,
launchedScenario.value,
@@ -360,6 +365,7 @@ try {
browserScenarioCli: true,
browserScenarioAttachCleanup: "disconnected-external",
browserScenarioLaunchCleanup: "terminated-owned-process",
+ scenarioFailure,
verified: true,
};
} finally {
diff --git a/src/browser/BrowserScenarioPartialObservation.ts b/src/browser/BrowserScenarioPartialObservation.ts
new file mode 100644
index 000000000..a2e600bcd
--- /dev/null
+++ b/src/browser/BrowserScenarioPartialObservation.ts
@@ -0,0 +1,165 @@
+import type { BrowserScenario } from "../domain/browserScenario.js";
+import type {
+ BrowserScenarioCaptureData,
+ BrowserScenarioPartialObservation,
+} from "../domain/browserScenarioCapture.js";
+import type { BrowserScenarioStep } from "../domain/browserScenarioCaptureValues.js";
+import type { BrowserScenarioSessionPort } from "./BrowserScenarioSessionPort.js";
+import { AnalysisError } from "../domain/analysisErrorBase.js";
+import {
+ AnalysisCancelledError,
+ AnalysisTimeoutError,
+} from "../domain/analysisErrorCore.js";
+import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
+import { BrowserObservationError } from "../domain/browserObservationError.js";
+import { ProviderAdapterError } from "../domain/providerAdapterError.js";
+import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "./providerIdentities.js";
+
+const OPERATION = "capture_browser_scenario" as const;
+
+const browserScenarioLimitations = (
+ session: BrowserScenarioSessionPort,
+): string[] => [
+ "Event sequence records provider receipt order; simultaneous browser causality is not inferred.",
+ "Network content is retained only when selected; response bytes are browser-decoded, not compressed wire bytes.",
+ "Request bytes are limited to what Playwright exposes; not_exposed does not establish body absence or multipart file coverage.",
+ "Network content reads settle within 5 seconds each and never wait for unfinished responses or refetch them.",
+ "Playwright scenario events do not expose request initiator stacks; receipt order does not prove causality.",
+ "Storage values are hashed only after declared-secret redaction.",
+ ...(session.eventLimitations?.() ?? []),
+ ...(session.mode === "connect"
+ ? [
+ "CDP attachment cannot recover pre-attach events or guarantee launch-time context options.",
+ ]
+ : []),
+];
+
+/** Project one session into the common capture fields used by success and failure. */
+export const browserScenarioCaptureData = (input: {
+ readonly session: BrowserScenarioSessionPort;
+ readonly scenario: BrowserScenario;
+ readonly startedAt: number;
+ readonly steps: readonly BrowserScenarioStep[];
+ readonly cleanup:
+ | "terminated-owned-process"
+ | "disconnected-external"
+ | "incomplete";
+ readonly limitations?: readonly string[];
+}): BrowserScenarioCaptureData => ({
+ browser: {
+ mode: input.session.mode,
+ process_ownership: input.session.processOwnership,
+ cleanup: input.cleanup,
+ product: input.session.product,
+ version: input.session.version,
+ },
+ scenario: {
+ start_origin: new URL(input.scenario.start_url.url).origin,
+ action_count: input.scenario.actions.length,
+ secret_references: input.scenario.secrets
+ .map(({ secret_id: id }) => id)
+ .sort(),
+ network_content: input.scenario.capture.network,
+ },
+ duration_ms: Date.now() - input.startedAt,
+ steps: [...input.steps],
+ events: input.session.events(),
+ limitations: [
+ ...browserScenarioLimitations(input.session),
+ ...(input.limitations ?? []),
+ ],
+});
+
+/** Describe the cleanup failure without discarding its resource evidence. */
+export const browserScenarioCleanupObservation = (
+ cause: unknown,
+): { readonly reason: string; readonly resources: readonly string[] } =>
+ cause instanceof AnalysisError && cause.cleanup !== undefined
+ ? cause.cleanup
+ : {
+ reason: cause instanceof Error ? cause.message : String(cause),
+ resources:
+ cause instanceof AnalysisError && cause.cleanupResources.length > 0
+ ? cause.cleanupResources
+ : ["browser_transport"],
+ };
+
+/** Retain operation diagnostics and capture data after cleanup settles. */
+export const browserScenarioOperationFailure = (
+ cause: unknown,
+ partialObservation: BrowserScenarioPartialObservation,
+ cleanupFailure?: { readonly cause: unknown },
+): AnalysisError => {
+ const primary =
+ cause instanceof AnalysisError
+ ? cause
+ : new ProviderAdapterError(
+ PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id,
+ OPERATION,
+ { cause },
+ );
+ const primaryProjection = projectAnalysisError(primary);
+ const cleanupCause =
+ cleanupFailure === undefined
+ ? undefined
+ : cleanupFailure.cause instanceof AnalysisError
+ ? cleanupFailure.cause
+ : new ProviderAdapterError(
+ PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id,
+ "close_browser_scenario",
+ { cause: cleanupFailure.cause },
+ );
+ const cleanupObservation =
+ cleanupFailure === undefined
+ ? undefined
+ : browserScenarioCleanupObservation(cleanupFailure.cause);
+ const retainedOptions = {
+ cause: primary,
+ partialObservation,
+ ...(primary.capturedOutput === undefined
+ ? {}
+ : { capturedOutput: primary.capturedOutput }),
+ ...(primary.cleanup === undefined ? {} : { cleanup: primary.cleanup }),
+ };
+ if (cleanupCause === undefined) {
+ if (primary instanceof AnalysisCancelledError)
+ return new AnalysisCancelledError(primary.operation, retainedOptions);
+ if (primary instanceof AnalysisTimeoutError)
+ return new AnalysisTimeoutError(
+ primary.operation,
+ primary.timeoutMs,
+ retainedOptions,
+ );
+ if (primary instanceof BrowserObservationError)
+ return new BrowserObservationError(primary.operation, primary.reason, {
+ ...retainedOptions,
+ ...(primary.userMessage === undefined
+ ? {}
+ : { detail: primary.userMessage }),
+ });
+ }
+ const cleanup = cleanupObservation ?? primary.cleanup;
+ return new ProviderAdapterError(
+ PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY.id,
+ OPERATION,
+ {
+ ...retainedOptions,
+ cause:
+ cleanupCause === undefined
+ ? primary
+ : new AggregateError(
+ [primary, cleanupCause],
+ "Browser scenario operation and cleanup both failed",
+ { cause: primary },
+ ),
+ userMessage: primaryProjection.message,
+ ...(cleanup === undefined ? {} : { cleanup }),
+ diagnostics: {
+ primary_error: primaryProjection,
+ ...(cleanupCause === undefined
+ ? {}
+ : { cleanup_error: projectAnalysisError(cleanupCause) }),
+ },
+ },
+ );
+};
diff --git a/src/browser/PlaywrightBrowserScenarioProvider.test.ts b/src/browser/PlaywrightBrowserScenarioProvider.test.ts
index 96242a844..fd3cd8869 100644
--- a/src/browser/PlaywrightBrowserScenarioProvider.test.ts
+++ b/src/browser/PlaywrightBrowserScenarioProvider.test.ts
@@ -9,13 +9,19 @@ import { PlaywrightScenarioBrowserCleanupOwner } from "./PlaywrightScenarioBrows
import {
browserScenarioSchema,
type BrowserScenario,
- type BrowserScenarioAction,
} from "../domain/browserScenario.js";
+import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
import {
browserStepArtifactsSchema,
type BrowserScenarioEvent,
type BrowserStepArtifacts,
-} from "../domain/browserScenarioCapture.js";
+} from "../domain/browserScenarioCaptureValues.js";
+import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
+import {
+ AnalysisCancelledError,
+ AnalysisOutputError,
+ AnalysisTimeoutError,
+} from "../domain/analysisErrorCore.js";
const scenario = (
options: {
@@ -70,6 +76,13 @@ const scenario = (
type SnapshotKind = BrowserScenario["capture"]["after_each_step"][number];
+interface FakeSessionFailures {
+ readonly incomplete?: boolean;
+ readonly action?: Error;
+ readonly capture?: { readonly call: number; readonly error: Error };
+ readonly close?: Error;
+}
+
const artifactState = (
kind: SnapshotKind,
requested: ReadonlySet<SnapshotKind>,
@@ -101,13 +114,11 @@ class FakeSession implements BrowserScenarioSessionPort {
captureCalls = 0;
onPerform: (() => void) | undefined;
eventGaps: readonly string[] = [];
- private step = 0;
+ eventItems: readonly BrowserScenarioEvent[] = [];
constructor(
readonly mode: "launch" | "connect",
- private readonly incomplete = false,
- private readonly failAction = false,
- private readonly failCapture = false,
+ private readonly failures: FakeSessionFailures = {},
) {
this.processOwnership = mode === "launch" ? "provider-owned" : "external";
}
@@ -120,9 +131,7 @@ class FakeSession implements BrowserScenarioSessionPort {
return sanitizeBrowserUrl(value);
}
- setStep(index: number) {
- this.step = index;
- }
+ setStep(_index: number) {}
nextEventSequence() {
return 1;
@@ -137,7 +146,11 @@ class FakeSession implements BrowserScenarioSessionPort {
readonly dropped: number;
readonly items: readonly BrowserScenarioEvent[];
} {
- return { retained: 0, dropped: 0, items: [] };
+ return {
+ retained: this.eventItems.length,
+ dropped: 0,
+ items: this.eventItems,
+ };
}
eventLimitations(): readonly string[] {
@@ -148,7 +161,7 @@ class FakeSession implements BrowserScenarioSessionPort {
this.performCalls += 1;
this.onPerform?.();
if (signal?.aborted === true) throw new Error("request cancelled");
- if (this.failAction) throw new Error("fixture secret action failure");
+ if (this.failures.action !== undefined) throw this.failures.action;
}
capture(
@@ -156,25 +169,40 @@ class FakeSession implements BrowserScenarioSessionPort {
_signal?: AbortSignal,
): Promise<BrowserStepArtifacts> {
this.captureCalls += 1;
- if (this.failCapture) return Promise.reject(new Error("capture failed"));
+ if (this.failures.capture?.call === this.captureCalls)
+ return Promise.reject(this.failures.capture.error);
return Promise.resolve(
browserStepArtifactsSchema.parse({
- screenshot: artifactState("screenshot", requested, this.incomplete),
- dom: artifactState("dom", requested, this.incomplete),
+ screenshot: artifactState(
+ "screenshot",
+ requested,
+ this.failures.incomplete === true,
+ ),
+ dom: artifactState("dom", requested, this.failures.incomplete === true),
accessibility: artifactState(
"accessibility",
requested,
- this.incomplete,
+ this.failures.incomplete === true,
+ ),
+ url: artifactState("url", requested, this.failures.incomplete === true),
+ history: artifactState(
+ "history",
+ requested,
+ this.failures.incomplete === true,
+ ),
+ storage: artifactState(
+ "storage",
+ requested,
+ this.failures.incomplete === true,
),
- url: artifactState("url", requested, this.incomplete),
- history: artifactState("history", requested, this.incomplete),
- storage: artifactState("storage", requested, this.incomplete),
}),
);
}
close() {
this.closeCalls += 1;
+ if (this.failures.close !== undefined)
+ return Promise.reject(this.failures.close);
return Promise.resolve(
this.mode === "launch"
? ("terminated-owned-process" as const)
@@ -220,7 +248,7 @@ describe("PlaywrightBrowserScenarioProvider", () => {
});
it("makes missing and truncated captures ineligible for equality", async () => {
- const session = new FakeSession("launch", true);
+ const session = new FakeSession("launch", { incomplete: true });
const provider = new PlaywrightBrowserScenarioProvider({
open: () => Promise.resolve(session),
});
@@ -237,7 +265,9 @@ describe("PlaywrightBrowserScenarioProvider", () => {
});
it("records one failed step, cancels later actions, and still cleans up", async () => {
- const session = new FakeSession("launch", false, true);
+ const session = new FakeSession("launch", {
+ action: new Error("fixture secret action failure"),
+ });
const provider = new PlaywrightBrowserScenarioProvider({
open: () => Promise.resolve(session),
});
@@ -276,17 +306,187 @@ describe("PlaywrightBrowserScenarioProvider", () => {
});
expect(session.closeCalls).toBe(1);
});
+});
- it("closes the session when initial capture fails", async () => {
- const session = new FakeSession("launch", false, false, true);
+describe("PlaywrightBrowserScenarioProvider cleanup", () => {
+ it("returns a cleanup failure with completed observations when close fails", async () => {
+ const session = new FakeSession("launch", {
+ action: new Error("fixture secret action failure"),
+ close: new Error("browser close failed"),
+ });
const provider = new PlaywrightBrowserScenarioProvider({
open: () => Promise.resolve(session),
});
const result = await provider.captureScenario(scenario());
+
expect(result.ok).toBe(false);
+ if (result.ok) return;
+ const projection = projectAnalysisError(result.error);
+ expect(projection).toMatchObject({
+ code: "cleanup_incomplete",
+ details: {
+ cleanup: "incomplete",
+ resources: ["browser_transport"],
+ partial_observation: {
+ kind: "browser-scenario-observation",
+ capture: {
+ browser: { cleanup: "incomplete" },
+ steps: [
+ { step_id: "scenario_start" },
+ { step_id: "wait_0", status: "failed" },
+ ],
+ },
+ },
+ },
+ });
expect(session.closeCalls).toBe(1);
});
+});
+
+describe("PlaywrightBrowserScenarioProvider operation failures", () => {
+ it.each([{ cleanupFails: false }, { cleanupFails: true }])(
+ "retains typed partial data when operation fails and cleanupFails=$cleanupFails",
+ async ({ cleanupFails }) => {
+ const session = new FakeSession("launch", {
+ capture: {
+ call: 2,
+ error: new AnalysisOutputError(
+ "capture_browser_scenario",
+ "step capture failed",
+ {
+ capturedOutput: {
+ stdout: "partial stdout",
+ stderr: "",
+ truncated: false,
+ },
+ cleanup: {
+ reason: "capture cleanup is uncertain",
+ resources: ["capture_handle"],
+ },
+ },
+ ),
+ },
+ ...(cleanupFails ? { close: new Error("browser close failed") } : {}),
+ });
+ session.eventItems = [
+ {
+ sequence: 1,
+ step_index: 0,
+ kind: "console",
+ level: "log",
+ text: "navigation started",
+ url: sanitizeBrowserUrl("https://app.example.test/"),
+ },
+ ];
+ const provider = new PlaywrightBrowserScenarioProvider({
+ open: () => Promise.resolve(session),
+ });
+ const result = await provider.captureScenario(scenario());
+
+ expect(result.ok).toBe(false);
+ if (result.ok) return;
+ const projection = projectAnalysisError(result.error);
+ expect(projection).toMatchObject({
+ code: "cleanup_incomplete",
+ details: {
+ diagnostics: {
+ primary_error: {
+ code: "cleanup_incomplete",
+ details: {
+ operation: "capture_browser_scenario",
+ reason: "step capture failed",
+ },
+ },
+ ...(cleanupFails
+ ? { cleanup_error: { code: "execution_failure" } }
+ : {}),
+ },
+ partial_observation: {
+ kind: "browser-scenario-observation",
+ capture: {
+ browser: {
+ cleanup: cleanupFails
+ ? "incomplete"
+ : "terminated-owned-process",
+ },
+ steps: [{ step_id: "scenario_start" }],
+ events: {
+ items: [
+ expect.objectContaining({ text: "navigation started" }),
+ ],
+ },
+ },
+ },
+ },
+ });
+ expect(result.error.capturedOutput).toEqual({
+ stdout: "partial stdout",
+ stderr: "",
+ truncated: false,
+ });
+ expect(result.error.cleanup).toEqual(
+ cleanupFails
+ ? {
+ reason: "browser close failed",
+ resources: ["browser_transport"],
+ }
+ : {
+ reason: "capture cleanup is uncertain",
+ resources: ["capture_handle"],
+ },
+ );
+ expect(session.closeCalls).toBe(1);
+ },
+ );
+});
+
+describe("PlaywrightBrowserScenarioProvider cancellation and timeout", () => {
+ it.each([
+ {
+ name: "cancellation",
+ error: new AnalysisCancelledError("capture_browser_scenario"),
+ code: "cancelled",
+ category: "cancelled",
+ details: { operation: "capture_browser_scenario", cleanup: "complete" },
+ },
+ {
+ name: "timeout",
+ error: new AnalysisTimeoutError("capture_browser_scenario", 250),
+ code: "provider_timeout",
+ category: "timeout",
+ details: { operation: "capture_browser_scenario", timeout_ms: 250 },
+ },
+ ])(
+ "preserves $name category and details when cleanup succeeds",
+ async ({ error, code, category, details }) => {
+ const session = new FakeSession("launch", {
+ capture: { call: 1, error },
+ });
+ const provider = new PlaywrightBrowserScenarioProvider({
+ open: () => Promise.resolve(session),
+ });
+ const result = await provider.captureScenario(scenario());
+
+ expect(result.ok).toBe(false);
+ if (result.ok) return;
+ const projection = projectAnalysisError(result.error);
+ expect(projection).toMatchObject({
+ code,
+ category,
+ details: {
+ ...details,
+ partial_observation: {
+ kind: "browser-scenario-observation",
+ capture: { browser: { cleanup: "terminated-owned-process" } },
+ },
+ },
+ });
+ expect(session.closeCalls).toBe(1);
+ },
+ );
+});
+describe("PlaywrightBrowserScenarioProvider request cancellation", () => {
it("does not open a session for an already-cancelled request", async () => {
let opens = 0;
const provider = new PlaywrightBrowserScenarioProvider({
diff --git a/src/browser/PlaywrightBrowserScenarioProvider.ts b/src/browser/PlaywrightBrowserScenarioProvider.ts
index accc64d9d..b62d3504f 100644
--- a/src/browser/PlaywrightBrowserScenarioProvider.ts
+++ b/src/browser/PlaywrightBrowserScenarioProvider.ts
@@ -3,22 +3,23 @@ import type {
ExecutionOptions,
ProviderIdentity,
} from "../application/AnalysisProvider.js";
-import type {
- BrowserScenario,
- BrowserScenarioAction,
-} from "../domain/browserScenario.js";
+import type { BrowserScenario } from "../domain/browserScenario.js";
+import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
import {
browserScenarioCaptureSchema,
+ type BrowserScenarioCapture,
+ type BrowserScenarioPartialObservation,
+} from "../domain/browserScenarioCapture.js";
+import {
classifyBrowserScenarioCompleteness,
browserScenarioStepSchema,
browserStepArtifactsSchema,
- type BrowserScenarioCapture,
type BrowserScenarioCompleteness,
type BrowserScenarioCompletenessSection,
type BrowserScenarioStep,
type BrowserScenarioStepOutcome,
type BrowserStepArtifacts,
-} from "../domain/browserScenarioCapture.js";
+} from "../domain/browserScenarioCaptureValues.js";
import { AnalysisError } from "../domain/analysisErrorBase.js";
import { BrowserObservationError } from "../domain/browserObservationError.js";
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
@@ -27,6 +28,11 @@ import type {
BrowserScenarioSessionFactory,
BrowserScenarioSessionPort,
} from "./BrowserScenarioSessionPort.js";
+import {
+ browserScenarioCleanupObservation,
+ browserScenarioOperationFailure,
+ browserScenarioCaptureData,
+} from "./BrowserScenarioPartialObservation.js";
const OPERATION = "capture_browser_scenario" as const;
import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "./providerIdentities.js";
@@ -232,7 +238,12 @@ const runScenario = async (
options.signal === undefined ? {} : { signal: options.signal },
);
const steps: BrowserScenarioStep[] = [];
- let cleanup: "terminated-owned-process" | "disconnected-external" | undefined;
+ let operationOutcome:
+ | { readonly ok: true }
+ | {
+ readonly ok: false;
+ readonly cause: unknown;
+ } = { ok: true };
try {
steps.push(
await initialStep({
@@ -255,43 +266,72 @@ const runScenario = async (
steps.push(step);
failed ||= step.status !== "completed";
}
- } finally {
- cleanup = await session.close();
+ } catch (cause: unknown) {
+ operationOutcome = { ok: false, cause };
+ }
+ let cleanupOutcome:
+ | {
+ readonly ok: true;
+ readonly cleanup: "terminated-owned-process" | "disconnected-external";
+ }
+ | { readonly ok: false; readonly cause: unknown };
+ try {
+ cleanupOutcome = { ok: true, cleanup: await session.close() };
+ } catch (cause: unknown) {
+ cleanupOutcome = { ok: false, cause };
}
- const events = session.events();
+ if (!cleanupOutcome.ok) {
+ const cleanup = browserScenarioCleanupObservation(cleanupOutcome.cause);
+ const partialObservation: BrowserScenarioPartialObservation = {
+ kind: "browser-scenario-observation",
+ capture: browserScenarioCaptureData({
+ session,
+ scenario,
+ startedAt,
+ steps,
+ cleanup: "incomplete",
+ limitations: [
+ "Scenario observations are partial because browser cleanup could not be confirmed.",
+ ],
+ }),
+ };
+ if (!operationOutcome.ok)
+ throw browserScenarioOperationFailure(
+ operationOutcome.cause,
+ partialObservation,
+ { cause: cleanupOutcome.cause },
+ );
+ throw new BrowserObservationError(OPERATION, "cleanup_failed", {
+ cause: cleanupOutcome.cause,
+ cleanup,
+ partialObservation,
+ detail: `Browser scenario cleanup could not be confirmed: ${cleanup.reason}. Collected steps and events are retained in partial_observation. Review the reported resources before retrying.`,
+ });
+ }
+ if (!operationOutcome.ok)
+ throw browserScenarioOperationFailure(operationOutcome.cause, {
+ kind: "browser-scenario-observation",
+ capture: browserScenarioCaptureData({
+ session,
+ scenario,
+ startedAt,
+ steps,
+ cleanup: cleanupOutcome.cleanup,
+ limitations: [
+ "Scenario capture ended after an operation failed; uncollected steps are not represented.",
+ ],
+ }),
+ });
const completeness = globalCompleteness(scenario, session, steps);
return browserScenarioCaptureSchema.parse({
- browser: {
- mode: session.mode,
- process_ownership: session.processOwnership,
- cleanup,
- product: session.product,
- version: session.version,
- },
- scenario: {
- start_origin: new URL(scenario.start_url.url).origin,
- action_count: scenario.actions.length,
- secret_references: scenario.secrets.map(({ secret_id: id }) => id).sort(),
- network_content: scenario.capture.network,
- },
- duration_ms: Date.now() - startedAt,
- steps,
- events,
+ ...browserScenarioCaptureData({
+ session,
+ scenario,
+ startedAt,
+ steps,
+ cleanup: cleanupOutcome.cleanup,
+ }),
completeness,
- limitations: [
- "Event sequence records provider receipt order; simultaneous browser causality is not inferred.",
- "Network content is retained only when selected; response bytes are browser-decoded, not compressed wire bytes.",
- "Request bytes are limited to what Playwright exposes; not_exposed does not establish body absence or multipart file coverage.",
- "Network content reads settle within 5 seconds each and never wait for unfinished responses or refetch them.",
- "Playwright scenario events do not expose request initiator stacks; receipt order does not prove causality.",
- "Storage values are hashed only after declared-secret redaction.",
- ...(session.eventLimitations?.() ?? []),
- ...(session.mode === "connect"
- ? [
- "CDP attachment cannot recover pre-attach events or guarantee launch-time context options.",
- ]
- : []),
- ],
});
};
diff --git a/src/browser/PlaywrightScenarioSession.ts b/src/browser/PlaywrightScenarioSession.ts
index d1081abbf..1c31ef344 100644
--- a/src/browser/PlaywrightScenarioSession.ts
+++ b/src/browser/PlaywrightScenarioSession.ts
@@ -1,9 +1,7 @@
import { type BrowserContext, type Page } from "playwright-core";
-import type {
- BrowserScenario,
- BrowserScenarioAction,
-} from "../domain/browserScenario.js";
+import type { BrowserScenario } from "../domain/browserScenario.js";
+import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
import { BrowserObservationError } from "../domain/browserObservationError.js";
import type {
BrowserScenarioSessionFactory,
@@ -19,6 +17,10 @@ import { performPlaywrightScenarioAction } from "./PlaywrightScenarioActions.js"
import { capturePlaywrightStepArtifacts } from "./PlaywrightScenarioArtifacts.js";
import { PlaywrightScenarioEvents } from "./PlaywrightScenarioEvents.js";
import { withPlaywrightExecutionBoundary } from "./PlaywrightExecutionBoundary.js";
+import {
+ browserScenarioCaptureData,
+ browserScenarioOperationFailure,
+} from "./BrowserScenarioPartialObservation.js";
const OPERATION = "capture_browser_scenario" as const;
@@ -135,6 +137,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort {
const secrets = BrowserScenarioSecrets.resolve(scenario, environment);
if (secrets === undefined)
throw new BrowserObservationError(OPERATION, "secret_unavailable");
+ const startedAt = Date.now();
const opening = openPlaywrightScenarioBrowser(scenario, environment);
let opened: OpenedScenarioBrowser;
let events: PlaywrightScenarioEvents | undefined;
@@ -155,6 +158,7 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort {
});
throw cause;
}
+ let session: PlaywrightScenarioSession;
try {
await withPlaywrightExecutionBoundary(
() => initializePage(opened.context, opened.page, scenario, secrets),
@@ -170,11 +174,24 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort {
network: scenario.capture.network,
...(options.signal === undefined ? {} : { signal: options.signal }),
});
- const session = new PlaywrightScenarioSession(opened, {
+ session = new PlaywrightScenarioSession(opened, {
mode: scenario.browser.mode,
secrets,
eventCapture: events,
});
+ } catch (cause: unknown) {
+ return failBrowserScenarioOperation(
+ () =>
+ opened.cleanup.close(
+ events === undefined
+ ? undefined
+ : () => events?.finish() ?? Promise.resolve(),
+ options.signal,
+ ),
+ cause,
+ );
+ }
+ try {
await withPlaywrightExecutionBoundary(
() =>
opened.page.goto(secrets.url(scenario.start_url), {
@@ -186,15 +203,33 @@ export class PlaywrightScenarioSession implements BrowserScenarioSessionPort {
);
return session;
} catch (cause: unknown) {
- return failBrowserScenarioOperation(
- () =>
- opened.cleanup.close(
- events === undefined
- ? undefined
- : () => events?.finish() ?? Promise.resolve(),
- options.signal,
- ),
+ let cleanup:
+ | "terminated-owned-process"
+ | "disconnected-external"
+ | "incomplete";
+ let cleanupFailure: { readonly cause: unknown } | undefined;
+ try {
+ cleanup = await session.close();
+ } catch (failure: unknown) {
+ cleanup = "incomplete";
+ cleanupFailure = { cause: failure };
+ }
+ throw browserScenarioOperationFailure(
cause,
+ {
+ kind: "browser-scenario-observation",
+ capture: browserScenarioCaptureData({
+ session,
+ scenario,
+ startedAt,
+ steps: [],
+ cleanup,
+ limitations: [
+ "Scenario navigation failed before the initial state was captured.",
+ ],
+ }),
+ },
+ cleanupFailure,
);
}
}
diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts
index 6c9178c55..97638ecd8 100644
--- a/src/domain/analysisErrorBase.ts
+++ b/src/domain/analysisErrorBase.ts
@@ -89,6 +89,7 @@ import type { NativeCallPartialObservation } from "./native/nativeCallPartialObs
import type { ElectronActivePartialObservation } from "./javascript/electronActiveObservation.js";
import type { JavaScriptRuntimeObservation } from "./javascript/javascriptRuntimeObservation.js";
import type { ArtifactInventoryPartialObservation } from "./artifactPartialObservation.js";
+import type { BrowserScenarioPartialObservation } from "./browserScenarioCapture.js";
/** Provider-neutral evidence collected before a typed analysis failure. */
export type AnalysisPartialObservation =
@@ -96,5 +97,6 @@ export type AnalysisPartialObservation =
| NativeCallPartialObservation
| FileOffsetPartialObservation
| ElectronActivePartialObservation
+ | BrowserScenarioPartialObservation
| JavaScriptRuntimeObservation
| ArtifactInventoryPartialObservation;
diff --git a/src/domain/browserScenarioCapture.ts b/src/domain/browserScenarioCapture.ts
index efd3744ba..87e2626de 100644
--- a/src/domain/browserScenarioCapture.ts
+++ b/src/domain/browserScenarioCapture.ts
@@ -108,3 +108,25 @@ export const browserScenarioCaptureSchema = z
export type BrowserScenarioCapture = z.infer<
typeof browserScenarioCaptureSchema
>;
+
+/** Capture fields shared by complete results and retained partial observations. */
+export type BrowserScenarioCaptureData = Omit<
+ BrowserScenarioCapture,
+ "browser" | "steps" | "completeness" | "events"
+> & {
+ readonly browser: Omit<BrowserScenarioCapture["browser"], "cleanup"> & {
+ readonly cleanup:
+ | BrowserScenarioCapture["browser"]["cleanup"]
+ | "incomplete";
+ };
+ readonly steps: readonly BrowserScenarioCapture["steps"][number][];
+ readonly events: Omit<BrowserScenarioCapture["events"], "items"> & {
+ readonly items: readonly BrowserScenarioCapture["events"]["items"][number][];
+ };
+};
+
+/** Browser scenario data retained when an operation fails before a complete result. */
+export interface BrowserScenarioPartialObservation {
+ readonly kind: "browser-scenario-observation";
+ readonly capture: BrowserScenarioCaptureData;
+}