The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
test(runner): improve blank-page diagnostics and infrastructure coverage (#13824)
Dotta9a76390cfa4e
diff --git a/server/src/services/workspace-runtime-exposure.test.ts b/server/src/services/workspace-runtime-exposure.test.tsindex 9f8dfeac42..951b6f5e4a 100644--- a/server/src/services/workspace-runtime-exposure.test.ts+++ b/server/src/services/workspace-runtime-exposure.test.ts@@ -131,18 +131,10 @@ setInterval(() => {}, 1000);* exposure-port pool. The start must surface the failure terminally after a single* allocation.*/-const SYNTHETIC_EADDRINUSE_ON_BASE_PORT_GUEST = `-import http from "node:http";+const SYNTHETIC_EADDRINUSE_ON_ASSIGNED_PORT_GUEST = `const p = Number(process.env.PORT);-if (p === 42000) {- process.stderr.write("node:events:497\\nError: listen EADDRINUSE: address already in use 127.0.0.1:" + p + "\\n");- process.exit(1);-}-const health = (rq, r) => { if (rq.url === "/api/health") { r.setHeader("content-type", "application/json"); r.end(JSON.stringify({ status: "ok" })); return true; } return false; };-for (const q of [p, p + 10000]) {- http.createServer((rq, r) => { if (health(rq, r)) return; r.statusCode = 200; r.end("ok"); }).listen(q, "127.0.0.1");-}-setInterval(() => {}, 1000);+process.stderr.write("node:events:497\\nError: listen EADDRINUSE: address already in use 127.0.0.1:" + p + "\\n");+process.exit(1);`;/**@@ -189,7 +181,7 @@ beforeAll(async () => {// collision the host cannot confirm. The start must not quarantine the pair.await fs.writeFile(path.join(guestDir, "dev-runner-eaddrinuse-synthetic.mjs"),- SYNTHETIC_EADDRINUSE_ON_BASE_PORT_GUEST,+ SYNTHETIC_EADDRINUSE_ON_ASSIGNED_PORT_GUEST,);// A guest that fails on a fixed auxiliary port, not on its assigned app or HMR// port. It models an unrelated helper listener that an external process holds.@@ -757,7 +749,7 @@ describe("the deployed failure shape: loopback app port, wildcard HMR (PAP-17256}, 20_000);});-describe("recovers when a guest loses its assigned exposure port during startup (PAP-17256)", () => {+describe.each([false, true])("guest exposure-port collision reporting (base pair occupied: %s)", (basePairOccupied) => {// The quarantine decision itself is unit-tested through// `classifyExposureHostCollisions` below. A deterministic end-to-end quarantine// test is not reachable here: a real host listener that holds the assigned port@@ -773,7 +765,12 @@ describe("recovers when a guest loses its assigned exposure port during startupreturn broker.reserve(runtimeId, requested);},};- installDeps({ broker: recordingBroker });+ installDeps({+ broker: recordingBroker,+ isPortAvailable: (port) => basePairOccupied &&+ [RUNTIME_EXPOSURE_APP_PORT_MIN, deriveViteHmrPort(RUNTIME_EXPOSURE_APP_PORT_MIN)].includes(port)+ ? Promise.resolve(false) : isLoopbackPortFree(port),+ });const logs: string[] = [];const error = await startRuntimeServicesForWorkspaceControl({@@ -788,12 +785,14 @@ describe("recovers when a guest loses its assigned exposure port during startup},}).then(() => null, (err: unknown) => err as Error);- // No host listener owns 42000, so the printed EADDRINUSE line is unverified.+ // The assigned pair is free, so the printed EADDRINUSE line is unverified.// The start fails terminally after ONE allocation and never burns the pool.expect(error).not.toBeNull();- expect(error!.message).toContain("42000");+ expect(error!.message).toContain(String(reservedAppPorts[0]));expect(error!.message).toContain("not verified");- expect(reservedAppPorts).toEqual([42_000]);+ expect(reservedAppPorts).toHaveLength(1);+ expect(reservedAppPorts[0]).toBeGreaterThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MIN + (basePairOccupied ? 1 : 0));+ expect(reservedAppPorts[0]).toBeLessThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MAX);// No quarantine and no re-allocation happened for the unverified claim.const diagnosis = logs.join("");@@ -811,7 +810,12 @@ describe("recovers when a guest loses its assigned exposure port during startupreturn broker.reserve(runtimeId, requested);},};- installDeps({ broker: recordingBroker });+ installDeps({+ broker: recordingBroker,+ isPortAvailable: (port) => basePairOccupied &&+ [RUNTIME_EXPOSURE_APP_PORT_MIN, deriveViteHmrPort(RUNTIME_EXPOSURE_APP_PORT_MIN)].includes(port)+ ? Promise.resolve(false) : isLoopbackPortFree(port),+ });const logs: string[] = [];const error = await startRuntimeServicesForWorkspaceControl({@@ -831,7 +835,9 @@ describe("recovers when a guest loses its assigned exposure port during startup// burns the bounded retries on a valid pair.expect(error).not.toBeNull();expect(error!.message).toContain("39999");- expect(reservedAppPorts).toEqual([42_000]);+ expect(reservedAppPorts).toHaveLength(1);+ expect(reservedAppPorts[0]).toBeGreaterThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MIN + (basePairOccupied ? 1 : 0));+ expect(reservedAppPorts[0]).toBeLessThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MAX);// No quarantine and no re-allocation happened for the auxiliary conflict.const diagnosis = logs.join("");@@ -849,7 +855,12 @@ describe("recovers when a guest loses its assigned exposure port during startupreturn broker.reserve(runtimeId, requested);},};- installDeps({ broker: recordingBroker });+ installDeps({+ broker: recordingBroker,+ isPortAvailable: (port) => basePairOccupied &&+ [RUNTIME_EXPOSURE_APP_PORT_MIN, deriveViteHmrPort(RUNTIME_EXPOSURE_APP_PORT_MIN)].includes(port)+ ? Promise.resolve(false) : isLoopbackPortFree(port),+ });const logs: string[] = [];const error = await startRuntimeServicesForWorkspaceControl({@@ -864,13 +875,15 @@ describe("recovers when a guest loses its assigned exposure port during startup},}).then(() => null, (err: unknown) => err as Error);- // The assigned port 42000 appears on a benign line, but EADDRINUSE names only+ // The assigned port appears on a benign line, but EADDRINUSE names only// the auxiliary port 39999. The parser matches the error and the port on the// same line, so the assigned pair is not a collision. The start fails// terminally after ONE allocation and never quarantines the valid pair.expect(error).not.toBeNull();expect(error!.message).toContain("39999");- expect(reservedAppPorts).toEqual([42_000]);+ expect(reservedAppPorts).toHaveLength(1);+ expect(reservedAppPorts[0]).toBeGreaterThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MIN + (basePairOccupied ? 1 : 0));+ expect(reservedAppPorts[0]).toBeLessThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MAX);const diagnosis = logs.join("");expect(diagnosis).not.toContain("Quarantined pair");diff --git a/tests/e2e/task-reload.spec.ts b/tests/e2e/task-reload.spec.tsnew file mode 100644index 0000000000..68f02b332f--- /dev/null+++ b/tests/e2e/task-reload.spec.ts@@ -0,0 +1,44 @@+import { expect, test } from "@playwright/test";+import { observeBrowserBootstrap } from "../runner-e2e/browser-bootstrap-diagnostics";+import { json, setup } from "./agent-chat.shared";++test.use({ serviceWorkers: "allow", trace: "retain-on-failure" });++test("saved task content survives same-URL navigation and reload with a controlling service worker", async ({ page, request }) => {+ test.setTimeout(120_000);+ const fixture = await setup(request);+ const diagnostics = observeBrowserBootstrap(page);+ const title = "Task reload continuity";+ const comment = "Persisted context remains available after a full page load.";+ try {+ // Unassigned backlog work exercises the actual UI without a provider call.+ const issue = await json(await request.post(`/api/companies/${fixture.company.id}/issues`, {+ data: { title, status: "backlog" },+ }));+ await json(await request.post(`/api/issues/${issue.id}/comments`, { data: { body: comment } }));+ const route = `/${fixture.company.issuePrefix}/issues/${issue.identifier}`;+ const assertLoaded = async () => {+ await expect(page.getByRole("heading", { name: title, exact: true })).toBeVisible({ timeout: 30_000 });+ await expect(page.getByText(comment, { exact: true })).toBeVisible();+ await expect(page.getByTestId("task-chat-composer-input")).toBeVisible();+ };+ await page.goto(route, { waitUntil: "domcontentloaded" });+ await assertLoaded();+ await page.waitForFunction(() => Boolean(navigator.serviceWorker?.controller));+ for (let index = 0; index < 3; index += 1) {+ // The warm-continuity eval used same-URL goto, not just location.reload.+ await page.goto(route, { waitUntil: "domcontentloaded" });+ await assertLoaded();+ await page.reload({ waitUntil: "domcontentloaded" });+ await assertLoaded();+ }+ expect(await json(await request.get(`/api/issues/${issue.id}/runs`))).toEqual([]);+ } finally {+ await test.info().attach("browser-bootstrap", {+ contentType: "application/json",+ body: Buffer.from(JSON.stringify(await diagnostics.snapshot(), null, 2)),+ });+ diagnostics.dispose();+ await fixture.restore();+ }+});diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.mdindex cbe7695cc7..d10684439e 100644--- a/tests/runner-e2e/README.md+++ b/tests/runner-e2e/README.md@@ -985,3 +985,19 @@ Current proof and remaining decisions are recorded inthe original worker-loss attempts quarantined both providers. The version 9 crasheval requires a usable fresh conversation after verified cleanup. A passingquarantine guard alone is not a recovered workflow.++### Blank-page investigation++Private `browser-diagnostics.json` includes the final document readiness, whether+`#root` mounted content, whether a service worker controls the page, outstanding+script/style paths, and recent module 304/error statuses. These fields contain no+response bodies, headers, or query strings. A 304 is ordinary cache validation;+recording it does not change the grade or retry the page. The public report still+uses the existing evidence allowlist.++The provider-free `tests/e2e/task-reload.spec.ts` regression opens a persisted task+with the production service worker, navigates to the same URL, and reloads it. It+requires the saved content and usable composer to remain visible. Run it with the+standard `tests/e2e/playwright.config.ts`; no provider or Daytona credentials are+needed. Browser-support tests separately exercise blank-root/pending-module+failure evidence, so a future blank page is distinguishable from a loaded task.diff --git a/tests/runner-e2e/browser-bootstrap-diagnostics.spec.ts b/tests/runner-e2e/browser-bootstrap-diagnostics.spec.tsnew file mode 100644index 0000000000..ef52d54b1d--- /dev/null+++ b/tests/runner-e2e/browser-bootstrap-diagnostics.spec.ts@@ -0,0 +1,44 @@+import { expect, test } from "@playwright/test";+import { observeBrowserBootstrap } from "./browser-bootstrap-diagnostics.js";++const html = '<div id="root"></div><script type="module" src="/entry.js?private=do-not-record"></script>';++test("records an empty root and pending module, then recognizes completed startup", async ({ page }) => {+ let release!: () => void;+ const gate = new Promise<void>(resolve => { release = resolve; });+ await page.route("http://bootstrap.test/**", async route => {+ if (new URL(route.request().url()).pathname === "/entry.js") {+ await gate;+ await route.fulfill({ contentType: "text/javascript", body: 'document.getElementById("root").innerHTML = "<main>Ready</main>";' });+ } else await route.fulfill({ contentType: "text/html", body: html });+ });+ const diagnostics = observeBrowserBootstrap(page);+ try {+ await page.goto("http://bootstrap.test/", { waitUntil: "commit" });+ await expect.poll(async () => (await diagnostics.snapshot()).pendingModules).toEqual(["/entry.js"]);+ const before = await diagnostics.snapshot();+ expect(before.state).toMatchObject({ rootPresent: true, rootChildCount: 0, serviceWorkerControlled: false });+ expect(JSON.stringify(before)).not.toContain("do-not-record");+ release();+ await expect(page.getByRole("main")).toHaveText("Ready");+ await expect.poll(async () => (await diagnostics.snapshot()).pendingModuleCount).toBe(0);+ expect((await diagnostics.snapshot()).state?.rootChildCount).toBe(1);+ } finally { release(); diagnostics.dispose(); }+});++for (const status of [304, 503]) {+ test(`retains module status ${status} without credentials and clears it on navigation`, async ({ page }) => {+ await page.route("http://bootstrap.test/**", async route => {+ const pathname = new URL(route.request().url()).pathname;+ await route.fulfill(pathname === "/entry.js" ? { status, body: "" }+ : { contentType: "text/html", body: pathname === "/ready" ? '<div id="root">Ready</div>' : html });+ });+ const diagnostics = observeBrowserBootstrap(page);+ try {+ await page.goto("http://bootstrap.test/", { waitUntil: "load" });+ expect((await diagnostics.snapshot()).moduleResponses).toEqual([{ path: "/entry.js", status }]);+ await page.goto("http://bootstrap.test/ready");+ expect((await diagnostics.snapshot()).moduleResponses).toEqual([]);+ } finally { diagnostics.dispose(); }+ });+}diff --git a/tests/runner-e2e/browser-bootstrap-diagnostics.ts b/tests/runner-e2e/browser-bootstrap-diagnostics.tsnew file mode 100644index 0000000000..838beecf70--- /dev/null+++ b/tests/runner-e2e/browser-bootstrap-diagnostics.ts@@ -0,0 +1,51 @@+import type { Page, Request, Response } from "@playwright/test";++/** Private failure evidence only: no response bodies, headers, or query strings. */+export function observeBrowserBootstrap(page: Page) {+ const pending = new Map<Request, string>();+ const moduleResponses: { path: string; status: number }[] = [];+ const pathname = (url: string) => {+ try { return new URL(url).pathname; } catch { return "[invalid URL]"; }+ };+ const isModule = (request: Request) => ["script", "stylesheet"].includes(request.resourceType());+ const onRequest = (request: Request) => {+ if (isModule(request)) pending.set(request, pathname(request.url()));+ };+ const onFinished = (request: Request) => { pending.delete(request); };+ const onResponse = (response: Response) => {+ // A 304 is normally valid, but is useful evidence when module evaluation+ // never mounts the app. It must not itself turn a successful run into a fail.+ if (isModule(response.request()) && (response.status() === 304 || response.status() >= 400)) {+ moduleResponses.push({ path: pathname(response.url()), status: response.status() });+ if (moduleResponses.length > 50) moduleResponses.shift();+ }+ };+ const onNavigation = (frame: ReturnType<Page["mainFrame"]>) => {+ if (frame !== page.mainFrame()) return;+ pending.clear();+ moduleResponses.length = 0;+ };+ page.on("request", onRequest);+ page.on("requestfinished", onFinished);+ page.on("requestfailed", onFinished);+ page.on("response", onResponse);+ page.on("framenavigated", onNavigation);+ return {+ async snapshot() {+ const state = await page.evaluate(() => ({+ readyState: document.readyState,+ rootPresent: document.getElementById("root") !== null,+ rootChildCount: document.getElementById("root")?.childElementCount ?? 0,+ serviceWorkerControlled: Boolean(navigator.serviceWorker?.controller),+ })).catch(() => null);+ return { state, pendingModuleCount: pending.size, pendingModules: [...pending.values()].slice(0, 50), moduleResponses: [...moduleResponses] };+ },+ dispose() {+ page.off("request", onRequest);+ page.off("requestfinished", onFinished);+ page.off("requestfailed", onFinished);+ page.off("response", onResponse);+ page.off("framenavigated", onNavigation);+ },+ };+}diff --git a/tests/runner-e2e/playwright-support.config.ts b/tests/runner-e2e/playwright-support.config.tsindex 53e6b3a47c..32dd9e6b29 100644--- a/tests/runner-e2e/playwright-support.config.ts+++ b/tests/runner-e2e/playwright-support.config.ts@@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test";/** Browser-only harness regressions: no Paperclip instance or provider credentials. */export default defineConfig({testDir: ".",- testMatch: ["screenshot-readiness.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts"],+ testMatch: ["screenshot-readiness.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts"],workers: 1,retries: 0,timeout: 10_000,diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.tsindex e5008fd746..9649bf1da5 100644--- a/tests/runner-e2e/runner.spec.ts+++ b/tests/runner-e2e/runner.spec.ts@@ -1,3 +1,4 @@+import { observeBrowserBootstrap } from "./browser-bootstrap-diagnostics.js";import { runContinuationFlow } from "./continuation-flow.js";import { runEverydayFlow } from "./everyday-flow.js";import { createTaskThroughUi, submitTaskReply } from "./user-actions.js";@@ -533,6 +534,7 @@ for (const execution of executions) {const consoleDiagnostics: Array<Record<string, unknown>> = [];const networkDiagnostics: Array<Record<string, unknown>> = [];const pageLifecycleDiagnostics: Array<Record<string, unknown>> = [];+ const browserBootstrap = observeBrowserBootstrap(page);let fixtures: LiveFixtureValues | undefined;let reviewProvider: Awaited<ReturnType<typeof setupConnectionReview>> | undefined;let issue: IssueRecord | undefined;@@ -2449,6 +2451,7 @@ for (const execution of executions) {console: consoleDiagnostics,network: networkDiagnostics,lifecycle: pageLifecycleDiagnostics,+ bootstrap: await browserBootstrap.snapshot(),},secrets,);@@ -2459,6 +2462,7 @@ for (const execution of executions) {);failureClassOverride = "secret_leak";}+ browserBootstrap.dispose();if (fixtures) {await captureRuntimeLeases().catch((error) => {networkDiagnostics.push({diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.tsindex 6e80147d17..1a9c021dd0 100644--- a/tests/runner-e2e/workflow-security.test.ts+++ b/tests/runner-e2e/workflow-security.test.ts@@ -682,6 +682,23 @@ describe("public repository paid workflow security", () => {);});+ it("resolves each trusted reporting lockfile before frozen install and AWS credentials", async () => {+ const workflow = await readFile(+ path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8",+ );+ for (const jobName of ["report", "publish_history"]) {+ const job = workflow.split(`\n ${jobName}:`)[1]!.split(/\n [a-z_]+:/u)[0]!;+ const resolve = job.indexOf("pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile");+ const install = job.indexOf("pnpm install --frozen-lockfile");+ expect(job).toContain("ref: ${{ github.sha }}");+ expect(job).not.toContain("resolved-target-lockfile");+ expect(resolve, jobName).toBeGreaterThan(-1);+ expect(install, jobName).toBeGreaterThan(resolve);+ const credentials = job.indexOf("aws-actions/configure-aws-credentials@");+ if (credentials >= 0) expect(install).toBeLessThan(credentials);+ }+ });+it("binds rerun evidence and Pages artifacts to the exact workflow attempt", async () => {const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),