The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
test(e2e): shorten and split Smoke Lab coverage (#12506)
Dotta39eafad47d77
diff --git a/doc/connections/SMOKE-LAB-BROWSER-RUNNER.md b/doc/connections/SMOKE-LAB-BROWSER-RUNNER.mdindex f5f491b2e6..65bfe876cc 100644--- a/doc/connections/SMOKE-LAB-BROWSER-RUNNER.md+++ b/doc/connections/SMOKE-LAB-BROWSER-RUNNER.md@@ -4,9 +4,10 @@**Scenario source of truth:** [`tests/e2e/smoke-lab.catalog.ts`](../../tests/e2e/smoke-lab.catalog.ts) — **do not fork the step list.**This is the manual/agent counterpart to the deterministic Playwright CI mirror-(`tests/e2e/smoke-lab.spec.ts`, S4). Where the CI mirror runs headless in a-throwaway instance for a red/green gate, this runbook has a **QA agent drive a-real browser** through the same P1–P7 lifecycle against a live instance,+(`tests/e2e/smoke-lab.shared.ts`, split between the P1–P4 and P5–P7 specs).+Where the CI mirror runs headless in a throwaway instance for a red/green gate,+this runbook has a **QA agent drive a real browser** through the same P1–P7+lifecycle against a live instance,**typing demo credentials into the fake OAuth provider's real consent page**, andrecords every step + a viewable screenshot to the Smoke Lab results API so the runshows up in the **Smoke Lab tab** and the **dashboard "Integration smoke" card**.diff --git a/doc/connections/SMOKE-LAB-TUTORIAL.md b/doc/connections/SMOKE-LAB-TUTORIAL.mdindex c6093a9f43..0299fbda05 100644--- a/doc/connections/SMOKE-LAB-TUTORIAL.md+++ b/doc/connections/SMOKE-LAB-TUTORIAL.md@@ -12,9 +12,10 @@ button or label is quoted, that's the exact text in the product.> Companion docs: the automated counterparts live in> [`SMOKE-LAB-BROWSER-RUNNER.md`](./SMOKE-LAB-BROWSER-RUNNER.md) (the agent-driven-> browser runner) and `tests/e2e/smoke-lab.spec.ts` (the headless CI mirror). The-> daily recurring routine that runs the browser smoke for you is described in-> [§8](#8-the-daily-routine-hands-off).+> browser runner), `tests/e2e/smoke-lab.shared.ts` (the shared headless runner),+> and the `tests/e2e/smoke-lab-p1-p4.spec.ts` and+> `tests/e2e/smoke-lab-p5-p7.spec.ts` CI halves. The daily recurring routine that+> runs the browser smoke for you is described in [§8](#8-the-daily-routine-hands-off).---diff --git a/scripts/e2e-shard-durations.json b/scripts/e2e-shard-durations.jsonindex b23c25efdb..af5549c94d 100644--- a/scripts/e2e-shard-durations.json+++ b/scripts/e2e-shard-durations.json@@ -1,5 +1,5 @@{- "$comment": "Per-spec wall-clock durations (ms) for the Playwright e2e lane, used by scripts/e2e-shard.mjs to balance specs across the PR shard matrix. Averaged from two real PR runs of .github/workflows/pr.yml (actions runs 29775184389 and 29769465682, 2026-07-20). Specs missing here get the median weight, so the manifest only needs occasional refreshes: pull the 'Run e2e tests' logs from a recent PR run and sum the Playwright list-reporter durations per spec file.",+ "$comment": "Per-spec wall-clock durations (ms) for the Playwright e2e lane, used by scripts/e2e-shard.mjs to balance specs across the PR shard matrix. Averaged from two real PR runs of .github/workflows/pr.yml (actions runs 29775184389 and 29769465682, 2026-07-20). The split Smoke Lab weights scale its prior 168s CI duration by the measured local before/after ratio and scenario count; refresh them from the next two real PR runs. Specs missing here get the median weight, so the manifest only needs occasional refreshes: pull the 'Run e2e tests' logs from a recent PR run and sum the Playwright list-reporter durations per spec file.","unit": "ms","durations": {"tests/e2e/app-not-connected.spec.ts": 26750,@@ -15,6 +15,7 @@"tests/e2e/planning-mode-visual-verification.spec.ts": 23250,"tests/e2e/sidebar-takeover.spec.ts": 16650,"tests/e2e/signoff-policy.spec.ts": 9696,- "tests/e2e/smoke-lab.spec.ts": 168000+ "tests/e2e/smoke-lab-p1-p4.spec.ts": 71000,+ "tests/e2e/smoke-lab-p5-p7.spec.ts": 53000}}diff --git a/tests/e2e/smoke-lab-p1-p4.spec.ts b/tests/e2e/smoke-lab-p1-p4.spec.tsnew file mode 100644index 0000000000..51afd51fd6--- /dev/null+++ b/tests/e2e/smoke-lab-p1-p4.spec.ts@@ -0,0 +1,4 @@+import { ciSmokeLabScenarios } from "./smoke-lab.catalog";+import { defineSmokeLabSuite } from "./smoke-lab.shared";++defineSmokeLabSuite("P1-P4", ciSmokeLabScenarios.slice(0, 4));diff --git a/tests/e2e/smoke-lab-p5-p7.spec.ts b/tests/e2e/smoke-lab-p5-p7.spec.tsnew file mode 100644index 0000000000..4f3b704ad9--- /dev/null+++ b/tests/e2e/smoke-lab-p5-p7.spec.ts@@ -0,0 +1,4 @@+import { ciSmokeLabScenarios } from "./smoke-lab.catalog";+import { defineSmokeLabSuite } from "./smoke-lab.shared";++defineSmokeLabSuite("P5-P7", ciSmokeLabScenarios.slice(4));diff --git a/tests/e2e/smoke-lab.spec.ts b/tests/e2e/smoke-lab.shared.tssimilarity index 90%rename from tests/e2e/smoke-lab.spec.tsrename to tests/e2e/smoke-lab.shared.tsindex 3b6c0904f0..d69568727f 100644--- a/tests/e2e/smoke-lab.spec.ts+++ b/tests/e2e/smoke-lab.shared.ts@@ -85,14 +85,14 @@ async function enableSmokeLab(request: APIRequestContext) {await json(await request.patch("/api/instance/settings/experimental", { data: { enableSmokeLab: true, enableApps: true } }));}-async function createSmokeRun(request: APIRequestContext, companyId: string) {+async function createSmokeRun(request: APIRequestContext, companyId: string, scenarioCount: number) {const result = await json<{ run: SmokeRun }>(await request.post(`/api/companies/${companyId}/smoke-lab/runs`, {data: {trigger: "ci",summary: {catalog: "tests/e2e/smoke-lab.catalog.ts",- scenarioCount: ciSmokeLabScenarios.length,+ scenarioCount,},},}),@@ -180,11 +180,12 @@ async function runRecordedStep(scenario: SmokeLabScenario,step: string,action: () => Promise<string | null | undefined>,+ captureSuccessScreenshot = false,) {const start = Date.now();try {const screenshotHint = await action();- const screenshotPath = await screenshot(page, scenario, step);+ const screenshotPath = captureSuccessScreenshot ? await screenshot(page, scenario, step) : null;await recordStep(request, seed.companyId, runId, {path: scenario.path,scenarioStep: step,@@ -207,8 +208,11 @@ async function runRecordedStep(}}-async function startAndInstallFixtures(request: APIRequestContext, companyId: string): Promise<FixtureInstall> {+async function startSmokeLabServices(request: APIRequestContext, companyId: string) {await json(await request.post(`/api/companies/${companyId}/smoke-lab/services/start`));+}++async function installFixtures(request: APIRequestContext, companyId: string): Promise<FixtureInstall> {return await json<FixtureInstall>(await request.post(`/api/companies/${companyId}/smoke-lab/install-fixtures`));}@@ -300,36 +304,37 @@ async function gatewayFetch(request: APIRequestContext, path: string, token: strreturn await request.get(path, { headers });}-test.describe.serial("Smoke Lab scenario catalog mirror", () => {- // The lifecycle test walks all CI-safe scenarios, and each scenario records- // eight steps with a real navigation and a full-page screenshot. On a loaded- // CI runner the whole walk needs a little more than four minutes, so the- // earlier 240s budget could expire on the final step. Six minutes gives- // headroom for runner variance without slowing a healthy run, because this is- // a ceiling, not the normal run time.+export const defineSmokeLabSuite = (label: string, scenarios: SmokeLabScenario[]) => test.describe.serial(`Smoke Lab scenario catalog mirror (${label})`, () => {+ // The lifecycle test walks all CI-safe scenarios and records eight steps per+ // scenario. Each scenario keeps one representative success screenshot, while+ // every failed step still captures its own evidence. On a loaded CI runner+ // the whole walk can take several minutes, so this ceiling keeps enough+ // headroom for runner variance without slowing a healthy run.test.setTimeout(360_000);- test("records the P1-P7 CI-safe Smoke Lab lifecycle into the results API @smoke-lab", async ({ page, request }) => {+ test(`records the ${label} CI-safe Smoke Lab lifecycle into the results API @smoke-lab`, async ({ page, request }) => {const seed = await newCompany(request, "catalog");const scout = await createScout(request, seed.companyId);await enableSmokeLab(request);- const smokeRun = await createSmokeRun(request, seed.companyId);+ const smokeRun = await createSmokeRun(request, seed.companyId, scenarios.length);const failed: string[] = [];+ const isPartialRun = scenarios.length < ciSmokeLabScenarios.length;try {- for (const scenario of ciSmokeLabScenarios) {- const fixtures = await startAndInstallFixtures(request, seed.companyId);+ await startSmokeLabServices(request, seed.companyId);++ for (const scenario of scenarios) {+ const fixtures = await installFixtures(request, seed.companyId);const connection = connectionForScenario(fixtures, scenario);await runRecordedStep(page, request, seed, smokeRun.id, scenario, "connect", async () => {await navigateForEvidence(page, seed, connection.id, scenario);return scenario.lifecycle.connect;- });+ }, true);await runRecordedStep(page, request, seed, smokeRun.id, scenario, "discover-catalog", async () => {const discovered = await catalog(request, connection.id);expect(discovered.catalog.map((entry) => entry.toolName)).toContain(scenario.lifecycle.allowedRead.name);- await navigateForEvidence(page, seed, connection.id, scenario);return `${scenario.lifecycle.discoverCatalog}: ${discovered.catalog.length} entries`;});@@ -440,20 +445,26 @@ test.describe.serial("Smoke Lab scenario catalog mirror", () => {} finally {await updateSmokeRun(request, seed.companyId, smokeRun.id, failed.length > 0 ? "failed" : "passed", {catalog: "tests/e2e/smoke-lab.catalog.ts",- scenarioCount: ciSmokeLabScenarios.length,+ scenarioCount: scenarios.length,+ catalogScenarioCount: ciSmokeLabScenarios.length,+ partial: isPartialRun,failed,}).catch(() => undefined);}- const completed = await json<{ run: SmokeRun; steps: Array<{ path: string; status: string; screenshotArtifactRef: Json | null }> }>(+ const completed = await json<{+ run: SmokeRun;+ steps: Array<{ path: string; scenarioStep: string; status: string; screenshotArtifactRef: Json | null }>;+ }>(await request.get(`/api/companies/${seed.companyId}/smoke-lab/runs/${smokeRun.id}`),);expect(completed.run.status).toBe("passed");- for (const scenario of ciSmokeLabScenarios) {+ for (const scenario of scenarios) {const steps = completed.steps.filter((step) => step.path === scenario.path);expect(steps.length, `${scenario.path} should record lifecycle steps`).toBeGreaterThanOrEqual(8);expect(steps.every((step) => step.status === "pass")).toBe(true);- expect(steps.every((step) => step.screenshotArtifactRef?.kind === "playwright_screenshot")).toBe(true);+ const screenshotSteps = steps.filter((step) => step.screenshotArtifactRef?.kind === "playwright_screenshot");+ expect(screenshotSteps.map((step) => step.scenarioStep)).toEqual(["connect"]);}});});diff --git a/ui/src/pages/tools/smoke-lab-matrix.test.ts b/ui/src/pages/tools/smoke-lab-matrix.test.tsindex 1b3a3560aa..49a55df11f 100644--- a/ui/src/pages/tools/smoke-lab-matrix.test.ts+++ b/ui/src/pages/tools/smoke-lab-matrix.test.ts@@ -80,6 +80,11 @@ describe("runHealth + failingPaths", () => {expect(runHealth(run({ status: "passed" }), steps)).toBe("green");});+ it("is amber for a passed partial-catalog run", () => {+ const steps = [step({ path: "P1", scenarioStep: "connect", status: "pass" })];+ expect(runHealth(run({ status: "passed", summary: { partial: true } }), steps)).toBe("amber");+ });+it("is amber for a running run or an empty run", () => {expect(runHealth(run({ status: "running" }), [])).toBe("amber");expect(runHealth(run({ status: "passed" }), [])).toBe("amber");diff --git a/ui/src/pages/tools/smoke-lab-matrix.ts b/ui/src/pages/tools/smoke-lab-matrix.tsindex c609e4c37d..392dc40fe5 100644--- a/ui/src/pages/tools/smoke-lab-matrix.ts+++ b/ui/src/pages/tools/smoke-lab-matrix.ts@@ -92,6 +92,7 @@ export function runHealth(run: SmokeRun | undefined, steps: SmokeRunStep[]): Smoif (!run) return "unknown";if (run.status === "failed") return "red";if (steps.some((s) => s.status === "fail")) return "red";+ if (run.summary.partial === true) return "amber";if (run.status === "cancelled") return "amber";if (run.status === "running") return "amber";if (steps.length === 0) return "amber";