Skip to content

The open-source app everyone uses to manage agents at work

test(e2e): shorten and split Smoke Lab coverage (#12506)

Dotta39eafad47d77
diff --git a/doc/connections/SMOKE-LAB-BROWSER-RUNNER.md b/doc/connections/SMOKE-LAB-BROWSER-RUNNER.md
index f5f491b2e6..65bfe876cc 100644
--- a/doc/connections/SMOKE-LAB-BROWSER-RUNNER.md
+++ b/doc/connections/SMOKE-LAB-BROWSER-RUNNER.md
@@ -4,9 +4,10 @@
**Scenario source of truth:** [`tests/e2e/smoke-lab.catalog.ts`](../../tests/e2e/smoke-lab.catalog.ts) — **do not fork the step list.**
This is the manual/agent counterpart to the deterministic Playwright CI mirror
-(`tests/e2e/smoke-lab.spec.ts`, S4). Where the CI mirror runs headless in a
-throwaway instance for a red/green gate, this runbook has a **QA agent drive a
-real browser** through the same P1–P7 lifecycle against a live instance,
+(`tests/e2e/smoke-lab.shared.ts`, split between the P1–P4 and P5–P7 specs).
+Where the CI mirror runs headless in a throwaway instance for a red/green gate,
+this runbook has a **QA agent drive a real browser** through the same P1–P7
+lifecycle against a live instance,
**typing demo credentials into the fake OAuth provider's real consent page**, and
records every step + a viewable screenshot to the Smoke Lab results API so the run
shows up in the **Smoke Lab tab** and the **dashboard "Integration smoke" card**.
diff --git a/doc/connections/SMOKE-LAB-TUTORIAL.md b/doc/connections/SMOKE-LAB-TUTORIAL.md
index c6093a9f43..0299fbda05 100644
--- a/doc/connections/SMOKE-LAB-TUTORIAL.md
+++ b/doc/connections/SMOKE-LAB-TUTORIAL.md
@@ -12,9 +12,10 @@ button or label is quoted, that's the exact text in the product.
> Companion docs: the automated counterparts live in
> [`SMOKE-LAB-BROWSER-RUNNER.md`](./SMOKE-LAB-BROWSER-RUNNER.md) (the agent-driven
-> browser runner) and `tests/e2e/smoke-lab.spec.ts` (the headless CI mirror). The
-> daily recurring routine that runs the browser smoke for you is described in
-> [§8](#8-the-daily-routine-hands-off).
+> browser runner), `tests/e2e/smoke-lab.shared.ts` (the shared headless runner),
+> and the `tests/e2e/smoke-lab-p1-p4.spec.ts` and
+> `tests/e2e/smoke-lab-p5-p7.spec.ts` CI halves. The daily recurring routine that
+> runs the browser smoke for you is described in [§8](#8-the-daily-routine-hands-off).
---
diff --git a/scripts/e2e-shard-durations.json b/scripts/e2e-shard-durations.json
index b23c25efdb..af5549c94d 100644
--- a/scripts/e2e-shard-durations.json
+++ b/scripts/e2e-shard-durations.json
@@ -1,5 +1,5 @@
{
- "$comment": "Per-spec wall-clock durations (ms) for the Playwright e2e lane, used by scripts/e2e-shard.mjs to balance specs across the PR shard matrix. Averaged from two real PR runs of .github/workflows/pr.yml (actions runs 29775184389 and 29769465682, 2026-07-20). Specs missing here get the median weight, so the manifest only needs occasional refreshes: pull the 'Run e2e tests' logs from a recent PR run and sum the Playwright list-reporter durations per spec file.",
+ "$comment": "Per-spec wall-clock durations (ms) for the Playwright e2e lane, used by scripts/e2e-shard.mjs to balance specs across the PR shard matrix. Averaged from two real PR runs of .github/workflows/pr.yml (actions runs 29775184389 and 29769465682, 2026-07-20). The split Smoke Lab weights scale its prior 168s CI duration by the measured local before/after ratio and scenario count; refresh them from the next two real PR runs. Specs missing here get the median weight, so the manifest only needs occasional refreshes: pull the 'Run e2e tests' logs from a recent PR run and sum the Playwright list-reporter durations per spec file.",
"unit": "ms",
"durations": {
"tests/e2e/app-not-connected.spec.ts": 26750,
@@ -15,6 +15,7 @@
"tests/e2e/planning-mode-visual-verification.spec.ts": 23250,
"tests/e2e/sidebar-takeover.spec.ts": 16650,
"tests/e2e/signoff-policy.spec.ts": 9696,
- "tests/e2e/smoke-lab.spec.ts": 168000
+ "tests/e2e/smoke-lab-p1-p4.spec.ts": 71000,
+ "tests/e2e/smoke-lab-p5-p7.spec.ts": 53000
}
}
diff --git a/tests/e2e/smoke-lab-p1-p4.spec.ts b/tests/e2e/smoke-lab-p1-p4.spec.ts
new file mode 100644
index 0000000000..51afd51fd6
--- /dev/null
+++ b/tests/e2e/smoke-lab-p1-p4.spec.ts
@@ -0,0 +1,4 @@
+import { ciSmokeLabScenarios } from "./smoke-lab.catalog";
+import { defineSmokeLabSuite } from "./smoke-lab.shared";
+
+defineSmokeLabSuite("P1-P4", ciSmokeLabScenarios.slice(0, 4));
diff --git a/tests/e2e/smoke-lab-p5-p7.spec.ts b/tests/e2e/smoke-lab-p5-p7.spec.ts
new file mode 100644
index 0000000000..4f3b704ad9
--- /dev/null
+++ b/tests/e2e/smoke-lab-p5-p7.spec.ts
@@ -0,0 +1,4 @@
+import { ciSmokeLabScenarios } from "./smoke-lab.catalog";
+import { defineSmokeLabSuite } from "./smoke-lab.shared";
+
+defineSmokeLabSuite("P5-P7", ciSmokeLabScenarios.slice(4));
diff --git a/tests/e2e/smoke-lab.spec.ts b/tests/e2e/smoke-lab.shared.ts
similarity index 90%
rename from tests/e2e/smoke-lab.spec.ts
rename to tests/e2e/smoke-lab.shared.ts
index 3b6c0904f0..d69568727f 100644
--- a/tests/e2e/smoke-lab.spec.ts
+++ b/tests/e2e/smoke-lab.shared.ts
@@ -85,14 +85,14 @@ async function enableSmokeLab(request: APIRequestContext) {
await json(await request.patch("/api/instance/settings/experimental", { data: { enableSmokeLab: true, enableApps: true } }));
}
-async function createSmokeRun(request: APIRequestContext, companyId: string) {
+async function createSmokeRun(request: APIRequestContext, companyId: string, scenarioCount: number) {
const result = await json<{ run: SmokeRun }>(
await request.post(`/api/companies/${companyId}/smoke-lab/runs`, {
data: {
trigger: "ci",
summary: {
catalog: "tests/e2e/smoke-lab.catalog.ts",
- scenarioCount: ciSmokeLabScenarios.length,
+ scenarioCount,
},
},
}),
@@ -180,11 +180,12 @@ async function runRecordedStep(
scenario: SmokeLabScenario,
step: string,
action: () => Promise<string | null | undefined>,
+ captureSuccessScreenshot = false,
) {
const start = Date.now();
try {
const screenshotHint = await action();
- const screenshotPath = await screenshot(page, scenario, step);
+ const screenshotPath = captureSuccessScreenshot ? await screenshot(page, scenario, step) : null;
await recordStep(request, seed.companyId, runId, {
path: scenario.path,
scenarioStep: step,
@@ -207,8 +208,11 @@ async function runRecordedStep(
}
}
-async function startAndInstallFixtures(request: APIRequestContext, companyId: string): Promise<FixtureInstall> {
+async function startSmokeLabServices(request: APIRequestContext, companyId: string) {
await json(await request.post(`/api/companies/${companyId}/smoke-lab/services/start`));
+}
+
+async function installFixtures(request: APIRequestContext, companyId: string): Promise<FixtureInstall> {
return await json<FixtureInstall>(await request.post(`/api/companies/${companyId}/smoke-lab/install-fixtures`));
}
@@ -300,36 +304,37 @@ async function gatewayFetch(request: APIRequestContext, path: string, token: str
return await request.get(path, { headers });
}
-test.describe.serial("Smoke Lab scenario catalog mirror", () => {
- // The lifecycle test walks all CI-safe scenarios, and each scenario records
- // eight steps with a real navigation and a full-page screenshot. On a loaded
- // CI runner the whole walk needs a little more than four minutes, so the
- // earlier 240s budget could expire on the final step. Six minutes gives
- // headroom for runner variance without slowing a healthy run, because this is
- // a ceiling, not the normal run time.
+export const defineSmokeLabSuite = (label: string, scenarios: SmokeLabScenario[]) => test.describe.serial(`Smoke Lab scenario catalog mirror (${label})`, () => {
+ // The lifecycle test walks all CI-safe scenarios and records eight steps per
+ // scenario. Each scenario keeps one representative success screenshot, while
+ // every failed step still captures its own evidence. On a loaded CI runner
+ // the whole walk can take several minutes, so this ceiling keeps enough
+ // headroom for runner variance without slowing a healthy run.
test.setTimeout(360_000);
- test("records the P1-P7 CI-safe Smoke Lab lifecycle into the results API @smoke-lab", async ({ page, request }) => {
+ test(`records the ${label} CI-safe Smoke Lab lifecycle into the results API @smoke-lab`, async ({ page, request }) => {
const seed = await newCompany(request, "catalog");
const scout = await createScout(request, seed.companyId);
await enableSmokeLab(request);
- const smokeRun = await createSmokeRun(request, seed.companyId);
+ const smokeRun = await createSmokeRun(request, seed.companyId, scenarios.length);
const failed: string[] = [];
+ const isPartialRun = scenarios.length < ciSmokeLabScenarios.length;
try {
- for (const scenario of ciSmokeLabScenarios) {
- const fixtures = await startAndInstallFixtures(request, seed.companyId);
+ await startSmokeLabServices(request, seed.companyId);
+
+ for (const scenario of scenarios) {
+ const fixtures = await installFixtures(request, seed.companyId);
const connection = connectionForScenario(fixtures, scenario);
await runRecordedStep(page, request, seed, smokeRun.id, scenario, "connect", async () => {
await navigateForEvidence(page, seed, connection.id, scenario);
return scenario.lifecycle.connect;
- });
+ }, true);
await runRecordedStep(page, request, seed, smokeRun.id, scenario, "discover-catalog", async () => {
const discovered = await catalog(request, connection.id);
expect(discovered.catalog.map((entry) => entry.toolName)).toContain(scenario.lifecycle.allowedRead.name);
- await navigateForEvidence(page, seed, connection.id, scenario);
return `${scenario.lifecycle.discoverCatalog}: ${discovered.catalog.length} entries`;
});
@@ -440,20 +445,26 @@ test.describe.serial("Smoke Lab scenario catalog mirror", () => {
} finally {
await updateSmokeRun(request, seed.companyId, smokeRun.id, failed.length > 0 ? "failed" : "passed", {
catalog: "tests/e2e/smoke-lab.catalog.ts",
- scenarioCount: ciSmokeLabScenarios.length,
+ scenarioCount: scenarios.length,
+ catalogScenarioCount: ciSmokeLabScenarios.length,
+ partial: isPartialRun,
failed,
}).catch(() => undefined);
}
- const completed = await json<{ run: SmokeRun; steps: Array<{ path: string; status: string; screenshotArtifactRef: Json | null }> }>(
+ const completed = await json<{
+ run: SmokeRun;
+ steps: Array<{ path: string; scenarioStep: string; status: string; screenshotArtifactRef: Json | null }>;
+ }>(
await request.get(`/api/companies/${seed.companyId}/smoke-lab/runs/${smokeRun.id}`),
);
expect(completed.run.status).toBe("passed");
- for (const scenario of ciSmokeLabScenarios) {
+ for (const scenario of scenarios) {
const steps = completed.steps.filter((step) => step.path === scenario.path);
expect(steps.length, `${scenario.path} should record lifecycle steps`).toBeGreaterThanOrEqual(8);
expect(steps.every((step) => step.status === "pass")).toBe(true);
- expect(steps.every((step) => step.screenshotArtifactRef?.kind === "playwright_screenshot")).toBe(true);
+ const screenshotSteps = steps.filter((step) => step.screenshotArtifactRef?.kind === "playwright_screenshot");
+ expect(screenshotSteps.map((step) => step.scenarioStep)).toEqual(["connect"]);
}
});
});
diff --git a/ui/src/pages/tools/smoke-lab-matrix.test.ts b/ui/src/pages/tools/smoke-lab-matrix.test.ts
index 1b3a3560aa..49a55df11f 100644
--- a/ui/src/pages/tools/smoke-lab-matrix.test.ts
+++ b/ui/src/pages/tools/smoke-lab-matrix.test.ts
@@ -80,6 +80,11 @@ describe("runHealth + failingPaths", () => {
expect(runHealth(run({ status: "passed" }), steps)).toBe("green");
});
+ it("is amber for a passed partial-catalog run", () => {
+ const steps = [step({ path: "P1", scenarioStep: "connect", status: "pass" })];
+ expect(runHealth(run({ status: "passed", summary: { partial: true } }), steps)).toBe("amber");
+ });
+
it("is amber for a running run or an empty run", () => {
expect(runHealth(run({ status: "running" }), [])).toBe("amber");
expect(runHealth(run({ status: "passed" }), [])).toBe("amber");
diff --git a/ui/src/pages/tools/smoke-lab-matrix.ts b/ui/src/pages/tools/smoke-lab-matrix.ts
index c609e4c37d..392dc40fe5 100644
--- a/ui/src/pages/tools/smoke-lab-matrix.ts
+++ b/ui/src/pages/tools/smoke-lab-matrix.ts
@@ -92,6 +92,7 @@ export function runHealth(run: SmokeRun | undefined, steps: SmokeRunStep[]): Smo
if (!run) return "unknown";
if (run.status === "failed") return "red";
if (steps.some((s) => s.status === "fail")) return "red";
+ if (run.summary.partial === true) return "amber";
if (run.status === "cancelled") return "amber";
if (run.status === "running") return "amber";
if (steps.length === 0) return "amber";