Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

test(e2e): failing step tests and schedule waits report the server state (#16949)

Othman Abu Ajamiehbe794137856a
diff --git a/packages/tests-e2e/fixtures/api.ts b/packages/tests-e2e/fixtures/api.ts
index 709ae92d1b9..62b838e9739 100644
--- a/packages/tests-e2e/fixtures/api.ts
+++ b/packages/tests-e2e/fixtures/api.ts
@@ -33,6 +33,63 @@ async function deleteStaleData({ request, session }: { request: APIRequestContex
return { flows: staleFlows.length, connections: staleConnections.length };
}
+async function describeFlow({ request, session, flowId }: { request: APIRequestContext; session: E2eSession; flowId: string }) {
+ const flow = await readJson<FlowState>({ request, session, path: `/api/v1/flows/${flowId}`, params: {} });
+ const runs = await readJson<{ data: RunSummary[] }>({
+ request,
+ session,
+ path: '/api/v1/flow-runs',
+ params: { projectId: session.projectId, flowId, limit: 5 },
+ });
+ const flowLine = 'error' in flow
+ ? flow.error
+ : JSON.stringify({
+ status: flow.body.status,
+ operationStatus: flow.body.operationStatus,
+ publishedVersionId: flow.body.publishedVersionId,
+ versionState: flow.body.version.state,
+ trigger: flow.body.version.trigger.settings.triggerName,
+ triggerInput: flow.body.version.trigger.settings.input,
+ });
+ const runsLine = 'error' in runs
+ ? runs.error
+ : JSON.stringify(runs.body.data.map(({ status, environment, created }) => ({ status, environment, created })));
+ return redact({ session, text: `flow: ${flowLine}\nruns: ${runsLine}` });
+}
+
+async function describeRun({ request, session, runId }: { request: APIRequestContext; session: E2eSession; runId: string }) {
+ const run = await readJson<RunDetails>({ request, session, path: `/api/v1/flow-runs/${runId}`, params: {} });
+ if ('error' in run) {
+ return redact({ session, text: run.error });
+ }
+ const failedSteps = Object.entries(run.body.steps ?? {})
+ .filter(([, step]) => step.status === 'FAILED')
+ .map(([name, step]) => `${name}: ${(step.errorMessage ?? 'no error message').slice(0, 500)}`);
+ return redact({ session, text: [`run ${run.body.status}`, ...failedSteps].join('; ') });
+}
+
+async function readJson<T>({ request, session, path, params }: {
+ request: APIRequestContext;
+ session: E2eSession;
+ path: string;
+ params: Record<string, string | number>;
+}): Promise<{ body: T } | { error: string }> {
+ const response = await request
+ .get(path, { headers: { Authorization: `Bearer ${session.token}` }, params })
+ .catch((error: unknown) => new Error(String(error)));
+ if (response instanceof Error) {
+ return { error: `GET ${path} failed: ${response.message}` };
+ }
+ if (!response.ok()) {
+ return { error: `GET ${path} returned ${response.status()}` };
+ }
+ return { body: await response.json() };
+}
+
+function redact({ session, text }: { session: E2eSession; text: string }) {
+ return text.replaceAll(session.token, '***');
+}
+
async function deleteById({ request, session, resource, id }: { request: APIRequestContext; session: E2eSession; resource: Resource; id: string }) {
const response = await request.delete(`/api/v1/${resource}/${id}`, {
headers: { Authorization: `Bearer ${session.token}` },
@@ -67,11 +124,19 @@ const STALE_AFTER_MS = 60 * 60 * 1000;
export const STORAGE_STATE = path.join(__dirname, '..', '.auth', 'user.json');
export const E2E_PREFIX = 'E2E ';
-export const e2eApi = { readSession, deleteFlow, deleteConnection, deleteStaleData };
+export const e2eApi = { readSession, deleteFlow, deleteConnection, deleteStaleData, describeFlow, describeRun };
export type E2eSession = { token: string; projectId: string };
type Resource = 'flows' | 'app-connections';
type FlowSummary = { id: string; created: string; version: { displayName: string } };
type ConnectionSummary = { id: string; created: string; displayName: string };
+type FlowState = {
+ status: string;
+ operationStatus: string;
+ publishedVersionId: string | null;
+ version: { state: string; trigger: { settings: { triggerName?: string; input?: unknown } } };
+};
+type RunSummary = { status: string; environment: string; created: string };
+type RunDetails = { status: string; steps?: Record<string, { status: string; errorMessage?: string }> };
type StorageState = { origins: { localStorage: { name: string; value: string }[] }[] };
diff --git a/packages/tests-e2e/fixtures/index.ts b/packages/tests-e2e/fixtures/index.ts
index d526b90da5b..c24888837ed 100644
--- a/packages/tests-e2e/fixtures/index.ts
+++ b/packages/tests-e2e/fixtures/index.ts
@@ -16,8 +16,8 @@ export const test = base.extend<CustomFixtures>({
await use(new AutomationsPage(page));
},
- builderPage: async ({ page }, use) => {
- await use(new BuilderPage(page));
+ builderPage: async ({ page, request, session }, use) => {
+ await use(new BuilderPage({ page, describeRun: (runId) => e2eApi.describeRun({ request, session, runId }) }));
},
runsPage: async ({ page }, use) => {
diff --git a/packages/tests-e2e/pages/builder.page.ts b/packages/tests-e2e/pages/builder.page.ts
index 9bfcbe5b249..1f486048d5e 100644
--- a/packages/tests-e2e/pages/builder.page.ts
+++ b/packages/tests-e2e/pages/builder.page.ts
@@ -1,9 +1,16 @@
-import { expect, Locator } from '@playwright/test';
+import { expect, Locator, Page } from '@playwright/test';
import { BasePage } from './base';
export class BuilderPage extends BasePage {
url = `/flows`;
+ private readonly describeRun: (runId: string) => Promise<string>;
+
+ constructor({ page, describeRun }: { page: Page; describeRun: (runId: string) => Promise<string> }) {
+ super(page);
+ this.describeRun = describeRun;
+ }
+
async nameNewFlow(name: string) {
const breadcrumb = this.page.getByRole('navigation', { name: 'breadcrumb' });
await breadcrumb.getByText('Untitled', { exact: true }).click();
@@ -86,10 +93,18 @@ export class BuilderPage extends BasePage {
}
async testStep() {
+ const started = this.page.waitForResponse(
+ (response) => response.request().method() === 'POST' && new URL(response.url()).pathname === STEP_TEST,
+ );
await this.sidebar().getByTestId('test-step-button').click();
+ const response = await started;
+ expect(response.ok(), `Starting the step test failed with ${response.status()}: ${await response.text()}`).toBe(true);
+ const run: { id: string } = await response.json();
const succeeded = this.sidebar().getByText('Tested Successfully');
- await succeeded.or(this.sidebar().getByText('Test Failed')).first().waitFor({ timeout: 60000 });
- await expect(succeeded, 'The step test failed').toBeVisible();
+ await succeeded.or(this.sidebar().getByText('Test Failed')).first().waitFor({ timeout: 60000 }).catch(() => undefined);
+ if (!(await succeeded.isVisible())) {
+ throw new Error(`The step test did not succeed: ${await this.describeRun(run.id)}`);
+ }
}
async expandStepOutput(key: string) {
@@ -110,7 +125,14 @@ export class BuilderPage extends BasePage {
async publish() {
const publishButton = this.page.getByRole('button', { name: 'Publish' });
+ const published = this.page.waitForResponse(
+ (response) => response.request().method() === 'POST'
+ && FLOW_SAVE.test(new URL(response.url()).pathname)
+ && response.request().postDataJSON()?.type === 'LOCK_AND_PUBLISH',
+ );
await publishButton.click();
+ const response = await published;
+ expect(response.ok(), `Publishing failed with ${response.status()}: ${await response.text()}`).toBe(true);
await expect(publishButton).toBeHidden({ timeout: 60000 });
}
@@ -154,3 +176,4 @@ export class BuilderPage extends BasePage {
}
const FLOW_SAVE = /\/api\/v1\/flows\/[^/]+$/;
+const STEP_TEST = '/api/v1/sample-data/test-step';
diff --git a/packages/tests-e2e/scenarios/flows/schedule-trigger.spec.ts b/packages/tests-e2e/scenarios/flows/schedule-trigger.spec.ts
index 41a37dc5059..34784950158 100644
--- a/packages/tests-e2e/scenarios/flows/schedule-trigger.spec.ts
+++ b/packages/tests-e2e/scenarios/flows/schedule-trigger.spec.ts
@@ -1,8 +1,9 @@
import { test } from '../../fixtures';
+import { e2eApi } from '../../fixtures/api';
import { e2eFlows } from '../../fixtures/flows';
test.describe('Schedule trigger', () => {
- test('a published flow on a one-minute schedule runs by itself', async ({ automationsPage, builderPage, runsPage, flowCleanup }) => {
+ test('a published flow on a one-minute schedule runs by itself', async ({ automationsPage, builderPage, runsPage, flowCleanup, request, session }) => {
test.setTimeout(240000);
const flowName = e2eFlows.uniqueName('schedule');
@@ -14,12 +15,17 @@ test.describe('Schedule trigger', () => {
return id;
});
- await test.step('Publish the flow', async () => {
+ const publishedAt = await test.step('Publish the flow', async () => {
await builderPage.publish();
+ return Date.now();
});
await test.step('See a scheduled run succeed in the run history', async () => {
- await runsPage.expectLatestRun({ flowId, flowName, status: 'Succeeded', timeout: 150000 });
+ await runsPage.expectLatestRun({ flowId, flowName, status: 'Succeeded', timeout: 150000 }).catch(async (error: Error) => {
+ const flowState = await e2eApi.describeFlow({ request, session, flowId });
+ const seconds = Math.round((Date.now() - publishedAt) / 1000);
+ throw new Error(`${error.message}\n${seconds} s after publish\n${flowState}`);
+ });
});
});
});