AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
test(e2e): failing step tests and schedule waits report the server state (#16949)
Othman Abu Ajamiehbe794137856a
diff --git a/packages/tests-e2e/fixtures/api.ts b/packages/tests-e2e/fixtures/api.tsindex 709ae92d1b9..62b838e9739 100644--- a/packages/tests-e2e/fixtures/api.ts+++ b/packages/tests-e2e/fixtures/api.ts@@ -33,6 +33,63 @@ async function deleteStaleData({ request, session }: { request: APIRequestContexreturn { flows: staleFlows.length, connections: staleConnections.length };}+async function describeFlow({ request, session, flowId }: { request: APIRequestContext; session: E2eSession; flowId: string }) {+ const flow = await readJson<FlowState>({ request, session, path: `/api/v1/flows/${flowId}`, params: {} });+ const runs = await readJson<{ data: RunSummary[] }>({+ request,+ session,+ path: '/api/v1/flow-runs',+ params: { projectId: session.projectId, flowId, limit: 5 },+ });+ const flowLine = 'error' in flow+ ? flow.error+ : JSON.stringify({+ status: flow.body.status,+ operationStatus: flow.body.operationStatus,+ publishedVersionId: flow.body.publishedVersionId,+ versionState: flow.body.version.state,+ trigger: flow.body.version.trigger.settings.triggerName,+ triggerInput: flow.body.version.trigger.settings.input,+ });+ const runsLine = 'error' in runs+ ? runs.error+ : JSON.stringify(runs.body.data.map(({ status, environment, created }) => ({ status, environment, created })));+ return redact({ session, text: `flow: ${flowLine}\nruns: ${runsLine}` });+}++async function describeRun({ request, session, runId }: { request: APIRequestContext; session: E2eSession; runId: string }) {+ const run = await readJson<RunDetails>({ request, session, path: `/api/v1/flow-runs/${runId}`, params: {} });+ if ('error' in run) {+ return redact({ session, text: run.error });+ }+ const failedSteps = Object.entries(run.body.steps ?? {})+ .filter(([, step]) => step.status === 'FAILED')+ .map(([name, step]) => `${name}: ${(step.errorMessage ?? 'no error message').slice(0, 500)}`);+ return redact({ session, text: [`run ${run.body.status}`, ...failedSteps].join('; ') });+}++async function readJson<T>({ request, session, path, params }: {+ request: APIRequestContext;+ session: E2eSession;+ path: string;+ params: Record<string, string | number>;+}): Promise<{ body: T } | { error: string }> {+ const response = await request+ .get(path, { headers: { Authorization: `Bearer ${session.token}` }, params })+ .catch((error: unknown) => new Error(String(error)));+ if (response instanceof Error) {+ return { error: `GET ${path} failed: ${response.message}` };+ }+ if (!response.ok()) {+ return { error: `GET ${path} returned ${response.status()}` };+ }+ return { body: await response.json() };+}++function redact({ session, text }: { session: E2eSession; text: string }) {+ return text.replaceAll(session.token, '***');+}+async function deleteById({ request, session, resource, id }: { request: APIRequestContext; session: E2eSession; resource: Resource; id: string }) {const response = await request.delete(`/api/v1/${resource}/${id}`, {headers: { Authorization: `Bearer ${session.token}` },@@ -67,11 +124,19 @@ const STALE_AFTER_MS = 60 * 60 * 1000;export const STORAGE_STATE = path.join(__dirname, '..', '.auth', 'user.json');export const E2E_PREFIX = 'E2E ';-export const e2eApi = { readSession, deleteFlow, deleteConnection, deleteStaleData };+export const e2eApi = { readSession, deleteFlow, deleteConnection, deleteStaleData, describeFlow, describeRun };export type E2eSession = { token: string; projectId: string };type Resource = 'flows' | 'app-connections';type FlowSummary = { id: string; created: string; version: { displayName: string } };type ConnectionSummary = { id: string; created: string; displayName: string };+type FlowState = {+ status: string;+ operationStatus: string;+ publishedVersionId: string | null;+ version: { state: string; trigger: { settings: { triggerName?: string; input?: unknown } } };+};+type RunSummary = { status: string; environment: string; created: string };+type RunDetails = { status: string; steps?: Record<string, { status: string; errorMessage?: string }> };type StorageState = { origins: { localStorage: { name: string; value: string }[] }[] };diff --git a/packages/tests-e2e/fixtures/index.ts b/packages/tests-e2e/fixtures/index.tsindex d526b90da5b..c24888837ed 100644--- a/packages/tests-e2e/fixtures/index.ts+++ b/packages/tests-e2e/fixtures/index.ts@@ -16,8 +16,8 @@ export const test = base.extend<CustomFixtures>({await use(new AutomationsPage(page));},- builderPage: async ({ page }, use) => {- await use(new BuilderPage(page));+ builderPage: async ({ page, request, session }, use) => {+ await use(new BuilderPage({ page, describeRun: (runId) => e2eApi.describeRun({ request, session, runId }) }));},runsPage: async ({ page }, use) => {diff --git a/packages/tests-e2e/pages/builder.page.ts b/packages/tests-e2e/pages/builder.page.tsindex 9bfcbe5b249..1f486048d5e 100644--- a/packages/tests-e2e/pages/builder.page.ts+++ b/packages/tests-e2e/pages/builder.page.ts@@ -1,9 +1,16 @@-import { expect, Locator } from '@playwright/test';+import { expect, Locator, Page } from '@playwright/test';import { BasePage } from './base';export class BuilderPage extends BasePage {url = `/flows`;+ private readonly describeRun: (runId: string) => Promise<string>;++ constructor({ page, describeRun }: { page: Page; describeRun: (runId: string) => Promise<string> }) {+ super(page);+ this.describeRun = describeRun;+ }+async nameNewFlow(name: string) {const breadcrumb = this.page.getByRole('navigation', { name: 'breadcrumb' });await breadcrumb.getByText('Untitled', { exact: true }).click();@@ -86,10 +93,18 @@ export class BuilderPage extends BasePage {}async testStep() {+ const started = this.page.waitForResponse(+ (response) => response.request().method() === 'POST' && new URL(response.url()).pathname === STEP_TEST,+ );await this.sidebar().getByTestId('test-step-button').click();+ const response = await started;+ expect(response.ok(), `Starting the step test failed with ${response.status()}: ${await response.text()}`).toBe(true);+ const run: { id: string } = await response.json();const succeeded = this.sidebar().getByText('Tested Successfully');- await succeeded.or(this.sidebar().getByText('Test Failed')).first().waitFor({ timeout: 60000 });- await expect(succeeded, 'The step test failed').toBeVisible();+ await succeeded.or(this.sidebar().getByText('Test Failed')).first().waitFor({ timeout: 60000 }).catch(() => undefined);+ if (!(await succeeded.isVisible())) {+ throw new Error(`The step test did not succeed: ${await this.describeRun(run.id)}`);+ }}async expandStepOutput(key: string) {@@ -110,7 +125,14 @@ export class BuilderPage extends BasePage {async publish() {const publishButton = this.page.getByRole('button', { name: 'Publish' });+ const published = this.page.waitForResponse(+ (response) => response.request().method() === 'POST'+ && FLOW_SAVE.test(new URL(response.url()).pathname)+ && response.request().postDataJSON()?.type === 'LOCK_AND_PUBLISH',+ );await publishButton.click();+ const response = await published;+ expect(response.ok(), `Publishing failed with ${response.status()}: ${await response.text()}`).toBe(true);await expect(publishButton).toBeHidden({ timeout: 60000 });}@@ -154,3 +176,4 @@ export class BuilderPage extends BasePage {}const FLOW_SAVE = /\/api\/v1\/flows\/[^/]+$/;+const STEP_TEST = '/api/v1/sample-data/test-step';diff --git a/packages/tests-e2e/scenarios/flows/schedule-trigger.spec.ts b/packages/tests-e2e/scenarios/flows/schedule-trigger.spec.tsindex 41a37dc5059..34784950158 100644--- a/packages/tests-e2e/scenarios/flows/schedule-trigger.spec.ts+++ b/packages/tests-e2e/scenarios/flows/schedule-trigger.spec.ts@@ -1,8 +1,9 @@import { test } from '../../fixtures';+import { e2eApi } from '../../fixtures/api';import { e2eFlows } from '../../fixtures/flows';test.describe('Schedule trigger', () => {- test('a published flow on a one-minute schedule runs by itself', async ({ automationsPage, builderPage, runsPage, flowCleanup }) => {+ test('a published flow on a one-minute schedule runs by itself', async ({ automationsPage, builderPage, runsPage, flowCleanup, request, session }) => {test.setTimeout(240000);const flowName = e2eFlows.uniqueName('schedule');@@ -14,12 +15,17 @@ test.describe('Schedule trigger', () => {return id;});- await test.step('Publish the flow', async () => {+ const publishedAt = await test.step('Publish the flow', async () => {await builderPage.publish();+ return Date.now();});await test.step('See a scheduled run succeed in the run history', async () => {- await runsPage.expectLatestRun({ flowId, flowName, status: 'Succeeded', timeout: 150000 });+ await runsPage.expectLatestRun({ flowId, flowName, status: 'Succeeded', timeout: 150000 }).catch(async (error: Error) => {+ const flowState = await e2eApi.describeFlow({ request, session, flowId });+ const seconds = Math.round((Date.now() - publishedAt) / 1000);+ throw new Error(`${error.message}\n${seconds} s after publish\n${flowState}`);+ });});});});