AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(connections): hold the flow locks for the whole connection replace (#16972)
Abdul22a1c134e2d3
diff --git a/packages/server/api/src/app/app-connection/app-connection-service/app-connection.handler.ts b/packages/server/api/src/app/app-connection/app-connection-service/app-connection.handler.tsindex cb0758667f6..22d2e4e917d 100644--- a/packages/server/api/src/app/app-connection/app-connection-service/app-connection.handler.ts+++ b/packages/server/api/src/app/app-connection/app-connection-service/app-connection.handler.ts@@ -1,13 +1,13 @@import { ActivepiecesError, ErrorCode, FlowId, isNil, PlatformId, ProjectId, tryCatch, unique, UserId } from '@activepieces/core-utils'import { PropertyType } from '@activepieces/pieces-framework'-import { AppConnection, AppConnectionStatus, AppConnectionType, AppConnectionValue, AppConnectionWithoutSensitiveData, EngineResponse, EngineResponseStatus, ExecuteRefreshTokenAuthResponse, Flow, FlowOperationType, flowStructureUtil, FlowVersion, PopulatedFlow, WorkerJobType } from '@activepieces/shared'+import { AppConnection, AppConnectionStatus, AppConnectionType, AppConnectionValue, AppConnectionWithoutSensitiveData, EngineResponse, EngineResponseStatus, ExecuteRefreshTokenAuthResponse, PopulatedFlow, WorkerJobType } from '@activepieces/shared'import dayjs from 'dayjs'import { FastifyBaseLogger } from 'fastify'import { lru, LRU } from 'tiny-lru'import { ArrayContains } from 'typeorm'import { distributedLock } from '../../database/redis-connections'import { flowService } from '../../flows/flow/flow.service'-import { flowVersionRepo, flowVersionService } from '../../flows/flow-version/flow-version.service'+import { flowVersionRepo } from '../../flows/flow-version/flow-version.service'import { encryptUtils } from '../../helper/encryption'import { exceptionHandler } from '../../helper/exception-handler'import { getPiecePackageWithoutArchive, pieceMetadataService } from '../../pieces/metadata/piece-metadata-service'@@ -24,12 +24,16 @@ export const appConnectionHandler = (log: FastifyBaseLogger) => ({: new Set<FlowId>()await Promise.all(flows.map(async (flow) => {- const replaceParams = { flow, latestVersion: flow.version, userId, platformId, appConnection, newAppConnection, log }- if (flowIdsToRepublish.has(flow.id)) {- await replaceConnectionInPublishedAndDraft(replaceParams)- return- }- await replaceConnectionInDraft(replaceParams)+ await flowService(log).replaceConnection({+ id: flow.id,+ projectId: flow.projectId,+ platformId,+ userId,+ sourceExternalId: appConnection.externalId,+ targetExternalId: newAppConnection.externalId,+ applyToPublishedVersion: flowIdsToRepublish.has(flow.id),+ auditExplanation: `replaced connection "${appConnection.displayName}" with "${newAppConnection.displayName}".`,+ })}))},@@ -331,92 +335,6 @@ async function findFlowIdsToRepublish({ flows, externalId }: FindFlowIdsToRepublreturn new Set(rows.map((row) => row.flowId))}-async function replaceConnectionInPublishedAndDraft(params: ReplaceConnectionInPublishedAndDraftParams): Promise<void> {- const { flow, latestVersion, userId, platformId, appConnection, newAppConnection, log } = params- const projectId = flow.projectId- const publishedVersionId = flow.publishedVersionId- if (isNil(publishedVersionId)) {- return- }-- const { error } = await tryCatch(async () => {- const publishedVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: flow.id, versionId: publishedVersionId })-- await flowService(log).update({- id: flow.id,- projectId,- platformId,- userId,- previousFlow: flow,- operation: {- type: FlowOperationType.IMPORT_FLOW,- request: replaceConnectionInFlowVersion({ flowVersion: publishedVersion, appConnection, newAppConnection }),- },- auditExplanation: replaceConnectionAuditExplanation({ appConnection, newAppConnection }),- })-- const { status } = await flowService(log).getOneOrThrow({ id: flow.id, projectId })- await flowService(log).update({- id: flow.id,- projectId,- platformId,- userId,- operation: {- type: FlowOperationType.LOCK_AND_PUBLISH,- request: { status },- },- skipRequiredActionsCheck: true,- })- })-- const hadUnpublishedChanges = latestVersion.id !== publishedVersionId- const republishFailed = !isNil(error)- if (hadUnpublishedChanges || republishFailed) {- await replaceConnectionInDraft({ ...params, restoreDraft: true })- }- if (republishFailed) {- throw error- }-}--async function replaceConnectionInDraft({ flow, latestVersion, userId, platformId, appConnection, newAppConnection, restoreDraft = false, log }: ReplaceConnectionInDraftParams): Promise<void> {- const usesConnection = latestVersion.connectionIds.includes(appConnection.externalId)- || flowStructureUtil.extractConnectionIds(latestVersion).includes(appConnection.externalId)- if (!usesConnection && !restoreDraft) {- return- }-- await flowService(log).update({- id: flow.id,- projectId: flow.projectId,- platformId,- userId,- operation: {- type: FlowOperationType.IMPORT_FLOW,- request: replaceConnectionInFlowVersion({ flowVersion: latestVersion, appConnection, newAppConnection }),- },- auditExplanation: replaceConnectionAuditExplanation({ appConnection, newAppConnection }),- })-}--function replaceConnectionInFlowVersion({ flowVersion, appConnection, newAppConnection }: ReplaceConnectionInFlowVersionParams): FlowVersion {- return flowStructureUtil.replaceConnection({- flowVersion,- sourceExternalId: appConnection.externalId,- targetExternalId: newAppConnection.externalId,- })-}--function replaceConnectionAuditExplanation({ appConnection, newAppConnection }: Omit<ReplaceConnectionInFlowVersionParams, 'flowVersion'>): string {- return `replaced connection "${appConnection.displayName}" with "${newAppConnection.displayName}".`-}--type ReplaceConnectionInFlowVersionParams = {- flowVersion: FlowVersion- appConnection: AppConnectionWithoutSensitiveData- newAppConnection: AppConnectionWithoutSensitiveData-}-type ReplaceConnectionInFlowsParams = {flows: PopulatedFlow[]appConnection: AppConnectionWithoutSensitiveData@@ -431,17 +349,3 @@ type FindFlowIdsToRepublishParams = {externalId: string}-type ReplaceConnectionInDraftParams = {- flow: Flow- latestVersion: FlowVersion- userId: UserId- platformId: PlatformId- appConnection: AppConnectionWithoutSensitiveData- newAppConnection: AppConnectionWithoutSensitiveData- restoreDraft?: boolean- log: FastifyBaseLogger-}--type ReplaceConnectionInPublishedAndDraftParams = ReplaceConnectionInDraftParams & {- flow: PopulatedFlow-}diff --git a/packages/server/api/src/app/flows/flow/flow.service.ts b/packages/server/api/src/app/flows/flow/flow.service.tsindex 10e45396a93..ac7ace3064b 100644--- a/packages/server/api/src/app/flows/flow/flow.service.ts+++ b/packages/server/api/src/app/flows/flow/flow.service.ts@@ -1,6 +1,6 @@import { ActivepiecesError, apId, Cursor, ErrorCode, FlowId, FlowVersionId, isNil, Metadata, PlatformId, ProjectId, SeekPage, tryCatch, UserId } from '@activepieces/core-utils'import { apDayjs, apDayjsDuration } from '@activepieces/server-utils'-import { CreateFlowRequest, Flow, FlowCreator, FlowOperationRequest, FlowOperationStatus, FlowOperationType, flowPieceUtil, FlowStatus, FlowTrigger, FlowTriggerType, FlowVersion, FlowVersionState, PopulatedFlow, requiredActionsUtil, SharedTemplate, TelemetryEventName, TemplateStatus, TemplateType, TriggerSource, UncategorizedFolderId, UserWithMetaInformation } from '@activepieces/shared'+import { CreateFlowRequest, Flow, FlowCreator, FlowOperationRequest, FlowOperationStatus, FlowOperationType, flowPieceUtil, FlowStatus, flowStructureUtil, FlowTrigger, FlowTriggerType, FlowVersion, FlowVersionState, PopulatedFlow, requiredActionsUtil, SharedTemplate, TelemetryEventName, TemplateStatus, TemplateType, TriggerSource, UncategorizedFolderId, UserWithMetaInformation } from '@activepieces/shared'import dayjs from 'dayjs'import { FastifyBaseLogger } from 'fastify'import { EntityManager, In, IsNull, Not } from 'typeorm'@@ -356,47 +356,18 @@ export const flowService = (log: FastifyBaseLogger) => ({switch (operation.type) {case FlowOperationType.LOCK_AND_PUBLISH: {- const flow = await this.getOneOrThrow({ id, projectId })- const flowVersionToPublish = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })- if (!skipRequiredActionsCheck) {- await assertRequiredActionsPresent({- projectId,- platformId,- flowVersion: flowVersionToPublish,- log,- })- }- const requestedStatus = operation.request.status ?? FlowStatus.ENABLED- if (requestedStatus === FlowStatus.ENABLED) {- assertTriggerIsSelected({ trigger: flowVersionToPublish.trigger })- await triggerUtils(log).getPieceTriggerOrThrow({ flowVersion: flowVersionToPublish, projectId })- }- const route = await publishHooksFactory.get(log).routePublish({ flow, projectId, platformId, userId })- if (route === 'NEEDS_APPROVAL') {- await publishHooksFactory.get(log).submitForApproval({- flow,- flowVersionToPublish,- userId,- projectId,- platformId,- requestedStatus,- })- break- }await withStatusLock({id,log,- fn: async () => {- const { isRepublish, isTriggerUnchanged } = await this.updatedPublishedVersionId({- id,- userId,- projectId,- platformId,- flowVersionToPublish,- requestedStatus,- })- await changeStatus({ id, projectId, newStatus: requestedStatus, isRepublish, isTriggerUnchanged }, log)- },+ fn: () => publish({+ id,+ projectId,+ platformId,+ userId,+ requestedStatus: operation.request.status ?? FlowStatus.ENABLED,+ skipRequiredActionsCheck,+ log,+ }),})break}@@ -461,28 +432,7 @@ export const flowService = (log: FastifyBaseLogger) => ({break}default: {- await withDraftLock({ flowId: id, log, fn: async () => {- const { version: lastVersion, createdNewDraft } = await createNewDraftIfVersionIsPublished({- flowId: id,- projectId,- platformId,- userId,- log,- })- const { error } = await tryCatch(() => flowVersionService(log).applyOperation({- userId,- projectId,- platformId,- flowVersion: lastVersion,- userOperation: operation,- }))- if (!isNil(error)) {- if (createdNewDraft) {- await tryCatch(() => flowVersionRepo().delete({ id: lastVersion.id, flowId: id }))- }- throw error- }- } })+ await withDraftLock({ flowId: id, log, fn: () => writeDraft({ id, projectId, platformId, userId, operation, log }) })}}@@ -509,6 +459,31 @@ export const flowService = (log: FastifyBaseLogger) => ({}return updatedFlow},++ async replaceConnection({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, auditExplanation }: ReplaceConnectionParams): Promise<void> {+ const flowBeforeReplace = await this.getOnePopulatedOrThrow({ id, projectId })+ const { appliedOperations, error } = await replaceConnectionForDraftAndPublishedVersions({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, log })+ if (appliedOperations.length > 0) {+ await flowSideEffects(log).onDraftWritten({ flowId: id, projectId, editorSessionId: null })+ appliedOperations.forEach(({ operation, flowAfterOperation }, index) => {+ const flowBeforeOperation = index === 0 ? flowBeforeReplace : appliedOperations[index - 1].flowAfterOperation+ flowSideEffects(log).onOperationApplied({+ platformId,+ projectId,+ userId,+ flow: flowAfterOperation,+ previousVersion: flowBeforeOperation.version,+ previousStatus: flowBeforeOperation.status,+ operation,+ auditExplanation,+ })+ })+ }+ if (!isNil(error)) {+ throw error+ }+ },+async updatedPublishedVersionId({id,userId,@@ -840,18 +815,144 @@ const lockFlowVersionIfNotLocked = async ({}-async function withDraftLock({ flowId, log, fn }: {+async function withDraftLock<T>({ flowId, log, fn }: {flowId: FlowIdlog: FastifyBaseLogger- fn: () => Promise<void>-}): Promise<void> {- await distributedLock(log).runExclusive({+ fn: () => Promise<T>+}): Promise<T> {+ return distributedLock(log).runExclusive({key: `flow-draft-update-${flowId}`,timeoutInSeconds: DRAFT_UPDATE_LOCK_TIMEOUT_SECONDS,fn,})}+async function writeDraft({ id, projectId, platformId, userId, operation, log }: WriteDraftParams): Promise<void> {+ const { version: lastVersion, createdNewDraft } = await createNewDraftIfVersionIsPublished({+ flowId: id,+ projectId,+ platformId,+ userId,+ log,+ })+ const { error } = await tryCatch(() => flowVersionService(log).applyOperation({+ userId,+ projectId,+ platformId,+ flowVersion: lastVersion,+ userOperation: operation,+ }))+ if (!isNil(error)) {+ if (createdNewDraft) {+ await tryCatch(() => flowVersionRepo().delete({ id: lastVersion.id, flowId: id }))+ }+ throw error+ }+}++async function publish({ id, projectId, platformId, userId, requestedStatus, skipRequiredActionsCheck, log }: PublishParams): Promise<void> {+ const flow = await flowService(log).getOneOrThrow({ id, projectId })+ const flowVersionToPublish = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })+ if (!skipRequiredActionsCheck) {+ await assertRequiredActionsPresent({+ projectId,+ platformId,+ flowVersion: flowVersionToPublish,+ log,+ })+ }+ if (requestedStatus === FlowStatus.ENABLED) {+ assertTriggerIsSelected({ trigger: flowVersionToPublish.trigger })+ await triggerUtils(log).getPieceTriggerOrThrow({ flowVersion: flowVersionToPublish, projectId })+ }+ const route = await publishHooksFactory.get(log).routePublish({ flow, projectId, platformId, userId })+ if (route === 'NEEDS_APPROVAL') {+ await publishHooksFactory.get(log).submitForApproval({+ flow,+ flowVersionToPublish,+ userId,+ projectId,+ platformId,+ requestedStatus,+ })+ return+ }+ const { isRepublish, isTriggerUnchanged } = await flowService(log).updatedPublishedVersionId({+ id,+ userId,+ projectId,+ platformId,+ flowVersionToPublish,+ requestedStatus,+ })+ await changeStatus({ id, projectId, newStatus: requestedStatus, isRepublish, isTriggerUnchanged }, log)+}++async function replaceConnectionForDraftAndPublishedVersions({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, log }: ReplaceConnectionForDraftAndPublishedVersionsParams): Promise<ReplaceConnectionResult> {+ return withDraftLock({+ flowId: id,+ log,+ fn: async () => {+ if (!applyToPublishedVersion) {+ return replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log })+ }+ return withStatusLock({+ id,+ log,+ fn: async () => {+ const flow = await flowService(log).getOneOrThrow({ id, projectId })+ const publishedVersionId = flow.publishedVersionId+ if (isNil(publishedVersionId)) {+ return replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log })+ }+ const latestVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })+ const publishedVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: publishedVersionId })+ const replaceInDraftOperation = createImportOperationToReplaceConnection({ flowVersion: latestVersion, sourceExternalId, targetExternalId })+ const replaceInPublishedVersionOperation = createImportOperationToReplaceConnection({ flowVersion: publishedVersion, sourceExternalId, targetExternalId })+ const appliedOperations: AppliedOperation[] = []+ const { error } = await tryCatch(async () => {+ await writeDraft({ id, projectId, platformId, userId, operation: replaceInPublishedVersionOperation, log })+ appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: replaceInPublishedVersionOperation, log }))+ await publish({ id, projectId, platformId, userId, requestedStatus: flow.status, skipRequiredActionsCheck: true, log })+ appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: { type: FlowOperationType.LOCK_AND_PUBLISH, request: { status: flow.status } }, log }))+ })+ const hadUnpublishedChanges = latestVersion.id !== publishedVersionId+ if (!hadUnpublishedChanges && isNil(error)) {+ return { appliedOperations, error }+ }+ const { error: restoreDraftError } = await tryCatch(async () => {+ await writeDraft({ id, projectId, platformId, userId, operation: replaceInDraftOperation, log })+ appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: replaceInDraftOperation, log }))+ })+ return { appliedOperations, error: error ?? restoreDraftError }+ },+ })+ },+ })+}++async function replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log }: ReplaceConnectionInDraftParams): Promise<ReplaceConnectionResult> {+ const latestVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })+ if (!latestVersion.connectionIds.includes(sourceExternalId)) {+ return { appliedOperations: [], error: null }+ }+ const replaceInDraftOperation = createImportOperationToReplaceConnection({ flowVersion: latestVersion, sourceExternalId, targetExternalId })+ await writeDraft({ id, projectId, platformId, userId, operation: replaceInDraftOperation, log })+ return { appliedOperations: [await recordAppliedOperation({ id, projectId, operation: replaceInDraftOperation, log })], error: null }+}++async function recordAppliedOperation({ id, projectId, operation, log }: RecordAppliedOperationParams): Promise<AppliedOperation> {+ const flowAfterOperation = await flowService(log).getOnePopulatedOrThrow({ id, projectId })+ return { operation, flowAfterOperation }+}++function createImportOperationToReplaceConnection({ flowVersion, sourceExternalId, targetExternalId }: CreateImportOperationToReplaceConnectionParams): FlowOperationRequest {+ return {+ type: FlowOperationType.IMPORT_FLOW,+ request: flowStructureUtil.replaceConnection({ flowVersion, sourceExternalId, targetExternalId }),+ }+}+async function applyStatusChange(params: ChangeStatusParams, log: FastifyBaseLogger): Promise<void> {await withStatusLock({ id: params.id, log, fn: () => changeStatus(params, log) })}@@ -1026,6 +1127,65 @@ type UpdateParams = EventEmissionParams & {auditExplanation?: string}+type ReplaceConnectionParams = {+ id: FlowId+ projectId: ProjectId+ platformId: PlatformId+ userId: UserId+ sourceExternalId: string+ targetExternalId: string+ applyToPublishedVersion: boolean+ auditExplanation: string+}++type ReplaceConnectionForDraftAndPublishedVersionsParams = Omit<ReplaceConnectionParams, 'auditExplanation'> & {+ log: FastifyBaseLogger+}++type ReplaceConnectionInDraftParams = Omit<ReplaceConnectionForDraftAndPublishedVersionsParams, 'applyToPublishedVersion'>++type ReplaceConnectionResult = {+ appliedOperations: AppliedOperation[]+ error: Error | null+}++type AppliedOperation = {+ operation: FlowOperationRequest+ flowAfterOperation: PopulatedFlow+}++type RecordAppliedOperationParams = {+ id: FlowId+ projectId: ProjectId+ operation: FlowOperationRequest+ log: FastifyBaseLogger+}++type CreateImportOperationToReplaceConnectionParams = {+ flowVersion: FlowVersion+ sourceExternalId: string+ targetExternalId: string+}++type WriteDraftParams = {+ id: FlowId+ projectId: ProjectId+ platformId: PlatformId+ userId: UserId | null+ operation: FlowOperationRequest+ log: FastifyBaseLogger+}++type PublishParams = {+ id: FlowId+ projectId: ProjectId+ platformId: PlatformId+ userId: UserId | null+ requestedStatus: FlowStatus+ skipRequiredActionsCheck: boolean+ log: FastifyBaseLogger+}+type UpdatePublishedVersionIdParams = {id: FlowIduserId: UserId | nulldiff --git a/packages/tests-e2e/scenarios/ce/connections/replace-connection-in-200-flows.spec.ts b/packages/tests-e2e/scenarios/ce/connections/replace-connection-in-200-flows.spec.tsnew file mode 100644index 00000000000..48c6f846a29--- /dev/null+++ b/packages/tests-e2e/scenarios/ce/connections/replace-connection-in-200-flows.spec.ts@@ -0,0 +1,272 @@+import type { APIRequestContext, Page } from '@playwright/test';+import { expect, test } from '../../../fixtures';+import { E2eSession } from '../../../fixtures/api';+import { e2eFlows } from '../../../fixtures/flows';++test.describe('Replace connection at scale', () => {+ test.skip(!process.env.E2E_REPLACE_CONNECTION_LOAD, 'Set E2E_REPLACE_CONNECTION_LOAD=true to run this load test');++ test('replaces one Hugging Face connection in 200 published flows from the Connections page', async ({ page, request, session, flowCleanup, connectionCleanup }) => {+ test.setTimeout(LOAD_TEST_TIMEOUT_MS);+ const api = createApiClient({ request, session });++ const [webhookVersion, huggingFaceVersion] = await Promise.all([+ api.getPieceVersion(WEBHOOK_PIECE),+ api.getPieceVersion(HUGGING_FACE_PIECE),+ ]);+ const sourceConnection = await api.createHuggingFaceConnection(e2eFlows.uniqueName('HF source'));+ connectionCleanup.track(sourceConnection.id);+ const targetConnection = await api.createHuggingFaceConnection(e2eFlows.uniqueName('HF target'));+ connectionCleanup.track(targetConnection.id);++ const seedStartedAt = Date.now();+ await runWithConcurrency({+ count: FLOW_COUNT,+ concurrency: SEED_CONCURRENCY,+ task: (index) => api.createPublishedFlow({+ displayName: e2eFlows.uniqueName(`replace load #${index + 1}`),+ trigger: buildWebhookTrigger({+ webhookVersion,+ huggingFaceVersion,+ connectionExternalId: sourceConnection.externalId,+ }),+ onCreated: (flowId) => flowCleanup.track(flowId),+ }),+ });+ logTiming(`Created and published ${FLOW_COUNT} flows`, seedStartedAt);++ expect(await api.countFlowsUsingConnection({ externalId: sourceConnection.externalId, published: false })).toBe(FLOW_COUNT);+ expect(await api.countFlowsUsingConnection({ externalId: sourceConnection.externalId, published: true })).toBe(FLOW_COUNT);++ page.setDefaultTimeout(ACTION_TIMEOUT_MS);+ await page.goto(`/projects/${session.projectId}/connections`);+ await page.getByRole('button', { name: 'Replace', exact: true }).click();+ const dialog = page.getByRole('dialog');+ await pickOption({ page, trigger: dialog.getByRole('combobox').filter({ hasText: 'Select a piece' }), search: 'Hugging Face' });+ await pickOption({ page, trigger: dialog.getByRole('combobox').filter({ hasText: 'Choose connection to replace' }).first(), search: sourceConnection.displayName });+ await pickOption({ page, trigger: dialog.getByRole('combobox').filter({ hasText: 'Choose connection to replace with' }), search: targetConnection.displayName });+ const reviewStartedAt = Date.now();+ await dialog.getByRole('button', { name: 'Next' }).click();+ await expect(dialog.getByText(/A replace changes up to 200 flows\./)).toBeVisible({ timeout: REVIEW_STEP_TIMEOUT_MS });+ logTiming('Loaded the review step', reviewStartedAt);++ await dialog.getByRole('combobox').filter({ hasText: 'Draft only' }).click();+ await page.getByRole('option', { name: 'Draft and published' }).click();++ const replaceStartedAt = Date.now();+ const replaceResponse = page.waitForResponse(+ (response) => response.url().includes('/v1/app-connections/replace'),+ { timeout: REPLACE_TIMEOUT_MS },+ );+ await dialog.getByRole('button', { name: 'Replace', exact: true }).click();+ const response = await replaceResponse;+ logTiming(`Replace request finished with HTTP ${response.status()}`, replaceStartedAt);+ expect(response.status(), await response.text()).toBe(204);++ expect(await api.countFlowsUsingConnection({ externalId: sourceConnection.externalId, published: false })).toBe(0);+ expect(await api.countFlowsUsingConnection({ externalId: sourceConnection.externalId, published: true })).toBe(0);+ expect(await api.countFlowsUsingConnection({ externalId: targetConnection.externalId, published: false })).toBe(FLOW_COUNT);+ expect(await api.countFlowsUsingConnection({ externalId: targetConnection.externalId, published: true })).toBe(FLOW_COUNT);+ });+});++function createApiClient({ request, session }: { request: APIRequestContext; session: E2eSession }) {+ const { projectId } = session;+ const headers = { Authorization: `Bearer ${session.token}` };++ async function call<T>({ method, path, data, params }: CallParams): Promise<T> {+ const response = await request.fetch(`/api${path}`, { method, headers, data, params, timeout: REPLACE_TIMEOUT_MS });+ if (!response.ok()) {+ throw new Error(`${method} ${path} failed with ${response.status()}: ${await response.text()}`);+ }+ const body = await response.text();+ return (body ? JSON.parse(body) : undefined) as T;+ }++ return {+ async getPieceVersion(pieceName: string): Promise<string> {+ const piece = await call<{ version: string }>({ method: 'GET', path: `/v1/pieces/${encodeURIComponent(pieceName)}`, params: { projectId } });+ return piece.version;+ },+ async createHuggingFaceConnection(displayName: string): Promise<Connection> {+ return call<Connection>({+ method: 'POST',+ path: '/v1/app-connections',+ data: {+ externalId: `hf_${Math.random().toString(36).slice(2)}`,+ displayName,+ pieceName: HUGGING_FACE_PIECE,+ projectId,+ type: 'SECRET_TEXT',+ value: { type: 'SECRET_TEXT', secret_text: 'e2e-not-a-real-token' },+ },+ });+ },+ async createPublishedFlow({ displayName, trigger, onCreated }: CreatePublishedFlowParams): Promise<string> {+ const flow = await call<{ id: string; version: { schemaVersion: string | null } }>({+ method: 'POST',+ path: '/v1/flows',+ data: { displayName, projectId },+ });+ onCreated(flow.id);+ await call({+ method: 'POST',+ path: `/v1/flows/${flow.id}`,+ data: {+ type: 'IMPORT_FLOW',+ request: { displayName, trigger, schemaVersion: flow.version.schemaVersion, notes: null },+ },+ });+ await call({+ method: 'POST',+ path: `/v1/flows/${flow.id}`,+ data: { type: 'LOCK_AND_PUBLISH', request: {} },+ });+ return flow.id;+ },+ async countFlowsUsingConnection({ externalId, published }: { externalId: string; published: boolean }): Promise<number> {+ let count = 0;+ let cursor: string | undefined = undefined;+ do {+ const page: { data: unknown[]; next: string | null } = await call({+ method: 'GET',+ path: '/v1/flows',+ params: {+ projectId,+ connectionExternalIds: externalId,+ limit: 1000,+ ...(published ? { versionState: 'LOCKED' } : {}),+ ...(cursor ? { cursor } : {}),+ },+ });+ count += page.data.length;+ cursor = page.next ?? undefined;+ } while (cursor);+ return count;+ },+ };+}++function buildWebhookTrigger({ webhookVersion, huggingFaceVersion, connectionExternalId }: BuildWebhookTriggerParams) {+ const actions = Array.from({ length: STEPS_PER_FLOW }, (_, index) => ({+ name: `step_${index + 1}`,+ type: 'PIECE',+ valid: true,+ displayName: `Text Classification ${index + 1}`,+ lastUpdatedDate: new Date().toISOString(),+ settings: {+ pieceName: HUGGING_FACE_PIECE,+ pieceVersion: huggingFaceVersion,+ actionName: 'text_classification',+ input: {+ auth: `{{connections['${connectionExternalId}']}}`,+ classificationMode: 'zero-shot',+ text: 'The replace connection load test sends this text.',+ },+ propertySettings: {},+ errorHandlingOptions: {+ continueOnFailure: { value: false },+ retryOnFailure: { value: false },+ },+ },+ }));+ const firstAction = actions.reduceRight<Record<string, unknown> | undefined>(+ (nextAction, action) => ({ ...action, ...(nextAction ? { nextAction } : {}) }),+ undefined,+ );+ return {+ name: 'trigger',+ type: 'PIECE_TRIGGER',+ valid: true,+ displayName: 'Catch Webhook',+ lastUpdatedDate: new Date().toISOString(),+ settings: {+ pieceName: WEBHOOK_PIECE,+ pieceVersion: webhookVersion,+ triggerName: 'catch_webhook',+ input: { authType: 'none' },+ propertySettings: {},+ },+ nextAction: firstAction,+ };+}++async function pickOption({ page, trigger, search }: PickOptionParams): Promise<void> {+ await trigger.click();+ await page.keyboard.type(search);+ await page.getByRole('option', { name: search }).first().click();+}++async function runWithConcurrency<T>({ count, concurrency, task }: RunWithConcurrencyParams<T>): Promise<T[]> {+ const results: T[] = new Array(count);+ const failures: unknown[] = [];+ let nextIndex = 0;+ async function worker(): Promise<void> {+ while (nextIndex < count && failures.length === 0) {+ const index = nextIndex++;+ try {+ results[index] = await task(index);+ } catch (error) {+ failures.push(error);+ }+ }+ }+ await Promise.all(Array.from({ length: Math.min(concurrency, count) }, () => worker()));+ if (failures.length > 0) {+ throw failures[0];+ }+ return results;+}++function logTiming(label: string, startedAt: number): void {+ const seconds = ((Date.now() - startedAt) / 1000).toFixed(1);+ console.log(`⏱ ${label} in ${seconds}s`);+ test.info().annotations.push({ type: 'timing', description: `${label}: ${seconds}s` });+}++const FLOW_COUNT = 200;+const STEPS_PER_FLOW = 10;+const SEED_CONCURRENCY = 10;+const WEBHOOK_PIECE = '@activepieces/piece-webhook';+const HUGGING_FACE_PIECE = '@activepieces/piece-hugging-face';+const LOAD_TEST_TIMEOUT_MS = 60 * 60 * 1000;+const ACTION_TIMEOUT_MS = 30 * 1000;+const REVIEW_STEP_TIMEOUT_MS = 5 * 60 * 1000;+const REPLACE_TIMEOUT_MS = 30 * 60 * 1000;++type Connection = {+ id: string;+ externalId: string;+ displayName: string;+};++type CreatePublishedFlowParams = {+ displayName: string;+ trigger: unknown;+ onCreated: (flowId: string) => void;+};++type CallParams = {+ method: 'GET' | 'POST' | 'DELETE';+ path: string;+ data?: unknown;+ params?: Record<string, string | number>;+};++type BuildWebhookTriggerParams = {+ webhookVersion: string;+ huggingFaceVersion: string;+ connectionExternalId: string;+};++type PickOptionParams = {+ page: Page;+ trigger: ReturnType<Page['getByRole']>;+ search: string;+};++type RunWithConcurrencyParams<T> = {+ count: number;+ concurrency: number;+ task: (index: number) => Promise<T>;+};