Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(connections): hold the flow locks for the whole connection replace (#16972)

Abdul22a1c134e2d3
diff --git a/packages/server/api/src/app/app-connection/app-connection-service/app-connection.handler.ts b/packages/server/api/src/app/app-connection/app-connection-service/app-connection.handler.ts
index cb0758667f6..22d2e4e917d 100644
--- a/packages/server/api/src/app/app-connection/app-connection-service/app-connection.handler.ts
+++ b/packages/server/api/src/app/app-connection/app-connection-service/app-connection.handler.ts
@@ -1,13 +1,13 @@
import { ActivepiecesError, ErrorCode, FlowId, isNil, PlatformId, ProjectId, tryCatch, unique, UserId } from '@activepieces/core-utils'
import { PropertyType } from '@activepieces/pieces-framework'
-import { AppConnection, AppConnectionStatus, AppConnectionType, AppConnectionValue, AppConnectionWithoutSensitiveData, EngineResponse, EngineResponseStatus, ExecuteRefreshTokenAuthResponse, Flow, FlowOperationType, flowStructureUtil, FlowVersion, PopulatedFlow, WorkerJobType } from '@activepieces/shared'
+import { AppConnection, AppConnectionStatus, AppConnectionType, AppConnectionValue, AppConnectionWithoutSensitiveData, EngineResponse, EngineResponseStatus, ExecuteRefreshTokenAuthResponse, PopulatedFlow, WorkerJobType } from '@activepieces/shared'
import dayjs from 'dayjs'
import { FastifyBaseLogger } from 'fastify'
import { lru, LRU } from 'tiny-lru'
import { ArrayContains } from 'typeorm'
import { distributedLock } from '../../database/redis-connections'
import { flowService } from '../../flows/flow/flow.service'
-import { flowVersionRepo, flowVersionService } from '../../flows/flow-version/flow-version.service'
+import { flowVersionRepo } from '../../flows/flow-version/flow-version.service'
import { encryptUtils } from '../../helper/encryption'
import { exceptionHandler } from '../../helper/exception-handler'
import { getPiecePackageWithoutArchive, pieceMetadataService } from '../../pieces/metadata/piece-metadata-service'
@@ -24,12 +24,16 @@ export const appConnectionHandler = (log: FastifyBaseLogger) => ({
: new Set<FlowId>()
await Promise.all(flows.map(async (flow) => {
- const replaceParams = { flow, latestVersion: flow.version, userId, platformId, appConnection, newAppConnection, log }
- if (flowIdsToRepublish.has(flow.id)) {
- await replaceConnectionInPublishedAndDraft(replaceParams)
- return
- }
- await replaceConnectionInDraft(replaceParams)
+ await flowService(log).replaceConnection({
+ id: flow.id,
+ projectId: flow.projectId,
+ platformId,
+ userId,
+ sourceExternalId: appConnection.externalId,
+ targetExternalId: newAppConnection.externalId,
+ applyToPublishedVersion: flowIdsToRepublish.has(flow.id),
+ auditExplanation: `replaced connection "${appConnection.displayName}" with "${newAppConnection.displayName}".`,
+ })
}))
},
@@ -331,92 +335,6 @@ async function findFlowIdsToRepublish({ flows, externalId }: FindFlowIdsToRepubl
return new Set(rows.map((row) => row.flowId))
}
-async function replaceConnectionInPublishedAndDraft(params: ReplaceConnectionInPublishedAndDraftParams): Promise<void> {
- const { flow, latestVersion, userId, platformId, appConnection, newAppConnection, log } = params
- const projectId = flow.projectId
- const publishedVersionId = flow.publishedVersionId
- if (isNil(publishedVersionId)) {
- return
- }
-
- const { error } = await tryCatch(async () => {
- const publishedVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: flow.id, versionId: publishedVersionId })
-
- await flowService(log).update({
- id: flow.id,
- projectId,
- platformId,
- userId,
- previousFlow: flow,
- operation: {
- type: FlowOperationType.IMPORT_FLOW,
- request: replaceConnectionInFlowVersion({ flowVersion: publishedVersion, appConnection, newAppConnection }),
- },
- auditExplanation: replaceConnectionAuditExplanation({ appConnection, newAppConnection }),
- })
-
- const { status } = await flowService(log).getOneOrThrow({ id: flow.id, projectId })
- await flowService(log).update({
- id: flow.id,
- projectId,
- platformId,
- userId,
- operation: {
- type: FlowOperationType.LOCK_AND_PUBLISH,
- request: { status },
- },
- skipRequiredActionsCheck: true,
- })
- })
-
- const hadUnpublishedChanges = latestVersion.id !== publishedVersionId
- const republishFailed = !isNil(error)
- if (hadUnpublishedChanges || republishFailed) {
- await replaceConnectionInDraft({ ...params, restoreDraft: true })
- }
- if (republishFailed) {
- throw error
- }
-}
-
-async function replaceConnectionInDraft({ flow, latestVersion, userId, platformId, appConnection, newAppConnection, restoreDraft = false, log }: ReplaceConnectionInDraftParams): Promise<void> {
- const usesConnection = latestVersion.connectionIds.includes(appConnection.externalId)
- || flowStructureUtil.extractConnectionIds(latestVersion).includes(appConnection.externalId)
- if (!usesConnection && !restoreDraft) {
- return
- }
-
- await flowService(log).update({
- id: flow.id,
- projectId: flow.projectId,
- platformId,
- userId,
- operation: {
- type: FlowOperationType.IMPORT_FLOW,
- request: replaceConnectionInFlowVersion({ flowVersion: latestVersion, appConnection, newAppConnection }),
- },
- auditExplanation: replaceConnectionAuditExplanation({ appConnection, newAppConnection }),
- })
-}
-
-function replaceConnectionInFlowVersion({ flowVersion, appConnection, newAppConnection }: ReplaceConnectionInFlowVersionParams): FlowVersion {
- return flowStructureUtil.replaceConnection({
- flowVersion,
- sourceExternalId: appConnection.externalId,
- targetExternalId: newAppConnection.externalId,
- })
-}
-
-function replaceConnectionAuditExplanation({ appConnection, newAppConnection }: Omit<ReplaceConnectionInFlowVersionParams, 'flowVersion'>): string {
- return `replaced connection "${appConnection.displayName}" with "${newAppConnection.displayName}".`
-}
-
-type ReplaceConnectionInFlowVersionParams = {
- flowVersion: FlowVersion
- appConnection: AppConnectionWithoutSensitiveData
- newAppConnection: AppConnectionWithoutSensitiveData
-}
-
type ReplaceConnectionInFlowsParams = {
flows: PopulatedFlow[]
appConnection: AppConnectionWithoutSensitiveData
@@ -431,17 +349,3 @@ type FindFlowIdsToRepublishParams = {
externalId: string
}
-type ReplaceConnectionInDraftParams = {
- flow: Flow
- latestVersion: FlowVersion
- userId: UserId
- platformId: PlatformId
- appConnection: AppConnectionWithoutSensitiveData
- newAppConnection: AppConnectionWithoutSensitiveData
- restoreDraft?: boolean
- log: FastifyBaseLogger
-}
-
-type ReplaceConnectionInPublishedAndDraftParams = ReplaceConnectionInDraftParams & {
- flow: PopulatedFlow
-}
diff --git a/packages/server/api/src/app/flows/flow/flow.service.ts b/packages/server/api/src/app/flows/flow/flow.service.ts
index 10e45396a93..ac7ace3064b 100644
--- a/packages/server/api/src/app/flows/flow/flow.service.ts
+++ b/packages/server/api/src/app/flows/flow/flow.service.ts
@@ -1,6 +1,6 @@
import { ActivepiecesError, apId, Cursor, ErrorCode, FlowId, FlowVersionId, isNil, Metadata, PlatformId, ProjectId, SeekPage, tryCatch, UserId } from '@activepieces/core-utils'
import { apDayjs, apDayjsDuration } from '@activepieces/server-utils'
-import { CreateFlowRequest, Flow, FlowCreator, FlowOperationRequest, FlowOperationStatus, FlowOperationType, flowPieceUtil, FlowStatus, FlowTrigger, FlowTriggerType, FlowVersion, FlowVersionState, PopulatedFlow, requiredActionsUtil, SharedTemplate, TelemetryEventName, TemplateStatus, TemplateType, TriggerSource, UncategorizedFolderId, UserWithMetaInformation } from '@activepieces/shared'
+import { CreateFlowRequest, Flow, FlowCreator, FlowOperationRequest, FlowOperationStatus, FlowOperationType, flowPieceUtil, FlowStatus, flowStructureUtil, FlowTrigger, FlowTriggerType, FlowVersion, FlowVersionState, PopulatedFlow, requiredActionsUtil, SharedTemplate, TelemetryEventName, TemplateStatus, TemplateType, TriggerSource, UncategorizedFolderId, UserWithMetaInformation } from '@activepieces/shared'
import dayjs from 'dayjs'
import { FastifyBaseLogger } from 'fastify'
import { EntityManager, In, IsNull, Not } from 'typeorm'
@@ -356,47 +356,18 @@ export const flowService = (log: FastifyBaseLogger) => ({
switch (operation.type) {
case FlowOperationType.LOCK_AND_PUBLISH: {
- const flow = await this.getOneOrThrow({ id, projectId })
- const flowVersionToPublish = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })
- if (!skipRequiredActionsCheck) {
- await assertRequiredActionsPresent({
- projectId,
- platformId,
- flowVersion: flowVersionToPublish,
- log,
- })
- }
- const requestedStatus = operation.request.status ?? FlowStatus.ENABLED
- if (requestedStatus === FlowStatus.ENABLED) {
- assertTriggerIsSelected({ trigger: flowVersionToPublish.trigger })
- await triggerUtils(log).getPieceTriggerOrThrow({ flowVersion: flowVersionToPublish, projectId })
- }
- const route = await publishHooksFactory.get(log).routePublish({ flow, projectId, platformId, userId })
- if (route === 'NEEDS_APPROVAL') {
- await publishHooksFactory.get(log).submitForApproval({
- flow,
- flowVersionToPublish,
- userId,
- projectId,
- platformId,
- requestedStatus,
- })
- break
- }
await withStatusLock({
id,
log,
- fn: async () => {
- const { isRepublish, isTriggerUnchanged } = await this.updatedPublishedVersionId({
- id,
- userId,
- projectId,
- platformId,
- flowVersionToPublish,
- requestedStatus,
- })
- await changeStatus({ id, projectId, newStatus: requestedStatus, isRepublish, isTriggerUnchanged }, log)
- },
+ fn: () => publish({
+ id,
+ projectId,
+ platformId,
+ userId,
+ requestedStatus: operation.request.status ?? FlowStatus.ENABLED,
+ skipRequiredActionsCheck,
+ log,
+ }),
})
break
}
@@ -461,28 +432,7 @@ export const flowService = (log: FastifyBaseLogger) => ({
break
}
default: {
- await withDraftLock({ flowId: id, log, fn: async () => {
- const { version: lastVersion, createdNewDraft } = await createNewDraftIfVersionIsPublished({
- flowId: id,
- projectId,
- platformId,
- userId,
- log,
- })
- const { error } = await tryCatch(() => flowVersionService(log).applyOperation({
- userId,
- projectId,
- platformId,
- flowVersion: lastVersion,
- userOperation: operation,
- }))
- if (!isNil(error)) {
- if (createdNewDraft) {
- await tryCatch(() => flowVersionRepo().delete({ id: lastVersion.id, flowId: id }))
- }
- throw error
- }
- } })
+ await withDraftLock({ flowId: id, log, fn: () => writeDraft({ id, projectId, platformId, userId, operation, log }) })
}
}
@@ -509,6 +459,31 @@ export const flowService = (log: FastifyBaseLogger) => ({
}
return updatedFlow
},
+
+ async replaceConnection({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, auditExplanation }: ReplaceConnectionParams): Promise<void> {
+ const flowBeforeReplace = await this.getOnePopulatedOrThrow({ id, projectId })
+ const { appliedOperations, error } = await replaceConnectionForDraftAndPublishedVersions({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, log })
+ if (appliedOperations.length > 0) {
+ await flowSideEffects(log).onDraftWritten({ flowId: id, projectId, editorSessionId: null })
+ appliedOperations.forEach(({ operation, flowAfterOperation }, index) => {
+ const flowBeforeOperation = index === 0 ? flowBeforeReplace : appliedOperations[index - 1].flowAfterOperation
+ flowSideEffects(log).onOperationApplied({
+ platformId,
+ projectId,
+ userId,
+ flow: flowAfterOperation,
+ previousVersion: flowBeforeOperation.version,
+ previousStatus: flowBeforeOperation.status,
+ operation,
+ auditExplanation,
+ })
+ })
+ }
+ if (!isNil(error)) {
+ throw error
+ }
+ },
+
async updatedPublishedVersionId({
id,
userId,
@@ -840,18 +815,144 @@ const lockFlowVersionIfNotLocked = async ({
}
-async function withDraftLock({ flowId, log, fn }: {
+async function withDraftLock<T>({ flowId, log, fn }: {
flowId: FlowId
log: FastifyBaseLogger
- fn: () => Promise<void>
-}): Promise<void> {
- await distributedLock(log).runExclusive({
+ fn: () => Promise<T>
+}): Promise<T> {
+ return distributedLock(log).runExclusive({
key: `flow-draft-update-${flowId}`,
timeoutInSeconds: DRAFT_UPDATE_LOCK_TIMEOUT_SECONDS,
fn,
})
}
+async function writeDraft({ id, projectId, platformId, userId, operation, log }: WriteDraftParams): Promise<void> {
+ const { version: lastVersion, createdNewDraft } = await createNewDraftIfVersionIsPublished({
+ flowId: id,
+ projectId,
+ platformId,
+ userId,
+ log,
+ })
+ const { error } = await tryCatch(() => flowVersionService(log).applyOperation({
+ userId,
+ projectId,
+ platformId,
+ flowVersion: lastVersion,
+ userOperation: operation,
+ }))
+ if (!isNil(error)) {
+ if (createdNewDraft) {
+ await tryCatch(() => flowVersionRepo().delete({ id: lastVersion.id, flowId: id }))
+ }
+ throw error
+ }
+}
+
+async function publish({ id, projectId, platformId, userId, requestedStatus, skipRequiredActionsCheck, log }: PublishParams): Promise<void> {
+ const flow = await flowService(log).getOneOrThrow({ id, projectId })
+ const flowVersionToPublish = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })
+ if (!skipRequiredActionsCheck) {
+ await assertRequiredActionsPresent({
+ projectId,
+ platformId,
+ flowVersion: flowVersionToPublish,
+ log,
+ })
+ }
+ if (requestedStatus === FlowStatus.ENABLED) {
+ assertTriggerIsSelected({ trigger: flowVersionToPublish.trigger })
+ await triggerUtils(log).getPieceTriggerOrThrow({ flowVersion: flowVersionToPublish, projectId })
+ }
+ const route = await publishHooksFactory.get(log).routePublish({ flow, projectId, platformId, userId })
+ if (route === 'NEEDS_APPROVAL') {
+ await publishHooksFactory.get(log).submitForApproval({
+ flow,
+ flowVersionToPublish,
+ userId,
+ projectId,
+ platformId,
+ requestedStatus,
+ })
+ return
+ }
+ const { isRepublish, isTriggerUnchanged } = await flowService(log).updatedPublishedVersionId({
+ id,
+ userId,
+ projectId,
+ platformId,
+ flowVersionToPublish,
+ requestedStatus,
+ })
+ await changeStatus({ id, projectId, newStatus: requestedStatus, isRepublish, isTriggerUnchanged }, log)
+}
+
+async function replaceConnectionForDraftAndPublishedVersions({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, applyToPublishedVersion, log }: ReplaceConnectionForDraftAndPublishedVersionsParams): Promise<ReplaceConnectionResult> {
+ return withDraftLock({
+ flowId: id,
+ log,
+ fn: async () => {
+ if (!applyToPublishedVersion) {
+ return replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log })
+ }
+ return withStatusLock({
+ id,
+ log,
+ fn: async () => {
+ const flow = await flowService(log).getOneOrThrow({ id, projectId })
+ const publishedVersionId = flow.publishedVersionId
+ if (isNil(publishedVersionId)) {
+ return replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log })
+ }
+ const latestVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })
+ const publishedVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: publishedVersionId })
+ const replaceInDraftOperation = createImportOperationToReplaceConnection({ flowVersion: latestVersion, sourceExternalId, targetExternalId })
+ const replaceInPublishedVersionOperation = createImportOperationToReplaceConnection({ flowVersion: publishedVersion, sourceExternalId, targetExternalId })
+ const appliedOperations: AppliedOperation[] = []
+ const { error } = await tryCatch(async () => {
+ await writeDraft({ id, projectId, platformId, userId, operation: replaceInPublishedVersionOperation, log })
+ appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: replaceInPublishedVersionOperation, log }))
+ await publish({ id, projectId, platformId, userId, requestedStatus: flow.status, skipRequiredActionsCheck: true, log })
+ appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: { type: FlowOperationType.LOCK_AND_PUBLISH, request: { status: flow.status } }, log }))
+ })
+ const hadUnpublishedChanges = latestVersion.id !== publishedVersionId
+ if (!hadUnpublishedChanges && isNil(error)) {
+ return { appliedOperations, error }
+ }
+ const { error: restoreDraftError } = await tryCatch(async () => {
+ await writeDraft({ id, projectId, platformId, userId, operation: replaceInDraftOperation, log })
+ appliedOperations.push(await recordAppliedOperation({ id, projectId, operation: replaceInDraftOperation, log }))
+ })
+ return { appliedOperations, error: error ?? restoreDraftError }
+ },
+ })
+ },
+ })
+}
+
+async function replaceConnectionInDraft({ id, projectId, platformId, userId, sourceExternalId, targetExternalId, log }: ReplaceConnectionInDraftParams): Promise<ReplaceConnectionResult> {
+ const latestVersion = await flowVersionService(log).getFlowVersionOrThrow({ flowId: id, versionId: undefined })
+ if (!latestVersion.connectionIds.includes(sourceExternalId)) {
+ return { appliedOperations: [], error: null }
+ }
+ const replaceInDraftOperation = createImportOperationToReplaceConnection({ flowVersion: latestVersion, sourceExternalId, targetExternalId })
+ await writeDraft({ id, projectId, platformId, userId, operation: replaceInDraftOperation, log })
+ return { appliedOperations: [await recordAppliedOperation({ id, projectId, operation: replaceInDraftOperation, log })], error: null }
+}
+
+async function recordAppliedOperation({ id, projectId, operation, log }: RecordAppliedOperationParams): Promise<AppliedOperation> {
+ const flowAfterOperation = await flowService(log).getOnePopulatedOrThrow({ id, projectId })
+ return { operation, flowAfterOperation }
+}
+
+function createImportOperationToReplaceConnection({ flowVersion, sourceExternalId, targetExternalId }: CreateImportOperationToReplaceConnectionParams): FlowOperationRequest {
+ return {
+ type: FlowOperationType.IMPORT_FLOW,
+ request: flowStructureUtil.replaceConnection({ flowVersion, sourceExternalId, targetExternalId }),
+ }
+}
+
async function applyStatusChange(params: ChangeStatusParams, log: FastifyBaseLogger): Promise<void> {
await withStatusLock({ id: params.id, log, fn: () => changeStatus(params, log) })
}
@@ -1026,6 +1127,65 @@ type UpdateParams = EventEmissionParams & {
auditExplanation?: string
}
+type ReplaceConnectionParams = {
+ id: FlowId
+ projectId: ProjectId
+ platformId: PlatformId
+ userId: UserId
+ sourceExternalId: string
+ targetExternalId: string
+ applyToPublishedVersion: boolean
+ auditExplanation: string
+}
+
+type ReplaceConnectionForDraftAndPublishedVersionsParams = Omit<ReplaceConnectionParams, 'auditExplanation'> & {
+ log: FastifyBaseLogger
+}
+
+type ReplaceConnectionInDraftParams = Omit<ReplaceConnectionForDraftAndPublishedVersionsParams, 'applyToPublishedVersion'>
+
+type ReplaceConnectionResult = {
+ appliedOperations: AppliedOperation[]
+ error: Error | null
+}
+
+type AppliedOperation = {
+ operation: FlowOperationRequest
+ flowAfterOperation: PopulatedFlow
+}
+
+type RecordAppliedOperationParams = {
+ id: FlowId
+ projectId: ProjectId
+ operation: FlowOperationRequest
+ log: FastifyBaseLogger
+}
+
+type CreateImportOperationToReplaceConnectionParams = {
+ flowVersion: FlowVersion
+ sourceExternalId: string
+ targetExternalId: string
+}
+
+type WriteDraftParams = {
+ id: FlowId
+ projectId: ProjectId
+ platformId: PlatformId
+ userId: UserId | null
+ operation: FlowOperationRequest
+ log: FastifyBaseLogger
+}
+
+type PublishParams = {
+ id: FlowId
+ projectId: ProjectId
+ platformId: PlatformId
+ userId: UserId | null
+ requestedStatus: FlowStatus
+ skipRequiredActionsCheck: boolean
+ log: FastifyBaseLogger
+}
+
type UpdatePublishedVersionIdParams = {
id: FlowId
userId: UserId | null
diff --git a/packages/tests-e2e/scenarios/ce/connections/replace-connection-in-200-flows.spec.ts b/packages/tests-e2e/scenarios/ce/connections/replace-connection-in-200-flows.spec.ts
new file mode 100644
index 00000000000..48c6f846a29
--- /dev/null
+++ b/packages/tests-e2e/scenarios/ce/connections/replace-connection-in-200-flows.spec.ts
@@ -0,0 +1,272 @@
+import type { APIRequestContext, Page } from '@playwright/test';
+import { expect, test } from '../../../fixtures';
+import { E2eSession } from '../../../fixtures/api';
+import { e2eFlows } from '../../../fixtures/flows';
+
+test.describe('Replace connection at scale', () => {
+ test.skip(!process.env.E2E_REPLACE_CONNECTION_LOAD, 'Set E2E_REPLACE_CONNECTION_LOAD=true to run this load test');
+
+ test('replaces one Hugging Face connection in 200 published flows from the Connections page', async ({ page, request, session, flowCleanup, connectionCleanup }) => {
+ test.setTimeout(LOAD_TEST_TIMEOUT_MS);
+ const api = createApiClient({ request, session });
+
+ const [webhookVersion, huggingFaceVersion] = await Promise.all([
+ api.getPieceVersion(WEBHOOK_PIECE),
+ api.getPieceVersion(HUGGING_FACE_PIECE),
+ ]);
+ const sourceConnection = await api.createHuggingFaceConnection(e2eFlows.uniqueName('HF source'));
+ connectionCleanup.track(sourceConnection.id);
+ const targetConnection = await api.createHuggingFaceConnection(e2eFlows.uniqueName('HF target'));
+ connectionCleanup.track(targetConnection.id);
+
+ const seedStartedAt = Date.now();
+ await runWithConcurrency({
+ count: FLOW_COUNT,
+ concurrency: SEED_CONCURRENCY,
+ task: (index) => api.createPublishedFlow({
+ displayName: e2eFlows.uniqueName(`replace load #${index + 1}`),
+ trigger: buildWebhookTrigger({
+ webhookVersion,
+ huggingFaceVersion,
+ connectionExternalId: sourceConnection.externalId,
+ }),
+ onCreated: (flowId) => flowCleanup.track(flowId),
+ }),
+ });
+ logTiming(`Created and published ${FLOW_COUNT} flows`, seedStartedAt);
+
+ expect(await api.countFlowsUsingConnection({ externalId: sourceConnection.externalId, published: false })).toBe(FLOW_COUNT);
+ expect(await api.countFlowsUsingConnection({ externalId: sourceConnection.externalId, published: true })).toBe(FLOW_COUNT);
+
+ page.setDefaultTimeout(ACTION_TIMEOUT_MS);
+ await page.goto(`/projects/${session.projectId}/connections`);
+ await page.getByRole('button', { name: 'Replace', exact: true }).click();
+ const dialog = page.getByRole('dialog');
+ await pickOption({ page, trigger: dialog.getByRole('combobox').filter({ hasText: 'Select a piece' }), search: 'Hugging Face' });
+ await pickOption({ page, trigger: dialog.getByRole('combobox').filter({ hasText: 'Choose connection to replace' }).first(), search: sourceConnection.displayName });
+ await pickOption({ page, trigger: dialog.getByRole('combobox').filter({ hasText: 'Choose connection to replace with' }), search: targetConnection.displayName });
+ const reviewStartedAt = Date.now();
+ await dialog.getByRole('button', { name: 'Next' }).click();
+ await expect(dialog.getByText(/A replace changes up to 200 flows\./)).toBeVisible({ timeout: REVIEW_STEP_TIMEOUT_MS });
+ logTiming('Loaded the review step', reviewStartedAt);
+
+ await dialog.getByRole('combobox').filter({ hasText: 'Draft only' }).click();
+ await page.getByRole('option', { name: 'Draft and published' }).click();
+
+ const replaceStartedAt = Date.now();
+ const replaceResponse = page.waitForResponse(
+ (response) => response.url().includes('/v1/app-connections/replace'),
+ { timeout: REPLACE_TIMEOUT_MS },
+ );
+ await dialog.getByRole('button', { name: 'Replace', exact: true }).click();
+ const response = await replaceResponse;
+ logTiming(`Replace request finished with HTTP ${response.status()}`, replaceStartedAt);
+ expect(response.status(), await response.text()).toBe(204);
+
+ expect(await api.countFlowsUsingConnection({ externalId: sourceConnection.externalId, published: false })).toBe(0);
+ expect(await api.countFlowsUsingConnection({ externalId: sourceConnection.externalId, published: true })).toBe(0);
+ expect(await api.countFlowsUsingConnection({ externalId: targetConnection.externalId, published: false })).toBe(FLOW_COUNT);
+ expect(await api.countFlowsUsingConnection({ externalId: targetConnection.externalId, published: true })).toBe(FLOW_COUNT);
+ });
+});
+
+function createApiClient({ request, session }: { request: APIRequestContext; session: E2eSession }) {
+ const { projectId } = session;
+ const headers = { Authorization: `Bearer ${session.token}` };
+
+ async function call<T>({ method, path, data, params }: CallParams): Promise<T> {
+ const response = await request.fetch(`/api${path}`, { method, headers, data, params, timeout: REPLACE_TIMEOUT_MS });
+ if (!response.ok()) {
+ throw new Error(`${method} ${path} failed with ${response.status()}: ${await response.text()}`);
+ }
+ const body = await response.text();
+ return (body ? JSON.parse(body) : undefined) as T;
+ }
+
+ return {
+ async getPieceVersion(pieceName: string): Promise<string> {
+ const piece = await call<{ version: string }>({ method: 'GET', path: `/v1/pieces/${encodeURIComponent(pieceName)}`, params: { projectId } });
+ return piece.version;
+ },
+ async createHuggingFaceConnection(displayName: string): Promise<Connection> {
+ return call<Connection>({
+ method: 'POST',
+ path: '/v1/app-connections',
+ data: {
+ externalId: `hf_${Math.random().toString(36).slice(2)}`,
+ displayName,
+ pieceName: HUGGING_FACE_PIECE,
+ projectId,
+ type: 'SECRET_TEXT',
+ value: { type: 'SECRET_TEXT', secret_text: 'e2e-not-a-real-token' },
+ },
+ });
+ },
+ async createPublishedFlow({ displayName, trigger, onCreated }: CreatePublishedFlowParams): Promise<string> {
+ const flow = await call<{ id: string; version: { schemaVersion: string | null } }>({
+ method: 'POST',
+ path: '/v1/flows',
+ data: { displayName, projectId },
+ });
+ onCreated(flow.id);
+ await call({
+ method: 'POST',
+ path: `/v1/flows/${flow.id}`,
+ data: {
+ type: 'IMPORT_FLOW',
+ request: { displayName, trigger, schemaVersion: flow.version.schemaVersion, notes: null },
+ },
+ });
+ await call({
+ method: 'POST',
+ path: `/v1/flows/${flow.id}`,
+ data: { type: 'LOCK_AND_PUBLISH', request: {} },
+ });
+ return flow.id;
+ },
+ async countFlowsUsingConnection({ externalId, published }: { externalId: string; published: boolean }): Promise<number> {
+ let count = 0;
+ let cursor: string | undefined = undefined;
+ do {
+ const page: { data: unknown[]; next: string | null } = await call({
+ method: 'GET',
+ path: '/v1/flows',
+ params: {
+ projectId,
+ connectionExternalIds: externalId,
+ limit: 1000,
+ ...(published ? { versionState: 'LOCKED' } : {}),
+ ...(cursor ? { cursor } : {}),
+ },
+ });
+ count += page.data.length;
+ cursor = page.next ?? undefined;
+ } while (cursor);
+ return count;
+ },
+ };
+}
+
+function buildWebhookTrigger({ webhookVersion, huggingFaceVersion, connectionExternalId }: BuildWebhookTriggerParams) {
+ const actions = Array.from({ length: STEPS_PER_FLOW }, (_, index) => ({
+ name: `step_${index + 1}`,
+ type: 'PIECE',
+ valid: true,
+ displayName: `Text Classification ${index + 1}`,
+ lastUpdatedDate: new Date().toISOString(),
+ settings: {
+ pieceName: HUGGING_FACE_PIECE,
+ pieceVersion: huggingFaceVersion,
+ actionName: 'text_classification',
+ input: {
+ auth: `{{connections['${connectionExternalId}']}}`,
+ classificationMode: 'zero-shot',
+ text: 'The replace connection load test sends this text.',
+ },
+ propertySettings: {},
+ errorHandlingOptions: {
+ continueOnFailure: { value: false },
+ retryOnFailure: { value: false },
+ },
+ },
+ }));
+ const firstAction = actions.reduceRight<Record<string, unknown> | undefined>(
+ (nextAction, action) => ({ ...action, ...(nextAction ? { nextAction } : {}) }),
+ undefined,
+ );
+ return {
+ name: 'trigger',
+ type: 'PIECE_TRIGGER',
+ valid: true,
+ displayName: 'Catch Webhook',
+ lastUpdatedDate: new Date().toISOString(),
+ settings: {
+ pieceName: WEBHOOK_PIECE,
+ pieceVersion: webhookVersion,
+ triggerName: 'catch_webhook',
+ input: { authType: 'none' },
+ propertySettings: {},
+ },
+ nextAction: firstAction,
+ };
+}
+
+async function pickOption({ page, trigger, search }: PickOptionParams): Promise<void> {
+ await trigger.click();
+ await page.keyboard.type(search);
+ await page.getByRole('option', { name: search }).first().click();
+}
+
+async function runWithConcurrency<T>({ count, concurrency, task }: RunWithConcurrencyParams<T>): Promise<T[]> {
+ const results: T[] = new Array(count);
+ const failures: unknown[] = [];
+ let nextIndex = 0;
+ async function worker(): Promise<void> {
+ while (nextIndex < count && failures.length === 0) {
+ const index = nextIndex++;
+ try {
+ results[index] = await task(index);
+ } catch (error) {
+ failures.push(error);
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: Math.min(concurrency, count) }, () => worker()));
+ if (failures.length > 0) {
+ throw failures[0];
+ }
+ return results;
+}
+
+function logTiming(label: string, startedAt: number): void {
+ const seconds = ((Date.now() - startedAt) / 1000).toFixed(1);
+ console.log(`⏱ ${label} in ${seconds}s`);
+ test.info().annotations.push({ type: 'timing', description: `${label}: ${seconds}s` });
+}
+
+const FLOW_COUNT = 200;
+const STEPS_PER_FLOW = 10;
+const SEED_CONCURRENCY = 10;
+const WEBHOOK_PIECE = '@activepieces/piece-webhook';
+const HUGGING_FACE_PIECE = '@activepieces/piece-hugging-face';
+const LOAD_TEST_TIMEOUT_MS = 60 * 60 * 1000;
+const ACTION_TIMEOUT_MS = 30 * 1000;
+const REVIEW_STEP_TIMEOUT_MS = 5 * 60 * 1000;
+const REPLACE_TIMEOUT_MS = 30 * 60 * 1000;
+
+type Connection = {
+ id: string;
+ externalId: string;
+ displayName: string;
+};
+
+type CreatePublishedFlowParams = {
+ displayName: string;
+ trigger: unknown;
+ onCreated: (flowId: string) => void;
+};
+
+type CallParams = {
+ method: 'GET' | 'POST' | 'DELETE';
+ path: string;
+ data?: unknown;
+ params?: Record<string, string | number>;
+};
+
+type BuildWebhookTriggerParams = {
+ webhookVersion: string;
+ huggingFaceVersion: string;
+ connectionExternalId: string;
+};
+
+type PickOptionParams = {
+ page: Page;
+ trigger: ReturnType<Page['getByRole']>;
+ search: string;
+};
+
+type RunWithConcurrencyParams<T> = {
+ count: number;
+ concurrency: number;
+ task: (index: number) => Promise<T>;
+};