AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(runs): scope run metadata updates to the run's own project (#15647)
Hazem Adeldaba436a6e40
diff --git a/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts b/packages/server/api/src/app/flows/flow-run/flow-runs-queue.tsindex 9c8d111f8be..738f73d6ebc 100644--- a/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts+++ b/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts@@ -1,4 +1,4 @@-import { apId, isNil, sanitizeObjectForPostgresql, spreadIfDefined } from '@activepieces/core-utils'+import { apId, assertNotNullOrUndefined, isNil, sanitizeObjectForPostgresql, spreadIfDefined } from '@activepieces/core-utils'import { FlowRun, FlowRunStatus, isFlowRunStateTerminal, RunTimeline } from '@activepieces/shared'import { Queue, Worker } from 'bullmq'import { FastifyBaseLogger } from 'fastify'@@ -11,7 +11,7 @@ import { projectService } from '../../project/project-service'import { resumeService } from '../../waitpoints/resume-service'import { waitpointService } from '../../waitpoints/waitpoint-service'import { WaitpointStatus } from '../../waitpoints/waitpoint-types'-import { QueueName, redisMetadataKey, RunsMetadataJobData, RunsMetadataQueueConfig, runsMetadataQueueFactory, RunsMetadataUpsertData } from '../../workers/job'+import { legacyRedisMetadataKey, QueueName, redisMetadataKey, runsMetadataDeduplicationId, RunsMetadataJobData, RunsMetadataQueueConfig, runsMetadataQueueFactory, RunsMetadataUpsertData } from '../../workers/job'import { flowService } from '../flow/flow.service'import { flowRunRepo } from './flow-run-service'import { flowRunSideEffects } from './flow-run-side-effects'@@ -37,14 +37,16 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({job: { id: job.id },flowRun: { id: job.data.runId },}, '[runsMetadataQueue#worker] Saving runs metadata')- const key = redisMetadataKey(job.data.runId)+ assertNotNullOrUndefined(job.data.projectId, 'projectId')+ const key = redisMetadataKey({ projectId: job.data.projectId, runId: job.data.runId })await distributedLock(log).runExclusive({key: `runs_metadata_${job.data.runId}`,timeoutInSeconds: 30,fn: async () => {try {- await runsMetadataQueue(log).get().removeDeduplicationKey(job.data.runId)+ await runsMetadataQueue(log).get().removeDeduplicationKey(runsMetadataDeduplicationId({ projectId: job.data.projectId, runId: job.data.runId }))const rawRunMetadata = await distributedStore.hgetJson<RunsMetadataUpsertData>(key)+ ?? await claimMetadataWrittenBeforeTheKeyWasScoped(job.data)if (isNil(rawRunMetadata) || Object.keys(rawRunMetadata).length === 0) {log.info({job: { id: job.id },@@ -54,13 +56,13 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({}const runMetadata = sanitizeObjectForPostgresql(rawRunMetadata)- const existingFlowRun = await flowRunRepo().findOneBy({ id: job.data.runId })+ const flowRunScope = { id: job.data.runId, projectId: job.data.projectId }+ const existingFlowRun = await flowRunRepo().findOneBy(flowRunScope)let savedFlowRun: FlowRunif (!isNil(existingFlowRun)) {const timeline = buildTimeline({ existingFlowRun, runMetadata })- await flowRunRepo().update(job.data.runId, {+ await flowRunRepo().update(flowRunScope, {...spreadIfDefined('timeline', timeline),- ...spreadIfDefined('projectId', runMetadata.projectId),...spreadIfDefined('flowId', runMetadata.flowId),...spreadIfDefined('flowVersionId', runMetadata.flowVersionId),...spreadIfDefined('environment', runMetadata.environment),@@ -76,7 +78,7 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({...spreadIfDefined('updated', runMetadata.updated),...spreadIfDefined('stepsCount', runMetadata.stepsCount),})- const updatedFlowRun = await flowRunRepo().findOneBy({ id: job.data.runId })+ const updatedFlowRun = await flowRunRepo().findOneBy(flowRunScope)if (isNil(updatedFlowRun)) {log.info({job: { id: job.id },@@ -87,16 +89,27 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({savedFlowRun = updatedFlowRun}else {+ const idBelongsToAnotherProject = await flowRunRepo().countBy({ id: job.data.runId }) > 0+ if (idBelongsToAnotherProject) {+ log.warn({+ job: { id: job.id },+ flowRun: { id: job.data.runId },+ project: { id: job.data.projectId },+ }, '[runsMetadataQueue#worker] Run metadata reported for a run owned by another project, skipping job')+ await discardMetadata({ key, requestId: runMetadata.requestId })+ return+ }const flowId = runMetadata.flowId- const flowExists = !isNil(flowId) && await flowService(log).exists(flowId)+ const flowExists = !isNil(flowId) && await flowService(log).exists({ id: flowId, projectId: job.data.projectId })if (!flowExists) {log.info({job: { id: job.id },flowRun: { id: job.data.runId },}, '[runsMetadataQueue#worker] Flow does not exist (deleted), skipping job')+ await discardMetadata({ key, requestId: runMetadata.requestId })return}- savedFlowRun = await flowRunRepo().save(runMetadata)+ savedFlowRun = await flowRunRepo().save({ ...runMetadata, ...flowRunScope })}const parentRunId = savedFlowRun.parentRunId@@ -182,6 +195,23 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({})+async function discardMetadata({ key, requestId }: DiscardMetadataParams): Promise<void> {+ if (isNil(requestId)) {+ return+ }+ await distributedStore.deleteKeyIfFieldValueMatches(key, 'requestId', requestId)+}++async function claimMetadataWrittenBeforeTheKeyWasScoped({ runId, projectId }: RunsMetadataJobData): Promise<RunsMetadataUpsertData | null> {+ const legacyKey = legacyRedisMetadataKey(runId)+ const legacyMetadata = await distributedStore.hgetJson<RunsMetadataUpsertData>(legacyKey)+ if (isNil(legacyMetadata) || legacyMetadata.projectId !== projectId) {+ return null+ }+ await distributedStore.delete(legacyKey)+ return legacyMetadata+}+function buildTimeline({ existingFlowRun, runMetadata }: BuildTimelineParams): RunTimeline | undefined {return buildRunTimeline({existingTimeline: existingFlowRun.timeline,@@ -254,6 +284,11 @@ type ChildRunFailsParentParams = {willRetry?: boolean}+type DiscardMetadataParams = {+ key: string+ requestId?: string+}+type BuildTimelineParams = {existingFlowRun: FlowRunrunMetadata: RunsMetadataUpsertDatadiff --git a/packages/server/api/src/app/flows/flow/flow.service.ts b/packages/server/api/src/app/flows/flow/flow.service.tsindex 2210795d442..97a068aa05b 100644--- a/packages/server/api/src/app/flows/flow/flow.service.ts+++ b/packages/server/api/src/app/flows/flow/flow.service.ts@@ -230,9 +230,10 @@ export const flowService = (log: FastifyBaseLogger) => ({}))return paginationHelper.createPage(populatedFlows, isNil(sortBy) ? paginationResult.cursor : null)},- async exists(id: FlowId): Promise<boolean> {+ async exists({ id, projectId }: FlowExistsParams): Promise<boolean> {return flowRepo().existsBy({id,+ projectId,})},async getOneById(id: string): Promise<Flow | null> {@@ -982,6 +983,11 @@ type LockFlowVersionIfNotLockedParams = {log: FastifyBaseLogger}+type FlowExistsParams = {+ id: FlowId+ projectId: ProjectId+}+type ExistsByProjectAndStatusParams = {projectId: ProjectIdstatus: FlowStatusdiff --git a/packages/server/api/src/app/workers/job/runs-metadata-queue-factory.ts b/packages/server/api/src/app/workers/job/runs-metadata-queue-factory.tsindex b865d6e9b99..6ccc6fbee15 100644--- a/packages/server/api/src/app/workers/job/runs-metadata-queue-factory.ts+++ b/packages/server/api/src/app/workers/job/runs-metadata-queue-factory.ts@@ -6,7 +6,11 @@ import Redis from 'ioredis'import { DistributedStore } from '../../database/redis/distributed-store-factory'import { QueueName } from './index'-export const redisMetadataKey = (runId: ApId): string => `runs_metadata:${runId}`+export const redisMetadataKey = ({ projectId, runId }: RedisMetadataKeyParams): string => `runs_metadata:${projectId}:${runId}`++export const legacyRedisMetadataKey = (runId: ApId): string => `runs_metadata:${runId}`++export const runsMetadataDeduplicationId = ({ projectId, runId }: RedisMetadataKeyParams): string => `${projectId}-${runId}`export const runsMetadataQueueFactory = ({createRedisConnection,@@ -41,7 +45,7 @@ export const runsMetadataQueueFactory = ({const cleanedParams = stripToRunsMetadataUpsertData(params)- await distributedStore.merge(redisMetadataKey(cleanedParams.id), {+ await distributedStore.merge(redisMetadataKey({ projectId: cleanedParams.projectId, runId: cleanedParams.id }), {...cleanedParams,requestId: apId(),})@@ -49,7 +53,7 @@ export const runsMetadataQueueFactory = ({await queueInstance.add('update-run-metadata',{ runId: cleanedParams.id, projectId: cleanedParams.projectId },- { deduplication: { id: cleanedParams.id } },+ { deduplication: { id: runsMetadataDeduplicationId({ projectId: cleanedParams.projectId, runId: cleanedParams.id }) } },)},@@ -84,6 +88,11 @@ function stripToRunsMetadataUpsertData(params: RunsMetadataUpsertData): RunsMetareturn result as RunsMetadataUpsertData}+type RedisMetadataKeyParams = {+ projectId: ApId+ runId: ApId+}+type RunsMetadataQueueFactoryParams = {createRedisConnection: () => Promise<Redis>distributedStore: DistributedStorediff --git a/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.tsindex 6f2fe0a1326..e50040a8fda 100644--- a/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts+++ b/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts@@ -130,7 +130,7 @@ describe('Resume flow run', () => {const runId = flowRun.idconst requestId = apId()- await distributedStore.merge(redisMetadataKey(runId), {+ await distributedStore.merge(redisMetadataKey({ projectId: ctx.project.id, runId }), {id: runId,projectId: ctx.project.id,flowId: flow.id,@@ -257,7 +257,7 @@ describe('Resume flow run', () => {environment: RunEnvironment.PRODUCTION,status: FlowRunStatus.RUNNING,}- await distributedStore.merge(redisMetadataKey(runId), runMetadata)+ await distributedStore.merge(redisMetadataKey({ projectId: ctx.project.id, runId }), runMetadata)await db.save('waitpoint', {id: apId(),@@ -316,7 +316,7 @@ describe('Resume flow run', () => {environment: RunEnvironment.PRODUCTION,status: FlowRunStatus.RUNNING,}- await distributedStore.merge(redisMetadataKey(runId), runMetadata)+ await distributedStore.merge(redisMetadataKey({ projectId: ctx.project.id, runId }), runMetadata)await db.save('waitpoint', {id: apId(),diff --git a/packages/server/api/test/integration/ce/flows/flow-run/runs-metadata-tenant-isolation.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/runs-metadata-tenant-isolation.test.tsnew file mode 100644index 00000000000..40a7cb63a25--- /dev/null+++ b/packages/server/api/test/integration/ce/flows/flow-run/runs-metadata-tenant-isolation.test.ts@@ -0,0 +1,146 @@+import { FlowRun, FlowRunStatus, FlowVersionState, RunEnvironment } from '@activepieces/shared'+import { FastifyInstance } from 'fastify'+import { engineRunCallbackService } from '../../../../../src/app/flows/flow-run/engine-run-callback-service'+import { distributedStore } from '../../../../../src/app/database/redis-connections'+import { runsMetadataQueue } from '../../../../../src/app/flows/flow-run/flow-runs-queue'+import { legacyRedisMetadataKey } from '../../../../../src/app/workers/job'+import { db } from '../../../../helpers/db'+import { createMockFlow, createMockFlowRun, createMockFlowVersion, mockAndSaveBasicSetup } from '../../../../helpers/mocks'+import { setupTestEnvironment, teardownTestEnvironment } from '../../../../helpers/test-setup'++let app: FastifyInstance++beforeAll(async () => {+ app = await setupTestEnvironment()+})++afterAll(async () => {+ await teardownTestEnvironment()+})++async function createFlow(projectId: string) {+ const flow = createMockFlow({ projectId })+ await db.save('flow', flow)+ const flowVersion = createMockFlowVersion({ flowId: flow.id, state: FlowVersionState.LOCKED })+ await db.save('flow_version', flowVersion)+ return { flow, flowVersion }+}++async function createRunningFlowRun(projectId: string) {+ const { flow, flowVersion } = await createFlow(projectId)+ const flowRun = createMockFlowRun({+ projectId,+ flowId: flow.id,+ flowVersionId: flowVersion.id,+ status: FlowRunStatus.RUNNING,+ environment: RunEnvironment.PRODUCTION,+ finishTime: null,+ })+ await db.save('flow_run', flowRun)+ return flowRun+}++// A rejected report is a non-event, and the queue's own counts are not a usable signal under+// the in-memory redis used in tests — a drain check there reports empty immediately and makes+// these assertions pass even with the guards removed. So the barrier is a later report from the+// run's own project: once its marker lands, the earlier foreign report has been processed too.+async function waitUntilTheOwnerReportLands({ flowRun, marker }: { flowRun: FlowRun, marker: string }): Promise<void> {+ await engineRunCallbackService(app.log).uploadRunLog({+ projectId: flowRun.projectId,+ request: { runId: flowRun.id, tags: [marker] },+ })+ await vi.waitUntil(+ async () => (await db.findOneBy<{ tags: string[] }>('flow_run', { id: flowRun.id }))?.tags?.includes(marker) === true,+ { timeout: 15_000, interval: 100 },+ )+}++describe('runs metadata tenant isolation', () => {+ it('does not let a report from another project rewrite an existing run', async () => {+ const { mockProject: attacker } = await mockAndSaveBasicSetup()+ const { mockProject: victim } = await mockAndSaveBasicSetup()+ const victimRun = await createRunningFlowRun(victim.id)++ await engineRunCallbackService(app.log).uploadRunLog({+ projectId: attacker.id,+ request: {+ runId: victimRun.id,+ status: FlowRunStatus.FAILED,+ finishTime: new Date().toISOString(),+ },+ })+ await waitUntilTheOwnerReportLands({ flowRun: victimRun, marker: 'barrier-update' })++ const run = await db.findOneBy<{ status: string, projectId: string }>('flow_run', { id: victimRun.id })+ expect(run?.status).toBe(FlowRunStatus.RUNNING)+ expect(run?.projectId).toBe(victim.id)+ }, 60_000)++ it('does not let a report poison the metadata another project later applies', async () => {+ const { mockProject: attacker } = await mockAndSaveBasicSetup()+ const { mockProject: victim } = await mockAndSaveBasicSetup()+ const victimRun = await createRunningFlowRun(victim.id)++ await engineRunCallbackService(app.log).uploadRunLog({+ projectId: attacker.id,+ request: {+ runId: victimRun.id,+ status: FlowRunStatus.FAILED,+ finishTime: new Date().toISOString(),+ failedStep: { name: 'step_1', displayName: 'Injected', message: 'injected by another project' },+ },+ })+ await waitUntilTheOwnerReportLands({ flowRun: victimRun, marker: 'barrier-poison' })++ const run = await db.findOneBy<{ status: string, failedStep: unknown }>('flow_run', { id: victimRun.id })+ expect(run?.status).toBe(FlowRunStatus.RUNNING)+ expect(run?.failedStep).toBeNull()+ }, 60_000)++ it('still applies metadata written under the pre-upgrade key, but only for its owner', async () => {+ const { mockProject: attacker } = await mockAndSaveBasicSetup()+ const { mockProject: victim } = await mockAndSaveBasicSetup()+ const ownedRun = await createRunningFlowRun(victim.id)+ const foreignRun = await createRunningFlowRun(victim.id)++ await distributedStore.merge(legacyRedisMetadataKey(ownedRun.id), {+ id: ownedRun.id, projectId: victim.id, status: FlowRunStatus.SUCCEEDED, requestId: 'legacy-owned',+ })+ await distributedStore.merge(legacyRedisMetadataKey(foreignRun.id), {+ id: foreignRun.id, projectId: attacker.id, status: FlowRunStatus.FAILED, requestId: 'legacy-foreign',+ })+ await runsMetadataQueue(app.log).get().add('update-run-metadata', { runId: ownedRun.id, projectId: victim.id })+ await runsMetadataQueue(app.log).get().add('update-run-metadata', { runId: foreignRun.id, projectId: victim.id })+ await vi.waitUntil(+ async () => (await db.findOneBy<{ status: string }>('flow_run', { id: ownedRun.id }))?.status === FlowRunStatus.SUCCEEDED,+ { timeout: 15_000, interval: 100 },+ )++ const owned = await db.findOneBy<{ status: string }>('flow_run', { id: ownedRun.id })+ const foreign = await db.findOneBy<{ status: string }>('flow_run', { id: foreignRun.id })+ expect(owned?.status).toBe(FlowRunStatus.SUCCEEDED)+ expect(foreign?.status).toBe(FlowRunStatus.RUNNING)+ // a non-owner must not consume the pending metadata it refused to apply+ const foreignLegacy = await distributedStore.hgetJson(legacyRedisMetadataKey(foreignRun.id))+ expect(foreignLegacy).not.toBeNull()+ }, 60_000)++ it('still applies a report from the run own project', async () => {+ const { mockProject } = await mockAndSaveBasicSetup()+ const flowRun = await createRunningFlowRun(mockProject.id)++ await engineRunCallbackService(app.log).uploadRunLog({+ projectId: mockProject.id,+ request: {+ runId: flowRun.id,+ status: FlowRunStatus.SUCCEEDED,+ finishTime: new Date().toISOString(),+ },+ })++ await vi.waitUntil(+ async () => (await db.findOneBy<{ status: string }>('flow_run', { id: flowRun.id }))?.status === FlowRunStatus.SUCCEEDED,+ { timeout: 10_000, interval: 100 },+ )+ }, 60_000)+})