Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(runs): scope run metadata updates to the run's own project (#15647)

Hazem Adeldaba436a6e40
diff --git a/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts b/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts
index 9c8d111f8be..738f73d6ebc 100644
--- a/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts
+++ b/packages/server/api/src/app/flows/flow-run/flow-runs-queue.ts
@@ -1,4 +1,4 @@
-import { apId, isNil, sanitizeObjectForPostgresql, spreadIfDefined } from '@activepieces/core-utils'
+import { apId, assertNotNullOrUndefined, isNil, sanitizeObjectForPostgresql, spreadIfDefined } from '@activepieces/core-utils'
import { FlowRun, FlowRunStatus, isFlowRunStateTerminal, RunTimeline } from '@activepieces/shared'
import { Queue, Worker } from 'bullmq'
import { FastifyBaseLogger } from 'fastify'
@@ -11,7 +11,7 @@ import { projectService } from '../../project/project-service'
import { resumeService } from '../../waitpoints/resume-service'
import { waitpointService } from '../../waitpoints/waitpoint-service'
import { WaitpointStatus } from '../../waitpoints/waitpoint-types'
-import { QueueName, redisMetadataKey, RunsMetadataJobData, RunsMetadataQueueConfig, runsMetadataQueueFactory, RunsMetadataUpsertData } from '../../workers/job'
+import { legacyRedisMetadataKey, QueueName, redisMetadataKey, runsMetadataDeduplicationId, RunsMetadataJobData, RunsMetadataQueueConfig, runsMetadataQueueFactory, RunsMetadataUpsertData } from '../../workers/job'
import { flowService } from '../flow/flow.service'
import { flowRunRepo } from './flow-run-service'
import { flowRunSideEffects } from './flow-run-side-effects'
@@ -37,14 +37,16 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({
job: { id: job.id },
flowRun: { id: job.data.runId },
}, '[runsMetadataQueue#worker] Saving runs metadata')
- const key = redisMetadataKey(job.data.runId)
+ assertNotNullOrUndefined(job.data.projectId, 'projectId')
+ const key = redisMetadataKey({ projectId: job.data.projectId, runId: job.data.runId })
await distributedLock(log).runExclusive({
key: `runs_metadata_${job.data.runId}`,
timeoutInSeconds: 30,
fn: async () => {
try {
- await runsMetadataQueue(log).get().removeDeduplicationKey(job.data.runId)
+ await runsMetadataQueue(log).get().removeDeduplicationKey(runsMetadataDeduplicationId({ projectId: job.data.projectId, runId: job.data.runId }))
const rawRunMetadata = await distributedStore.hgetJson<RunsMetadataUpsertData>(key)
+ ?? await claimMetadataWrittenBeforeTheKeyWasScoped(job.data)
if (isNil(rawRunMetadata) || Object.keys(rawRunMetadata).length === 0) {
log.info({
job: { id: job.id },
@@ -54,13 +56,13 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({
}
const runMetadata = sanitizeObjectForPostgresql(rawRunMetadata)
- const existingFlowRun = await flowRunRepo().findOneBy({ id: job.data.runId })
+ const flowRunScope = { id: job.data.runId, projectId: job.data.projectId }
+ const existingFlowRun = await flowRunRepo().findOneBy(flowRunScope)
let savedFlowRun: FlowRun
if (!isNil(existingFlowRun)) {
const timeline = buildTimeline({ existingFlowRun, runMetadata })
- await flowRunRepo().update(job.data.runId, {
+ await flowRunRepo().update(flowRunScope, {
...spreadIfDefined('timeline', timeline),
- ...spreadIfDefined('projectId', runMetadata.projectId),
...spreadIfDefined('flowId', runMetadata.flowId),
...spreadIfDefined('flowVersionId', runMetadata.flowVersionId),
...spreadIfDefined('environment', runMetadata.environment),
@@ -76,7 +78,7 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({
...spreadIfDefined('updated', runMetadata.updated),
...spreadIfDefined('stepsCount', runMetadata.stepsCount),
})
- const updatedFlowRun = await flowRunRepo().findOneBy({ id: job.data.runId })
+ const updatedFlowRun = await flowRunRepo().findOneBy(flowRunScope)
if (isNil(updatedFlowRun)) {
log.info({
job: { id: job.id },
@@ -87,16 +89,27 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({
savedFlowRun = updatedFlowRun
}
else {
+ const idBelongsToAnotherProject = await flowRunRepo().countBy({ id: job.data.runId }) > 0
+ if (idBelongsToAnotherProject) {
+ log.warn({
+ job: { id: job.id },
+ flowRun: { id: job.data.runId },
+ project: { id: job.data.projectId },
+ }, '[runsMetadataQueue#worker] Run metadata reported for a run owned by another project, skipping job')
+ await discardMetadata({ key, requestId: runMetadata.requestId })
+ return
+ }
const flowId = runMetadata.flowId
- const flowExists = !isNil(flowId) && await flowService(log).exists(flowId)
+ const flowExists = !isNil(flowId) && await flowService(log).exists({ id: flowId, projectId: job.data.projectId })
if (!flowExists) {
log.info({
job: { id: job.id },
flowRun: { id: job.data.runId },
}, '[runsMetadataQueue#worker] Flow does not exist (deleted), skipping job')
+ await discardMetadata({ key, requestId: runMetadata.requestId })
return
}
- savedFlowRun = await flowRunRepo().save(runMetadata)
+ savedFlowRun = await flowRunRepo().save({ ...runMetadata, ...flowRunScope })
}
const parentRunId = savedFlowRun.parentRunId
@@ -182,6 +195,23 @@ export const runsMetadataQueue = (log: FastifyBaseLogger) => ({
})
+async function discardMetadata({ key, requestId }: DiscardMetadataParams): Promise<void> {
+ if (isNil(requestId)) {
+ return
+ }
+ await distributedStore.deleteKeyIfFieldValueMatches(key, 'requestId', requestId)
+}
+
+async function claimMetadataWrittenBeforeTheKeyWasScoped({ runId, projectId }: RunsMetadataJobData): Promise<RunsMetadataUpsertData | null> {
+ const legacyKey = legacyRedisMetadataKey(runId)
+ const legacyMetadata = await distributedStore.hgetJson<RunsMetadataUpsertData>(legacyKey)
+ if (isNil(legacyMetadata) || legacyMetadata.projectId !== projectId) {
+ return null
+ }
+ await distributedStore.delete(legacyKey)
+ return legacyMetadata
+}
+
function buildTimeline({ existingFlowRun, runMetadata }: BuildTimelineParams): RunTimeline | undefined {
return buildRunTimeline({
existingTimeline: existingFlowRun.timeline,
@@ -254,6 +284,11 @@ type ChildRunFailsParentParams = {
willRetry?: boolean
}
+type DiscardMetadataParams = {
+ key: string
+ requestId?: string
+}
+
type BuildTimelineParams = {
existingFlowRun: FlowRun
runMetadata: RunsMetadataUpsertData
diff --git a/packages/server/api/src/app/flows/flow/flow.service.ts b/packages/server/api/src/app/flows/flow/flow.service.ts
index 2210795d442..97a068aa05b 100644
--- a/packages/server/api/src/app/flows/flow/flow.service.ts
+++ b/packages/server/api/src/app/flows/flow/flow.service.ts
@@ -230,9 +230,10 @@ export const flowService = (log: FastifyBaseLogger) => ({
}))
return paginationHelper.createPage(populatedFlows, isNil(sortBy) ? paginationResult.cursor : null)
},
- async exists(id: FlowId): Promise<boolean> {
+ async exists({ id, projectId }: FlowExistsParams): Promise<boolean> {
return flowRepo().existsBy({
id,
+ projectId,
})
},
async getOneById(id: string): Promise<Flow | null> {
@@ -982,6 +983,11 @@ type LockFlowVersionIfNotLockedParams = {
log: FastifyBaseLogger
}
+type FlowExistsParams = {
+ id: FlowId
+ projectId: ProjectId
+}
+
type ExistsByProjectAndStatusParams = {
projectId: ProjectId
status: FlowStatus
diff --git a/packages/server/api/src/app/workers/job/runs-metadata-queue-factory.ts b/packages/server/api/src/app/workers/job/runs-metadata-queue-factory.ts
index b865d6e9b99..6ccc6fbee15 100644
--- a/packages/server/api/src/app/workers/job/runs-metadata-queue-factory.ts
+++ b/packages/server/api/src/app/workers/job/runs-metadata-queue-factory.ts
@@ -6,7 +6,11 @@ import Redis from 'ioredis'
import { DistributedStore } from '../../database/redis/distributed-store-factory'
import { QueueName } from './index'
-export const redisMetadataKey = (runId: ApId): string => `runs_metadata:${runId}`
+export const redisMetadataKey = ({ projectId, runId }: RedisMetadataKeyParams): string => `runs_metadata:${projectId}:${runId}`
+
+export const legacyRedisMetadataKey = (runId: ApId): string => `runs_metadata:${runId}`
+
+export const runsMetadataDeduplicationId = ({ projectId, runId }: RedisMetadataKeyParams): string => `${projectId}-${runId}`
export const runsMetadataQueueFactory = ({
createRedisConnection,
@@ -41,7 +45,7 @@ export const runsMetadataQueueFactory = ({
const cleanedParams = stripToRunsMetadataUpsertData(params)
- await distributedStore.merge(redisMetadataKey(cleanedParams.id), {
+ await distributedStore.merge(redisMetadataKey({ projectId: cleanedParams.projectId, runId: cleanedParams.id }), {
...cleanedParams,
requestId: apId(),
})
@@ -49,7 +53,7 @@ export const runsMetadataQueueFactory = ({
await queueInstance.add(
'update-run-metadata',
{ runId: cleanedParams.id, projectId: cleanedParams.projectId },
- { deduplication: { id: cleanedParams.id } },
+ { deduplication: { id: runsMetadataDeduplicationId({ projectId: cleanedParams.projectId, runId: cleanedParams.id }) } },
)
},
@@ -84,6 +88,11 @@ function stripToRunsMetadataUpsertData(params: RunsMetadataUpsertData): RunsMeta
return result as RunsMetadataUpsertData
}
+type RedisMetadataKeyParams = {
+ projectId: ApId
+ runId: ApId
+}
+
type RunsMetadataQueueFactoryParams = {
createRedisConnection: () => Promise<Redis>
distributedStore: DistributedStore
diff --git a/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts
index 6f2fe0a1326..e50040a8fda 100644
--- a/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts
+++ b/packages/server/api/test/integration/ce/flows/flow-run/resume-flow-run.test.ts
@@ -130,7 +130,7 @@ describe('Resume flow run', () => {
const runId = flowRun.id
const requestId = apId()
- await distributedStore.merge(redisMetadataKey(runId), {
+ await distributedStore.merge(redisMetadataKey({ projectId: ctx.project.id, runId }), {
id: runId,
projectId: ctx.project.id,
flowId: flow.id,
@@ -257,7 +257,7 @@ describe('Resume flow run', () => {
environment: RunEnvironment.PRODUCTION,
status: FlowRunStatus.RUNNING,
}
- await distributedStore.merge(redisMetadataKey(runId), runMetadata)
+ await distributedStore.merge(redisMetadataKey({ projectId: ctx.project.id, runId }), runMetadata)
await db.save('waitpoint', {
id: apId(),
@@ -316,7 +316,7 @@ describe('Resume flow run', () => {
environment: RunEnvironment.PRODUCTION,
status: FlowRunStatus.RUNNING,
}
- await distributedStore.merge(redisMetadataKey(runId), runMetadata)
+ await distributedStore.merge(redisMetadataKey({ projectId: ctx.project.id, runId }), runMetadata)
await db.save('waitpoint', {
id: apId(),
diff --git a/packages/server/api/test/integration/ce/flows/flow-run/runs-metadata-tenant-isolation.test.ts b/packages/server/api/test/integration/ce/flows/flow-run/runs-metadata-tenant-isolation.test.ts
new file mode 100644
index 00000000000..40a7cb63a25
--- /dev/null
+++ b/packages/server/api/test/integration/ce/flows/flow-run/runs-metadata-tenant-isolation.test.ts
@@ -0,0 +1,146 @@
+import { FlowRun, FlowRunStatus, FlowVersionState, RunEnvironment } from '@activepieces/shared'
+import { FastifyInstance } from 'fastify'
+import { engineRunCallbackService } from '../../../../../src/app/flows/flow-run/engine-run-callback-service'
+import { distributedStore } from '../../../../../src/app/database/redis-connections'
+import { runsMetadataQueue } from '../../../../../src/app/flows/flow-run/flow-runs-queue'
+import { legacyRedisMetadataKey } from '../../../../../src/app/workers/job'
+import { db } from '../../../../helpers/db'
+import { createMockFlow, createMockFlowRun, createMockFlowVersion, mockAndSaveBasicSetup } from '../../../../helpers/mocks'
+import { setupTestEnvironment, teardownTestEnvironment } from '../../../../helpers/test-setup'
+
+let app: FastifyInstance
+
+beforeAll(async () => {
+ app = await setupTestEnvironment()
+})
+
+afterAll(async () => {
+ await teardownTestEnvironment()
+})
+
+async function createFlow(projectId: string) {
+ const flow = createMockFlow({ projectId })
+ await db.save('flow', flow)
+ const flowVersion = createMockFlowVersion({ flowId: flow.id, state: FlowVersionState.LOCKED })
+ await db.save('flow_version', flowVersion)
+ return { flow, flowVersion }
+}
+
+async function createRunningFlowRun(projectId: string) {
+ const { flow, flowVersion } = await createFlow(projectId)
+ const flowRun = createMockFlowRun({
+ projectId,
+ flowId: flow.id,
+ flowVersionId: flowVersion.id,
+ status: FlowRunStatus.RUNNING,
+ environment: RunEnvironment.PRODUCTION,
+ finishTime: null,
+ })
+ await db.save('flow_run', flowRun)
+ return flowRun
+}
+
+// A rejected report is a non-event, and the queue's own counts are not a usable signal under
+// the in-memory redis used in tests — a drain check there reports empty immediately and makes
+// these assertions pass even with the guards removed. So the barrier is a later report from the
+// run's own project: once its marker lands, the earlier foreign report has been processed too.
+async function waitUntilTheOwnerReportLands({ flowRun, marker }: { flowRun: FlowRun, marker: string }): Promise<void> {
+ await engineRunCallbackService(app.log).uploadRunLog({
+ projectId: flowRun.projectId,
+ request: { runId: flowRun.id, tags: [marker] },
+ })
+ await vi.waitUntil(
+ async () => (await db.findOneBy<{ tags: string[] }>('flow_run', { id: flowRun.id }))?.tags?.includes(marker) === true,
+ { timeout: 15_000, interval: 100 },
+ )
+}
+
+describe('runs metadata tenant isolation', () => {
+ it('does not let a report from another project rewrite an existing run', async () => {
+ const { mockProject: attacker } = await mockAndSaveBasicSetup()
+ const { mockProject: victim } = await mockAndSaveBasicSetup()
+ const victimRun = await createRunningFlowRun(victim.id)
+
+ await engineRunCallbackService(app.log).uploadRunLog({
+ projectId: attacker.id,
+ request: {
+ runId: victimRun.id,
+ status: FlowRunStatus.FAILED,
+ finishTime: new Date().toISOString(),
+ },
+ })
+ await waitUntilTheOwnerReportLands({ flowRun: victimRun, marker: 'barrier-update' })
+
+ const run = await db.findOneBy<{ status: string, projectId: string }>('flow_run', { id: victimRun.id })
+ expect(run?.status).toBe(FlowRunStatus.RUNNING)
+ expect(run?.projectId).toBe(victim.id)
+ }, 60_000)
+
+ it('does not let a report poison the metadata another project later applies', async () => {
+ const { mockProject: attacker } = await mockAndSaveBasicSetup()
+ const { mockProject: victim } = await mockAndSaveBasicSetup()
+ const victimRun = await createRunningFlowRun(victim.id)
+
+ await engineRunCallbackService(app.log).uploadRunLog({
+ projectId: attacker.id,
+ request: {
+ runId: victimRun.id,
+ status: FlowRunStatus.FAILED,
+ finishTime: new Date().toISOString(),
+ failedStep: { name: 'step_1', displayName: 'Injected', message: 'injected by another project' },
+ },
+ })
+ await waitUntilTheOwnerReportLands({ flowRun: victimRun, marker: 'barrier-poison' })
+
+ const run = await db.findOneBy<{ status: string, failedStep: unknown }>('flow_run', { id: victimRun.id })
+ expect(run?.status).toBe(FlowRunStatus.RUNNING)
+ expect(run?.failedStep).toBeNull()
+ }, 60_000)
+
+ it('still applies metadata written under the pre-upgrade key, but only for its owner', async () => {
+ const { mockProject: attacker } = await mockAndSaveBasicSetup()
+ const { mockProject: victim } = await mockAndSaveBasicSetup()
+ const ownedRun = await createRunningFlowRun(victim.id)
+ const foreignRun = await createRunningFlowRun(victim.id)
+
+ await distributedStore.merge(legacyRedisMetadataKey(ownedRun.id), {
+ id: ownedRun.id, projectId: victim.id, status: FlowRunStatus.SUCCEEDED, requestId: 'legacy-owned',
+ })
+ await distributedStore.merge(legacyRedisMetadataKey(foreignRun.id), {
+ id: foreignRun.id, projectId: attacker.id, status: FlowRunStatus.FAILED, requestId: 'legacy-foreign',
+ })
+ await runsMetadataQueue(app.log).get().add('update-run-metadata', { runId: ownedRun.id, projectId: victim.id })
+ await runsMetadataQueue(app.log).get().add('update-run-metadata', { runId: foreignRun.id, projectId: victim.id })
+ await vi.waitUntil(
+ async () => (await db.findOneBy<{ status: string }>('flow_run', { id: ownedRun.id }))?.status === FlowRunStatus.SUCCEEDED,
+ { timeout: 15_000, interval: 100 },
+ )
+
+ const owned = await db.findOneBy<{ status: string }>('flow_run', { id: ownedRun.id })
+ const foreign = await db.findOneBy<{ status: string }>('flow_run', { id: foreignRun.id })
+ expect(owned?.status).toBe(FlowRunStatus.SUCCEEDED)
+ expect(foreign?.status).toBe(FlowRunStatus.RUNNING)
+ // a non-owner must not consume the pending metadata it refused to apply
+ const foreignLegacy = await distributedStore.hgetJson(legacyRedisMetadataKey(foreignRun.id))
+ expect(foreignLegacy).not.toBeNull()
+ }, 60_000)
+
+ it('still applies a report from the run own project', async () => {
+ const { mockProject } = await mockAndSaveBasicSetup()
+ const flowRun = await createRunningFlowRun(mockProject.id)
+
+ await engineRunCallbackService(app.log).uploadRunLog({
+ projectId: mockProject.id,
+ request: {
+ runId: flowRun.id,
+ status: FlowRunStatus.SUCCEEDED,
+ finishTime: new Date().toISOString(),
+ },
+ })
+
+ await vi.waitUntil(
+ async () => (await db.findOneBy<{ status: string }>('flow_run', { id: flowRun.id }))?.status === FlowRunStatus.SUCCEEDED,
+ { timeout: 10_000, interval: 100 },
+ )
+ }, 60_000)
+})