Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

feat(ai): carry AI step files by id instead of inline bytes (#15491)

Abdul7aedffb4bf06
diff --git a/bun.lock b/bun.lock
index 1747f33905c..f4d02eb1acd 100644
--- a/bun.lock
+++ b/bun.lock
@@ -10984,6 +10984,7 @@
"dotenv": "16.4.7",
"env-var": "7.5.0",
"nanoid": "3.3.18",
+ "p-limit": "2.3.0",
"proxy-chain": "2.7.1",
"socket.io": "4.7.5",
"socket.io-client": "4.8.1",
diff --git a/packages/core/execution/src/lib/workers/job-data.ts b/packages/core/execution/src/lib/workers/job-data.ts
index 05f799e96a3..57f97da272a 100644
--- a/packages/core/execution/src/lib/workers/job-data.ts
+++ b/packages/core/execution/src/lib/workers/job-data.ts
@@ -398,8 +398,8 @@ export const AiStepWebSearch = z.object({
export type AiStepWebSearch = z.infer<typeof AiStepWebSearch>
export const AiStepFile = z.object({
- mimeType: z.string(),
- base64: z.string(),
+ fileId: z.string(),
+ mimeType: z.string().optional(),
filename: z.string().optional(),
})
export type AiStepFile = z.infer<typeof AiStepFile>
diff --git a/packages/core/execution/src/lib/workers/worker-contract.ts b/packages/core/execution/src/lib/workers/worker-contract.ts
index eae72c40ef0..c30c71ceddd 100644
--- a/packages/core/execution/src/lib/workers/worker-contract.ts
+++ b/packages/core/execution/src/lib/workers/worker-contract.ts
@@ -97,6 +97,7 @@ export type WorkerToApiContract = {
resumeFlowStep(input: ResumeFlowStepRequest): Promise<void>
resolveAiProvider(input: ResolveAiProviderRequest): Promise<ResolveAiProviderResponse>
saveFlowStepFile(input: SaveFlowStepFileRequest): Promise<SaveFlowStepFileResponse>
+ readFlowStepFile(input: ReadFlowStepFileRequest): Promise<ReadFlowStepFileResponse>
resumeAiStep(input: ResumeAiStepRequest): Promise<void>
updateFlowStepProgress(input: UpdateFlowStepProgressRequest): Promise<void>
executePieceTool(input: ExecutePieceToolRequest): Promise<ExecutePieceToolResponse>
@@ -436,6 +437,18 @@ export type SaveFlowStepFileRequest = {
fileName: string
}
+export type ReadFlowStepFileRequest = {
+ projectId: string
+ platformId: string
+ fileId: string
+}
+
+export type ReadFlowStepFileResponse = {
+ data: Buffer
+ mimeType?: string
+ fileName?: string
+}
+
export type SaveFlowStepFileResponse = {
fileId: string
url: string
diff --git a/packages/core/execution/test/workers/ai-job-data.test.ts b/packages/core/execution/test/workers/ai-job-data.test.ts
index 20e029577c0..ae1a5e4f2b8 100644
--- a/packages/core/execution/test/workers/ai-job-data.test.ts
+++ b/packages/core/execution/test/workers/ai-job-data.test.ts
@@ -44,13 +44,13 @@ describe('what an AI step job accepts', () => {
...base,
action: AiStepAction.EXTRACT_STRUCTURED_DATA,
text: 'an invoice',
- files: [{ mimeType: 'application/pdf', base64: 'eA==' }],
+ files: [{ fileId: 'file-1', mimeType: 'application/pdf' }],
schema: { mode: 'simple', fields: [{ name: 'total' }] },
}).success).toBe(true)
expect(ExecuteAiJobData.safeParse({
...base,
action: AiStepAction.GENERATE_IMAGE,
- files: [{ mimeType: 'image/png', base64: 'eA==' }],
+ files: [{ fileId: 'file-2', mimeType: 'image/png' }],
advancedOptions: { size: '1024x1024' },
}).success).toBe(true)
})
diff --git a/packages/server/api/src/app/ai/ai-rpc-handlers.ts b/packages/server/api/src/app/ai/ai-rpc-handlers.ts
index 83b006a88d0..7dfd3b8fda7 100644
--- a/packages/server/api/src/app/ai/ai-rpc-handlers.ts
+++ b/packages/server/api/src/app/ai/ai-rpc-handlers.ts
@@ -1,5 +1,5 @@
import { ActivepiecesError, ErrorCode, isNil, sanitizeObjectForPostgresql, tryCatch } from '@activepieces/core-utils'
-import { FileCompression, FileSizeError, FileType, ResolveAiProviderRequest, ResolveAiProviderResponse, ResumeAiStepRequest, SaveFlowStepFileRequest, SaveFlowStepFileResponse, spreadIfDefined } from '@activepieces/shared'
+import { FileCompression, FileSizeError, FileType, ReadFlowStepFileRequest, ReadFlowStepFileResponse, ResolveAiProviderRequest, ResolveAiProviderResponse, ResumeAiStepRequest, SaveFlowStepFileRequest, SaveFlowStepFileResponse, spreadIfDefined } from '@activepieces/shared'
import { FastifyBaseLogger } from 'fastify'
import { fileService } from '../file/file.service'
import { filesService } from '../file/files-service'
@@ -51,6 +51,20 @@ export const aiRpcHandlers = (log: FastifyBaseLogger) => ({
return { fileId: file.id, url }
},
+ async readFlowStepFile(input: ReadFlowStepFileRequest): Promise<ReadFlowStepFileResponse> {
+ await assertProjectBelongsToPlatform({ ...input, log })
+ const file = await fileService(log).getDataOrThrow({
+ projectId: input.projectId,
+ fileId: input.fileId,
+ type: FileType.FLOW_STEP_FILE,
+ })
+ return {
+ data: file.data,
+ ...spreadIfDefined('mimeType', file.metadata?.['mimeType']),
+ ...spreadIfDefined('fileName', file.fileName),
+ }
+ },
+
async resumeAiStep(input: ResumeAiStepRequest): Promise<void> {
const resumeFields = { flowRun: { id: input.flowRunId }, waitpoint: { id: input.waitpointId } }
const flowRun = await flowRunService(log).getOne({ id: input.flowRunId, projectId: input.projectId })
diff --git a/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts b/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts
index 7e1cff0270d..4ebe7d1f753 100644
--- a/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts
+++ b/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts
@@ -400,6 +400,9 @@ export function createHandlers(log: FastifyBaseLogger, assignment: WorkerGroupAs
async saveFlowStepFile(input) {
return aiRpcHandlers(log).saveFlowStepFile(input)
},
+ async readFlowStepFile(input) {
+ return aiRpcHandlers(log).readFlowStepFile(input)
+ },
async resumeAiStep(input) {
return aiRpcHandlers(log).resumeAiStep(input)
},
diff --git a/packages/server/worker/package.json b/packages/server/worker/package.json
index a82ee2f3f96..d6163f202cb 100644
--- a/packages/server/worker/package.json
+++ b/packages/server/worker/package.json
@@ -29,6 +29,7 @@
"dayjs": "1.11.9",
"env-var": "7.5.0",
"nanoid": "3.3.18",
+ "p-limit": "2.3.0",
"proxy-chain": "2.7.1",
"socket.io": "4.7.5",
"dotenv": "16.4.7",
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/ai-files.ts b/packages/server/worker/src/lib/execute/jobs/ai/ai-files.ts
new file mode 100644
index 00000000000..c2ed7c0f785
--- /dev/null
+++ b/packages/server/worker/src/lib/execute/jobs/ai/ai-files.ts
@@ -0,0 +1,41 @@
+import { spreadIfDefined } from '@activepieces/core-utils'
+import { AiStepFile, ExtractStructuredDataJobData, GenerateImageJobData } from '@activepieces/shared'
+import pLimit from 'p-limit'
+import { JobContext } from '../../types'
+
+export async function resolveAiFiles({ ctx, data }: {
+ ctx: JobContext
+ data: AiStepWithFiles
+}): Promise<ResolvedAiFile[]> {
+ const files = data.files ?? []
+ const readAtOnce = pLimit(FILES_READ_AT_ONCE)
+ return Promise.all(files.map((file) => readAtOnce(() => resolveAiFile({ ctx, data, file }))))
+}
+
+async function resolveAiFile({ ctx, data, file }: {
+ ctx: JobContext
+ data: AiStepWithFiles
+ file: AiStepFile
+}): Promise<ResolvedAiFile> {
+ const stored = await ctx.apiClient.readFlowStepFile({
+ projectId: data.projectId,
+ platformId: data.platformId,
+ fileId: file.fileId,
+ })
+ return {
+ mimeType: file.mimeType ?? stored.mimeType ?? UNKNOWN_MIME_TYPE,
+ base64: stored.data.toString('base64'),
+ ...spreadIfDefined('filename', file.filename ?? stored.fileName),
+ }
+}
+
+const UNKNOWN_MIME_TYPE = 'application/octet-stream'
+const FILES_READ_AT_ONCE = 5
+
+export type AiStepWithFiles = ExtractStructuredDataJobData | GenerateImageJobData
+
+export type ResolvedAiFile = {
+ mimeType: string
+ base64: string
+ filename?: string
+}
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
index 92d5b449d91..92c9e4c7b9b 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
@@ -3,6 +3,7 @@ import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'
import { AiStepAction, ClassifyTextJobData, EngineResponseStatus, ExecuteAiJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse, WorkerJobType } from '@activepieces/shared'
import { generateText, ModelMessage, stepCountIs } from 'ai'
import { FireAndForgetJobResult, JobContext, JobHandler, JobResultKind } from '../../types'
+import { resolveAiFiles } from './ai-files'
import { extractStructuredData } from './extract-structured-data'
import { generateImageStep } from './generate-image'
@@ -56,9 +57,9 @@ async function runAiStep(ctx: JobContext, data: ExecuteAiJobData): Promise<unkno
const flowStep = flowStepMetadata(data)
switch (data.action) {
case AiStepAction.EXTRACT_STRUCTURED_DATA:
- return { answer: await extractStructuredData({ data, resolved, flowStep }) }
+ return { answer: await extractStructuredData({ data, resolved, flowStep, files: await resolveAiFiles({ ctx, data }) }) }
case AiStepAction.GENERATE_IMAGE:
- return { answer: await generateImageStep({ ctx, data, resolved, flowStep }) }
+ return { answer: await generateImageStep({ ctx, data, resolved, flowStep, inputImages: await resolveAiFiles({ ctx, data }) }) }
case AiStepAction.ASK_AI:
case AiStepAction.SUMMARIZE_TEXT:
case AiStepAction.CLASSIFY_TEXT:
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
index df7b6c24731..2557b4ee6f1 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
@@ -1,14 +1,15 @@
import { isNil, spreadIfDefined } from '@activepieces/core-utils'
import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'
-import { AiStepFile, ExtractStructuredDataJobData, ResolveAiProviderResponse } from '@activepieces/shared'
+import { ExtractStructuredDataJobData, ResolveAiProviderResponse } from '@activepieces/shared'
import { generateText, jsonSchema, ModelMessage, tool, UserModelMessage } from 'ai'
+import { ResolvedAiFile } from './ai-files'
-export async function extractStructuredData({ data, resolved, flowStep }: {
+export async function extractStructuredData({ data, resolved, flowStep, files }: {
data: ExtractStructuredDataJobData
resolved: ResolveAiProviderResponse
flowStep: FlowStepMetadata
+ files: ResolvedAiFile[]
}): Promise<unknown> {
- const files = data.files ?? []
if (isNil(data.text) && files.length === 0) {
throw new Error('Please provide text or image/PDF to extract data from.')
}
@@ -65,7 +66,7 @@ function buildSchema(data: ExtractStructuredDataJobData): { schemaDefinition: Re
return { schemaDefinition: jsonSchema({ type: 'object', properties, required } as Parameters<typeof jsonSchema>[0]), sanitizedNameMap }
}
-function buildMessages({ data, files }: { data: ExtractStructuredDataJobData, files: AiStepFile[] }): ModelMessage[] {
+function buildMessages({ data, files }: { data: ExtractStructuredDataJobData, files: ResolvedAiFile[] }): ModelMessage[] {
let textContent = data.prompt ?? 'Extract the following data from the provided data.'
if (data.text) {
textContent += `\n\nText to analyze:\n${data.text}`
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts
index dbcec313e3c..3cb15121ca7 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts
@@ -1,16 +1,17 @@
import { AIProviderName, isNil } from '@activepieces/core-utils'
import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'
-import { AI_PROVIDER_CAPABILITIES, AiStepFile, GenerateImageJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse } from '@activepieces/shared'
+import { AI_PROVIDER_CAPABILITIES, GenerateImageJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse } from '@activepieces/shared'
import { generateImage, generateText, ImageModel, ImagePart, LanguageModel } from 'ai'
import { JobContext } from '../../types'
+import { ResolvedAiFile } from './ai-files'
-export async function generateImageStep({ ctx, data, resolved, flowStep }: {
+export async function generateImageStep({ ctx, data, resolved, flowStep, inputImages }: {
ctx: JobContext
data: GenerateImageJobData
resolved: ResolveAiProviderResponse
flowStep: FlowStepMetadata
+ inputImages: ResolvedAiFile[]
}): Promise<string> {
- const inputImages = data.files ?? []
const image = await getGeneratedImage({ data, resolved, inputImages, flowStep })
const imageData = !isNil(image.base64) && image.base64.length > 0
? Buffer.from(image.base64, 'base64')
@@ -28,7 +29,7 @@ export async function generateImageStep({ ctx, data, resolved, flowStep }: {
async function getGeneratedImage({ data, resolved, inputImages, flowStep }: {
data: GenerateImageJobData
resolved: ResolveAiProviderResponse
- inputImages: AiStepFile[]
+ inputImages: ResolvedAiFile[]
flowStep: FlowStepMetadata
}): Promise<GeneratedImage> {
const model = createImageCapableModel({ resolved, modelId: data.modelId, flowStep })
@@ -77,7 +78,7 @@ function createImageCapableModel({ resolved, modelId, flowStep }: {
async function generateImageUsingGenerateText({ model, prompt, inputImages }: {
model: LanguageModel
prompt: string
- inputImages: AiStepFile[]
+ inputImages: ResolvedAiFile[]
}): Promise<GeneratedImage> {
const imageParts = inputImages.map<ImagePart>((file) => ({
type: 'image',
diff --git a/packages/server/worker/test/lib/execute/jobs/ai/ai-files.test.ts b/packages/server/worker/test/lib/execute/jobs/ai/ai-files.test.ts
new file mode 100644
index 00000000000..75190a9c2b8
--- /dev/null
+++ b/packages/server/worker/test/lib/execute/jobs/ai/ai-files.test.ts
@@ -0,0 +1,144 @@
+import { AiStepAction, ExecuteAiJobData, ReadFlowStepFileRequest, ReadFlowStepFileResponse } from '@activepieces/shared'
+import { describe, expect, it } from 'vitest'
+import { resolveAiFiles } from '../../../../../src/lib/execute/jobs/ai/ai-files'
+
+const PDF_BYTES = Buffer.from('%PDF-1.7 a small invoice')
+
+function contextReading(stored: Record<string, ReadFlowStepFileResponse>): Parameters<typeof resolveAiFiles>[0]['ctx'] {
+ const requests: ReadFlowStepFileRequest[] = []
+ const ctx = {
+ apiClient: {
+ readFlowStepFile: async (input: ReadFlowStepFileRequest) => {
+ requests.push(input)
+ const found = stored[input.fileId]
+ if (found === undefined) {
+ throw new Error(`no such file ${input.fileId}`)
+ }
+ return found
+ },
+ },
+ requests,
+ }
+ return ctx as unknown as Parameters<typeof resolveAiFiles>[0]['ctx']
+}
+
+function jobData(files: ExecuteAiJobData['files']): ExecuteAiJobData {
+ return {
+ requestId: 'request-1',
+ projectId: 'project-1',
+ platformId: 'platform-1',
+ flowId: 'flow-1',
+ flowRunId: 'run-1',
+ waitpointId: 'waitpoint-1',
+ action: AiStepAction.EXTRACT_STRUCTURED_DATA,
+ provider: 'openai',
+ modelId: 'gpt-4o',
+ files,
+ } as ExecuteAiJobData
+}
+
+describe('resolveAiFiles', () => {
+ it('fetches the bytes by id, scoped to the job project and platform', async () => {
+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES, mimeType: 'application/pdf' } })
+
+ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1' }]) })
+
+ expect(resolved).toEqual([{ mimeType: 'application/pdf', base64: PDF_BYTES.toString('base64') }])
+ expect((ctx as unknown as { requests: ReadFlowStepFileRequest[] }).requests).toEqual([
+ { projectId: 'project-1', platformId: 'platform-1', fileId: 'file-1' },
+ ])
+ })
+
+ it('prefers the mime type the caller sent over the one the file was stored with', async () => {
+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES, mimeType: 'application/octet-stream' } })
+
+ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1', mimeType: 'application/pdf' }]) })
+
+ expect(resolved[0].mimeType).toBe('application/pdf')
+ })
+
+ it('falls back to the stored mime type when the caller sent none', async () => {
+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES, mimeType: 'image/png' } })
+
+ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1' }]) })
+
+ expect(resolved[0].mimeType).toBe('image/png')
+ })
+
+ it('falls back to an unknown type rather than guessing, so a file of no known kind is skipped downstream', async () => {
+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES } })
+
+ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1' }]) })
+
+ expect(resolved[0].mimeType).toBe('application/octet-stream')
+ })
+
+ it('keeps the stored file name when the caller did not name the file', async () => {
+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES, mimeType: 'application/pdf', fileName: 'invoice.pdf' } })
+
+ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1' }]) })
+
+ expect(resolved[0].filename).toBe('invoice.pdf')
+ })
+
+ it('resolves several files in order, so a multi-file extract keeps its inputs aligned', async () => {
+ const ctx = contextReading({
+ 'file-1': { data: Buffer.from('one'), mimeType: 'application/pdf' },
+ 'file-2': { data: Buffer.from('two'), mimeType: 'image/png' },
+ 'file-3': { data: Buffer.from('three'), mimeType: 'application/pdf' },
+ })
+
+ const resolved = await resolveAiFiles({
+ ctx,
+ data: jobData([{ fileId: 'file-1' }, { fileId: 'file-2' }, { fileId: 'file-3' }]),
+ })
+
+ expect(resolved.map((file) => file.base64)).toEqual([
+ Buffer.from('one').toString('base64'),
+ Buffer.from('two').toString('base64'),
+ Buffer.from('three').toString('base64'),
+ ])
+ })
+
+ it('resolves to nothing when the step carries no files', async () => {
+ const ctx = contextReading({})
+
+ expect(await resolveAiFiles({ ctx, data: jobData(undefined) })).toEqual([])
+ })
+})
+
+describe('how many files resolveAiFiles reads at once', () => {
+ it('paces the reads instead of opening one call per attachment', async () => {
+ let inFlight = 0
+ let mostAtOnce = 0
+ const ctx = {
+ apiClient: {
+ readFlowStepFile: async () => {
+ inFlight += 1
+ mostAtOnce = Math.max(mostAtOnce, inFlight)
+ await new Promise((resolve) => setTimeout(resolve, 1))
+ inFlight -= 1
+ return { data: PDF_BYTES, mimeType: 'application/pdf' }
+ },
+ },
+ } as unknown as Parameters<typeof resolveAiFiles>[0]['ctx']
+ const files = Array.from({ length: 40 }, (_, index) => ({ fileId: `file-${index}` }))
+
+ const resolved = await resolveAiFiles({ ctx, data: jobData(files) })
+
+ expect(resolved).toHaveLength(40)
+ expect(mostAtOnce).toBeLessThanOrEqual(5)
+ })
+
+ it('still returns every attachment, in the order they were given', async () => {
+ const ctx = contextReading(Object.fromEntries(
+ Array.from({ length: 12 }, (_, index) => [`file-${index}`, { data: Buffer.from(`file ${index}`), mimeType: 'application/pdf' }]),
+ ))
+ const files = Array.from({ length: 12 }, (_, index) => ({ fileId: `file-${index}` }))
+
+ const resolved = await resolveAiFiles({ ctx, data: jobData(files) })
+
+ expect(resolved.map((file) => Buffer.from(file.base64, 'base64').toString()))
+ .toEqual(files.map((_, index) => `file ${index}`))
+ })
+})
diff --git a/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts b/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts
index 579a7093443..58604370526 100644
--- a/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts
+++ b/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts
@@ -72,6 +72,7 @@ describe('flowStep reaches every model built on the worker', () => {
}),
resolved: RESOLVED,
flowStep: FLOW_STEP,
+ files: [],
})
expect(createModelCalls).toHaveLength(1)
@@ -90,6 +91,7 @@ describe('flowStep reaches every model built on the worker', () => {
}),
resolved: { ...RESOLVED, provider: AIProviderName.GOOGLE },
flowStep: FLOW_STEP,
+ inputImages: [],
})
expect(createModelCalls).toHaveLength(1)