AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
feat(ai): carry AI step files by id instead of inline bytes (#15491)
Abdul7aedffb4bf06
diff --git a/bun.lock b/bun.lockindex 1747f33905c..f4d02eb1acd 100644--- a/bun.lock+++ b/bun.lock@@ -10984,6 +10984,7 @@"dotenv": "16.4.7","env-var": "7.5.0","nanoid": "3.3.18",+ "p-limit": "2.3.0","proxy-chain": "2.7.1","socket.io": "4.7.5","socket.io-client": "4.8.1",diff --git a/packages/core/execution/src/lib/workers/job-data.ts b/packages/core/execution/src/lib/workers/job-data.tsindex 05f799e96a3..57f97da272a 100644--- a/packages/core/execution/src/lib/workers/job-data.ts+++ b/packages/core/execution/src/lib/workers/job-data.ts@@ -398,8 +398,8 @@ export const AiStepWebSearch = z.object({export type AiStepWebSearch = z.infer<typeof AiStepWebSearch>export const AiStepFile = z.object({- mimeType: z.string(),- base64: z.string(),+ fileId: z.string(),+ mimeType: z.string().optional(),filename: z.string().optional(),})export type AiStepFile = z.infer<typeof AiStepFile>diff --git a/packages/core/execution/src/lib/workers/worker-contract.ts b/packages/core/execution/src/lib/workers/worker-contract.tsindex eae72c40ef0..c30c71ceddd 100644--- a/packages/core/execution/src/lib/workers/worker-contract.ts+++ b/packages/core/execution/src/lib/workers/worker-contract.ts@@ -97,6 +97,7 @@ export type WorkerToApiContract = {resumeFlowStep(input: ResumeFlowStepRequest): Promise<void>resolveAiProvider(input: ResolveAiProviderRequest): Promise<ResolveAiProviderResponse>saveFlowStepFile(input: SaveFlowStepFileRequest): Promise<SaveFlowStepFileResponse>+ readFlowStepFile(input: ReadFlowStepFileRequest): Promise<ReadFlowStepFileResponse>resumeAiStep(input: ResumeAiStepRequest): Promise<void>updateFlowStepProgress(input: UpdateFlowStepProgressRequest): Promise<void>executePieceTool(input: ExecutePieceToolRequest): Promise<ExecutePieceToolResponse>@@ -436,6 +437,18 @@ export type SaveFlowStepFileRequest = {fileName: string}+export type ReadFlowStepFileRequest = {+ projectId: string+ platformId: string+ fileId: string+}++export type ReadFlowStepFileResponse = {+ data: Buffer+ mimeType?: string+ fileName?: string+}+export type SaveFlowStepFileResponse = {fileId: stringurl: stringdiff --git a/packages/core/execution/test/workers/ai-job-data.test.ts b/packages/core/execution/test/workers/ai-job-data.test.tsindex 20e029577c0..ae1a5e4f2b8 100644--- a/packages/core/execution/test/workers/ai-job-data.test.ts+++ b/packages/core/execution/test/workers/ai-job-data.test.ts@@ -44,13 +44,13 @@ describe('what an AI step job accepts', () => {...base,action: AiStepAction.EXTRACT_STRUCTURED_DATA,text: 'an invoice',- files: [{ mimeType: 'application/pdf', base64: 'eA==' }],+ files: [{ fileId: 'file-1', mimeType: 'application/pdf' }],schema: { mode: 'simple', fields: [{ name: 'total' }] },}).success).toBe(true)expect(ExecuteAiJobData.safeParse({...base,action: AiStepAction.GENERATE_IMAGE,- files: [{ mimeType: 'image/png', base64: 'eA==' }],+ files: [{ fileId: 'file-2', mimeType: 'image/png' }],advancedOptions: { size: '1024x1024' },}).success).toBe(true)})diff --git a/packages/server/api/src/app/ai/ai-rpc-handlers.ts b/packages/server/api/src/app/ai/ai-rpc-handlers.tsindex 83b006a88d0..7dfd3b8fda7 100644--- a/packages/server/api/src/app/ai/ai-rpc-handlers.ts+++ b/packages/server/api/src/app/ai/ai-rpc-handlers.ts@@ -1,5 +1,5 @@import { ActivepiecesError, ErrorCode, isNil, sanitizeObjectForPostgresql, tryCatch } from '@activepieces/core-utils'-import { FileCompression, FileSizeError, FileType, ResolveAiProviderRequest, ResolveAiProviderResponse, ResumeAiStepRequest, SaveFlowStepFileRequest, SaveFlowStepFileResponse, spreadIfDefined } from '@activepieces/shared'+import { FileCompression, FileSizeError, FileType, ReadFlowStepFileRequest, ReadFlowStepFileResponse, ResolveAiProviderRequest, ResolveAiProviderResponse, ResumeAiStepRequest, SaveFlowStepFileRequest, SaveFlowStepFileResponse, spreadIfDefined } from '@activepieces/shared'import { FastifyBaseLogger } from 'fastify'import { fileService } from '../file/file.service'import { filesService } from '../file/files-service'@@ -51,6 +51,20 @@ export const aiRpcHandlers = (log: FastifyBaseLogger) => ({return { fileId: file.id, url }},+ async readFlowStepFile(input: ReadFlowStepFileRequest): Promise<ReadFlowStepFileResponse> {+ await assertProjectBelongsToPlatform({ ...input, log })+ const file = await fileService(log).getDataOrThrow({+ projectId: input.projectId,+ fileId: input.fileId,+ type: FileType.FLOW_STEP_FILE,+ })+ return {+ data: file.data,+ ...spreadIfDefined('mimeType', file.metadata?.['mimeType']),+ ...spreadIfDefined('fileName', file.fileName),+ }+ },+async resumeAiStep(input: ResumeAiStepRequest): Promise<void> {const resumeFields = { flowRun: { id: input.flowRunId }, waitpoint: { id: input.waitpointId } }const flowRun = await flowRunService(log).getOne({ id: input.flowRunId, projectId: input.projectId })diff --git a/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts b/packages/server/api/src/app/workers/rpc/worker-rpc-service.tsindex 7e1cff0270d..4ebe7d1f753 100644--- a/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts+++ b/packages/server/api/src/app/workers/rpc/worker-rpc-service.ts@@ -400,6 +400,9 @@ export function createHandlers(log: FastifyBaseLogger, assignment: WorkerGroupAsasync saveFlowStepFile(input) {return aiRpcHandlers(log).saveFlowStepFile(input)},+ async readFlowStepFile(input) {+ return aiRpcHandlers(log).readFlowStepFile(input)+ },async resumeAiStep(input) {return aiRpcHandlers(log).resumeAiStep(input)},diff --git a/packages/server/worker/package.json b/packages/server/worker/package.jsonindex a82ee2f3f96..d6163f202cb 100644--- a/packages/server/worker/package.json+++ b/packages/server/worker/package.json@@ -29,6 +29,7 @@"dayjs": "1.11.9","env-var": "7.5.0","nanoid": "3.3.18",+ "p-limit": "2.3.0","proxy-chain": "2.7.1","socket.io": "4.7.5","dotenv": "16.4.7",diff --git a/packages/server/worker/src/lib/execute/jobs/ai/ai-files.ts b/packages/server/worker/src/lib/execute/jobs/ai/ai-files.tsnew file mode 100644index 00000000000..c2ed7c0f785--- /dev/null+++ b/packages/server/worker/src/lib/execute/jobs/ai/ai-files.ts@@ -0,0 +1,41 @@+import { spreadIfDefined } from '@activepieces/core-utils'+import { AiStepFile, ExtractStructuredDataJobData, GenerateImageJobData } from '@activepieces/shared'+import pLimit from 'p-limit'+import { JobContext } from '../../types'++export async function resolveAiFiles({ ctx, data }: {+ ctx: JobContext+ data: AiStepWithFiles+}): Promise<ResolvedAiFile[]> {+ const files = data.files ?? []+ const readAtOnce = pLimit(FILES_READ_AT_ONCE)+ return Promise.all(files.map((file) => readAtOnce(() => resolveAiFile({ ctx, data, file }))))+}++async function resolveAiFile({ ctx, data, file }: {+ ctx: JobContext+ data: AiStepWithFiles+ file: AiStepFile+}): Promise<ResolvedAiFile> {+ const stored = await ctx.apiClient.readFlowStepFile({+ projectId: data.projectId,+ platformId: data.platformId,+ fileId: file.fileId,+ })+ return {+ mimeType: file.mimeType ?? stored.mimeType ?? UNKNOWN_MIME_TYPE,+ base64: stored.data.toString('base64'),+ ...spreadIfDefined('filename', file.filename ?? stored.fileName),+ }+}++const UNKNOWN_MIME_TYPE = 'application/octet-stream'+const FILES_READ_AT_ONCE = 5++export type AiStepWithFiles = ExtractStructuredDataJobData | GenerateImageJobData++export type ResolvedAiFile = {+ mimeType: string+ base64: string+ filename?: string+}diff --git a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.tsindex 92d5b449d91..92c9e4c7b9b 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts@@ -3,6 +3,7 @@ import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'import { AiStepAction, ClassifyTextJobData, EngineResponseStatus, ExecuteAiJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse, WorkerJobType } from '@activepieces/shared'import { generateText, ModelMessage, stepCountIs } from 'ai'import { FireAndForgetJobResult, JobContext, JobHandler, JobResultKind } from '../../types'+import { resolveAiFiles } from './ai-files'import { extractStructuredData } from './extract-structured-data'import { generateImageStep } from './generate-image'@@ -56,9 +57,9 @@ async function runAiStep(ctx: JobContext, data: ExecuteAiJobData): Promise<unknoconst flowStep = flowStepMetadata(data)switch (data.action) {case AiStepAction.EXTRACT_STRUCTURED_DATA:- return { answer: await extractStructuredData({ data, resolved, flowStep }) }+ return { answer: await extractStructuredData({ data, resolved, flowStep, files: await resolveAiFiles({ ctx, data }) }) }case AiStepAction.GENERATE_IMAGE:- return { answer: await generateImageStep({ ctx, data, resolved, flowStep }) }+ return { answer: await generateImageStep({ ctx, data, resolved, flowStep, inputImages: await resolveAiFiles({ ctx, data }) }) }case AiStepAction.ASK_AI:case AiStepAction.SUMMARIZE_TEXT:case AiStepAction.CLASSIFY_TEXT:diff --git a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.tsindex df7b6c24731..2557b4ee6f1 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts@@ -1,14 +1,15 @@import { isNil, spreadIfDefined } from '@activepieces/core-utils'import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'-import { AiStepFile, ExtractStructuredDataJobData, ResolveAiProviderResponse } from '@activepieces/shared'+import { ExtractStructuredDataJobData, ResolveAiProviderResponse } from '@activepieces/shared'import { generateText, jsonSchema, ModelMessage, tool, UserModelMessage } from 'ai'+import { ResolvedAiFile } from './ai-files'-export async function extractStructuredData({ data, resolved, flowStep }: {+export async function extractStructuredData({ data, resolved, flowStep, files }: {data: ExtractStructuredDataJobDataresolved: ResolveAiProviderResponseflowStep: FlowStepMetadata+ files: ResolvedAiFile[]}): Promise<unknown> {- const files = data.files ?? []if (isNil(data.text) && files.length === 0) {throw new Error('Please provide text or image/PDF to extract data from.')}@@ -65,7 +66,7 @@ function buildSchema(data: ExtractStructuredDataJobData): { schemaDefinition: Rereturn { schemaDefinition: jsonSchema({ type: 'object', properties, required } as Parameters<typeof jsonSchema>[0]), sanitizedNameMap }}-function buildMessages({ data, files }: { data: ExtractStructuredDataJobData, files: AiStepFile[] }): ModelMessage[] {+function buildMessages({ data, files }: { data: ExtractStructuredDataJobData, files: ResolvedAiFile[] }): ModelMessage[] {let textContent = data.prompt ?? 'Extract the following data from the provided data.'if (data.text) {textContent += `\n\nText to analyze:\n${data.text}`diff --git a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.tsindex dbcec313e3c..3cb15121ca7 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts@@ -1,16 +1,17 @@import { AIProviderName, isNil } from '@activepieces/core-utils'import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'-import { AI_PROVIDER_CAPABILITIES, AiStepFile, GenerateImageJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse } from '@activepieces/shared'+import { AI_PROVIDER_CAPABILITIES, GenerateImageJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse } from '@activepieces/shared'import { generateImage, generateText, ImageModel, ImagePart, LanguageModel } from 'ai'import { JobContext } from '../../types'+import { ResolvedAiFile } from './ai-files'-export async function generateImageStep({ ctx, data, resolved, flowStep }: {+export async function generateImageStep({ ctx, data, resolved, flowStep, inputImages }: {ctx: JobContextdata: GenerateImageJobDataresolved: ResolveAiProviderResponseflowStep: FlowStepMetadata+ inputImages: ResolvedAiFile[]}): Promise<string> {- const inputImages = data.files ?? []const image = await getGeneratedImage({ data, resolved, inputImages, flowStep })const imageData = !isNil(image.base64) && image.base64.length > 0? Buffer.from(image.base64, 'base64')@@ -28,7 +29,7 @@ export async function generateImageStep({ ctx, data, resolved, flowStep }: {async function getGeneratedImage({ data, resolved, inputImages, flowStep }: {data: GenerateImageJobDataresolved: ResolveAiProviderResponse- inputImages: AiStepFile[]+ inputImages: ResolvedAiFile[]flowStep: FlowStepMetadata}): Promise<GeneratedImage> {const model = createImageCapableModel({ resolved, modelId: data.modelId, flowStep })@@ -77,7 +78,7 @@ function createImageCapableModel({ resolved, modelId, flowStep }: {async function generateImageUsingGenerateText({ model, prompt, inputImages }: {model: LanguageModelprompt: string- inputImages: AiStepFile[]+ inputImages: ResolvedAiFile[]}): Promise<GeneratedImage> {const imageParts = inputImages.map<ImagePart>((file) => ({type: 'image',diff --git a/packages/server/worker/test/lib/execute/jobs/ai/ai-files.test.ts b/packages/server/worker/test/lib/execute/jobs/ai/ai-files.test.tsnew file mode 100644index 00000000000..75190a9c2b8--- /dev/null+++ b/packages/server/worker/test/lib/execute/jobs/ai/ai-files.test.ts@@ -0,0 +1,144 @@+import { AiStepAction, ExecuteAiJobData, ReadFlowStepFileRequest, ReadFlowStepFileResponse } from '@activepieces/shared'+import { describe, expect, it } from 'vitest'+import { resolveAiFiles } from '../../../../../src/lib/execute/jobs/ai/ai-files'++const PDF_BYTES = Buffer.from('%PDF-1.7 a small invoice')++function contextReading(stored: Record<string, ReadFlowStepFileResponse>): Parameters<typeof resolveAiFiles>[0]['ctx'] {+ const requests: ReadFlowStepFileRequest[] = []+ const ctx = {+ apiClient: {+ readFlowStepFile: async (input: ReadFlowStepFileRequest) => {+ requests.push(input)+ const found = stored[input.fileId]+ if (found === undefined) {+ throw new Error(`no such file ${input.fileId}`)+ }+ return found+ },+ },+ requests,+ }+ return ctx as unknown as Parameters<typeof resolveAiFiles>[0]['ctx']+}++function jobData(files: ExecuteAiJobData['files']): ExecuteAiJobData {+ return {+ requestId: 'request-1',+ projectId: 'project-1',+ platformId: 'platform-1',+ flowId: 'flow-1',+ flowRunId: 'run-1',+ waitpointId: 'waitpoint-1',+ action: AiStepAction.EXTRACT_STRUCTURED_DATA,+ provider: 'openai',+ modelId: 'gpt-4o',+ files,+ } as ExecuteAiJobData+}++describe('resolveAiFiles', () => {+ it('fetches the bytes by id, scoped to the job project and platform', async () => {+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES, mimeType: 'application/pdf' } })++ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1' }]) })++ expect(resolved).toEqual([{ mimeType: 'application/pdf', base64: PDF_BYTES.toString('base64') }])+ expect((ctx as unknown as { requests: ReadFlowStepFileRequest[] }).requests).toEqual([+ { projectId: 'project-1', platformId: 'platform-1', fileId: 'file-1' },+ ])+ })++ it('prefers the mime type the caller sent over the one the file was stored with', async () => {+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES, mimeType: 'application/octet-stream' } })++ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1', mimeType: 'application/pdf' }]) })++ expect(resolved[0].mimeType).toBe('application/pdf')+ })++ it('falls back to the stored mime type when the caller sent none', async () => {+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES, mimeType: 'image/png' } })++ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1' }]) })++ expect(resolved[0].mimeType).toBe('image/png')+ })++ it('falls back to an unknown type rather than guessing, so a file of no known kind is skipped downstream', async () => {+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES } })++ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1' }]) })++ expect(resolved[0].mimeType).toBe('application/octet-stream')+ })++ it('keeps the stored file name when the caller did not name the file', async () => {+ const ctx = contextReading({ 'file-1': { data: PDF_BYTES, mimeType: 'application/pdf', fileName: 'invoice.pdf' } })++ const resolved = await resolveAiFiles({ ctx, data: jobData([{ fileId: 'file-1' }]) })++ expect(resolved[0].filename).toBe('invoice.pdf')+ })++ it('resolves several files in order, so a multi-file extract keeps its inputs aligned', async () => {+ const ctx = contextReading({+ 'file-1': { data: Buffer.from('one'), mimeType: 'application/pdf' },+ 'file-2': { data: Buffer.from('two'), mimeType: 'image/png' },+ 'file-3': { data: Buffer.from('three'), mimeType: 'application/pdf' },+ })++ const resolved = await resolveAiFiles({+ ctx,+ data: jobData([{ fileId: 'file-1' }, { fileId: 'file-2' }, { fileId: 'file-3' }]),+ })++ expect(resolved.map((file) => file.base64)).toEqual([+ Buffer.from('one').toString('base64'),+ Buffer.from('two').toString('base64'),+ Buffer.from('three').toString('base64'),+ ])+ })++ it('resolves to nothing when the step carries no files', async () => {+ const ctx = contextReading({})++ expect(await resolveAiFiles({ ctx, data: jobData(undefined) })).toEqual([])+ })+})++describe('how many files resolveAiFiles reads at once', () => {+ it('paces the reads instead of opening one call per attachment', async () => {+ let inFlight = 0+ let mostAtOnce = 0+ const ctx = {+ apiClient: {+ readFlowStepFile: async () => {+ inFlight += 1+ mostAtOnce = Math.max(mostAtOnce, inFlight)+ await new Promise((resolve) => setTimeout(resolve, 1))+ inFlight -= 1+ return { data: PDF_BYTES, mimeType: 'application/pdf' }+ },+ },+ } as unknown as Parameters<typeof resolveAiFiles>[0]['ctx']+ const files = Array.from({ length: 40 }, (_, index) => ({ fileId: `file-${index}` }))++ const resolved = await resolveAiFiles({ ctx, data: jobData(files) })++ expect(resolved).toHaveLength(40)+ expect(mostAtOnce).toBeLessThanOrEqual(5)+ })++ it('still returns every attachment, in the order they were given', async () => {+ const ctx = contextReading(Object.fromEntries(+ Array.from({ length: 12 }, (_, index) => [`file-${index}`, { data: Buffer.from(`file ${index}`), mimeType: 'application/pdf' }]),+ ))+ const files = Array.from({ length: 12 }, (_, index) => ({ fileId: `file-${index}` }))++ const resolved = await resolveAiFiles({ ctx, data: jobData(files) })++ expect(resolved.map((file) => Buffer.from(file.base64, 'base64').toString()))+ .toEqual(files.map((_, index) => `file ${index}`))+ })+})diff --git a/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts b/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.tsindex 579a7093443..58604370526 100644--- a/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts+++ b/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts@@ -72,6 +72,7 @@ describe('flowStep reaches every model built on the worker', () => {}),resolved: RESOLVED,flowStep: FLOW_STEP,+ files: [],})expect(createModelCalls).toHaveLength(1)@@ -90,6 +91,7 @@ describe('flowStep reaches every model built on the worker', () => {}),resolved: { ...RESOLVED, provider: AIProviderName.GOOGLE },flowStep: FLOW_STEP,+ inputImages: [],})expect(createModelCalls).toHaveLength(1)