AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
feat(ai): worker path for Extract Structured Data and Generate Image (#15489)
Abdul9d428ed78ee4
diff --git a/packages/core/ai-providers/src/index.ts b/packages/core/ai-providers/src/index.tsindex 18c864b6e89..1bb8e995d3a 100644--- a/packages/core/ai-providers/src/index.ts+++ b/packages/core/ai-providers/src/index.ts@@ -1,4 +1,4 @@-export { createLanguageModel, buildOpenAICompatibleHeaders } from './lib/create-language-model'-export type { CreateLanguageModelParams, LanguageModelOptions } from './lib/create-language-model'+export { createImageModel, createLanguageModel, buildOpenAICompatibleHeaders } from './lib/create-language-model'+export type { CreateImageModelParams, CreateLanguageModelParams, ImageModelOptions, LanguageModelOptions } from './lib/create-language-model'export { createCloudflareGatewayModel } from './lib/create-cloudflare-gateway-model'export type { CreateCloudflareGatewayModelParams, CloudflareGatewayMetadata, CloudflareGatewayRouting } from './lib/create-cloudflare-gateway-model'diff --git a/packages/core/ai-providers/src/lib/create-language-model.ts b/packages/core/ai-providers/src/lib/create-language-model.tsindex 287db8519c4..e7d4a32b95a 100644--- a/packages/core/ai-providers/src/lib/create-language-model.ts+++ b/packages/core/ai-providers/src/lib/create-language-model.ts@@ -10,7 +10,7 @@ import { createGoogleGenerativeAI } from '@ai-sdk/google'import { createOpenAI } from '@ai-sdk/openai'import { createOpenAICompatible } from '@ai-sdk/openai-compatible'import { createOpenRouter, OpenRouterChatSettings } from '@openrouter/ai-sdk-provider'-import { LanguageModel } from 'ai'+import { ImageModel, LanguageModel } from 'ai'const VERTEX_MAAS_SUFFIX = '-maas'const VERTEX_ANTHROPIC_PREFIX = 'claude'@@ -107,6 +107,43 @@ export function createLanguageModel({ provider, auth, config, modelId, options =}}+export function createImageModel({ provider, auth, config, modelId, options = {} }: CreateImageModelParams): ImageModel | undefined {+ const observed = spreadIfDefined('fetch', observedProviderFetch(options.onOutcome))+ switch (provider) {+ case AIProviderName.OPENAI: {+ const { apiKey } = auth as BaseAIProviderAuthConfig+ return createOpenAI({ apiKey, ...observed }).imageModel(modelId)+ }+ case AIProviderName.AZURE: {+ const { apiKey } = auth as BaseAIProviderAuthConfig+ const { resourceName, apiVersion } = config as AzureProviderConfig+ return createAzure({ resourceName, apiKey, apiVersion, ...observed }).imageModel(modelId)+ }+ case AIProviderName.BEDROCK: {+ const { accessKeyId, secretAccessKey } = auth as BedrockProviderAuthConfig+ const { region } = config as BedrockProviderConfig+ return createAmazonBedrock({ region, accessKeyId, secretAccessKey, ...observed }).imageModel(modelId)+ }+ case AIProviderName.VERTEX: {+ const { serviceAccountJson } = auth as VertexProviderAuthConfig+ const { project, region } = config as VertexProviderConfig+ return createVertex({ project, location: region, googleAuthOptions: { credentials: parseServiceAccount(serviceAccountJson) }, ...observed }).imageModel(modelId)+ }+ case AIProviderName.CUSTOM: {+ const { apiKey } = auth as BaseAIProviderAuthConfig+ const { apiKeyHeader, baseUrl, defaultHeaders } = config as OpenAICompatibleProviderConfig+ return createOpenAICompatible({+ name: 'openai-compatible',+ baseURL: baseUrl,+ headers: buildOpenAICompatibleHeaders({ apiKeyHeader, apiKey, defaultHeaders, extraHeaders: options.extraHeaders }),+ ...observed,+ }).imageModel(modelId)+ }+ default:+ return undefined+ }+}+function vertexClientFor({ modelId }: { modelId: string }): typeof createVertex | typeof createVertexAnthropic | typeof createVertexMaas {if (modelId.includes('/') || modelId.endsWith(VERTEX_MAAS_SUFFIX)) {return createVertexMaas@@ -176,6 +213,19 @@ export type LanguageModelOptions = {extraHeaders?: Record<string, string>}+export type CreateImageModelParams = {+ provider: AIProviderName+ auth: unknown+ config: unknown+ modelId: string+ options?: ImageModelOptions+}++export type ImageModelOptions = {+ onOutcome?: ProviderOutcomeReporter+ extraHeaders?: Record<string, string>+}+export type CreateLanguageModelParams = {provider: AIProviderNameauth: unknowndiff --git a/packages/core/execution/src/lib/workers/job-data.ts b/packages/core/execution/src/lib/workers/job-data.tsindex 88555d77bfb..05f799e96a3 100644--- a/packages/core/execution/src/lib/workers/job-data.ts+++ b/packages/core/execution/src/lib/workers/job-data.ts@@ -374,6 +374,8 @@ export enum AiStepAction {ASK_AI = 'ASK_AI',SUMMARIZE_TEXT = 'SUMMARIZE_TEXT',CLASSIFY_TEXT = 'CLASSIFY_TEXT',+ EXTRACT_STRUCTURED_DATA = 'EXTRACT_STRUCTURED_DATA',+ GENERATE_IMAGE = 'GENERATE_IMAGE',}export const AiStepWebSearchOptions = z.object({@@ -395,6 +397,19 @@ export const AiStepWebSearch = z.object({})export type AiStepWebSearch = z.infer<typeof AiStepWebSearch>+export const AiStepFile = z.object({+ mimeType: z.string(),+ base64: z.string(),+ filename: z.string().optional(),+})+export type AiStepFile = z.infer<typeof AiStepFile>++export const AiStepSchema = z.object({+ mode: z.enum(['simple', 'advanced']),+ fields: z.unknown(),+})+export type AiStepSchema = z.infer<typeof AiStepSchema>+const AiStepJobBase = z.object({schemaVersion: z.number(),jobType: z.literal(WorkerJobType.EXECUTE_AI),@@ -407,7 +422,7 @@ const AiStepJobBase = z.object({provider: z.enum(AIProviderName),providerConfigId: z.string().optional(),modelId: z.string(),- prompt: z.string(),+ prompt: z.string().optional(),maxOutputTokens: z.number().optional(),temperature: z.number().optional(),webSearch: AiStepWebSearch.optional(),@@ -432,10 +447,27 @@ export const ClassifyTextJobData = AiStepJobBase.extend({})export type ClassifyTextJobData = z.infer<typeof ClassifyTextJobData>+export const ExtractStructuredDataJobData = AiStepJobBase.extend({+ action: z.literal(AiStepAction.EXTRACT_STRUCTURED_DATA),+ text: z.string().optional(),+ files: z.array(AiStepFile).optional(),+ schema: AiStepSchema.optional(),+})+export type ExtractStructuredDataJobData = z.infer<typeof ExtractStructuredDataJobData>++export const GenerateImageJobData = AiStepJobBase.extend({+ action: z.literal(AiStepAction.GENERATE_IMAGE),+ files: z.array(AiStepFile).optional(),+ advancedOptions: z.record(z.string(), z.unknown()).optional(),+})+export type GenerateImageJobData = z.infer<typeof GenerateImageJobData>+export const ExecuteAiJobData = z.discriminatedUnion('action', [AskAiJobData,SummarizeTextJobData,ClassifyTextJobData,+ ExtractStructuredDataJobData,+ GenerateImageJobData,])export type ExecuteAiJobData = z.infer<typeof ExecuteAiJobData>diff --git a/packages/core/execution/src/lib/workers/worker-contract.ts b/packages/core/execution/src/lib/workers/worker-contract.tsindex b3a04950f7f..eae72c40ef0 100644--- a/packages/core/execution/src/lib/workers/worker-contract.ts+++ b/packages/core/execution/src/lib/workers/worker-contract.ts@@ -433,8 +433,7 @@ export type SaveFlowStepFileRequest = {platformId: stringflowRunId: stringdata: Buffer- mediaType: string- fileName?: string+ fileName: string}export type SaveFlowStepFileResponse = {diff --git a/packages/core/execution/test/workers/ai-job-data.test.ts b/packages/core/execution/test/workers/ai-job-data.test.tsindex 45c7762ab5a..20e029577c0 100644--- a/packages/core/execution/test/workers/ai-job-data.test.ts+++ b/packages/core/execution/test/workers/ai-job-data.test.ts@@ -39,8 +39,27 @@ describe('what an AI step job accepts', () => {expect(parsed).toMatchObject({ action: AiStepAction.CLASSIFY_TEXT, text: 'warm', categories: ['sunny', 'rainy'] })})- it('rejects what it rejected before: no prompt, or an action nobody ships', () => {- expect(ExecuteAiJobData.safeParse({ ...base, action: AiStepAction.ASK_AI, prompt: undefined }).success).toBe(false)+ it('takes the two actions that carry files and a schema', () => {+ expect(ExecuteAiJobData.safeParse({+ ...base,+ action: AiStepAction.EXTRACT_STRUCTURED_DATA,+ text: 'an invoice',+ files: [{ mimeType: 'application/pdf', base64: 'eA==' }],+ schema: { mode: 'simple', fields: [{ name: 'total' }] },+ }).success).toBe(true)+ expect(ExecuteAiJobData.safeParse({+ ...base,+ action: AiStepAction.GENERATE_IMAGE,+ files: [{ mimeType: 'image/png', base64: 'eA==' }],+ advancedOptions: { size: '1024x1024' },+ }).success).toBe(true)+ })++ it('no longer needs a prompt, because Extract and Generate Image do not have one', () => {+ expect(ExecuteAiJobData.safeParse({ ...base, action: AiStepAction.EXTRACT_STRUCTURED_DATA, prompt: undefined }).success).toBe(true)+ })++ it('still rejects an action nobody ships', () => {expect(ExecuteAiJobData.safeParse({ ...base, action: 'TRANSLATE' }).success).toBe(false)})})diff --git a/packages/server/api/src/app/ai/ai-execute-controller.ts b/packages/server/api/src/app/ai/ai-execute-controller.tsindex 4d08f9a02d7..559267e7a40 100644--- a/packages/server/api/src/app/ai/ai-execute-controller.ts+++ b/packages/server/api/src/app/ai/ai-execute-controller.ts@@ -1,14 +1,18 @@import { ActivepiecesError, AIProviderName, apId, ErrorCode } from '@activepieces/core-utils'-import { AiStepAction, AiStepWebSearch, ExecuteAiJobData, LATEST_JOB_DATA_SCHEMA_VERSION, PrincipalType, spreadIfDefined, WorkerJobType } from '@activepieces/shared'+import { AiStepAction, AiStepFile, AiStepSchema, AiStepWebSearch, ExecuteAiJobData, LATEST_JOB_DATA_SCHEMA_VERSION, maxSocketHttpBufferSizeBytes, PrincipalType, spreadIfDefined, WorkerJobType } from '@activepieces/shared'import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'import { StatusCodes } from 'http-status-codes'import { z } from 'zod'import { securityAccess } from '../core/security/authorization/fastify-security'+import { system } from '../helper/system/system'+import { AppSystemProp } from '../helper/system/system-props'import { assertCreditsAndAppSumoNotExceeded } from '../platform/billing-provider'import { jobQueue, JobType } from '../workers/job-queue/job-queue'export const aiExecuteController: FastifyPluginAsyncZod = async (app) => {- app.post('/execute', ExecuteAiRoute, async (request, reply) => {+ const bodyLimit = maxSocketHttpBufferSizeBytes(system.getNumberOrThrow(AppSystemProp.MAX_FILE_SIZE_MB))++ app.post('/execute', { ...ExecuteAiRoute, bodyLimit }, async (request, reply) => {if (request.principal.type !== PrincipalType.ENGINE) {throw new ActivepiecesError({code: ErrorCode.AUTHORIZATION,@@ -50,7 +54,7 @@ function aiJobFor({ body, requestId, projectId, platformId }: {waitpointId: body.waitpointId,provider: body.provider,modelId: body.modelId,- prompt: body.prompt,+ ...spreadIfDefined('prompt', body.prompt),...spreadIfDefined('providerConfigId', body.providerConfigId),...spreadIfDefined('maxOutputTokens', body.maxOutputTokens),...spreadIfDefined('temperature', body.temperature),@@ -69,6 +73,21 @@ function aiJobFor({ body, requestId, projectId, platformId }: {...spreadIfDefined('text', body.text),...spreadIfDefined('categories', body.categories),}+ case AiStepAction.EXTRACT_STRUCTURED_DATA:+ return {+ ...shared,+ action: AiStepAction.EXTRACT_STRUCTURED_DATA,+ ...spreadIfDefined('text', body.text),+ ...spreadIfDefined('files', body.files),+ ...spreadIfDefined('schema', body.schema),+ }+ case AiStepAction.GENERATE_IMAGE:+ return {+ ...shared,+ action: AiStepAction.GENERATE_IMAGE,+ ...spreadIfDefined('files', body.files),+ ...spreadIfDefined('advancedOptions', body.advancedOptions),+ }}}@@ -82,9 +101,12 @@ const ExecuteAiRequest = z.object({provider: z.enum(AIProviderName),providerConfigId: z.string().optional(),modelId: z.string(),- prompt: z.string(),+ prompt: z.string().optional(),text: z.string().optional(),categories: z.array(z.string()).optional(),+ files: z.array(AiStepFile).optional(),+ schema: AiStepSchema.optional(),+ advancedOptions: z.record(z.string(), z.unknown()).optional(),conversation: z.array(z.record(z.string(), z.unknown())).optional(),maxOutputTokens: z.number().optional(),temperature: z.number().optional(),diff --git a/packages/server/api/src/app/ai/ai-rpc-handlers.ts b/packages/server/api/src/app/ai/ai-rpc-handlers.tsindex e2ecb42f0a0..83b006a88d0 100644--- a/packages/server/api/src/app/ai/ai-rpc-handlers.ts+++ b/packages/server/api/src/app/ai/ai-rpc-handlers.ts@@ -1,5 +1,5 @@import { ActivepiecesError, ErrorCode, isNil, sanitizeObjectForPostgresql, tryCatch } from '@activepieces/core-utils'-import { FileCompression, FileType, ResolveAiProviderRequest, ResolveAiProviderResponse, ResumeAiStepRequest, SaveFlowStepFileRequest, SaveFlowStepFileResponse, spreadIfDefined } from '@activepieces/shared'+import { FileCompression, FileSizeError, FileType, ResolveAiProviderRequest, ResolveAiProviderResponse, ResumeAiStepRequest, SaveFlowStepFileRequest, SaveFlowStepFileResponse, spreadIfDefined } from '@activepieces/shared'import { FastifyBaseLogger } from 'fastify'import { fileService } from '../file/file.service'import { filesService } from '../file/files-service'@@ -29,12 +29,10 @@ export const aiRpcHandlers = (log: FastifyBaseLogger) => ({async saveFlowStepFile(input: SaveFlowStepFileRequest): Promise<SaveFlowStepFileResponse> {await assertProjectBelongsToPlatform({ ...input, log })- const maxFileSizeInBytes = system.getNumberOrThrow(AppSystemProp.MAX_FILE_SIZE_MB) * 1024 * 1024+ const maxFileSizeMb = system.getNumberOrThrow(AppSystemProp.MAX_FILE_SIZE_MB)+ const maxFileSizeInBytes = maxFileSizeMb * BYTES_PER_MBif (input.data.length > maxFileSizeInBytes) {- throw new ActivepiecesError({- code: ErrorCode.VALIDATION,- params: { message: `This AI step produced a ${Math.ceil(input.data.length / 1024 / 1024)}MB file, over the ${maxFileSizeInBytes / 1024 / 1024}MB limit` },- })+ throw new FileSizeError(Math.ceil(input.data.length / BYTES_PER_MB * 100) / 100, maxFileSizeMb)}const file = await fileService(log).save({projectId: input.projectId,@@ -44,7 +42,6 @@ export const aiRpcHandlers = (log: FastifyBaseLogger) => ({type: FileType.FLOW_STEP_FILE,fileName: input.fileName,compression: FileCompression.NONE,- metadata: { mimetype: input.mediaType },})const url = await filesService.constructReadUrl({fileId: file.id,@@ -98,3 +95,4 @@ function isFlowRunGone(error: unknown): boolean {}const FLOW_RUN_ENTITY_TYPE = 'flow_run'+const BYTES_PER_MB = 1024 * 1024diff --git a/packages/server/utils/src/ai-utils.ts b/packages/server/utils/src/ai-utils.tsindex cad9f908006..44b10bb3063 100644--- a/packages/server/utils/src/ai-utils.ts+++ b/packages/server/utils/src/ai-utils.ts@@ -1,5 +1,5 @@import { AIProviderName, isNil, observedProviderFetch, ProviderOutcomeReporter, spreadIfDefined } from '@activepieces/core-utils';-import { CloudflareGatewayMetadata, createCloudflareGatewayModel, createLanguageModel } from '@activepieces/ai-providers';+import { CloudflareGatewayMetadata, createCloudflareGatewayModel, createImageModel, createLanguageModel } from '@activepieces/ai-providers';import { AI_PROVIDER_CAPABILITIES, AIWebSearchMode, BaseAIProviderAuthConfig, getEffectiveProviderAndModel } from '@activepieces/shared';import { createAnthropic } from '@ai-sdk/anthropic'import { createAzure } from '@ai-sdk/azure'@@ -7,7 +7,7 @@ import { createGoogleGenerativeAI } from '@ai-sdk/google'import { createOpenAI } from '@ai-sdk/openai'import { SharedV3ProviderOptions } from '@ai-sdk/provider'import { createOpenRouter, OpenRouterChatSettings } from '@openrouter/ai-sdk-provider'-import { EmbeddingModel, LanguageModel, ToolSet } from 'ai'+import { EmbeddingModel, ImageModel, LanguageModel, ToolSet } from 'ai'const DEFAULT_WEB_SEARCH_RESULTS = 5const MIN_OPENROUTER_WEB_SEARCH_RESULTS = 1@@ -165,6 +165,26 @@ function flowStepMetadataHeaders(flowStep?: FlowStepMetadata): Record<string, st}}+function createModelForImages({ provider, auth, config, modelId, flowStep }: {+ provider: AIProviderName+ auth: Record<string, unknown>+ config: Record<string, unknown>+ modelId: string+ flowStep?: FlowStepMetadata+}): ImageModel | undefined {+ if (provider === AIProviderName.CLOUDFLARE_GATEWAY) {+ return createCloudflareGatewayModel({+ auth,+ config,+ modelId,+ isImage: true,+ routing: isNil(flowStep) ? 'compat' : 'submodel',+ ...spreadIfDefined('metadata', cloudflareGatewayMetadata(flowStep)),+ })+ }+ return createImageModel({ provider, auth, config, modelId })+}+function readStringField(source: Record<string, unknown>, key: string): string {const value = source[key]return typeof value === 'string' ? value : ''@@ -238,6 +258,7 @@ const OPENROUTER_EMBEDDING_PROVIDER_OPTIONS: SharedV3ProviderOptions = {export const aiUtils = {createModel,+ createModelForImages,createEmbeddingModel,toStorageEmbedding,supportsWebSearch,diff --git a/packages/server/utils/test/ai-utils.test.ts b/packages/server/utils/test/ai-utils.test.tsindex ff177b29ea9..0c438da1328 100644--- a/packages/server/utils/test/ai-utils.test.ts+++ b/packages/server/utils/test/ai-utils.test.ts@@ -112,3 +112,38 @@ describe('aiUtils.buildWebSearchToolsOrThrow', () => {expect(aiUtils.buildWebSearchToolsOrThrow({ provider: AIProviderName.AZURE, auth: { apiKey: 'key' }, webSearchEnabled: false })).toEqual({})})})++describe('aiUtils.createModelForImages', () => {+ const gatewayConfig = { accountId: 'account', gatewayId: 'gateway' }++ it('builds an image model for a Cloudflare Gateway model, as the piece does', () => {+ const model = aiUtils.createModelForImages({+ provider: AIProviderName.CLOUDFLARE_GATEWAY,+ auth: { apiKey: 'key' },+ config: gatewayConfig,+ modelId: 'openai/dall-e-3',+ })++ expect(model).toBeDefined()+ })++ it('builds an image model for a gateway submodel it has no dedicated branch for', () => {+ const model = aiUtils.createModelForImages({+ provider: AIProviderName.CLOUDFLARE_GATEWAY,+ auth: { apiKey: 'key' },+ config: gatewayConfig,+ modelId: 'workers-ai/flux',+ })++ expect(model).toBeDefined()+ })++ it('returns nothing for a provider with no image model, so the caller can fall back to text', () => {+ expect(aiUtils.createModelForImages({+ provider: AIProviderName.GOOGLE,+ auth: { apiKey: 'key' },+ config: {},+ modelId: 'gemini-2.5-flash',+ })).toBeUndefined()+ })+})diff --git a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.tsindex 5667f62f04b..3a5c44d8023 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts@@ -1,8 +1,10 @@import { AIProviderName, isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils'import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'-import { AiStepAction, ClassifyTextJobData, EngineResponseStatus, ExecuteAiJobData, getEffectiveProviderAndModel, WorkerJobType } from '@activepieces/shared'+import { AiStepAction, ClassifyTextJobData, EngineResponseStatus, ExecuteAiJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse, WorkerJobType } from '@activepieces/shared'import { generateText, ModelMessage, stepCountIs } from 'ai'import { FireAndForgetJobResult, JobContext, JobHandler, JobResultKind } from '../../types'+import { extractStructuredData } from './extract-structured-data'+import { generateImageStep } from './generate-image'export const executeAiJob: JobHandler<ExecuteAiJobData, FireAndForgetJobResult> = {jobType: WorkerJobType.EXECUTE_AI,@@ -45,12 +47,26 @@ async function waitBeforeRetry(attempt: number): Promise<void> {}async function runAiStep(ctx: JobContext, data: ExecuteAiJobData): Promise<unknown> {- const { provider, auth, config } = await ctx.apiClient.resolveAiProvider({+ const resolved = await ctx.apiClient.resolveAiProvider({projectId: data.projectId,platformId: data.platformId,provider: data.provider,...spreadIfDefined('providerConfigId', data.providerConfigId),})+ switch (data.action) {+ case AiStepAction.EXTRACT_STRUCTURED_DATA:+ return { answer: await extractStructuredData({ data, resolved }) }+ case AiStepAction.GENERATE_IMAGE:+ return { answer: await generateImageStep({ ctx, data, resolved }) }+ case AiStepAction.ASK_AI:+ case AiStepAction.SUMMARIZE_TEXT:+ case AiStepAction.CLASSIFY_TEXT:+ return runTextStep({ data, resolved })+ }+}++async function runTextStep({ data, resolved }: { data: ExecuteAiJobData, resolved: ResolveAiProviderResponse }): Promise<unknown> {+ const { provider, auth, config } = resolvedconst webSearchEnabled = data.webSearch?.enabled ?? falseconst webSearchOptions = data.webSearch?.optionsconst { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider, model: data.modelId })@@ -91,12 +107,15 @@ function buildMessages(data: ExecuteAiJobData): ModelMessage[] {switch (data.action) {case AiStepAction.ASK_AI: {const history = (data.conversation ?? []) as ModelMessage[]- return [...history, { role: 'user', content: data.prompt }]+ return [...history, { role: 'user', content: data.prompt ?? '' }]}case AiStepAction.SUMMARIZE_TEXT:return [{ role: 'user', content: `${data.prompt} Summarize the following text : ${data.text ?? ''}` }]case AiStepAction.CLASSIFY_TEXT:return [{ role: 'user', content: classificationPrompt(data) }]+ case AiStepAction.EXTRACT_STRUCTURED_DATA:+ case AiStepAction.GENERATE_IMAGE:+ throw new Error(`${data.action} does not build plain messages`)}}@@ -110,7 +129,7 @@ function toStepOutput({ data, text, sources }: { data: ExecuteAiJobData, text: scase AiStepAction.ASK_AI: {const conversation = isNil(data.conversation) ? undefined : [...(data.conversation as ModelMessage[]),- { role: 'user' as const, content: data.prompt },+ { role: 'user' as const, content: data.prompt ?? '' },{ role: 'assistant' as const, content: text },]const answer = data.webSearch?.enabled === true && data.webSearch.options?.includeSources === true@@ -127,6 +146,9 @@ function toStepOutput({ data, text, sources }: { data: ExecuteAiJobData, text: s}return { answer: label }}+ case AiStepAction.EXTRACT_STRUCTURED_DATA:+ case AiStepAction.GENERATE_IMAGE:+ throw new Error(`${data.action} returns its own output shape`)}}diff --git a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.tsnew file mode 100644index 00000000000..312ece2a8d9--- /dev/null+++ b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts@@ -0,0 +1,106 @@+import { isNil, spreadIfDefined } from '@activepieces/core-utils'+import { aiUtils } from '@activepieces/server-utils'+import { AiStepFile, ExtractStructuredDataJobData, ResolveAiProviderResponse } from '@activepieces/shared'+import { generateText, jsonSchema, ModelMessage, tool, UserModelMessage } from 'ai'++export async function extractStructuredData({ data, resolved }: {+ data: ExtractStructuredDataJobData+ resolved: ResolveAiProviderResponse+}): Promise<unknown> {+ const files = data.files ?? []+ if (isNil(data.text) && files.length === 0) {+ throw new Error('Please provide text or image/PDF to extract data from.')+ }+ const model = aiUtils.createModel({+ provider: resolved.provider,+ auth: resolved.auth,+ config: resolved.config,+ modelId: data.modelId,+ })+ const { schemaDefinition, sanitizedNameMap } = buildSchema(data)+ const extractionTool = tool({+ description: 'Extract structured data from the provided content',+ inputSchema: schemaDefinition,+ execute: async (extracted) => extracted,+ })++ try {+ const result = await generateText({+ model,+ ...spreadIfDefined('maxOutputTokens', data.maxOutputTokens),+ tools: { extractData: extractionTool },+ toolChoice: 'required',+ messages: buildMessages({ data, files }),+ })+ const toolCalls = result.toolCalls+ if (isNil(toolCalls) || toolCalls.length === 0) {+ throw new Error('No structured data could be extracted from the input.')+ }+ return restoreFieldNames({ extracted: toolCalls[0].input, sanitizedNameMap })+ }+ catch (error) {+ throw new Error(`Failed to extract structured data: ${error instanceof Error ? error.message : 'Unknown error'}`)+ }+}++function buildSchema(data: ExtractStructuredDataJobData): { schemaDefinition: ReturnType<typeof jsonSchema>, sanitizedNameMap: Record<string, string> } {+ const fields = data.schema?.fields+ if (data.schema?.mode === 'advanced') {+ return { schemaDefinition: jsonSchema(fields as Parameters<typeof jsonSchema>[0]), sanitizedNameMap: {} }+ }+ const simpleFields = (fields ?? []) as SimpleField[]+ const sanitizedNameMap: Record<string, string> = {}+ const properties: Record<string, { type: string, description?: string }> = {}+ const required: string[] = []+ for (const field of simpleFields) {+ const sanitizedFieldName = field.name.replace(/[^a-zA-Z0-9_.-]/g, '_')+ sanitizedNameMap[sanitizedFieldName] = field.name+ properties[sanitizedFieldName] = { type: field.type, description: field.description }+ if (field.isRequired) {+ required.push(sanitizedFieldName)+ }+ }+ return { schemaDefinition: jsonSchema({ type: 'object', properties, required } as Parameters<typeof jsonSchema>[0]), sanitizedNameMap }+}++function buildMessages({ data, files }: { data: ExtractStructuredDataJobData, files: AiStepFile[] }): ModelMessage[] {+ let textContent = data.prompt ?? 'Extract the following data from the provided data.'+ if (data.text) {+ textContent += `\n\nText to analyze:\n${data.text}`+ }+ const contentParts: UserModelMessage['content'] = [{ type: 'text', text: textContent }]+ for (const file of files) {+ if (file.base64.length === 0) {+ continue+ }+ if (file.mimeType.startsWith('image')) {+ contentParts.push({ type: 'image', image: `data:${file.mimeType};base64,${file.base64}` })+ }+ else if (file.mimeType.startsWith('application/pdf')) {+ contentParts.push({+ type: 'file',+ data: `data:${file.mimeType};base64,${file.base64}`,+ mediaType: file.mimeType,+ ...spreadIfDefined('filename', file.filename),+ })+ }+ }+ return [{ role: 'user', content: contentParts }]+}++function restoreFieldNames({ extracted, sanitizedNameMap }: { extracted: unknown, sanitizedNameMap: Record<string, string> }): unknown {+ if (Object.keys(sanitizedNameMap).length === 0 || typeof extracted !== 'object' || extracted === null) {+ return extracted+ }+ return Object.entries(extracted).reduce<Record<string, unknown>>((restored, [key, value]) => {+ restored[sanitizedNameMap[key] ?? key] = value+ return restored+ }, {})+}++type SimpleField = {+ name: string+ description?: string+ type: string+ isRequired: boolean+}diff --git a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.tsnew file mode 100644index 00000000000..84bb18e7ab4--- /dev/null+++ b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts@@ -0,0 +1,150 @@+import { AIProviderName, isNil } from '@activepieces/core-utils'+import { aiUtils } from '@activepieces/server-utils'+import { AI_PROVIDER_CAPABILITIES, AiStepFile, GenerateImageJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse } from '@activepieces/shared'+import { generateImage, generateText, ImageModel, ImagePart, LanguageModel } from 'ai'+import { JobContext } from '../../types'++export async function generateImageStep({ ctx, data, resolved }: {+ ctx: JobContext+ data: GenerateImageJobData+ resolved: ResolveAiProviderResponse+}): Promise<string> {+ const inputImages = data.files ?? []+ const image = await getGeneratedImage({ data, resolved, inputImages })+ const imageData = !isNil(image.base64) && image.base64.length > 0+ ? Buffer.from(image.base64, 'base64')+ : Buffer.from(image.uint8Array)+ const { url } = await ctx.apiClient.saveFlowStepFile({+ projectId: data.projectId,+ platformId: data.platformId,+ flowRunId: data.flowRunId,+ data: imageData,+ fileName: 'image.png',+ })+ return url+}++async function getGeneratedImage({ data, resolved, inputImages }: {+ data: GenerateImageJobData+ resolved: ResolveAiProviderResponse+ inputImages: AiStepFile[]+}): Promise<GeneratedImage> {+ const model = createImageCapableModel({ resolved, modelId: data.modelId })+ const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider: resolved.provider, model: data.modelId })+ const resolvedProvider = effectiveProvider ?? resolved.provider+ const prompt = data.prompt ?? ''+ const hasInputImages = inputImages.length > 0++ return withImageInputErrorContext({ modelId: data.modelId, hasInputImages }, async () => {+ if (GENERATES_IMAGES_THROUGH_TEXT.has(resolvedProvider)) {+ if (model.kind !== 'language') {+ throw new Error(`Model "${data.modelId}" resolves to ${resolvedProvider}, which generates images through text, but the provider returned an image-only model`)+ }+ return generateImageUsingGenerateText({ model: model.model, prompt, inputImages })+ }+ if (model.kind !== 'image') {+ throw new Error(`Provider ${resolvedProvider} does not support image models`)+ }+ const { image } = await generateImage({+ model: model.model,+ prompt: hasInputImages+ ? { text: prompt, images: inputImages.map((file) => Buffer.from(file.base64, 'base64')) }+ : prompt,+ providerOptions: { [resolvedProvider]: { ...stripLegacyImageField(data.advancedOptions) } } as Parameters<typeof generateImage>[0]['providerOptions'],+ })+ return image+ })+}++function createImageCapableModel({ resolved, modelId }: {+ resolved: ResolveAiProviderResponse+ modelId: string+}): ImageCapableModel {+ if (!AI_PROVIDER_CAPABILITIES[resolved.provider].supportsImageGeneration) {+ throw new Error(`Provider ${resolved.provider} does not support image models`)+ }+ const { provider, auth, config } = resolved+ const imageModel = aiUtils.createModelForImages({ provider, auth, config, modelId })+ if (!isNil(imageModel)) {+ return { kind: 'image', model: imageModel }+ }+ return { kind: 'language', model: aiUtils.createModel({ provider, auth, config, modelId }) }+}++async function generateImageUsingGenerateText({ model, prompt, inputImages }: {+ model: LanguageModel+ prompt: string+ inputImages: AiStepFile[]+}): Promise<GeneratedImage> {+ const imageParts = inputImages.map<ImagePart>((file) => ({+ type: 'image',+ image: `data:${file.mimeType};base64,${file.base64}`,+ }))+ const result = await generateText({+ model,+ providerOptions: {+ google: { responseModalities: ['TEXT', 'IMAGE'] },+ openrouter: { modalities: ['image', 'text'] },+ },+ messages: [{ role: 'user', content: [{ type: 'text', text: prompt }, ...imageParts] }],+ })+ assertImageGenerationSuccess(result)+ return result.files[0]+}++async function withImageInputErrorContext<T>({ modelId, hasInputImages }: { modelId: string, hasInputImages: boolean }, run: () => Promise<T>): Promise<T> {+ try {+ return await run()+ }+ catch (error) {+ if (!hasInputImages) {+ throw error+ }+ const original = error instanceof Error ? error.message : String(error)+ throw new Error(+ `Image generation failed for model "${modelId}". ` ++ 'This model may not support input images. Try a model that supports image editing — ' ++ 'for example gpt-image-1, dall-e-2, or a Gemini Nano Banana model — or remove the input images. ' ++ `Original error: ${original}`,+ )+ }+}++function assertImageGenerationSuccess(result: Awaited<ReturnType<typeof generateText>>): void {+ const body = result.response.body+ const responseBody = typeof body === 'object' && body !== null && 'candidates' in body ? body : { candidates: [] }+ const responseCandidates = Array.isArray(responseBody.candidates) ? responseBody.candidates : []+ for (const candidate of responseCandidates) {+ if (candidate.finishReason !== 'STOP') {+ throw new Error('Image generation failed Reason:\n ' + JSON.stringify(responseCandidates, null, 2))+ }+ }+ if (isNil(result.files) || result.files.length === 0) {+ throw new Error('No image generated')+ }+}++function stripLegacyImageField(advancedOptions: Record<string, unknown> | undefined): Record<string, unknown> | undefined {+ if (isNil(advancedOptions)) {+ return advancedOptions+ }+ const { image: _legacy, ...rest } = advancedOptions+ return rest+}++const GENERATES_IMAGES_THROUGH_TEXT: ReadonlySet<AIProviderName | string> = new Set([+ AIProviderName.GOOGLE,+ AIProviderName.ACTIVEPIECES,+ AIProviderName.OPENROUTER,+ AIProviderName.CLOUDFLARE_GATEWAY,+])++type ImageCapableModel =+ | { kind: 'image', model: ImageModel }+ | { kind: 'language', model: LanguageModel }++type GeneratedImage = {+ base64?: string+ uint8Array: Uint8Array+ mediaType: string+}