Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

feat(ai): worker path for Extract Structured Data and Generate Image (#15489)

Abdul9d428ed78ee4
diff --git a/packages/core/ai-providers/src/index.ts b/packages/core/ai-providers/src/index.ts
index 18c864b6e89..1bb8e995d3a 100644
--- a/packages/core/ai-providers/src/index.ts
+++ b/packages/core/ai-providers/src/index.ts
@@ -1,4 +1,4 @@
-export { createLanguageModel, buildOpenAICompatibleHeaders } from './lib/create-language-model'
-export type { CreateLanguageModelParams, LanguageModelOptions } from './lib/create-language-model'
+export { createImageModel, createLanguageModel, buildOpenAICompatibleHeaders } from './lib/create-language-model'
+export type { CreateImageModelParams, CreateLanguageModelParams, ImageModelOptions, LanguageModelOptions } from './lib/create-language-model'
export { createCloudflareGatewayModel } from './lib/create-cloudflare-gateway-model'
export type { CreateCloudflareGatewayModelParams, CloudflareGatewayMetadata, CloudflareGatewayRouting } from './lib/create-cloudflare-gateway-model'
diff --git a/packages/core/ai-providers/src/lib/create-language-model.ts b/packages/core/ai-providers/src/lib/create-language-model.ts
index 287db8519c4..e7d4a32b95a 100644
--- a/packages/core/ai-providers/src/lib/create-language-model.ts
+++ b/packages/core/ai-providers/src/lib/create-language-model.ts
@@ -10,7 +10,7 @@ import { createGoogleGenerativeAI } from '@ai-sdk/google'
import { createOpenAI } from '@ai-sdk/openai'
import { createOpenAICompatible } from '@ai-sdk/openai-compatible'
import { createOpenRouter, OpenRouterChatSettings } from '@openrouter/ai-sdk-provider'
-import { LanguageModel } from 'ai'
+import { ImageModel, LanguageModel } from 'ai'
const VERTEX_MAAS_SUFFIX = '-maas'
const VERTEX_ANTHROPIC_PREFIX = 'claude'
@@ -107,6 +107,43 @@ export function createLanguageModel({ provider, auth, config, modelId, options =
}
}
+export function createImageModel({ provider, auth, config, modelId, options = {} }: CreateImageModelParams): ImageModel | undefined {
+ const observed = spreadIfDefined('fetch', observedProviderFetch(options.onOutcome))
+ switch (provider) {
+ case AIProviderName.OPENAI: {
+ const { apiKey } = auth as BaseAIProviderAuthConfig
+ return createOpenAI({ apiKey, ...observed }).imageModel(modelId)
+ }
+ case AIProviderName.AZURE: {
+ const { apiKey } = auth as BaseAIProviderAuthConfig
+ const { resourceName, apiVersion } = config as AzureProviderConfig
+ return createAzure({ resourceName, apiKey, apiVersion, ...observed }).imageModel(modelId)
+ }
+ case AIProviderName.BEDROCK: {
+ const { accessKeyId, secretAccessKey } = auth as BedrockProviderAuthConfig
+ const { region } = config as BedrockProviderConfig
+ return createAmazonBedrock({ region, accessKeyId, secretAccessKey, ...observed }).imageModel(modelId)
+ }
+ case AIProviderName.VERTEX: {
+ const { serviceAccountJson } = auth as VertexProviderAuthConfig
+ const { project, region } = config as VertexProviderConfig
+ return createVertex({ project, location: region, googleAuthOptions: { credentials: parseServiceAccount(serviceAccountJson) }, ...observed }).imageModel(modelId)
+ }
+ case AIProviderName.CUSTOM: {
+ const { apiKey } = auth as BaseAIProviderAuthConfig
+ const { apiKeyHeader, baseUrl, defaultHeaders } = config as OpenAICompatibleProviderConfig
+ return createOpenAICompatible({
+ name: 'openai-compatible',
+ baseURL: baseUrl,
+ headers: buildOpenAICompatibleHeaders({ apiKeyHeader, apiKey, defaultHeaders, extraHeaders: options.extraHeaders }),
+ ...observed,
+ }).imageModel(modelId)
+ }
+ default:
+ return undefined
+ }
+}
+
function vertexClientFor({ modelId }: { modelId: string }): typeof createVertex | typeof createVertexAnthropic | typeof createVertexMaas {
if (modelId.includes('/') || modelId.endsWith(VERTEX_MAAS_SUFFIX)) {
return createVertexMaas
@@ -176,6 +213,19 @@ export type LanguageModelOptions = {
extraHeaders?: Record<string, string>
}
+export type CreateImageModelParams = {
+ provider: AIProviderName
+ auth: unknown
+ config: unknown
+ modelId: string
+ options?: ImageModelOptions
+}
+
+export type ImageModelOptions = {
+ onOutcome?: ProviderOutcomeReporter
+ extraHeaders?: Record<string, string>
+}
+
export type CreateLanguageModelParams = {
provider: AIProviderName
auth: unknown
diff --git a/packages/core/execution/src/lib/workers/job-data.ts b/packages/core/execution/src/lib/workers/job-data.ts
index 88555d77bfb..05f799e96a3 100644
--- a/packages/core/execution/src/lib/workers/job-data.ts
+++ b/packages/core/execution/src/lib/workers/job-data.ts
@@ -374,6 +374,8 @@ export enum AiStepAction {
ASK_AI = 'ASK_AI',
SUMMARIZE_TEXT = 'SUMMARIZE_TEXT',
CLASSIFY_TEXT = 'CLASSIFY_TEXT',
+ EXTRACT_STRUCTURED_DATA = 'EXTRACT_STRUCTURED_DATA',
+ GENERATE_IMAGE = 'GENERATE_IMAGE',
}
export const AiStepWebSearchOptions = z.object({
@@ -395,6 +397,19 @@ export const AiStepWebSearch = z.object({
})
export type AiStepWebSearch = z.infer<typeof AiStepWebSearch>
+export const AiStepFile = z.object({
+ mimeType: z.string(),
+ base64: z.string(),
+ filename: z.string().optional(),
+})
+export type AiStepFile = z.infer<typeof AiStepFile>
+
+export const AiStepSchema = z.object({
+ mode: z.enum(['simple', 'advanced']),
+ fields: z.unknown(),
+})
+export type AiStepSchema = z.infer<typeof AiStepSchema>
+
const AiStepJobBase = z.object({
schemaVersion: z.number(),
jobType: z.literal(WorkerJobType.EXECUTE_AI),
@@ -407,7 +422,7 @@ const AiStepJobBase = z.object({
provider: z.enum(AIProviderName),
providerConfigId: z.string().optional(),
modelId: z.string(),
- prompt: z.string(),
+ prompt: z.string().optional(),
maxOutputTokens: z.number().optional(),
temperature: z.number().optional(),
webSearch: AiStepWebSearch.optional(),
@@ -432,10 +447,27 @@ export const ClassifyTextJobData = AiStepJobBase.extend({
})
export type ClassifyTextJobData = z.infer<typeof ClassifyTextJobData>
+export const ExtractStructuredDataJobData = AiStepJobBase.extend({
+ action: z.literal(AiStepAction.EXTRACT_STRUCTURED_DATA),
+ text: z.string().optional(),
+ files: z.array(AiStepFile).optional(),
+ schema: AiStepSchema.optional(),
+})
+export type ExtractStructuredDataJobData = z.infer<typeof ExtractStructuredDataJobData>
+
+export const GenerateImageJobData = AiStepJobBase.extend({
+ action: z.literal(AiStepAction.GENERATE_IMAGE),
+ files: z.array(AiStepFile).optional(),
+ advancedOptions: z.record(z.string(), z.unknown()).optional(),
+})
+export type GenerateImageJobData = z.infer<typeof GenerateImageJobData>
+
export const ExecuteAiJobData = z.discriminatedUnion('action', [
AskAiJobData,
SummarizeTextJobData,
ClassifyTextJobData,
+ ExtractStructuredDataJobData,
+ GenerateImageJobData,
])
export type ExecuteAiJobData = z.infer<typeof ExecuteAiJobData>
diff --git a/packages/core/execution/src/lib/workers/worker-contract.ts b/packages/core/execution/src/lib/workers/worker-contract.ts
index b3a04950f7f..eae72c40ef0 100644
--- a/packages/core/execution/src/lib/workers/worker-contract.ts
+++ b/packages/core/execution/src/lib/workers/worker-contract.ts
@@ -433,8 +433,7 @@ export type SaveFlowStepFileRequest = {
platformId: string
flowRunId: string
data: Buffer
- mediaType: string
- fileName?: string
+ fileName: string
}
export type SaveFlowStepFileResponse = {
diff --git a/packages/core/execution/test/workers/ai-job-data.test.ts b/packages/core/execution/test/workers/ai-job-data.test.ts
index 45c7762ab5a..20e029577c0 100644
--- a/packages/core/execution/test/workers/ai-job-data.test.ts
+++ b/packages/core/execution/test/workers/ai-job-data.test.ts
@@ -39,8 +39,27 @@ describe('what an AI step job accepts', () => {
expect(parsed).toMatchObject({ action: AiStepAction.CLASSIFY_TEXT, text: 'warm', categories: ['sunny', 'rainy'] })
})
- it('rejects what it rejected before: no prompt, or an action nobody ships', () => {
- expect(ExecuteAiJobData.safeParse({ ...base, action: AiStepAction.ASK_AI, prompt: undefined }).success).toBe(false)
+ it('takes the two actions that carry files and a schema', () => {
+ expect(ExecuteAiJobData.safeParse({
+ ...base,
+ action: AiStepAction.EXTRACT_STRUCTURED_DATA,
+ text: 'an invoice',
+ files: [{ mimeType: 'application/pdf', base64: 'eA==' }],
+ schema: { mode: 'simple', fields: [{ name: 'total' }] },
+ }).success).toBe(true)
+ expect(ExecuteAiJobData.safeParse({
+ ...base,
+ action: AiStepAction.GENERATE_IMAGE,
+ files: [{ mimeType: 'image/png', base64: 'eA==' }],
+ advancedOptions: { size: '1024x1024' },
+ }).success).toBe(true)
+ })
+
+ it('no longer needs a prompt, because Extract and Generate Image do not have one', () => {
+ expect(ExecuteAiJobData.safeParse({ ...base, action: AiStepAction.EXTRACT_STRUCTURED_DATA, prompt: undefined }).success).toBe(true)
+ })
+
+ it('still rejects an action nobody ships', () => {
expect(ExecuteAiJobData.safeParse({ ...base, action: 'TRANSLATE' }).success).toBe(false)
})
})
diff --git a/packages/server/api/src/app/ai/ai-execute-controller.ts b/packages/server/api/src/app/ai/ai-execute-controller.ts
index 4d08f9a02d7..559267e7a40 100644
--- a/packages/server/api/src/app/ai/ai-execute-controller.ts
+++ b/packages/server/api/src/app/ai/ai-execute-controller.ts
@@ -1,14 +1,18 @@
import { ActivepiecesError, AIProviderName, apId, ErrorCode } from '@activepieces/core-utils'
-import { AiStepAction, AiStepWebSearch, ExecuteAiJobData, LATEST_JOB_DATA_SCHEMA_VERSION, PrincipalType, spreadIfDefined, WorkerJobType } from '@activepieces/shared'
+import { AiStepAction, AiStepFile, AiStepSchema, AiStepWebSearch, ExecuteAiJobData, LATEST_JOB_DATA_SCHEMA_VERSION, maxSocketHttpBufferSizeBytes, PrincipalType, spreadIfDefined, WorkerJobType } from '@activepieces/shared'
import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'
import { StatusCodes } from 'http-status-codes'
import { z } from 'zod'
import { securityAccess } from '../core/security/authorization/fastify-security'
+import { system } from '../helper/system/system'
+import { AppSystemProp } from '../helper/system/system-props'
import { assertCreditsAndAppSumoNotExceeded } from '../platform/billing-provider'
import { jobQueue, JobType } from '../workers/job-queue/job-queue'
export const aiExecuteController: FastifyPluginAsyncZod = async (app) => {
- app.post('/execute', ExecuteAiRoute, async (request, reply) => {
+ const bodyLimit = maxSocketHttpBufferSizeBytes(system.getNumberOrThrow(AppSystemProp.MAX_FILE_SIZE_MB))
+
+ app.post('/execute', { ...ExecuteAiRoute, bodyLimit }, async (request, reply) => {
if (request.principal.type !== PrincipalType.ENGINE) {
throw new ActivepiecesError({
code: ErrorCode.AUTHORIZATION,
@@ -50,7 +54,7 @@ function aiJobFor({ body, requestId, projectId, platformId }: {
waitpointId: body.waitpointId,
provider: body.provider,
modelId: body.modelId,
- prompt: body.prompt,
+ ...spreadIfDefined('prompt', body.prompt),
...spreadIfDefined('providerConfigId', body.providerConfigId),
...spreadIfDefined('maxOutputTokens', body.maxOutputTokens),
...spreadIfDefined('temperature', body.temperature),
@@ -69,6 +73,21 @@ function aiJobFor({ body, requestId, projectId, platformId }: {
...spreadIfDefined('text', body.text),
...spreadIfDefined('categories', body.categories),
}
+ case AiStepAction.EXTRACT_STRUCTURED_DATA:
+ return {
+ ...shared,
+ action: AiStepAction.EXTRACT_STRUCTURED_DATA,
+ ...spreadIfDefined('text', body.text),
+ ...spreadIfDefined('files', body.files),
+ ...spreadIfDefined('schema', body.schema),
+ }
+ case AiStepAction.GENERATE_IMAGE:
+ return {
+ ...shared,
+ action: AiStepAction.GENERATE_IMAGE,
+ ...spreadIfDefined('files', body.files),
+ ...spreadIfDefined('advancedOptions', body.advancedOptions),
+ }
}
}
@@ -82,9 +101,12 @@ const ExecuteAiRequest = z.object({
provider: z.enum(AIProviderName),
providerConfigId: z.string().optional(),
modelId: z.string(),
- prompt: z.string(),
+ prompt: z.string().optional(),
text: z.string().optional(),
categories: z.array(z.string()).optional(),
+ files: z.array(AiStepFile).optional(),
+ schema: AiStepSchema.optional(),
+ advancedOptions: z.record(z.string(), z.unknown()).optional(),
conversation: z.array(z.record(z.string(), z.unknown())).optional(),
maxOutputTokens: z.number().optional(),
temperature: z.number().optional(),
diff --git a/packages/server/api/src/app/ai/ai-rpc-handlers.ts b/packages/server/api/src/app/ai/ai-rpc-handlers.ts
index e2ecb42f0a0..83b006a88d0 100644
--- a/packages/server/api/src/app/ai/ai-rpc-handlers.ts
+++ b/packages/server/api/src/app/ai/ai-rpc-handlers.ts
@@ -1,5 +1,5 @@
import { ActivepiecesError, ErrorCode, isNil, sanitizeObjectForPostgresql, tryCatch } from '@activepieces/core-utils'
-import { FileCompression, FileType, ResolveAiProviderRequest, ResolveAiProviderResponse, ResumeAiStepRequest, SaveFlowStepFileRequest, SaveFlowStepFileResponse, spreadIfDefined } from '@activepieces/shared'
+import { FileCompression, FileSizeError, FileType, ResolveAiProviderRequest, ResolveAiProviderResponse, ResumeAiStepRequest, SaveFlowStepFileRequest, SaveFlowStepFileResponse, spreadIfDefined } from '@activepieces/shared'
import { FastifyBaseLogger } from 'fastify'
import { fileService } from '../file/file.service'
import { filesService } from '../file/files-service'
@@ -29,12 +29,10 @@ export const aiRpcHandlers = (log: FastifyBaseLogger) => ({
async saveFlowStepFile(input: SaveFlowStepFileRequest): Promise<SaveFlowStepFileResponse> {
await assertProjectBelongsToPlatform({ ...input, log })
- const maxFileSizeInBytes = system.getNumberOrThrow(AppSystemProp.MAX_FILE_SIZE_MB) * 1024 * 1024
+ const maxFileSizeMb = system.getNumberOrThrow(AppSystemProp.MAX_FILE_SIZE_MB)
+ const maxFileSizeInBytes = maxFileSizeMb * BYTES_PER_MB
if (input.data.length > maxFileSizeInBytes) {
- throw new ActivepiecesError({
- code: ErrorCode.VALIDATION,
- params: { message: `This AI step produced a ${Math.ceil(input.data.length / 1024 / 1024)}MB file, over the ${maxFileSizeInBytes / 1024 / 1024}MB limit` },
- })
+ throw new FileSizeError(Math.ceil(input.data.length / BYTES_PER_MB * 100) / 100, maxFileSizeMb)
}
const file = await fileService(log).save({
projectId: input.projectId,
@@ -44,7 +42,6 @@ export const aiRpcHandlers = (log: FastifyBaseLogger) => ({
type: FileType.FLOW_STEP_FILE,
fileName: input.fileName,
compression: FileCompression.NONE,
- metadata: { mimetype: input.mediaType },
})
const url = await filesService.constructReadUrl({
fileId: file.id,
@@ -98,3 +95,4 @@ function isFlowRunGone(error: unknown): boolean {
}
const FLOW_RUN_ENTITY_TYPE = 'flow_run'
+const BYTES_PER_MB = 1024 * 1024
diff --git a/packages/server/utils/src/ai-utils.ts b/packages/server/utils/src/ai-utils.ts
index cad9f908006..44b10bb3063 100644
--- a/packages/server/utils/src/ai-utils.ts
+++ b/packages/server/utils/src/ai-utils.ts
@@ -1,5 +1,5 @@
import { AIProviderName, isNil, observedProviderFetch, ProviderOutcomeReporter, spreadIfDefined } from '@activepieces/core-utils';
-import { CloudflareGatewayMetadata, createCloudflareGatewayModel, createLanguageModel } from '@activepieces/ai-providers';
+import { CloudflareGatewayMetadata, createCloudflareGatewayModel, createImageModel, createLanguageModel } from '@activepieces/ai-providers';
import { AI_PROVIDER_CAPABILITIES, AIWebSearchMode, BaseAIProviderAuthConfig, getEffectiveProviderAndModel } from '@activepieces/shared';
import { createAnthropic } from '@ai-sdk/anthropic'
import { createAzure } from '@ai-sdk/azure'
@@ -7,7 +7,7 @@ import { createGoogleGenerativeAI } from '@ai-sdk/google'
import { createOpenAI } from '@ai-sdk/openai'
import { SharedV3ProviderOptions } from '@ai-sdk/provider'
import { createOpenRouter, OpenRouterChatSettings } from '@openrouter/ai-sdk-provider'
-import { EmbeddingModel, LanguageModel, ToolSet } from 'ai'
+import { EmbeddingModel, ImageModel, LanguageModel, ToolSet } from 'ai'
const DEFAULT_WEB_SEARCH_RESULTS = 5
const MIN_OPENROUTER_WEB_SEARCH_RESULTS = 1
@@ -165,6 +165,26 @@ function flowStepMetadataHeaders(flowStep?: FlowStepMetadata): Record<string, st
}
}
+function createModelForImages({ provider, auth, config, modelId, flowStep }: {
+ provider: AIProviderName
+ auth: Record<string, unknown>
+ config: Record<string, unknown>
+ modelId: string
+ flowStep?: FlowStepMetadata
+}): ImageModel | undefined {
+ if (provider === AIProviderName.CLOUDFLARE_GATEWAY) {
+ return createCloudflareGatewayModel({
+ auth,
+ config,
+ modelId,
+ isImage: true,
+ routing: isNil(flowStep) ? 'compat' : 'submodel',
+ ...spreadIfDefined('metadata', cloudflareGatewayMetadata(flowStep)),
+ })
+ }
+ return createImageModel({ provider, auth, config, modelId })
+}
+
function readStringField(source: Record<string, unknown>, key: string): string {
const value = source[key]
return typeof value === 'string' ? value : ''
@@ -238,6 +258,7 @@ const OPENROUTER_EMBEDDING_PROVIDER_OPTIONS: SharedV3ProviderOptions = {
export const aiUtils = {
createModel,
+ createModelForImages,
createEmbeddingModel,
toStorageEmbedding,
supportsWebSearch,
diff --git a/packages/server/utils/test/ai-utils.test.ts b/packages/server/utils/test/ai-utils.test.ts
index ff177b29ea9..0c438da1328 100644
--- a/packages/server/utils/test/ai-utils.test.ts
+++ b/packages/server/utils/test/ai-utils.test.ts
@@ -112,3 +112,38 @@ describe('aiUtils.buildWebSearchToolsOrThrow', () => {
expect(aiUtils.buildWebSearchToolsOrThrow({ provider: AIProviderName.AZURE, auth: { apiKey: 'key' }, webSearchEnabled: false })).toEqual({})
})
})
+
+describe('aiUtils.createModelForImages', () => {
+ const gatewayConfig = { accountId: 'account', gatewayId: 'gateway' }
+
+ it('builds an image model for a Cloudflare Gateway model, as the piece does', () => {
+ const model = aiUtils.createModelForImages({
+ provider: AIProviderName.CLOUDFLARE_GATEWAY,
+ auth: { apiKey: 'key' },
+ config: gatewayConfig,
+ modelId: 'openai/dall-e-3',
+ })
+
+ expect(model).toBeDefined()
+ })
+
+ it('builds an image model for a gateway submodel it has no dedicated branch for', () => {
+ const model = aiUtils.createModelForImages({
+ provider: AIProviderName.CLOUDFLARE_GATEWAY,
+ auth: { apiKey: 'key' },
+ config: gatewayConfig,
+ modelId: 'workers-ai/flux',
+ })
+
+ expect(model).toBeDefined()
+ })
+
+ it('returns nothing for a provider with no image model, so the caller can fall back to text', () => {
+ expect(aiUtils.createModelForImages({
+ provider: AIProviderName.GOOGLE,
+ auth: { apiKey: 'key' },
+ config: {},
+ modelId: 'gemini-2.5-flash',
+ })).toBeUndefined()
+ })
+})
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
index 5667f62f04b..3a5c44d8023 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
@@ -1,8 +1,10 @@
import { AIProviderName, isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils'
import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'
-import { AiStepAction, ClassifyTextJobData, EngineResponseStatus, ExecuteAiJobData, getEffectiveProviderAndModel, WorkerJobType } from '@activepieces/shared'
+import { AiStepAction, ClassifyTextJobData, EngineResponseStatus, ExecuteAiJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse, WorkerJobType } from '@activepieces/shared'
import { generateText, ModelMessage, stepCountIs } from 'ai'
import { FireAndForgetJobResult, JobContext, JobHandler, JobResultKind } from '../../types'
+import { extractStructuredData } from './extract-structured-data'
+import { generateImageStep } from './generate-image'
export const executeAiJob: JobHandler<ExecuteAiJobData, FireAndForgetJobResult> = {
jobType: WorkerJobType.EXECUTE_AI,
@@ -45,12 +47,26 @@ async function waitBeforeRetry(attempt: number): Promise<void> {
}
async function runAiStep(ctx: JobContext, data: ExecuteAiJobData): Promise<unknown> {
- const { provider, auth, config } = await ctx.apiClient.resolveAiProvider({
+ const resolved = await ctx.apiClient.resolveAiProvider({
projectId: data.projectId,
platformId: data.platformId,
provider: data.provider,
...spreadIfDefined('providerConfigId', data.providerConfigId),
})
+ switch (data.action) {
+ case AiStepAction.EXTRACT_STRUCTURED_DATA:
+ return { answer: await extractStructuredData({ data, resolved }) }
+ case AiStepAction.GENERATE_IMAGE:
+ return { answer: await generateImageStep({ ctx, data, resolved }) }
+ case AiStepAction.ASK_AI:
+ case AiStepAction.SUMMARIZE_TEXT:
+ case AiStepAction.CLASSIFY_TEXT:
+ return runTextStep({ data, resolved })
+ }
+}
+
+async function runTextStep({ data, resolved }: { data: ExecuteAiJobData, resolved: ResolveAiProviderResponse }): Promise<unknown> {
+ const { provider, auth, config } = resolved
const webSearchEnabled = data.webSearch?.enabled ?? false
const webSearchOptions = data.webSearch?.options
const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider, model: data.modelId })
@@ -91,12 +107,15 @@ function buildMessages(data: ExecuteAiJobData): ModelMessage[] {
switch (data.action) {
case AiStepAction.ASK_AI: {
const history = (data.conversation ?? []) as ModelMessage[]
- return [...history, { role: 'user', content: data.prompt }]
+ return [...history, { role: 'user', content: data.prompt ?? '' }]
}
case AiStepAction.SUMMARIZE_TEXT:
return [{ role: 'user', content: `${data.prompt} Summarize the following text : ${data.text ?? ''}` }]
case AiStepAction.CLASSIFY_TEXT:
return [{ role: 'user', content: classificationPrompt(data) }]
+ case AiStepAction.EXTRACT_STRUCTURED_DATA:
+ case AiStepAction.GENERATE_IMAGE:
+ throw new Error(`${data.action} does not build plain messages`)
}
}
@@ -110,7 +129,7 @@ function toStepOutput({ data, text, sources }: { data: ExecuteAiJobData, text: s
case AiStepAction.ASK_AI: {
const conversation = isNil(data.conversation) ? undefined : [
...(data.conversation as ModelMessage[]),
- { role: 'user' as const, content: data.prompt },
+ { role: 'user' as const, content: data.prompt ?? '' },
{ role: 'assistant' as const, content: text },
]
const answer = data.webSearch?.enabled === true && data.webSearch.options?.includeSources === true
@@ -127,6 +146,9 @@ function toStepOutput({ data, text, sources }: { data: ExecuteAiJobData, text: s
}
return { answer: label }
}
+ case AiStepAction.EXTRACT_STRUCTURED_DATA:
+ case AiStepAction.GENERATE_IMAGE:
+ throw new Error(`${data.action} returns its own output shape`)
}
}
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
new file mode 100644
index 00000000000..312ece2a8d9
--- /dev/null
+++ b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
@@ -0,0 +1,106 @@
+import { isNil, spreadIfDefined } from '@activepieces/core-utils'
+import { aiUtils } from '@activepieces/server-utils'
+import { AiStepFile, ExtractStructuredDataJobData, ResolveAiProviderResponse } from '@activepieces/shared'
+import { generateText, jsonSchema, ModelMessage, tool, UserModelMessage } from 'ai'
+
+export async function extractStructuredData({ data, resolved }: {
+ data: ExtractStructuredDataJobData
+ resolved: ResolveAiProviderResponse
+}): Promise<unknown> {
+ const files = data.files ?? []
+ if (isNil(data.text) && files.length === 0) {
+ throw new Error('Please provide text or image/PDF to extract data from.')
+ }
+ const model = aiUtils.createModel({
+ provider: resolved.provider,
+ auth: resolved.auth,
+ config: resolved.config,
+ modelId: data.modelId,
+ })
+ const { schemaDefinition, sanitizedNameMap } = buildSchema(data)
+ const extractionTool = tool({
+ description: 'Extract structured data from the provided content',
+ inputSchema: schemaDefinition,
+ execute: async (extracted) => extracted,
+ })
+
+ try {
+ const result = await generateText({
+ model,
+ ...spreadIfDefined('maxOutputTokens', data.maxOutputTokens),
+ tools: { extractData: extractionTool },
+ toolChoice: 'required',
+ messages: buildMessages({ data, files }),
+ })
+ const toolCalls = result.toolCalls
+ if (isNil(toolCalls) || toolCalls.length === 0) {
+ throw new Error('No structured data could be extracted from the input.')
+ }
+ return restoreFieldNames({ extracted: toolCalls[0].input, sanitizedNameMap })
+ }
+ catch (error) {
+ throw new Error(`Failed to extract structured data: ${error instanceof Error ? error.message : 'Unknown error'}`)
+ }
+}
+
+function buildSchema(data: ExtractStructuredDataJobData): { schemaDefinition: ReturnType<typeof jsonSchema>, sanitizedNameMap: Record<string, string> } {
+ const fields = data.schema?.fields
+ if (data.schema?.mode === 'advanced') {
+ return { schemaDefinition: jsonSchema(fields as Parameters<typeof jsonSchema>[0]), sanitizedNameMap: {} }
+ }
+ const simpleFields = (fields ?? []) as SimpleField[]
+ const sanitizedNameMap: Record<string, string> = {}
+ const properties: Record<string, { type: string, description?: string }> = {}
+ const required: string[] = []
+ for (const field of simpleFields) {
+ const sanitizedFieldName = field.name.replace(/[^a-zA-Z0-9_.-]/g, '_')
+ sanitizedNameMap[sanitizedFieldName] = field.name
+ properties[sanitizedFieldName] = { type: field.type, description: field.description }
+ if (field.isRequired) {
+ required.push(sanitizedFieldName)
+ }
+ }
+ return { schemaDefinition: jsonSchema({ type: 'object', properties, required } as Parameters<typeof jsonSchema>[0]), sanitizedNameMap }
+}
+
+function buildMessages({ data, files }: { data: ExtractStructuredDataJobData, files: AiStepFile[] }): ModelMessage[] {
+ let textContent = data.prompt ?? 'Extract the following data from the provided data.'
+ if (data.text) {
+ textContent += `\n\nText to analyze:\n${data.text}`
+ }
+ const contentParts: UserModelMessage['content'] = [{ type: 'text', text: textContent }]
+ for (const file of files) {
+ if (file.base64.length === 0) {
+ continue
+ }
+ if (file.mimeType.startsWith('image')) {
+ contentParts.push({ type: 'image', image: `data:${file.mimeType};base64,${file.base64}` })
+ }
+ else if (file.mimeType.startsWith('application/pdf')) {
+ contentParts.push({
+ type: 'file',
+ data: `data:${file.mimeType};base64,${file.base64}`,
+ mediaType: file.mimeType,
+ ...spreadIfDefined('filename', file.filename),
+ })
+ }
+ }
+ return [{ role: 'user', content: contentParts }]
+}
+
+function restoreFieldNames({ extracted, sanitizedNameMap }: { extracted: unknown, sanitizedNameMap: Record<string, string> }): unknown {
+ if (Object.keys(sanitizedNameMap).length === 0 || typeof extracted !== 'object' || extracted === null) {
+ return extracted
+ }
+ return Object.entries(extracted).reduce<Record<string, unknown>>((restored, [key, value]) => {
+ restored[sanitizedNameMap[key] ?? key] = value
+ return restored
+ }, {})
+}
+
+type SimpleField = {
+ name: string
+ description?: string
+ type: string
+ isRequired: boolean
+}
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts
new file mode 100644
index 00000000000..84bb18e7ab4
--- /dev/null
+++ b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts
@@ -0,0 +1,150 @@
+import { AIProviderName, isNil } from '@activepieces/core-utils'
+import { aiUtils } from '@activepieces/server-utils'
+import { AI_PROVIDER_CAPABILITIES, AiStepFile, GenerateImageJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse } from '@activepieces/shared'
+import { generateImage, generateText, ImageModel, ImagePart, LanguageModel } from 'ai'
+import { JobContext } from '../../types'
+
+export async function generateImageStep({ ctx, data, resolved }: {
+ ctx: JobContext
+ data: GenerateImageJobData
+ resolved: ResolveAiProviderResponse
+}): Promise<string> {
+ const inputImages = data.files ?? []
+ const image = await getGeneratedImage({ data, resolved, inputImages })
+ const imageData = !isNil(image.base64) && image.base64.length > 0
+ ? Buffer.from(image.base64, 'base64')
+ : Buffer.from(image.uint8Array)
+ const { url } = await ctx.apiClient.saveFlowStepFile({
+ projectId: data.projectId,
+ platformId: data.platformId,
+ flowRunId: data.flowRunId,
+ data: imageData,
+ fileName: 'image.png',
+ })
+ return url
+}
+
+async function getGeneratedImage({ data, resolved, inputImages }: {
+ data: GenerateImageJobData
+ resolved: ResolveAiProviderResponse
+ inputImages: AiStepFile[]
+}): Promise<GeneratedImage> {
+ const model = createImageCapableModel({ resolved, modelId: data.modelId })
+ const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider: resolved.provider, model: data.modelId })
+ const resolvedProvider = effectiveProvider ?? resolved.provider
+ const prompt = data.prompt ?? ''
+ const hasInputImages = inputImages.length > 0
+
+ return withImageInputErrorContext({ modelId: data.modelId, hasInputImages }, async () => {
+ if (GENERATES_IMAGES_THROUGH_TEXT.has(resolvedProvider)) {
+ if (model.kind !== 'language') {
+ throw new Error(`Model "${data.modelId}" resolves to ${resolvedProvider}, which generates images through text, but the provider returned an image-only model`)
+ }
+ return generateImageUsingGenerateText({ model: model.model, prompt, inputImages })
+ }
+ if (model.kind !== 'image') {
+ throw new Error(`Provider ${resolvedProvider} does not support image models`)
+ }
+ const { image } = await generateImage({
+ model: model.model,
+ prompt: hasInputImages
+ ? { text: prompt, images: inputImages.map((file) => Buffer.from(file.base64, 'base64')) }
+ : prompt,
+ providerOptions: { [resolvedProvider]: { ...stripLegacyImageField(data.advancedOptions) } } as Parameters<typeof generateImage>[0]['providerOptions'],
+ })
+ return image
+ })
+}
+
+function createImageCapableModel({ resolved, modelId }: {
+ resolved: ResolveAiProviderResponse
+ modelId: string
+}): ImageCapableModel {
+ if (!AI_PROVIDER_CAPABILITIES[resolved.provider].supportsImageGeneration) {
+ throw new Error(`Provider ${resolved.provider} does not support image models`)
+ }
+ const { provider, auth, config } = resolved
+ const imageModel = aiUtils.createModelForImages({ provider, auth, config, modelId })
+ if (!isNil(imageModel)) {
+ return { kind: 'image', model: imageModel }
+ }
+ return { kind: 'language', model: aiUtils.createModel({ provider, auth, config, modelId }) }
+}
+
+async function generateImageUsingGenerateText({ model, prompt, inputImages }: {
+ model: LanguageModel
+ prompt: string
+ inputImages: AiStepFile[]
+}): Promise<GeneratedImage> {
+ const imageParts = inputImages.map<ImagePart>((file) => ({
+ type: 'image',
+ image: `data:${file.mimeType};base64,${file.base64}`,
+ }))
+ const result = await generateText({
+ model,
+ providerOptions: {
+ google: { responseModalities: ['TEXT', 'IMAGE'] },
+ openrouter: { modalities: ['image', 'text'] },
+ },
+ messages: [{ role: 'user', content: [{ type: 'text', text: prompt }, ...imageParts] }],
+ })
+ assertImageGenerationSuccess(result)
+ return result.files[0]
+}
+
+async function withImageInputErrorContext<T>({ modelId, hasInputImages }: { modelId: string, hasInputImages: boolean }, run: () => Promise<T>): Promise<T> {
+ try {
+ return await run()
+ }
+ catch (error) {
+ if (!hasInputImages) {
+ throw error
+ }
+ const original = error instanceof Error ? error.message : String(error)
+ throw new Error(
+ `Image generation failed for model "${modelId}". ` +
+ 'This model may not support input images. Try a model that supports image editing — ' +
+ 'for example gpt-image-1, dall-e-2, or a Gemini Nano Banana model — or remove the input images. ' +
+ `Original error: ${original}`,
+ )
+ }
+}
+
+function assertImageGenerationSuccess(result: Awaited<ReturnType<typeof generateText>>): void {
+ const body = result.response.body
+ const responseBody = typeof body === 'object' && body !== null && 'candidates' in body ? body : { candidates: [] }
+ const responseCandidates = Array.isArray(responseBody.candidates) ? responseBody.candidates : []
+ for (const candidate of responseCandidates) {
+ if (candidate.finishReason !== 'STOP') {
+ throw new Error('Image generation failed Reason:\n ' + JSON.stringify(responseCandidates, null, 2))
+ }
+ }
+ if (isNil(result.files) || result.files.length === 0) {
+ throw new Error('No image generated')
+ }
+}
+
+function stripLegacyImageField(advancedOptions: Record<string, unknown> | undefined): Record<string, unknown> | undefined {
+ if (isNil(advancedOptions)) {
+ return advancedOptions
+ }
+ const { image: _legacy, ...rest } = advancedOptions
+ return rest
+}
+
+const GENERATES_IMAGES_THROUGH_TEXT: ReadonlySet<AIProviderName | string> = new Set([
+ AIProviderName.GOOGLE,
+ AIProviderName.ACTIVEPIECES,
+ AIProviderName.OPENROUTER,
+ AIProviderName.CLOUDFLARE_GATEWAY,
+])
+
+type ImageCapableModel =
+ | { kind: 'image', model: ImageModel }
+ | { kind: 'language', model: LanguageModel }
+
+type GeneratedImage = {
+ base64?: string
+ uint8Array: Uint8Array
+ mediaType: string
+}