AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(ai): stop Extract and Generate Image reaching the wrong AI provider (#15490)
Abdul420f5f6c260f
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.tsindex 3a5c44d8023..92d5b449d91 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts@@ -53,19 +53,20 @@ async function runAiStep(ctx: JobContext, data: ExecuteAiJobData): Promise<unknoprovider: data.provider,...spreadIfDefined('providerConfigId', data.providerConfigId),})+ const flowStep = flowStepMetadata(data)switch (data.action) {case AiStepAction.EXTRACT_STRUCTURED_DATA:- return { answer: await extractStructuredData({ data, resolved }) }+ return { answer: await extractStructuredData({ data, resolved, flowStep }) }case AiStepAction.GENERATE_IMAGE:- return { answer: await generateImageStep({ ctx, data, resolved }) }+ return { answer: await generateImageStep({ ctx, data, resolved, flowStep }) }case AiStepAction.ASK_AI:case AiStepAction.SUMMARIZE_TEXT:case AiStepAction.CLASSIFY_TEXT:- return runTextStep({ data, resolved })+ return runTextStep({ data, resolved, flowStep })}}-async function runTextStep({ data, resolved }: { data: ExecuteAiJobData, resolved: ResolveAiProviderResponse }): Promise<unknown> {+async function runTextStep({ data, resolved, flowStep }: { data: ExecuteAiJobData, resolved: ResolveAiProviderResponse, flowStep: FlowStepMetadata }): Promise<unknown> {const { provider, auth, config } = resolvedconst webSearchEnabled = data.webSearch?.enabled ?? falseconst webSearchOptions = data.webSearch?.options@@ -76,7 +77,7 @@ async function runTextStep({ data, resolved }: { data: ExecuteAiJobData, resolveauth,config,modelId: data.modelId,- flowStep: flowStepMetadata(data),+ flowStep,openaiResponsesModel: webSearchEnabled && (effectiveProvider ?? provider) === AIProviderName.OPENAI,webSearchEnabled,webSearchOptions,diff --git a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.tsindex 312ece2a8d9..df7b6c24731 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts@@ -1,11 +1,12 @@import { isNil, spreadIfDefined } from '@activepieces/core-utils'-import { aiUtils } from '@activepieces/server-utils'+import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'import { AiStepFile, ExtractStructuredDataJobData, ResolveAiProviderResponse } from '@activepieces/shared'import { generateText, jsonSchema, ModelMessage, tool, UserModelMessage } from 'ai'-export async function extractStructuredData({ data, resolved }: {+export async function extractStructuredData({ data, resolved, flowStep }: {data: ExtractStructuredDataJobDataresolved: ResolveAiProviderResponse+ flowStep: FlowStepMetadata}): Promise<unknown> {const files = data.files ?? []if (isNil(data.text) && files.length === 0) {@@ -16,6 +17,7 @@ export async function extractStructuredData({ data, resolved }: {auth: resolved.auth,config: resolved.config,modelId: data.modelId,+ flowStep,})const { schemaDefinition, sanitizedNameMap } = buildSchema(data)const extractionTool = tool({diff --git a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.tsindex 84bb18e7ab4..dbcec313e3c 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts@@ -1,16 +1,17 @@import { AIProviderName, isNil } from '@activepieces/core-utils'-import { aiUtils } from '@activepieces/server-utils'+import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'import { AI_PROVIDER_CAPABILITIES, AiStepFile, GenerateImageJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse } from '@activepieces/shared'import { generateImage, generateText, ImageModel, ImagePart, LanguageModel } from 'ai'import { JobContext } from '../../types'-export async function generateImageStep({ ctx, data, resolved }: {+export async function generateImageStep({ ctx, data, resolved, flowStep }: {ctx: JobContextdata: GenerateImageJobDataresolved: ResolveAiProviderResponse+ flowStep: FlowStepMetadata}): Promise<string> {const inputImages = data.files ?? []- const image = await getGeneratedImage({ data, resolved, inputImages })+ const image = await getGeneratedImage({ data, resolved, inputImages, flowStep })const imageData = !isNil(image.base64) && image.base64.length > 0? Buffer.from(image.base64, 'base64'): Buffer.from(image.uint8Array)@@ -24,12 +25,13 @@ export async function generateImageStep({ ctx, data, resolved }: {return url}-async function getGeneratedImage({ data, resolved, inputImages }: {+async function getGeneratedImage({ data, resolved, inputImages, flowStep }: {data: GenerateImageJobDataresolved: ResolveAiProviderResponseinputImages: AiStepFile[]+ flowStep: FlowStepMetadata}): Promise<GeneratedImage> {- const model = createImageCapableModel({ resolved, modelId: data.modelId })+ const model = createImageCapableModel({ resolved, modelId: data.modelId, flowStep })const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider: resolved.provider, model: data.modelId })const resolvedProvider = effectiveProvider ?? resolved.providerconst prompt = data.prompt ?? ''@@ -56,19 +58,20 @@ async function getGeneratedImage({ data, resolved, inputImages }: {})}-function createImageCapableModel({ resolved, modelId }: {+function createImageCapableModel({ resolved, modelId, flowStep }: {resolved: ResolveAiProviderResponsemodelId: string+ flowStep: FlowStepMetadata}): ImageCapableModel {if (!AI_PROVIDER_CAPABILITIES[resolved.provider].supportsImageGeneration) {throw new Error(`Provider ${resolved.provider} does not support image models`)}const { provider, auth, config } = resolved- const imageModel = aiUtils.createModelForImages({ provider, auth, config, modelId })+ const imageModel = aiUtils.createModelForImages({ provider, auth, config, modelId, flowStep })if (!isNil(imageModel)) {return { kind: 'image', model: imageModel }}- return { kind: 'language', model: aiUtils.createModel({ provider, auth, config, modelId }) }+ return { kind: 'language', model: aiUtils.createModel({ provider, auth, config, modelId, flowStep }) }}async function generateImageUsingGenerateText({ model, prompt, inputImages }: {diff --git a/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts b/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.tsnew file mode 100644index 00000000000..579a7093443--- /dev/null+++ b/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts@@ -0,0 +1,160 @@+import { AIProviderName } from '@activepieces/core-utils'+import { AiStepAction, ExecuteAiJobData, ResolveAiProviderResponse } from '@activepieces/shared'+import { beforeEach, describe, expect, it, vi } from 'vitest'++const createModelCalls: Record<string, unknown>[] = []++vi.mock('@activepieces/server-utils', async (importOriginal) => ({+ ...(await importOriginal<Record<string, unknown>>()),+ aiUtils: {+ createModel: (args: Record<string, unknown>) => {+ createModelCalls.push(args)+ return { modelId: args['modelId'] }+ },+ createModelForImages: () => undefined,+ buildWebSearchToolsOrThrow: () => ({}),+ },+}))++vi.mock('ai', async (importOriginal) => ({+ ...(await importOriginal<Record<string, unknown>>()),+ generateText: async () => ({+ text: 'ok',+ toolCalls: [{ input: { total: 42 } }],+ response: { body: {} },+ files: [{ base64: Buffer.from('png').toString('base64'), uint8Array: new Uint8Array() }],+ }),+}))++const { extractStructuredData } = await import('../../../../../src/lib/execute/jobs/ai/extract-structured-data')+const { generateImageStep } = await import('../../../../../src/lib/execute/jobs/ai/generate-image')+const { executeAiJob } = await import('../../../../../src/lib/execute/jobs/ai/execute-ai')++const FLOW_STEP = {+ projectId: 'project-1',+ platformId: 'platform-1',+ flowId: 'flow-1',+ runId: 'run-1',+}++function jobData(action: AiStepAction, overrides: Partial<ExecuteAiJobData> = {}): ExecuteAiJobData {+ return {+ requestId: 'request-1',+ projectId: FLOW_STEP.projectId,+ platformId: FLOW_STEP.platformId,+ flowId: FLOW_STEP.flowId,+ flowRunId: FLOW_STEP.runId,+ waitpointId: 'waitpoint-1',+ action,+ provider: AIProviderName.MISTRAL,+ modelId: 'mistral-large-latest',+ ...overrides,+ } as ExecuteAiJobData+}++const RESOLVED: ResolveAiProviderResponse = {+ provider: AIProviderName.MISTRAL,+ providerConfigId: 'config-1',+ auth: { apiKey: 'key' },+ config: {},+}++describe('flowStep reaches every model built on the worker', () => {+ beforeEach(() => {+ createModelCalls.length = 0+ })++ it('extract structured data builds its model with the flow step, so Mistral is not re-routed through OpenRouter', async () => {+ await extractStructuredData({+ data: jobData(AiStepAction.EXTRACT_STRUCTURED_DATA, {+ text: 'an invoice total of 42',+ schema: { mode: 'simple', fields: [{ name: 'total', type: 'number', isRequired: true }] },+ }),+ resolved: RESOLVED,+ flowStep: FLOW_STEP,+ })++ expect(createModelCalls).toHaveLength(1)+ expect(createModelCalls[0]['flowStep']).toEqual(FLOW_STEP)+ })++ it('generate image builds its language fallback with the flow step, keeping gateway routing on submodel', async () => {+ const ctx = { apiClient: { saveFlowStepFile: async () => ({ fileId: 'file-1', url: 'https://files.example/file-1' }) } }++ await generateImageStep({+ ctx: ctx as unknown as Parameters<typeof generateImageStep>[0]['ctx'],+ data: jobData(AiStepAction.GENERATE_IMAGE, {+ prompt: 'a cat',+ provider: AIProviderName.GOOGLE,+ modelId: 'gemini-2.5-flash-image',+ }),+ resolved: { ...RESOLVED, provider: AIProviderName.GOOGLE },+ flowStep: FLOW_STEP,+ })++ expect(createModelCalls).toHaveLength(1)+ expect(createModelCalls[0]['flowStep']).toEqual(FLOW_STEP)+ })++ it('builds the flow step inside the job itself, so extract keeps it without the caller supplying one', async () => {+ const ctx = jobContext()++ await executeAiJob.execute(ctx, jobData(AiStepAction.EXTRACT_STRUCTURED_DATA, {+ text: 'an invoice total of 42',+ schema: { mode: 'simple', fields: [{ name: 'total', type: 'number', isRequired: true }] },+ }))++ expect(createModelCalls).toHaveLength(1)+ expect(createModelCalls[0]['flowStep']).toEqual(FLOW_STEP)+ })++ it('builds the flow step inside the job itself for generate image too', async () => {+ const ctx = jobContext({ provider: AIProviderName.GOOGLE })++ await executeAiJob.execute(ctx, jobData(AiStepAction.GENERATE_IMAGE, {+ prompt: 'a cat',+ provider: AIProviderName.GOOGLE,+ modelId: 'gemini-2.5-flash-image',+ }))++ expect(createModelCalls).toHaveLength(1)+ expect(createModelCalls[0]['flowStep']).toEqual(FLOW_STEP)+ })+})++function jobContext({ provider = AIProviderName.MISTRAL }: { provider?: AIProviderName } = {}): Parameters<typeof executeAiJob.execute>[0] {+ return {+ apiClient: {+ resolveAiProvider: async () => ({ ...RESOLVED, provider }),+ resumeAiStep: async () => undefined,+ saveFlowStepFile: async () => ({ fileId: 'file-1', url: 'https://files.example/file-1' }),+ },+ log: { warn: () => undefined, error: () => undefined },+ } as unknown as Parameters<typeof executeAiJob.execute>[0]+}++describe('the image model gets the flow step too, so gateway routing is not silently compat', () => {+ it('hands the flow step to the image model builder, not just the language fallback', async () => {+ const forImages: Record<string, unknown>[] = []+ const serverUtils = await import('@activepieces/server-utils')+ const original = serverUtils.aiUtils.createModelForImages+ serverUtils.aiUtils.createModelForImages = (args: Record<string, unknown>) => {+ forImages.push(args)+ return undefined+ }++ try {+ await executeAiJob.execute(jobContext({ provider: AIProviderName.GOOGLE }), jobData(AiStepAction.GENERATE_IMAGE, {+ prompt: 'a cat',+ provider: AIProviderName.GOOGLE,+ modelId: 'gemini-2.5-flash-image',+ }))+ }+ finally {+ serverUtils.aiUtils.createModelForImages = original+ }++ expect(forImages).toHaveLength(1)+ expect(forImages[0]['flowStep']).toEqual(FLOW_STEP)+ })+})