Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(ai): stop Extract and Generate Image reaching the wrong AI provider (#15490)

Abdul420f5f6c260f
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
index 3a5c44d8023..92d5b449d91 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
@@ -53,19 +53,20 @@ async function runAiStep(ctx: JobContext, data: ExecuteAiJobData): Promise<unkno
provider: data.provider,
...spreadIfDefined('providerConfigId', data.providerConfigId),
})
+ const flowStep = flowStepMetadata(data)
switch (data.action) {
case AiStepAction.EXTRACT_STRUCTURED_DATA:
- return { answer: await extractStructuredData({ data, resolved }) }
+ return { answer: await extractStructuredData({ data, resolved, flowStep }) }
case AiStepAction.GENERATE_IMAGE:
- return { answer: await generateImageStep({ ctx, data, resolved }) }
+ return { answer: await generateImageStep({ ctx, data, resolved, flowStep }) }
case AiStepAction.ASK_AI:
case AiStepAction.SUMMARIZE_TEXT:
case AiStepAction.CLASSIFY_TEXT:
- return runTextStep({ data, resolved })
+ return runTextStep({ data, resolved, flowStep })
}
}
-async function runTextStep({ data, resolved }: { data: ExecuteAiJobData, resolved: ResolveAiProviderResponse }): Promise<unknown> {
+async function runTextStep({ data, resolved, flowStep }: { data: ExecuteAiJobData, resolved: ResolveAiProviderResponse, flowStep: FlowStepMetadata }): Promise<unknown> {
const { provider, auth, config } = resolved
const webSearchEnabled = data.webSearch?.enabled ?? false
const webSearchOptions = data.webSearch?.options
@@ -76,7 +77,7 @@ async function runTextStep({ data, resolved }: { data: ExecuteAiJobData, resolve
auth,
config,
modelId: data.modelId,
- flowStep: flowStepMetadata(data),
+ flowStep,
openaiResponsesModel: webSearchEnabled && (effectiveProvider ?? provider) === AIProviderName.OPENAI,
webSearchEnabled,
webSearchOptions,
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
index 312ece2a8d9..df7b6c24731 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
@@ -1,11 +1,12 @@
import { isNil, spreadIfDefined } from '@activepieces/core-utils'
-import { aiUtils } from '@activepieces/server-utils'
+import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'
import { AiStepFile, ExtractStructuredDataJobData, ResolveAiProviderResponse } from '@activepieces/shared'
import { generateText, jsonSchema, ModelMessage, tool, UserModelMessage } from 'ai'
-export async function extractStructuredData({ data, resolved }: {
+export async function extractStructuredData({ data, resolved, flowStep }: {
data: ExtractStructuredDataJobData
resolved: ResolveAiProviderResponse
+ flowStep: FlowStepMetadata
}): Promise<unknown> {
const files = data.files ?? []
if (isNil(data.text) && files.length === 0) {
@@ -16,6 +17,7 @@ export async function extractStructuredData({ data, resolved }: {
auth: resolved.auth,
config: resolved.config,
modelId: data.modelId,
+ flowStep,
})
const { schemaDefinition, sanitizedNameMap } = buildSchema(data)
const extractionTool = tool({
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts
index 84bb18e7ab4..dbcec313e3c 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/generate-image.ts
@@ -1,16 +1,17 @@
import { AIProviderName, isNil } from '@activepieces/core-utils'
-import { aiUtils } from '@activepieces/server-utils'
+import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'
import { AI_PROVIDER_CAPABILITIES, AiStepFile, GenerateImageJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse } from '@activepieces/shared'
import { generateImage, generateText, ImageModel, ImagePart, LanguageModel } from 'ai'
import { JobContext } from '../../types'
-export async function generateImageStep({ ctx, data, resolved }: {
+export async function generateImageStep({ ctx, data, resolved, flowStep }: {
ctx: JobContext
data: GenerateImageJobData
resolved: ResolveAiProviderResponse
+ flowStep: FlowStepMetadata
}): Promise<string> {
const inputImages = data.files ?? []
- const image = await getGeneratedImage({ data, resolved, inputImages })
+ const image = await getGeneratedImage({ data, resolved, inputImages, flowStep })
const imageData = !isNil(image.base64) && image.base64.length > 0
? Buffer.from(image.base64, 'base64')
: Buffer.from(image.uint8Array)
@@ -24,12 +25,13 @@ export async function generateImageStep({ ctx, data, resolved }: {
return url
}
-async function getGeneratedImage({ data, resolved, inputImages }: {
+async function getGeneratedImage({ data, resolved, inputImages, flowStep }: {
data: GenerateImageJobData
resolved: ResolveAiProviderResponse
inputImages: AiStepFile[]
+ flowStep: FlowStepMetadata
}): Promise<GeneratedImage> {
- const model = createImageCapableModel({ resolved, modelId: data.modelId })
+ const model = createImageCapableModel({ resolved, modelId: data.modelId, flowStep })
const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider: resolved.provider, model: data.modelId })
const resolvedProvider = effectiveProvider ?? resolved.provider
const prompt = data.prompt ?? ''
@@ -56,19 +58,20 @@ async function getGeneratedImage({ data, resolved, inputImages }: {
})
}
-function createImageCapableModel({ resolved, modelId }: {
+function createImageCapableModel({ resolved, modelId, flowStep }: {
resolved: ResolveAiProviderResponse
modelId: string
+ flowStep: FlowStepMetadata
}): ImageCapableModel {
if (!AI_PROVIDER_CAPABILITIES[resolved.provider].supportsImageGeneration) {
throw new Error(`Provider ${resolved.provider} does not support image models`)
}
const { provider, auth, config } = resolved
- const imageModel = aiUtils.createModelForImages({ provider, auth, config, modelId })
+ const imageModel = aiUtils.createModelForImages({ provider, auth, config, modelId, flowStep })
if (!isNil(imageModel)) {
return { kind: 'image', model: imageModel }
}
- return { kind: 'language', model: aiUtils.createModel({ provider, auth, config, modelId }) }
+ return { kind: 'language', model: aiUtils.createModel({ provider, auth, config, modelId, flowStep }) }
}
async function generateImageUsingGenerateText({ model, prompt, inputImages }: {
diff --git a/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts b/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts
new file mode 100644
index 00000000000..579a7093443
--- /dev/null
+++ b/packages/server/worker/test/lib/execute/jobs/ai/flow-step-routing.test.ts
@@ -0,0 +1,160 @@
+import { AIProviderName } from '@activepieces/core-utils'
+import { AiStepAction, ExecuteAiJobData, ResolveAiProviderResponse } from '@activepieces/shared'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const createModelCalls: Record<string, unknown>[] = []
+
+vi.mock('@activepieces/server-utils', async (importOriginal) => ({
+ ...(await importOriginal<Record<string, unknown>>()),
+ aiUtils: {
+ createModel: (args: Record<string, unknown>) => {
+ createModelCalls.push(args)
+ return { modelId: args['modelId'] }
+ },
+ createModelForImages: () => undefined,
+ buildWebSearchToolsOrThrow: () => ({}),
+ },
+}))
+
+vi.mock('ai', async (importOriginal) => ({
+ ...(await importOriginal<Record<string, unknown>>()),
+ generateText: async () => ({
+ text: 'ok',
+ toolCalls: [{ input: { total: 42 } }],
+ response: { body: {} },
+ files: [{ base64: Buffer.from('png').toString('base64'), uint8Array: new Uint8Array() }],
+ }),
+}))
+
+const { extractStructuredData } = await import('../../../../../src/lib/execute/jobs/ai/extract-structured-data')
+const { generateImageStep } = await import('../../../../../src/lib/execute/jobs/ai/generate-image')
+const { executeAiJob } = await import('../../../../../src/lib/execute/jobs/ai/execute-ai')
+
+const FLOW_STEP = {
+ projectId: 'project-1',
+ platformId: 'platform-1',
+ flowId: 'flow-1',
+ runId: 'run-1',
+}
+
+function jobData(action: AiStepAction, overrides: Partial<ExecuteAiJobData> = {}): ExecuteAiJobData {
+ return {
+ requestId: 'request-1',
+ projectId: FLOW_STEP.projectId,
+ platformId: FLOW_STEP.platformId,
+ flowId: FLOW_STEP.flowId,
+ flowRunId: FLOW_STEP.runId,
+ waitpointId: 'waitpoint-1',
+ action,
+ provider: AIProviderName.MISTRAL,
+ modelId: 'mistral-large-latest',
+ ...overrides,
+ } as ExecuteAiJobData
+}
+
+const RESOLVED: ResolveAiProviderResponse = {
+ provider: AIProviderName.MISTRAL,
+ providerConfigId: 'config-1',
+ auth: { apiKey: 'key' },
+ config: {},
+}
+
+describe('flowStep reaches every model built on the worker', () => {
+ beforeEach(() => {
+ createModelCalls.length = 0
+ })
+
+ it('extract structured data builds its model with the flow step, so Mistral is not re-routed through OpenRouter', async () => {
+ await extractStructuredData({
+ data: jobData(AiStepAction.EXTRACT_STRUCTURED_DATA, {
+ text: 'an invoice total of 42',
+ schema: { mode: 'simple', fields: [{ name: 'total', type: 'number', isRequired: true }] },
+ }),
+ resolved: RESOLVED,
+ flowStep: FLOW_STEP,
+ })
+
+ expect(createModelCalls).toHaveLength(1)
+ expect(createModelCalls[0]['flowStep']).toEqual(FLOW_STEP)
+ })
+
+ it('generate image builds its language fallback with the flow step, keeping gateway routing on submodel', async () => {
+ const ctx = { apiClient: { saveFlowStepFile: async () => ({ fileId: 'file-1', url: 'https://files.example/file-1' }) } }
+
+ await generateImageStep({
+ ctx: ctx as unknown as Parameters<typeof generateImageStep>[0]['ctx'],
+ data: jobData(AiStepAction.GENERATE_IMAGE, {
+ prompt: 'a cat',
+ provider: AIProviderName.GOOGLE,
+ modelId: 'gemini-2.5-flash-image',
+ }),
+ resolved: { ...RESOLVED, provider: AIProviderName.GOOGLE },
+ flowStep: FLOW_STEP,
+ })
+
+ expect(createModelCalls).toHaveLength(1)
+ expect(createModelCalls[0]['flowStep']).toEqual(FLOW_STEP)
+ })
+
+ it('builds the flow step inside the job itself, so extract keeps it without the caller supplying one', async () => {
+ const ctx = jobContext()
+
+ await executeAiJob.execute(ctx, jobData(AiStepAction.EXTRACT_STRUCTURED_DATA, {
+ text: 'an invoice total of 42',
+ schema: { mode: 'simple', fields: [{ name: 'total', type: 'number', isRequired: true }] },
+ }))
+
+ expect(createModelCalls).toHaveLength(1)
+ expect(createModelCalls[0]['flowStep']).toEqual(FLOW_STEP)
+ })
+
+ it('builds the flow step inside the job itself for generate image too', async () => {
+ const ctx = jobContext({ provider: AIProviderName.GOOGLE })
+
+ await executeAiJob.execute(ctx, jobData(AiStepAction.GENERATE_IMAGE, {
+ prompt: 'a cat',
+ provider: AIProviderName.GOOGLE,
+ modelId: 'gemini-2.5-flash-image',
+ }))
+
+ expect(createModelCalls).toHaveLength(1)
+ expect(createModelCalls[0]['flowStep']).toEqual(FLOW_STEP)
+ })
+})
+
+function jobContext({ provider = AIProviderName.MISTRAL }: { provider?: AIProviderName } = {}): Parameters<typeof executeAiJob.execute>[0] {
+ return {
+ apiClient: {
+ resolveAiProvider: async () => ({ ...RESOLVED, provider }),
+ resumeAiStep: async () => undefined,
+ saveFlowStepFile: async () => ({ fileId: 'file-1', url: 'https://files.example/file-1' }),
+ },
+ log: { warn: () => undefined, error: () => undefined },
+ } as unknown as Parameters<typeof executeAiJob.execute>[0]
+}
+
+describe('the image model gets the flow step too, so gateway routing is not silently compat', () => {
+ it('hands the flow step to the image model builder, not just the language fallback', async () => {
+ const forImages: Record<string, unknown>[] = []
+ const serverUtils = await import('@activepieces/server-utils')
+ const original = serverUtils.aiUtils.createModelForImages
+ serverUtils.aiUtils.createModelForImages = (args: Record<string, unknown>) => {
+ forImages.push(args)
+ return undefined
+ }
+
+ try {
+ await executeAiJob.execute(jobContext({ provider: AIProviderName.GOOGLE }), jobData(AiStepAction.GENERATE_IMAGE, {
+ prompt: 'a cat',
+ provider: AIProviderName.GOOGLE,
+ modelId: 'gemini-2.5-flash-image',
+ }))
+ }
+ finally {
+ serverUtils.aiUtils.createModelForImages = original
+ }
+
+ expect(forImages).toHaveLength(1)
+ expect(forImages[0]['flowStep']).toEqual(FLOW_STEP)
+ })
+})