Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

test(chat): cover the fallback when a chosen AI key can't serve a chat run (#15911)

Hazem Adelad0f389ed23f
diff --git a/packages/server/api/src/app/ee/agent/rpc/agent-config-rpc.ts b/packages/server/api/src/app/ee/agent/rpc/agent-config-rpc.ts
index 5e7680671f7..c2e80fc9fe3 100644
--- a/packages/server/api/src/app/ee/agent/rpc/agent-config-rpc.ts
+++ b/packages/server/api/src/app/ee/agent/rpc/agent-config-rpc.ts
@@ -1,6 +1,6 @@
import { ActivepiecesError, ErrorCode, isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils'
-import { agentAiUtils, aiUtils, ModelTierSurface } from '@activepieces/server-utils'
-import { AgentConfigResponse, AgentConversationStatus, AgentRunSource, AiProviderToolChoices, AiProviderToolConfig, AIProviderWithoutSensitiveData, GetAgentConfigRequest, GetEnabledAiToolsResponse, GetProviderConfigResponse, PersistedAgentMessage, PersistedAgentPartType, PersistedAgentRole } from '@activepieces/shared'
+import { agentAiUtils, aiUtils } from '@activepieces/server-utils'
+import { AgentConfigResponse, AgentConversationStatus, AgentRunSource, GetAgentConfigRequest, GetEnabledAiToolsResponse, PersistedAgentMessage, PersistedAgentPartType, PersistedAgentRole } from '@activepieces/shared'
import { ModelMessage } from 'ai'
import { FastifyBaseLogger } from 'fastify'
import { agentApprovalGate } from '.././agent-approval-gate'
@@ -12,7 +12,6 @@ import { chatPersonalizationService } from '.././personalization/chat-personaliz
import { agentPrompt } from '.././prompt/agent-prompt'
import { agentSurfaceNotes } from '.././prompt/agent-surface-notes'
import { UserIdentity } from '.././prompt/agent-user-identity'
-import { aiProviderService, ProviderScope } from '../../../ai/ai-provider-service'
import { aiToolConfigService } from '../../../ai/ai-tool-config-service'
import { appConnectionService } from '../../../app-connection/app-connection-service/app-connection-service'
import { system } from '../../../helper/system/system'
@@ -21,6 +20,7 @@ import { platformService } from '../../../platform/platform.service'
import { userService } from '../../../user/user-service'
import { smtpEmailSender } from '../../helper/email/email-sender/smtp-email-sender'
+import { chosenProviders } from './chosen-providers'
import { CONNECTION_INVENTORY_LIMIT, loadOrStartConversation } from './rpc-shared'
@@ -95,20 +95,20 @@ export const agentConfigRpc = (log: FastifyBaseLogger) => ({
const aiTools: GetEnabledAiToolsResponse = dryRun ? {} : enabledAiTools
const surface = agentHelpers.surfaceOf({ source: requestedSource })
- const chosenProviders = dryRun
+ const chosen = dryRun
? { search: null, image: null }
- : await resolveChosenProviders({ platformId, choices: providerChoices, surface, scope: runScope, log })
+ : await chosenProviders.resolveForRun({ platformId, choices: providerChoices, surface, scope: runScope, log })
const actingRun = !dryRun && !discoveryOnly
const emailEnabled = actingRun && carriesChatContext && smtpEmailSender(log).isSmtpConfigured()
const agentsAvailable = actingRun && agentsSurfaceOn
const fetchAvailable = !dryRun
// Tavily takes precedence over native LLM search; native is only the no-Tavily fallback.
const tavilySearchAvailable = !isNil(aiTools.webSearch)
- const webSearchAvailable = fetchAvailable && (tavilySearchAvailable || aiUtils.supportsWebSearch((chosenProviders.search?.credentials ?? providerConfig).provider))
+ const webSearchAvailable = fetchAvailable && (tavilySearchAvailable || aiUtils.supportsWebSearch((chosen.search?.credentials ?? providerConfig).provider))
const generatesImagesOnProvider = actingRun && isNil(aiTools.imageGeneration)
const imageModelId = !generatesImagesOnProvider
? undefined
- : chosenProviders.image?.modelId ?? await agentHelpers.resolveImageModelId({ platformId, providerConfig, scope: runScope, log })
+ : chosen.image?.modelId ?? await agentHelpers.resolveImageModelId({ platformId, providerConfig, scope: runScope, log })
const lockResult = await agentHelpers.conversationRepo()
.createQueryBuilder()
@@ -253,9 +253,9 @@ export const agentConfigRpc = (log: FastifyBaseLogger) => ({
modelId: resolvedModelId,
fastModelId,
...spreadIfDefined('imageModelId', imageModelId),
- ...spreadIfDefined('searchCredentials', chosenProviders.search?.credentials),
- ...spreadIfDefined('searchModelId', chosenProviders.search?.modelId),
- ...spreadIfDefined('imageCredentials', chosenProviders.image?.credentials),
+ ...spreadIfDefined('searchCredentials', chosen.search?.credentials),
+ ...spreadIfDefined('searchModelId', chosen.search?.modelId),
+ ...spreadIfDefined('imageCredentials', chosen.image?.credentials),
systemPrompt: systemPromptText,
messages: messagesForLlm,
allMessages,
@@ -275,61 +275,3 @@ export const agentConfigRpc = (log: FastifyBaseLogger) => ({
},
})
-
-async function resolveChosenProviders({ platformId, choices, surface, scope, log }: { platformId: string, choices: AiProviderToolChoices, surface: ModelTierSurface, scope: ProviderScope, log: FastifyBaseLogger }): Promise<ChosenProviders> {
- if (isNil(choices.webSearch) && isNil(choices.imageGeneration)) {
- return { search: null, image: null }
- }
- const configs = await aiProviderService(log).listConfigs(platformId)
- const [search, image] = await Promise.all([
- resolveChosenProvider({ platformId, configs, choice: choices.webSearch, scope, log }).then((credentials) => withSearchModel({ platformId, credentials, surface, scope, log })),
- resolveChosenProvider({ platformId, configs, choice: choices.imageGeneration, scope, log }).then((credentials) => withImageModel({ credentials, modelId: choices.imageGeneration?.modelId, log })),
- ])
- return { search, image }
-}
-
-async function withSearchModel({ platformId, credentials, surface, scope, log }: { platformId: string, credentials: GetProviderConfigResponse | null, surface: ModelTierSurface, scope: ProviderScope, log: FastifyBaseLogger }): Promise<ChosenProvider | null> {
- if (isNil(credentials)) {
- return null
- }
- const { data: modelId, error } = await tryCatch(() => agentHelpers.resolveFastModelId({ platformId, providerConfig: credentials, surface, scope, log }))
- if (error) {
- log.warn({ error, aiProvider: { id: credentials.configId } }, '[agentRpc#getAgentConfig] Chosen search provider has no usable model, using the chat provider')
- return null
- }
- return { credentials, modelId }
-}
-
-function withImageModel({ credentials, modelId, log }: { credentials: GetProviderConfigResponse | null, modelId: string | undefined, log: FastifyBaseLogger }): ChosenProvider | null {
- if (isNil(credentials) || isNil(modelId)) {
- return null
- }
- const modelAllowed = credentials.modelScope !== 'selected' || credentials.modelIds.includes(modelId)
- if (!modelAllowed) {
- log.warn({ aiProvider: { id: credentials.configId } }, '[agentRpc#getAgentConfig] Chosen image model is no longer allowed on its key, using the chat provider')
- return null
- }
- return { credentials, modelId }
-}
-
-async function resolveChosenProvider({ platformId, configs, choice, scope, log }: { platformId: string, configs: AIProviderWithoutSensitiveData[], choice: AiProviderToolConfig | undefined, scope: ProviderScope, log: FastifyBaseLogger }): Promise<GetProviderConfigResponse | null> {
- const row = isNil(choice) ? undefined : configs.find((config) => config.id === choice.aiProviderId)
- if (isNil(row)) {
- return null
- }
- const { data, error } = await tryCatch(() => aiProviderService(log).getConfigOrThrow({ platformId, provider: row.provider, scope, configId: row.id }))
- if (error) {
- log.warn({ error, aiProvider: { id: row.id } }, '[agentRpc#getAgentConfig] Chosen AI provider cannot serve this run, using the chat provider')
- }
- return data ?? null
-}
-
-type ChosenProvider = {
- credentials: GetProviderConfigResponse
- modelId: string
-}
-
-type ChosenProviders = {
- search: ChosenProvider | null
- image: ChosenProvider | null
-}
diff --git a/packages/server/api/src/app/ee/agent/rpc/chosen-providers.ts b/packages/server/api/src/app/ee/agent/rpc/chosen-providers.ts
new file mode 100644
index 00000000000..d209400b000
--- /dev/null
+++ b/packages/server/api/src/app/ee/agent/rpc/chosen-providers.ts
@@ -0,0 +1,68 @@
+import { isNil, tryCatch } from '@activepieces/core-utils'
+import { ModelTierSurface } from '@activepieces/server-utils'
+import { AiProviderToolChoices, AiProviderToolConfig, AIProviderWithoutSensitiveData, GetProviderConfigResponse } from '@activepieces/shared'
+import { FastifyBaseLogger } from 'fastify'
+import { aiProviderService, ProviderScope } from '../../../ai/ai-provider-service'
+import { agentHelpers } from '../agent-helpers'
+
+async function resolveForRun({ platformId, choices, surface, scope, log }: { platformId: string, choices: AiProviderToolChoices, surface: ModelTierSurface, scope: ProviderScope, log: FastifyBaseLogger }): Promise<ChosenProviders> {
+ if (isNil(choices.webSearch) && isNil(choices.imageGeneration)) {
+ return { search: null, image: null }
+ }
+ const configs = await aiProviderService(log).listConfigs(platformId)
+ const [search, image] = await Promise.all([
+ keyCredentialsForRun({ platformId, configs, choice: choices.webSearch, scope, log }).then((credentials) => searchKeyWithFastModel({ platformId, credentials, surface, scope, log })),
+ keyCredentialsForRun({ platformId, configs, choice: choices.imageGeneration, scope, log }).then((credentials) => imageKeyIfModelAllowed({ credentials, modelId: choices.imageGeneration?.modelId, log })),
+ ])
+ return { search, image }
+}
+
+async function keyCredentialsForRun({ platformId, configs, choice, scope, log }: { platformId: string, configs: AIProviderWithoutSensitiveData[], choice: AiProviderToolConfig | undefined, scope: ProviderScope, log: FastifyBaseLogger }): Promise<GetProviderConfigResponse | null> {
+ const row = isNil(choice) ? undefined : configs.find((config) => config.id === choice.aiProviderId)
+ if (isNil(row)) {
+ return null
+ }
+ const { data, error } = await tryCatch(() => aiProviderService(log).getConfigOrThrow({ platformId, provider: row.provider, scope, configId: row.id }))
+ if (error) {
+ log.warn({ error, aiProvider: { id: row.id } }, '[chosenProviders#resolveForRun] Chosen AI provider cannot serve this run, using the chat provider')
+ }
+ return data ?? null
+}
+
+async function searchKeyWithFastModel({ platformId, credentials, surface, scope, log }: { platformId: string, credentials: GetProviderConfigResponse | null, surface: ModelTierSurface, scope: ProviderScope, log: FastifyBaseLogger }): Promise<ChosenProvider | null> {
+ if (isNil(credentials)) {
+ return null
+ }
+ const { data: modelId, error } = await tryCatch(() => agentHelpers.resolveFastModelId({ platformId, providerConfig: credentials, surface, scope, log }))
+ if (error) {
+ log.warn({ error, aiProvider: { id: credentials.configId } }, '[chosenProviders#resolveForRun] Chosen search provider has no usable model, using the chat provider')
+ return null
+ }
+ return { credentials, modelId }
+}
+
+function imageKeyIfModelAllowed({ credentials, modelId, log }: { credentials: GetProviderConfigResponse | null, modelId: string | undefined, log: FastifyBaseLogger }): ChosenProvider | null {
+ if (isNil(credentials) || isNil(modelId)) {
+ return null
+ }
+ const modelAllowed = credentials.modelScope !== 'selected' || credentials.modelIds.includes(modelId)
+ if (!modelAllowed) {
+ log.warn({ aiProvider: { id: credentials.configId } }, '[chosenProviders#resolveForRun] Chosen image model is no longer allowed on its key, using the chat provider')
+ return null
+ }
+ return { credentials, modelId }
+}
+
+export const chosenProviders = {
+ resolveForRun,
+}
+
+type ChosenProvider = {
+ credentials: GetProviderConfigResponse
+ modelId: string
+}
+
+type ChosenProviders = {
+ search: ChosenProvider | null
+ image: ChosenProvider | null
+}
diff --git a/packages/server/api/test/unit/app/ee/agent/chosen-providers.test.ts b/packages/server/api/test/unit/app/ee/agent/chosen-providers.test.ts
new file mode 100644
index 00000000000..883f43e4a2d
--- /dev/null
+++ b/packages/server/api/test/unit/app/ee/agent/chosen-providers.test.ts
@@ -0,0 +1,105 @@
+import { AIProviderName } from '@activepieces/core-utils'
+import { GetProviderConfigResponse } from '@activepieces/shared'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockListConfigs, mockGetConfigOrThrow, mockResolveFastModelId } = vi.hoisted(() => ({
+ mockListConfigs: vi.fn(),
+ mockGetConfigOrThrow: vi.fn(),
+ mockResolveFastModelId: vi.fn(),
+}))
+
+vi.mock('../../../../../src/app/ai/ai-provider-service', () => ({
+ aiProviderService: () => ({
+ listConfigs: mockListConfigs,
+ getConfigOrThrow: mockGetConfigOrThrow,
+ }),
+}))
+
+vi.mock('../../../../../src/app/ee/agent/agent-helpers', () => ({
+ agentHelpers: { resolveFastModelId: mockResolveFastModelId },
+}))
+
+const { chosenProviders } = await import('../../../../../src/app/ee/agent/rpc/chosen-providers')
+
+const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as never
+const platformId = 'platform-1'
+const scope = { type: 'project', projectId: 'project-1' } as const
+
+function credentialsFor({ modelScope, modelIds }: { modelScope: 'all' | 'selected', modelIds: string[] }): GetProviderConfigResponse {
+ return { provider: AIProviderName.GOOGLE, configId: 'key-1', platformId, auth: { apiKey: 'k' }, config: {}, modelScope, modelIds }
+}
+
+describe('chosenProviders.resolveForRun', () => {
+ beforeEach(() => {
+ mockListConfigs.mockReset().mockResolvedValue([{ id: 'key-1', provider: AIProviderName.GOOGLE }])
+ mockGetConfigOrThrow.mockReset().mockResolvedValue(credentialsFor({ modelScope: 'all', modelIds: [] }))
+ mockResolveFastModelId.mockReset().mockResolvedValue('gemini-2.5-flash')
+ })
+
+ it('skips every lookup when nothing was chosen', async () => {
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: {}, surface: 'chat', scope, log })
+
+ expect(chosen).toEqual({ search: null, image: null })
+ expect(mockListConfigs).not.toHaveBeenCalled()
+ })
+
+ it('uses the chosen key and its fast model for search, checked against the run scope', async () => {
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { webSearch: { aiProviderId: 'key-1' } }, surface: 'chat', scope, log })
+
+ expect(mockGetConfigOrThrow).toHaveBeenCalledWith({ platformId, provider: AIProviderName.GOOGLE, scope, configId: 'key-1' })
+ expect(chosen.search).toMatchObject({ credentials: { configId: 'key-1' }, modelId: 'gemini-2.5-flash' })
+ })
+
+ it('falls back to the chat provider when the search key has no usable model', async () => {
+ mockResolveFastModelId.mockRejectedValue(new Error('no text model'))
+
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { webSearch: { aiProviderId: 'key-1' } }, surface: 'chat', scope, log })
+
+ expect(chosen.search).toBeNull()
+ })
+
+ it('falls back when the key no longer serves the run\'s project', async () => {
+ mockGetConfigOrThrow.mockRejectedValue(new Error('scoped away'))
+
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { webSearch: { aiProviderId: 'key-1' }, imageGeneration: { aiProviderId: 'key-1', modelId: 'img-1' } }, surface: 'chat', scope, log })
+
+ expect(chosen).toEqual({ search: null, image: null })
+ })
+
+ it('never resolves a key that is not on the platform', async () => {
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { webSearch: { aiProviderId: 'foreign-key' } }, surface: 'chat', scope, log })
+
+ expect(chosen.search).toBeNull()
+ expect(mockGetConfigOrThrow).not.toHaveBeenCalled()
+ })
+
+ it('keeps an image model the key still allows', async () => {
+ mockGetConfigOrThrow.mockResolvedValue(credentialsFor({ modelScope: 'selected', modelIds: ['img-1'] }))
+
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { imageGeneration: { aiProviderId: 'key-1', modelId: 'img-1' } }, surface: 'chat', scope, log })
+
+ expect(chosen.image).toMatchObject({ credentials: { configId: 'key-1' }, modelId: 'img-1' })
+ })
+
+ it('keeps any image model on a key that allows every model', async () => {
+ mockGetConfigOrThrow.mockResolvedValue(credentialsFor({ modelScope: 'all', modelIds: [] }))
+
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { imageGeneration: { aiProviderId: 'key-1', modelId: 'img-1' } }, surface: 'chat', scope, log })
+
+ expect(chosen.image).toMatchObject({ modelId: 'img-1' })
+ })
+
+ it('ignores an image choice that names no model', async () => {
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { imageGeneration: { aiProviderId: 'key-1' } }, surface: 'chat', scope, log })
+
+ expect(chosen.image).toBeNull()
+ })
+
+ it('falls back when the saved image model was removed from the key', async () => {
+ mockGetConfigOrThrow.mockResolvedValue(credentialsFor({ modelScope: 'selected', modelIds: ['img-2'] }))
+
+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { imageGeneration: { aiProviderId: 'key-1', modelId: 'img-1' } }, surface: 'chat', scope, log })
+
+ expect(chosen.image).toBeNull()
+ })
+})