AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
test(chat): cover the fallback when a chosen AI key can't serve a chat run (#15911)
Hazem Adelad0f389ed23f
diff --git a/packages/server/api/src/app/ee/agent/rpc/agent-config-rpc.ts b/packages/server/api/src/app/ee/agent/rpc/agent-config-rpc.tsindex 5e7680671f7..c2e80fc9fe3 100644--- a/packages/server/api/src/app/ee/agent/rpc/agent-config-rpc.ts+++ b/packages/server/api/src/app/ee/agent/rpc/agent-config-rpc.ts@@ -1,6 +1,6 @@import { ActivepiecesError, ErrorCode, isNil, spreadIfDefined, tryCatch } from '@activepieces/core-utils'-import { agentAiUtils, aiUtils, ModelTierSurface } from '@activepieces/server-utils'-import { AgentConfigResponse, AgentConversationStatus, AgentRunSource, AiProviderToolChoices, AiProviderToolConfig, AIProviderWithoutSensitiveData, GetAgentConfigRequest, GetEnabledAiToolsResponse, GetProviderConfigResponse, PersistedAgentMessage, PersistedAgentPartType, PersistedAgentRole } from '@activepieces/shared'+import { agentAiUtils, aiUtils } from '@activepieces/server-utils'+import { AgentConfigResponse, AgentConversationStatus, AgentRunSource, GetAgentConfigRequest, GetEnabledAiToolsResponse, PersistedAgentMessage, PersistedAgentPartType, PersistedAgentRole } from '@activepieces/shared'import { ModelMessage } from 'ai'import { FastifyBaseLogger } from 'fastify'import { agentApprovalGate } from '.././agent-approval-gate'@@ -12,7 +12,6 @@ import { chatPersonalizationService } from '.././personalization/chat-personalizimport { agentPrompt } from '.././prompt/agent-prompt'import { agentSurfaceNotes } from '.././prompt/agent-surface-notes'import { UserIdentity } from '.././prompt/agent-user-identity'-import { aiProviderService, ProviderScope } from '../../../ai/ai-provider-service'import { aiToolConfigService } from '../../../ai/ai-tool-config-service'import { appConnectionService } from '../../../app-connection/app-connection-service/app-connection-service'import { system } from '../../../helper/system/system'@@ -21,6 +20,7 @@ import { platformService } from '../../../platform/platform.service'import { userService } from '../../../user/user-service'import { smtpEmailSender } from '../../helper/email/email-sender/smtp-email-sender'+import { chosenProviders } from './chosen-providers'import { CONNECTION_INVENTORY_LIMIT, loadOrStartConversation } from './rpc-shared'@@ -95,20 +95,20 @@ export const agentConfigRpc = (log: FastifyBaseLogger) => ({const aiTools: GetEnabledAiToolsResponse = dryRun ? {} : enabledAiToolsconst surface = agentHelpers.surfaceOf({ source: requestedSource })- const chosenProviders = dryRun+ const chosen = dryRun? { search: null, image: null }- : await resolveChosenProviders({ platformId, choices: providerChoices, surface, scope: runScope, log })+ : await chosenProviders.resolveForRun({ platformId, choices: providerChoices, surface, scope: runScope, log })const actingRun = !dryRun && !discoveryOnlyconst emailEnabled = actingRun && carriesChatContext && smtpEmailSender(log).isSmtpConfigured()const agentsAvailable = actingRun && agentsSurfaceOnconst fetchAvailable = !dryRun// Tavily takes precedence over native LLM search; native is only the no-Tavily fallback.const tavilySearchAvailable = !isNil(aiTools.webSearch)- const webSearchAvailable = fetchAvailable && (tavilySearchAvailable || aiUtils.supportsWebSearch((chosenProviders.search?.credentials ?? providerConfig).provider))+ const webSearchAvailable = fetchAvailable && (tavilySearchAvailable || aiUtils.supportsWebSearch((chosen.search?.credentials ?? providerConfig).provider))const generatesImagesOnProvider = actingRun && isNil(aiTools.imageGeneration)const imageModelId = !generatesImagesOnProvider? undefined- : chosenProviders.image?.modelId ?? await agentHelpers.resolveImageModelId({ platformId, providerConfig, scope: runScope, log })+ : chosen.image?.modelId ?? await agentHelpers.resolveImageModelId({ platformId, providerConfig, scope: runScope, log })const lockResult = await agentHelpers.conversationRepo().createQueryBuilder()@@ -253,9 +253,9 @@ export const agentConfigRpc = (log: FastifyBaseLogger) => ({modelId: resolvedModelId,fastModelId,...spreadIfDefined('imageModelId', imageModelId),- ...spreadIfDefined('searchCredentials', chosenProviders.search?.credentials),- ...spreadIfDefined('searchModelId', chosenProviders.search?.modelId),- ...spreadIfDefined('imageCredentials', chosenProviders.image?.credentials),+ ...spreadIfDefined('searchCredentials', chosen.search?.credentials),+ ...spreadIfDefined('searchModelId', chosen.search?.modelId),+ ...spreadIfDefined('imageCredentials', chosen.image?.credentials),systemPrompt: systemPromptText,messages: messagesForLlm,allMessages,@@ -275,61 +275,3 @@ export const agentConfigRpc = (log: FastifyBaseLogger) => ({},})--async function resolveChosenProviders({ platformId, choices, surface, scope, log }: { platformId: string, choices: AiProviderToolChoices, surface: ModelTierSurface, scope: ProviderScope, log: FastifyBaseLogger }): Promise<ChosenProviders> {- if (isNil(choices.webSearch) && isNil(choices.imageGeneration)) {- return { search: null, image: null }- }- const configs = await aiProviderService(log).listConfigs(platformId)- const [search, image] = await Promise.all([- resolveChosenProvider({ platformId, configs, choice: choices.webSearch, scope, log }).then((credentials) => withSearchModel({ platformId, credentials, surface, scope, log })),- resolveChosenProvider({ platformId, configs, choice: choices.imageGeneration, scope, log }).then((credentials) => withImageModel({ credentials, modelId: choices.imageGeneration?.modelId, log })),- ])- return { search, image }-}--async function withSearchModel({ platformId, credentials, surface, scope, log }: { platformId: string, credentials: GetProviderConfigResponse | null, surface: ModelTierSurface, scope: ProviderScope, log: FastifyBaseLogger }): Promise<ChosenProvider | null> {- if (isNil(credentials)) {- return null- }- const { data: modelId, error } = await tryCatch(() => agentHelpers.resolveFastModelId({ platformId, providerConfig: credentials, surface, scope, log }))- if (error) {- log.warn({ error, aiProvider: { id: credentials.configId } }, '[agentRpc#getAgentConfig] Chosen search provider has no usable model, using the chat provider')- return null- }- return { credentials, modelId }-}--function withImageModel({ credentials, modelId, log }: { credentials: GetProviderConfigResponse | null, modelId: string | undefined, log: FastifyBaseLogger }): ChosenProvider | null {- if (isNil(credentials) || isNil(modelId)) {- return null- }- const modelAllowed = credentials.modelScope !== 'selected' || credentials.modelIds.includes(modelId)- if (!modelAllowed) {- log.warn({ aiProvider: { id: credentials.configId } }, '[agentRpc#getAgentConfig] Chosen image model is no longer allowed on its key, using the chat provider')- return null- }- return { credentials, modelId }-}--async function resolveChosenProvider({ platformId, configs, choice, scope, log }: { platformId: string, configs: AIProviderWithoutSensitiveData[], choice: AiProviderToolConfig | undefined, scope: ProviderScope, log: FastifyBaseLogger }): Promise<GetProviderConfigResponse | null> {- const row = isNil(choice) ? undefined : configs.find((config) => config.id === choice.aiProviderId)- if (isNil(row)) {- return null- }- const { data, error } = await tryCatch(() => aiProviderService(log).getConfigOrThrow({ platformId, provider: row.provider, scope, configId: row.id }))- if (error) {- log.warn({ error, aiProvider: { id: row.id } }, '[agentRpc#getAgentConfig] Chosen AI provider cannot serve this run, using the chat provider')- }- return data ?? null-}--type ChosenProvider = {- credentials: GetProviderConfigResponse- modelId: string-}--type ChosenProviders = {- search: ChosenProvider | null- image: ChosenProvider | null-}diff --git a/packages/server/api/src/app/ee/agent/rpc/chosen-providers.ts b/packages/server/api/src/app/ee/agent/rpc/chosen-providers.tsnew file mode 100644index 00000000000..d209400b000--- /dev/null+++ b/packages/server/api/src/app/ee/agent/rpc/chosen-providers.ts@@ -0,0 +1,68 @@+import { isNil, tryCatch } from '@activepieces/core-utils'+import { ModelTierSurface } from '@activepieces/server-utils'+import { AiProviderToolChoices, AiProviderToolConfig, AIProviderWithoutSensitiveData, GetProviderConfigResponse } from '@activepieces/shared'+import { FastifyBaseLogger } from 'fastify'+import { aiProviderService, ProviderScope } from '../../../ai/ai-provider-service'+import { agentHelpers } from '../agent-helpers'++async function resolveForRun({ platformId, choices, surface, scope, log }: { platformId: string, choices: AiProviderToolChoices, surface: ModelTierSurface, scope: ProviderScope, log: FastifyBaseLogger }): Promise<ChosenProviders> {+ if (isNil(choices.webSearch) && isNil(choices.imageGeneration)) {+ return { search: null, image: null }+ }+ const configs = await aiProviderService(log).listConfigs(platformId)+ const [search, image] = await Promise.all([+ keyCredentialsForRun({ platformId, configs, choice: choices.webSearch, scope, log }).then((credentials) => searchKeyWithFastModel({ platformId, credentials, surface, scope, log })),+ keyCredentialsForRun({ platformId, configs, choice: choices.imageGeneration, scope, log }).then((credentials) => imageKeyIfModelAllowed({ credentials, modelId: choices.imageGeneration?.modelId, log })),+ ])+ return { search, image }+}++async function keyCredentialsForRun({ platformId, configs, choice, scope, log }: { platformId: string, configs: AIProviderWithoutSensitiveData[], choice: AiProviderToolConfig | undefined, scope: ProviderScope, log: FastifyBaseLogger }): Promise<GetProviderConfigResponse | null> {+ const row = isNil(choice) ? undefined : configs.find((config) => config.id === choice.aiProviderId)+ if (isNil(row)) {+ return null+ }+ const { data, error } = await tryCatch(() => aiProviderService(log).getConfigOrThrow({ platformId, provider: row.provider, scope, configId: row.id }))+ if (error) {+ log.warn({ error, aiProvider: { id: row.id } }, '[chosenProviders#resolveForRun] Chosen AI provider cannot serve this run, using the chat provider')+ }+ return data ?? null+}++async function searchKeyWithFastModel({ platformId, credentials, surface, scope, log }: { platformId: string, credentials: GetProviderConfigResponse | null, surface: ModelTierSurface, scope: ProviderScope, log: FastifyBaseLogger }): Promise<ChosenProvider | null> {+ if (isNil(credentials)) {+ return null+ }+ const { data: modelId, error } = await tryCatch(() => agentHelpers.resolveFastModelId({ platformId, providerConfig: credentials, surface, scope, log }))+ if (error) {+ log.warn({ error, aiProvider: { id: credentials.configId } }, '[chosenProviders#resolveForRun] Chosen search provider has no usable model, using the chat provider')+ return null+ }+ return { credentials, modelId }+}++function imageKeyIfModelAllowed({ credentials, modelId, log }: { credentials: GetProviderConfigResponse | null, modelId: string | undefined, log: FastifyBaseLogger }): ChosenProvider | null {+ if (isNil(credentials) || isNil(modelId)) {+ return null+ }+ const modelAllowed = credentials.modelScope !== 'selected' || credentials.modelIds.includes(modelId)+ if (!modelAllowed) {+ log.warn({ aiProvider: { id: credentials.configId } }, '[chosenProviders#resolveForRun] Chosen image model is no longer allowed on its key, using the chat provider')+ return null+ }+ return { credentials, modelId }+}++export const chosenProviders = {+ resolveForRun,+}++type ChosenProvider = {+ credentials: GetProviderConfigResponse+ modelId: string+}++type ChosenProviders = {+ search: ChosenProvider | null+ image: ChosenProvider | null+}diff --git a/packages/server/api/test/unit/app/ee/agent/chosen-providers.test.ts b/packages/server/api/test/unit/app/ee/agent/chosen-providers.test.tsnew file mode 100644index 00000000000..883f43e4a2d--- /dev/null+++ b/packages/server/api/test/unit/app/ee/agent/chosen-providers.test.ts@@ -0,0 +1,105 @@+import { AIProviderName } from '@activepieces/core-utils'+import { GetProviderConfigResponse } from '@activepieces/shared'+import { beforeEach, describe, expect, it, vi } from 'vitest'++const { mockListConfigs, mockGetConfigOrThrow, mockResolveFastModelId } = vi.hoisted(() => ({+ mockListConfigs: vi.fn(),+ mockGetConfigOrThrow: vi.fn(),+ mockResolveFastModelId: vi.fn(),+}))++vi.mock('../../../../../src/app/ai/ai-provider-service', () => ({+ aiProviderService: () => ({+ listConfigs: mockListConfigs,+ getConfigOrThrow: mockGetConfigOrThrow,+ }),+}))++vi.mock('../../../../../src/app/ee/agent/agent-helpers', () => ({+ agentHelpers: { resolveFastModelId: mockResolveFastModelId },+}))++const { chosenProviders } = await import('../../../../../src/app/ee/agent/rpc/chosen-providers')++const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as never+const platformId = 'platform-1'+const scope = { type: 'project', projectId: 'project-1' } as const++function credentialsFor({ modelScope, modelIds }: { modelScope: 'all' | 'selected', modelIds: string[] }): GetProviderConfigResponse {+ return { provider: AIProviderName.GOOGLE, configId: 'key-1', platformId, auth: { apiKey: 'k' }, config: {}, modelScope, modelIds }+}++describe('chosenProviders.resolveForRun', () => {+ beforeEach(() => {+ mockListConfigs.mockReset().mockResolvedValue([{ id: 'key-1', provider: AIProviderName.GOOGLE }])+ mockGetConfigOrThrow.mockReset().mockResolvedValue(credentialsFor({ modelScope: 'all', modelIds: [] }))+ mockResolveFastModelId.mockReset().mockResolvedValue('gemini-2.5-flash')+ })++ it('skips every lookup when nothing was chosen', async () => {+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: {}, surface: 'chat', scope, log })++ expect(chosen).toEqual({ search: null, image: null })+ expect(mockListConfigs).not.toHaveBeenCalled()+ })++ it('uses the chosen key and its fast model for search, checked against the run scope', async () => {+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { webSearch: { aiProviderId: 'key-1' } }, surface: 'chat', scope, log })++ expect(mockGetConfigOrThrow).toHaveBeenCalledWith({ platformId, provider: AIProviderName.GOOGLE, scope, configId: 'key-1' })+ expect(chosen.search).toMatchObject({ credentials: { configId: 'key-1' }, modelId: 'gemini-2.5-flash' })+ })++ it('falls back to the chat provider when the search key has no usable model', async () => {+ mockResolveFastModelId.mockRejectedValue(new Error('no text model'))++ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { webSearch: { aiProviderId: 'key-1' } }, surface: 'chat', scope, log })++ expect(chosen.search).toBeNull()+ })++ it('falls back when the key no longer serves the run\'s project', async () => {+ mockGetConfigOrThrow.mockRejectedValue(new Error('scoped away'))++ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { webSearch: { aiProviderId: 'key-1' }, imageGeneration: { aiProviderId: 'key-1', modelId: 'img-1' } }, surface: 'chat', scope, log })++ expect(chosen).toEqual({ search: null, image: null })+ })++ it('never resolves a key that is not on the platform', async () => {+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { webSearch: { aiProviderId: 'foreign-key' } }, surface: 'chat', scope, log })++ expect(chosen.search).toBeNull()+ expect(mockGetConfigOrThrow).not.toHaveBeenCalled()+ })++ it('keeps an image model the key still allows', async () => {+ mockGetConfigOrThrow.mockResolvedValue(credentialsFor({ modelScope: 'selected', modelIds: ['img-1'] }))++ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { imageGeneration: { aiProviderId: 'key-1', modelId: 'img-1' } }, surface: 'chat', scope, log })++ expect(chosen.image).toMatchObject({ credentials: { configId: 'key-1' }, modelId: 'img-1' })+ })++ it('keeps any image model on a key that allows every model', async () => {+ mockGetConfigOrThrow.mockResolvedValue(credentialsFor({ modelScope: 'all', modelIds: [] }))++ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { imageGeneration: { aiProviderId: 'key-1', modelId: 'img-1' } }, surface: 'chat', scope, log })++ expect(chosen.image).toMatchObject({ modelId: 'img-1' })+ })++ it('ignores an image choice that names no model', async () => {+ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { imageGeneration: { aiProviderId: 'key-1' } }, surface: 'chat', scope, log })++ expect(chosen.image).toBeNull()+ })++ it('falls back when the saved image model was removed from the key', async () => {+ mockGetConfigOrThrow.mockResolvedValue(credentialsFor({ modelScope: 'selected', modelIds: ['img-2'] }))++ const chosen = await chosenProviders.resolveForRun({ platformId, choices: { imageGeneration: { aiProviderId: 'key-1', modelId: 'img-1' } }, surface: 'chat', scope, log })++ expect(chosen.image).toBeNull()+ })+})