Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(agents): an outage at your own AI provider no longer shows up as a platform failure (#16086)

Hazem Adel289c089876e2
diff --git a/packages/server/worker/src/lib/execute/jobs/ee/agent/run-agent-turn.ts b/packages/server/worker/src/lib/execute/jobs/ee/agent/run-agent-turn.ts
index 474b6530744..41dd461f8e3 100644
--- a/packages/server/worker/src/lib/execute/jobs/ee/agent/run-agent-turn.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ee/agent/run-agent-turn.ts
@@ -403,7 +403,12 @@ export function classifyAgentRunError({ error, provider }: { error: unknown, pro
if (apiError.statusCode === 400 && provider !== AIProviderName.ACTIVEPIECES && MODEL_UNAVAILABLE_PATTERNS.some((pattern) => pattern.test(message))) {
return 'user'
}
- const blamesTheUser = USER_FAULT_STATUS_CODES.has(apiError.statusCode ?? 0) && provider !== AIProviderName.ACTIVEPIECES
+ const ownsTheProvider = provider !== AIProviderName.ACTIVEPIECES
+ const providerUnavailable = ownsTheProvider && (serverSideFault || apiError.statusCode === 429)
+ if (providerUnavailable) {
+ return 'provider'
+ }
+ const blamesTheUser = USER_FAULT_STATUS_CODES.has(apiError.statusCode ?? 0) && ownsTheProvider
return blamesTheUser ? 'user' : 'internal'
}
@@ -527,7 +532,7 @@ export type AgentTurnResult = {
toolCalls: AgentTurnToolCall[]
}
-type AgentRunErrorClass = 'credit' | 'user' | 'internal'
+type AgentRunErrorClass = 'credit' | 'user' | 'provider' | 'internal'
export function firstStepUsesFastModel({ source, dryRun, runsASavedAgent }: { source: AgentRunSource, dryRun?: boolean, runsASavedAgent: boolean }): boolean {
return dryRun !== true && !(source === AgentRunSource.FLOW_STEP && runsASavedAgent)
diff --git a/packages/server/worker/test/lib/execute/jobs/ee/agent/execute-agent-run-config-failure.test.ts b/packages/server/worker/test/lib/execute/jobs/ee/agent/execute-agent-run-config-failure.test.ts
index c45965cda5c..25702cb0cb0 100644
--- a/packages/server/worker/test/lib/execute/jobs/ee/agent/execute-agent-run-config-failure.test.ts
+++ b/packages/server/worker/test/lib/execute/jobs/ee/agent/execute-agent-run-config-failure.test.ts
@@ -1,5 +1,6 @@
import { ActivepiecesError, ErrorCode } from '@activepieces/core-utils'
import { AgentEvent, AgentEventType, AgentRunSource, AI_PROVIDER_ENTITY_TYPES, EngineResponseStatus, ExecuteAgentRunJobData, LATEST_JOB_DATA_SCHEMA_VERSION, WorkerJobType } from '@activepieces/shared'
+import { APICallError, RetryError } from 'ai'
import { describe, expect, it } from 'vitest'
import { executeAgentRunJob } from '../../../../../../src/lib/execute/jobs/ee/agent/execute-agent-run'
import { JobContext } from '../../../../../../src/lib/execute/types'
@@ -99,6 +100,20 @@ describe('executeAgentRunJob — a config failure must not swallow the turn', ()
expect(events[0]).toMatchObject({ type: AgentEventType.ERROR, data: { code: ErrorCode.QUOTA_EXCEEDED } })
})
+ it('completes as a user failure when the customer\'s own provider is overloaded, so it is not reported as ours', async () => {
+ const overloaded = new RetryError({
+ message: 'Failed after 4 attempts',
+ reason: 'maxRetriesExceeded',
+ errors: [new APICallError({ message: 'This model is currently experiencing high demand.', url: 'https://provider.test/v1/chat', requestBodyValues: {}, statusCode: 503 })],
+ })
+ const { ctx, events } = buildContext(overloaded)
+
+ const result = await executeAgentRunJob.execute(ctx, buildJobData({ source: AgentRunSource.CHAT }))
+
+ expect(result.status).toBe(EngineResponseStatus.USER_FAILURE)
+ expect(events.map((event) => event.type)).toEqual([AgentEventType.ERROR, AgentEventType.FINISHED])
+ })
+
it('still fails the job on an unrecognised error, so our own bugs are not laundered', async () => {
const { ctx, events } = buildContext(new Error('Cannot read properties of undefined'))
diff --git a/packages/server/worker/test/lib/execute/jobs/ee/agent/run-agent-turn-guards.test.ts b/packages/server/worker/test/lib/execute/jobs/ee/agent/run-agent-turn-guards.test.ts
index cfced625699..0bf5490a6ce 100644
--- a/packages/server/worker/test/lib/execute/jobs/ee/agent/run-agent-turn-guards.test.ts
+++ b/packages/server/worker/test/lib/execute/jobs/ee/agent/run-agent-turn-guards.test.ts
@@ -66,7 +66,7 @@ describe('classifyAgentRunError', () => {
it.each([
[400, 'internal'], [401, 'user'], [402, 'credit'], [403, 'user'], [404, 'user'], [408, 'internal'],
- [409, 'internal'], [413, 'internal'], [422, 'internal'], [429, 'internal'], [500, 'internal'], [503, 'internal'],
+ [409, 'internal'], [413, 'internal'], [422, 'internal'], [429, 'provider'], [500, 'provider'], [503, 'provider'],
])('classifies a provider %i as %s', (statusCode, expected) => {
expect(classify(apiError({ statusCode, message: 'the provider said no' }))).toBe(expected)
})
@@ -129,8 +129,20 @@ describe('classifyAgentRunError', () => {
})
it('does not let a 5xx error page mentioning credits masquerade as a billing failure', () => {
- expect(classify(apiError({ statusCode: 500, message: 'Bad gateway', responseBody: '<html>Buy more credits</html>' }))).toBe('internal')
- expect(classify(apiError({ statusCode: 503, message: 'Unavailable', responseBody: 'trace-id 402 upstream down' }))).toBe('internal')
+ expect(classify(apiError({ statusCode: 500, message: 'Bad gateway', responseBody: '<html>Buy more credits</html>' }), AIProviderName.ACTIVEPIECES)).toBe('internal')
+ expect(classify(apiError({ statusCode: 503, message: 'Unavailable', responseBody: 'trace-id 402 upstream down' }), AIProviderName.ACTIVEPIECES)).toBe('internal')
+ })
+
+ it('blames an outage at the customer\'s own provider on the provider, but keeps one at ours internal', () => {
+ const highDemand = new RetryError({
+ message: 'Failed after 4 attempts',
+ reason: 'maxRetriesExceeded',
+ errors: [apiError({ statusCode: 503, message: 'This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.' })],
+ })
+ expect(classify(highDemand, AIProviderName.GOOGLE)).toBe('provider')
+ expect(classify(highDemand, AIProviderName.ACTIVEPIECES)).toBe('internal')
+ expect(classify(apiError({ statusCode: 429, message: 'Too Many Requests' }), AIProviderName.OPENAI)).toBe('provider')
+ expect(classify(apiError({ statusCode: 429, message: 'Too Many Requests' }), AIProviderName.ACTIVEPIECES)).toBe('internal')
})
it('reports a real quota rejection as credit, so the client can offer a top-up', () => {