The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Only add Bedrock cache points for Claude models (#27929)
Charles Bochet25a22aedc8df
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/provider-options.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/provider-options.util.spec.tsindex f7f0eb83528..fa3f1fab93c 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/provider-options.util.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/provider-options.util.spec.ts@@ -11,6 +11,12 @@ import {AI_SDK_BEDROCK,AI_SDK_OPENAI,} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';+import { ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';++const BEDROCK_CLAUDE = {+ sdkPackage: AI_SDK_BEDROCK,+ modelFamily: ModelFamily.CLAUDE,+};describe('provider-options.util', () => {describe('getCallLevelProviderOptions', () => {@@ -154,13 +160,22 @@ describe('provider-options.util', () => {});describe('getCacheProviderOptions', () => {- it('returns cache point provider options for Bedrock models', () => {- expect(getCacheProviderOptions(AI_SDK_BEDROCK)).toEqual({+ it('returns cache point provider options for Bedrock Claude models', () => {+ expect(getCacheProviderOptions(BEDROCK_CLAUDE)).toEqual({bedrock: {cachePoint: { type: 'default' },},});});++ it('returns no cache point for Bedrock models that do not support it', () => {+ expect(+ getCacheProviderOptions({+ sdkPackage: AI_SDK_BEDROCK,+ modelFamily: ModelFamily.GPT,+ }),+ ).toBeUndefined();+ });});describe('injectCacheBreakpoint', () => {@@ -179,7 +194,7 @@ describe('provider-options.util', () => {},},],- AI_SDK_BEDROCK,+ BEDROCK_CLAUDE,),).toEqual([{ role: 'user', content: 'first' },@@ -214,7 +229,7 @@ describe('provider-options.util', () => {},{ role: 'tool', content: [] },],- AI_SDK_BEDROCK,+ BEDROCK_CLAUDE,),).toEqual([{ role: 'user', content: 'first' },@@ -244,7 +259,7 @@ describe('provider-options.util', () => {providerOptions: { bedrock: { guardrail: 'off' } },},],- AI_SDK_BEDROCK,+ BEDROCK_CLAUDE,),).toEqual([{@@ -262,7 +277,7 @@ describe('provider-options.util', () => {]);});- it('leaves messages untouched for non-Bedrock providers', () => {+ it('leaves messages untouched for models without Bedrock cache points', () => {const messages: ModelMessage[] = [{ role: 'user', content: 'first' },{@@ -272,8 +287,18 @@ describe('provider-options.util', () => {},];- expect(injectCacheBreakpoint(messages, AI_SDK_ANTHROPIC)).toBe(messages);- expect(injectCacheBreakpoint(messages, AI_SDK_OPENAI)).toBe(messages);+ expect(+ injectCacheBreakpoint(messages, {+ sdkPackage: AI_SDK_ANTHROPIC,+ modelFamily: ModelFamily.CLAUDE,+ }),+ ).toBe(messages);+ expect(+ injectCacheBreakpoint(messages, {+ sdkPackage: AI_SDK_BEDROCK,+ modelFamily: ModelFamily.GPT,+ }),+ ).toBe(messages);});});});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util.tsindex a5def3a735d..7715a759634 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util.ts@@ -6,22 +6,22 @@ import { isDefined } from 'twenty-shared/utils';import {AI_SDK_ANTHROPIC,AI_SDK_AZURE,- AI_SDK_BEDROCK,AI_SDK_OPENAI,} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';+import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';+import { supportsBedrockCachePoint } from 'src/engine/metadata-modules/ai/ai-models/utils/supports-bedrock-cache-point.util';++type CacheModel = Pick<AiModelConfig, 'sdkPackage' | 'modelFamily'>;const BEDROCK_CACHE_POINT = { cachePoint: { type: 'default' } };export const getCacheProviderOptions = (- sdkPackage: AiSdkPackage,-): ProviderOptions | undefined => {- switch (sdkPackage) {- case AI_SDK_BEDROCK:- return { bedrock: { ...BEDROCK_CACHE_POINT } };- default:- return undefined;- }-};+ model: CacheModel,+): ProviderOptions | undefined =>+ supportsBedrockCachePoint(model)+ ? { bedrock: { ...BEDROCK_CACHE_POINT } }+ : undefined;+export const getCallLevelProviderOptions = ({sdkPackage,providerOptions,@@ -84,9 +84,11 @@ const omitBedrockCachePoint = (export const injectCacheBreakpoint = (messages: ModelMessage[],- sdkPackage: AiSdkPackage,+ model: CacheModel,): ModelMessage[] => {- if (messages.length === 0 || sdkPackage !== AI_SDK_BEDROCK) return messages;+ if (messages.length === 0 || !supportsBedrockCachePoint(model)) {+ return messages;+ }const lastIdx = messages.length - 1;diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.tsindex 90d575734f2..4105de83beb 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts@@ -428,7 +428,7 @@ export class ChatExecutionService {const systemMessage: SystemModelMessage = {role: 'system',content: systemPrompt,- providerOptions: getCacheProviderOptions(registeredModel.sdkPackage),+ providerOptions: getCacheProviderOptions(modelConfig),};const sanitizedMessages = this.sanitizeMessagePartsForModel(@@ -573,7 +573,7 @@ export class ChatExecutionService {return {messages: injectCacheBreakpoint(messages.map(sortKeysDeeply),- registeredModel.sdkPackage,+ modelConfig,),};},diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.tsindex 27f40d23e05..0e9ad262be2 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts@@ -7,6 +7,12 @@ import {injectCacheBreakpoint,} from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util';import { AI_SDK_BEDROCK } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';+import { ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';++const BEDROCK_CLAUDE = {+ sdkPackage: AI_SDK_BEDROCK,+ modelFamily: ModelFamily.CLAUDE,+};const BEDROCK_MAX_CACHE_POINTS = 4;const TOOL_STEPS = 4;@@ -81,7 +87,7 @@ describe('injectCacheBreakpoint against the Bedrock Converse payload', () => {instructions: {role: 'system',content: 'system prompt',- providerOptions: getCacheProviderOptions(AI_SDK_BEDROCK),+ providerOptions: getCacheProviderOptions(BEDROCK_CLAUDE),},messages: [{ role: 'user', content: 'find my first targets' }],tools: {@@ -93,7 +99,7 @@ describe('injectCacheBreakpoint against the Bedrock Converse payload', () => {},stopWhen: isStepCount(TOOL_STEPS + 1),prepareStep: ({ messages }) => ({- messages: injectCacheBreakpoint(messages, AI_SDK_BEDROCK),+ messages: injectCacheBreakpoint(messages, BEDROCK_CLAUDE),}),});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/bedrock-cache-point-model-families.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/bedrock-cache-point-model-families.constant.tsnew file mode 100644index 00000000000..6b28ee80297--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/bedrock-cache-point-model-families.constant.ts@@ -0,0 +1,5 @@+import { ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';++export const BEDROCK_CACHE_POINT_MODEL_FAMILIES: ModelFamily[] = [+ ModelFamily.CLAUDE,+];diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/supports-bedrock-cache-point.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/supports-bedrock-cache-point.util.tsnew file mode 100644index 00000000000..fa47cab141a--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/supports-bedrock-cache-point.util.ts@@ -0,0 +1,13 @@+import { isDefined } from 'twenty-shared/utils';++import { AI_SDK_BEDROCK } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';+import { BEDROCK_CACHE_POINT_MODEL_FAMILIES } from 'src/engine/metadata-modules/ai/ai-models/constants/bedrock-cache-point-model-families.constant';+import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';++export const supportsBedrockCachePoint = ({+ sdkPackage,+ modelFamily,+}: Pick<AiModelConfig, 'sdkPackage' | 'modelFamily'>): boolean =>+ sdkPackage === AI_SDK_BEDROCK &&+ isDefined(modelFamily) &&+ BEDROCK_CACHE_POINT_MODEL_FAMILIES.includes(modelFamily);