Skip to content

The open alternative to Salesforce, designed for AI.

Only add Bedrock cache points for Claude models (#27929)

Charles Bochet25a22aedc8df
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/provider-options.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/provider-options.util.spec.ts
index f7f0eb83528..fa3f1fab93c 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/provider-options.util.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/provider-options.util.spec.ts
@@ -11,6 +11,12 @@ import {
AI_SDK_BEDROCK,
AI_SDK_OPENAI,
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';
+import { ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';
+
+const BEDROCK_CLAUDE = {
+ sdkPackage: AI_SDK_BEDROCK,
+ modelFamily: ModelFamily.CLAUDE,
+};
describe('provider-options.util', () => {
describe('getCallLevelProviderOptions', () => {
@@ -154,13 +160,22 @@ describe('provider-options.util', () => {
});
describe('getCacheProviderOptions', () => {
- it('returns cache point provider options for Bedrock models', () => {
- expect(getCacheProviderOptions(AI_SDK_BEDROCK)).toEqual({
+ it('returns cache point provider options for Bedrock Claude models', () => {
+ expect(getCacheProviderOptions(BEDROCK_CLAUDE)).toEqual({
bedrock: {
cachePoint: { type: 'default' },
},
});
});
+
+ it('returns no cache point for Bedrock models that do not support it', () => {
+ expect(
+ getCacheProviderOptions({
+ sdkPackage: AI_SDK_BEDROCK,
+ modelFamily: ModelFamily.GPT,
+ }),
+ ).toBeUndefined();
+ });
});
describe('injectCacheBreakpoint', () => {
@@ -179,7 +194,7 @@ describe('provider-options.util', () => {
},
},
],
- AI_SDK_BEDROCK,
+ BEDROCK_CLAUDE,
),
).toEqual([
{ role: 'user', content: 'first' },
@@ -214,7 +229,7 @@ describe('provider-options.util', () => {
},
{ role: 'tool', content: [] },
],
- AI_SDK_BEDROCK,
+ BEDROCK_CLAUDE,
),
).toEqual([
{ role: 'user', content: 'first' },
@@ -244,7 +259,7 @@ describe('provider-options.util', () => {
providerOptions: { bedrock: { guardrail: 'off' } },
},
],
- AI_SDK_BEDROCK,
+ BEDROCK_CLAUDE,
),
).toEqual([
{
@@ -262,7 +277,7 @@ describe('provider-options.util', () => {
]);
});
- it('leaves messages untouched for non-Bedrock providers', () => {
+ it('leaves messages untouched for models without Bedrock cache points', () => {
const messages: ModelMessage[] = [
{ role: 'user', content: 'first' },
{
@@ -272,8 +287,18 @@ describe('provider-options.util', () => {
},
];
- expect(injectCacheBreakpoint(messages, AI_SDK_ANTHROPIC)).toBe(messages);
- expect(injectCacheBreakpoint(messages, AI_SDK_OPENAI)).toBe(messages);
+ expect(
+ injectCacheBreakpoint(messages, {
+ sdkPackage: AI_SDK_ANTHROPIC,
+ modelFamily: ModelFamily.CLAUDE,
+ }),
+ ).toBe(messages);
+ expect(
+ injectCacheBreakpoint(messages, {
+ sdkPackage: AI_SDK_BEDROCK,
+ modelFamily: ModelFamily.GPT,
+ }),
+ ).toBe(messages);
});
});
});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util.ts
index a5def3a735d..7715a759634 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util.ts
@@ -6,22 +6,22 @@ import { isDefined } from 'twenty-shared/utils';
import {
AI_SDK_ANTHROPIC,
AI_SDK_AZURE,
- AI_SDK_BEDROCK,
AI_SDK_OPENAI,
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';
+import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';
+import { supportsBedrockCachePoint } from 'src/engine/metadata-modules/ai/ai-models/utils/supports-bedrock-cache-point.util';
+
+type CacheModel = Pick<AiModelConfig, 'sdkPackage' | 'modelFamily'>;
const BEDROCK_CACHE_POINT = { cachePoint: { type: 'default' } };
export const getCacheProviderOptions = (
- sdkPackage: AiSdkPackage,
-): ProviderOptions | undefined => {
- switch (sdkPackage) {
- case AI_SDK_BEDROCK:
- return { bedrock: { ...BEDROCK_CACHE_POINT } };
- default:
- return undefined;
- }
-};
+ model: CacheModel,
+): ProviderOptions | undefined =>
+ supportsBedrockCachePoint(model)
+ ? { bedrock: { ...BEDROCK_CACHE_POINT } }
+ : undefined;
+
export const getCallLevelProviderOptions = ({
sdkPackage,
providerOptions,
@@ -84,9 +84,11 @@ const omitBedrockCachePoint = (
export const injectCacheBreakpoint = (
messages: ModelMessage[],
- sdkPackage: AiSdkPackage,
+ model: CacheModel,
): ModelMessage[] => {
- if (messages.length === 0 || sdkPackage !== AI_SDK_BEDROCK) return messages;
+ if (messages.length === 0 || !supportsBedrockCachePoint(model)) {
+ return messages;
+ }
const lastIdx = messages.length - 1;
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
index 90d575734f2..4105de83beb 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
@@ -428,7 +428,7 @@ export class ChatExecutionService {
const systemMessage: SystemModelMessage = {
role: 'system',
content: systemPrompt,
- providerOptions: getCacheProviderOptions(registeredModel.sdkPackage),
+ providerOptions: getCacheProviderOptions(modelConfig),
};
const sanitizedMessages = this.sanitizeMessagePartsForModel(
@@ -573,7 +573,7 @@ export class ChatExecutionService {
return {
messages: injectCacheBreakpoint(
messages.map(sortKeysDeeply),
- registeredModel.sdkPackage,
+ modelConfig,
),
};
},
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts
index 27f40d23e05..0e9ad262be2 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts
@@ -7,6 +7,12 @@ import {
injectCacheBreakpoint,
} from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/provider-options.util';
import { AI_SDK_BEDROCK } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';
+import { ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';
+
+const BEDROCK_CLAUDE = {
+ sdkPackage: AI_SDK_BEDROCK,
+ modelFamily: ModelFamily.CLAUDE,
+};
const BEDROCK_MAX_CACHE_POINTS = 4;
const TOOL_STEPS = 4;
@@ -81,7 +87,7 @@ describe('injectCacheBreakpoint against the Bedrock Converse payload', () => {
instructions: {
role: 'system',
content: 'system prompt',
- providerOptions: getCacheProviderOptions(AI_SDK_BEDROCK),
+ providerOptions: getCacheProviderOptions(BEDROCK_CLAUDE),
},
messages: [{ role: 'user', content: 'find my first targets' }],
tools: {
@@ -93,7 +99,7 @@ describe('injectCacheBreakpoint against the Bedrock Converse payload', () => {
},
stopWhen: isStepCount(TOOL_STEPS + 1),
prepareStep: ({ messages }) => ({
- messages: injectCacheBreakpoint(messages, AI_SDK_BEDROCK),
+ messages: injectCacheBreakpoint(messages, BEDROCK_CLAUDE),
}),
});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/bedrock-cache-point-model-families.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/bedrock-cache-point-model-families.constant.ts
new file mode 100644
index 00000000000..6b28ee80297
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/bedrock-cache-point-model-families.constant.ts
@@ -0,0 +1,5 @@
+import { ModelFamily } from 'src/engine/metadata-modules/ai/ai-models/types/model-family.enum';
+
+export const BEDROCK_CACHE_POINT_MODEL_FAMILIES: ModelFamily[] = [
+ ModelFamily.CLAUDE,
+];
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/supports-bedrock-cache-point.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/supports-bedrock-cache-point.util.ts
new file mode 100644
index 00000000000..fa47cab141a
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/supports-bedrock-cache-point.util.ts
@@ -0,0 +1,13 @@
+import { isDefined } from 'twenty-shared/utils';
+
+import { AI_SDK_BEDROCK } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';
+import { BEDROCK_CACHE_POINT_MODEL_FAMILIES } from 'src/engine/metadata-modules/ai/ai-models/constants/bedrock-cache-point-model-families.constant';
+import { type AiModelConfig } from 'src/engine/metadata-modules/ai/ai-models/types/ai-model-config.type';
+
+export const supportsBedrockCachePoint = ({
+ sdkPackage,
+ modelFamily,
+}: Pick<AiModelConfig, 'sdkPackage' | 'modelFamily'>): boolean =>
+ sdkPackage === AI_SDK_BEDROCK &&
+ isDefined(modelFamily) &&
+ BEDROCK_CACHE_POINT_MODEL_FAMILIES.includes(modelFamily);