Skip to content

The open alternative to Salesforce, designed for AI.

fix(ai-chat): stop Bedrock cache points from stacking across AI SDK steps (#26355)

Etienne21df6055d351
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts
new file mode 100644
index 00000000000..01176553314
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts
@@ -0,0 +1,116 @@
+import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock';
+import { generateText, isStepCount, tool } from 'ai';
+import { z } from 'zod';
+
+import {
+ getCacheProviderOptions,
+ injectCacheBreakpoint,
+} from 'src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util';
+import { AI_SDK_BEDROCK } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';
+
+const BEDROCK_MAX_CACHE_POINTS = 4;
+const TOOL_STEPS = 4;
+
+type ConverseBlock = Record<string, unknown>;
+type ConverseBody = {
+ system?: ConverseBlock[];
+ messages?: { role: string; content?: ConverseBlock[] }[];
+ toolConfig?: { tools?: ConverseBlock[] };
+};
+
+const countCachePoints = (blocks: ConverseBlock[] = []) =>
+ blocks.filter((block) => 'cachePoint' in block).length;
+
+const countPayloadCachePoints = (body: ConverseBody) =>
+ countCachePoints(body.system) +
+ countCachePoints(body.toolConfig?.tools) +
+ (body.messages ?? []).reduce(
+ (total, message) => total + countCachePoints(message.content),
+ 0,
+ );
+
+describe('injectCacheBreakpoint against the Bedrock Converse payload', () => {
+ beforeAll(() => jest.useRealTimers());
+
+ it('never sends more cache points than Bedrock accepts, however many tool steps a turn takes', async () => {
+ const cachePointsPerCall: number[] = [];
+ const payloads: ConverseBody[] = [];
+ let call = 0;
+
+ const bedrock = createAmazonBedrock({
+ region: 'us-west-2',
+ accessKeyId: 'test-access-key-id',
+ secretAccessKey: 'test-secret-access-key',
+ fetch: (async (_url: string, init: { body: string }) => {
+ const body: ConverseBody = JSON.parse(init.body);
+
+ payloads.push(body);
+ cachePointsPerCall.push(countPayloadCachePoints(body));
+ call += 1;
+
+ const isLastCall = call > TOOL_STEPS;
+
+ return new Response(
+ JSON.stringify({
+ output: {
+ message: {
+ role: 'assistant',
+ content: isLastCall
+ ? [{ text: 'done' }]
+ : [
+ {
+ toolUse: {
+ toolUseId: `call-${call}`,
+ name: 'search',
+ input: {},
+ },
+ },
+ ],
+ },
+ },
+ stopReason: isLastCall ? 'end_turn' : 'tool_use',
+ usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+ }),
+ { status: 200, headers: { 'content-type': 'application/json' } },
+ );
+ }) as typeof fetch,
+ });
+
+ await generateText({
+ model: bedrock('us.anthropic.claude-sonnet-5'),
+ instructions: {
+ role: 'system',
+ content: 'system prompt',
+ providerOptions: getCacheProviderOptions(AI_SDK_BEDROCK),
+ },
+ messages: [{ role: 'user', content: 'find my first targets' }],
+ tools: {
+ search: tool({
+ description: 'search records',
+ inputSchema: z.object({}),
+ execute: async () => 'ok',
+ }),
+ },
+ stopWhen: isStepCount(TOOL_STEPS + 1),
+ prepareStep: ({ messages }) => ({
+ messages: injectCacheBreakpoint(messages, AI_SDK_BEDROCK),
+ }),
+ });
+
+ expect(
+ cachePointsPerCall.every((count) => count <= BEDROCK_MAX_CACHE_POINTS),
+ ).toBe(true);
+ expect(cachePointsPerCall).toEqual(new Array(TOOL_STEPS + 1).fill(2));
+
+ const lastPayload = payloads[payloads.length - 1];
+ const messages = lastPayload.messages ?? [];
+
+ expect(countCachePoints(lastPayload.system)).toBe(1);
+ expect(
+ messages.slice(0, -1).flatMap((message) => message.content ?? []),
+ ).not.toContainEqual({ cachePoint: { type: 'default' } });
+ expect(messages[messages.length - 1].content).toContainEqual({
+ cachePoint: { type: 'default' },
+ });
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/provider-options.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/provider-options.util.spec.ts
index 6b0ac46d3a1..18532c99a68 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/provider-options.util.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/provider-options.util.spec.ts
@@ -1,3 +1,5 @@
+import { type ModelMessage } from 'ai';
+
import {
getCallLevelProviderOptions,
getCacheProviderOptions,
@@ -195,5 +197,83 @@ describe('provider-options.util', () => {
},
]);
});
+
+ it('moves the cache point off earlier messages that already carry one', () => {
+ expect(
+ injectCacheBreakpoint(
+ [
+ {
+ role: 'user',
+ content: 'first',
+ providerOptions: { bedrock: { cachePoint: { type: 'default' } } },
+ },
+ {
+ role: 'tool',
+ content: [],
+ providerOptions: { bedrock: { cachePoint: { type: 'default' } } },
+ },
+ { role: 'tool', content: [] },
+ ],
+ AI_SDK_BEDROCK,
+ ),
+ ).toEqual([
+ { role: 'user', content: 'first' },
+ { role: 'tool', content: [] },
+ {
+ role: 'tool',
+ content: [],
+ providerOptions: { bedrock: { cachePoint: { type: 'default' } } },
+ },
+ ]);
+ });
+
+ it('keeps bedrock provider options other than the cache point', () => {
+ expect(
+ injectCacheBreakpoint(
+ [
+ {
+ role: 'user',
+ content: 'first',
+ providerOptions: {
+ bedrock: { cachePoint: { type: 'default' }, guardrail: 'on' },
+ },
+ },
+ {
+ role: 'user',
+ content: 'last',
+ providerOptions: { bedrock: { guardrail: 'off' } },
+ },
+ ],
+ AI_SDK_BEDROCK,
+ ),
+ ).toEqual([
+ {
+ role: 'user',
+ content: 'first',
+ providerOptions: { bedrock: { guardrail: 'on' } },
+ },
+ {
+ role: 'user',
+ content: 'last',
+ providerOptions: {
+ bedrock: { guardrail: 'off', cachePoint: { type: 'default' } },
+ },
+ },
+ ]);
+ });
+
+ it('leaves messages untouched for non-Bedrock providers', () => {
+ const messages: ModelMessage[] = [
+ { role: 'user', content: 'first' },
+ {
+ role: 'user',
+ content: 'last',
+ providerOptions: { bedrock: { cachePoint: { type: 'default' } } },
+ },
+ ];
+
+ expect(injectCacheBreakpoint(messages, AI_SDK_ANTHROPIC)).toBe(messages);
+ expect(injectCacheBreakpoint(messages, AI_SDK_OPENAI)).toBe(messages);
+ });
});
});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util.ts
index c4d6650742f..a5def3a735d 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util.ts
@@ -1,6 +1,7 @@
import { type ProviderOptions } from '@ai-sdk/provider-utils';
import { type ModelMessage } from 'ai';
import { type AiSdkPackage } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
import {
AI_SDK_ANTHROPIC,
@@ -9,12 +10,14 @@ import {
AI_SDK_OPENAI,
} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';
+const BEDROCK_CACHE_POINT = { cachePoint: { type: 'default' } };
+
export const getCacheProviderOptions = (
sdkPackage: AiSdkPackage,
): ProviderOptions | undefined => {
switch (sdkPackage) {
case AI_SDK_BEDROCK:
- return { bedrock: { cachePoint: { type: 'default' } } };
+ return { bedrock: { ...BEDROCK_CACHE_POINT } };
default:
return undefined;
}
@@ -56,26 +59,47 @@ export const getCallLevelProviderOptions = ({
}
};
+const omitBedrockCachePoint = (
+ providerOptions: ProviderOptions | undefined,
+): ProviderOptions | undefined => {
+ const bedrock = providerOptions?.bedrock;
+
+ if (!isDefined(providerOptions) || !isDefined(bedrock)) {
+ return providerOptions;
+ }
+
+ const { bedrock: _bedrock, ...otherProviderOptions } = providerOptions;
+ const { cachePoint: _cachePoint, ...otherBedrockOptions } = bedrock;
+
+ if (Object.keys(otherBedrockOptions).length > 0) {
+ return { ...otherProviderOptions, bedrock: otherBedrockOptions };
+ }
+
+ if (Object.keys(otherProviderOptions).length > 0) {
+ return otherProviderOptions;
+ }
+
+ return undefined;
+};
+
export const injectCacheBreakpoint = (
messages: ModelMessage[],
sdkPackage: AiSdkPackage,
): ModelMessage[] => {
- if (messages.length === 0) return messages;
-
- const cacheOptions = getCacheProviderOptions(sdkPackage);
-
- if (!cacheOptions) return messages;
+ if (messages.length === 0 || sdkPackage !== AI_SDK_BEDROCK) return messages;
const lastIdx = messages.length - 1;
return messages.map((message, index) => {
- if (index !== lastIdx) return message;
+ const providerOptions = omitBedrockCachePoint(message.providerOptions);
+
+ if (index !== lastIdx) return { ...message, providerOptions };
return {
...message,
providerOptions: {
- ...(message.providerOptions ?? {}),
- ...cacheOptions,
+ ...providerOptions,
+ bedrock: { ...providerOptions?.bedrock, ...BEDROCK_CACHE_POINT },
},
};
});