The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(ai-chat): stop Bedrock cache points from stacking across AI SDK steps (#26355)
Etienne21df6055d351
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.tsnew file mode 100644index 00000000000..01176553314--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/inject-cache-breakpoint-bedrock-payload.spec.ts@@ -0,0 +1,116 @@+import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock';+import { generateText, isStepCount, tool } from 'ai';+import { z } from 'zod';++import {+ getCacheProviderOptions,+ injectCacheBreakpoint,+} from 'src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util';+import { AI_SDK_BEDROCK } from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';++const BEDROCK_MAX_CACHE_POINTS = 4;+const TOOL_STEPS = 4;++type ConverseBlock = Record<string, unknown>;+type ConverseBody = {+ system?: ConverseBlock[];+ messages?: { role: string; content?: ConverseBlock[] }[];+ toolConfig?: { tools?: ConverseBlock[] };+};++const countCachePoints = (blocks: ConverseBlock[] = []) =>+ blocks.filter((block) => 'cachePoint' in block).length;++const countPayloadCachePoints = (body: ConverseBody) =>+ countCachePoints(body.system) ++ countCachePoints(body.toolConfig?.tools) ++ (body.messages ?? []).reduce(+ (total, message) => total + countCachePoints(message.content),+ 0,+ );++describe('injectCacheBreakpoint against the Bedrock Converse payload', () => {+ beforeAll(() => jest.useRealTimers());++ it('never sends more cache points than Bedrock accepts, however many tool steps a turn takes', async () => {+ const cachePointsPerCall: number[] = [];+ const payloads: ConverseBody[] = [];+ let call = 0;++ const bedrock = createAmazonBedrock({+ region: 'us-west-2',+ accessKeyId: 'test-access-key-id',+ secretAccessKey: 'test-secret-access-key',+ fetch: (async (_url: string, init: { body: string }) => {+ const body: ConverseBody = JSON.parse(init.body);++ payloads.push(body);+ cachePointsPerCall.push(countPayloadCachePoints(body));+ call += 1;++ const isLastCall = call > TOOL_STEPS;++ return new Response(+ JSON.stringify({+ output: {+ message: {+ role: 'assistant',+ content: isLastCall+ ? [{ text: 'done' }]+ : [+ {+ toolUse: {+ toolUseId: `call-${call}`,+ name: 'search',+ input: {},+ },+ },+ ],+ },+ },+ stopReason: isLastCall ? 'end_turn' : 'tool_use',+ usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },+ }),+ { status: 200, headers: { 'content-type': 'application/json' } },+ );+ }) as typeof fetch,+ });++ await generateText({+ model: bedrock('us.anthropic.claude-sonnet-5'),+ instructions: {+ role: 'system',+ content: 'system prompt',+ providerOptions: getCacheProviderOptions(AI_SDK_BEDROCK),+ },+ messages: [{ role: 'user', content: 'find my first targets' }],+ tools: {+ search: tool({+ description: 'search records',+ inputSchema: z.object({}),+ execute: async () => 'ok',+ }),+ },+ stopWhen: isStepCount(TOOL_STEPS + 1),+ prepareStep: ({ messages }) => ({+ messages: injectCacheBreakpoint(messages, AI_SDK_BEDROCK),+ }),+ });++ expect(+ cachePointsPerCall.every((count) => count <= BEDROCK_MAX_CACHE_POINTS),+ ).toBe(true);+ expect(cachePointsPerCall).toEqual(new Array(TOOL_STEPS + 1).fill(2));++ const lastPayload = payloads[payloads.length - 1];+ const messages = lastPayload.messages ?? [];++ expect(countCachePoints(lastPayload.system)).toBe(1);+ expect(+ messages.slice(0, -1).flatMap((message) => message.content ?? []),+ ).not.toContainEqual({ cachePoint: { type: 'default' } });+ expect(messages[messages.length - 1].content).toContainEqual({+ cachePoint: { type: 'default' },+ });+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/provider-options.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/provider-options.util.spec.tsindex 6b0ac46d3a1..18532c99a68 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/provider-options.util.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/provider-options.util.spec.ts@@ -1,3 +1,5 @@+import { type ModelMessage } from 'ai';+import {getCallLevelProviderOptions,getCacheProviderOptions,@@ -195,5 +197,83 @@ describe('provider-options.util', () => {},]);});++ it('moves the cache point off earlier messages that already carry one', () => {+ expect(+ injectCacheBreakpoint(+ [+ {+ role: 'user',+ content: 'first',+ providerOptions: { bedrock: { cachePoint: { type: 'default' } } },+ },+ {+ role: 'tool',+ content: [],+ providerOptions: { bedrock: { cachePoint: { type: 'default' } } },+ },+ { role: 'tool', content: [] },+ ],+ AI_SDK_BEDROCK,+ ),+ ).toEqual([+ { role: 'user', content: 'first' },+ { role: 'tool', content: [] },+ {+ role: 'tool',+ content: [],+ providerOptions: { bedrock: { cachePoint: { type: 'default' } } },+ },+ ]);+ });++ it('keeps bedrock provider options other than the cache point', () => {+ expect(+ injectCacheBreakpoint(+ [+ {+ role: 'user',+ content: 'first',+ providerOptions: {+ bedrock: { cachePoint: { type: 'default' }, guardrail: 'on' },+ },+ },+ {+ role: 'user',+ content: 'last',+ providerOptions: { bedrock: { guardrail: 'off' } },+ },+ ],+ AI_SDK_BEDROCK,+ ),+ ).toEqual([+ {+ role: 'user',+ content: 'first',+ providerOptions: { bedrock: { guardrail: 'on' } },+ },+ {+ role: 'user',+ content: 'last',+ providerOptions: {+ bedrock: { guardrail: 'off', cachePoint: { type: 'default' } },+ },+ },+ ]);+ });++ it('leaves messages untouched for non-Bedrock providers', () => {+ const messages: ModelMessage[] = [+ { role: 'user', content: 'first' },+ {+ role: 'user',+ content: 'last',+ providerOptions: { bedrock: { cachePoint: { type: 'default' } } },+ },+ ];++ expect(injectCacheBreakpoint(messages, AI_SDK_ANTHROPIC)).toBe(messages);+ expect(injectCacheBreakpoint(messages, AI_SDK_OPENAI)).toBe(messages);+ });});});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util.tsindex c4d6650742f..a5def3a735d 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/provider-options.util.ts@@ -1,6 +1,7 @@import { type ProviderOptions } from '@ai-sdk/provider-utils';import { type ModelMessage } from 'ai';import { type AiSdkPackage } from 'twenty-shared/ai';+import { isDefined } from 'twenty-shared/utils';import {AI_SDK_ANTHROPIC,@@ -9,12 +10,14 @@ import {AI_SDK_OPENAI,} from 'src/engine/metadata-modules/ai/ai-models/constants/ai-sdk-package.const';+const BEDROCK_CACHE_POINT = { cachePoint: { type: 'default' } };+export const getCacheProviderOptions = (sdkPackage: AiSdkPackage,): ProviderOptions | undefined => {switch (sdkPackage) {case AI_SDK_BEDROCK:- return { bedrock: { cachePoint: { type: 'default' } } };+ return { bedrock: { ...BEDROCK_CACHE_POINT } };default:return undefined;}@@ -56,26 +59,47 @@ export const getCallLevelProviderOptions = ({}};+const omitBedrockCachePoint = (+ providerOptions: ProviderOptions | undefined,+): ProviderOptions | undefined => {+ const bedrock = providerOptions?.bedrock;++ if (!isDefined(providerOptions) || !isDefined(bedrock)) {+ return providerOptions;+ }++ const { bedrock: _bedrock, ...otherProviderOptions } = providerOptions;+ const { cachePoint: _cachePoint, ...otherBedrockOptions } = bedrock;++ if (Object.keys(otherBedrockOptions).length > 0) {+ return { ...otherProviderOptions, bedrock: otherBedrockOptions };+ }++ if (Object.keys(otherProviderOptions).length > 0) {+ return otherProviderOptions;+ }++ return undefined;+};+export const injectCacheBreakpoint = (messages: ModelMessage[],sdkPackage: AiSdkPackage,): ModelMessage[] => {- if (messages.length === 0) return messages;-- const cacheOptions = getCacheProviderOptions(sdkPackage);-- if (!cacheOptions) return messages;+ if (messages.length === 0 || sdkPackage !== AI_SDK_BEDROCK) return messages;const lastIdx = messages.length - 1;return messages.map((message, index) => {- if (index !== lastIdx) return message;+ const providerOptions = omitBedrockCachePoint(message.providerOptions);++ if (index !== lastIdx) return { ...message, providerOptions };return {...message,providerOptions: {- ...(message.providerOptions ?? {}),- ...cacheOptions,+ ...providerOptions,+ bedrock: { ...providerOptions?.bedrock, ...BEDROCK_CACHE_POINT },},};});