Skip to content

The open alternative to Salesforce, designed for AI.

fix(ai-chat) - fix browser context injection (#20809)

Etienne2fcf3e3c2b3d
diff --git a/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts b/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts
index 32cc1bccc7f..e1cbed6694f 100644
--- a/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts
+++ b/packages/twenty-front/src/modules/ai/hooks/useAgentChat.ts
@@ -22,6 +22,7 @@ import {
} from '@/ai/states/agentChatDraftsByThreadIdState';
import { agentChatErrorComponentFamilyState } from '@/ai/states/agentChatErrorComponentFamilyState';
import { agentChatInputState } from '@/ai/states/agentChatInputState';
+import { agentChatLastSentBrowsingContextFamilyState } from '@/ai/states/agentChatLastSentBrowsingContextFamilyState';
import { agentChatMessagesComponentFamilyState } from '@/ai/states/agentChatMessagesComponentFamilyState';
import { agentChatSelectedFilesState } from '@/ai/states/agentChatSelectedFilesState';
import { agentChatUploadedFilesState } from '@/ai/states/agentChatUploadedFilesState';
@@ -96,6 +97,17 @@ export const useAgentChat = (
}));
const browsingContext = getBrowsingContext();
+ const lastSentBrowsingContextAtom =
+ agentChatLastSentBrowsingContextFamilyState.atomFamily(threadId);
+ const lastSentBrowsingContext = store.get(lastSentBrowsingContextAtom);
+ const isBrowsingContextChanged =
+ lastSentBrowsingContext === undefined
+ ? browsingContext !== null
+ : JSON.stringify(browsingContext) !==
+ JSON.stringify(lastSentBrowsingContext);
+ const browsingContextToSend = isBrowsingContextChanged
+ ? browsingContext
+ : null;
const messageId = v4();
const optimisticMessageCreatedAt = new Date().toISOString();
const rollbackOptimisticUnarchive = applyOptimisticUnarchive(
@@ -151,13 +163,17 @@ export const useAgentChat = (
threadId,
text: contentToSend,
messageId,
- browsingContext: browsingContext ?? null,
+ browsingContext: browsingContextToSend,
modelId: modelIdForRequest ?? undefined,
fileAttachments:
fileAttachments.length > 0 ? fileAttachments : undefined,
},
});
+ if (isBrowsingContextChanged) {
+ store.set(lastSentBrowsingContextAtom, browsingContext);
+ }
+
if (data?.sendChatMessage?.queued) {
const latestMessages = store.get(messagesAtom);
diff --git a/packages/twenty-front/src/modules/ai/states/agentChatLastSentBrowsingContextFamilyState.ts b/packages/twenty-front/src/modules/ai/states/agentChatLastSentBrowsingContextFamilyState.ts
new file mode 100644
index 00000000000..eb98f90f52e
--- /dev/null
+++ b/packages/twenty-front/src/modules/ai/states/agentChatLastSentBrowsingContextFamilyState.ts
@@ -0,0 +1,8 @@
+import { createAtomFamilyState } from '@/ui/utilities/state/jotai/utils/createAtomFamilyState';
+import { type BrowsingContext } from '@/ai/types/BrowsingContext';
+
+export const agentChatLastSentBrowsingContextFamilyState =
+ createAtomFamilyState<BrowsingContext | null | undefined, string>({
+ key: 'ai/agentChatLastSentBrowsingContextFamilyState',
+ defaultValue: undefined,
+ });
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const.ts
index cee49dbf140..4fbf9366262 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/chat-system-prompts.const.ts
@@ -52,6 +52,9 @@ For simple CRUD operations (find/create/update/delete a record), you do NOT need
- Validate assumptions before making changes
`,
+ // Browsing context hint
+ BROWSING_CONTEXT_INSTRUCTION: `A <browsing_context> tag may appear in the user's last message. Only use it when directly relevant to the question.`,
+
// Response formatting and record references
RESPONSE_FORMAT: `
Format responses with markdown for clarity (headings, lists, code blocks, tables).
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
index 47cfba114db..a1cc0098a39 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/chat-execution.service.ts
@@ -123,10 +123,6 @@ export class ChatExecutionService {
onCodeExecutionUpdate,
};
- const contextString = browsingContext
- ? this.buildContextFromBrowsingContext(workspace, browsingContext)
- : undefined;
-
const toolCatalog = await this.toolRegistry.buildToolIndex(
workspace.id,
roleId,
@@ -226,11 +222,22 @@ export class ChatExecutionService {
}
}
+ if (isDefined(browsingContext)) {
+ const contextString = this.buildContextFromBrowsingContext(
+ workspace,
+ browsingContext,
+ );
+
+ processedMessages = this.injectBrowsingContextIntoLastUserMessage(
+ processedMessages,
+ contextString,
+ );
+ }
+
const systemPrompt = this.systemPromptBuilder.buildFullPrompt(
toolCatalog,
skillCatalog,
preloadedToolNames,
- contextString,
storedFiles,
workspace.aiAdditionalInstructions ?? undefined,
userContext,
@@ -417,6 +424,34 @@ export class ChatExecutionService {
};
}
+ private injectBrowsingContextIntoLastUserMessage(
+ messages: UIMessage[],
+ contextString: string,
+ ): UIMessage[] {
+ const lastUserIndex = messages
+ .map((message) => message.role)
+ .lastIndexOf('user');
+
+ if (lastUserIndex === -1) {
+ return messages;
+ }
+
+ const lastUserMessage = messages[lastUserIndex];
+ const browsingContextPart = {
+ type: 'text' as const,
+ text: `<browsing_context note="Only use this if the user explicitly asks about the current page, record, or view. Do not call any tools based on this context.">\n${contextString}\n</browsing_context>`,
+ };
+
+ return [
+ ...messages.slice(0, lastUserIndex),
+ {
+ ...lastUserMessage,
+ parts: [...lastUserMessage.parts, browsingContextPart],
+ },
+ ...messages.slice(lastUserIndex + 1),
+ ];
+ }
+
private buildContextFromBrowsingContext(
workspace: WorkspaceEntity,
browsingContext: BrowsingContextType,
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts
index e2802ed6a4c..8eed08ff8c3 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/services/system-prompt-builder.service.ts
@@ -136,7 +136,6 @@ export class SystemPromptBuilderService {
toolCatalog: ToolIndexEntry[],
skillCatalog: FlatSkill[],
preloadedTools: string[],
- contextString?: string,
storedFiles?: Array<{
filename: string;
fileId: string;
@@ -146,6 +145,7 @@ export class SystemPromptBuilderService {
): string {
const parts: string[] = [
CHAT_SYSTEM_PROMPTS.BASE,
+ CHAT_SYSTEM_PROMPTS.BROWSING_CONTEXT_INSTRUCTION,
CHAT_SYSTEM_PROMPTS.RESPONSE_FORMAT,
];
@@ -164,12 +164,6 @@ export class SystemPromptBuilderService {
parts.push(this.buildUploadedFilesSection(storedFiles));
}
- if (contextString) {
- parts.push(
- `\nCONTEXT (what the user is currently viewing):\n${contextString}`,
- );
- }
-
return parts.join('\n');
}