The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix(ai): degrade unsupported runAgent attachments instead of failing the run (#26177)
Abdul Rahman74ee19ead7e4
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/run-agent-attachment.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/run-agent-attachment.service.spec.tsindex ac2aefc451c..7af001e3f80 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/run-agent-attachment.service.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/run-agent-attachment.service.spec.ts@@ -2,6 +2,7 @@ import { FileFolder } from 'twenty-shared/types';import { In, Like } from 'typeorm';import { RunAgentAttachmentService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service';+import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception';const WORKSPACE_ID = 'workspace-id';const FILE_ID = '6fdbd0ba-9a10-4b71-b0b7-2b1d64f8dc27';@@ -27,6 +28,7 @@ describe('RunAgentAttachmentService', () => {{ role: 'user', content: 'hi', attachments: [{ fileId: FILE_ID }] },],workspaceId: WORKSPACE_ID,+ modalities: ['image'],});expect(fileRepository.find).toHaveBeenCalledWith(WORKSPACE_ID, {@@ -51,8 +53,115 @@ describe('RunAgentAttachmentService', () => {await service.buildModelMessagesOrThrow({messages: [{ role: 'user', content: 'hello' }],workspaceId: WORKSPACE_ID,+ modalities: ['image'],}),).toEqual([{ role: 'user', content: 'hello' }]);expect(fileRepository.find).not.toHaveBeenCalled();});++ it('hands a supported attachment to the model as a file part', async () => {+ const fileRepository = {+ find: jest+ .fn()+ .mockResolvedValue([{ id: FILE_ID, mimeType: 'image/png' }]),+ };+ const service = new RunAgentAttachmentService(+ fileRepository as never,+ {+ signFileByIdUrl: jest+ .fn()+ .mockResolvedValue('https://twenty.test/file'),+ } as never,+ );++ expect(+ await service.buildModelMessagesOrThrow({+ messages: [+ {+ role: 'user',+ content: 'what is in this?',+ attachments: [{ fileId: FILE_ID, filename: 'screenshot.png' }],+ },+ ],+ workspaceId: WORKSPACE_ID,+ modalities: ['image'],+ }),+ ).toEqual([+ {+ role: 'user',+ content: [+ { type: 'text', text: 'what is in this?' },+ {+ type: 'file',+ data: 'https://twenty.test/file',+ mediaType: 'image/png',+ filename: 'screenshot.png',+ },+ ],+ },+ ]);+ });++ it('degrades an attachment the resolved model cannot read into text', async () => {+ const fileRepository = {+ find: jest+ .fn()+ .mockResolvedValue([{ id: FILE_ID, mimeType: 'application/pdf' }]),+ };+ const service = new RunAgentAttachmentService(+ fileRepository as never,+ {+ signFileByIdUrl: jest+ .fn()+ .mockResolvedValue('https://twenty.test/file'),+ } as never,+ );++ expect(+ await service.buildModelMessagesOrThrow({+ messages: [+ {+ role: 'user',+ content: 'summarize this',+ attachments: [{ fileId: FILE_ID, filename: 'report.pdf' }],+ },+ ],+ workspaceId: WORKSPACE_ID,+ modalities: ['image'],+ }),+ ).toEqual([+ {+ role: 'user',+ content: [+ { type: 'text', text: 'summarize this' },+ {+ type: 'text',+ text: '[Attached file: report.pdf (type: application/pdf) — file type is not supported for direct analysis]',+ },+ ],+ },+ ]);+ });++ it('still refuses a file id that resolves to nothing in the workspace', async () => {+ const fileRepository = { find: jest.fn().mockResolvedValue([]) };+ const service = new RunAgentAttachmentService(+ fileRepository as never,+ {+ signFileByIdUrl: jest.fn(),+ } as never,+ );++ await expect(+ service.buildModelMessagesOrThrow({+ messages: [+ { role: 'user', content: 'hi', attachments: [{ fileId: FILE_ID }] },+ ],+ workspaceId: WORKSPACE_ID,+ modalities: ['image'],+ }),+ ).rejects.toThrow(+ expect.objectContaining({ code: AiExceptionCode.INVALID_AGENT_INPUT }),+ );+ });});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.tsindex f590d2afffc..908cea19000 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts@@ -386,6 +386,9 @@ export class AgentAsyncExecutorService {await this.runAgentAttachmentService.buildModelMessagesOrThrow({messages,workspaceId,+ modalities: this.aiModelRegistryService.getModelConfig(+ registeredModel.modelId,+ )?.modalities,});const textResponse = await generateText({diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service.tsindex 0c3193c203d..e63a177aebe 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service.ts@@ -26,9 +26,11 @@ export class RunAgentAttachmentService {async buildModelMessagesOrThrow({messages,workspaceId,+ modalities,}: {messages: RunAgentMessage[];workspaceId: string;+ modalities: string[] | undefined;}): Promise<ModelMessage[]> {const fileIds = extractRunAgentAttachmentFileIdsOrThrow(messages);@@ -37,7 +39,11 @@ export class RunAgentAttachmentService {: new Map<string, ResolvedRunAgentAttachment>();return messages.map((message) =>- buildRunAgentModelMessageOrThrow({ message, attachmentsByFileId }),+ buildRunAgentModelMessageOrThrow({+ message,+ attachmentsByFileId,+ modalities,+ }),);}diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-run-agent-model-message.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-run-agent-model-message.util.spec.tsindex 75bb5ecea22..62ae63ff37a 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-run-agent-model-message.util.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-run-agent-model-message.util.spec.ts@@ -3,9 +3,12 @@ import { buildRunAgentModelMessageOrThrow } from 'src/engine/metadata-modules/aiconst FILE_ID = '6fdbd0ba-9a10-4b71-b0b7-2b1d64f8dc27';const FILE_URL = `https://twenty.test/file/agent-chat/${FILE_ID}?token=t`;+const PDF_FILE_ID = 'a2a0d40b-bd5f-4f12-9a0b-06ec8c3f5fd1';+const PDF_FILE_URL = `https://twenty.test/file/agent-chat/${PDF_FILE_ID}?token=t`;const attachmentsByFileId = new Map<string, ResolvedRunAgentAttachment>([[FILE_ID, { mediaType: 'image/png', url: FILE_URL }],+ [PDF_FILE_ID, { mediaType: 'application/pdf', url: PDF_FILE_URL }],]);describe('buildRunAgentModelMessageOrThrow', () => {@@ -14,6 +17,7 @@ describe('buildRunAgentModelMessageOrThrow', () => {buildRunAgentModelMessageOrThrow({message: { role: 'assistant', content: 'hi' },attachmentsByFileId,+ modalities: ['image'],}),).toEqual({ role: 'assistant', content: 'hi' });});@@ -27,6 +31,7 @@ describe('buildRunAgentModelMessageOrThrow', () => {attachments: [{ fileId: FILE_ID, filename: 'screenshot.png' }],},attachmentsByFileId,+ modalities: ['image'],}),).toEqual({role: 'user',@@ -50,6 +55,7 @@ describe('buildRunAgentModelMessageOrThrow', () => {attachments: [{ fileId: FILE_ID }],},attachmentsByFileId,+ modalities: ['image'],});expect(message.content).toEqual([@@ -66,7 +72,123 @@ describe('buildRunAgentModelMessageOrThrow', () => {attachments: [{ fileId: FILE_ID }],},attachmentsByFileId: new Map(),+ modalities: ['image'],}),).toThrow(FILE_ID);});++ it('degrades an attachment the model cannot read into a text part', () => {+ expect(+ buildRunAgentModelMessageOrThrow({+ message: {+ role: 'user',+ content: 'summarize this',+ attachments: [{ fileId: PDF_FILE_ID, filename: 'report.pdf' }],+ },+ attachmentsByFileId,+ modalities: ['image'],+ }),+ ).toEqual({+ role: 'user',+ content: [+ { type: 'text', text: 'summarize this' },+ {+ type: 'text',+ text: '[Attached file: report.pdf (type: application/pdf) — file type is not supported for direct analysis]',+ },+ ],+ });+ });++ it('names an unsupported attachment that carries no filename', () => {+ expect(+ buildRunAgentModelMessageOrThrow({+ message: {+ role: 'user',+ content: '',+ attachments: [{ fileId: PDF_FILE_ID }],+ },+ attachmentsByFileId,+ modalities: ['image'],+ }).content,+ ).toEqual([+ {+ type: 'text',+ text: '[Attached file: uploaded_file (type: application/pdf) — file type is not supported for direct analysis]',+ },+ ]);+ });++ it('keeps the supported attachment and degrades only the unsupported one', () => {+ expect(+ buildRunAgentModelMessageOrThrow({+ message: {+ role: 'user',+ content: 'compare these',+ attachments: [+ { fileId: FILE_ID, filename: 'screenshot.png' },+ { fileId: PDF_FILE_ID, filename: 'report.pdf' },+ ],+ },+ attachmentsByFileId,+ modalities: ['image'],+ }),+ ).toEqual({+ role: 'user',+ content: [+ { type: 'text', text: 'compare these' },+ {+ type: 'file',+ data: FILE_URL,+ mediaType: 'image/png',+ filename: 'screenshot.png',+ },+ {+ type: 'text',+ text: '[Attached file: report.pdf (type: application/pdf) — file type is not supported for direct analysis]',+ },+ ],+ });+ });++ it('degrades every attachment when the model declares no modalities', () => {+ expect(+ buildRunAgentModelMessageOrThrow({+ message: {+ role: 'user',+ content: 'what is in this?',+ attachments: [{ fileId: FILE_ID, filename: 'screenshot.png' }],+ },+ attachmentsByFileId,+ modalities: undefined,+ }).content,+ ).toEqual([+ { type: 'text', text: 'what is in this?' },+ {+ type: 'text',+ text: '[Attached file: screenshot.png (type: image/png) — file type is not supported for direct analysis]',+ },+ ]);+ });++ it('labels an attachment stored without a media type as unknown', () => {+ expect(+ buildRunAgentModelMessageOrThrow({+ message: {+ role: 'user',+ content: '',+ attachments: [{ fileId: FILE_ID, filename: 'mystery.bin' }],+ },+ attachmentsByFileId: new Map([+ [FILE_ID, { mediaType: '', url: FILE_URL }],+ ]),+ modalities: ['image'],+ }).content,+ ).toEqual([+ {+ type: 'text',+ text: '[Attached file: mystery.bin (type: unknown) — file type is not supported for direct analysis]',+ },+ ]);+ });});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-run-agent-model-message.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-run-agent-model-message.util.tsindex 6bfa9d860d2..5b0bbe39164 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-run-agent-model-message.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-run-agent-model-message.util.ts@@ -1,10 +1,12 @@import { isNonEmptyString } from '@sniptt/guards';-import { type FilePart, type ModelMessage } from 'ai';+import { type FilePart, type ModelMessage, type TextPart } from 'ai';import { type RunAgentMessage } from 'twenty-shared/application';import { FileFolder } from 'twenty-shared/types';import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';import { type ResolvedRunAgentAttachment } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/resolved-run-agent-attachment.type';+import { formatUnsupportedFilePlaceholder } from 'src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util';+import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util';import {AiException,AiExceptionCode,@@ -13,9 +15,11 @@ import {export const buildRunAgentModelMessageOrThrow = ({message,attachmentsByFileId,+ modalities,}: {message: RunAgentMessage;attachmentsByFileId: Map<string, ResolvedRunAgentAttachment>;+ modalities: string[] | undefined;}): ModelMessage => {const attachments = message.attachments ?? [];@@ -23,7 +27,9 @@ export const buildRunAgentModelMessageOrThrow = ({return { role: message.role, content: message.content };}- const fileParts = attachments.map((attachment): FilePart => {+ const supportedMediaTypes = getNativeMimeTypesForModalities(modalities);++ const attachmentParts = attachments.map((attachment): FilePart | TextPart => {const resolvedAttachment = attachmentsByFileId.get(attachment.fileId);if (!isDefined(resolvedAttachment)) {@@ -33,6 +39,16 @@ export const buildRunAgentModelMessageOrThrow = ({);}+ if (!supportedMediaTypes.has(resolvedAttachment.mediaType)) {+ return {+ type: 'text',+ text: formatUnsupportedFilePlaceholder({+ filename: attachment.filename,+ mediaType: resolvedAttachment.mediaType,+ }),+ };+ }+return {type: 'file',data: resolvedAttachment.url,@@ -47,7 +63,7 @@ export const buildRunAgentModelMessageOrThrow = ({...(isNonEmptyString(message.content)? [{ type: 'text' as const, text: message.content }]: []),- ...fileParts,+ ...attachmentParts,],};};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util.tsindex edd3a5a365c..eac75df4eb1 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util.ts@@ -1,7 +1,8 @@import { type ExtendedUIMessage, isExtendedFileUIPart } from 'twenty-shared/ai';import { CODE_INTERPRETER_MIME_TYPES } from 'src/engine/metadata-modules/ai/ai-chat/constants/code-interpreter-mime-types.constant';-import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util';+import { formatUnsupportedFilePlaceholder } from 'src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util';+import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util';export const replaceUnsupportedFileParts = (messages: ExtendedUIMessage[],@@ -29,10 +30,12 @@ export const replaceUnsupportedFileParts = (if (isSupported) {newParts.push(part);} else {- const filename = part.filename ?? 'uploaded_file';newParts.push({type: 'text',- text: `[Attached file: ${filename} (type: ${mimeType || 'unknown'}) — file type is not supported for direct analysis]`,+ text: formatUnsupportedFilePlaceholder({+ filename: part.filename,+ mediaType: mimeType,+ }),});}} else {diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/modality-to-mime-types.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/modality-to-mime-types.constant.tssimilarity index 100%rename from packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/modality-to-mime-types.constant.tsrename to packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/modality-to-mime-types.constant.tsdiff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/format-unsupported-file-placeholder.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/format-unsupported-file-placeholder.util.spec.tsnew file mode 100644index 00000000000..c98db63c3ce--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/format-unsupported-file-placeholder.util.spec.ts@@ -0,0 +1,36 @@+import { formatUnsupportedFilePlaceholder } from 'src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util';++describe('formatUnsupportedFilePlaceholder', () => {+ it('names the file and its type', () => {+ expect(+ formatUnsupportedFilePlaceholder({+ filename: 'report.pdf',+ mediaType: 'application/pdf',+ }),+ ).toBe(+ '[Attached file: report.pdf (type: application/pdf) — file type is not supported for direct analysis]',+ );+ });++ it('falls back to a generic name when the file carries none', () => {+ expect(+ formatUnsupportedFilePlaceholder({+ filename: undefined,+ mediaType: 'application/pdf',+ }),+ ).toBe(+ '[Attached file: uploaded_file (type: application/pdf) — file type is not supported for direct analysis]',+ );+ });++ it('labels a missing media type as unknown', () => {+ expect(+ formatUnsupportedFilePlaceholder({+ filename: 'mystery.bin',+ mediaType: '',+ }),+ ).toBe(+ '[Attached file: mystery.bin (type: unknown) — file type is not supported for direct analysis]',+ );+ });+});diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-native-mime-types-for-modalities.util.spec.tssimilarity index 96%rename from packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/get-native-mime-types-for-modalities.util.spec.tsrename to packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-native-mime-types-for-modalities.util.spec.tsindex 2aa6e3d9f40..19c269ccbb5 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts@@ -1,4 +1,4 @@-import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util';+import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util';describe('getNativeMimeTypesForModalities', () => {it('returns an empty set when no modalities are provided', () => {diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util.tsnew file mode 100644index 00000000000..b9c81f67ce9--- /dev/null+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util.ts@@ -0,0 +1,21 @@+import { isNonEmptyString } from '@sniptt/guards';++const DEFAULT_ATTACHMENT_FILENAME = 'uploaded_file';+const DEFAULT_ATTACHMENT_MEDIA_TYPE = 'unknown';++export const formatUnsupportedFilePlaceholder = ({+ filename,+ mediaType,+}: {+ filename: string | undefined;+ mediaType: string | undefined;+}): string => {+ const displayedFilename = isNonEmptyString(filename)+ ? filename+ : DEFAULT_ATTACHMENT_FILENAME;+ const displayedMediaType = isNonEmptyString(mediaType)+ ? mediaType+ : DEFAULT_ATTACHMENT_MEDIA_TYPE;++ return `[Attached file: ${displayedFilename} (type: ${displayedMediaType}) — file type is not supported for direct analysis]`;+};diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util.tssimilarity index 80%rename from packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util.tsrename to packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util.tsindex 5461cfc9173..d0341dfc9f4 100644--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util.ts@@ -1,4 +1,4 @@-import { MODALITY_TO_MIME_TYPES } from 'src/engine/metadata-modules/ai/ai-chat/constants/modality-to-mime-types.constant';+import { MODALITY_TO_MIME_TYPES } from 'src/engine/metadata-modules/ai/ai-models/constants/modality-to-mime-types.constant';export const getNativeMimeTypesForModalities = (modalities: string[] = [],