Skip to content

The open alternative to Salesforce, designed for AI.

fix(ai): degrade unsupported runAgent attachments instead of failing the run (#26177)

Abdul Rahman74ee19ead7e4
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/run-agent-attachment.service.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/run-agent-attachment.service.spec.ts
index ac2aefc451c..7af001e3f80 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/run-agent-attachment.service.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/__tests__/run-agent-attachment.service.spec.ts
@@ -2,6 +2,7 @@ import { FileFolder } from 'twenty-shared/types';
import { In, Like } from 'typeorm';
import { RunAgentAttachmentService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service';
+import { AiExceptionCode } from 'src/engine/metadata-modules/ai/ai.exception';
const WORKSPACE_ID = 'workspace-id';
const FILE_ID = '6fdbd0ba-9a10-4b71-b0b7-2b1d64f8dc27';
@@ -27,6 +28,7 @@ describe('RunAgentAttachmentService', () => {
{ role: 'user', content: 'hi', attachments: [{ fileId: FILE_ID }] },
],
workspaceId: WORKSPACE_ID,
+ modalities: ['image'],
});
expect(fileRepository.find).toHaveBeenCalledWith(WORKSPACE_ID, {
@@ -51,8 +53,115 @@ describe('RunAgentAttachmentService', () => {
await service.buildModelMessagesOrThrow({
messages: [{ role: 'user', content: 'hello' }],
workspaceId: WORKSPACE_ID,
+ modalities: ['image'],
}),
).toEqual([{ role: 'user', content: 'hello' }]);
expect(fileRepository.find).not.toHaveBeenCalled();
});
+
+ it('hands a supported attachment to the model as a file part', async () => {
+ const fileRepository = {
+ find: jest
+ .fn()
+ .mockResolvedValue([{ id: FILE_ID, mimeType: 'image/png' }]),
+ };
+ const service = new RunAgentAttachmentService(
+ fileRepository as never,
+ {
+ signFileByIdUrl: jest
+ .fn()
+ .mockResolvedValue('https://twenty.test/file'),
+ } as never,
+ );
+
+ expect(
+ await service.buildModelMessagesOrThrow({
+ messages: [
+ {
+ role: 'user',
+ content: 'what is in this?',
+ attachments: [{ fileId: FILE_ID, filename: 'screenshot.png' }],
+ },
+ ],
+ workspaceId: WORKSPACE_ID,
+ modalities: ['image'],
+ }),
+ ).toEqual([
+ {
+ role: 'user',
+ content: [
+ { type: 'text', text: 'what is in this?' },
+ {
+ type: 'file',
+ data: 'https://twenty.test/file',
+ mediaType: 'image/png',
+ filename: 'screenshot.png',
+ },
+ ],
+ },
+ ]);
+ });
+
+ it('degrades an attachment the resolved model cannot read into text', async () => {
+ const fileRepository = {
+ find: jest
+ .fn()
+ .mockResolvedValue([{ id: FILE_ID, mimeType: 'application/pdf' }]),
+ };
+ const service = new RunAgentAttachmentService(
+ fileRepository as never,
+ {
+ signFileByIdUrl: jest
+ .fn()
+ .mockResolvedValue('https://twenty.test/file'),
+ } as never,
+ );
+
+ expect(
+ await service.buildModelMessagesOrThrow({
+ messages: [
+ {
+ role: 'user',
+ content: 'summarize this',
+ attachments: [{ fileId: FILE_ID, filename: 'report.pdf' }],
+ },
+ ],
+ workspaceId: WORKSPACE_ID,
+ modalities: ['image'],
+ }),
+ ).toEqual([
+ {
+ role: 'user',
+ content: [
+ { type: 'text', text: 'summarize this' },
+ {
+ type: 'text',
+ text: '[Attached file: report.pdf (type: application/pdf) — file type is not supported for direct analysis]',
+ },
+ ],
+ },
+ ]);
+ });
+
+ it('still refuses a file id that resolves to nothing in the workspace', async () => {
+ const fileRepository = { find: jest.fn().mockResolvedValue([]) };
+ const service = new RunAgentAttachmentService(
+ fileRepository as never,
+ {
+ signFileByIdUrl: jest.fn(),
+ } as never,
+ );
+
+ await expect(
+ service.buildModelMessagesOrThrow({
+ messages: [
+ { role: 'user', content: 'hi', attachments: [{ fileId: FILE_ID }] },
+ ],
+ workspaceId: WORKSPACE_ID,
+ modalities: ['image'],
+ }),
+ ).rejects.toThrow(
+ expect.objectContaining({ code: AiExceptionCode.INVALID_AGENT_INPUT }),
+ );
+ });
});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
index f590d2afffc..908cea19000 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
@@ -386,6 +386,9 @@ export class AgentAsyncExecutorService {
await this.runAgentAttachmentService.buildModelMessagesOrThrow({
messages,
workspaceId,
+ modalities: this.aiModelRegistryService.getModelConfig(
+ registeredModel.modelId,
+ )?.modalities,
});
const textResponse = await generateText({
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service.ts
index 0c3193c203d..e63a177aebe 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service.ts
@@ -26,9 +26,11 @@ export class RunAgentAttachmentService {
async buildModelMessagesOrThrow({
messages,
workspaceId,
+ modalities,
}: {
messages: RunAgentMessage[];
workspaceId: string;
+ modalities: string[] | undefined;
}): Promise<ModelMessage[]> {
const fileIds = extractRunAgentAttachmentFileIdsOrThrow(messages);
@@ -37,7 +39,11 @@ export class RunAgentAttachmentService {
: new Map<string, ResolvedRunAgentAttachment>();
return messages.map((message) =>
- buildRunAgentModelMessageOrThrow({ message, attachmentsByFileId }),
+ buildRunAgentModelMessageOrThrow({
+ message,
+ attachmentsByFileId,
+ modalities,
+ }),
);
}
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-run-agent-model-message.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-run-agent-model-message.util.spec.ts
index 75bb5ecea22..62ae63ff37a 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-run-agent-model-message.util.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-run-agent-model-message.util.spec.ts
@@ -3,9 +3,12 @@ import { buildRunAgentModelMessageOrThrow } from 'src/engine/metadata-modules/ai
const FILE_ID = '6fdbd0ba-9a10-4b71-b0b7-2b1d64f8dc27';
const FILE_URL = `https://twenty.test/file/agent-chat/${FILE_ID}?token=t`;
+const PDF_FILE_ID = 'a2a0d40b-bd5f-4f12-9a0b-06ec8c3f5fd1';
+const PDF_FILE_URL = `https://twenty.test/file/agent-chat/${PDF_FILE_ID}?token=t`;
const attachmentsByFileId = new Map<string, ResolvedRunAgentAttachment>([
[FILE_ID, { mediaType: 'image/png', url: FILE_URL }],
+ [PDF_FILE_ID, { mediaType: 'application/pdf', url: PDF_FILE_URL }],
]);
describe('buildRunAgentModelMessageOrThrow', () => {
@@ -14,6 +17,7 @@ describe('buildRunAgentModelMessageOrThrow', () => {
buildRunAgentModelMessageOrThrow({
message: { role: 'assistant', content: 'hi' },
attachmentsByFileId,
+ modalities: ['image'],
}),
).toEqual({ role: 'assistant', content: 'hi' });
});
@@ -27,6 +31,7 @@ describe('buildRunAgentModelMessageOrThrow', () => {
attachments: [{ fileId: FILE_ID, filename: 'screenshot.png' }],
},
attachmentsByFileId,
+ modalities: ['image'],
}),
).toEqual({
role: 'user',
@@ -50,6 +55,7 @@ describe('buildRunAgentModelMessageOrThrow', () => {
attachments: [{ fileId: FILE_ID }],
},
attachmentsByFileId,
+ modalities: ['image'],
});
expect(message.content).toEqual([
@@ -66,7 +72,123 @@ describe('buildRunAgentModelMessageOrThrow', () => {
attachments: [{ fileId: FILE_ID }],
},
attachmentsByFileId: new Map(),
+ modalities: ['image'],
}),
).toThrow(FILE_ID);
});
+
+ it('degrades an attachment the model cannot read into a text part', () => {
+ expect(
+ buildRunAgentModelMessageOrThrow({
+ message: {
+ role: 'user',
+ content: 'summarize this',
+ attachments: [{ fileId: PDF_FILE_ID, filename: 'report.pdf' }],
+ },
+ attachmentsByFileId,
+ modalities: ['image'],
+ }),
+ ).toEqual({
+ role: 'user',
+ content: [
+ { type: 'text', text: 'summarize this' },
+ {
+ type: 'text',
+ text: '[Attached file: report.pdf (type: application/pdf) — file type is not supported for direct analysis]',
+ },
+ ],
+ });
+ });
+
+ it('names an unsupported attachment that carries no filename', () => {
+ expect(
+ buildRunAgentModelMessageOrThrow({
+ message: {
+ role: 'user',
+ content: '',
+ attachments: [{ fileId: PDF_FILE_ID }],
+ },
+ attachmentsByFileId,
+ modalities: ['image'],
+ }).content,
+ ).toEqual([
+ {
+ type: 'text',
+ text: '[Attached file: uploaded_file (type: application/pdf) — file type is not supported for direct analysis]',
+ },
+ ]);
+ });
+
+ it('keeps the supported attachment and degrades only the unsupported one', () => {
+ expect(
+ buildRunAgentModelMessageOrThrow({
+ message: {
+ role: 'user',
+ content: 'compare these',
+ attachments: [
+ { fileId: FILE_ID, filename: 'screenshot.png' },
+ { fileId: PDF_FILE_ID, filename: 'report.pdf' },
+ ],
+ },
+ attachmentsByFileId,
+ modalities: ['image'],
+ }),
+ ).toEqual({
+ role: 'user',
+ content: [
+ { type: 'text', text: 'compare these' },
+ {
+ type: 'file',
+ data: FILE_URL,
+ mediaType: 'image/png',
+ filename: 'screenshot.png',
+ },
+ {
+ type: 'text',
+ text: '[Attached file: report.pdf (type: application/pdf) — file type is not supported for direct analysis]',
+ },
+ ],
+ });
+ });
+
+ it('degrades every attachment when the model declares no modalities', () => {
+ expect(
+ buildRunAgentModelMessageOrThrow({
+ message: {
+ role: 'user',
+ content: 'what is in this?',
+ attachments: [{ fileId: FILE_ID, filename: 'screenshot.png' }],
+ },
+ attachmentsByFileId,
+ modalities: undefined,
+ }).content,
+ ).toEqual([
+ { type: 'text', text: 'what is in this?' },
+ {
+ type: 'text',
+ text: '[Attached file: screenshot.png (type: image/png) — file type is not supported for direct analysis]',
+ },
+ ]);
+ });
+
+ it('labels an attachment stored without a media type as unknown', () => {
+ expect(
+ buildRunAgentModelMessageOrThrow({
+ message: {
+ role: 'user',
+ content: '',
+ attachments: [{ fileId: FILE_ID, filename: 'mystery.bin' }],
+ },
+ attachmentsByFileId: new Map([
+ [FILE_ID, { mediaType: '', url: FILE_URL }],
+ ]),
+ modalities: ['image'],
+ }).content,
+ ).toEqual([
+ {
+ type: 'text',
+ text: '[Attached file: mystery.bin (type: unknown) — file type is not supported for direct analysis]',
+ },
+ ]);
+ });
});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-run-agent-model-message.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-run-agent-model-message.util.ts
index 6bfa9d860d2..5b0bbe39164 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-run-agent-model-message.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-run-agent-model-message.util.ts
@@ -1,10 +1,12 @@
import { isNonEmptyString } from '@sniptt/guards';
-import { type FilePart, type ModelMessage } from 'ai';
+import { type FilePart, type ModelMessage, type TextPart } from 'ai';
import { type RunAgentMessage } from 'twenty-shared/application';
import { FileFolder } from 'twenty-shared/types';
import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';
import { type ResolvedRunAgentAttachment } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/resolved-run-agent-attachment.type';
+import { formatUnsupportedFilePlaceholder } from 'src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util';
+import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util';
import {
AiException,
AiExceptionCode,
@@ -13,9 +15,11 @@ import {
export const buildRunAgentModelMessageOrThrow = ({
message,
attachmentsByFileId,
+ modalities,
}: {
message: RunAgentMessage;
attachmentsByFileId: Map<string, ResolvedRunAgentAttachment>;
+ modalities: string[] | undefined;
}): ModelMessage => {
const attachments = message.attachments ?? [];
@@ -23,7 +27,9 @@ export const buildRunAgentModelMessageOrThrow = ({
return { role: message.role, content: message.content };
}
- const fileParts = attachments.map((attachment): FilePart => {
+ const supportedMediaTypes = getNativeMimeTypesForModalities(modalities);
+
+ const attachmentParts = attachments.map((attachment): FilePart | TextPart => {
const resolvedAttachment = attachmentsByFileId.get(attachment.fileId);
if (!isDefined(resolvedAttachment)) {
@@ -33,6 +39,16 @@ export const buildRunAgentModelMessageOrThrow = ({
);
}
+ if (!supportedMediaTypes.has(resolvedAttachment.mediaType)) {
+ return {
+ type: 'text',
+ text: formatUnsupportedFilePlaceholder({
+ filename: attachment.filename,
+ mediaType: resolvedAttachment.mediaType,
+ }),
+ };
+ }
+
return {
type: 'file',
data: resolvedAttachment.url,
@@ -47,7 +63,7 @@ export const buildRunAgentModelMessageOrThrow = ({
...(isNonEmptyString(message.content)
? [{ type: 'text' as const, text: message.content }]
: []),
- ...fileParts,
+ ...attachmentParts,
],
};
};
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util.ts
index edd3a5a365c..eac75df4eb1 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/replace-unsupported-file-parts.util.ts
@@ -1,7 +1,8 @@
import { type ExtendedUIMessage, isExtendedFileUIPart } from 'twenty-shared/ai';
import { CODE_INTERPRETER_MIME_TYPES } from 'src/engine/metadata-modules/ai/ai-chat/constants/code-interpreter-mime-types.constant';
-import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util';
+import { formatUnsupportedFilePlaceholder } from 'src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util';
+import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util';
export const replaceUnsupportedFileParts = (
messages: ExtendedUIMessage[],
@@ -29,10 +30,12 @@ export const replaceUnsupportedFileParts = (
if (isSupported) {
newParts.push(part);
} else {
- const filename = part.filename ?? 'uploaded_file';
newParts.push({
type: 'text',
- text: `[Attached file: ${filename} (type: ${mimeType || 'unknown'}) — file type is not supported for direct analysis]`,
+ text: formatUnsupportedFilePlaceholder({
+ filename: part.filename,
+ mediaType: mimeType,
+ }),
});
}
} else {
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/modality-to-mime-types.constant.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/modality-to-mime-types.constant.ts
similarity index 100%
rename from packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/constants/modality-to-mime-types.constant.ts
rename to packages/twenty-server/src/engine/metadata-modules/ai/ai-models/constants/modality-to-mime-types.constant.ts
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/format-unsupported-file-placeholder.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/format-unsupported-file-placeholder.util.spec.ts
new file mode 100644
index 00000000000..c98db63c3ce
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/format-unsupported-file-placeholder.util.spec.ts
@@ -0,0 +1,36 @@
+import { formatUnsupportedFilePlaceholder } from 'src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util';
+
+describe('formatUnsupportedFilePlaceholder', () => {
+ it('names the file and its type', () => {
+ expect(
+ formatUnsupportedFilePlaceholder({
+ filename: 'report.pdf',
+ mediaType: 'application/pdf',
+ }),
+ ).toBe(
+ '[Attached file: report.pdf (type: application/pdf) — file type is not supported for direct analysis]',
+ );
+ });
+
+ it('falls back to a generic name when the file carries none', () => {
+ expect(
+ formatUnsupportedFilePlaceholder({
+ filename: undefined,
+ mediaType: 'application/pdf',
+ }),
+ ).toBe(
+ '[Attached file: uploaded_file (type: application/pdf) — file type is not supported for direct analysis]',
+ );
+ });
+
+ it('labels a missing media type as unknown', () => {
+ expect(
+ formatUnsupportedFilePlaceholder({
+ filename: 'mystery.bin',
+ mediaType: '',
+ }),
+ ).toBe(
+ '[Attached file: mystery.bin (type: unknown) — file type is not supported for direct analysis]',
+ );
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts
similarity index 96%
rename from packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts
rename to packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts
index 2aa6e3d9f40..19c269ccbb5 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/__tests__/get-native-mime-types-for-modalities.util.spec.ts
@@ -1,4 +1,4 @@
-import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util';
+import { getNativeMimeTypesForModalities } from 'src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util';
describe('getNativeMimeTypesForModalities', () => {
it('returns an empty set when no modalities are provided', () => {
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util.ts
new file mode 100644
index 00000000000..b9c81f67ce9
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/format-unsupported-file-placeholder.util.ts
@@ -0,0 +1,21 @@
+import { isNonEmptyString } from '@sniptt/guards';
+
+const DEFAULT_ATTACHMENT_FILENAME = 'uploaded_file';
+const DEFAULT_ATTACHMENT_MEDIA_TYPE = 'unknown';
+
+export const formatUnsupportedFilePlaceholder = ({
+ filename,
+ mediaType,
+}: {
+ filename: string | undefined;
+ mediaType: string | undefined;
+}): string => {
+ const displayedFilename = isNonEmptyString(filename)
+ ? filename
+ : DEFAULT_ATTACHMENT_FILENAME;
+ const displayedMediaType = isNonEmptyString(mediaType)
+ ? mediaType
+ : DEFAULT_ATTACHMENT_MEDIA_TYPE;
+
+ return `[Attached file: ${displayedFilename} (type: ${displayedMediaType}) — file type is not supported for direct analysis]`;
+};
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util.ts
similarity index 80%
rename from packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util.ts
rename to packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util.ts
index 5461cfc9173..d0341dfc9f4 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-chat/utils/get-native-mime-types-for-modalities.util.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-models/utils/get-native-mime-types-for-modalities.util.ts
@@ -1,4 +1,4 @@
-import { MODALITY_TO_MIME_TYPES } from 'src/engine/metadata-modules/ai/ai-chat/constants/modality-to-mime-types.constant';
+import { MODALITY_TO_MIME_TYPES } from 'src/engine/metadata-modules/ai/ai-models/constants/modality-to-mime-types.constant';
export const getNativeMimeTypesForModalities = (
modalities: string[] = [],