The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Fix PDF Upload edge case (#18533)
neo773b21fb4aa6f11
diff --git a/packages/twenty-server/jest.config.mjs b/packages/twenty-server/jest.config.mjsindex cb114e3aef6..22d28de204e 100644--- a/packages/twenty-server/jest.config.mjs+++ b/packages/twenty-server/jest.config.mjs@@ -10,7 +10,9 @@ const jestConfig = {rootDir: './',testEnvironment: 'node',setupFilesAfterEnv: ['./setupTests.ts'],- transformIgnorePatterns: ['/node_modules/'],+ transformIgnorePatterns: [+ '/node_modules/(?!(file-type|@file-type|strtok3|token-types|@borewit|@tokenizer|uint8array-extras|read-next-line)/)',+ ],testRegex: '.*\\.spec\\.ts$',transform: {'^.+\\.(t|j)s$': [@@ -42,6 +44,7 @@ const jestConfig = {moduleNameMapper: {'^src/(.*)': '<rootDir>/src/$1','^test/(.*)': '<rootDir>/test/$1',+ '^file-type$': '<rootDir>/node_modules/file-type/index.js',},moduleFileExtensions: ['js', 'json', 'ts'],modulePathIgnorePatterns: ['<rootDir>/dist'],diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.jsonindex 3696a9a324b..c80bab474b5 100644--- a/packages/twenty-server/package.json+++ b/packages/twenty-server/package.json@@ -39,6 +39,7 @@"@envelop/on-resolve": "4.1.0","@esbuild-plugins/node-modules-polyfill": "^0.2.2","@faker-js/faker": "9.8.0",+ "@file-type/pdf": "^0.2.0","@graphql-tools/schema": "10.0.4","@graphql-tools/utils": "9.2.1","@graphql-yoga/nestjs": "patch:@graphql-yoga/[email protected]#./patches/@graphql-yoga+nestjs+2.1.0.patch",@@ -106,7 +107,7 @@"dotenv": "16.4.5","express": "4.22.1","express-session": "^1.18.2",- "file-type": "16.5.4",+ "file-type": "^21.3.1","fuse.js": "^7.1.0","gaxios": "5.1.3","google-auth-library": "8.9.0",diff --git a/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-migrate-workspace-pictures.command.ts b/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-migrate-workspace-pictures.command.tsindex cafc79ba69f..70f443aedbe 100644--- a/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-migrate-workspace-pictures.command.ts+++ b/packages/twenty-server/src/database/commands/upgrade-version-command/1-18/1-18-migrate-workspace-pictures.command.ts@@ -1,7 +1,8 @@import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';import { isNonEmptyString } from '@sniptt/guards';-import FileType from 'file-type';+import { FileTypeParser } from 'file-type';+import { detectPdf } from '@file-type/pdf';import { Command } from 'nest-commander';import { STANDARD_OBJECTS } from 'twenty-shared/metadata';import { FileFolder } from 'twenty-shared/types';@@ -204,7 +205,8 @@ export class MigrateWorkspacePicturesCommand extends ActiveOrSuspendedWorkspacesconst httpClient = this.secureHttpClientService.getHttpClient();const buffer = await getImageBufferFromUrl(logoUrl, httpClient);- const type = await FileType.fromBuffer(buffer);+ const parser = new FileTypeParser({ customDetectors: [detectPdf] });+ const type = await parser.fromBuffer(buffer);if (!isDefined(type) || !type.mime.startsWith('image/')) {this.logger.warn(diff --git a/packages/twenty-server/src/engine/core-modules/file/file-core-picture/services/file-core-picture.service.ts b/packages/twenty-server/src/engine/core-modules/file/file-core-picture/services/file-core-picture.service.tsindex 06394be2b5d..808076ab826 100644--- a/packages/twenty-server/src/engine/core-modules/file/file-core-picture/services/file-core-picture.service.ts+++ b/packages/twenty-server/src/engine/core-modules/file/file-core-picture/services/file-core-picture.service.ts@@ -4,7 +4,8 @@ import { InjectRepository } from '@nestjs/typeorm';import { buffer as streamToBuffer } from 'node:stream/consumers';import { isNonEmptyString } from '@sniptt/guards';-import FileType from 'file-type';+import { FileTypeParser } from 'file-type';+import { detectPdf } from '@file-type/pdf';import { FileFolder } from 'twenty-shared/types';import { isDefined } from 'twenty-shared/utils';import { Like, type QueryRunner, Repository } from 'typeorm';@@ -195,7 +196,8 @@ export class FileCorePictureService {const buffer = await getImageBufferFromUrl(imageUrl, httpClient);- const type = await FileType.fromBuffer(buffer);+ const parser = new FileTypeParser({ customDetectors: [detectPdf] });+ const type = await parser.fromBuffer(buffer);if (!isDefined(type) || !type.mime.startsWith('image/')) {return undefined;diff --git a/packages/twenty-server/src/engine/core-modules/file/utils/__tests__/extract-file-info.utils.spec.ts b/packages/twenty-server/src/engine/core-modules/file/utils/__tests__/extract-file-info.utils.spec.tsindex efdd025e5e0..8aa8fcf5392 100644--- a/packages/twenty-server/src/engine/core-modules/file/utils/__tests__/extract-file-info.utils.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/file/utils/__tests__/extract-file-info.utils.spec.ts@@ -1,182 +1,104 @@-import { extractFileInfo } from 'src/engine/core-modules/file/utils/extract-file-info.utils';+import { extractFileInfo } from '../extract-file-info.utils';-// Mock detectableMimeTypes to work around ESM/CommonJS interop issues in Jest-jest.mock('file-type', () => {- const actual = jest.requireActual('file-type');-- return {- ...actual,- mimeTypes: actual.mimeTypes,- };-});+const pngBuffer = Buffer.from([+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49,+ 0x48, 0x44, 0x52,+]);+const pdfBuffer = Buffer.from('%PDF-1.4\n', 'utf-8');+const textBuffer = Buffer.from('Hello, world!', 'utf-8');+const zipBuffer = Buffer.from([0x50, 0x4b, 0x03, 0x04]);describe('extractFileInfo', () => {- // Real PNG file header (magic numbers)- const pngBuffer = Buffer.from([- 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d,- 0x49, 0x48, 0x44, 0x52,- ]);-- // Real PDF file header- const pdfBuffer = Buffer.from('%PDF-1.4\n', 'utf-8');-- // Plain text buffer (no magic numbers)- const textBuffer = Buffer.from('Hello, world!', 'utf-8');-- // Real ZIP file header (for testing docx, xlsx, etc.)- const zipBuffer = Buffer.from([0x50, 0x4b, 0x03, 0x04]);-- it('should detect PNG from buffer magic numbers', async () => {- const result = await extractFileInfo({- file: pngBuffer,+ it.each([+ {+ name: 'PNG',+ buffer: pngBuffer,filename: 'image.png',- });-- expect(result).toEqual({- mimeType: 'image/png',ext: 'png',- });- });-- it('should detect PDF from buffer magic numbers', async () => {- const result = await extractFileInfo({- file: pdfBuffer,+ mime: 'image/png',+ },+ {+ name: 'PDF',+ buffer: pdfBuffer,filename: 'document.pdf',- });-- expect(result).toEqual({- mimeType: 'application/pdf',ext: 'pdf',- });- });-- it('should use extension-based lookup for text files', async () => {- const result = await extractFileInfo({- file: textBuffer,- filename: 'document.txt',- });-- expect(result).toEqual({- mimeType: 'text/plain',- ext: 'txt',- });- });-- it('should handle CSV files using extension', async () => {- const result = await extractFileInfo({- file: textBuffer,- filename: 'data.csv',- });-- expect(result).toEqual({- mimeType: 'text/csv',- ext: 'csv',- });- });-- it('should handle JSON files using extension', async () => {- const result = await extractFileInfo({- file: Buffer.from('{"key": "value"}'),+ mime: 'application/pdf',+ },+ {+ name: 'ZIP',+ buffer: zipBuffer,+ filename: 'archive.zip',+ ext: 'zip',+ mime: 'application/zip',+ },+ {+ name: 'PNG (mismatched extension)',+ buffer: pngBuffer,+ filename: 'image.txt',+ ext: 'png',+ mime: 'image/png',+ },+ ])(+ 'should detect $name from buffer magic numbers',+ async ({ buffer, filename, ext, mime }) => {+ const result = await extractFileInfo({ file: buffer, filename });++ expect(result).toEqual({ mimeType: mime, ext });+ },+ );++ it.each([+ { name: 'text', filename: 'document.txt', ext: 'txt', mime: 'text/plain' },+ { name: 'CSV', filename: 'data.csv', ext: 'csv', mime: 'text/csv' },+ {+ name: 'JSON',filename: 'config.json',- });-- expect(result).toEqual({- mimeType: 'application/json',ext: 'json',- });- });-- it('should return application/octet-stream for unknown extensions', async () => {- const result = await extractFileInfo({- file: textBuffer,+ mime: 'application/json',+ },+ {+ name: 'markdown',+ filename: 'README.md',+ ext: 'md',+ mime: 'text/markdown',+ },+ { name: 'HTML', filename: 'index.html', ext: 'html', mime: 'text/html' },+ {+ name: 'unknown extension',filename: 'file.unknown',- });-- expect(result).toEqual({- mimeType: 'application/octet-stream',ext: 'unknown',- });- });-- it('should return application/octet-stream for files without extension', async () => {- const result = await extractFileInfo({- file: textBuffer,+ mime: 'application/octet-stream',+ },+ {+ name: 'no extension',filename: 'file-without-extension',- });-- expect(result).toEqual({- mimeType: 'application/octet-stream',ext: '',- });- });-- it('should detect ZIP files from buffer', async () => {- const result = await extractFileInfo({- file: zipBuffer,- filename: 'archive.zip',- });-- expect(result).toEqual({- mimeType: 'application/zip',- ext: 'zip',- });- });-- it('should throw error when PNG extension is used with non-PNG buffer', async () => {- await expect(- extractFileInfo({- file: textBuffer,- filename: 'fake-image.png',- }),- ).rejects.toThrow(- "File content does not match its extension. The file has extension 'png' (expected mime type: image/png), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.",- );- });-- it('should throw error when PDF extension is used with non-PDF buffer', async () => {- await expect(- extractFileInfo({- file: textBuffer,- filename: 'fake-document.pdf',- }),- ).rejects.toThrow(- "File content does not match its extension. The file has extension 'pdf' (expected mime type: application/pdf), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.",- );- });-- it('should handle markdown files using extension', async () => {- const result = await extractFileInfo({- file: Buffer.from('# Heading\n\nContent'),- filename: 'README.md',- });-- expect(result).toEqual({- mimeType: 'text/markdown',- ext: 'md',- });- });-- it('should handle HTML files using extension', async () => {- const result = await extractFileInfo({- file: Buffer.from('<html><body>Test</body></html>'),- filename: 'index.html',- });-- expect(result).toEqual({- mimeType: 'text/html',- ext: 'html',- });- });-- it('should prefer detected type over declared extension', async () => {- const result = await extractFileInfo({- file: pngBuffer,- filename: 'image.txt',- });-- expect(result).toEqual({- mimeType: 'image/png',- ext: 'png',- });- });+ mime: 'application/octet-stream',+ },+ ])(+ 'should fall back to extension for $name files',+ async ({ filename, ext, mime }) => {+ const result = await extractFileInfo({ file: textBuffer, filename });++ expect(result).toEqual({ mimeType: mime, ext });+ },+ );++ it.each([+ { ext: 'png', filename: 'fake-image.png', expectedMime: 'image/png' },+ {+ ext: 'pdf',+ filename: 'fake-document.pdf',+ expectedMime: 'application/pdf',+ },+ ])(+ 'should throw when $ext extension does not match buffer content',+ async ({ filename, ext, expectedMime }) => {+ await expect(+ extractFileInfo({ file: textBuffer, filename }),+ ).rejects.toThrow(+ `File content does not match its extension. The file has extension '${ext}' (expected mime type: ${expectedMime}), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.`,+ );+ },+ );});diff --git a/packages/twenty-server/src/engine/core-modules/file/utils/extract-file-info.utils.ts b/packages/twenty-server/src/engine/core-modules/file/utils/extract-file-info.utils.tsindex cb1c5a4e8cf..2f93f9530f7 100644--- a/packages/twenty-server/src/engine/core-modules/file/utils/extract-file-info.utils.ts+++ b/packages/twenty-server/src/engine/core-modules/file/utils/extract-file-info.utils.ts@@ -1,6 +1,6 @@import { msg } from '@lingui/core/macro';import { isNonEmptyString } from '@sniptt/guards';-import FileType, { type MimeType } from 'file-type';+import { FileTypeParser, supportedMimeTypes } from 'file-type';import { lookup } from 'mrmime';import { isDefined } from 'twenty-shared/utils';@@ -9,6 +9,7 @@ import {FileStorageExceptionCode,} from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';+import { detectPdf } from '@file-type/pdf';import { buildFileInfo } from 'src/engine/core-modules/file/utils/build-file-info.utils';export const extractFileInfo = async ({@@ -20,8 +21,12 @@ export const extractFileInfo = async ({}) => {const { ext: declaredExt } = buildFileInfo(filename);+ const fileParser = new FileTypeParser({+ customDetectors: [detectPdf],+ });+const { ext: detectedExt, mime: detectedMime } =- (await FileType.fromBuffer(file)) ?? {};+ (await fileParser.fromBuffer(file)) ?? {};if (isDefined(detectedExt) && isDefined(detectedMime)) {return {@@ -39,7 +44,7 @@ export const extractFileInfo = async ({if (mimeTypeFromExtension &&- FileType.mimeTypes.has(mimeTypeFromExtension as MimeType)+ supportedMimeTypes.has(mimeTypeFromExtension)) {throw new FileStorageException(`File content does not match its extension. The file has extension '${ext}' (expected mime type: ${mimeTypeFromExtension}), but the file content could not be detected as this type. The file may be corrupted, have the wrong extension, or be a security risk.`,diff --git a/yarn.lock b/yarn.lockindex 6059c55ef7a..49051cfe035 100644--- a/yarn.lock+++ b/yarn.lock@@ -5873,6 +5873,17 @@ __metadata:languageName: nodelinkType: hard+"@file-type/pdf@npm:^0.2.0":+ version: 0.2.0+ resolution: "@file-type/pdf@npm:0.2.0"+ dependencies:+ "@borewit/text-codec": "npm:^0.2.1"+ read-next-line: "npm:^0.5.0"+ sax: "npm:^1.4.1"+ checksum: 10c0/9f2f446a04ca2fff4f5a02aeb6065d72f4fc95b6bce5efc6a8ec77530c5b36192f9aa718b65e00398921cf5a0d505cda5e9208d3ac81aad63f7d60a535212a89+ languageName: node+ linkType: hard+"@floating-ui/core@npm:^1.6.0":version: 1.6.7resolution: "@floating-ui/core@npm:1.6.7"@@ -36527,6 +36538,18 @@ __metadata:languageName: nodelinkType: hard+"file-type@npm:^21.3.1":+ version: 21.3.1+ resolution: "file-type@npm:21.3.1"+ dependencies:+ "@tokenizer/inflate": "npm:^0.4.1"+ strtok3: "npm:^10.3.4"+ token-types: "npm:^6.1.1"+ uint8array-extras: "npm:^1.4.0"+ checksum: 10c0/66a8eda781c803c6fc372464abba88cee564cfe30f029716f06909da1564bc51d0a4e8d34654b881dc751cdb0513338b3c0747e06a608cee0dd5c5499b018d47+ languageName: node+ linkType: hard+"filelist@npm:^1.0.4":version: 1.0.4resolution: "filelist@npm:1.0.4"@@ -51234,6 +51257,13 @@ __metadata:languageName: nodelinkType: hard+"read-next-line@npm:^0.5.0":+ version: 0.5.0+ resolution: "read-next-line@npm:0.5.0"+ checksum: 10c0/a658dd647a655767a81d237474dac2cce6f15fff14e8963a41980bafb30c11298e27d6210a06031ecef62a28edf703cc2e3fce8d28ec411bb309ea71403c6992+ languageName: node+ linkType: hard+"read-only-stream@npm:^2.0.0":version: 2.0.0resolution: "read-only-stream@npm:2.0.0"@@ -52880,6 +52910,13 @@ __metadata:languageName: nodelinkType: hard+"sax@npm:^1.4.1":+ version: 1.5.0+ resolution: "sax@npm:1.5.0"+ checksum: 10c0/bc3b60a7bfecd40b18256596e96b32df2488339ae1e00a77f842b568f0831228a16c3bd357ec500241ec0b9dc7a475a1286427795c4a8c50bb8e8878f3435dd8+ languageName: node+ linkType: hard+"saxes@npm:^6.0.0":version: 6.0.0resolution: "saxes@npm:6.0.0"@@ -56594,6 +56631,7 @@ __metadata:"@envelop/on-resolve": "npm:4.1.0""@esbuild-plugins/node-modules-polyfill": "npm:^0.2.2""@faker-js/faker": "npm:^9.8.0"+ "@file-type/pdf": "npm:^0.2.0""@graphql-tools/schema": "npm:10.0.4""@graphql-tools/utils": "npm:9.2.1""@graphql-yoga/nestjs": "patch:@graphql-yoga/[email protected]#./patches/@graphql-yoga+nestjs+2.1.0.patch"@@ -56700,7 +56738,7 @@ __metadata:dotenv: "npm:16.4.5"express: "npm:4.22.1"express-session: "npm:^1.18.2"- file-type: "npm:16.5.4"+ file-type: "npm:^21.3.1"fuse.js: "npm:^7.1.0"gaxios: "npm:5.1.3"google-auth-library: "npm:8.9.0"