The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix: keep completeFileUpload below the gateway timeout (#25320)
martmull6d5ce1397d67
diff --git a/packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-check-timeout.util.ts b/packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-check-timeout.util.tsindex 75d1cc86aa7..8d71e2cb50c 100644--- a/packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-check-timeout.util.ts+++ b/packages/twenty-server/src/engine/core-modules/admin-panel/utils/health-check-timeout.util.ts@@ -1,16 +1,13 @@import { HEALTH_INDICATORS_TIMEOUT } from 'src/engine/core-modules/admin-panel/constants/health-indicators-timeout.conts';+import { withDeadline } from 'src/utils/with-deadline';export const withHealthCheckTimeout = async <T>(promise: Promise<T>,errorMessage: string,): Promise<T> => {- return Promise.race([+ return withDeadline({promise,- new Promise<T>((_, reject) =>- setTimeout(- () => reject(new Error(errorMessage)),- HEALTH_INDICATORS_TIMEOUT,- ),- ),- ]);+ timeoutMs: HEALTH_INDICATORS_TIMEOUT,+ createTimeoutError: () => new Error(errorMessage),+ });};diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts b/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.tsindex d269f9a9200..c32cf4ec616 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant.ts@@ -1,3 +1,10 @@export const FILE_STORAGE_S3_REQUEST_TIMEOUT_MS = 5 * 60 * 1000;export const FILE_STORAGE_S3_CONNECTION_TIMEOUT_MS = 30 * 1000;export const FILE_STORAGE_S3_MAX_SOCKETS = 200;++export const FILE_STORAGE_S3_METADATA_REQUEST_TIMEOUT_MS = 15 * 1000;+export const FILE_STORAGE_S3_METADATA_CONNECTION_TIMEOUT_MS = 5 * 1000;+export const FILE_STORAGE_S3_METADATA_MAX_ATTEMPTS = 2;+export const FILE_STORAGE_S3_METADATA_MAX_SOCKETS = 50;++export const FILE_STORAGE_S3_SLOW_REQUEST_THRESHOLD_MS = 2 * 1000;diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/local.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/local.driver.spec.tsindex b95ea808d23..4d370b603ee 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/local.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/local.driver.spec.ts@@ -179,6 +179,56 @@ describe('LocalDriver', () => {});});+ describe('readFilePrefix', () => {+ it('should read only the leading bytes', async () => {+ const storagePath = await createTempDirectory('local-driver-storage-');+ const folderPath = path.join(storagePath, 'workspace', 'app');++ await mkdir(folderPath, { recursive: true });+ await writeFile(path.join(folderPath, 'file.txt'), '0123456789');++ const driver = new LocalDriver({ storagePath });++ await expect(+ driver.readFilePrefix({+ filePath: 'workspace/app/file.txt',+ byteCount: 4,+ }),+ ).resolves.toEqual(Buffer.from('0123'));+ });++ it('should return the whole file when it is shorter than requested', async () => {+ const storagePath = await createTempDirectory('local-driver-storage-');+ const folderPath = path.join(storagePath, 'workspace', 'app');++ await mkdir(folderPath, { recursive: true });+ await writeFile(path.join(folderPath, 'file.txt'), '01');++ const driver = new LocalDriver({ storagePath });++ await expect(+ driver.readFilePrefix({+ filePath: 'workspace/app/file.txt',+ byteCount: 4,+ }),+ ).resolves.toEqual(Buffer.from('01'));+ });++ it('should reject a missing file with FILE_NOT_FOUND', async () => {+ const storagePath = await createTempDirectory('local-driver-storage-');+ const driver = new LocalDriver({ storagePath });++ await expect(+ driver.readFilePrefix({+ filePath: 'workspace/app/missing.txt',+ byteCount: 4,+ }),+ ).rejects.toMatchObject({+ code: FileStorageExceptionCode.FILE_NOT_FOUND,+ });+ });+ });+describe('getFileMetadata', () => {it('should return the file size', async () => {const storagePath = await createTempDirectory('local-driver-storage-');diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/s3.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/s3.driver.spec.tsindex 1babdf95d83..e7567276905 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/s3.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/s3.driver.spec.ts@@ -1,9 +1,16 @@-import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3';+import {+ GetObjectCommand,+ HeadObjectCommand,+ PutObjectCommand,+ S3,+} from '@aws-sdk/client-s3';import { getSignedUrl } from '@aws-sdk/s3-request-presigner';import { Readable } from 'stream';+import { FILE_STORAGE_S3_METADATA_MAX_ATTEMPTS } from 'src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant';import { S3Driver } from 'src/engine/core-modules/file-storage/drivers/s3.driver';+import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';const mockS3Send = jest.fn();@@ -54,6 +61,104 @@ describe('S3Driver.readFile', () => {});});+describe('S3Driver.readFilePrefix', () => {+ afterEach(() => {+ jest.clearAllMocks();+ });++ it('should request only the leading byte range and buffer it', async () => {+ mockS3Send.mockResolvedValue({+ Body: {+ transformToByteArray: jest+ .fn()+ .mockResolvedValue(new Uint8Array(Buffer.from('prefix'))),+ },+ });++ const driver = new S3Driver({+ bucketName: 'test-bucket',+ region: 'us-east-1',+ });++ const prefix = await driver.readFilePrefix({+ filePath: 'attachments/file.pdf',+ byteCount: 64,+ });++ expect(prefix).toEqual(Buffer.from('prefix'));+ expect(mockS3Send).toHaveBeenCalledTimes(1);+ expect(mockS3Send.mock.calls[0][0]).toBeInstanceOf(GetObjectCommand);+ expect(mockS3Send).toHaveBeenCalledWith(+ expect.objectContaining({+ input: expect.objectContaining({+ Bucket: 'test-bucket',+ Key: 'attachments/file.pdf',+ Range: 'bytes=0-63',+ }),+ }),+ );+ });++ it('should return an empty buffer when the object is empty', async () => {+ mockS3Send.mockRejectedValue(+ Object.assign(new Error('The requested range is not satisfiable'), {+ name: 'InvalidRange',+ }),+ );++ const driver = new S3Driver({+ bucketName: 'test-bucket',+ region: 'us-east-1',+ });++ await expect(+ driver.readFilePrefix({ filePath: 'attachments/empty', byteCount: 64 }),+ ).resolves.toEqual(Buffer.alloc(0));+ });++ it('should map a missing key to FILE_NOT_FOUND', async () => {+ mockS3Send.mockRejectedValue(+ Object.assign(new Error('missing'), { name: 'NoSuchKey' }),+ );++ const driver = new S3Driver({+ bucketName: 'test-bucket',+ region: 'us-east-1',+ });++ await expect(+ driver.readFilePrefix({ filePath: 'attachments/missing', byteCount: 64 }),+ ).rejects.toMatchObject({ code: FileStorageExceptionCode.FILE_NOT_FOUND });+ });+});++describe('S3Driver.getFileMetadata', () => {+ afterEach(() => {+ jest.clearAllMocks();+ });++ it('should issue HeadObject through a client with a capped attempt count', async () => {+ mockS3Send.mockResolvedValue({ ContentLength: 42 });++ const driver = new S3Driver({+ bucketName: 'test-bucket',+ region: 'us-east-1',+ });++ await expect(+ driver.getFileMetadata({ filePath: 'attachments/file.pdf' }),+ ).resolves.toEqual({ size: 42 });++ expect(mockS3Send.mock.calls[0][0]).toBeInstanceOf(HeadObjectCommand);+ expect(S3).toHaveBeenCalledWith(+ expect.objectContaining({+ maxAttempts: FILE_STORAGE_S3_METADATA_MAX_ATTEMPTS,+ responseChecksumValidation: 'WHEN_REQUIRED',+ }),+ );+ });+});+describe('S3Driver.getPresignedUrl', () => {afterEach(() => {jest.clearAllMocks();diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/validated-storage.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/validated-storage.driver.spec.tsindex 70911976cfa..6be1359b192 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/validated-storage.driver.spec.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/validated-storage.driver.spec.ts@@ -7,6 +7,7 @@ import { ValidatedStorageDriver } from 'src/engine/core-modules/file-storage/driconst createMockDriver = (): jest.Mocked<StorageDriver> => ({readFile: jest.fn().mockResolvedValue(Readable.from([])),+ readFilePrefix: jest.fn().mockResolvedValue(Buffer.alloc(0)),writeFile: jest.fn().mockResolvedValue(undefined),writeFileStream: jest.fn().mockResolvedValue(undefined),getFileMetadata: jest.fn().mockResolvedValue(null),@@ -154,6 +155,21 @@ describe('ValidatedStorageDriver', () => {expect(mockDelegate.writeFileStream).toHaveBeenCalledWith(params);});+ it('should delegate readFilePrefix', async () => {+ mockDelegate.readFilePrefix.mockResolvedValue(Buffer.from('abc'));++ const result = await driver.readFilePrefix({+ filePath: 'folder/file.txt',+ byteCount: 3,+ });++ expect(result).toEqual(Buffer.from('abc'));+ expect(mockDelegate.readFilePrefix).toHaveBeenCalledWith({+ filePath: 'folder/file.txt',+ byteCount: 3,+ });+ });+it('should delegate getFileMetadata', async () => {mockDelegate.getFileMetadata.mockResolvedValue({ size: 1024 });@@ -200,6 +216,16 @@ describe('ValidatedStorageDriver', () => {expect(mockDelegate.readFile).not.toHaveBeenCalled();});+ it('should reject readFilePrefix with traversal', async () => {+ await expect(+ driver.readFilePrefix({ filePath: '../etc/passwd', byteCount: 16 }),+ ).rejects.toMatchObject({+ code: FileStorageExceptionCode.ACCESS_DENIED,+ });++ expect(mockDelegate.readFilePrefix).not.toHaveBeenCalled();+ });+it('should reject writeFile with traversal', async () => {await expect(driver.writeFile({diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface.tsindex 805ab3358cb..060258ac4cf 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface.ts@@ -7,6 +7,10 @@ export interface StorageDriver {filePath: string;byteRange?: ByteRange;}): Promise<Readable>;+ readFilePrefix(params: {+ filePath: string;+ byteCount: number;+ }): Promise<Buffer>;writeFile(params: {filePath: string;sourceFile: Buffer | Uint8Array | string;diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/local.driver.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/local.driver.tsindex 8c4c9d08087..6f4d78f485c 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/local.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/local.driver.ts@@ -44,15 +44,12 @@ export class LocalDriver implements StorageDriver {}}- async readFile(params: {- filePath: string;- byteRange?: ByteRange;- }): Promise<Readable> {- const joinedPath = join(this.options.storagePath, params.filePath);- let filePath: string;+ private resolveReadableRealPathOrThrow(filePath: string): string {+ const joinedPath = join(this.options.storagePath, filePath);+ let realPath: string;try {- filePath = realpathSync(path.resolve(joinedPath));+ realPath = realpathSync(path.resolve(joinedPath));} catch {throw new FileStorageException('File not found',@@ -60,7 +57,16 @@ export class LocalDriver implements StorageDriver {);}- this.assertRealPathIsWithinStorage(filePath);+ this.assertRealPathIsWithinStorage(realPath);++ return realPath;+ }++ async readFile(params: {+ filePath: string;+ byteRange?: ByteRange;+ }): Promise<Readable> {+ const filePath = this.resolveReadableRealPathOrThrow(params.filePath);try {return createReadStream(@@ -81,6 +87,41 @@ export class LocalDriver implements StorageDriver {}}+ async readFilePrefix(params: {+ filePath: string;+ byteCount: number;+ }): Promise<Buffer> {+ const filePath = this.resolveReadableRealPathOrThrow(params.filePath);+ let fileHandle: fs.FileHandle;++ try {+ fileHandle = await fs.open(filePath, 'r');+ } catch (error) {+ if (error.code === 'ENOENT') {+ throw new FileStorageException(+ 'File not found',+ FileStorageExceptionCode.FILE_NOT_FOUND,+ );+ }++ throw error;+ }++ try {+ const buffer = Buffer.alloc(params.byteCount);+ const { bytesRead } = await fileHandle.read(+ buffer,+ 0,+ params.byteCount,+ 0,+ );++ return buffer.subarray(0, bytesRead);+ } finally {+ await fileHandle.close();+ }+ }+// Resolves the on-disk path for a write, creating the parent folder and// enforcing storage containment + symlink rejection.private async resolveWritableRealPathOrThrow(diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.tsindex 38cc5f86374..0f8c11ddd06 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts@@ -27,7 +27,12 @@ import { isDefined } from 'twenty-shared/utils';import {FILE_STORAGE_S3_CONNECTION_TIMEOUT_MS,FILE_STORAGE_S3_MAX_SOCKETS,+ FILE_STORAGE_S3_METADATA_CONNECTION_TIMEOUT_MS,+ FILE_STORAGE_S3_METADATA_MAX_ATTEMPTS,+ FILE_STORAGE_S3_METADATA_MAX_SOCKETS,+ FILE_STORAGE_S3_METADATA_REQUEST_TIMEOUT_MS,FILE_STORAGE_S3_REQUEST_TIMEOUT_MS,+ FILE_STORAGE_S3_SLOW_REQUEST_THRESHOLD_MS,} from 'src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant';import { type StorageDriver } from 'src/engine/core-modules/file-storage/drivers/interfaces/storage-driver.interface';import {@@ -48,6 +53,7 @@ export interface S3DriverOptions extends S3ClientConfig {export class S3Driver implements StorageDriver {private s3Client: S3;+ private metadataClient: S3;private presignClient: S3 | undefined;private bucketName: string;private readonly logger = new Logger(S3Driver.name);@@ -73,6 +79,19 @@ export class S3Driver implements StorageDriver {});this.s3Client = new S3({ ...s3Options, region, endpoint, requestHandler });++ this.metadataClient = new S3({+ ...s3Options,+ region,+ endpoint,+ maxAttempts: FILE_STORAGE_S3_METADATA_MAX_ATTEMPTS,+ responseChecksumValidation: 'WHEN_REQUIRED',+ requestHandler: buildAwsRequestHandlerOptions({+ requestTimeoutMs: FILE_STORAGE_S3_METADATA_REQUEST_TIMEOUT_MS,+ connectionTimeoutMs: FILE_STORAGE_S3_METADATA_CONNECTION_TIMEOUT_MS,+ maxSockets: FILE_STORAGE_S3_METADATA_MAX_SOCKETS,+ }),+ });this.bucketName = bucketName;if (presignEnabled) {@@ -123,6 +142,52 @@ export class S3Driver implements StorageDriver {}}+ async readFilePrefix(params: {+ filePath: string;+ byteCount: number;+ }): Promise<Buffer> {+ const command = new GetObjectCommand({+ Key: params.filePath,+ Bucket: this.bucketName,+ Range: `bytes=0-${params.byteCount - 1}`,+ });++ try {+ return await this.measureRequest(+ { operation: 'GetObjectPrefix', key: params.filePath },+ async () => {+ try {+ const file = await this.metadataClient.send(command);++ if (!isDefined(file?.Body)) {+ throw new FileStorageException(+ 'Unable to get file body',+ FileStorageExceptionCode.FILE_NOT_FOUND,+ );+ }++ return Buffer.from(await file.Body.transformToByteArray());+ } catch (error) {+ if (error.name === 'InvalidRange') {+ return Buffer.alloc(0);+ }++ throw error;+ }+ },+ );+ } catch (error) {+ if (error.name === 'NoSuchKey') {+ throw new FileStorageException(+ 'File not found',+ FileStorageExceptionCode.FILE_NOT_FOUND,+ );+ }++ throw error;+ }+ }+async writeFile(params: {filePath: string;sourceFile: Buffer | Uint8Array | string;@@ -161,20 +226,54 @@ export class S3Driver implements StorageDriver {async getFileMetadata(params: {filePath: string;}): Promise<{ size: number } | null> {+ return this.measureRequest(+ { operation: 'HeadObject', key: params.filePath },+ async () => {+ try {+ const head = await this.metadataClient.send(+ new HeadObjectCommand({+ Bucket: this.bucketName,+ Key: params.filePath,+ }),+ );++ return { size: head.ContentLength ?? 0 };+ } catch (error) {+ if (error instanceof NotFound) {+ return null;+ }++ throw error;+ }+ },+ );+ }++ private async measureRequest<TResult>(+ { operation, key }: { operation: string; key: string },+ request: () => Promise<TResult>,+ ): Promise<TResult> {+ const startedAt = Date.now();+try {- const head = await this.s3Client.send(- new HeadObjectCommand({- Bucket: this.bucketName,- Key: params.filePath,- }),- );+ const result = await request();+ const durationMs = Date.now() - startedAt;+ const message = `S3 ${operation} ${key} succeeded in ${durationMs}ms`;- return { size: head.ContentLength ?? 0 };- } catch (error) {- if (error instanceof NotFound) {- return null;+ if (durationMs >= FILE_STORAGE_S3_SLOW_REQUEST_THRESHOLD_MS) {+ this.logger.warn(message);+ } else {+ this.logger.debug(message);}+ return result;+ } catch (error) {+ const durationMs = Date.now() - startedAt;++ this.logger.warn(+ `S3 ${operation} ${key} failed after ${durationMs}ms: ${error?.name ?? 'Error'} ${error?.message ?? ''}`,+ );+throw error;}}diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/validated-storage.driver.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/validated-storage.driver.tsindex 62ff94ff010..0b6576ba112 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/validated-storage.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/validated-storage.driver.ts@@ -17,6 +17,15 @@ export class ValidatedStorageDriver implements StorageDriver {return this.delegate.readFile(params);}+ async readFilePrefix(params: {+ filePath: string;+ byteCount: number;+ }): Promise<Buffer> {+ assertStoragePathIsSafe(params.filePath);++ return this.delegate.readFilePrefix(params);+ }+async writeFile(params: {filePath: string;sourceFile: Buffer | Uint8Array | string;diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/services/file-storage.service.ts b/packages/twenty-server/src/engine/core-modules/file-storage/services/file-storage.service.tsindex 22cda530872..ff0c3bd13d7 100644--- a/packages/twenty-server/src/engine/core-modules/file-storage/services/file-storage.service.ts+++ b/packages/twenty-server/src/engine/core-modules/file-storage/services/file-storage.service.ts@@ -392,6 +392,20 @@ export class FileStorageService {});}+ readFilePrefix(+ params: ResourceIdentifier & { byteCount: number },+ ): Promise<Buffer> {+ const driver = this.fileStorageDriverFactory.getCurrentDriver();++ const { onStorageFilePath } =+ this.validateAndBuildFileStoragePathOrThrow(params);++ return driver.readFilePrefix({+ filePath: onStorageFilePath,+ byteCount: params.byteCount,+ });+ }+downloadFile(params: ResourceIdentifier & { localPath: string },): Promise<void> {diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/constants/complete-file-upload-deadline.constant.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/constants/complete-file-upload-deadline.constant.tsnew file mode 100644index 00000000000..76eb69324eb--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/constants/complete-file-upload-deadline.constant.ts@@ -0,0 +1 @@+export const COMPLETE_FILE_UPLOAD_DEADLINE_MS = 60 * 1000;diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.exception.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.exception.tsindex 8e3cb63077f..0ccc4cf7523 100644--- a/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.exception.ts+++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.exception.ts@@ -8,6 +8,7 @@ export enum FileUploadExceptionCode {FILE_NOT_UPLOADED = 'FILE_NOT_UPLOADED',FILE_SIZE_MISMATCH = 'FILE_SIZE_MISMATCH',FILE_TOO_LARGE = 'FILE_TOO_LARGE',+ STORAGE_TIMEOUT = 'STORAGE_TIMEOUT',}export class FileUploadException extends CustomException<FileUploadExceptionCode> {diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/filters/file-upload-api-exception.filter.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/filters/file-upload-api-exception.filter.tsindex 1c314b08552..a98819e7cb5 100644--- a/packages/twenty-server/src/engine/core-modules/file/file-upload/filters/file-upload-api-exception.filter.ts+++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/filters/file-upload-api-exception.filter.ts@@ -35,6 +35,12 @@ export class FileUploadApiExceptionFilter implements ExceptionFilter {response,413,);+ case FileUploadExceptionCode.STORAGE_TIMEOUT:+ return this.httpExceptionHandlerService.handleError(+ exception,+ response,+ 504,+ );case FileUploadExceptionCode.BAD_REQUEST:case FileUploadExceptionCode.FILE_NOT_UPLOADED:case FileUploadExceptionCode.FILE_SIZE_MISMATCH:diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload-completion.service.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload-completion.service.tsindex f2481315bf3..29902cd4cb9 100644--- a/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload-completion.service.ts+++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload-completion.service.ts@@ -22,7 +22,6 @@ import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/fileimport { sanitizeFile } from 'src/engine/core-modules/file/utils/sanitize-file.utils';import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';-import { readReadablePrefix } from 'src/utils/read-readable-prefix';import { streamToBuffer } from 'src/utils/stream-to-buffer';export type BatchCompleteUploadRequest = {@@ -152,18 +151,14 @@ export class FileUploadCompletionService {resourcePath,filename,}: FileUploadStorageLocation & { filename: string }): Promise<string> {- const stream = await this.fileStorageService.readFile({+ const prefix = await this.fileStorageService.readFilePrefix({fileFolder,applicationUniversalIdentifier,workspaceId,resourcePath,+ byteCount: FILE_CONTENT_SNIFF_BYTE_COUNT,});- const prefix = await readReadablePrefix(- stream,- FILE_CONTENT_SNIFF_BYTE_COUNT,- );-const { mimeType } = await extractFileInfoOrThrow({file: prefix,filename,diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.tsindex 5a7c585e9ed..000e5e0fc3b 100644--- a/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.ts+++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/services/file-upload.service.ts@@ -1,4 +1,4 @@-import { Injectable } from '@nestjs/common';+import { Injectable, Logger } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { Readable, Transform } from 'stream';@@ -18,6 +18,7 @@ import { ApplicationService } from 'src/engine/core-modules/application/applicatimport { FileStorageService } from 'src/engine/core-modules/file-storage/services/file-storage.service';import { FileWithSignedUrlDTO } from 'src/engine/core-modules/file/dtos/file-with-sign-url.dto';import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';+import { COMPLETE_FILE_UPLOAD_DEADLINE_MS } from 'src/engine/core-modules/file/file-upload/constants/complete-file-upload-deadline.constant';import { FileUploadTargetDTO } from 'src/engine/core-modules/file/file-upload/dtos/file-upload-target.dto';import {FileUploadException,@@ -32,6 +33,7 @@ import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/fileimport { FieldMetadataEntity } from 'src/engine/metadata-modules/field-metadata/field-metadata.entity';import { InjectWorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/inject-workspace-scoped-repository.decorator';import { WorkspaceScopedRepository } from 'src/engine/twenty-orm/workspace-scoped-repository/workspace-scoped-repository';+import { withDeadline } from 'src/utils/with-deadline';export const DIRECT_UPLOAD_FILE_FOLDERS = [FileFolder.FilesField,@@ -43,6 +45,8 @@ export const DIRECT_UPLOAD_FILE_FOLDERS = [@Injectable()export class FileUploadService {+ private readonly logger = new Logger(FileUploadService.name);+constructor(private readonly fileStorageService: FileStorageService,private readonly fileUrlService: FileUrlService,@@ -271,8 +275,8 @@ export class FileUploadService {file,});- const completedFile =- await this.fileUploadCompletionService.completeUploadedFile({+ const completedFile = await withDeadline({+ promise: this.fileUploadCompletionService.completeUploadedFile({workspaceId,file,storageLocation: {@@ -281,7 +285,24 @@ export class FileUploadService {workspaceId,resourcePath,},- });+ }),+ timeoutMs: COMPLETE_FILE_UPLOAD_DEADLINE_MS,+ createTimeoutError: () =>+ new FileUploadException(+ `Completion of file ${fileId} exceeded ${COMPLETE_FILE_UPLOAD_DEADLINE_MS}ms waiting on storage`,+ FileUploadExceptionCode.STORAGE_TIMEOUT,+ {+ userFriendlyMessage: msg`File storage took too long to respond. Please retry.`,+ },+ ),+ onSettleAfterDeadline: (settlement) => {+ this.logger.warn(+ settlement.status === 'fulfilled'+ ? `Completion of file ${fileId} succeeded after the deadline had been reported to the client`+ : `Completion of file ${fileId} failed after the deadline had been reported to the client: ${settlement.error}`,+ );+ },+ });return this.toFileWithSignedUrl({file: { ...file, ...completedFile, status: FILE_STATUS.UPLOADED },diff --git a/packages/twenty-server/src/utils/__tests__/read-readable-prefix.spec.ts b/packages/twenty-server/src/utils/__tests__/read-readable-prefix.spec.tsdeleted file mode 100644index e94af5071f1..00000000000--- a/packages/twenty-server/src/utils/__tests__/read-readable-prefix.spec.ts+++ /dev/null@@ -1,62 +0,0 @@-import { Readable } from 'stream';--import { readReadablePrefix } from 'src/utils/read-readable-prefix';--describe('readReadablePrefix', () => {- it('should return the whole content when it is shorter than the limit', async () => {- const prefix = await readReadablePrefix(- Readable.from(Buffer.from('hello')),- 1024,- );-- expect(prefix.toString()).toBe('hello');- });-- it('should stop at the limit and not buffer the rest of a large source', async () => {- let producedBytes = 0;-- const stream = new Readable({- read() {- producedBytes += 1024;- this.push(Buffer.alloc(1024, 0x61));-- if (producedBytes >= 1024 * 1024) {- this.push(null);- }- },- });-- const prefix = await readReadablePrefix(stream, 4096);-- expect(prefix.length).toBe(4096);- expect(stream.destroyed).toBe(true);- expect(producedBytes).toBeLessThan(1024 * 1024);- });-- it('should bound the buffer to maxBytes even when a single chunk overshoots', async () => {- const prefix = await readReadablePrefix(- Readable.from(Buffer.alloc(64 * 1024, 0x61)),- 4096,- );-- expect(prefix.length).toBe(4096);- });-- it('should reject when the stream errors before the limit', async () => {- const stream = new Readable({- read() {- this.destroy(new Error('storage exploded'));- },- });-- await expect(readReadablePrefix(stream, 4096)).rejects.toThrow(- 'storage exploded',- );- });-- it('should return an empty buffer for an empty stream', async () => {- const prefix = await readReadablePrefix(Readable.from([]), 4096);-- expect(prefix.length).toBe(0);- });-});diff --git a/packages/twenty-server/src/utils/__tests__/with-deadline.spec.ts b/packages/twenty-server/src/utils/__tests__/with-deadline.spec.tsnew file mode 100644index 00000000000..6908789eb2b--- /dev/null+++ b/packages/twenty-server/src/utils/__tests__/with-deadline.spec.ts@@ -0,0 +1,118 @@+import { withDeadline } from 'src/utils/with-deadline';++describe('withDeadline', () => {+ beforeEach(() => {+ jest.useFakeTimers();+ });++ afterEach(() => {+ jest.useRealTimers();+ });++ it('should resolve with the promise value when it settles in time', async () => {+ const result = withDeadline({+ promise: Promise.resolve('done'),+ timeoutMs: 1000,+ createTimeoutError: () => new Error('too slow'),+ });++ await expect(result).resolves.toBe('done');+ });++ it('should reject with the timeout error when the deadline passes first', async () => {+ const result = withDeadline({+ promise: new Promise<never>(() => {}),+ timeoutMs: 1000,+ createTimeoutError: () => new Error('too slow'),+ });++ jest.advanceTimersByTime(1000);++ await expect(result).rejects.toThrow('too slow');+ });++ it('should propagate the promise rejection when it fails in time', async () => {+ const result = withDeadline({+ promise: Promise.reject(new Error('boom')),+ timeoutMs: 1000,+ createTimeoutError: () => new Error('too slow'),+ });++ await expect(result).rejects.toThrow('boom');+ });++ it('should report a settlement that happens after the deadline', async () => {+ const onSettleAfterDeadline = jest.fn();+ let resolveLate: (value: string) => void = () => {};+ const latePromise = new Promise<string>((resolve) => {+ resolveLate = resolve;+ });++ const result = withDeadline({+ promise: latePromise,+ timeoutMs: 1000,+ createTimeoutError: () => new Error('too slow'),+ onSettleAfterDeadline,+ });++ jest.advanceTimersByTime(1000);++ await expect(result).rejects.toThrow('too slow');++ resolveLate('late');+ await latePromise;++ expect(onSettleAfterDeadline).toHaveBeenCalledWith({ status: 'fulfilled' });+ });++ it('should report a rejection that happens after the deadline', async () => {+ const onSettleAfterDeadline = jest.fn();+ const lateError = new Error('late failure');+ let rejectLate: (error: Error) => void = () => {};+ const latePromise = new Promise<string>((_, reject) => {+ rejectLate = reject;+ });++ const result = withDeadline({+ promise: latePromise,+ timeoutMs: 1000,+ createTimeoutError: () => new Error('too slow'),+ onSettleAfterDeadline,+ });++ jest.advanceTimersByTime(1000);++ await expect(result).rejects.toThrow('too slow');++ rejectLate(lateError);+ await expect(latePromise).rejects.toBe(lateError);++ expect(onSettleAfterDeadline).toHaveBeenCalledWith({+ status: 'rejected',+ error: lateError,+ });+ });++ it('should not report a settlement that happens before the deadline', async () => {+ const onSettleAfterDeadline = jest.fn();++ await withDeadline({+ promise: Promise.resolve('done'),+ timeoutMs: 1000,+ createTimeoutError: () => new Error('too slow'),+ onSettleAfterDeadline,+ });++ expect(onSettleAfterDeadline).not.toHaveBeenCalled();+ });++ it('should clear the timer once the promise settles', async () => {+ await withDeadline({+ promise: Promise.resolve('done'),+ timeoutMs: 1000,+ createTimeoutError: () => new Error('too slow'),+ });++ expect(jest.getTimerCount()).toBe(0);+ });+});diff --git a/packages/twenty-server/src/utils/read-readable-prefix.ts b/packages/twenty-server/src/utils/read-readable-prefix.tsdeleted file mode 100644index caadcfa416e..00000000000--- a/packages/twenty-server/src/utils/read-readable-prefix.ts+++ /dev/null@@ -1,55 +0,0 @@-import { type Readable } from 'stream';--export const readReadablePrefix = async (- stream: Readable,- maxBytes: number,-): Promise<Buffer> => {- const chunks: Buffer[] = [];- let collected = 0;- let settled = false;-- return new Promise((resolve, reject) => {- const onData = (chunk: Buffer) => {- if (settled) {- return;- }-- const remaining = maxBytes - collected;- const boundedChunk =- chunk.length > remaining ? chunk.subarray(0, remaining) : chunk;-- chunks.push(boundedChunk);- collected += boundedChunk.length;-- if (collected >= maxBytes) {- settled = true;- stream.off('data', onData);- stream.off('end', onEnd);- stream.destroy();- resolve(Buffer.concat(chunks));- }- };-- const onEnd = () => {- if (settled) {- return;- }-- settled = true;- resolve(Buffer.concat(chunks));- };-- const onError = (error: Error) => {- if (settled) {- return;- }-- settled = true;- reject(error);- };-- stream.on('data', onData);- stream.on('end', onEnd);- stream.on('error', onError);- });-};diff --git a/packages/twenty-server/src/utils/with-deadline.ts b/packages/twenty-server/src/utils/with-deadline.tsnew file mode 100644index 00000000000..f0fccc51919--- /dev/null+++ b/packages/twenty-server/src/utils/with-deadline.ts@@ -0,0 +1,41 @@+import { isDefined } from 'twenty-shared/utils';++export type DeadlineLateSettlement =+ | { status: 'fulfilled' }+ | { status: 'rejected'; error: unknown };++export const withDeadline = async <TResult>({+ promise,+ timeoutMs,+ createTimeoutError,+ onSettleAfterDeadline,+}: {+ promise: Promise<TResult>;+ timeoutMs: number;+ createTimeoutError: () => Error;+ onSettleAfterDeadline?: (settlement: DeadlineLateSettlement) => void;+}): Promise<TResult> => {+ let timeoutId: NodeJS.Timeout | undefined;+ let hasTimedOut = false;++ const deadline = new Promise<never>((_, reject) => {+ timeoutId = setTimeout(() => {+ hasTimedOut = true;+ reject(createTimeoutError());+ }, timeoutMs);+ });++ try {+ return await Promise.race([promise, deadline]);+ } finally {+ clearTimeout(timeoutId);++ if (hasTimedOut && isDefined(onSettleAfterDeadline)) {+ promise.then(+ () => onSettleAfterDeadline({ status: 'fulfilled' }),+ (error: unknown) =>+ onSettleAfterDeadline({ status: 'rejected', error }),+ );+ }+ }+};