Skip to content

The open alternative to Salesforce, designed for AI.

fix(file-storage): create the missing file row when copying a file by path (#26582)

martmull97ddf1056450
diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/services/__tests__/file-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/file-storage/services/__tests__/file-storage.service.spec.ts
index 335db3ccb13..35a5ccb38d2 100644
--- a/packages/twenty-server/src/engine/core-modules/file-storage/services/__tests__/file-storage.service.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/file-storage/services/__tests__/file-storage.service.spec.ts
@@ -2,6 +2,7 @@ import { type QueryRunner } from 'typeorm';
import { FileFolder } from 'twenty-shared/types';
import { type FileStorageDriverFactory } from 'src/engine/core-modules/file-storage/file-storage-driver.factory';
+import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
import { FileStorageService } from 'src/engine/core-modules/file-storage/services/file-storage.service';
import { type FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
import { type WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
@@ -21,29 +22,73 @@ describe('FileStorageService', () => {
.mockImplementation((_workspaceId, entity) => entity),
};
+ const insertQueryBuilder = {
+ insert: jest.fn().mockReturnThis(),
+ values: jest.fn().mockReturnThis(),
+ orIgnore: jest.fn().mockReturnThis(),
+ returning: jest.fn().mockReturnThis(),
+ execute: jest.fn().mockResolvedValue({ raw: [{ id: 'created-row' }] }),
+ };
+
const fileRepository = {
findOne: jest.fn().mockResolvedValue(existingFile),
+ createQueryBuilder: jest.fn().mockReturnValue(insertQueryBuilder),
upsertAndReturnOne: jest
.fn()
.mockImplementation((_workspaceId, entity) => entity),
+ insertAndReturnOne: jest
+ .fn()
+ .mockImplementation((_workspaceId, entity) => entity),
withManager: jest.fn().mockReturnValue(transactionRepository),
};
- const driver = { writeFile: jest.fn().mockResolvedValue(undefined) };
+ const driver = {
+ writeFile: jest.fn().mockResolvedValue(undefined),
+ getFileMetadata: jest.fn().mockResolvedValue({ size: 42 }),
+ readFilePrefix: jest
+ .fn()
+ .mockResolvedValue(Buffer.from('export const main = () => {};')),
+ checkFileExists: jest.fn().mockResolvedValue(true),
+ copy: jest.fn().mockResolvedValue(undefined),
+ };
+
+ const usageLimitStockService = {
+ assertStockAvailable: jest.fn().mockResolvedValue(undefined),
+ acquireStock: jest.fn().mockResolvedValue(undefined),
+ };
const service = new FileStorageService(
{
getCurrentDriver: () => driver,
} as unknown as FileStorageDriverFactory,
fileRepository as unknown as WorkspaceScopedRepository<FileEntity>,
- {} as WorkspaceCacheService,
{
- assertStockAvailable: jest.fn().mockResolvedValue(undefined),
- acquireStock: jest.fn().mockResolvedValue(undefined),
- } as unknown as UsageLimitStockService,
+ getOrRecompute: jest.fn().mockResolvedValue({
+ flatApplicationMaps: {
+ byId: {
+ [APPLICATION_ID]: {
+ id: APPLICATION_ID,
+ universalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
+ deletedAt: null,
+ },
+ },
+ idByUniversalIdentifier: {
+ [APPLICATION_UNIVERSAL_IDENTIFIER]: APPLICATION_ID,
+ },
+ },
+ }),
+ } as unknown as WorkspaceCacheService,
+ usageLimitStockService as unknown as UsageLimitStockService,
);
- return { service, fileRepository, transactionRepository };
+ return {
+ service,
+ fileRepository,
+ transactionRepository,
+ insertQueryBuilder,
+ driver,
+ usageLimitStockService,
+ };
};
const writeFile = (
@@ -76,6 +121,22 @@ describe('FileStorageService', () => {
settings: { isTemporaryFile: false, toDelete: false },
});
+ const copyFileByPath = (service: FileStorageService) =>
+ service.copyFileByPath({
+ from: {
+ workspaceId: WORKSPACE_ID,
+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
+ fileFolder: FileFolder.Source,
+ resourcePath: 'original/src/index.ts',
+ },
+ to: {
+ workspaceId: WORKSPACE_ID,
+ applicationUniversalIdentifier: APPLICATION_UNIVERSAL_IDENTIFIER,
+ fileFolder: FileFolder.Source,
+ resourcePath: 'copy/src/index.ts',
+ },
+ });
+
it('should keep the identifier of the row already stored at that path', async () => {
const { service, fileRepository } = buildService({
id: 'the-row-the-application-points-at',
@@ -149,4 +210,97 @@ describe('FileStorageService', () => {
['path', 'workspaceId', 'applicationId'],
);
});
+
+ it('should copy a file with the size and mime type of its row', async () => {
+ const { service, fileRepository, driver } = buildService({
+ mimeType: 'application/typescript',
+ size: 12,
+ settings: { isTemporaryFile: false, toDelete: false },
+ });
+
+ await copyFileByPath(service);
+
+ expect(driver.getFileMetadata).not.toHaveBeenCalled();
+ expect(fileRepository.createQueryBuilder).not.toHaveBeenCalled();
+ expect(fileRepository.insertAndReturnOne).toHaveBeenCalledWith(
+ WORKSPACE_ID,
+ expect.objectContaining({
+ path: 'source/copy/src/index.ts',
+ mimeType: 'application/typescript',
+ size: 12,
+ }),
+ );
+ });
+
+ it('should create the missing row of a copied file from the stored file', async () => {
+ const { service, fileRepository, insertQueryBuilder, driver } =
+ buildService(null);
+
+ await copyFileByPath(service);
+
+ expect(insertQueryBuilder.values).toHaveBeenCalledWith({
+ workspaceId: WORKSPACE_ID,
+ path: 'source/original/src/index.ts',
+ applicationId: APPLICATION_ID,
+ mimeType: 'application/typescript',
+ size: 42,
+ status: 'UPLOADED',
+ settings: { isTemporaryFile: false, toDelete: false },
+ });
+ expect(insertQueryBuilder.orIgnore).toHaveBeenCalled();
+ expect(driver.copy).toHaveBeenCalledTimes(1);
+ expect(fileRepository.insertAndReturnOne).toHaveBeenCalledWith(
+ WORKSPACE_ID,
+ expect.objectContaining({
+ path: 'source/copy/src/index.ts',
+ mimeType: 'application/typescript',
+ size: 42,
+ }),
+ );
+ });
+
+ it('should charge the storage stock for both the created row and the copy', async () => {
+ const { service, usageLimitStockService } = buildService(null);
+
+ await copyFileByPath(service);
+
+ expect(usageLimitStockService.acquireStock).toHaveBeenCalledTimes(2);
+ expect(usageLimitStockService.acquireStock).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({
+ spenders: { applicationId: APPLICATION_ID },
+ cost: { bytes: 42, quantity: 1 },
+ }),
+ );
+ expect(usageLimitStockService.acquireStock).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({
+ spenders: { applicationId: APPLICATION_ID },
+ cost: { bytes: 42, quantity: 1 },
+ }),
+ );
+ });
+
+ it('should only charge the copy when a concurrent copy already created the missing row', async () => {
+ const { service, insertQueryBuilder, usageLimitStockService } =
+ buildService(null);
+
+ insertQueryBuilder.execute.mockResolvedValue({ raw: [] });
+
+ await copyFileByPath(service);
+
+ expect(usageLimitStockService.acquireStock).toHaveBeenCalledTimes(1);
+ });
+
+ it('should fail to copy a file missing from storage', async () => {
+ const { service, insertQueryBuilder, driver } = buildService(null);
+
+ driver.getFileMetadata.mockResolvedValue(null);
+
+ await expect(copyFileByPath(service)).rejects.toMatchObject({
+ code: FileStorageExceptionCode.FILE_NOT_FOUND,
+ });
+ expect(insertQueryBuilder.values).not.toHaveBeenCalled();
+ expect(driver.copy).not.toHaveBeenCalled();
+ });
});
diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/services/file-storage.service.ts b/packages/twenty-server/src/engine/core-modules/file-storage/services/file-storage.service.ts
index 58551c83630..53001fcc0ae 100644
--- a/packages/twenty-server/src/engine/core-modules/file-storage/services/file-storage.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/file-storage/services/file-storage.service.ts
@@ -6,7 +6,7 @@ import { type Readable } from 'stream';
import { v4 } from 'uuid';
import { FileFolder } from 'twenty-shared/types';
-import { isDefined } from 'twenty-shared/utils';
+import { isDefined, isNonEmptyArray } from 'twenty-shared/utils';
import {
type FindOptionsWhere,
Like,
@@ -31,8 +31,10 @@ import { validateFilePath } from 'src/engine/core-modules/file-storage/utils/val
import { validateFolderPath } from 'src/engine/core-modules/file-storage/utils/validate-folder-path.util';
import { validateStoragePathIsWithinWorkspaceOrThrow } from 'src/engine/core-modules/file-storage/utils/validate-storage-path-is-within-workspace-or-throw.util';
import { FileEntity } from 'src/engine/core-modules/file/entities/file.entity';
+import { FILE_CONTENT_SNIFF_BYTE_COUNT } from 'src/engine/core-modules/file/file-upload/constants/file-content-sniff.constant';
import { FileSettings } from 'src/engine/core-modules/file/types/file-settings.types';
import { FILE_STATUS } from 'src/engine/core-modules/file/types/file-status.types';
+import { extractFileInfoOrThrow } from 'src/engine/core-modules/file/utils/extract-file-info-or-throw.utils';
import { removeFileFolderFromFileEntityPath } from 'src/engine/core-modules/file/utils/remove-file-folder-from-file-entity-path.utils';
import { STOCK_METERS } from 'src/engine/core-modules/usage-limit/constants/usage-meters.constant';
import { UsageLimitStockService } from 'src/engine/core-modules/usage-limit/services/usage-limit-stock.service';
@@ -188,6 +190,63 @@ export class FileStorageService {
});
}
+ private async createFileRowFromStorageOrThrow({
+ resourceIdentifier,
+ filePath,
+ applicationId,
+ }: {
+ resourceIdentifier: ResourceIdentifier;
+ filePath: string;
+ applicationId: string;
+ }): Promise<Pick<FileEntity, 'mimeType' | 'size' | 'settings'>> {
+ const metadata = await this.getFileMetadata(resourceIdentifier);
+
+ if (!isDefined(metadata)) {
+ throw new FileStorageException(
+ `File not found at path "${filePath}"`,
+ FileStorageExceptionCode.FILE_NOT_FOUND,
+ );
+ }
+
+ const { mimeType } = await extractFileInfoOrThrow({
+ file: await this.readFilePrefix({
+ ...resourceIdentifier,
+ byteCount: FILE_CONTENT_SNIFF_BYTE_COUNT,
+ }),
+ filename: resourceIdentifier.resourcePath,
+ });
+
+ const fileRow = {
+ mimeType,
+ size: metadata.size,
+ settings: { isTemporaryFile: false, toDelete: false },
+ };
+
+ const insertResult = await this.fileRepository
+ .createQueryBuilder()
+ .insert()
+ .values({
+ ...fileRow,
+ workspaceId: resourceIdentifier.workspaceId,
+ path: filePath,
+ applicationId,
+ status: FILE_STATUS.UPLOADED,
+ })
+ .orIgnore()
+ .returning('id')
+ .execute();
+
+ if (isNonEmptyArray(insertResult.raw)) {
+ await this.applyStorageStockDelta({
+ workspaceId: resourceIdentifier.workspaceId,
+ applicationId,
+ delta: { bytes: metadata.size, quantity: 1 },
+ });
+ }
+
+ return fileRow;
+ }
+
private async applyStorageStockDelta({
workspaceId,
applicationId,
@@ -896,19 +955,18 @@ export class FileStorageService {
this.resolveApplicationIdOrThrow(to),
]);
- const sourceFile = await this.findFileByPath({
- fileRepository: this.fileRepository,
- workspaceId: from.workspaceId,
- filePath,
- applicationId: sourceApplicationId,
- });
-
- if (!isDefined(sourceFile)) {
- throw new FileStorageException(
- `File not found at path "${filePath}"`,
- FileStorageExceptionCode.FILE_NOT_FOUND,
- );
- }
+ const sourceFile =
+ (await this.findFileByPath({
+ fileRepository: this.fileRepository,
+ workspaceId: from.workspaceId,
+ filePath,
+ applicationId: sourceApplicationId,
+ })) ??
+ (await this.createFileRowFromStorageOrThrow({
+ resourceIdentifier: from,
+ filePath,
+ applicationId: sourceApplicationId,
+ }));
return this.copyFile({
from,
@@ -921,7 +979,7 @@ export class FileStorageService {
});
}
- async copy({
+ private async copy({
from,
to,
}: {