Skip to content

The open alternative to Salesforce, designed for AI.

fix: fall back to unconditional copy when S3 backend rejects CopySourceIfMatch (#26830)

Félix Malfaita54bba52559d
diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/s3.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/s3.driver.spec.ts
index e7567276905..24f02927ad6 100644
--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/s3.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/__tests__/s3.driver.spec.ts
@@ -1,4 +1,6 @@
import {
+ CopyObjectCommand,
+ DeleteObjectCommand,
GetObjectCommand,
HeadObjectCommand,
PutObjectCommand,
@@ -8,6 +10,8 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { Readable } from 'stream';
+import { isDefined } from 'twenty-shared/utils';
+
import { FILE_STORAGE_S3_METADATA_MAX_ATTEMPTS } from 'src/engine/core-modules/file-storage/constants/s3-client-timeouts.constant';
import { S3Driver } from 'src/engine/core-modules/file-storage/drivers/s3.driver';
import { FileStorageExceptionCode } from 'src/engine/core-modules/file-storage/interfaces/file-storage-exception';
@@ -326,3 +330,113 @@ describe('S3Driver.getPresignedUploadUrl', () => {
});
});
});
+
+describe('S3Driver.move', () => {
+ const notImplementedError = Object.assign(
+ new Error('Copy object not implemented with X-Amz-Copy-Source-If-Match'),
+ { name: 'NotImplemented', $metadata: { httpStatusCode: 501 } },
+ );
+
+ const moveParams = {
+ from: { folderPath: 'pending', filename: 'file.png' },
+ to: { folderPath: 'final', filename: 'file.png' },
+ ifMatchChecksum: '"etag"',
+ };
+
+ const getCopyCommands = () =>
+ mockS3Send.mock.calls
+ .map(([command]) => command)
+ .filter((command) => command instanceof CopyObjectCommand);
+
+ afterEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it('should copy conditionally on the inspected checksum', async () => {
+ mockS3Send.mockResolvedValue({ ETag: '"etag"' });
+
+ const driver = new S3Driver({
+ bucketName: 'test-bucket',
+ region: 'us-east-1',
+ });
+
+ await driver.move(moveParams);
+
+ const copyCommands = getCopyCommands();
+
+ expect(copyCommands).toHaveLength(1);
+ expect(copyCommands[0].input).toMatchObject({
+ CopySource: 'test-bucket/pending/file.png',
+ CopySourceIfMatch: '"etag"',
+ Key: 'final/file.png',
+ });
+ const lastCommand =
+ mockS3Send.mock.calls[mockS3Send.mock.calls.length - 1][0];
+
+ expect(lastCommand).toBeInstanceOf(DeleteObjectCommand);
+ });
+
+ it('should fall back to an unconditional copy when the backend does not implement CopySourceIfMatch', async () => {
+ mockS3Send.mockImplementation(async (command) => {
+ if (
+ command instanceof CopyObjectCommand &&
+ isDefined(command.input.CopySourceIfMatch)
+ ) {
+ throw notImplementedError;
+ }
+
+ return { ETag: '"etag"' };
+ });
+
+ const driver = new S3Driver({
+ bucketName: 'test-bucket',
+ region: 'us-east-1',
+ });
+
+ await driver.move(moveParams);
+ await driver.move(moveParams);
+
+ const copyCommands = getCopyCommands();
+
+ expect(copyCommands).toHaveLength(3);
+ expect(copyCommands[0].input.CopySourceIfMatch).toBe('"etag"');
+ expect(copyCommands[1].input.CopySourceIfMatch).toBeUndefined();
+ expect(copyCommands[2].input.CopySourceIfMatch).toBeUndefined();
+ });
+
+ it('should still report a failed precondition', async () => {
+ mockS3Send.mockImplementation(async (command) => {
+ if (command instanceof CopyObjectCommand) {
+ throw Object.assign(new Error('Precondition failed'), {
+ name: 'PreconditionFailed',
+ });
+ }
+
+ return { ETag: '"etag"' };
+ });
+
+ const driver = new S3Driver({
+ bucketName: 'test-bucket',
+ region: 'us-east-1',
+ });
+
+ await expect(driver.move(moveParams)).rejects.toMatchObject({
+ code: FileStorageExceptionCode.PRECONDITION_FAILED,
+ });
+ expect(getCopyCommands()).toHaveLength(1);
+ });
+
+ it('should refuse to copy when the source changed since it was inspected', async () => {
+ mockS3Send.mockResolvedValue({ ETag: '"replaced"' });
+
+ const driver = new S3Driver({
+ bucketName: 'test-bucket',
+ region: 'us-east-1',
+ });
+
+ await expect(driver.move(moveParams)).rejects.toMatchObject({
+ code: FileStorageExceptionCode.PRECONDITION_FAILED,
+ });
+ expect(getCopyCommands()).toHaveLength(0);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts
index 81239322c5c..341d42837e3 100644
--- a/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/file-storage/drivers/s3.driver.ts
@@ -57,6 +57,7 @@ export class S3Driver implements StorageDriver {
private presignClient: S3 | undefined;
private bucketName: string;
private readonly logger = new Logger(S3Driver.name);
+ private supportsConditionalCopy = true;
constructor(options: S3DriverOptions) {
const {
@@ -333,21 +334,30 @@ export class S3Driver implements StorageDriver {
const toKey = `${params.to.folderPath}/${params.to.filename}`;
try {
- await this.s3Client.send(
+ const head = await this.s3Client.send(
new HeadObjectCommand({
Bucket: this.bucketName,
Key: fromKey,
}),
);
- await this.s3Client.send(
- new CopyObjectCommand({
- CopySource: `${this.bucketName}/${fromKey}`,
- CopySourceIfMatch: params.ifMatchChecksum,
- Bucket: this.bucketName,
- Key: toKey,
- }),
- );
+ // Backends without CopySourceIfMatch support get an unconditional copy,
+ // so this is their only check that the object is still the inspected one.
+ if (
+ isDefined(params.ifMatchChecksum) &&
+ head.ETag !== params.ifMatchChecksum
+ ) {
+ throw new FileStorageException(
+ `Object at ${fromKey} changed since it was inspected`,
+ FileStorageExceptionCode.PRECONDITION_FAILED,
+ );
+ }
+
+ await this.copyObjectIfMatch({
+ fromKey,
+ toKey,
+ ifMatchChecksum: params.ifMatchChecksum,
+ });
await this.s3Client.send(
new DeleteObjectCommand({
@@ -374,6 +384,57 @@ export class S3Driver implements StorageDriver {
}
}
+ // Some S3-compatible backends (e.g. OVHcloud) reject CopySourceIfMatch
+ // with 501, so fall back to an unconditional copy rather than failing
+ // every upload completion on them.
+ private async copyObjectIfMatch({
+ fromKey,
+ toKey,
+ ifMatchChecksum,
+ }: {
+ fromKey: string;
+ toKey: string;
+ ifMatchChecksum?: string;
+ }): Promise<void> {
+ const copySource = `${this.bucketName}/${fromKey}`;
+
+ if (isDefined(ifMatchChecksum) && this.supportsConditionalCopy) {
+ try {
+ await this.s3Client.send(
+ new CopyObjectCommand({
+ CopySource: copySource,
+ CopySourceIfMatch: ifMatchChecksum,
+ Bucket: this.bucketName,
+ Key: toKey,
+ }),
+ );
+
+ return;
+ } catch (error) {
+ const isNotImplemented =
+ error.name === 'NotImplemented' ||
+ error.$metadata?.httpStatusCode === 501;
+
+ if (!isNotImplemented) {
+ throw error;
+ }
+
+ this.supportsConditionalCopy = false;
+ this.logger.warn(
+ 'S3 backend does not support CopySourceIfMatch, falling back to unconditional copies',
+ );
+ }
+ }
+
+ await this.s3Client.send(
+ new CopyObjectCommand({
+ CopySource: copySource,
+ Bucket: this.bucketName,
+ Key: toKey,
+ }),
+ );
+ }
+
async copy(params: {
from: { folderPath: string; filename?: string };
to: { folderPath: string; filename?: string };