Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(api): resolve s3 copy source under endpoint path (#3422)

Hampus8bda15159674
diff --git a/fluxer_api/src/api/infrastructure/StorageService.test.ts b/fluxer_api/src/api/infrastructure/StorageService.test.ts
index e4280f013..0d60e16f6 100644
--- a/fluxer_api/src/api/infrastructure/StorageService.test.ts
+++ b/fluxer_api/src/api/infrastructure/StorageService.test.ts
@@ -225,3 +225,55 @@ describe('StorageService.listObjects', () => {
);
});
});
+
+function copySourceFor(endpoint: string, forcePathStyle: boolean): Promise<unknown> {
+ const service = new StorageService({
+ endpoint,
+ forcePathStyle,
+ region: 'us-east-1',
+ accessKeyId: 'fluxer',
+ secretAccessKey: 'fluxer-secret',
+ });
+ let copySource: unknown;
+ Object.assign(service, {
+ client: {
+ send: async (command: {input: {CopySource?: unknown}}) => {
+ copySource = command.input.CopySource;
+ return {};
+ },
+ },
+ });
+ return service
+ .copyObject({
+ sourceBucket: 'fluxer-uploads',
+ sourceKey: 'a b/c+d.pdf',
+ destinationBucket: 'fluxer-cdn',
+ destinationKey: 'attachments/1/2/c.pdf',
+ })
+ .then(() => copySource);
+}
+
+describe('StorageService.copyObject copy source', () => {
+ it.each([
+ ['http://seaweedfs:8333', true],
+ ['http://seaweedfs:8333/', true],
+ ['https://s3.example.test', false],
+ ])(
+ 'names only the bucket and key for an endpoint without a path (%s, path style %s)',
+ async (endpoint, pathStyle) => {
+ await expect(copySourceFor(endpoint, pathStyle)).resolves.toBe('fluxer-uploads/a%20b/c%2Bd.pdf');
+ },
+ );
+
+ it('puts the endpoint path in front of the bucket with path-style addressing', async () => {
+ await expect(copySourceFor('https://s3.example.test/real%20bucket/sub/', true)).resolves.toBe(
+ 'real%20bucket/sub/fluxer-uploads/a%20b/c%2Bd.pdf',
+ );
+ });
+
+ it('puts the endpoint path between the bucket and the key with virtual-hosted addressing', async () => {
+ await expect(copySourceFor('https://s3.example.test/real%20bucket/sub', false)).resolves.toBe(
+ 'fluxer-uploads/real%20bucket/sub/a%20b/c%2Bd.pdf',
+ );
+ });
+});
diff --git a/fluxer_api/src/api/infrastructure/StorageService.ts b/fluxer_api/src/api/infrastructure/StorageService.ts
index d1b3434d7..2c94cfd1f 100644
--- a/fluxer_api/src/api/infrastructure/StorageService.ts
+++ b/fluxer_api/src/api/infrastructure/StorageService.ts
@@ -115,6 +115,13 @@ const REJECTED_SERVER_SIDE_COPY_ERRORS = new Set([
'MethodNotAllowed',
]);
+function encodedEndpointPath(endpoint: string): Array<string> {
+ return (URL.parse(endpoint)?.pathname ?? '')
+ .split('/')
+ .filter((segment) => segment !== '')
+ .map((segment) => encodeURIComponent(decodeURIComponent(segment)));
+}
+
function isRejectedServerSideCopy(error: unknown): boolean {
return (
error instanceof S3ServiceException &&
@@ -126,6 +133,7 @@ export class StorageService implements IStorageService {
private readonly client: S3Client;
private readonly presignClient: S3Client;
private readonly provider: S3ProviderSettings;
+ private readonly endpointPath: Array<string>;
constructor(provider?: S3ProviderSettings) {
this.provider = provider ?? {
@@ -136,6 +144,7 @@ export class StorageService implements IStorageService {
accessKeyId: Config.s3.accessKeyId,
secretAccessKey: Config.s3.secretAccessKey,
};
+ this.endpointPath = encodedEndpointPath(this.provider.endpoint);
this.client = buildPooledS3Client({
endpoint: this.provider.endpoint,
region: this.provider.region,
@@ -492,7 +501,7 @@ export class StorageService implements IStorageService {
new CopyObjectCommand({
Bucket: destinationBucket,
Key: destinationKey,
- CopySource: `${encodeURIComponent(sourceBucket)}/${sourceKey.split('/').map(encodeURIComponent).join('/')}`,
+ CopySource: this.copySource(sourceBucket, sourceKey),
ContentType: newContentType,
MetadataDirective: newContentType ? 'REPLACE' : undefined,
}),
@@ -508,6 +517,14 @@ export class StorageService implements IStorageService {
}
}
+ private copySource(bucket: string, key: string): string {
+ const encodedBucket = encodeURIComponent(bucket);
+ const location = this.provider.forcePathStyle
+ ? [...this.endpointPath, encodedBucket]
+ : [encodedBucket, ...this.endpointPath];
+ return [...location, ...key.split('/').map(encodeURIComponent)].join('/');
+ }
+
private async copyObjectThroughApi(
{
sourceBucket,
diff --git a/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts b/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts
new file mode 100644
index 000000000..0749db71b
--- /dev/null
+++ b/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts
@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {StorageService} from '@app/api/infrastructure/StorageService';
+import {server} from '@app/api/test/msw/server';
+import {HttpResponse, http} from 'msw';
+import {beforeEach, describe, expect, it} from 'vitest';
+
+const HOST = 's3.single-bucket.test';
+const REAL_BUCKET = 'realbucket';
+
+function errorResponse(code: string, status: number): HttpResponse<string> {
+ return new HttpResponse(
+ `<?xml version="1.0" encoding="UTF-8"?><Error><Code>${code}</Code><Message></Message><RequestId>tx0</RequestId></Error>`,
+ {status, headers: {'Content-Type': 'application/xml'}},
+ );
+}
+
+function splitBucket(path: string): {bucket: string; key: string} {
+ const [bucket = '', ...key] = path.split('/');
+ return {bucket, key: key.join('/')};
+}
+
+function singleBucketStore() {
+ const objects = new Map<string, {body: Uint8Array; contentType: string}>();
+ const resolve = (url: URL): {bucket: string; key: string} => {
+ const path = decodeURIComponent(url.pathname.slice(1));
+ if (url.hostname === HOST) {
+ return splitBucket(path);
+ }
+ return {bucket: url.hostname.slice(0, -HOST.length - 1), key: path};
+ };
+ server.use(
+ http.all(/^https:\/\/([a-z0-9-]+\.)?s3\.single-bucket\.test\//u, async ({request}) => {
+ const target = resolve(new URL(request.url));
+ if (target.bucket !== REAL_BUCKET) {
+ return errorResponse('NoSuchBucket', 404);
+ }
+ if (request.method === 'GET') {
+ const object = objects.get(target.key);
+ if (!object) {
+ return errorResponse('NoSuchKey', 404);
+ }
+ return new HttpResponse(object.body, {
+ status: 200,
+ headers: {'Content-Type': object.contentType, 'Content-Length': String(object.body.length)},
+ });
+ }
+ if (request.method !== 'PUT') {
+ return errorResponse('NotImplemented', 501);
+ }
+ const copySource = request.headers.get('x-amz-copy-source');
+ if (!copySource) {
+ objects.set(target.key, {
+ body: new Uint8Array(await request.arrayBuffer()),
+ contentType: request.headers.get('content-type') ?? 'application/octet-stream',
+ });
+ return new HttpResponse(null, {status: 200, headers: {ETag: '"x"'}});
+ }
+ const source = splitBucket(decodeURIComponent(copySource.replace(/^\//u, '')));
+ if (source.bucket !== REAL_BUCKET) {
+ return errorResponse('NoSuchBucket', 404);
+ }
+ const object = objects.get(source.key);
+ if (!object) {
+ return errorResponse('NoSuchKey', 404);
+ }
+ objects.set(target.key, {...object, contentType: request.headers.get('content-type') ?? object.contentType});
+ return new HttpResponse(
+ '<?xml version="1.0" encoding="UTF-8"?><CopyObjectResult><ETag>"x"</ETag></CopyObjectResult>',
+ {status: 200, headers: {'Content-Type': 'application/xml'}},
+ );
+ }),
+ );
+ return objects;
+}
+
+function storage(endpoint: string, forcePathStyle: boolean): StorageService {
+ return new StorageService({
+ endpoint,
+ forcePathStyle,
+ region: 'us-east-1',
+ accessKeyId: 'TEST',
+ secretAccessKey: 'TEST',
+ });
+}
+
+const PDF = new TextEncoder().encode('%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n%%EOF\n');
+
+describe('StorageService with every bucket inside one real bucket', () => {
+ let objects: ReturnType<typeof singleBucketStore>;
+
+ beforeEach(() => {
+ objects = singleBucketStore();
+ });
+
+ it('copies a non-media attachment when the endpoint names the bucket with path-style addressing', async () => {
+ const service = storage(`https://${HOST}/${REAL_BUCKET}`, true);
+ await service.uploadObject({bucket: 'fluxer-uploads', key: 'upload 1', body: PDF});
+
+ await expect(
+ service.copyObjectWithMetadataStripping({
+ sourceBucket: 'fluxer-uploads',
+ sourceKey: 'upload 1',
+ destinationBucket: 'fluxer-cdn',
+ destinationKey: 'attachments/1/2/file.pdf',
+ contentType: 'application/pdf',
+ }),
+ ).resolves.toBeNull();
+
+ expect(objects.get('fluxer-cdn/attachments/1/2/file.pdf')?.contentType).toBe('application/pdf');
+ await expect(service.readObject('fluxer-cdn', 'attachments/1/2/file.pdf')).resolves.toEqual(PDF);
+ });
+
+ it('copies within a bucket whose virtual host keeps the endpoint path as a key prefix', async () => {
+ const service = storage(`https://${HOST}/prefix`, false);
+ await service.uploadObject({bucket: REAL_BUCKET, key: 'upload-2', body: PDF});
+
+ await service.copyObject({
+ sourceBucket: REAL_BUCKET,
+ sourceKey: 'upload-2',
+ destinationBucket: REAL_BUCKET,
+ destinationKey: 'attachments/1/3/file.pdf',
+ newContentType: 'application/pdf',
+ });
+
+ expect(objects.get('prefix/attachments/1/3/file.pdf')?.contentType).toBe('application/pdf');
+ await expect(service.readObject(REAL_BUCKET, 'attachments/1/3/file.pdf')).resolves.toEqual(PDF);
+ });
+});
diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdx
index dde6b8baf..cf414dab4 100644
--- a/fluxer_docs/src/content/docs/operator/configuration.mdx
+++ b/fluxer_docs/src/content/docs/operator/configuration.mdx
@@ -514,6 +514,8 @@ Every uploaded file lands in an S3-compatible object store, which the stack prov
Default `http://localhost:3900`. The S3 API address. `media-proxy` defaults to empty instead. Compose forwards it from `.env` with a default of `http://seaweedfs:8333`, the bundled store.
+An endpoint with a bucket in its path, such as `https://s3.example.com/my-bucket` with `FLUXER_S3_FORCE_PATH_STYLE=true`, keeps every Fluxer bucket inside that one bucket. Each bucket name becomes a key prefix there, so uploads land under `fluxer-uploads/` in `my-bucket`.
+
#### `FLUXER_S3_PUBLIC_ENDPOINT`
No default. The host substituted into presigned URLs, which are the only place it appears. Compose forwards it from `.env`, and uses `FLUXER_S3_ENDPOINT` when it is unset, except on `app-proxy`, which gets only a value `.env` sets and adds it to its Content Security Policy. `admin` reads it to find the origin that report downloads are presigned against. See `FLUXER_S3_BUCKET_REPORTS`.