A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(api): resolve s3 copy source under endpoint path (#3422)
Hampus8bda15159674
diff --git a/fluxer_api/src/api/infrastructure/StorageService.test.ts b/fluxer_api/src/api/infrastructure/StorageService.test.tsindex e4280f013..0d60e16f6 100644--- a/fluxer_api/src/api/infrastructure/StorageService.test.ts+++ b/fluxer_api/src/api/infrastructure/StorageService.test.ts@@ -225,3 +225,55 @@ describe('StorageService.listObjects', () => {);});});++function copySourceFor(endpoint: string, forcePathStyle: boolean): Promise<unknown> {+ const service = new StorageService({+ endpoint,+ forcePathStyle,+ region: 'us-east-1',+ accessKeyId: 'fluxer',+ secretAccessKey: 'fluxer-secret',+ });+ let copySource: unknown;+ Object.assign(service, {+ client: {+ send: async (command: {input: {CopySource?: unknown}}) => {+ copySource = command.input.CopySource;+ return {};+ },+ },+ });+ return service+ .copyObject({+ sourceBucket: 'fluxer-uploads',+ sourceKey: 'a b/c+d.pdf',+ destinationBucket: 'fluxer-cdn',+ destinationKey: 'attachments/1/2/c.pdf',+ })+ .then(() => copySource);+}++describe('StorageService.copyObject copy source', () => {+ it.each([+ ['http://seaweedfs:8333', true],+ ['http://seaweedfs:8333/', true],+ ['https://s3.example.test', false],+ ])(+ 'names only the bucket and key for an endpoint without a path (%s, path style %s)',+ async (endpoint, pathStyle) => {+ await expect(copySourceFor(endpoint, pathStyle)).resolves.toBe('fluxer-uploads/a%20b/c%2Bd.pdf');+ },+ );++ it('puts the endpoint path in front of the bucket with path-style addressing', async () => {+ await expect(copySourceFor('https://s3.example.test/real%20bucket/sub/', true)).resolves.toBe(+ 'real%20bucket/sub/fluxer-uploads/a%20b/c%2Bd.pdf',+ );+ });++ it('puts the endpoint path between the bucket and the key with virtual-hosted addressing', async () => {+ await expect(copySourceFor('https://s3.example.test/real%20bucket/sub', false)).resolves.toBe(+ 'fluxer-uploads/real%20bucket/sub/a%20b/c%2Bd.pdf',+ );+ });+});diff --git a/fluxer_api/src/api/infrastructure/StorageService.ts b/fluxer_api/src/api/infrastructure/StorageService.tsindex d1b3434d7..2c94cfd1f 100644--- a/fluxer_api/src/api/infrastructure/StorageService.ts+++ b/fluxer_api/src/api/infrastructure/StorageService.ts@@ -115,6 +115,13 @@ const REJECTED_SERVER_SIDE_COPY_ERRORS = new Set(['MethodNotAllowed',]);+function encodedEndpointPath(endpoint: string): Array<string> {+ return (URL.parse(endpoint)?.pathname ?? '')+ .split('/')+ .filter((segment) => segment !== '')+ .map((segment) => encodeURIComponent(decodeURIComponent(segment)));+}+function isRejectedServerSideCopy(error: unknown): boolean {return (error instanceof S3ServiceException &&@@ -126,6 +133,7 @@ export class StorageService implements IStorageService {private readonly client: S3Client;private readonly presignClient: S3Client;private readonly provider: S3ProviderSettings;+ private readonly endpointPath: Array<string>;constructor(provider?: S3ProviderSettings) {this.provider = provider ?? {@@ -136,6 +144,7 @@ export class StorageService implements IStorageService {accessKeyId: Config.s3.accessKeyId,secretAccessKey: Config.s3.secretAccessKey,};+ this.endpointPath = encodedEndpointPath(this.provider.endpoint);this.client = buildPooledS3Client({endpoint: this.provider.endpoint,region: this.provider.region,@@ -492,7 +501,7 @@ export class StorageService implements IStorageService {new CopyObjectCommand({Bucket: destinationBucket,Key: destinationKey,- CopySource: `${encodeURIComponent(sourceBucket)}/${sourceKey.split('/').map(encodeURIComponent).join('/')}`,+ CopySource: this.copySource(sourceBucket, sourceKey),ContentType: newContentType,MetadataDirective: newContentType ? 'REPLACE' : undefined,}),@@ -508,6 +517,14 @@ export class StorageService implements IStorageService {}}+ private copySource(bucket: string, key: string): string {+ const encodedBucket = encodeURIComponent(bucket);+ const location = this.provider.forcePathStyle+ ? [...this.endpointPath, encodedBucket]+ : [encodedBucket, ...this.endpointPath];+ return [...location, ...key.split('/').map(encodeURIComponent)].join('/');+ }+private async copyObjectThroughApi({sourceBucket,diff --git a/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts b/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.tsnew file mode 100644index 000000000..0749db71b--- /dev/null+++ b/fluxer_api/src/api/infrastructure/StorageServiceEndpointPath.test.ts@@ -0,0 +1,129 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {StorageService} from '@app/api/infrastructure/StorageService';+import {server} from '@app/api/test/msw/server';+import {HttpResponse, http} from 'msw';+import {beforeEach, describe, expect, it} from 'vitest';++const HOST = 's3.single-bucket.test';+const REAL_BUCKET = 'realbucket';++function errorResponse(code: string, status: number): HttpResponse<string> {+ return new HttpResponse(+ `<?xml version="1.0" encoding="UTF-8"?><Error><Code>${code}</Code><Message></Message><RequestId>tx0</RequestId></Error>`,+ {status, headers: {'Content-Type': 'application/xml'}},+ );+}++function splitBucket(path: string): {bucket: string; key: string} {+ const [bucket = '', ...key] = path.split('/');+ return {bucket, key: key.join('/')};+}++function singleBucketStore() {+ const objects = new Map<string, {body: Uint8Array; contentType: string}>();+ const resolve = (url: URL): {bucket: string; key: string} => {+ const path = decodeURIComponent(url.pathname.slice(1));+ if (url.hostname === HOST) {+ return splitBucket(path);+ }+ return {bucket: url.hostname.slice(0, -HOST.length - 1), key: path};+ };+ server.use(+ http.all(/^https:\/\/([a-z0-9-]+\.)?s3\.single-bucket\.test\//u, async ({request}) => {+ const target = resolve(new URL(request.url));+ if (target.bucket !== REAL_BUCKET) {+ return errorResponse('NoSuchBucket', 404);+ }+ if (request.method === 'GET') {+ const object = objects.get(target.key);+ if (!object) {+ return errorResponse('NoSuchKey', 404);+ }+ return new HttpResponse(object.body, {+ status: 200,+ headers: {'Content-Type': object.contentType, 'Content-Length': String(object.body.length)},+ });+ }+ if (request.method !== 'PUT') {+ return errorResponse('NotImplemented', 501);+ }+ const copySource = request.headers.get('x-amz-copy-source');+ if (!copySource) {+ objects.set(target.key, {+ body: new Uint8Array(await request.arrayBuffer()),+ contentType: request.headers.get('content-type') ?? 'application/octet-stream',+ });+ return new HttpResponse(null, {status: 200, headers: {ETag: '"x"'}});+ }+ const source = splitBucket(decodeURIComponent(copySource.replace(/^\//u, '')));+ if (source.bucket !== REAL_BUCKET) {+ return errorResponse('NoSuchBucket', 404);+ }+ const object = objects.get(source.key);+ if (!object) {+ return errorResponse('NoSuchKey', 404);+ }+ objects.set(target.key, {...object, contentType: request.headers.get('content-type') ?? object.contentType});+ return new HttpResponse(+ '<?xml version="1.0" encoding="UTF-8"?><CopyObjectResult><ETag>"x"</ETag></CopyObjectResult>',+ {status: 200, headers: {'Content-Type': 'application/xml'}},+ );+ }),+ );+ return objects;+}++function storage(endpoint: string, forcePathStyle: boolean): StorageService {+ return new StorageService({+ endpoint,+ forcePathStyle,+ region: 'us-east-1',+ accessKeyId: 'TEST',+ secretAccessKey: 'TEST',+ });+}++const PDF = new TextEncoder().encode('%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\n%%EOF\n');++describe('StorageService with every bucket inside one real bucket', () => {+ let objects: ReturnType<typeof singleBucketStore>;++ beforeEach(() => {+ objects = singleBucketStore();+ });++ it('copies a non-media attachment when the endpoint names the bucket with path-style addressing', async () => {+ const service = storage(`https://${HOST}/${REAL_BUCKET}`, true);+ await service.uploadObject({bucket: 'fluxer-uploads', key: 'upload 1', body: PDF});++ await expect(+ service.copyObjectWithMetadataStripping({+ sourceBucket: 'fluxer-uploads',+ sourceKey: 'upload 1',+ destinationBucket: 'fluxer-cdn',+ destinationKey: 'attachments/1/2/file.pdf',+ contentType: 'application/pdf',+ }),+ ).resolves.toBeNull();++ expect(objects.get('fluxer-cdn/attachments/1/2/file.pdf')?.contentType).toBe('application/pdf');+ await expect(service.readObject('fluxer-cdn', 'attachments/1/2/file.pdf')).resolves.toEqual(PDF);+ });++ it('copies within a bucket whose virtual host keeps the endpoint path as a key prefix', async () => {+ const service = storage(`https://${HOST}/prefix`, false);+ await service.uploadObject({bucket: REAL_BUCKET, key: 'upload-2', body: PDF});++ await service.copyObject({+ sourceBucket: REAL_BUCKET,+ sourceKey: 'upload-2',+ destinationBucket: REAL_BUCKET,+ destinationKey: 'attachments/1/3/file.pdf',+ newContentType: 'application/pdf',+ });++ expect(objects.get('prefix/attachments/1/3/file.pdf')?.contentType).toBe('application/pdf');+ await expect(service.readObject(REAL_BUCKET, 'attachments/1/3/file.pdf')).resolves.toEqual(PDF);+ });+});diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdxindex dde6b8baf..cf414dab4 100644--- a/fluxer_docs/src/content/docs/operator/configuration.mdx+++ b/fluxer_docs/src/content/docs/operator/configuration.mdx@@ -514,6 +514,8 @@ Every uploaded file lands in an S3-compatible object store, which the stack provDefault `http://localhost:3900`. The S3 API address. `media-proxy` defaults to empty instead. Compose forwards it from `.env` with a default of `http://seaweedfs:8333`, the bundled store.+An endpoint with a bucket in its path, such as `https://s3.example.com/my-bucket` with `FLUXER_S3_FORCE_PATH_STYLE=true`, keeps every Fluxer bucket inside that one bucket. Each bucket name becomes a key prefix there, so uploads land under `fluxer-uploads/` in `my-bucket`.+#### `FLUXER_S3_PUBLIC_ENDPOINT`No default. The host substituted into presigned URLs, which are the only place it appears. Compose forwards it from `.env`, and uses `FLUXER_S3_ENDPOINT` when it is unset, except on `app-proxy`, which gets only a value `.env` sets and adds it to its Content Security Policy. `admin` reads it to find the origin that report downloads are presigned against. See `FLUXER_S3_BUCKET_REPORTS`.