Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.
TypeScript albumsexif-extractiongeocodinglive-photomotion-photophoto-galleryphoto-sharingwebgl-viewer
fix(storage): decode and paginate multipart upload listings
Timothy Yin487a884b2bf9
diff --git a/packages/storage/src/s3-edge.ts b/packages/storage/src/s3-edge.tsindex c85eabbf..6dabdbe9 100644--- a/packages/storage/src/s3-edge.ts+++ b/packages/storage/src/s3-edge.ts@@ -289,18 +289,47 @@ export function createCloudS3()},async recoverMultipart(key: string) {- const result = await client.send(- new ListMultipartUploadsCommand({- Bucket,- Prefix: safeKey(key),- MaxUploads: 100,- }),- )- if (result.IsTruncated)- throw new Error('Too many multipart sessions for one object')- return (result.Uploads ?? [])- .filter((upload) => upload.Key === key && upload.UploadId)- .map((upload) => upload.UploadId!)+ const prefix = safeKey(key),+ ids = new Set<string>(),+ cursors = new Set<string>()+ let keyMarker: string | undefined, uploadIdMarker: string | undefined+ // Bound provider pagination, and never return a partial list for cleanup.+ for (let page = 0; page < 20; page++) {+ const result = await client.send(+ new ListMultipartUploadsCommand({+ Bucket,+ Prefix: prefix,+ MaxUploads: 100,+ EncodingType: 'url',+ KeyMarker: keyMarker,+ UploadIdMarker: uploadIdMarker,+ }),+ )+ // The SDK leaves S3 EncodingType=url values encoded. Decode exactly once;+ // decoding unlabelled keys would confuse literal percent-escaped names.+ const decodeKey = (value: string) =>+ result.EncodingType === 'url' ? decodeURIComponent(value) : value+ for (const upload of result.Uploads ?? []) {+ if (+ upload.Key !== undefined &&+ decodeKey(upload.Key) === key &&+ upload.UploadId+ )+ ids.add(upload.UploadId)+ }+ if (ids.size > 100)+ throw new Error('Too many multipart sessions for one object')+ if (!result.IsTruncated) return [...ids]+ if (result.NextKeyMarker === undefined)+ throw new Error('Missing multipart pagination marker')+ keyMarker = decodeKey(result.NextKeyMarker)+ uploadIdMarker = result.NextUploadIdMarker+ const cursor = JSON.stringify([keyMarker, uploadIdMarker])+ if (cursors.has(cursor))+ throw new Error('Repeated multipart pagination marker')+ cursors.add(cursor)+ }+ throw new Error('Multipart pagination limit exceeded')},async multipart(key: string, contentType: string) {const r = await client.send(diff --git a/tests/cloud-storage-probe.test.ts b/tests/cloud-storage-probe.test.tsindex 62116238..98869380 100644--- a/tests/cloud-storage-probe.test.ts+++ b/tests/cloud-storage-probe.test.ts@@ -258,3 +258,80 @@ test('Multipart diagnostics identify each operation and redact arbitrary provide)assert(absent.calls.includes('abort'))})++test('Multipart recovery matches URL-encoded keys and follows exact pagination markers', async () => {+ const key = 'chronoframe/connection-check/a%2Fb space+测试'+ const requests: URL[] = []+ const xml = (s: string) =>+ new Response(s, { headers: { 'content-type': 'application/xml' } })+ const client = createCloudS3(config, {+ fetch: async (input) => {+ const url = new URL(input instanceof Request ? input.url : String(input))+ requests.push(url)+ assert.equal(url.searchParams.get('prefix'), key)+ assert.equal(url.searchParams.get('encoding-type'), 'url')+ if (requests.length === 1)+ return xml(+ `<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><EncodingType>url</EncodingType><IsTruncated>true</IsTruncated><NextKeyMarker>${encodeURIComponent(key)}</NextKeyMarker><NextUploadIdMarker>cursor-id</NextUploadIdMarker><Upload><Key>${encodeURIComponent(key + '-other')}</Key><UploadId>foreign</UploadId></Upload></ListMultipartUploadsResult>`,+ )+ assert.equal(url.searchParams.get('key-marker'), key)+ assert.equal(url.searchParams.get('upload-id-marker'), 'cursor-id')+ return xml(+ `<ListMultipartUploadsResult><EncodingType>url</EncodingType><IsTruncated>false</IsTruncated><Upload><Key>${encodeURIComponent(key)}</Key><UploadId>expected</UploadId></Upload><Upload><Key>${encodeURIComponent(key + '-other')}</Key><UploadId>foreign</UploadId></Upload></ListMultipartUploadsResult>`,+ )+ },+ })+ assert.deepEqual(await client.recoverMultipart(key), ['expected'])+ assert.equal(requests.length, 2)+})++test('Multipart recovery preserves literal percent keys and rejects incomplete or looping listings', async () => {+ const key = 'test/literal%2Fname'+ const make = (body: string) =>+ createCloudS3(config, {+ fetch: async () =>+ new Response(body, { headers: { 'content-type': 'application/xml' } }),+ })+ assert.deepEqual(+ await make(+ `<ListMultipartUploadsResult><Upload><Key>${key}</Key><UploadId>literal</UploadId></Upload></ListMultipartUploadsResult>`,+ ).recoverMultipart(key),+ ['literal'],+ )+ assert.deepEqual(+ await make(+ '<ListMultipartUploadsResult><EncodingType>url</EncodingType><Upload><Key>test%2Fliteral%252Fname</Key><UploadId>encoded</UploadId></Upload></ListMultipartUploadsResult>',+ ).recoverMultipart(key),+ ['encoded'],+ )+ await assert.rejects(+ make(+ '<ListMultipartUploadsResult><EncodingType>url</EncodingType><Upload><Key>test%ZZ</Key><UploadId>bad</UploadId></Upload></ListMultipartUploadsResult>',+ ).recoverMultipart(key),+ )+ await assert.rejects(+ make(+ '<ListMultipartUploadsResult><IsTruncated>true</IsTruncated></ListMultipartUploadsResult>',+ ).recoverMultipart(key),+ /Missing multipart pagination marker/,+ )+ await assert.rejects(+ make(+ '<ListMultipartUploadsResult><IsTruncated>true</IsTruncated><NextKeyMarker>same</NextKeyMarker><NextUploadIdMarker>same</NextUploadIdMarker></ListMultipartUploadsResult>',+ ).recoverMultipart(key),+ /Repeated multipart pagination marker/,+ )+ let calls = 0+ const many = createCloudS3(config, {+ fetch: async () =>+ new Response(+ `<ListMultipartUploadsResult><IsTruncated>true</IsTruncated><NextKeyMarker>key-${++calls}</NextKeyMarker></ListMultipartUploadsResult>`,+ { headers: { 'content-type': 'application/xml' } },+ ),+ })+ await assert.rejects(+ many.recoverMultipart(key),+ /Multipart pagination limit exceeded/,+ )+ assert.equal(calls, 20)+})diff --git a/tests/s3-worker-runtime.test.ts b/tests/s3-worker-runtime.test.tsindex 6a8f2302..3684581b 100644--- a/tests/s3-worker-runtime.test.ts+++ b/tests/s3-worker-runtime.test.ts@@ -38,10 +38,18 @@ test('S3 browser distribution parses multipart/CORS XML in real workerd without)if (request.method === 'DELETE')return new Response(null, { status: 204 })- if (u.searchParams.has('uploads'))+ if (u.searchParams.has('uploads')) {+ assert.equal(u.searchParams.get('encoding-type'), 'url')+ if (!u.searchParams.has('key-marker'))+ return xml(+ '<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><EncodingType>url</EncodingType><IsTruncated>true</IsTruncated><NextKeyMarker>test%2Ffile</NextKeyMarker><NextUploadIdMarker>marker</NextUploadIdMarker><Upload><Key>test%2Ffile-other</Key><UploadId>foreign-upload</UploadId></Upload></ListMultipartUploadsResult>',+ )+ assert.equal(u.searchParams.get('key-marker'), 'test/file')+ assert.equal(u.searchParams.get('upload-id-marker'), 'marker')return xml(- '<ListMultipartUploadsResult><Upload><Key>test/file</Key><UploadId>test-upload</UploadId></Upload></ListMultipartUploadsResult>',+ '<ListMultipartUploadsResult><EncodingType>url</EncodingType><IsTruncated>false</IsTruncated><Upload><Key>test%2Ffile</Key><UploadId>test-upload</UploadId></Upload></ListMultipartUploadsResult>',)+ }if (u.searchParams.has('uploadId'))return xml('<ListPartsResult><Part><PartNumber>1</PartNumber><ETag>test-etag</ETag><Size>32</Size></Part></ListPartsResult>',