Skip to content

Self-hosted personal gallery application with online photo management and albums, supporting Live/Motion Photos, EXIF parsing, geolocation recognition, and an explore map.

fix(storage): decode and paginate multipart upload listings

Timothy Yin487a884b2bf9
diff --git a/packages/storage/src/s3-edge.ts b/packages/storage/src/s3-edge.ts
index c85eabbf..6dabdbe9 100644
--- a/packages/storage/src/s3-edge.ts
+++ b/packages/storage/src/s3-edge.ts
@@ -289,18 +289,47 @@ export function createCloudS3(
)
},
async recoverMultipart(key: string) {
- const result = await client.send(
- new ListMultipartUploadsCommand({
- Bucket,
- Prefix: safeKey(key),
- MaxUploads: 100,
- }),
- )
- if (result.IsTruncated)
- throw new Error('Too many multipart sessions for one object')
- return (result.Uploads ?? [])
- .filter((upload) => upload.Key === key && upload.UploadId)
- .map((upload) => upload.UploadId!)
+ const prefix = safeKey(key),
+ ids = new Set<string>(),
+ cursors = new Set<string>()
+ let keyMarker: string | undefined, uploadIdMarker: string | undefined
+ // Bound provider pagination, and never return a partial list for cleanup.
+ for (let page = 0; page < 20; page++) {
+ const result = await client.send(
+ new ListMultipartUploadsCommand({
+ Bucket,
+ Prefix: prefix,
+ MaxUploads: 100,
+ EncodingType: 'url',
+ KeyMarker: keyMarker,
+ UploadIdMarker: uploadIdMarker,
+ }),
+ )
+ // The SDK leaves S3 EncodingType=url values encoded. Decode exactly once;
+ // decoding unlabelled keys would confuse literal percent-escaped names.
+ const decodeKey = (value: string) =>
+ result.EncodingType === 'url' ? decodeURIComponent(value) : value
+ for (const upload of result.Uploads ?? []) {
+ if (
+ upload.Key !== undefined &&
+ decodeKey(upload.Key) === key &&
+ upload.UploadId
+ )
+ ids.add(upload.UploadId)
+ }
+ if (ids.size > 100)
+ throw new Error('Too many multipart sessions for one object')
+ if (!result.IsTruncated) return [...ids]
+ if (result.NextKeyMarker === undefined)
+ throw new Error('Missing multipart pagination marker')
+ keyMarker = decodeKey(result.NextKeyMarker)
+ uploadIdMarker = result.NextUploadIdMarker
+ const cursor = JSON.stringify([keyMarker, uploadIdMarker])
+ if (cursors.has(cursor))
+ throw new Error('Repeated multipart pagination marker')
+ cursors.add(cursor)
+ }
+ throw new Error('Multipart pagination limit exceeded')
},
async multipart(key: string, contentType: string) {
const r = await client.send(
diff --git a/tests/cloud-storage-probe.test.ts b/tests/cloud-storage-probe.test.ts
index 62116238..98869380 100644
--- a/tests/cloud-storage-probe.test.ts
+++ b/tests/cloud-storage-probe.test.ts
@@ -258,3 +258,80 @@ test('Multipart diagnostics identify each operation and redact arbitrary provide
)
assert(absent.calls.includes('abort'))
})
+
+test('Multipart recovery matches URL-encoded keys and follows exact pagination markers', async () => {
+ const key = 'chronoframe/connection-check/a%2Fb space+测试'
+ const requests: URL[] = []
+ const xml = (s: string) =>
+ new Response(s, { headers: { 'content-type': 'application/xml' } })
+ const client = createCloudS3(config, {
+ fetch: async (input) => {
+ const url = new URL(input instanceof Request ? input.url : String(input))
+ requests.push(url)
+ assert.equal(url.searchParams.get('prefix'), key)
+ assert.equal(url.searchParams.get('encoding-type'), 'url')
+ if (requests.length === 1)
+ return xml(
+ `<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><EncodingType>url</EncodingType><IsTruncated>true</IsTruncated><NextKeyMarker>${encodeURIComponent(key)}</NextKeyMarker><NextUploadIdMarker>cursor-id</NextUploadIdMarker><Upload><Key>${encodeURIComponent(key + '-other')}</Key><UploadId>foreign</UploadId></Upload></ListMultipartUploadsResult>`,
+ )
+ assert.equal(url.searchParams.get('key-marker'), key)
+ assert.equal(url.searchParams.get('upload-id-marker'), 'cursor-id')
+ return xml(
+ `<ListMultipartUploadsResult><EncodingType>url</EncodingType><IsTruncated>false</IsTruncated><Upload><Key>${encodeURIComponent(key)}</Key><UploadId>expected</UploadId></Upload><Upload><Key>${encodeURIComponent(key + '-other')}</Key><UploadId>foreign</UploadId></Upload></ListMultipartUploadsResult>`,
+ )
+ },
+ })
+ assert.deepEqual(await client.recoverMultipart(key), ['expected'])
+ assert.equal(requests.length, 2)
+})
+
+test('Multipart recovery preserves literal percent keys and rejects incomplete or looping listings', async () => {
+ const key = 'test/literal%2Fname'
+ const make = (body: string) =>
+ createCloudS3(config, {
+ fetch: async () =>
+ new Response(body, { headers: { 'content-type': 'application/xml' } }),
+ })
+ assert.deepEqual(
+ await make(
+ `<ListMultipartUploadsResult><Upload><Key>${key}</Key><UploadId>literal</UploadId></Upload></ListMultipartUploadsResult>`,
+ ).recoverMultipart(key),
+ ['literal'],
+ )
+ assert.deepEqual(
+ await make(
+ '<ListMultipartUploadsResult><EncodingType>url</EncodingType><Upload><Key>test%2Fliteral%252Fname</Key><UploadId>encoded</UploadId></Upload></ListMultipartUploadsResult>',
+ ).recoverMultipart(key),
+ ['encoded'],
+ )
+ await assert.rejects(
+ make(
+ '<ListMultipartUploadsResult><EncodingType>url</EncodingType><Upload><Key>test%ZZ</Key><UploadId>bad</UploadId></Upload></ListMultipartUploadsResult>',
+ ).recoverMultipart(key),
+ )
+ await assert.rejects(
+ make(
+ '<ListMultipartUploadsResult><IsTruncated>true</IsTruncated></ListMultipartUploadsResult>',
+ ).recoverMultipart(key),
+ /Missing multipart pagination marker/,
+ )
+ await assert.rejects(
+ make(
+ '<ListMultipartUploadsResult><IsTruncated>true</IsTruncated><NextKeyMarker>same</NextKeyMarker><NextUploadIdMarker>same</NextUploadIdMarker></ListMultipartUploadsResult>',
+ ).recoverMultipart(key),
+ /Repeated multipart pagination marker/,
+ )
+ let calls = 0
+ const many = createCloudS3(config, {
+ fetch: async () =>
+ new Response(
+ `<ListMultipartUploadsResult><IsTruncated>true</IsTruncated><NextKeyMarker>key-${++calls}</NextKeyMarker></ListMultipartUploadsResult>`,
+ { headers: { 'content-type': 'application/xml' } },
+ ),
+ })
+ await assert.rejects(
+ many.recoverMultipart(key),
+ /Multipart pagination limit exceeded/,
+ )
+ assert.equal(calls, 20)
+})
diff --git a/tests/s3-worker-runtime.test.ts b/tests/s3-worker-runtime.test.ts
index 6a8f2302..3684581b 100644
--- a/tests/s3-worker-runtime.test.ts
+++ b/tests/s3-worker-runtime.test.ts
@@ -38,10 +38,18 @@ test('S3 browser distribution parses multipart/CORS XML in real workerd without
)
if (request.method === 'DELETE')
return new Response(null, { status: 204 })
- if (u.searchParams.has('uploads'))
+ if (u.searchParams.has('uploads')) {
+ assert.equal(u.searchParams.get('encoding-type'), 'url')
+ if (!u.searchParams.has('key-marker'))
+ return xml(
+ '<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><EncodingType>url</EncodingType><IsTruncated>true</IsTruncated><NextKeyMarker>test%2Ffile</NextKeyMarker><NextUploadIdMarker>marker</NextUploadIdMarker><Upload><Key>test%2Ffile-other</Key><UploadId>foreign-upload</UploadId></Upload></ListMultipartUploadsResult>',
+ )
+ assert.equal(u.searchParams.get('key-marker'), 'test/file')
+ assert.equal(u.searchParams.get('upload-id-marker'), 'marker')
return xml(
- '<ListMultipartUploadsResult><Upload><Key>test/file</Key><UploadId>test-upload</UploadId></Upload></ListMultipartUploadsResult>',
+ '<ListMultipartUploadsResult><EncodingType>url</EncodingType><IsTruncated>false</IsTruncated><Upload><Key>test%2Ffile</Key><UploadId>test-upload</UploadId></Upload></ListMultipartUploadsResult>',
)
+ }
if (u.searchParams.has('uploadId'))
return xml(
'<ListPartsResult><Part><PartNumber>1</PartNumber><ETag>test-etag</ETag><Size>32</Size></Part></ListPartsResult>',