Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): accept a percent-encoded bucket/key separator in copy sources (#2060)
TiMa8ed218c5790
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 64fb3e6c30..f73e094a0a 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -1926,15 +1926,8 @@ public class S3Controller {private Response handleCopyObject(String copySource, String destBucket, String destKey,String contentType, HttpHeaders httpHeaders) {- // copySource format: /bucket/key or bucket/key, where key is URL-encoded- String source = copySource.startsWith("/") ? copySource.substring(1) : copySource;-- int slashIndex = source.indexOf('/');- if (slashIndex <= 0) {- throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400);- }- String sourceBucket = decodeCopySourceComponent(source.substring(0, slashIndex), copySource);- ParsedCopySource sourceObject = parseCopySourceObject(source.substring(slashIndex + 1), copySource);+ CopySourceRef sourceObject = parseCopySource(copySource);+ String sourceBucket = sourceObject.bucket();String copyContentEncoding = toPersistedContentEncoding(httpHeaders.getHeaderString("Content-Encoding"));String copyContentDisposition = httpHeaders.getHeaderString("Content-Disposition");String copyCacheControl = httpHeaders.getHeaderString("Cache-Control");@@ -1985,15 +1978,8 @@ public class S3Controller {private Response handleUploadPartCopy(String copySource, String destBucket, String destKey,String uploadId, int partNumber, HttpHeaders httpHeaders) {- // copySource format: /bucket/key or bucket/key, where key is URL-encoded- String source = copySource.startsWith("/") ? copySource.substring(1) : copySource;-- int slashIndex = source.indexOf('/');- if (slashIndex <= 0) {- throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400);- }- String sourceBucket = decodeCopySourceComponent(source.substring(0, slashIndex), copySource);- ParsedCopySource sourceObject = parseCopySourceObject(source.substring(slashIndex + 1), copySource);+ CopySourceRef sourceObject = parseCopySource(copySource);+ String sourceBucket = sourceObject.bucket();String copySourceRange = httpHeaders.getHeaderString("x-amz-copy-source-range");String eTag = s3Service.uploadPartCopy(destBucket, destKey, uploadId, partNumber,sourceBucket, sourceObject.objectKey(), sourceObject.versionId(), copySourceRange,@@ -2879,6 +2865,72 @@ public class S3Controller {return Response.ok().build();}+ /**+ * Splits a raw {@code x-amz-copy-source} header into decoded source bucket, object key and optional+ * {@code versionId}. Shared by {@code CopyObject} and {@code UploadPartCopy}.+ *+ * <p>The bucket/key separator may arrive as a literal {@code '/'} or percent-encoded as {@code %2F}:+ * the AWS SDK for .NET encodes the whole copy source, so a v4 client sends+ * {@code bucket%2Ffolder%2Fkey.txt} with no literal slash at all. Both forms name the same object, so+ * whichever separator appears first delimits the bucket. Bucket names admit neither {@code '/'} nor+ * {@code '%'}, so the first occurrence of either is unambiguously the separator and never part of the+ * bucket name.+ *+ * <p>The split still happens before decoding, so an encoded {@code %2F} inside the key stays key+ * content instead of being promoted to a path separator. {@code parseCopySourceObject} documents how+ * the remainder is split into key and {@code versionId}.+ *+ * @param copySource raw header value, with or without a leading separator+ * @return decoded bucket, key and {@code versionId} ({@code null} version when absent)+ * @throws AwsException {@code InvalidArgument} when no separator follows a non-empty bucket, or when a+ * component is not valid percent-encoding+ */+ private CopySourceRef parseCopySource(String copySource) {+ String source = stripLeadingSeparator(copySource);+ int separator = indexOfBucketSeparator(source);+ if (separator <= 0) {+ throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400);+ }+ int keyStart = separator + (source.charAt(separator) == '/' ? 1 : 3);+ String bucket = decodeCopySourceComponent(source.substring(0, separator), copySource);+ ParsedCopySource object = parseCopySourceObject(source.substring(keyStart), copySource);+ return new CopySourceRef(bucket, object.objectKey(), object.versionId());+ }++ /** Drops the optional leading separator, in either its literal or percent-encoded form. */+ private static String stripLeadingSeparator(String copySource) {+ if (copySource.startsWith("/")) {+ return copySource.substring(1);+ }+ if (indexOfEncodedSlash(copySource) == 0) {+ return copySource.substring(3);+ }+ return copySource;+ }++ /** Index of the bucket/key separator: the first literal {@code '/'} or {@code %2F}, or -1 if neither. */+ private static int indexOfBucketSeparator(String source) {+ int literal = source.indexOf('/');+ int encoded = indexOfEncodedSlash(source);+ if (literal < 0) {+ return encoded;+ }+ if (encoded < 0) {+ return literal;+ }+ return Math.min(literal, encoded);+ }++ /** Index of the first {@code %2F} or {@code %2f} sequence, or -1 when there is none. */+ private static int indexOfEncodedSlash(String source) {+ for (int i = source.indexOf('%'); i >= 0 && i + 2 < source.length(); i = source.indexOf('%', i + 1)) {+ if (source.charAt(i + 1) == '2' && (source.charAt(i + 2) == 'F' || source.charAt(i + 2) == 'f')) {+ return i;+ }+ }+ return -1;+ }+/*** Splits the raw {@code CopyObject}/{@code UploadPartCopy} copy-source remainder into decoded S3 object* key and optional source {@code versionId}.@@ -2944,4 +2996,7 @@ public class S3Controller {private record ParsedCopySource(String objectKey, String versionId) {}++ private record CopySourceRef(String bucket, String objectKey, String versionId) {+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.javaindex 9e931dc757..fefe54c97b 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java@@ -479,6 +479,64 @@ class S3IntegrationTest {given().delete("/" + bucket);}+ @Test+ @Order(119)+ void copyObjectWithPercentEncodedBucketSeparatorSucceeds() {+ // The AWS SDK for .NET percent-encodes the whole copy source, so a v4 client sends+ // "bucket%2Ffolder%2Fkey.txt" with no literal slash anywhere in the header.+ String bucket = "copy-encoded-separator-bucket";+ String srcKey = "folder/file.txt";++ given().put("/" + bucket).then().statusCode(200);+ given()+ .contentType("text/plain")+ .body("encoded separator")+ .when()+ .put("/" + bucket + "/" + srcKey)+ .then()+ .statusCode(200);++ // Fully encoded, no leading separator: the .NET wire format.+ given()+ .header("x-amz-copy-source", bucket + "%2Ffolder%2Ffile.txt")+ .when()+ .put("/" + bucket + "/copied/dotnet.txt")+ .then()+ .statusCode(200)+ .body(containsString("CopyObjectResult"));++ // Lowercase %2f, and a leading encoded separator, name the same object.+ given()+ .header("x-amz-copy-source", "%2F" + bucket + "%2ffolder%2ffile.txt")+ .when()+ .put("/" + bucket + "/copied/lowercase.txt")+ .then()+ .statusCode(200)+ .body(containsString("CopyObjectResult"));++ // Mixed: encoded bucket separator, literal slash inside the key.+ given()+ .header("x-amz-copy-source", bucket + "%2Ffolder/file.txt")+ .when()+ .put("/" + bucket + "/copied/mixed.txt")+ .then()+ .statusCode(200)+ .body(containsString("CopyObjectResult"));++ for (String copied : new String[] {"dotnet", "lowercase", "mixed"}) {+ given()+ .when()+ .get("/" + bucket + "/copied/" + copied + ".txt")+ .then()+ .statusCode(200)+ .body(equalTo("encoded separator"));+ given().delete("/" + bucket + "/copied/" + copied + ".txt");+ }++ given().delete("/" + bucket + "/" + srcKey);+ given().delete("/" + bucket);+ }+@Test@Order(120)void copyObjectWithQuestionMarkInSourceKeySucceeds() {@@ -550,6 +608,15 @@ class S3IntegrationTest {.then().statusCode(400).body(containsString("InvalidArgument"));++ // Decoding still rejects a malformed key reached through an encoded separator.+ given()+ .header("x-amz-copy-source", "test-bucket%2F%ZZinvalid")+ .when()+ .put("/test-bucket/dest-key")+ .then()+ .statusCode(400)+ .body(containsString("InvalidArgument"));}@Test@@ -562,6 +629,23 @@ class S3IntegrationTest {.then().statusCode(400).body(containsString("InvalidArgument"));++ // A source with no separator at all names no bucket, encoded form included.+ given()+ .header("x-amz-copy-source", "key-only-no-bucket")+ .when()+ .put("/test-bucket/dest-key")+ .then()+ .statusCode(400)+ .body(containsString("InvalidArgument"));++ given()+ .header("x-amz-copy-source", "%2Fkey-only-no-bucket")+ .when()+ .put("/test-bucket/dest-key")+ .then()+ .statusCode(400)+ .body(containsString("InvalidArgument"));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.javaindex 7c9bd55e34..eee89b3910 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java@@ -247,11 +247,24 @@ class S3MultipartIntegrationTest {.body(containsString("<CopyPartResult")).body(containsString("<ETag>"));+ // Percent-encoded bucket/key separator: the AWS SDK for .NET encodes the whole+ // copy source, so the header carries no literal slash.+ given()+ .header("x-amz-copy-source", BUCKET + "%2Fsource-for-copy.bin")+ .header("x-amz-copy-source-range", "bytes=2-5")+ .when()+ .put("/" + BUCKET + "/copy-dest.bin?uploadId=" + copyUploadId + "&partNumber=3")+ .then()+ .statusCode(200)+ .body(containsString("<CopyPartResult"))+ .body(containsString("<ETag>"));+// Complete the uploadString completeXml = """<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>etag1</ETag></Part><Part><PartNumber>2</PartNumber><ETag>etag2</ETag></Part>+ <Part><PartNumber>3</PartNumber><ETag>etag3</ETag></Part></CompleteMultipartUpload>""";given().contentType("application/xml")@@ -261,13 +274,15 @@ class S3MultipartIntegrationTest {.then().statusCode(200);- // Verify contents: full source + ranged slice+ // Verify contents: full source, ranged slice, then the same slice copied+ // through the percent-encoded separator. The last four bytes prove the+ // encoded source resolved to the same object, not just that it returned 200.given().when().get("/" + BUCKET + "/copy-dest.bin").then().statusCode(200)- .body(equalTo("ABCDEFGHIJCDEF"));+ .body(equalTo("ABCDEFGHIJCDEFCDEF"));}@Test