Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): parse copy-source versionId for CopyObject and part copy iss… (#751)
Asha Somayajulace2012de45b7
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 8c94c22749..55c734a8c6 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -1511,14 +1511,15 @@ public class S3Controller {throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400);}String sourceBucket = decodedSource.substring(0, slashIndex);- String sourceKey = decodedSource.substring(slashIndex + 1);-+ String pathAfterBucket = decodedSource.substring(slashIndex + 1);+ ParsedCopySource sourceObject = parseCopySourceObject(pathAfterBucket);String copyContentEncoding = toPersistedContentEncoding(httpHeaders.getHeaderString("Content-Encoding"));String copyContentDisposition = httpHeaders.getHeaderString("Content-Disposition");String copyCacheControl = httpHeaders.getHeaderString("Cache-Control");String copyServerSideEncryption = httpHeaders.getHeaderString("x-amz-server-side-encryption");String cannedAcl = httpHeaders.getHeaderString("x-amz-acl");- S3Object copy = s3Service.copyObject(sourceBucket, sourceKey, destBucket, destKey,+ S3Object copy = s3Service.copyObject(sourceBucket, sourceObject.objectKey(), destBucket, destKey,+ sourceObject.versionId(),new CopyObjectOptions().withMetadataDirective(httpHeaders.getHeaderString("x-amz-metadata-directive")).withReplacementMetadata(extractUserMetadata(httpHeaders))@@ -1556,10 +1557,11 @@ public class S3Controller {throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400);}String sourceBucket = decodedSource.substring(0, slashIndex);- String sourceKey = decodedSource.substring(slashIndex + 1);+ String pathAfterBucket = decodedSource.substring(slashIndex + 1);+ ParsedCopySource sourceObject = parseCopySourceObject(pathAfterBucket);String copySourceRange = httpHeaders.getHeaderString("x-amz-copy-source-range");String eTag = s3Service.uploadPartCopy(destBucket, destKey, uploadId, partNumber,- sourceBucket, sourceKey, copySourceRange);+ sourceBucket, sourceObject.objectKey(), sourceObject.versionId(), copySourceRange);String xml = new XmlBuilder().raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").start("CopyPartResult", AwsNamespaces.S3)@@ -2209,4 +2211,46 @@ public class S3Controller {s3Service.putBucketWebsite(bucket, new WebsiteConfiguration(indexDoc, errorDoc));return Response.ok().build();}++ /**+ * Splits the {@code CopyObject}/{@code UploadPartCopy} copy-source remainder into S3 object key and+ * optional source {@code versionId}.+ * <ul>+ * <li><b>Input:</b> decoded {@code x-amz-copy-source} with bucket already removed (substring after+ * the {@code '/'} that follows the bucket). Both {@code handleCopyObject} and+ * {@code handleUploadPartCopy} compute this as {@code pathAfterBucket}.</li>+ * <li><b>Key:</b> substring before the first {@code '?'} if any; keys may contain more {@code '/'}+ * segments.</li>+ * <li><b>{@code versionId}:</b> first {@code versionId} query pair, when present (raw value after+ * {@code '='}). Other query pairs are ignored.</li>+ * </ul>+ *+ * @param pathAfterBucket object key alone, or key with query (for example {@code dir/k.txt?versionId=uuid})+ * @return key without trailing query plus {@code versionId} value, or {@code null} version when absent+ */+ private ParsedCopySource parseCopySourceObject(String pathAfterBucket) {+ int queryStart = pathAfterBucket.indexOf('?');+ if (queryStart < 0) {+ return new ParsedCopySource(pathAfterBucket, null);+ }+ String objectKey = pathAfterBucket.substring(0, queryStart);+ String query = pathAfterBucket.substring(queryStart + 1);+ String versionId = null;+ for (String pair : query.split("&")) {+ int eq = pair.indexOf('=');+ if (eq <= 0) {+ continue;+ }+ String name = pair.substring(0, eq);+ String value = pair.substring(eq + 1);+ if ("versionId".equals(name)) {+ versionId = value;+ break;+ }+ }+ return new ParsedCopySource(objectKey, versionId);+ }++ private record ParsedCopySource(String objectKey, String versionId) {+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 64eab0060b..93034b96bb 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -636,6 +636,11 @@ public class S3Service {return copyObject(sourceBucket, sourceKey, destBucket, destKey, new CopyObjectOptions());}+ public S3Object copyObject(String sourceBucket, String sourceKey,+ String destBucket, String destKey, String versionId) {+ return copyObject(sourceBucket, sourceKey, destBucket, destKey, null, new CopyObjectOptions());+ }+public S3Object copyObject(String sourceBucket, String sourceKey,String destBucket, String destKey,String metadataDirective, Map<String, String> replacementMetadata,@@ -666,50 +671,35 @@ public class S3Service {.withAcl(acl));}+ public S3Object copyObject(String sourceBucket, String sourceKey,+ String destBucket, String destKey, String versionId,+ String metadataDirective, Map<String, String> replacementMetadata,+ String storageClass, String contentType, String contentEncoding,+ String contentDisposition, String cacheControl, String serverSideEncryption, String acl) {+ return copyObject(sourceBucket, sourceKey, destBucket, destKey, versionId,+ new CopyObjectOptions()+ .withMetadataDirective(metadataDirective)+ .withReplacementMetadata(replacementMetadata)+ .withStorageClass(storageClass)+ .withContentType(contentType)+ .withContentEncoding(contentEncoding)+ .withContentDisposition(contentDisposition)+ .withCacheControl(cacheControl)+ .withServerSideEncryption(serverSideEncryption)+ .withAcl(acl));+ }+ public S3Object copyObject(String sourceBucket, String sourceKey,+ String destBucket, String destKey, String versionId, CopyObjectOptions options)+ {+ S3Object source = getObject(sourceBucket, sourceKey, versionId);+ return copyS3Object(sourceBucket, sourceKey,+ destBucket, destKey, source, options);+ }+public S3Object copyObject(String sourceBucket, String sourceKey,String destBucket, String destKey, CopyObjectOptions options) {S3Object source = getObject(sourceBucket, sourceKey);- ensureBucketExists(destBucket);- CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();- String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());-- boolean replaceMetadata = "REPLACE".equalsIgnoreCase(effectiveOptions.getMetadataDirective());- Map<String, String> metadata = replaceMetadata ? new LinkedHashMap<>() : new LinkedHashMap<>(source.getMetadata());- if (replaceMetadata && effectiveOptions.getReplacementMetadata() != null) {- metadata.putAll(effectiveOptions.getReplacementMetadata());- }-- String effectiveContentType = replaceMetadata && effectiveOptions.getContentType() != null- ? effectiveOptions.getContentType()- : source.getContentType();- String effectiveStorageClass = effectiveOptions.getStorageClass() != null- ? effectiveOptions.getStorageClass()- : source.getStorageClass();- String effectiveContentEncoding = replaceMetadata && effectiveOptions.getContentEncoding() != null- ? effectiveOptions.getContentEncoding()- : source.getContentEncoding();- String effectiveContentDisposition = replaceMetadata && effectiveOptions.getContentDisposition() != null- ? effectiveOptions.getContentDisposition()- : source.getContentDisposition();- String effectiveCacheControl = replaceMetadata && effectiveOptions.getCacheControl() != null- ? effectiveOptions.getCacheControl()- : source.getCacheControl();- String effectiveServerSideEncryption = normalizedServerSideEncryption != null- ? normalizedServerSideEncryption- : source.getServerSideEncryption();- S3Object copy = storeObject(destBucket, destKey, source.getData(), effectiveContentType, metadata,- source.getChecksum(), source.getParts(),- new PutObjectOptions()- .withStorageClass(effectiveStorageClass)- .withContentEncoding(effectiveContentEncoding)- .withContentDisposition(effectiveContentDisposition)- .withCacheControl(effectiveCacheControl)- .withServerSideEncryption(effectiveServerSideEncryption)- .withAcl(effectiveOptions.getAcl()));- copy.setETag(source.getETag());- LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey);- fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy);- return copy;+ return copyS3Object(sourceBucket, sourceKey, destBucket, destKey, source, options);}// --- Versioning Operations ---@@ -1093,8 +1083,9 @@ public class S3Service {}public String uploadPartCopy(String destBucket, String destKey, String uploadId, int partNumber,- String sourceBucket, String sourceKey, String copySourceRange) {- S3Object source = getObject(sourceBucket, sourceKey);+ String sourceBucket, String sourceKey, String sourceVersionId,+ String copySourceRange) {+ S3Object source = getObject(sourceBucket, sourceKey, sourceVersionId);byte[] data = source.getData();if (copySourceRange != null && !copySourceRange.isBlank()) {@@ -2026,4 +2017,49 @@ public class S3Service {LOG.errorv(e, "Failed to delete directory: {0}", dir);}}++ private S3Object copyS3Object(String sourceBucket, String sourceKey,+ String destBucket, String destKey, S3Object source, CopyObjectOptions options) {+ ensureBucketExists(destBucket);+ CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();+ String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());++ boolean replaceMetadata = "REPLACE".equalsIgnoreCase(effectiveOptions.getMetadataDirective());+ Map<String, String> metadata = replaceMetadata ? new LinkedHashMap<>() : new LinkedHashMap<>(source.getMetadata());+ if (replaceMetadata && effectiveOptions.getReplacementMetadata() != null) {+ metadata.putAll(effectiveOptions.getReplacementMetadata());+ }++ String effectiveContentType = replaceMetadata && effectiveOptions.getContentType() != null+ ? effectiveOptions.getContentType()+ : source.getContentType();+ String effectiveStorageClass = effectiveOptions.getStorageClass() != null+ ? effectiveOptions.getStorageClass()+ : source.getStorageClass();+ String effectiveContentEncoding = replaceMetadata && effectiveOptions.getContentEncoding() != null+ ? effectiveOptions.getContentEncoding()+ : source.getContentEncoding();+ String effectiveContentDisposition = replaceMetadata && effectiveOptions.getContentDisposition() != null+ ? effectiveOptions.getContentDisposition()+ : source.getContentDisposition();+ String effectiveCacheControl = replaceMetadata && effectiveOptions.getCacheControl() != null+ ? effectiveOptions.getCacheControl()+ : source.getCacheControl();+ String effectiveServerSideEncryption = normalizedServerSideEncryption != null+ ? normalizedServerSideEncryption+ : source.getServerSideEncryption();+ S3Object copy = storeObject(destBucket, destKey, source.getData(), effectiveContentType, metadata,+ source.getChecksum(), source.getParts(),+ new PutObjectOptions()+ .withStorageClass(effectiveStorageClass)+ .withContentEncoding(effectiveContentEncoding)+ .withContentDisposition(effectiveContentDisposition)+ .withCacheControl(effectiveCacheControl)+ .withServerSideEncryption(effectiveServerSideEncryption)+ .withAcl(effectiveOptions.getAcl()));+ copy.setETag(source.getETag());+ LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey);+ fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy);+ return copy;+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectVersionedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectVersionedIntegrationTest.javanew file mode 100644index 0000000000..fbe1b8ade5--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectVersionedIntegrationTest.java@@ -0,0 +1,96 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;++/**+ * Mirrors the AWS CLI flow: versioned bucket, two puts, CopyObject with+ * {@code x-amz-copy-source: /bucket/key?versionId=} to restore an older version as the latest.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class S3CopyObjectVersionedIntegrationTest {++ private static final String BUCKET = "copy-version-repro-it";+ /** First object version id (after v1 upload, while still latest). */+ private static String v1VersionId;++ @Test+ @Order(1)+ void createBucket() {+ given().when().put("/" + BUCKET).then().statusCode(200);+ }++ @Test+ @Order(2)+ void enableVersioning() {+ String xml = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";+ given().body(xml).when().put("/" + BUCKET + "?versioning").then().statusCode(200);+ }++ @Test+ @Order(3)+ void uploadVersion1AndCaptureVersionId() {+ v1VersionId = given()+ .contentType("text/plain")+ .body("v1")+ .when()+ .put("/" + BUCKET + "/key")+ .then()+ .statusCode(200)+ .header("x-amz-version-id", notNullValue())+ .extract()+ .header("x-amz-version-id");+ }++ @Test+ @Order(4)+ void uploadVersion2() {+ given()+ .contentType("text/plain")+ .body("v2")+ .when()+ .put("/" + BUCKET + "/key")+ .then()+ .statusCode(200)+ .header("x-amz-version-id", notNullValue());+ }++ @Test+ @Order(5)+ void latestIsVersion2BeforeCopy() {+ given()+ .when()+ .get("/" + BUCKET + "/key")+ .then()+ .statusCode(200)+ .body(equalTo("v2"));+ }++ @Test+ @Order(6)+ void copyObjectFromV1RestoresV1AsLatest() {+ given()+ .header("x-amz-copy-source", "/" + BUCKET + "/key?versionId=" + v1VersionId)+ .when()+ .put("/" + BUCKET + "/key")+ .then()+ .statusCode(200)+ .body(containsString("CopyObjectResult"));++ given()+ .when()+ .get("/" + BUCKET + "/key")+ .then()+ .statusCode(200)+ .body(equalTo("v1"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3UploadPartCopyVersionedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3UploadPartCopyVersionedIntegrationTest.javanew file mode 100644index 0000000000..f023c57ca4--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3UploadPartCopyVersionedIntegrationTest.java@@ -0,0 +1,123 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;++/**+ * UploadPartCopy with {@code x-amz-copy-source} including {@code ?versionId=...} reads the matching+ * object version bytes (same contract as CopyObject).+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class S3UploadPartCopyVersionedIntegrationTest {++ private static final String BUCKET = "upload-part-copy-ver-it";+ private static final String SRC_KEY = "versioned-mp-source.txt";+ private static final String DEST_KEY = "versioned-mp-dest.bin";+ private static String sourceV1VersionId;+ private static String uploadId;++ @Test+ @Order(1)+ void createBucketAndVersioning() {+ given().when().put("/" + BUCKET).then().statusCode(200);++ String xml = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";+ given().body(xml).when().put("/" + BUCKET + "?versioning").then().statusCode(200);+ }++ @Test+ @Order(2)+ void putSourceVersion1CaptureVersionId() {+ sourceV1VersionId = given()+ .contentType("text/plain")+ .body("version-one-body")+ .when()+ .put("/" + BUCKET + "/" + SRC_KEY)+ .then()+ .statusCode(200)+ .header("x-amz-version-id", notNullValue())+ .extract()+ .header("x-amz-version-id");+ }++ @Test+ @Order(3)+ void overwriteSourceVersion2() {+ given()+ .contentType("text/plain")+ .body("version-two-body")+ .when()+ .put("/" + BUCKET + "/" + SRC_KEY)+ .then()+ .statusCode(200)+ .header("x-amz-version-id", notNullValue());++ given()+ .when()+ .get("/" + BUCKET + "/" + SRC_KEY)+ .then()+ .statusCode(200)+ .body(equalTo("version-two-body"));+ }++ @Test+ @Order(4)+ void initiateMultipartDest() {+ uploadId = given()+ .when()+ .post("/" + BUCKET + "/" + DEST_KEY + "?uploads")+ .then()+ .statusCode(200)+ .body(containsString("<UploadId>"))+ .extract()+ .xmlPath()+ .getString("InitiateMultipartUploadResult.UploadId");+ }++ @Test+ @Order(5)+ void uploadPartCopyUsesOlderVersion() {+ given()+ .header("x-amz-copy-source", "/" + BUCKET + "/" + SRC_KEY + "?versionId=" + sourceV1VersionId)+ .when()+ .put("/" + BUCKET + "/" + DEST_KEY + "?uploadId=" + uploadId + "&partNumber=1")+ .then()+ .statusCode(200)+ .body(containsString("<CopyPartResult"))+ .body(containsString("<ETag>"));+ }++ @Test+ @Order(6)+ void completeAndVerifyBodyFromVersion1() {+ String completeXml = """+ <CompleteMultipartUpload>+ <Part><PartNumber>1</PartNumber><ETag>placeholder</ETag></Part>+ </CompleteMultipartUpload>""";++ given()+ .contentType("application/xml")+ .body(completeXml)+ .when()+ .post("/" + BUCKET + "/" + DEST_KEY + "?uploadId=" + uploadId)+ .then()+ .statusCode(200)+ .body(containsString("<CompleteMultipartUploadResult"));++ given()+ .when()+ .get("/" + BUCKET + "/" + DEST_KEY)+ .then()+ .statusCode(200)+ .body(equalTo("version-one-body"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.javaindex 7e9304ceed..5d4803abfb 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.s3;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.s3.model.CopyObjectOptions;import io.github.hectorvent.floci.services.s3.model.S3Object;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;@@ -168,6 +169,22 @@ class S3VersioningServiceTest {assertEquals("NoSuchVersion", ex.getErrorCode());}+ @Test+ void copyObjectFromOlderVersionRestoresThatContentAsLatest() {+ s3Service.putBucketVersioning("versioned-bucket", "Enabled");+ S3Object v1 = s3Service.putObject("versioned-bucket", "key",+ "v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);+ s3Service.putObject("versioned-bucket", "key",+ "v2".getBytes(StandardCharsets.UTF_8), "text/plain", null);++ assertEquals("v2", new String(s3Service.getObject("versioned-bucket", "key").getData()));++ s3Service.copyObject("versioned-bucket", "key", "versioned-bucket", "key",+ v1.getVersionId(), new CopyObjectOptions());++ assertEquals("v1", new String(s3Service.getObject("versioned-bucket", "key").getData()));+ }+@Testvoid versionedFileUsesS3dataSuffixOnDisk() {S3Service diskService = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir, false);