Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): parse copy-source versionId for CopyObject and part copy iss… (#751)

Asha Somayajulace2012de45b7
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 8c94c22749..55c734a8c6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -1511,14 +1511,15 @@ public class S3Controller {
throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400);
}
String sourceBucket = decodedSource.substring(0, slashIndex);
- String sourceKey = decodedSource.substring(slashIndex + 1);
-
+ String pathAfterBucket = decodedSource.substring(slashIndex + 1);
+ ParsedCopySource sourceObject = parseCopySourceObject(pathAfterBucket);
String copyContentEncoding = toPersistedContentEncoding(httpHeaders.getHeaderString("Content-Encoding"));
String copyContentDisposition = httpHeaders.getHeaderString("Content-Disposition");
String copyCacheControl = httpHeaders.getHeaderString("Cache-Control");
String copyServerSideEncryption = httpHeaders.getHeaderString("x-amz-server-side-encryption");
String cannedAcl = httpHeaders.getHeaderString("x-amz-acl");
- S3Object copy = s3Service.copyObject(sourceBucket, sourceKey, destBucket, destKey,
+ S3Object copy = s3Service.copyObject(sourceBucket, sourceObject.objectKey(), destBucket, destKey,
+ sourceObject.versionId(),
new CopyObjectOptions()
.withMetadataDirective(httpHeaders.getHeaderString("x-amz-metadata-directive"))
.withReplacementMetadata(extractUserMetadata(httpHeaders))
@@ -1556,10 +1557,11 @@ public class S3Controller {
throw new AwsException("InvalidArgument", "Invalid copy source: " + copySource, 400);
}
String sourceBucket = decodedSource.substring(0, slashIndex);
- String sourceKey = decodedSource.substring(slashIndex + 1);
+ String pathAfterBucket = decodedSource.substring(slashIndex + 1);
+ ParsedCopySource sourceObject = parseCopySourceObject(pathAfterBucket);
String copySourceRange = httpHeaders.getHeaderString("x-amz-copy-source-range");
String eTag = s3Service.uploadPartCopy(destBucket, destKey, uploadId, partNumber,
- sourceBucket, sourceKey, copySourceRange);
+ sourceBucket, sourceObject.objectKey(), sourceObject.versionId(), copySourceRange);
String xml = new XmlBuilder()
.raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
.start("CopyPartResult", AwsNamespaces.S3)
@@ -2209,4 +2211,46 @@ public class S3Controller {
s3Service.putBucketWebsite(bucket, new WebsiteConfiguration(indexDoc, errorDoc));
return Response.ok().build();
}
+
+ /**
+ * Splits the {@code CopyObject}/{@code UploadPartCopy} copy-source remainder into S3 object key and
+ * optional source {@code versionId}.
+ * <ul>
+ * <li><b>Input:</b> decoded {@code x-amz-copy-source} with bucket already removed (substring after
+ * the {@code '/'} that follows the bucket). Both {@code handleCopyObject} and
+ * {@code handleUploadPartCopy} compute this as {@code pathAfterBucket}.</li>
+ * <li><b>Key:</b> substring before the first {@code '?'} if any; keys may contain more {@code '/'}
+ * segments.</li>
+ * <li><b>{@code versionId}:</b> first {@code versionId} query pair, when present (raw value after
+ * {@code '='}). Other query pairs are ignored.</li>
+ * </ul>
+ *
+ * @param pathAfterBucket object key alone, or key with query (for example {@code dir/k.txt?versionId=uuid})
+ * @return key without trailing query plus {@code versionId} value, or {@code null} version when absent
+ */
+ private ParsedCopySource parseCopySourceObject(String pathAfterBucket) {
+ int queryStart = pathAfterBucket.indexOf('?');
+ if (queryStart < 0) {
+ return new ParsedCopySource(pathAfterBucket, null);
+ }
+ String objectKey = pathAfterBucket.substring(0, queryStart);
+ String query = pathAfterBucket.substring(queryStart + 1);
+ String versionId = null;
+ for (String pair : query.split("&")) {
+ int eq = pair.indexOf('=');
+ if (eq <= 0) {
+ continue;
+ }
+ String name = pair.substring(0, eq);
+ String value = pair.substring(eq + 1);
+ if ("versionId".equals(name)) {
+ versionId = value;
+ break;
+ }
+ }
+ return new ParsedCopySource(objectKey, versionId);
+ }
+
+ private record ParsedCopySource(String objectKey, String versionId) {
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 64eab0060b..93034b96bb 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -636,6 +636,11 @@ public class S3Service {
return copyObject(sourceBucket, sourceKey, destBucket, destKey, new CopyObjectOptions());
}
+ public S3Object copyObject(String sourceBucket, String sourceKey,
+ String destBucket, String destKey, String versionId) {
+ return copyObject(sourceBucket, sourceKey, destBucket, destKey, null, new CopyObjectOptions());
+ }
+
public S3Object copyObject(String sourceBucket, String sourceKey,
String destBucket, String destKey,
String metadataDirective, Map<String, String> replacementMetadata,
@@ -666,50 +671,35 @@ public class S3Service {
.withAcl(acl));
}
+ public S3Object copyObject(String sourceBucket, String sourceKey,
+ String destBucket, String destKey, String versionId,
+ String metadataDirective, Map<String, String> replacementMetadata,
+ String storageClass, String contentType, String contentEncoding,
+ String contentDisposition, String cacheControl, String serverSideEncryption, String acl) {
+ return copyObject(sourceBucket, sourceKey, destBucket, destKey, versionId,
+ new CopyObjectOptions()
+ .withMetadataDirective(metadataDirective)
+ .withReplacementMetadata(replacementMetadata)
+ .withStorageClass(storageClass)
+ .withContentType(contentType)
+ .withContentEncoding(contentEncoding)
+ .withContentDisposition(contentDisposition)
+ .withCacheControl(cacheControl)
+ .withServerSideEncryption(serverSideEncryption)
+ .withAcl(acl));
+ }
+ public S3Object copyObject(String sourceBucket, String sourceKey,
+ String destBucket, String destKey, String versionId, CopyObjectOptions options)
+ {
+ S3Object source = getObject(sourceBucket, sourceKey, versionId);
+ return copyS3Object(sourceBucket, sourceKey,
+ destBucket, destKey, source, options);
+ }
+
public S3Object copyObject(String sourceBucket, String sourceKey,
String destBucket, String destKey, CopyObjectOptions options) {
S3Object source = getObject(sourceBucket, sourceKey);
- ensureBucketExists(destBucket);
- CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();
- String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());
-
- boolean replaceMetadata = "REPLACE".equalsIgnoreCase(effectiveOptions.getMetadataDirective());
- Map<String, String> metadata = replaceMetadata ? new LinkedHashMap<>() : new LinkedHashMap<>(source.getMetadata());
- if (replaceMetadata && effectiveOptions.getReplacementMetadata() != null) {
- metadata.putAll(effectiveOptions.getReplacementMetadata());
- }
-
- String effectiveContentType = replaceMetadata && effectiveOptions.getContentType() != null
- ? effectiveOptions.getContentType()
- : source.getContentType();
- String effectiveStorageClass = effectiveOptions.getStorageClass() != null
- ? effectiveOptions.getStorageClass()
- : source.getStorageClass();
- String effectiveContentEncoding = replaceMetadata && effectiveOptions.getContentEncoding() != null
- ? effectiveOptions.getContentEncoding()
- : source.getContentEncoding();
- String effectiveContentDisposition = replaceMetadata && effectiveOptions.getContentDisposition() != null
- ? effectiveOptions.getContentDisposition()
- : source.getContentDisposition();
- String effectiveCacheControl = replaceMetadata && effectiveOptions.getCacheControl() != null
- ? effectiveOptions.getCacheControl()
- : source.getCacheControl();
- String effectiveServerSideEncryption = normalizedServerSideEncryption != null
- ? normalizedServerSideEncryption
- : source.getServerSideEncryption();
- S3Object copy = storeObject(destBucket, destKey, source.getData(), effectiveContentType, metadata,
- source.getChecksum(), source.getParts(),
- new PutObjectOptions()
- .withStorageClass(effectiveStorageClass)
- .withContentEncoding(effectiveContentEncoding)
- .withContentDisposition(effectiveContentDisposition)
- .withCacheControl(effectiveCacheControl)
- .withServerSideEncryption(effectiveServerSideEncryption)
- .withAcl(effectiveOptions.getAcl()));
- copy.setETag(source.getETag());
- LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey);
- fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy);
- return copy;
+ return copyS3Object(sourceBucket, sourceKey, destBucket, destKey, source, options);
}
// --- Versioning Operations ---
@@ -1093,8 +1083,9 @@ public class S3Service {
}
public String uploadPartCopy(String destBucket, String destKey, String uploadId, int partNumber,
- String sourceBucket, String sourceKey, String copySourceRange) {
- S3Object source = getObject(sourceBucket, sourceKey);
+ String sourceBucket, String sourceKey, String sourceVersionId,
+ String copySourceRange) {
+ S3Object source = getObject(sourceBucket, sourceKey, sourceVersionId);
byte[] data = source.getData();
if (copySourceRange != null && !copySourceRange.isBlank()) {
@@ -2026,4 +2017,49 @@ public class S3Service {
LOG.errorv(e, "Failed to delete directory: {0}", dir);
}
}
+
+ private S3Object copyS3Object(String sourceBucket, String sourceKey,
+ String destBucket, String destKey, S3Object source, CopyObjectOptions options) {
+ ensureBucketExists(destBucket);
+ CopyObjectOptions effectiveOptions = options != null ? options : new CopyObjectOptions();
+ String normalizedServerSideEncryption = normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption());
+
+ boolean replaceMetadata = "REPLACE".equalsIgnoreCase(effectiveOptions.getMetadataDirective());
+ Map<String, String> metadata = replaceMetadata ? new LinkedHashMap<>() : new LinkedHashMap<>(source.getMetadata());
+ if (replaceMetadata && effectiveOptions.getReplacementMetadata() != null) {
+ metadata.putAll(effectiveOptions.getReplacementMetadata());
+ }
+
+ String effectiveContentType = replaceMetadata && effectiveOptions.getContentType() != null
+ ? effectiveOptions.getContentType()
+ : source.getContentType();
+ String effectiveStorageClass = effectiveOptions.getStorageClass() != null
+ ? effectiveOptions.getStorageClass()
+ : source.getStorageClass();
+ String effectiveContentEncoding = replaceMetadata && effectiveOptions.getContentEncoding() != null
+ ? effectiveOptions.getContentEncoding()
+ : source.getContentEncoding();
+ String effectiveContentDisposition = replaceMetadata && effectiveOptions.getContentDisposition() != null
+ ? effectiveOptions.getContentDisposition()
+ : source.getContentDisposition();
+ String effectiveCacheControl = replaceMetadata && effectiveOptions.getCacheControl() != null
+ ? effectiveOptions.getCacheControl()
+ : source.getCacheControl();
+ String effectiveServerSideEncryption = normalizedServerSideEncryption != null
+ ? normalizedServerSideEncryption
+ : source.getServerSideEncryption();
+ S3Object copy = storeObject(destBucket, destKey, source.getData(), effectiveContentType, metadata,
+ source.getChecksum(), source.getParts(),
+ new PutObjectOptions()
+ .withStorageClass(effectiveStorageClass)
+ .withContentEncoding(effectiveContentEncoding)
+ .withContentDisposition(effectiveContentDisposition)
+ .withCacheControl(effectiveCacheControl)
+ .withServerSideEncryption(effectiveServerSideEncryption)
+ .withAcl(effectiveOptions.getAcl()));
+ copy.setETag(source.getETag());
+ LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey);
+ fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy);
+ return copy;
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectVersionedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectVersionedIntegrationTest.java
new file mode 100644
index 0000000000..fbe1b8ade5
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3CopyObjectVersionedIntegrationTest.java
@@ -0,0 +1,96 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * Mirrors the AWS CLI flow: versioned bucket, two puts, CopyObject with
+ * {@code x-amz-copy-source: /bucket/key?versionId=} to restore an older version as the latest.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class S3CopyObjectVersionedIntegrationTest {
+
+ private static final String BUCKET = "copy-version-repro-it";
+ /** First object version id (after v1 upload, while still latest). */
+ private static String v1VersionId;
+
+ @Test
+ @Order(1)
+ void createBucket() {
+ given().when().put("/" + BUCKET).then().statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void enableVersioning() {
+ String xml = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";
+ given().body(xml).when().put("/" + BUCKET + "?versioning").then().statusCode(200);
+ }
+
+ @Test
+ @Order(3)
+ void uploadVersion1AndCaptureVersionId() {
+ v1VersionId = given()
+ .contentType("text/plain")
+ .body("v1")
+ .when()
+ .put("/" + BUCKET + "/key")
+ .then()
+ .statusCode(200)
+ .header("x-amz-version-id", notNullValue())
+ .extract()
+ .header("x-amz-version-id");
+ }
+
+ @Test
+ @Order(4)
+ void uploadVersion2() {
+ given()
+ .contentType("text/plain")
+ .body("v2")
+ .when()
+ .put("/" + BUCKET + "/key")
+ .then()
+ .statusCode(200)
+ .header("x-amz-version-id", notNullValue());
+ }
+
+ @Test
+ @Order(5)
+ void latestIsVersion2BeforeCopy() {
+ given()
+ .when()
+ .get("/" + BUCKET + "/key")
+ .then()
+ .statusCode(200)
+ .body(equalTo("v2"));
+ }
+
+ @Test
+ @Order(6)
+ void copyObjectFromV1RestoresV1AsLatest() {
+ given()
+ .header("x-amz-copy-source", "/" + BUCKET + "/key?versionId=" + v1VersionId)
+ .when()
+ .put("/" + BUCKET + "/key")
+ .then()
+ .statusCode(200)
+ .body(containsString("CopyObjectResult"));
+
+ given()
+ .when()
+ .get("/" + BUCKET + "/key")
+ .then()
+ .statusCode(200)
+ .body(equalTo("v1"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3UploadPartCopyVersionedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3UploadPartCopyVersionedIntegrationTest.java
new file mode 100644
index 0000000000..f023c57ca4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3UploadPartCopyVersionedIntegrationTest.java
@@ -0,0 +1,123 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * UploadPartCopy with {@code x-amz-copy-source} including {@code ?versionId=...} reads the matching
+ * object version bytes (same contract as CopyObject).
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class S3UploadPartCopyVersionedIntegrationTest {
+
+ private static final String BUCKET = "upload-part-copy-ver-it";
+ private static final String SRC_KEY = "versioned-mp-source.txt";
+ private static final String DEST_KEY = "versioned-mp-dest.bin";
+ private static String sourceV1VersionId;
+ private static String uploadId;
+
+ @Test
+ @Order(1)
+ void createBucketAndVersioning() {
+ given().when().put("/" + BUCKET).then().statusCode(200);
+
+ String xml = "<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>";
+ given().body(xml).when().put("/" + BUCKET + "?versioning").then().statusCode(200);
+ }
+
+ @Test
+ @Order(2)
+ void putSourceVersion1CaptureVersionId() {
+ sourceV1VersionId = given()
+ .contentType("text/plain")
+ .body("version-one-body")
+ .when()
+ .put("/" + BUCKET + "/" + SRC_KEY)
+ .then()
+ .statusCode(200)
+ .header("x-amz-version-id", notNullValue())
+ .extract()
+ .header("x-amz-version-id");
+ }
+
+ @Test
+ @Order(3)
+ void overwriteSourceVersion2() {
+ given()
+ .contentType("text/plain")
+ .body("version-two-body")
+ .when()
+ .put("/" + BUCKET + "/" + SRC_KEY)
+ .then()
+ .statusCode(200)
+ .header("x-amz-version-id", notNullValue());
+
+ given()
+ .when()
+ .get("/" + BUCKET + "/" + SRC_KEY)
+ .then()
+ .statusCode(200)
+ .body(equalTo("version-two-body"));
+ }
+
+ @Test
+ @Order(4)
+ void initiateMultipartDest() {
+ uploadId = given()
+ .when()
+ .post("/" + BUCKET + "/" + DEST_KEY + "?uploads")
+ .then()
+ .statusCode(200)
+ .body(containsString("<UploadId>"))
+ .extract()
+ .xmlPath()
+ .getString("InitiateMultipartUploadResult.UploadId");
+ }
+
+ @Test
+ @Order(5)
+ void uploadPartCopyUsesOlderVersion() {
+ given()
+ .header("x-amz-copy-source", "/" + BUCKET + "/" + SRC_KEY + "?versionId=" + sourceV1VersionId)
+ .when()
+ .put("/" + BUCKET + "/" + DEST_KEY + "?uploadId=" + uploadId + "&partNumber=1")
+ .then()
+ .statusCode(200)
+ .body(containsString("<CopyPartResult"))
+ .body(containsString("<ETag>"));
+ }
+
+ @Test
+ @Order(6)
+ void completeAndVerifyBodyFromVersion1() {
+ String completeXml = """
+ <CompleteMultipartUpload>
+ <Part><PartNumber>1</PartNumber><ETag>placeholder</ETag></Part>
+ </CompleteMultipartUpload>""";
+
+ given()
+ .contentType("application/xml")
+ .body(completeXml)
+ .when()
+ .post("/" + BUCKET + "/" + DEST_KEY + "?uploadId=" + uploadId)
+ .then()
+ .statusCode(200)
+ .body(containsString("<CompleteMultipartUploadResult"));
+
+ given()
+ .when()
+ .get("/" + BUCKET + "/" + DEST_KEY)
+ .then()
+ .statusCode(200)
+ .body(equalTo("version-one-body"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java
index 7e9304ceed..5d4803abfb 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.s3;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.s3.model.CopyObjectOptions;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -168,6 +169,22 @@ class S3VersioningServiceTest {
assertEquals("NoSuchVersion", ex.getErrorCode());
}
+ @Test
+ void copyObjectFromOlderVersionRestoresThatContentAsLatest() {
+ s3Service.putBucketVersioning("versioned-bucket", "Enabled");
+ S3Object v1 = s3Service.putObject("versioned-bucket", "key",
+ "v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);
+ s3Service.putObject("versioned-bucket", "key",
+ "v2".getBytes(StandardCharsets.UTF_8), "text/plain", null);
+
+ assertEquals("v2", new String(s3Service.getObject("versioned-bucket", "key").getData()));
+
+ s3Service.copyObject("versioned-bucket", "key", "versioned-bucket", "key",
+ v1.getVersionId(), new CopyObjectOptions());
+
+ assertEquals("v1", new String(s3Service.getObject("versioned-bucket", "key").getData()));
+ }
+
@Test
void versionedFileUsesS3dataSuffixOnDisk() {
S3Service diskService = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir, false);