Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): persist Content-Encoding header on S3 objects (#57)
Roberto Perez Alcoleaff2f68d8ab20
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 92f24bb650..7ed3020145 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -344,8 +344,10 @@ public class S3Controller {byte[] data = decodeAwsChunked(body, contentEncoding, contentSha256);validateChecksumHeaders(httpHeaders, data);+ String persistedEncoding = toPersistedContentEncoding(contentEncoding);S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders),httpHeaders.getHeaderString("x-amz-storage-class"),+ persistedEncoding,lockMode, retainUntil, legalHold);var resp = Response.ok().header("ETag", obj.getETag());if (obj.getVersionId() != null) {@@ -823,6 +825,30 @@ public class S3Controller {}}+ /**+ * Strips the {@code aws-chunked} token from a {@code Content-Encoding} value before persisting it.+ * {@code aws-chunked} is a transfer-protocol marker used by AWS SDK v2 streaming uploads and is not+ * a real content encoding. For example, {@code gzip,aws-chunked} persists as {@code gzip};+ * a value of only {@code aws-chunked} persists as {@code null}.+ */+ private static String toPersistedContentEncoding(String contentEncoding) {+ if (contentEncoding == null) {+ return null;+ }+ String[] tokens = contentEncoding.split(",");+ StringBuilder result = new StringBuilder();+ for (String token : tokens) {+ String trimmed = token.trim();+ if (!trimmed.equalsIgnoreCase("aws-chunked")) {+ if (!result.isEmpty()) {+ result.append(",");+ }+ result.append(trimmed);+ }+ }+ return result.isEmpty() ? null : result.toString();+ }+// --- AWS Chunked Decoding ---/**@@ -1022,6 +1048,9 @@ public class S3Controller {if (obj.getStorageClass() != null) {resp.header("x-amz-storage-class", obj.getStorageClass());}+ if (obj.getContentEncoding() != null) {+ resp.header("Content-Encoding", obj.getContentEncoding());+ }if (obj.getMetadata() != null) {for (Map.Entry<String, String> entry : obj.getMetadata().entrySet()) {resp.header("x-amz-meta-" + entry.getKey(), entry.getValue());@@ -1056,11 +1085,13 @@ public class S3Controller {String sourceBucket = source.substring(0, slashIndex);String sourceKey = URLDecoder.decode(source.substring(slashIndex + 1), StandardCharsets.UTF_8);+ String copyContentEncoding = toPersistedContentEncoding(httpHeaders.getHeaderString("Content-Encoding"));S3Object copy = s3Service.copyObject(sourceBucket, sourceKey, destBucket, destKey,httpHeaders.getHeaderString("x-amz-metadata-directive"),extractUserMetadata(httpHeaders),httpHeaders.getHeaderString("x-amz-storage-class"),- contentType);+ contentType,+ copyContentEncoding);String xml = new XmlBuilder().raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").start("CopyObjectResult", AwsNamespaces.S3)diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 311552d7b5..b94fb6070d 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -150,8 +150,16 @@ public class S3Service {public S3Object putObject(String bucketName, String key, byte[] data,String contentType, Map<String, String> metadata, String storageClass,String objectLockMode, Instant retainUntilDate, String legalHoldStatus) {- S3Object object = storeObject(bucketName, key, data, contentType, metadata, storageClass, null, null,+ return putObject(bucketName, key, data, contentType, metadata, storageClass, null,objectLockMode, retainUntilDate, legalHoldStatus);+ }++ public S3Object putObject(String bucketName, String key, byte[] data,+ String contentType, Map<String, String> metadata, String storageClass,+ String contentEncoding,+ String objectLockMode, Instant retainUntilDate, String legalHoldStatus) {+ S3Object object = storeObject(bucketName, key, data, contentType, metadata, storageClass, null, null,+ objectLockMode, retainUntilDate, legalHoldStatus, contentEncoding);fireNotifications(bucketName, key, "ObjectCreated:Put", object);return object;}@@ -162,13 +170,22 @@ public class S3Service {private S3Object storeObject(String bucketName, String key, byte[] data,String contentType, Map<String, String> metadata) {return storeObject(bucketName, key, data, contentType, metadata, null, null, null,- null, null, null);+ null, null, null, null);}private S3Object storeObject(String bucketName, String key, byte[] data,String contentType, Map<String, String> metadata, String storageClass,S3Checksum checksum, List<Part> parts,String objectLockMode, Instant retainUntilDate, String legalHoldStatus) {+ return storeObject(bucketName, key, data, contentType, metadata, storageClass, checksum, parts,+ objectLockMode, retainUntilDate, legalHoldStatus, null);+ }++ private S3Object storeObject(String bucketName, String key, byte[] data,+ String contentType, Map<String, String> metadata, String storageClass,+ S3Checksum checksum, List<Part> parts,+ String objectLockMode, Instant retainUntilDate, String legalHoldStatus,+ String contentEncoding) {Bucket bucket = bucketStore.get(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));@@ -180,6 +197,7 @@ public class S3Service {object.setStorageClass(ObjectAttributeName.normalizeStorageClass(storageClass));object.setChecksum(checksum != null ? copyChecksum(checksum) : buildChecksum(data, parts, false));object.setParts(copyParts(parts));+ object.setContentEncoding(contentEncoding);if (bucket.isVersioningEnabled()) {String versionId = UUID.randomUUID().toString();@@ -462,6 +480,14 @@ public class S3Service {String destBucket, String destKey,String metadataDirective, Map<String, String> replacementMetadata,String storageClass, String contentType) {+ return copyObject(sourceBucket, sourceKey, destBucket, destKey, metadataDirective,+ replacementMetadata, storageClass, contentType, null);+ }++ public S3Object copyObject(String sourceBucket, String sourceKey,+ String destBucket, String destKey,+ String metadataDirective, Map<String, String> replacementMetadata,+ String storageClass, String contentType, String contentEncoding) {S3Object source = getObject(sourceBucket, sourceKey);ensureBucketExists(destBucket);@@ -473,8 +499,10 @@ public class S3Service {String effectiveContentType = replaceMetadata && contentType != null ? contentType : source.getContentType();String effectiveStorageClass = storageClass != null ? storageClass : source.getStorageClass();+ String effectiveContentEncoding = replaceMetadata && contentEncoding != null ? contentEncoding : source.getContentEncoding();S3Object copy = storeObject(destBucket, destKey, source.getData(), effectiveContentType, metadata,- effectiveStorageClass, source.getChecksum(), source.getParts(), null, null, null);+ effectiveStorageClass, source.getChecksum(), source.getParts(), null, null, null,+ effectiveContentEncoding);copy.setETag(source.getETag());LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey);fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy);@@ -1148,6 +1176,7 @@ public class S3Service {copy.setData(source.getData() != null ? Arrays.copyOf(source.getData(), source.getData().length) : null);copy.setMetadata(new HashMap<>(source.getMetadata()));copy.setContentType(source.getContentType());+ copy.setContentEncoding(source.getContentEncoding());copy.setSize(source.getSize());copy.setLastModified(source.getLastModified());copy.setETag(source.getETag());diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.javaindex 08c422d5cf..facd71ec83 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java@@ -21,6 +21,7 @@ public class S3Object {private byte[] data;private Map<String, String> metadata;private String contentType;+ private String contentEncoding;private long size;private Instant lastModified;private String eTag;@@ -77,6 +78,9 @@ public class S3Object {public String getContentType() { return contentType; }public void setContentType(String contentType) { this.contentType = contentType; }+ public String getContentEncoding() { return contentEncoding; }+ public void setContentEncoding(String contentEncoding) { this.contentEncoding = contentEncoding; }+public long getSize() { return size; }public void setSize(long size) { this.size = size; }diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.javaindex 9ee2ddd19e..ce9628633f 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java@@ -6,6 +6,8 @@ import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;+import io.restassured.config.DecoderConfig;+import io.restassured.config.RestAssuredConfig;import java.util.Arrays;import static io.restassured.RestAssured.given;@@ -732,4 +734,126 @@ class S3IntegrationTest {.then().statusCode(204);}++ @Test+ @Order(80)+ void createEncodingTestBucket() {+ given()+ .when()+ .put("/encoding-test-bucket")+ .then()+ .statusCode(200);+ }++ @Test+ @Order(81)+ void putObjectWithContentEncoding() {+ given()+ .contentType("text/plain")+ .header("Content-Encoding", "gzip")+ .body("compressed-content")+ .when()+ .put("/encoding-test-bucket/encoded.txt")+ .then()+ .statusCode(200)+ .header("ETag", notNullValue());+ }++ @Test+ @Order(82)+ void getObjectReturnsContentEncoding() {+ RestAssuredConfig noDecompress = RestAssuredConfig.config()+ .decoderConfig(DecoderConfig.decoderConfig().noContentDecoders());+ given()+ .config(noDecompress)+ .when()+ .get("/encoding-test-bucket/encoded.txt")+ .then()+ .statusCode(200)+ .header("Content-Encoding", equalTo("gzip"));+ }++ @Test+ @Order(83)+ void headObjectReturnsContentEncoding() {+ given()+ .when()+ .head("/encoding-test-bucket/encoded.txt")+ .then()+ .statusCode(200)+ .header("Content-Encoding", equalTo("gzip"));+ }++ @Test+ @Order(84)+ void copyObjectPreservesContentEncoding() {+ given()+ .header("x-amz-copy-source", "/encoding-test-bucket/encoded.txt")+ .when()+ .put("/encoding-test-bucket/encoded-copy.txt")+ .then()+ .statusCode(200)+ .body(containsString("CopyObjectResult"));++ given()+ .when()+ .head("/encoding-test-bucket/encoded-copy.txt")+ .then()+ .statusCode(200)+ .header("Content-Encoding", equalTo("gzip"));+ }++ @Test+ @Order(85)+ void copyObjectReplaceContentEncoding() {+ given()+ .header("x-amz-copy-source", "/encoding-test-bucket/encoded.txt")+ .header("x-amz-metadata-directive", "REPLACE")+ .header("Content-Encoding", "identity")+ .when()+ .put("/encoding-test-bucket/encoded-replace.txt")+ .then()+ .statusCode(200)+ .body(containsString("CopyObjectResult"));++ given()+ .when()+ .head("/encoding-test-bucket/encoded-replace.txt")+ .then()+ .statusCode(200)+ .header("Content-Encoding", equalTo("identity"));+ }++ @Test+ @Order(86)+ void putObjectWithCompositeEncoding_stripsAwsChunkedToken() {+ RestAssuredConfig noDecompress = RestAssuredConfig.config()+ .decoderConfig(DecoderConfig.decoderConfig().noContentDecoders());+ given()+ .contentType("text/plain")+ .header("Content-Encoding", "gzip,aws-chunked")+ .body("compressed-chunked-content")+ .when()+ .put("/encoding-test-bucket/composite-encoded.txt")+ .then()+ .statusCode(200);++ given()+ .config(noDecompress)+ .when()+ .head("/encoding-test-bucket/composite-encoded.txt")+ .then()+ .statusCode(200)+ .header("Content-Encoding", equalTo("gzip"));+ }++ @Test+ @Order(88)+ void cleanupContentEncodingBucket() {+ given().delete("/encoding-test-bucket/encoded.txt");+ given().delete("/encoding-test-bucket/encoded-copy.txt");+ given().delete("/encoding-test-bucket/encoded-replace.txt");+ given().delete("/encoding-test-bucket/composite-encoded.txt");+ given().delete("/encoding-test-bucket");+ }}