Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): persist Content-Encoding header on S3 objects (#57)

Roberto Perez Alcoleaff2f68d8ab20
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 92f24bb650..7ed3020145 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -344,8 +344,10 @@ public class S3Controller {
byte[] data = decodeAwsChunked(body, contentEncoding, contentSha256);
validateChecksumHeaders(httpHeaders, data);
+ String persistedEncoding = toPersistedContentEncoding(contentEncoding);
S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders),
httpHeaders.getHeaderString("x-amz-storage-class"),
+ persistedEncoding,
lockMode, retainUntil, legalHold);
var resp = Response.ok().header("ETag", obj.getETag());
if (obj.getVersionId() != null) {
@@ -823,6 +825,30 @@ public class S3Controller {
}
}
+ /**
+ * Strips the {@code aws-chunked} token from a {@code Content-Encoding} value before persisting it.
+ * {@code aws-chunked} is a transfer-protocol marker used by AWS SDK v2 streaming uploads and is not
+ * a real content encoding. For example, {@code gzip,aws-chunked} persists as {@code gzip};
+ * a value of only {@code aws-chunked} persists as {@code null}.
+ */
+ private static String toPersistedContentEncoding(String contentEncoding) {
+ if (contentEncoding == null) {
+ return null;
+ }
+ String[] tokens = contentEncoding.split(",");
+ StringBuilder result = new StringBuilder();
+ for (String token : tokens) {
+ String trimmed = token.trim();
+ if (!trimmed.equalsIgnoreCase("aws-chunked")) {
+ if (!result.isEmpty()) {
+ result.append(",");
+ }
+ result.append(trimmed);
+ }
+ }
+ return result.isEmpty() ? null : result.toString();
+ }
+
// --- AWS Chunked Decoding ---
/**
@@ -1022,6 +1048,9 @@ public class S3Controller {
if (obj.getStorageClass() != null) {
resp.header("x-amz-storage-class", obj.getStorageClass());
}
+ if (obj.getContentEncoding() != null) {
+ resp.header("Content-Encoding", obj.getContentEncoding());
+ }
if (obj.getMetadata() != null) {
for (Map.Entry<String, String> entry : obj.getMetadata().entrySet()) {
resp.header("x-amz-meta-" + entry.getKey(), entry.getValue());
@@ -1056,11 +1085,13 @@ public class S3Controller {
String sourceBucket = source.substring(0, slashIndex);
String sourceKey = URLDecoder.decode(source.substring(slashIndex + 1), StandardCharsets.UTF_8);
+ String copyContentEncoding = toPersistedContentEncoding(httpHeaders.getHeaderString("Content-Encoding"));
S3Object copy = s3Service.copyObject(sourceBucket, sourceKey, destBucket, destKey,
httpHeaders.getHeaderString("x-amz-metadata-directive"),
extractUserMetadata(httpHeaders),
httpHeaders.getHeaderString("x-amz-storage-class"),
- contentType);
+ contentType,
+ copyContentEncoding);
String xml = new XmlBuilder()
.raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
.start("CopyObjectResult", AwsNamespaces.S3)
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 311552d7b5..b94fb6070d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -150,8 +150,16 @@ public class S3Service {
public S3Object putObject(String bucketName, String key, byte[] data,
String contentType, Map<String, String> metadata, String storageClass,
String objectLockMode, Instant retainUntilDate, String legalHoldStatus) {
- S3Object object = storeObject(bucketName, key, data, contentType, metadata, storageClass, null, null,
+ return putObject(bucketName, key, data, contentType, metadata, storageClass, null,
objectLockMode, retainUntilDate, legalHoldStatus);
+ }
+
+ public S3Object putObject(String bucketName, String key, byte[] data,
+ String contentType, Map<String, String> metadata, String storageClass,
+ String contentEncoding,
+ String objectLockMode, Instant retainUntilDate, String legalHoldStatus) {
+ S3Object object = storeObject(bucketName, key, data, contentType, metadata, storageClass, null, null,
+ objectLockMode, retainUntilDate, legalHoldStatus, contentEncoding);
fireNotifications(bucketName, key, "ObjectCreated:Put", object);
return object;
}
@@ -162,13 +170,22 @@ public class S3Service {
private S3Object storeObject(String bucketName, String key, byte[] data,
String contentType, Map<String, String> metadata) {
return storeObject(bucketName, key, data, contentType, metadata, null, null, null,
- null, null, null);
+ null, null, null, null);
}
private S3Object storeObject(String bucketName, String key, byte[] data,
String contentType, Map<String, String> metadata, String storageClass,
S3Checksum checksum, List<Part> parts,
String objectLockMode, Instant retainUntilDate, String legalHoldStatus) {
+ return storeObject(bucketName, key, data, contentType, metadata, storageClass, checksum, parts,
+ objectLockMode, retainUntilDate, legalHoldStatus, null);
+ }
+
+ private S3Object storeObject(String bucketName, String key, byte[] data,
+ String contentType, Map<String, String> metadata, String storageClass,
+ S3Checksum checksum, List<Part> parts,
+ String objectLockMode, Instant retainUntilDate, String legalHoldStatus,
+ String contentEncoding) {
Bucket bucket = bucketStore.get(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket",
"The specified bucket does not exist.", 404));
@@ -180,6 +197,7 @@ public class S3Service {
object.setStorageClass(ObjectAttributeName.normalizeStorageClass(storageClass));
object.setChecksum(checksum != null ? copyChecksum(checksum) : buildChecksum(data, parts, false));
object.setParts(copyParts(parts));
+ object.setContentEncoding(contentEncoding);
if (bucket.isVersioningEnabled()) {
String versionId = UUID.randomUUID().toString();
@@ -462,6 +480,14 @@ public class S3Service {
String destBucket, String destKey,
String metadataDirective, Map<String, String> replacementMetadata,
String storageClass, String contentType) {
+ return copyObject(sourceBucket, sourceKey, destBucket, destKey, metadataDirective,
+ replacementMetadata, storageClass, contentType, null);
+ }
+
+ public S3Object copyObject(String sourceBucket, String sourceKey,
+ String destBucket, String destKey,
+ String metadataDirective, Map<String, String> replacementMetadata,
+ String storageClass, String contentType, String contentEncoding) {
S3Object source = getObject(sourceBucket, sourceKey);
ensureBucketExists(destBucket);
@@ -473,8 +499,10 @@ public class S3Service {
String effectiveContentType = replaceMetadata && contentType != null ? contentType : source.getContentType();
String effectiveStorageClass = storageClass != null ? storageClass : source.getStorageClass();
+ String effectiveContentEncoding = replaceMetadata && contentEncoding != null ? contentEncoding : source.getContentEncoding();
S3Object copy = storeObject(destBucket, destKey, source.getData(), effectiveContentType, metadata,
- effectiveStorageClass, source.getChecksum(), source.getParts(), null, null, null);
+ effectiveStorageClass, source.getChecksum(), source.getParts(), null, null, null,
+ effectiveContentEncoding);
copy.setETag(source.getETag());
LOG.debugv("Copied object: {0}/{1} -> {2}/{3}", sourceBucket, sourceKey, destBucket, destKey);
fireNotifications(destBucket, destKey, "ObjectCreated:Copy", copy);
@@ -1148,6 +1176,7 @@ public class S3Service {
copy.setData(source.getData() != null ? Arrays.copyOf(source.getData(), source.getData().length) : null);
copy.setMetadata(new HashMap<>(source.getMetadata()));
copy.setContentType(source.getContentType());
+ copy.setContentEncoding(source.getContentEncoding());
copy.setSize(source.getSize());
copy.setLastModified(source.getLastModified());
copy.setETag(source.getETag());
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
index 08c422d5cf..facd71ec83 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
@@ -21,6 +21,7 @@ public class S3Object {
private byte[] data;
private Map<String, String> metadata;
private String contentType;
+ private String contentEncoding;
private long size;
private Instant lastModified;
private String eTag;
@@ -77,6 +78,9 @@ public class S3Object {
public String getContentType() { return contentType; }
public void setContentType(String contentType) { this.contentType = contentType; }
+ public String getContentEncoding() { return contentEncoding; }
+ public void setContentEncoding(String contentEncoding) { this.contentEncoding = contentEncoding; }
+
public long getSize() { return size; }
public void setSize(long size) { this.size = size; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
index 9ee2ddd19e..ce9628633f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
@@ -6,6 +6,8 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
+import io.restassured.config.DecoderConfig;
+import io.restassured.config.RestAssuredConfig;
import java.util.Arrays;
import static io.restassured.RestAssured.given;
@@ -732,4 +734,126 @@ class S3IntegrationTest {
.then()
.statusCode(204);
}
+
+ @Test
+ @Order(80)
+ void createEncodingTestBucket() {
+ given()
+ .when()
+ .put("/encoding-test-bucket")
+ .then()
+ .statusCode(200);
+ }
+
+ @Test
+ @Order(81)
+ void putObjectWithContentEncoding() {
+ given()
+ .contentType("text/plain")
+ .header("Content-Encoding", "gzip")
+ .body("compressed-content")
+ .when()
+ .put("/encoding-test-bucket/encoded.txt")
+ .then()
+ .statusCode(200)
+ .header("ETag", notNullValue());
+ }
+
+ @Test
+ @Order(82)
+ void getObjectReturnsContentEncoding() {
+ RestAssuredConfig noDecompress = RestAssuredConfig.config()
+ .decoderConfig(DecoderConfig.decoderConfig().noContentDecoders());
+ given()
+ .config(noDecompress)
+ .when()
+ .get("/encoding-test-bucket/encoded.txt")
+ .then()
+ .statusCode(200)
+ .header("Content-Encoding", equalTo("gzip"));
+ }
+
+ @Test
+ @Order(83)
+ void headObjectReturnsContentEncoding() {
+ given()
+ .when()
+ .head("/encoding-test-bucket/encoded.txt")
+ .then()
+ .statusCode(200)
+ .header("Content-Encoding", equalTo("gzip"));
+ }
+
+ @Test
+ @Order(84)
+ void copyObjectPreservesContentEncoding() {
+ given()
+ .header("x-amz-copy-source", "/encoding-test-bucket/encoded.txt")
+ .when()
+ .put("/encoding-test-bucket/encoded-copy.txt")
+ .then()
+ .statusCode(200)
+ .body(containsString("CopyObjectResult"));
+
+ given()
+ .when()
+ .head("/encoding-test-bucket/encoded-copy.txt")
+ .then()
+ .statusCode(200)
+ .header("Content-Encoding", equalTo("gzip"));
+ }
+
+ @Test
+ @Order(85)
+ void copyObjectReplaceContentEncoding() {
+ given()
+ .header("x-amz-copy-source", "/encoding-test-bucket/encoded.txt")
+ .header("x-amz-metadata-directive", "REPLACE")
+ .header("Content-Encoding", "identity")
+ .when()
+ .put("/encoding-test-bucket/encoded-replace.txt")
+ .then()
+ .statusCode(200)
+ .body(containsString("CopyObjectResult"));
+
+ given()
+ .when()
+ .head("/encoding-test-bucket/encoded-replace.txt")
+ .then()
+ .statusCode(200)
+ .header("Content-Encoding", equalTo("identity"));
+ }
+
+ @Test
+ @Order(86)
+ void putObjectWithCompositeEncoding_stripsAwsChunkedToken() {
+ RestAssuredConfig noDecompress = RestAssuredConfig.config()
+ .decoderConfig(DecoderConfig.decoderConfig().noContentDecoders());
+ given()
+ .contentType("text/plain")
+ .header("Content-Encoding", "gzip,aws-chunked")
+ .body("compressed-chunked-content")
+ .when()
+ .put("/encoding-test-bucket/composite-encoded.txt")
+ .then()
+ .statusCode(200);
+
+ given()
+ .config(noDecompress)
+ .when()
+ .head("/encoding-test-bucket/composite-encoded.txt")
+ .then()
+ .statusCode(200)
+ .header("Content-Encoding", equalTo("gzip"));
+ }
+
+ @Test
+ @Order(88)
+ void cleanupContentEncodingBucket() {
+ given().delete("/encoding-test-bucket/encoded.txt");
+ given().delete("/encoding-test-bucket/encoded-copy.txt");
+ given().delete("/encoding-test-bucket/encoded-replace.txt");
+ given().delete("/encoding-test-bucket/composite-encoded.txt");
+ given().delete("/encoding-test-bucket");
+ }
}