Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): return correct checksum algorithm on PutObject (#877)
Eduardo Lugo Velae1fa8ae8c755
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 55c734a8c6..3e2915a13c 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -449,7 +449,7 @@ public class S3Controller {return handleUploadPartCopy(copySource, bucket, key, uploadId, partNumber, httpHeaders);}byte[] partData = decodeAwsChunked(body, contentEncoding, contentSha256);- validateChecksumHeaders(httpHeaders, partData);+ validateChecksumHeaders(httpHeaders, partData, httpHeaders.getHeaderString("x-amz-sdk-checksum-algorithm"));String eTag = s3Service.uploadPart(bucket, key, uploadId, partNumber, partData);return Response.ok().header("ETag", eTag).build();}@@ -469,7 +469,8 @@ public class S3Controller {Instant retainUntil = retainUntilStr != null ? Instant.parse(retainUntilStr) : null;byte[] data = decodeAwsChunked(body, contentEncoding, contentSha256);- validateChecksumHeaders(httpHeaders, data);+ String checksumAlgorithm = httpHeaders.getHeaderString("x-amz-sdk-checksum-algorithm");+ validateChecksumHeaders(httpHeaders, data, checksumAlgorithm);String persistedEncoding = toPersistedContentEncoding(contentEncoding);String contentDisposition = httpHeaders.getHeaderString("Content-Disposition");String cacheControl = httpHeaders.getHeaderString("Cache-Control");@@ -485,7 +486,9 @@ public class S3Controller {.withContentDisposition(contentDisposition).withCacheControl(cacheControl).withServerSideEncryption(serverSideEncryption)- .withAcl(cannedAcl));+ .withAcl(cannedAcl)+ .withChecksumAlgorithm(checksumAlgorithm)+ .withClientChecksum(extractChecksumFromHeaders(httpHeaders)));var resp = Response.ok().header("ETag", obj.getETag());if (obj.getVersionId() != null) {resp.header("x-amz-version-id", obj.getVersionId());@@ -1645,6 +1648,15 @@ public class S3Controller {if (checksum == null) {return;}+ if (checksum.getChecksumCRC32() != null) {+ resp.header("x-amz-checksum-crc32", checksum.getChecksumCRC32());+ }+ if (checksum.getChecksumCRC32C() != null) {+ resp.header("x-amz-checksum-crc32c", checksum.getChecksumCRC32C());+ }+ if (checksum.getChecksumCRC64NVME() != null) {+ resp.header("x-amz-checksum-crc64nvme", checksum.getChecksumCRC64NVME());+ }if (checksum.getChecksumSHA1() != null) {resp.header("x-amz-checksum-sha1", checksum.getChecksumSHA1());}@@ -1668,7 +1680,26 @@ public class S3Controller {return metadata;}- private void validateChecksumHeaders(HttpHeaders httpHeaders, byte[] data) {+ private S3Checksum extractChecksumFromHeaders(HttpHeaders httpHeaders) {+ String crc32 = httpHeaders.getHeaderString("x-amz-checksum-crc32");+ String crc32c = httpHeaders.getHeaderString("x-amz-checksum-crc32c");+ String crc64nvme = httpHeaders.getHeaderString("x-amz-checksum-crc64nvme");+ String sha1 = httpHeaders.getHeaderString("x-amz-checksum-sha1");+ String sha256 = httpHeaders.getHeaderString("x-amz-checksum-sha256");+ if (crc32 == null && crc32c == null && crc64nvme == null && sha1 == null && sha256 == null) {+ return null;+ }+ S3Checksum checksum = new S3Checksum();+ checksum.setChecksumCRC32(crc32);+ checksum.setChecksumCRC32C(crc32c);+ checksum.setChecksumCRC64NVME(crc64nvme);+ checksum.setChecksumSHA1(sha1);+ checksum.setChecksumSHA256(sha256);+ checksum.setChecksumType("FULL_OBJECT");+ return checksum;+ }++ private void validateChecksumHeaders(HttpHeaders httpHeaders, byte[] data, String algorithm) {String sha1 = httpHeaders.getHeaderString("x-amz-checksum-sha1");if (sha1 != null && !sha1.equals(S3Checksum.sha1Base64(data))) {throw new AwsException("BadDigest", "The SHA1 checksum you specified did not match the payload.", 400);@@ -1678,6 +1709,21 @@ public class S3Controller {if (sha256 != null && !sha256.equals(S3Checksum.sha256Base64(data))) {throw new AwsException("BadDigest", "The SHA256 checksum you specified did not match the payload.", 400);}++ String crc32 = httpHeaders.getHeaderString("x-amz-checksum-crc32");+ if (crc32 != null && !crc32.equals(S3Checksum.crc32Base64(data))) {+ throw new AwsException("BadDigest", "The CRC32 checksum you specified did not match the payload.", 400);+ }++ String crc32c = httpHeaders.getHeaderString("x-amz-checksum-crc32c");+ if (crc32c != null && !crc32c.equals(S3Checksum.crc32cBase64(data))) {+ throw new AwsException("BadDigest", "The CRC32C checksum you specified did not match the payload.", 400);+ }++ String crc64nvme = httpHeaders.getHeaderString("x-amz-checksum-crc64nvme");+ if (crc64nvme != null && !crc64nvme.equals(S3Checksum.crc64NvmeBase64(data))) {+ throw new AwsException("BadDigest", "The CRC64NVME checksum you specified did not match the payload.", 400);+ }}private Response xmlErrorResponse(AwsException e) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex ba6ef2b540..1f8aa97929 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -286,7 +286,10 @@ public class S3Service {object.getMetadata().putAll(metadata);}object.setStorageClass(ObjectAttributeName.normalizeStorageClass(effectiveOptions.getStorageClass()));- object.setChecksum(checksum != null ? copyChecksum(checksum) : buildChecksum(data, parts, false));+ S3Checksum resolvedChecksum = checksum != null ? copyChecksum(checksum)+ : effectiveOptions.getClientChecksum() != null ? copyChecksum(effectiveOptions.getClientChecksum())+ : buildChecksum(data, parts, false, effectiveOptions.getChecksumAlgorithm());+ object.setChecksum(resolvedChecksum);object.setParts(copyParts(parts));object.setContentEncoding(effectiveOptions.getContentEncoding());object.setContentDisposition(effectiveOptions.getContentDisposition());@@ -1812,9 +1815,19 @@ public class S3Service {}private static S3Checksum buildChecksum(byte[] data, List<Part> parts, boolean multipartUpload) {+ return buildChecksum(data, parts, multipartUpload, null);+ }++ private static S3Checksum buildChecksum(byte[] data, List<Part> parts, boolean multipartUpload, String algorithm) {S3Checksum checksum = new S3Checksum();- checksum.setChecksumSHA1(S3Checksum.sha1Base64(data));- checksum.setChecksumSHA256(S3Checksum.sha256Base64(data));+ String algo = (algorithm != null) ? algorithm.toUpperCase() : "CRC64NVME";+ switch (algo) {+ case "CRC32" -> checksum.setChecksumCRC32(S3Checksum.crc32Base64(data));+ case "CRC32C" -> checksum.setChecksumCRC32C(S3Checksum.crc32cBase64(data));+ case "SHA1" -> checksum.setChecksumSHA1(S3Checksum.sha1Base64(data));+ case "SHA256" -> checksum.setChecksumSHA256(S3Checksum.sha256Base64(data));+ default -> checksum.setChecksumCRC64NVME(S3Checksum.crc64NvmeBase64(data));+ }checksum.setChecksumType(multipartUpload || (parts != null && parts.size() > 1)? "COMPOSITE": "FULL_OBJECT");diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.javaindex 75726b7039..41e1de1623 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java@@ -12,6 +12,7 @@ public class PutObjectOptions {private String cacheControl;private String serverSideEncryption;private String acl;+ private String checksumAlgorithm;public String getStorageClass() { return storageClass; }public PutObjectOptions withStorageClass(String storageClass) { this.storageClass = storageClass; return this; }@@ -39,4 +40,11 @@ public class PutObjectOptions {public String getAcl() { return acl; }public PutObjectOptions withAcl(String acl) { this.acl = acl; return this; }++ public String getChecksumAlgorithm() { return checksumAlgorithm; }+ public PutObjectOptions withChecksumAlgorithm(String checksumAlgorithm) { this.checksumAlgorithm = checksumAlgorithm; return this; }++ private S3Checksum clientChecksum;+ public S3Checksum getClientChecksum() { return clientChecksum; }+ public PutObjectOptions withClientChecksum(S3Checksum clientChecksum) { this.clientChecksum = clientChecksum; return this; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.javaindex 40969cf324..08fae8e03e 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java@@ -5,10 +5,15 @@ import io.quarkus.runtime.annotations.RegisterForReflection;import java.security.MessageDigest;import java.security.NoSuchAlgorithmException;import java.util.Base64;+import java.util.zip.CRC32;+import java.util.zip.CRC32C;@RegisterForReflectionpublic class S3Checksum {+ private static final long CRC64_NVME_POLY = 0x9a6c9329ac4bc9b5L;+ private static final long[] CRC64_TABLE = buildCrc64Table();+private String checksumCRC32;private String checksumCRC32C;private String checksumCRC64NVME;@@ -39,6 +44,40 @@ public class S3Checksum {|| checksumSHA1 != null || checksumSHA256 != null;}+ public static String crc32Base64(byte[] data) {+ CRC32 crc = new CRC32();+ crc.update(data);+ long value = crc.getValue();+ byte[] bytes = new byte[]{+ (byte)(value >> 24), (byte)(value >> 16), (byte)(value >> 8), (byte) value+ };+ return Base64.getEncoder().encodeToString(bytes);+ }++ public static String crc32cBase64(byte[] data) {+ CRC32C crc = new CRC32C();+ crc.update(data);+ long value = crc.getValue();+ byte[] bytes = new byte[]{+ (byte)(value >> 24), (byte)(value >> 16), (byte)(value >> 8), (byte) value+ };+ return Base64.getEncoder().encodeToString(bytes);+ }++ public static String crc64NvmeBase64(byte[] data) {+ long crc = 0xFFFFFFFFFFFFFFFFL;+ for (byte b : data) {+ int idx = (int)((crc ^ b) & 0xFF);+ crc = CRC64_TABLE[idx] ^ (crc >>> 8);+ }+ crc ^= 0xFFFFFFFFFFFFFFFFL;+ byte[] bytes = new byte[]{+ (byte)(crc >> 56), (byte)(crc >> 48), (byte)(crc >> 40), (byte)(crc >> 32),+ (byte)(crc >> 24), (byte)(crc >> 16), (byte)(crc >> 8), (byte) crc+ };+ return Base64.getEncoder().encodeToString(bytes);+ }+public static String sha256Base64(byte[] data) {return digestBase64("SHA-256", data);}@@ -55,4 +94,20 @@ public class S3Checksum {throw new IllegalStateException("Missing digest algorithm: " + algorithm, e);}}++ private static long[] buildCrc64Table() {+ long[] table = new long[256];+ for (int i = 0; i < 256; i++) {+ long crc = i;+ for (int j = 0; j < 8; j++) {+ if ((crc & 1) != 0) {+ crc = (crc >>> 1) ^ CRC64_NVME_POLY;+ } else {+ crc >>>= 1;+ }+ }+ table[i] = crc;+ }+ return table;+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.javaindex 0a507c6870..39d647dc18 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java@@ -77,7 +77,7 @@ class S3IntegrationTest {.header("Content-Length", notNullValue()).header("x-amz-meta-owner", equalTo("team-a")).header("x-amz-storage-class", equalTo("STANDARD_IA"))- .header("x-amz-checksum-sha256", notNullValue())+ .header("x-amz-checksum-crc64nvme", notNullValue()).body(equalTo("Hello World from S3!"));}@@ -93,7 +93,7 @@ class S3IntegrationTest {.body(containsString("<GetObjectAttributesResponse")).body(containsString("<StorageClass>STANDARD_IA</StorageClass>")).body(containsString("<ObjectSize>20</ObjectSize>"))- .body(containsString("<ChecksumSHA256>"));+ .body(containsString("<ChecksumCRC64NVME>"));}@Test@@ -108,7 +108,7 @@ class S3IntegrationTest {.header("Content-Length", notNullValue()).header("x-amz-meta-owner", equalTo("team-a")).header("x-amz-storage-class", equalTo("STANDARD_IA"))- .header("x-amz-checksum-sha256", notNullValue());+ .header("x-amz-checksum-crc64nvme", notNullValue());}@Test@@ -1625,4 +1625,43 @@ class S3IntegrationTest {.statusCode(400).body(containsString("InvalidKey"));}++ @Test+ @Order(150)+ void putObjectRejectsMismatchedCRC32() {+ given()+ .body("hello")+ .header("x-amz-checksum-crc32", "INVALID==")+ .when()+ .put("/test-bucket/checksum-crc32-test.txt")+ .then()+ .statusCode(400)+ .body(containsString("BadDigest"));+ }++ @Test+ @Order(151)+ void putObjectRejectsMismatchedCRC32C() {+ given()+ .body("hello")+ .header("x-amz-checksum-crc32c", "INVALID==")+ .when()+ .put("/test-bucket/checksum-crc32c-test.txt")+ .then()+ .statusCode(400)+ .body(containsString("BadDigest"));+ }++ @Test+ @Order(152)+ void putObjectRejectsMismatchedCRC64NVME() {+ given()+ .body("hello")+ .header("x-amz-checksum-crc64nvme", "INVALID==")+ .when()+ .put("/test-bucket/checksum-crc64nvme-test.txt")+ .then()+ .statusCode(400)+ .body(containsString("BadDigest"));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.javaindex 703bc054eb..a7ce8cbb4e 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java@@ -148,7 +148,7 @@ class S3MultipartIntegrationTest {.body(containsString("<StorageClass>STANDARD_IA</StorageClass>")).body(containsString("<ObjectParts>")).body(containsString("<PartsCount>2</PartsCount>"))- .body(containsString("<ChecksumSHA256>"));+ .body(containsString("<ChecksumCRC64NVME>"));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.javaindex 31e34bf7b0..8d94343528 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java@@ -172,6 +172,6 @@ class S3MultipartServiceTest {assertEquals(1, attributes.getObjectParts().getParts().size());assertTrue(attributes.getObjectParts().isTruncated());assertEquals(1, attributes.getObjectParts().getNextPartNumberMarker());- assertNotNull(attributes.getObjectParts().getParts().get(0).getChecksum().getChecksumSHA256());+ assertNotNull(attributes.getObjectParts().getParts().get(0).getChecksum().getChecksumCRC64NVME());}}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex 9097bbd462..5085055170 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -362,7 +362,7 @@ class S3ServiceTest {assertEquals("STANDARD_IA", head.getStorageClass());assertEquals("team-a", head.getMetadata().get("owner"));assertNotNull(head.getChecksum());- assertNotNull(head.getChecksum().getChecksumSHA256());+ assertNotNull(head.getChecksum().getChecksumCRC64NVME());assertEquals("FULL_OBJECT", head.getChecksum().getChecksumType());assertEquals(stored.getETag(), head.getETag());}@@ -382,7 +382,7 @@ class S3ServiceTest {assertEquals(7L, attributes.getObjectSize());assertEquals("GLACIER", attributes.getStorageClass());assertNotNull(attributes.getChecksum());- assertNotNull(attributes.getChecksum().getChecksumSHA256());+ assertNotNull(attributes.getChecksum().getChecksumCRC64NVME());assertNull(attributes.getObjectParts());}