Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): return correct checksum algorithm on PutObject (#877)

Eduardo Lugo Velae1fa8ae8c755
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 55c734a8c6..3e2915a13c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -449,7 +449,7 @@ public class S3Controller {
return handleUploadPartCopy(copySource, bucket, key, uploadId, partNumber, httpHeaders);
}
byte[] partData = decodeAwsChunked(body, contentEncoding, contentSha256);
- validateChecksumHeaders(httpHeaders, partData);
+ validateChecksumHeaders(httpHeaders, partData, httpHeaders.getHeaderString("x-amz-sdk-checksum-algorithm"));
String eTag = s3Service.uploadPart(bucket, key, uploadId, partNumber, partData);
return Response.ok().header("ETag", eTag).build();
}
@@ -469,7 +469,8 @@ public class S3Controller {
Instant retainUntil = retainUntilStr != null ? Instant.parse(retainUntilStr) : null;
byte[] data = decodeAwsChunked(body, contentEncoding, contentSha256);
- validateChecksumHeaders(httpHeaders, data);
+ String checksumAlgorithm = httpHeaders.getHeaderString("x-amz-sdk-checksum-algorithm");
+ validateChecksumHeaders(httpHeaders, data, checksumAlgorithm);
String persistedEncoding = toPersistedContentEncoding(contentEncoding);
String contentDisposition = httpHeaders.getHeaderString("Content-Disposition");
String cacheControl = httpHeaders.getHeaderString("Cache-Control");
@@ -485,7 +486,9 @@ public class S3Controller {
.withContentDisposition(contentDisposition)
.withCacheControl(cacheControl)
.withServerSideEncryption(serverSideEncryption)
- .withAcl(cannedAcl));
+ .withAcl(cannedAcl)
+ .withChecksumAlgorithm(checksumAlgorithm)
+ .withClientChecksum(extractChecksumFromHeaders(httpHeaders)));
var resp = Response.ok().header("ETag", obj.getETag());
if (obj.getVersionId() != null) {
resp.header("x-amz-version-id", obj.getVersionId());
@@ -1645,6 +1648,15 @@ public class S3Controller {
if (checksum == null) {
return;
}
+ if (checksum.getChecksumCRC32() != null) {
+ resp.header("x-amz-checksum-crc32", checksum.getChecksumCRC32());
+ }
+ if (checksum.getChecksumCRC32C() != null) {
+ resp.header("x-amz-checksum-crc32c", checksum.getChecksumCRC32C());
+ }
+ if (checksum.getChecksumCRC64NVME() != null) {
+ resp.header("x-amz-checksum-crc64nvme", checksum.getChecksumCRC64NVME());
+ }
if (checksum.getChecksumSHA1() != null) {
resp.header("x-amz-checksum-sha1", checksum.getChecksumSHA1());
}
@@ -1668,7 +1680,26 @@ public class S3Controller {
return metadata;
}
- private void validateChecksumHeaders(HttpHeaders httpHeaders, byte[] data) {
+ private S3Checksum extractChecksumFromHeaders(HttpHeaders httpHeaders) {
+ String crc32 = httpHeaders.getHeaderString("x-amz-checksum-crc32");
+ String crc32c = httpHeaders.getHeaderString("x-amz-checksum-crc32c");
+ String crc64nvme = httpHeaders.getHeaderString("x-amz-checksum-crc64nvme");
+ String sha1 = httpHeaders.getHeaderString("x-amz-checksum-sha1");
+ String sha256 = httpHeaders.getHeaderString("x-amz-checksum-sha256");
+ if (crc32 == null && crc32c == null && crc64nvme == null && sha1 == null && sha256 == null) {
+ return null;
+ }
+ S3Checksum checksum = new S3Checksum();
+ checksum.setChecksumCRC32(crc32);
+ checksum.setChecksumCRC32C(crc32c);
+ checksum.setChecksumCRC64NVME(crc64nvme);
+ checksum.setChecksumSHA1(sha1);
+ checksum.setChecksumSHA256(sha256);
+ checksum.setChecksumType("FULL_OBJECT");
+ return checksum;
+ }
+
+ private void validateChecksumHeaders(HttpHeaders httpHeaders, byte[] data, String algorithm) {
String sha1 = httpHeaders.getHeaderString("x-amz-checksum-sha1");
if (sha1 != null && !sha1.equals(S3Checksum.sha1Base64(data))) {
throw new AwsException("BadDigest", "The SHA1 checksum you specified did not match the payload.", 400);
@@ -1678,6 +1709,21 @@ public class S3Controller {
if (sha256 != null && !sha256.equals(S3Checksum.sha256Base64(data))) {
throw new AwsException("BadDigest", "The SHA256 checksum you specified did not match the payload.", 400);
}
+
+ String crc32 = httpHeaders.getHeaderString("x-amz-checksum-crc32");
+ if (crc32 != null && !crc32.equals(S3Checksum.crc32Base64(data))) {
+ throw new AwsException("BadDigest", "The CRC32 checksum you specified did not match the payload.", 400);
+ }
+
+ String crc32c = httpHeaders.getHeaderString("x-amz-checksum-crc32c");
+ if (crc32c != null && !crc32c.equals(S3Checksum.crc32cBase64(data))) {
+ throw new AwsException("BadDigest", "The CRC32C checksum you specified did not match the payload.", 400);
+ }
+
+ String crc64nvme = httpHeaders.getHeaderString("x-amz-checksum-crc64nvme");
+ if (crc64nvme != null && !crc64nvme.equals(S3Checksum.crc64NvmeBase64(data))) {
+ throw new AwsException("BadDigest", "The CRC64NVME checksum you specified did not match the payload.", 400);
+ }
}
private Response xmlErrorResponse(AwsException e) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index ba6ef2b540..1f8aa97929 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -286,7 +286,10 @@ public class S3Service {
object.getMetadata().putAll(metadata);
}
object.setStorageClass(ObjectAttributeName.normalizeStorageClass(effectiveOptions.getStorageClass()));
- object.setChecksum(checksum != null ? copyChecksum(checksum) : buildChecksum(data, parts, false));
+ S3Checksum resolvedChecksum = checksum != null ? copyChecksum(checksum)
+ : effectiveOptions.getClientChecksum() != null ? copyChecksum(effectiveOptions.getClientChecksum())
+ : buildChecksum(data, parts, false, effectiveOptions.getChecksumAlgorithm());
+ object.setChecksum(resolvedChecksum);
object.setParts(copyParts(parts));
object.setContentEncoding(effectiveOptions.getContentEncoding());
object.setContentDisposition(effectiveOptions.getContentDisposition());
@@ -1812,9 +1815,19 @@ public class S3Service {
}
private static S3Checksum buildChecksum(byte[] data, List<Part> parts, boolean multipartUpload) {
+ return buildChecksum(data, parts, multipartUpload, null);
+ }
+
+ private static S3Checksum buildChecksum(byte[] data, List<Part> parts, boolean multipartUpload, String algorithm) {
S3Checksum checksum = new S3Checksum();
- checksum.setChecksumSHA1(S3Checksum.sha1Base64(data));
- checksum.setChecksumSHA256(S3Checksum.sha256Base64(data));
+ String algo = (algorithm != null) ? algorithm.toUpperCase() : "CRC64NVME";
+ switch (algo) {
+ case "CRC32" -> checksum.setChecksumCRC32(S3Checksum.crc32Base64(data));
+ case "CRC32C" -> checksum.setChecksumCRC32C(S3Checksum.crc32cBase64(data));
+ case "SHA1" -> checksum.setChecksumSHA1(S3Checksum.sha1Base64(data));
+ case "SHA256" -> checksum.setChecksumSHA256(S3Checksum.sha256Base64(data));
+ default -> checksum.setChecksumCRC64NVME(S3Checksum.crc64NvmeBase64(data));
+ }
checksum.setChecksumType(multipartUpload || (parts != null && parts.size() > 1)
? "COMPOSITE"
: "FULL_OBJECT");
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java
index 75726b7039..41e1de1623 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/PutObjectOptions.java
@@ -12,6 +12,7 @@ public class PutObjectOptions {
private String cacheControl;
private String serverSideEncryption;
private String acl;
+ private String checksumAlgorithm;
public String getStorageClass() { return storageClass; }
public PutObjectOptions withStorageClass(String storageClass) { this.storageClass = storageClass; return this; }
@@ -39,4 +40,11 @@ public class PutObjectOptions {
public String getAcl() { return acl; }
public PutObjectOptions withAcl(String acl) { this.acl = acl; return this; }
+
+ public String getChecksumAlgorithm() { return checksumAlgorithm; }
+ public PutObjectOptions withChecksumAlgorithm(String checksumAlgorithm) { this.checksumAlgorithm = checksumAlgorithm; return this; }
+
+ private S3Checksum clientChecksum;
+ public S3Checksum getClientChecksum() { return clientChecksum; }
+ public PutObjectOptions withClientChecksum(S3Checksum clientChecksum) { this.clientChecksum = clientChecksum; return this; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
index 40969cf324..08fae8e03e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java
@@ -5,10 +5,15 @@ import io.quarkus.runtime.annotations.RegisterForReflection;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
+import java.util.zip.CRC32;
+import java.util.zip.CRC32C;
@RegisterForReflection
public class S3Checksum {
+ private static final long CRC64_NVME_POLY = 0x9a6c9329ac4bc9b5L;
+ private static final long[] CRC64_TABLE = buildCrc64Table();
+
private String checksumCRC32;
private String checksumCRC32C;
private String checksumCRC64NVME;
@@ -39,6 +44,40 @@ public class S3Checksum {
|| checksumSHA1 != null || checksumSHA256 != null;
}
+ public static String crc32Base64(byte[] data) {
+ CRC32 crc = new CRC32();
+ crc.update(data);
+ long value = crc.getValue();
+ byte[] bytes = new byte[]{
+ (byte)(value >> 24), (byte)(value >> 16), (byte)(value >> 8), (byte) value
+ };
+ return Base64.getEncoder().encodeToString(bytes);
+ }
+
+ public static String crc32cBase64(byte[] data) {
+ CRC32C crc = new CRC32C();
+ crc.update(data);
+ long value = crc.getValue();
+ byte[] bytes = new byte[]{
+ (byte)(value >> 24), (byte)(value >> 16), (byte)(value >> 8), (byte) value
+ };
+ return Base64.getEncoder().encodeToString(bytes);
+ }
+
+ public static String crc64NvmeBase64(byte[] data) {
+ long crc = 0xFFFFFFFFFFFFFFFFL;
+ for (byte b : data) {
+ int idx = (int)((crc ^ b) & 0xFF);
+ crc = CRC64_TABLE[idx] ^ (crc >>> 8);
+ }
+ crc ^= 0xFFFFFFFFFFFFFFFFL;
+ byte[] bytes = new byte[]{
+ (byte)(crc >> 56), (byte)(crc >> 48), (byte)(crc >> 40), (byte)(crc >> 32),
+ (byte)(crc >> 24), (byte)(crc >> 16), (byte)(crc >> 8), (byte) crc
+ };
+ return Base64.getEncoder().encodeToString(bytes);
+ }
+
public static String sha256Base64(byte[] data) {
return digestBase64("SHA-256", data);
}
@@ -55,4 +94,20 @@ public class S3Checksum {
throw new IllegalStateException("Missing digest algorithm: " + algorithm, e);
}
}
+
+ private static long[] buildCrc64Table() {
+ long[] table = new long[256];
+ for (int i = 0; i < 256; i++) {
+ long crc = i;
+ for (int j = 0; j < 8; j++) {
+ if ((crc & 1) != 0) {
+ crc = (crc >>> 1) ^ CRC64_NVME_POLY;
+ } else {
+ crc >>>= 1;
+ }
+ }
+ table[i] = crc;
+ }
+ return table;
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
index 0a507c6870..39d647dc18 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
@@ -77,7 +77,7 @@ class S3IntegrationTest {
.header("Content-Length", notNullValue())
.header("x-amz-meta-owner", equalTo("team-a"))
.header("x-amz-storage-class", equalTo("STANDARD_IA"))
- .header("x-amz-checksum-sha256", notNullValue())
+ .header("x-amz-checksum-crc64nvme", notNullValue())
.body(equalTo("Hello World from S3!"));
}
@@ -93,7 +93,7 @@ class S3IntegrationTest {
.body(containsString("<GetObjectAttributesResponse"))
.body(containsString("<StorageClass>STANDARD_IA</StorageClass>"))
.body(containsString("<ObjectSize>20</ObjectSize>"))
- .body(containsString("<ChecksumSHA256>"));
+ .body(containsString("<ChecksumCRC64NVME>"));
}
@Test
@@ -108,7 +108,7 @@ class S3IntegrationTest {
.header("Content-Length", notNullValue())
.header("x-amz-meta-owner", equalTo("team-a"))
.header("x-amz-storage-class", equalTo("STANDARD_IA"))
- .header("x-amz-checksum-sha256", notNullValue());
+ .header("x-amz-checksum-crc64nvme", notNullValue());
}
@Test
@@ -1625,4 +1625,43 @@ class S3IntegrationTest {
.statusCode(400)
.body(containsString("InvalidKey"));
}
+
+ @Test
+ @Order(150)
+ void putObjectRejectsMismatchedCRC32() {
+ given()
+ .body("hello")
+ .header("x-amz-checksum-crc32", "INVALID==")
+ .when()
+ .put("/test-bucket/checksum-crc32-test.txt")
+ .then()
+ .statusCode(400)
+ .body(containsString("BadDigest"));
+ }
+
+ @Test
+ @Order(151)
+ void putObjectRejectsMismatchedCRC32C() {
+ given()
+ .body("hello")
+ .header("x-amz-checksum-crc32c", "INVALID==")
+ .when()
+ .put("/test-bucket/checksum-crc32c-test.txt")
+ .then()
+ .statusCode(400)
+ .body(containsString("BadDigest"));
+ }
+
+ @Test
+ @Order(152)
+ void putObjectRejectsMismatchedCRC64NVME() {
+ given()
+ .body("hello")
+ .header("x-amz-checksum-crc64nvme", "INVALID==")
+ .when()
+ .put("/test-bucket/checksum-crc64nvme-test.txt")
+ .then()
+ .statusCode(400)
+ .body(containsString("BadDigest"));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
index 703bc054eb..a7ce8cbb4e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartIntegrationTest.java
@@ -148,7 +148,7 @@ class S3MultipartIntegrationTest {
.body(containsString("<StorageClass>STANDARD_IA</StorageClass>"))
.body(containsString("<ObjectParts>"))
.body(containsString("<PartsCount>2</PartsCount>"))
- .body(containsString("<ChecksumSHA256>"));
+ .body(containsString("<ChecksumCRC64NVME>"));
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
index 31e34bf7b0..8d94343528 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
@@ -172,6 +172,6 @@ class S3MultipartServiceTest {
assertEquals(1, attributes.getObjectParts().getParts().size());
assertTrue(attributes.getObjectParts().isTruncated());
assertEquals(1, attributes.getObjectParts().getNextPartNumberMarker());
- assertNotNull(attributes.getObjectParts().getParts().get(0).getChecksum().getChecksumSHA256());
+ assertNotNull(attributes.getObjectParts().getParts().get(0).getChecksum().getChecksumCRC64NVME());
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index 9097bbd462..5085055170 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -362,7 +362,7 @@ class S3ServiceTest {
assertEquals("STANDARD_IA", head.getStorageClass());
assertEquals("team-a", head.getMetadata().get("owner"));
assertNotNull(head.getChecksum());
- assertNotNull(head.getChecksum().getChecksumSHA256());
+ assertNotNull(head.getChecksum().getChecksumCRC64NVME());
assertEquals("FULL_OBJECT", head.getChecksum().getChecksumType());
assertEquals(stored.getETag(), head.getETag());
}
@@ -382,7 +382,7 @@ class S3ServiceTest {
assertEquals(7L, attributes.getObjectSize());
assertEquals("GLACIER", attributes.getStorageClass());
assertNotNull(attributes.getChecksum());
- assertNotNull(attributes.getChecksum().getChecksumSHA256());
+ assertNotNull(attributes.getChecksum().getChecksumCRC64NVME());
assertNull(attributes.getObjectParts());
}