Light, fluffy, and always free - The AWS Local Emulator alternative
perf(s3): compute CRC64NVME eight bytes at a time (#4126)
TATSUNO “Taz” Yasuhirod6682cf45264
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.javaindex 0af5bd56f3..0bd6615c0b 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Checksum.java@@ -16,7 +16,8 @@ public class S3Checksum {private static final Pattern PART_COUNT_SUFFIX = Pattern.compile("-\\d+$");private static final long CRC64_NVME_POLY = 0x9a6c9329ac4bc9b5L;- private static final long[] CRC64_TABLE = buildCrc64Table();+ // Slicing-by-8: row n holds the CRC of each byte value followed by n zero bytes.+ private static final long[][] CRC64_TABLES = buildCrc64Tables();private String checksumCRC32;private String checksumCRC32C;@@ -146,9 +147,31 @@ public class S3Checksum {public static String crc64NvmeBase64(byte[] data) {long crc = 0xFFFFFFFFFFFFFFFFL;- for (byte b : data) {- int idx = (int)((crc ^ b) & 0xFF);- crc = CRC64_TABLE[idx] ^ (crc >>> 8);+ int offset = 0;+ int blocksEnd = data.length - data.length % Long.BYTES;+ while (offset < blocksEnd) {+ crc ^= (data[offset] & 0xFFL)+ | (data[offset + 1] & 0xFFL) << 8+ | (data[offset + 2] & 0xFFL) << 16+ | (data[offset + 3] & 0xFFL) << 24+ | (data[offset + 4] & 0xFFL) << 32+ | (data[offset + 5] & 0xFFL) << 40+ | (data[offset + 6] & 0xFFL) << 48+ | (data[offset + 7] & 0xFFL) << 56;+ crc = CRC64_TABLES[7][(int)(crc & 0xFF)]+ ^ CRC64_TABLES[6][(int)((crc >>> 8) & 0xFF)]+ ^ CRC64_TABLES[5][(int)((crc >>> 16) & 0xFF)]+ ^ CRC64_TABLES[4][(int)((crc >>> 24) & 0xFF)]+ ^ CRC64_TABLES[3][(int)((crc >>> 32) & 0xFF)]+ ^ CRC64_TABLES[2][(int)((crc >>> 40) & 0xFF)]+ ^ CRC64_TABLES[1][(int)((crc >>> 48) & 0xFF)]+ ^ CRC64_TABLES[0][(int)(crc >>> 56)];+ offset += Long.BYTES;+ }+ while (offset < data.length) {+ int idx = (int)((crc ^ data[offset]) & 0xFF);+ crc = CRC64_TABLES[0][idx] ^ (crc >>> 8);+ offset++;}crc ^= 0xFFFFFFFFFFFFFFFFL;byte[] bytes = new byte[]{@@ -175,8 +198,8 @@ public class S3Checksum {}}- private static long[] buildCrc64Table() {- long[] table = new long[256];+ private static long[][] buildCrc64Tables() {+ long[][] tables = new long[Long.BYTES][256];for (int i = 0; i < 256; i++) {long crc = i;for (int j = 0; j < 8; j++) {@@ -186,8 +209,15 @@ public class S3Checksum {crc >>>= 1;}}- table[i] = crc;+ tables[0][i] = crc;+ }+ for (int i = 0; i < 256; i++) {+ long crc = tables[0][i];+ for (int n = 1; n < Long.BYTES; n++) {+ crc = tables[0][(int)(crc & 0xFF)] ^ (crc >>> 8);+ tables[n][i] = crc;+ }}- return table;+ return tables;}}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.javaindex a2210da56b..6a61ee5eb7 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ChecksumTest.java@@ -6,8 +6,12 @@ import io.github.hectorvent.floci.services.s3.model.ChecksumType;import io.github.hectorvent.floci.services.s3.model.S3Checksum;import org.junit.jupiter.api.Test;+import java.nio.ByteBuffer;import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Base64;import java.util.List;+import java.util.Random;import static org.junit.jupiter.api.Assertions.*;@@ -65,6 +69,37 @@ class S3ChecksumTest {assertEquals(S3Checksum.crc64NvmeBase64(data), ChecksumAlgorithm.CRC64NVME.compute(data));}+ @Test+ void crc64NvmeMatchesTheCatalogueCheckValue() {+ assertEquals("rosUhgp5mIg=", S3Checksum.crc64NvmeBase64("123456789".getBytes(StandardCharsets.UTF_8)));+ }++ @Test+ void crc64NvmeMatchesABitwiseReferenceAtEveryLength() {+ Random random = new Random(64);+ List<Integer> lengths = new ArrayList<>();+ for (int length = 0; length <= 64; length++) {+ lengths.add(length);+ }+ lengths.addAll(List.of(1023, 1024, 1025, 65537, 1048579));+ for (int length : lengths) {+ byte[] data = new byte[length];+ random.nextBytes(data);+ assertEquals(bitwiseCrc64Nvme(data), S3Checksum.crc64NvmeBase64(data), "length " + length);+ }+ }++ private static String bitwiseCrc64Nvme(byte[] data) {+ long crc = ~0L;+ for (byte b : data) {+ crc ^= b & 0xFFL;+ for (int bit = 0; bit < 8; bit++) {+ crc = (crc & 1) != 0 ? (crc >>> 1) ^ 0x9a6c9329ac4bc9b5L : crc >>> 1;+ }+ }+ return Base64.getEncoder().encodeToString(ByteBuffer.allocate(Long.BYTES).putLong(~crc).array());+ }+@Testvoid objectWithoutDeclaredAlgorithmGetsCrc64Nvme() {byte[] data = "hello".getBytes(StandardCharsets.UTF_8);