Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): url-encode event record keys and add sequencer (#5192)
Salil Chincholikard0fec13cae55
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 5fd5fbaa79..e07019ec87 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -79,6 +79,7 @@ import java.util.Optional;import java.util.Set;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicLong;import java.util.concurrent.locks.Lock;import java.util.concurrent.locks.ReentrantLock;import java.util.concurrent.locks.ReentrantReadWriteLock;@@ -119,6 +120,8 @@ public class S3Service implements Resettable, ResourceProvider {}private static final Logger LOG = Logger.getLogger(S3Service.class);+ /** Last {@code s3.object.sequencer} handed out, in microseconds since the epoch. */+ private final AtomicLong lastEventSequencer = new AtomicLong();record RequestAuthorization(boolean signed, String accessKeyId, String sessionToken) {static RequestAuthorization unsigned() {@@ -937,6 +940,9 @@ public class S3Service implements Resettable, ResourceProvider {putObjectForAccount(bucketOwnerAccount, objectKey(bucketName, key), object);LOG.debugv("Put object: {0}/{1} ({2} bytes)", bucketName, key, body.size());}+ // Still under the bucket lock, so the sequencers of concurrent writes to one key follow+ // the order they were stored in, whenever their notifications are built.+ object.setEventSequencer(nextEventSequencer());return object;}@@ -2659,6 +2665,7 @@ public class S3Service implements Resettable, ResourceProvider {Bucket bucket = requireBucket(bucketName);S3Object[] notificationTarget = {null};ObjectAnnotation annotation;+ String sequencer;synchronized (bucket) {versionId = normalizeNullVersionId(versionId);S3Object parent = resolveParentObject(bucketName, key, versionId);@@ -2702,10 +2709,11 @@ public class S3Service implements Resettable, ResourceProvider {annotationStore.put(storeKey, annotation);LOG.debugv("Put annotation {0} on object: {1}/{2}", annotationName, bucketName, key);notificationTarget[0] = parent;+ sequencer = nextEventSequencer();}// Fired outside the bucket monitor (the storeObject callers' pattern): a slow SQS/SNS/// Lambda delivery must not block every other write and annotation op on the bucket.- fireNotifications(bucketName, key, "ObjectAnnotation:Put", notificationTarget[0]);+ fireNotifications(bucketName, key, "ObjectAnnotation:Put", notificationTarget[0], sequencer);return annotation;}@@ -2772,6 +2780,7 @@ public class S3Service implements Resettable, ResourceProvider {Bucket bucket = requireBucket(bucketName);S3Object[] notificationTarget = {null};String parentVersionId;+ String sequencer;synchronized (bucket) {versionId = normalizeNullVersionId(versionId);S3Object parent = resolveParentObject(bucketName, key, versionId);@@ -2797,9 +2806,10 @@ public class S3Service implements Resettable, ResourceProvider {LOG.debugv("Deleted annotation {0} from object: {1}/{2}", annotationName, bucketName, key);parentVersionId = parent.getVersionId();notificationTarget[0] = parent;+ sequencer = nextEventSequencer();}// Fired outside the bucket monitor, as in putObjectAnnotation.- fireNotifications(bucketName, key, "ObjectAnnotation:Delete", notificationTarget[0]);+ fireNotifications(bucketName, key, "ObjectAnnotation:Delete", notificationTarget[0], sequencer);return parentVersionId;}@@ -5364,6 +5374,15 @@ public class S3Service implements Resettable, ResourceProvider {}private void fireNotifications(String bucketName, String key, String eventName, S3Object obj) {+ fireNotifications(bucketName, key, eventName, obj, eventSequencer(eventName, obj));+ }++ /**+ * Fires the event with a {@code sequencer} the caller took under the bucket lock, so it orders+ * against other writes to the key the way they were applied, however late the event is built.+ */+ private void fireNotifications(String bucketName, String key, String eventName, S3Object obj,+ String sequencer) {if (s3UpdatedEvent != null && eventName.startsWith("ObjectCreated")) {s3UpdatedEvent.fire(new S3ObjectUpdatedEvent(bucketName, key));}@@ -5381,7 +5400,8 @@ public class S3Service implements Resettable, ResourceProvider {}String region = bucket.getRegion();- String eventJson = buildS3EventJson(bucketName, key, eventName, obj, region, bucket.isVersioningEnabled());+ String eventJson = buildS3EventJson(bucketName, key, eventName, obj, region, bucket.isVersioningEnabled(),+ sequencer);for (QueueNotification qn : config.getQueueConfigurations()) {if (qn.events().stream().anyMatch(p -> matchesEvent(p, eventName)) && qn.matchesKey(key)) {@@ -5534,7 +5554,7 @@ public class S3Service implements Resettable, ResourceProvider {}private String buildS3EventJson(String bucketName, String key, String eventName,- S3Object obj, String region, boolean isVersionEnabled) {+ S3Object obj, String region, boolean isVersionEnabled, String sequencer) {try {String eventTime = DateTimeFormatter.ISO_INSTANT.format(Instant.now());long size = obj != null ? obj.getSize() : 0;@@ -5546,13 +5566,14 @@ public class S3Service implements Resettable, ResourceProvider {bucketNode.put("arn", AwsArnUtils.Arn.global(bucketPartition(bucketName), "s3", "", bucketName).toString());ObjectNode objectNode = objectMapper.createObjectNode();- objectNode.put("key", key);+ objectNode.put("key", eventRecordKey(key));objectNode.put("size", size);objectNode.put("eTag", eTag);if(isVersionEnabled) {String versionId = obj !=null && obj.getVersionId()!=null ? obj.getVersionId() : "";objectNode.put("versionId", versionId);}+ objectNode.put("sequencer", sequencer);ObjectNode s3Node = objectMapper.createObjectNode();s3Node.put("s3SchemaVersion", "1.0");s3Node.put("configurationId", "emulator");@@ -5578,6 +5599,41 @@ public class S3Service implements Resettable, ResourceProvider {}}+ /**+ * The key as S3 writes it into an event record: form URL-encoded (a space is {@code +}, a+ * {@code +} is {@code %2B}) with {@code /} left as is, so consumers decode it the documented+ * way, with {@code unquote_plus} or {@code URLDecoder}.+ */+ static String eventRecordKey(String key) {+ if (key == null) {+ return null;+ }+ return URLEncoder.encode(key, StandardCharsets.UTF_8).replace("%2F", "/");+ }++ /**+ * The sequencer a created object was given when it was stored. Any other event names an object+ * that already existed, whose stored sequencer belongs to its own creation, so it gets a new one:+ * the delete paths fire inside the bucket lock, so that value is still taken under it.+ */+ private String eventSequencer(String eventName, S3Object obj) {+ if (eventName.startsWith("ObjectCreated") && obj != null && obj.getEventSequencer() != null) {+ return obj.getEventSequencer();+ }+ return nextEventSequencer();+ }++ /**+ * An 18-digit uppercase hex value that increases with every event this process emits, so two+ * events for one key compare in the order they happened, as S3's sequencer does. It is seeded+ * from the clock, so values keep increasing across a restart too.+ */+ private String nextEventSequencer() {+ long nowMicros = ChronoUnit.MICROS.between(Instant.EPOCH, Instant.now());+ long value = lastEventSequencer.updateAndGet(last -> Math.max(last + 1, nowMicros));+ return String.format(Locale.ROOT, "%018X", value);+ }+private void cleanupMultipart(String uploadId) {multipartUploads.remove(uploadId);if (inMemory) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.javaindex 4b00096795..8ebefd6417 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java@@ -53,6 +53,11 @@ public class S3Object {@JsonIgnoreprivate String dataGeneration;+ // Internal-only s3.object.sequencer of the write that stored this instance, assigned under the+ // bucket lock so concurrent writes to one key get values in the order they were stored.+ @JsonIgnore+ private String eventSequencer;+public S3Object() {this.metadata = new HashMap<>();this.storageClass = "STANDARD";@@ -167,6 +172,9 @@ public class S3Object {public String getDataGeneration() { return dataGeneration; }public void setDataGeneration(String dataGeneration) { this.dataGeneration = dataGeneration; }+ public String getEventSequencer() { return eventSequencer; }+ public void setEventSequencer(String eventSequencer) { this.eventSequencer = eventSequencer; }+public static String computeETag(byte[] data) {try {MessageDigest md = MessageDigest.getInstance("MD5");diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex 97ca39ce5a..dcac975cb9 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -25,6 +25,7 @@ import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.ValueSource;import java.io.IOException;+import java.net.URLDecoder;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.security.MessageDigest;@@ -833,6 +834,98 @@ class S3ServiceTest {assertNull(lambdaInvoker.payload);}+ @Test+ void eventRecordKeyIsFormUrlEncodedLikeS3() throws IOException {+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();+ S3Service service = bucketNotifyingOnCreation(lambdaInvoker, "notif-s3-encoded-key");++ service.putObject("test-bucket", "my folder/a+b=c (1).txt", "v1".getBytes(StandardCharsets.UTF_8),+ "text/plain", null);++ JsonNode record = onlyRecordedS3Event(lambdaInvoker);+ assertEquals("my+folder/a%2Bb%3Dc+%281%29.txt", record.path("s3").path("object").path("key").asText());+ assertEquals("my folder/a+b=c (1).txt", URLDecoder.decode(+ record.path("s3").path("object").path("key").asText(), StandardCharsets.UTF_8));+ }++ @Test+ void successiveEventsForOneKeyCarryIncreasingSequencers() throws IOException {+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();+ S3Service service = bucketNotifyingOnCreation(lambdaInvoker, "notif-s3-sequencer");++ service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);+ String first = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();+ service.putObject("test-bucket", "k.txt", "v2".getBytes(StandardCharsets.UTF_8), "text/plain", null);+ String second = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();++ assertTrue(first.matches("[0-9A-F]{18}"), first);+ assertTrue(second.matches("[0-9A-F]{18}"), second);+ assertTrue(second.compareTo(first) > 0, first + " then " + second);+ }++ @Test+ void createdEventCarriesTheSequencerAssignedWhenTheObjectWasStored() throws IOException {+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();+ S3Service service = bucketNotifyingOnCreation(lambdaInvoker, "notif-s3-sequencer-stored");++ S3Object stored = service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8),+ "text/plain", null);++ assertNotNull(stored.getEventSequencer());+ assertEquals(stored.getEventSequencer(),+ onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText());+ }++ @Test+ void removalOfAStoredVersionGetsANewerSequencerThanItsCreation() throws IOException {+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();+ S3Service service = versionedBucketNotifyingOnRemoval(lambdaInvoker, "notif-s3-sequencer-removal");++ S3Object stored = service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8),+ "text/plain", null);+ service.deleteObject("test-bucket", "k.txt", stored.getVersionId());+ String removed = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();++ assertTrue(removed.compareTo(stored.getEventSequencer()) > 0,+ stored.getEventSequencer() + " then " + removed);+ }++ @Test+ void annotationEventsCarrySequencersAfterTheObjectTheyAnnotate() throws IOException {+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();+ S3Service service = bucketNotifyingOn(lambdaInvoker, "notif-s3-sequencer-annotation",+ "s3:ObjectAnnotation:*");++ S3Object stored = service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8),+ "text/plain", null);+ service.putObjectAnnotation("test-bucket", "k.txt", "note", null,+ "{}".getBytes(StandardCharsets.UTF_8), null, null);+ String put = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();+ service.deleteObjectAnnotation("test-bucket", "k.txt", "note", null, null, false);+ String deleted = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();++ assertTrue(put.compareTo(stored.getEventSequencer()) > 0, stored.getEventSequencer() + " then " + put);+ assertTrue(deleted.compareTo(put) > 0, put + " then " + deleted);+ }++ private S3Service bucketNotifyingOnCreation(RecordingLambdaInvoker lambdaInvoker, String dataDir) {+ return bucketNotifyingOn(lambdaInvoker, dataDir, "s3:ObjectCreated:*");+ }++ private S3Service bucketNotifyingOn(RecordingLambdaInvoker lambdaInvoker, String dataDir, String event) {+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve(dataDir),+ false, lambdaInvoker, new RegionResolver("us-east-1", "000000000000"));+ service.createBucket("test-bucket", "us-east-1");+ NotificationConfiguration config = new NotificationConfiguration();+ config.getLambdaFunctionConfigurations().add(new LambdaNotification(+ "lambda-notif",+ "arn:aws:lambda:us-east-1:000000000000:function:s3-notif-test",+ List.of(event),+ List.of()));+ service.putBucketNotificationConfiguration("test-bucket", config, true);+ return service;+ }+private S3Service versionedBucketNotifyingOnRemoval(RecordingLambdaInvoker lambdaInvoker, String dataDir) {S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve(dataDir),false, lambdaInvoker, new RegionResolver("us-east-1", "000000000000"));