Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): url-encode event record keys and add sequencer (#5192)

Salil Chincholikard0fec13cae55
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 5fd5fbaa79..e07019ec87 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -79,6 +79,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -119,6 +120,8 @@ public class S3Service implements Resettable, ResourceProvider {
}
private static final Logger LOG = Logger.getLogger(S3Service.class);
+ /** Last {@code s3.object.sequencer} handed out, in microseconds since the epoch. */
+ private final AtomicLong lastEventSequencer = new AtomicLong();
record RequestAuthorization(boolean signed, String accessKeyId, String sessionToken) {
static RequestAuthorization unsigned() {
@@ -937,6 +940,9 @@ public class S3Service implements Resettable, ResourceProvider {
putObjectForAccount(bucketOwnerAccount, objectKey(bucketName, key), object);
LOG.debugv("Put object: {0}/{1} ({2} bytes)", bucketName, key, body.size());
}
+ // Still under the bucket lock, so the sequencers of concurrent writes to one key follow
+ // the order they were stored in, whenever their notifications are built.
+ object.setEventSequencer(nextEventSequencer());
return object;
}
@@ -2659,6 +2665,7 @@ public class S3Service implements Resettable, ResourceProvider {
Bucket bucket = requireBucket(bucketName);
S3Object[] notificationTarget = {null};
ObjectAnnotation annotation;
+ String sequencer;
synchronized (bucket) {
versionId = normalizeNullVersionId(versionId);
S3Object parent = resolveParentObject(bucketName, key, versionId);
@@ -2702,10 +2709,11 @@ public class S3Service implements Resettable, ResourceProvider {
annotationStore.put(storeKey, annotation);
LOG.debugv("Put annotation {0} on object: {1}/{2}", annotationName, bucketName, key);
notificationTarget[0] = parent;
+ sequencer = nextEventSequencer();
}
// Fired outside the bucket monitor (the storeObject callers' pattern): a slow SQS/SNS/
// Lambda delivery must not block every other write and annotation op on the bucket.
- fireNotifications(bucketName, key, "ObjectAnnotation:Put", notificationTarget[0]);
+ fireNotifications(bucketName, key, "ObjectAnnotation:Put", notificationTarget[0], sequencer);
return annotation;
}
@@ -2772,6 +2780,7 @@ public class S3Service implements Resettable, ResourceProvider {
Bucket bucket = requireBucket(bucketName);
S3Object[] notificationTarget = {null};
String parentVersionId;
+ String sequencer;
synchronized (bucket) {
versionId = normalizeNullVersionId(versionId);
S3Object parent = resolveParentObject(bucketName, key, versionId);
@@ -2797,9 +2806,10 @@ public class S3Service implements Resettable, ResourceProvider {
LOG.debugv("Deleted annotation {0} from object: {1}/{2}", annotationName, bucketName, key);
parentVersionId = parent.getVersionId();
notificationTarget[0] = parent;
+ sequencer = nextEventSequencer();
}
// Fired outside the bucket monitor, as in putObjectAnnotation.
- fireNotifications(bucketName, key, "ObjectAnnotation:Delete", notificationTarget[0]);
+ fireNotifications(bucketName, key, "ObjectAnnotation:Delete", notificationTarget[0], sequencer);
return parentVersionId;
}
@@ -5364,6 +5374,15 @@ public class S3Service implements Resettable, ResourceProvider {
}
private void fireNotifications(String bucketName, String key, String eventName, S3Object obj) {
+ fireNotifications(bucketName, key, eventName, obj, eventSequencer(eventName, obj));
+ }
+
+ /**
+ * Fires the event with a {@code sequencer} the caller took under the bucket lock, so it orders
+ * against other writes to the key the way they were applied, however late the event is built.
+ */
+ private void fireNotifications(String bucketName, String key, String eventName, S3Object obj,
+ String sequencer) {
if (s3UpdatedEvent != null && eventName.startsWith("ObjectCreated")) {
s3UpdatedEvent.fire(new S3ObjectUpdatedEvent(bucketName, key));
}
@@ -5381,7 +5400,8 @@ public class S3Service implements Resettable, ResourceProvider {
}
String region = bucket.getRegion();
- String eventJson = buildS3EventJson(bucketName, key, eventName, obj, region, bucket.isVersioningEnabled());
+ String eventJson = buildS3EventJson(bucketName, key, eventName, obj, region, bucket.isVersioningEnabled(),
+ sequencer);
for (QueueNotification qn : config.getQueueConfigurations()) {
if (qn.events().stream().anyMatch(p -> matchesEvent(p, eventName)) && qn.matchesKey(key)) {
@@ -5534,7 +5554,7 @@ public class S3Service implements Resettable, ResourceProvider {
}
private String buildS3EventJson(String bucketName, String key, String eventName,
- S3Object obj, String region, boolean isVersionEnabled) {
+ S3Object obj, String region, boolean isVersionEnabled, String sequencer) {
try {
String eventTime = DateTimeFormatter.ISO_INSTANT.format(Instant.now());
long size = obj != null ? obj.getSize() : 0;
@@ -5546,13 +5566,14 @@ public class S3Service implements Resettable, ResourceProvider {
bucketNode.put("arn", AwsArnUtils.Arn.global(bucketPartition(bucketName), "s3", "", bucketName).toString());
ObjectNode objectNode = objectMapper.createObjectNode();
- objectNode.put("key", key);
+ objectNode.put("key", eventRecordKey(key));
objectNode.put("size", size);
objectNode.put("eTag", eTag);
if(isVersionEnabled) {
String versionId = obj !=null && obj.getVersionId()!=null ? obj.getVersionId() : "";
objectNode.put("versionId", versionId);
}
+ objectNode.put("sequencer", sequencer);
ObjectNode s3Node = objectMapper.createObjectNode();
s3Node.put("s3SchemaVersion", "1.0");
s3Node.put("configurationId", "emulator");
@@ -5578,6 +5599,41 @@ public class S3Service implements Resettable, ResourceProvider {
}
}
+ /**
+ * The key as S3 writes it into an event record: form URL-encoded (a space is {@code +}, a
+ * {@code +} is {@code %2B}) with {@code /} left as is, so consumers decode it the documented
+ * way, with {@code unquote_plus} or {@code URLDecoder}.
+ */
+ static String eventRecordKey(String key) {
+ if (key == null) {
+ return null;
+ }
+ return URLEncoder.encode(key, StandardCharsets.UTF_8).replace("%2F", "/");
+ }
+
+ /**
+ * The sequencer a created object was given when it was stored. Any other event names an object
+ * that already existed, whose stored sequencer belongs to its own creation, so it gets a new one:
+ * the delete paths fire inside the bucket lock, so that value is still taken under it.
+ */
+ private String eventSequencer(String eventName, S3Object obj) {
+ if (eventName.startsWith("ObjectCreated") && obj != null && obj.getEventSequencer() != null) {
+ return obj.getEventSequencer();
+ }
+ return nextEventSequencer();
+ }
+
+ /**
+ * An 18-digit uppercase hex value that increases with every event this process emits, so two
+ * events for one key compare in the order they happened, as S3's sequencer does. It is seeded
+ * from the clock, so values keep increasing across a restart too.
+ */
+ private String nextEventSequencer() {
+ long nowMicros = ChronoUnit.MICROS.between(Instant.EPOCH, Instant.now());
+ long value = lastEventSequencer.updateAndGet(last -> Math.max(last + 1, nowMicros));
+ return String.format(Locale.ROOT, "%018X", value);
+ }
+
private void cleanupMultipart(String uploadId) {
multipartUploads.remove(uploadId);
if (inMemory) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
index 4b00096795..8ebefd6417 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
@@ -53,6 +53,11 @@ public class S3Object {
@JsonIgnore
private String dataGeneration;
+ // Internal-only s3.object.sequencer of the write that stored this instance, assigned under the
+ // bucket lock so concurrent writes to one key get values in the order they were stored.
+ @JsonIgnore
+ private String eventSequencer;
+
public S3Object() {
this.metadata = new HashMap<>();
this.storageClass = "STANDARD";
@@ -167,6 +172,9 @@ public class S3Object {
public String getDataGeneration() { return dataGeneration; }
public void setDataGeneration(String dataGeneration) { this.dataGeneration = dataGeneration; }
+ public String getEventSequencer() { return eventSequencer; }
+ public void setEventSequencer(String eventSequencer) { this.eventSequencer = eventSequencer; }
+
public static String computeETag(byte[] data) {
try {
MessageDigest md = MessageDigest.getInstance("MD5");
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index 97ca39ce5a..dcac975cb9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
+import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.MessageDigest;
@@ -833,6 +834,98 @@ class S3ServiceTest {
assertNull(lambdaInvoker.payload);
}
+ @Test
+ void eventRecordKeyIsFormUrlEncodedLikeS3() throws IOException {
+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();
+ S3Service service = bucketNotifyingOnCreation(lambdaInvoker, "notif-s3-encoded-key");
+
+ service.putObject("test-bucket", "my folder/a+b=c (1).txt", "v1".getBytes(StandardCharsets.UTF_8),
+ "text/plain", null);
+
+ JsonNode record = onlyRecordedS3Event(lambdaInvoker);
+ assertEquals("my+folder/a%2Bb%3Dc+%281%29.txt", record.path("s3").path("object").path("key").asText());
+ assertEquals("my folder/a+b=c (1).txt", URLDecoder.decode(
+ record.path("s3").path("object").path("key").asText(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void successiveEventsForOneKeyCarryIncreasingSequencers() throws IOException {
+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();
+ S3Service service = bucketNotifyingOnCreation(lambdaInvoker, "notif-s3-sequencer");
+
+ service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);
+ String first = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();
+ service.putObject("test-bucket", "k.txt", "v2".getBytes(StandardCharsets.UTF_8), "text/plain", null);
+ String second = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();
+
+ assertTrue(first.matches("[0-9A-F]{18}"), first);
+ assertTrue(second.matches("[0-9A-F]{18}"), second);
+ assertTrue(second.compareTo(first) > 0, first + " then " + second);
+ }
+
+ @Test
+ void createdEventCarriesTheSequencerAssignedWhenTheObjectWasStored() throws IOException {
+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();
+ S3Service service = bucketNotifyingOnCreation(lambdaInvoker, "notif-s3-sequencer-stored");
+
+ S3Object stored = service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8),
+ "text/plain", null);
+
+ assertNotNull(stored.getEventSequencer());
+ assertEquals(stored.getEventSequencer(),
+ onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText());
+ }
+
+ @Test
+ void removalOfAStoredVersionGetsANewerSequencerThanItsCreation() throws IOException {
+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();
+ S3Service service = versionedBucketNotifyingOnRemoval(lambdaInvoker, "notif-s3-sequencer-removal");
+
+ S3Object stored = service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8),
+ "text/plain", null);
+ service.deleteObject("test-bucket", "k.txt", stored.getVersionId());
+ String removed = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();
+
+ assertTrue(removed.compareTo(stored.getEventSequencer()) > 0,
+ stored.getEventSequencer() + " then " + removed);
+ }
+
+ @Test
+ void annotationEventsCarrySequencersAfterTheObjectTheyAnnotate() throws IOException {
+ RecordingLambdaInvoker lambdaInvoker = new RecordingLambdaInvoker();
+ S3Service service = bucketNotifyingOn(lambdaInvoker, "notif-s3-sequencer-annotation",
+ "s3:ObjectAnnotation:*");
+
+ S3Object stored = service.putObject("test-bucket", "k.txt", "v1".getBytes(StandardCharsets.UTF_8),
+ "text/plain", null);
+ service.putObjectAnnotation("test-bucket", "k.txt", "note", null,
+ "{}".getBytes(StandardCharsets.UTF_8), null, null);
+ String put = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();
+ service.deleteObjectAnnotation("test-bucket", "k.txt", "note", null, null, false);
+ String deleted = onlyRecordedS3Event(lambdaInvoker).path("s3").path("object").path("sequencer").asText();
+
+ assertTrue(put.compareTo(stored.getEventSequencer()) > 0, stored.getEventSequencer() + " then " + put);
+ assertTrue(deleted.compareTo(put) > 0, put + " then " + deleted);
+ }
+
+ private S3Service bucketNotifyingOnCreation(RecordingLambdaInvoker lambdaInvoker, String dataDir) {
+ return bucketNotifyingOn(lambdaInvoker, dataDir, "s3:ObjectCreated:*");
+ }
+
+ private S3Service bucketNotifyingOn(RecordingLambdaInvoker lambdaInvoker, String dataDir, String event) {
+ S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve(dataDir),
+ false, lambdaInvoker, new RegionResolver("us-east-1", "000000000000"));
+ service.createBucket("test-bucket", "us-east-1");
+ NotificationConfiguration config = new NotificationConfiguration();
+ config.getLambdaFunctionConfigurations().add(new LambdaNotification(
+ "lambda-notif",
+ "arn:aws:lambda:us-east-1:000000000000:function:s3-notif-test",
+ List.of(event),
+ List.of()));
+ service.putBucketNotificationConfiguration("test-bucket", config, true);
+ return service;
+ }
+
private S3Service versionedBucketNotifyingOnRemoval(RecordingLambdaInvoker lambdaInvoker, String dataDir) {
S3Service service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve(dataDir),
false, lambdaInvoker, new RegionResolver("us-east-1", "000000000000"));