Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): return CommonPrefixes in ListObjects when delimiter is specified (#133)
Beytullah Gunduz845ac8536321
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex d4781d6205..2a7aa9d61a 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -263,20 +263,22 @@ public class S3Controller {}int max = (maxKeys != null && maxKeys > 0) ? maxKeys : 1000;- List<S3Object> objects = s3Service.listObjects(bucket, prefix, delimiter, max);+ S3Service.ListObjectsResult result = s3Service.listObjectsWithPrefixes(bucket, prefix, delimiter, max);+ List<S3Object> objects = result.objects();+ List<String> commonPrefixes = result.commonPrefixes();boolean v2 = "2".equals(listType);XmlBuilder xml = new XmlBuilder().raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").start("ListBucketResult", AwsNamespaces.S3).elem("Name", bucket)- .elem("Prefix", prefix)+ .elem("Prefix", prefix != null ? prefix : "").elem("Delimiter", delimiter).elem("MaxKeys", max);if (v2) {- xml.elem("KeyCount", objects.size());+ xml.elem("KeyCount", objects.size() + commonPrefixes.size());}- xml.elem("IsTruncated", false);+ xml.elem("IsTruncated", result.isTruncated());for (S3Object obj : objects) {xml.start("Contents").elem("Key", obj.getKey())@@ -286,6 +288,11 @@ public class S3Controller {.elem("StorageClass", obj.getStorageClass()).end("Contents");}+ for (String cp : commonPrefixes) {+ xml.start("CommonPrefixes")+ .elem("Prefix", cp)+ .end("CommonPrefixes");+ }xml.end("ListBucketResult");return Response.ok(xml.build()).build();} catch (AwsException e) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 6d1ee64aa4..65f7349d66 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -439,7 +439,13 @@ public class S3Service {}}+ public record ListObjectsResult(List<S3Object> objects, List<String> commonPrefixes, boolean isTruncated) {}+public List<S3Object> listObjects(String bucketName, String prefix, String delimiter, int maxKeys) {+ return listObjectsWithPrefixes(bucketName, prefix, delimiter, maxKeys).objects();+ }++ public ListObjectsResult listObjectsWithPrefixes(String bucketName, String prefix, String delimiter, int maxKeys) {ensureBucketExists(bucketName);String keyPrefix = bucketName + "/";@@ -453,21 +459,59 @@ public class S3Service {.toList();allObjects = new ArrayList<>(allObjects);+ // see https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html+ List<String> commonPrefixes = List.of();+if (delimiter != null && !delimiter.isEmpty()) {- // Filter to only return objects at this level (simulate directory listing)- allObjects = allObjects.stream()- .filter(obj -> {- String remainder = obj.getKey().substring(prefix != null ? prefix.length() : 0);- return !remainder.contains(delimiter);- })- .toList();- }+ Set<String> prefixSet = new LinkedHashSet<>();+ List<S3Object> directObjects = new ArrayList<>();++ for (S3Object obj : allObjects) {+ String remainder = obj.getKey().substring(prefix != null ? prefix.length() : 0);+ int delimIdx = remainder.indexOf(delimiter);+ if (delimIdx >= 0) {+ String cp = (prefix != null ? prefix : "") + remainder.substring(0, delimIdx + delimiter.length());+ prefixSet.add(cp);+ } else {+ directObjects.add(obj);+ }+ }- if (maxKeys > 0 && allObjects.size() > maxKeys) {- allObjects = allObjects.subList(0, maxKeys);+ allObjects = directObjects;+ commonPrefixes = new ArrayList<>(prefixSet);+ Collections.sort(commonPrefixes);+ }++ allObjects.sort(Comparator.comparing(S3Object::getKey));++ // S3 counts both direct objects and common prefixes.+ // Each common prefix group (e.g. "docs/") uses one entry regardless of+ // how many keys it contains. Merge both sorted lists lexicographically+ // and stop at maxKeys to try to match S3 ListObjectsV2 behavior.+ // see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html+ boolean isTruncated = false;+ if (maxKeys > 0) {+ List<S3Object> limitedObjects = new ArrayList<>();+ List<String> limitedPrefixes = new ArrayList<>();+ int count = 0;+ int directObjectCount = 0;+ int commonPrefixCount = 0;+ while (count < maxKeys && (directObjectCount < allObjects.size() || commonPrefixCount < commonPrefixes.size())) {+ String objectKey = directObjectCount < allObjects.size() ? allObjects.get(directObjectCount).getKey() : null;+ String prefixKey = commonPrefixCount < commonPrefixes.size() ? commonPrefixes.get(commonPrefixCount) : null;+ if (objectKey != null && (prefixKey == null || objectKey.compareTo(prefixKey) <= 0)) {+ limitedObjects.add(allObjects.get(directObjectCount++));+ } else {+ limitedPrefixes.add(commonPrefixes.get(commonPrefixCount++));+ }+ count++;+ }+ isTruncated = directObjectCount < allObjects.size() || commonPrefixCount < commonPrefixes.size();+ allObjects = limitedObjects;+ commonPrefixes = limitedPrefixes;}- return allObjects;+ return new ListObjectsResult(allObjects, commonPrefixes, isTruncated);}public S3Object copyObject(String sourceBucket, String sourceKey,diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.javaindex ce9628633f..fa6f97708f 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java@@ -161,6 +161,23 @@ class S3IntegrationTest {@Test@Order(12)+ void listObjectsWithDelimiterReturnsCommonPrefixes() {+ given()+ .queryParam("delimiter", "/")+ .queryParam("list-type", "2")+ .when()+ .get("/test-bucket")+ .then()+ .statusCode(200)+ .body(containsString("<CommonPrefixes>"))+ .body(containsString("<Prefix>data/</Prefix>"))+ .body(containsString("<Key>greeting.txt</Key>"))+ .body(containsString("<KeyCount>2</KeyCount>"))+ .body(containsString("<IsTruncated>false</IsTruncated>"));+ }++ @Test+ @Order(13)void copyObject() {given().header("x-amz-copy-source", "/test-bucket/greeting.txt")@@ -186,7 +203,7 @@ class S3IntegrationTest {}@Test- @Order(13)+ @Order(14)void deleteObject() {given().when()@@ -203,7 +220,7 @@ class S3IntegrationTest {}@Test- @Order(14)+ @Order(15)void deleteNonEmptyBucketFails() {given().when()diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex df72c4e79b..0b27a18d83 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -177,6 +177,43 @@ class S3ServiceTest {assertEquals(2, docs.size());}+ @Test+ void listObjectsWithDelimiterReturnsCommonPrefixes() {+ s3Service.createBucket("test-bucket", "us-east-1");+ s3Service.putObject("test-bucket", "docs/a.txt", "a".getBytes(), null, null);+ s3Service.putObject("test-bucket", "docs/sub/deep.txt", "d".getBytes(), null, null);+ s3Service.putObject("test-bucket", "images/pic.jpg", "img".getBytes(), null, null);+ s3Service.putObject("test-bucket", "root.txt", "r".getBytes(), null, null);++ S3Service.ListObjectsResult result = s3Service.listObjectsWithPrefixes("test-bucket", null, "/", 1000);+ List<String> rootKeys = result.objects().stream().map(S3Object::getKey).toList();+ assertEquals(List.of("root.txt"), rootKeys);+ assertEquals(List.of("docs/", "images/"), result.commonPrefixes());+ assertFalse(result.isTruncated());++ S3Service.ListObjectsResult docsResult = s3Service.listObjectsWithPrefixes("test-bucket", "docs/", "/", 1000);+ List<String> docKeys = docsResult.objects().stream().map(S3Object::getKey).toList();+ assertEquals(List.of("docs/a.txt"), docKeys);+ assertEquals(List.of("docs/sub/"), docsResult.commonPrefixes());+ assertFalse(docsResult.isTruncated());+ }++ @Test+ void listObjectsWithDelimiterRespectsMaxKeysAcrossObjectsAndPrefixes() {+ s3Service.createBucket("test-bucket", "us-east-1");+ s3Service.putObject("test-bucket", "a.txt", "a".getBytes(), null, null);+ s3Service.putObject("test-bucket", "b.txt", "b".getBytes(), null, null);+ s3Service.putObject("test-bucket", "dir1/file.txt", "f1".getBytes(), null, null);+ s3Service.putObject("test-bucket", "dir2/file.txt", "f2".getBytes(), null, null);+ s3Service.putObject("test-bucket", "dir3/file.txt", "f3".getBytes(), null, null);++ S3Service.ListObjectsResult result = s3Service.listObjectsWithPrefixes("test-bucket", null, "/", 3);++ int totalReturned = result.objects().size() + result.commonPrefixes().size();+ assertEquals(3, totalReturned, "combined objects + commonPrefixes must not exceed maxKeys");+ assertTrue(result.isTruncated(), "result should be truncated when maxKeys < total entries");+ }+@Testvoid listObjectsInNonExistentBucketThrows() {assertThrows(AwsException.class, () ->