Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): return CommonPrefixes in ListObjects when delimiter is specified (#133)

Beytullah Gunduz845ac8536321
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index d4781d6205..2a7aa9d61a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -263,20 +263,22 @@ public class S3Controller {
}
int max = (maxKeys != null && maxKeys > 0) ? maxKeys : 1000;
- List<S3Object> objects = s3Service.listObjects(bucket, prefix, delimiter, max);
+ S3Service.ListObjectsResult result = s3Service.listObjectsWithPrefixes(bucket, prefix, delimiter, max);
+ List<S3Object> objects = result.objects();
+ List<String> commonPrefixes = result.commonPrefixes();
boolean v2 = "2".equals(listType);
XmlBuilder xml = new XmlBuilder()
.raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
.start("ListBucketResult", AwsNamespaces.S3)
.elem("Name", bucket)
- .elem("Prefix", prefix)
+ .elem("Prefix", prefix != null ? prefix : "")
.elem("Delimiter", delimiter)
.elem("MaxKeys", max);
if (v2) {
- xml.elem("KeyCount", objects.size());
+ xml.elem("KeyCount", objects.size() + commonPrefixes.size());
}
- xml.elem("IsTruncated", false);
+ xml.elem("IsTruncated", result.isTruncated());
for (S3Object obj : objects) {
xml.start("Contents")
.elem("Key", obj.getKey())
@@ -286,6 +288,11 @@ public class S3Controller {
.elem("StorageClass", obj.getStorageClass())
.end("Contents");
}
+ for (String cp : commonPrefixes) {
+ xml.start("CommonPrefixes")
+ .elem("Prefix", cp)
+ .end("CommonPrefixes");
+ }
xml.end("ListBucketResult");
return Response.ok(xml.build()).build();
} catch (AwsException e) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 6d1ee64aa4..65f7349d66 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -439,7 +439,13 @@ public class S3Service {
}
}
+ public record ListObjectsResult(List<S3Object> objects, List<String> commonPrefixes, boolean isTruncated) {}
+
public List<S3Object> listObjects(String bucketName, String prefix, String delimiter, int maxKeys) {
+ return listObjectsWithPrefixes(bucketName, prefix, delimiter, maxKeys).objects();
+ }
+
+ public ListObjectsResult listObjectsWithPrefixes(String bucketName, String prefix, String delimiter, int maxKeys) {
ensureBucketExists(bucketName);
String keyPrefix = bucketName + "/";
@@ -453,21 +459,59 @@ public class S3Service {
.toList();
allObjects = new ArrayList<>(allObjects);
+ // see https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html
+ List<String> commonPrefixes = List.of();
+
if (delimiter != null && !delimiter.isEmpty()) {
- // Filter to only return objects at this level (simulate directory listing)
- allObjects = allObjects.stream()
- .filter(obj -> {
- String remainder = obj.getKey().substring(prefix != null ? prefix.length() : 0);
- return !remainder.contains(delimiter);
- })
- .toList();
- }
+ Set<String> prefixSet = new LinkedHashSet<>();
+ List<S3Object> directObjects = new ArrayList<>();
+
+ for (S3Object obj : allObjects) {
+ String remainder = obj.getKey().substring(prefix != null ? prefix.length() : 0);
+ int delimIdx = remainder.indexOf(delimiter);
+ if (delimIdx >= 0) {
+ String cp = (prefix != null ? prefix : "") + remainder.substring(0, delimIdx + delimiter.length());
+ prefixSet.add(cp);
+ } else {
+ directObjects.add(obj);
+ }
+ }
- if (maxKeys > 0 && allObjects.size() > maxKeys) {
- allObjects = allObjects.subList(0, maxKeys);
+ allObjects = directObjects;
+ commonPrefixes = new ArrayList<>(prefixSet);
+ Collections.sort(commonPrefixes);
+ }
+
+ allObjects.sort(Comparator.comparing(S3Object::getKey));
+
+ // S3 counts both direct objects and common prefixes.
+ // Each common prefix group (e.g. "docs/") uses one entry regardless of
+ // how many keys it contains. Merge both sorted lists lexicographically
+ // and stop at maxKeys to try to match S3 ListObjectsV2 behavior.
+ // see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
+ boolean isTruncated = false;
+ if (maxKeys > 0) {
+ List<S3Object> limitedObjects = new ArrayList<>();
+ List<String> limitedPrefixes = new ArrayList<>();
+ int count = 0;
+ int directObjectCount = 0;
+ int commonPrefixCount = 0;
+ while (count < maxKeys && (directObjectCount < allObjects.size() || commonPrefixCount < commonPrefixes.size())) {
+ String objectKey = directObjectCount < allObjects.size() ? allObjects.get(directObjectCount).getKey() : null;
+ String prefixKey = commonPrefixCount < commonPrefixes.size() ? commonPrefixes.get(commonPrefixCount) : null;
+ if (objectKey != null && (prefixKey == null || objectKey.compareTo(prefixKey) <= 0)) {
+ limitedObjects.add(allObjects.get(directObjectCount++));
+ } else {
+ limitedPrefixes.add(commonPrefixes.get(commonPrefixCount++));
+ }
+ count++;
+ }
+ isTruncated = directObjectCount < allObjects.size() || commonPrefixCount < commonPrefixes.size();
+ allObjects = limitedObjects;
+ commonPrefixes = limitedPrefixes;
}
- return allObjects;
+ return new ListObjectsResult(allObjects, commonPrefixes, isTruncated);
}
public S3Object copyObject(String sourceBucket, String sourceKey,
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
index ce9628633f..fa6f97708f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
@@ -161,6 +161,23 @@ class S3IntegrationTest {
@Test
@Order(12)
+ void listObjectsWithDelimiterReturnsCommonPrefixes() {
+ given()
+ .queryParam("delimiter", "/")
+ .queryParam("list-type", "2")
+ .when()
+ .get("/test-bucket")
+ .then()
+ .statusCode(200)
+ .body(containsString("<CommonPrefixes>"))
+ .body(containsString("<Prefix>data/</Prefix>"))
+ .body(containsString("<Key>greeting.txt</Key>"))
+ .body(containsString("<KeyCount>2</KeyCount>"))
+ .body(containsString("<IsTruncated>false</IsTruncated>"));
+ }
+
+ @Test
+ @Order(13)
void copyObject() {
given()
.header("x-amz-copy-source", "/test-bucket/greeting.txt")
@@ -186,7 +203,7 @@ class S3IntegrationTest {
}
@Test
- @Order(13)
+ @Order(14)
void deleteObject() {
given()
.when()
@@ -203,7 +220,7 @@ class S3IntegrationTest {
}
@Test
- @Order(14)
+ @Order(15)
void deleteNonEmptyBucketFails() {
given()
.when()
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index df72c4e79b..0b27a18d83 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -177,6 +177,43 @@ class S3ServiceTest {
assertEquals(2, docs.size());
}
+ @Test
+ void listObjectsWithDelimiterReturnsCommonPrefixes() {
+ s3Service.createBucket("test-bucket", "us-east-1");
+ s3Service.putObject("test-bucket", "docs/a.txt", "a".getBytes(), null, null);
+ s3Service.putObject("test-bucket", "docs/sub/deep.txt", "d".getBytes(), null, null);
+ s3Service.putObject("test-bucket", "images/pic.jpg", "img".getBytes(), null, null);
+ s3Service.putObject("test-bucket", "root.txt", "r".getBytes(), null, null);
+
+ S3Service.ListObjectsResult result = s3Service.listObjectsWithPrefixes("test-bucket", null, "/", 1000);
+ List<String> rootKeys = result.objects().stream().map(S3Object::getKey).toList();
+ assertEquals(List.of("root.txt"), rootKeys);
+ assertEquals(List.of("docs/", "images/"), result.commonPrefixes());
+ assertFalse(result.isTruncated());
+
+ S3Service.ListObjectsResult docsResult = s3Service.listObjectsWithPrefixes("test-bucket", "docs/", "/", 1000);
+ List<String> docKeys = docsResult.objects().stream().map(S3Object::getKey).toList();
+ assertEquals(List.of("docs/a.txt"), docKeys);
+ assertEquals(List.of("docs/sub/"), docsResult.commonPrefixes());
+ assertFalse(docsResult.isTruncated());
+ }
+
+ @Test
+ void listObjectsWithDelimiterRespectsMaxKeysAcrossObjectsAndPrefixes() {
+ s3Service.createBucket("test-bucket", "us-east-1");
+ s3Service.putObject("test-bucket", "a.txt", "a".getBytes(), null, null);
+ s3Service.putObject("test-bucket", "b.txt", "b".getBytes(), null, null);
+ s3Service.putObject("test-bucket", "dir1/file.txt", "f1".getBytes(), null, null);
+ s3Service.putObject("test-bucket", "dir2/file.txt", "f2".getBytes(), null, null);
+ s3Service.putObject("test-bucket", "dir3/file.txt", "f3".getBytes(), null, null);
+
+ S3Service.ListObjectsResult result = s3Service.listObjectsWithPrefixes("test-bucket", null, "/", 3);
+
+ int totalReturned = result.objects().size() + result.commonPrefixes().size();
+ assertEquals(3, totalReturned, "combined objects + commonPrefixes must not exceed maxKeys");
+ assertTrue(result.isTruncated(), "result should be truncated when maxKeys < total entries");
+ }
+
@Test
void listObjectsInNonExistentBucketThrows() {
assertThrows(AwsException.class, () ->