Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

s3api: fix ListObjectsV2 dropping objects under a partial prefix (#10656)

hsdfat0dd33bd7ec01
diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go
index 11d8c9fa13..23785cb314 100644
--- a/weed/s3api/s3api_object_handlers_list.go
+++ b/weed/s3api/s3api_object_handlers_list.go
@@ -301,7 +301,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
// check filer
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
var lastEntryWasCommonPrefix bool
- var lastCommonPrefixName string
+ var lastCommonPrefix string
// Hoist versioning check out of per-entry callback
versioningState, _ := s3a.getVersioningState(bucket)
@@ -361,7 +361,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
delimitedPrefix := originalPrefix + delimitedPath[0] + delimiter
// Check if this CommonPrefix already exists
- if !lastEntryWasCommonPrefix || lastCommonPrefixName != delimitedPath[0] {
+ if !lastEntryWasCommonPrefix || lastCommonPrefix != delimitedPrefix {
// New CommonPrefix found
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: delimitedPrefix,
@@ -369,7 +369,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
cursor.maxKeys--
delimiterFound = true
lastEntryWasCommonPrefix = true
- lastCommonPrefixName = delimitedPath[0]
+ lastCommonPrefix = delimitedPrefix
} else {
// This directory object belongs to an existing CommonPrefix, skip it
delimiterFound = true
@@ -391,13 +391,14 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
} else if delimiter != "" { // A response can contain CommonPrefixes only if you specify a delimiter.
// Use raw dir and entry.Name (not encoded) to ensure consistent handling
// Encoding will be applied after sorting if encodingTypeUrl is set
+ dirPrefix := fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):]
commonPrefixes = append(commonPrefixes, PrefixEntry{
- Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
+ Prefix: dirPrefix,
})
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
lastEntryWasCommonPrefix = true
- lastCommonPrefixName = entry.Name
+ lastCommonPrefix = dirPrefix
}
} else {
var delimiterFound bool
@@ -429,7 +430,7 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
cursor.maxKeys--
delimiterFound = true
lastEntryWasCommonPrefix = true
- lastCommonPrefixName = delimitedPath[0]
+ lastCommonPrefix = delimitedPrefix
} else {
// This object belongs to an existing CommonPrefix, skip it
// but continue processing to maintain correct flow
@@ -454,9 +455,8 @@ func (s3a *S3ApiServer) listFilerEntries(ctx context.Context, req listObjectsReq
return doErr
}
- // Adjust nextMarker for CommonPrefixes to include trailing slash (AWS S3 compliance)
if cursor.isTruncated {
- nextMarker = buildTruncatedNextMarker(requestDir, prefix, nextMarker, lastEntryWasCommonPrefix, lastCommonPrefixName)
+ nextMarker = buildTruncatedNextMarker(requestDir, nextMarker, lastEntryWasCommonPrefix, lastCommonPrefix)
}
if cursor.isTruncated {
@@ -535,13 +535,9 @@ func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, al
// something wrong
return "", prefix, marker
}
- if strings.HasPrefix(marker, prefix+"/") {
- alignedDir = prefix
- alignedPrefix = ""
- alignedMarker = marker[len(alignedDir)+1:]
- return
- }
-
+ // Resolve the listing dir from the prefix, not the marker: a partial name prefix like
+ // "data/a" also matches siblings such as "data/ab/", which narrowing to the marker's
+ // subtree would drop.
alignedDir, alignedPrefix = toDirAndName(prefix)
if alignedDir != "" {
alignedMarker = marker[len(alignedDir)+1:]
@@ -571,19 +567,12 @@ func toParentAndDescendants(dirAndName string) (dir, name string) {
return
}
-func buildTruncatedNextMarker(requestDir, prefix, nextMarker string, lastEntryWasCommonPrefix bool, lastCommonPrefixName string) string {
- if lastEntryWasCommonPrefix && lastCommonPrefixName != "" {
- // For CommonPrefixes, NextMarker should include the trailing slash
- if requestDir != "" {
- if prefix != "" {
- return requestDir + "/" + prefix + "/" + lastCommonPrefixName + "/"
- }
- return requestDir + "/" + lastCommonPrefixName + "/"
- }
- if prefix != "" {
- return prefix + "/" + lastCommonPrefixName + "/"
- }
- return lastCommonPrefixName + "/"
+func buildTruncatedNextMarker(requestDir, nextMarker string, lastEntryWasCommonPrefix bool, lastCommonPrefix string) string {
+ // The emitted CommonPrefix is already the full key path with its trailing delimiter.
+ // Rebuilding it from requestDir plus the listing prefix breaks when that prefix is a
+ // partial name, not a directory.
+ if lastEntryWasCommonPrefix && lastCommonPrefix != "" {
+ return lastCommonPrefix
}
if requestDir != "" {
diff --git a/weed/s3api/s3api_object_handlers_list_test.go b/weed/s3api/s3api_object_handlers_list_test.go
index 8d5d990cd9..9cffaee857 100644
--- a/weed/s3api/s3api_object_handlers_list_test.go
+++ b/weed/s3api/s3api_object_handlers_list_test.go
@@ -207,6 +207,27 @@ func Test_normalizePrefixMarker(t *testing.T) {
"parent",
"parentDir/data/0e/0e149049a2137b0cc12e",
},
+ {"partial name prefix, marker resumes inside a matching subdirectory",
+ args{"data/a",
+ "data/a/1"},
+ "data",
+ "a",
+ "a/1",
+ },
+ {"partial name prefix, marker resumes inside a matching sibling directory",
+ args{"data/a",
+ "data/ab/1"},
+ "data",
+ "a",
+ "ab/1",
+ },
+ {"top-level partial name prefix, marker resumes inside a matching subdirectory",
+ args{"a",
+ "a/1"},
+ "",
+ "a",
+ "a/1",
+ },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -220,22 +241,26 @@ func Test_normalizePrefixMarker(t *testing.T) {
func TestBuildTruncatedNextMarker(t *testing.T) {
t.Run("does not duplicate prefix segment in next continuation token", func(t *testing.T) {
- prefix := "export_2026-02-10_17-00-23"
nextMarker := "export_2026-02-10_17-00-23/4156000e.jpg"
- actual := buildTruncatedNextMarker("xemu", prefix, nextMarker, false, "")
+ actual := buildTruncatedNextMarker("xemu", nextMarker, false, "")
assert.Equal(t, "xemu/export_2026-02-10_17-00-23/4156000e.jpg", actual)
})
t.Run("keeps common prefix marker trailing slash", func(t *testing.T) {
- actual := buildTruncatedNextMarker("xemu", "export_2026-02-10_17-00-23", "", true, "nested")
+ actual := buildTruncatedNextMarker("xemu", "", true, "xemu/export_2026-02-10_17-00-23/nested/")
assert.Equal(t, "xemu/export_2026-02-10_17-00-23/nested/", actual)
})
- t.Run("includes prefix for common prefix marker when request dir is empty", func(t *testing.T) {
- actual := buildTruncatedNextMarker("", "foo", "", true, "bar")
+ t.Run("keeps common prefix marker when request dir is empty", func(t *testing.T) {
+ actual := buildTruncatedNextMarker("", "", true, "foo/bar/")
assert.Equal(t, "foo/bar/", actual)
})
+
+ t.Run("does not fold a partial name prefix into the common prefix marker", func(t *testing.T) {
+ actual := buildTruncatedNextMarker("data", "", true, "data/ab/")
+ assert.Equal(t, "data/ab/", actual)
+ })
}
func TestAllowUnorderedParameterValidation(t *testing.T) {