Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: fix operations/stat for directories wth large parent dirs

Nick Craig-Wood0a44cbff3739
diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go
index 8e714efbf..0b2bd5a16 100644
--- a/fs/operations/lsjson.go
+++ b/fs/operations/lsjson.go
@@ -319,6 +319,38 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)
//
// Remove trailing / as rclone listings won't have them
remote = strings.TrimRight(remote, "/")
+
+ // For bucket-based backends with ListP, try listing the
+ // directory itself first using ListP. This is much cheaper than
+ // listing the parent when the parent has many entries.
+ //
+ // Bucket-based backends don't have directory IDs or other
+ // metadata that would be lost by returning a synthetic entry.
+ //
+ // If the listing returns empty we fall through to listing the
+ // parent to distinguish between an empty directory and a
+ // non-existent one.
+ features := fsrc.Features()
+ if features.BucketBased && features.ListP != nil {
+ errDirFound := errors.New("directory found")
+ err = features.ListP(ctx, remote, func(entries fs.DirEntries) error {
+ accounting.Stats(ctx).Listed(int64(len(entries)))
+ if len(entries) > 0 {
+ return errDirFound
+ }
+ return nil
+ })
+ if err == errDirFound {
+ return lj.entry(ctx, fs.NewDir(remote, time.Now()))
+ } else if err == fs.ErrorDirNotFound {
+ return nil, nil
+ }
+ // Fall through to parent listing
+ // - on other errors (err != nil)
+ // - empty listing (err == nil)
+ }
+
+ // List the parent to find the directory entry with proper metadata.
parent := path.Dir(remote)
if parent == "." || parent == "/" {
parent = ""
diff --git a/fs/operations/lsjson_test.go b/fs/operations/lsjson_test.go
index 75d4728ba..e32aeefd4 100644
--- a/fs/operations/lsjson_test.go
+++ b/fs/operations/lsjson_test.go
@@ -2,11 +2,14 @@ package operations_test
import (
"context"
+ "io"
"sort"
+ "strings"
"testing"
"time"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
@@ -404,3 +407,58 @@ func TestStatJSON(t *testing.T) {
assert.True(t, err != nil || f.Features().BucketBased, "Need an error for non bucket based backends")
})
}
+
+// TestStatJSONMemory tests StatJSON against the memory backend.
+//
+// The memory backend is bucket based and implements ListP, so this
+// exercises the fast path in StatJSON which lists the target directory
+// itself rather than its (potentially huge) parent. It also checks that
+// the entries listed on that path are accounted in the Listed stats.
+func TestStatJSONMemory(t *testing.T) {
+ ctx := context.Background()
+ f, err := fs.NewFs(ctx, ":memory:")
+ require.NoError(t, err)
+
+ // Check the memory backend triggers the fast path
+ require.True(t, f.Features().BucketBased, "memory backend should be bucket based")
+ require.NotNil(t, f.Features().ListP, "memory backend should implement ListP")
+
+ // Put a file into a subdirectory
+ _, err = operations.Rcat(ctx, f, "sub/file1", io.NopCloser(strings.NewReader("hello")), t1, nil)
+ require.NoError(t, err)
+
+ t.Run("Dir", func(t *testing.T) {
+ accounting.GlobalStats().ResetCounters()
+ got, err := operations.StatJSON(ctx, f, "sub", &operations.ListJSONOpt{})
+ require.NoError(t, err)
+ require.NotNil(t, got)
+ assert.Equal(t, "sub", got.Path)
+ assert.Equal(t, "sub", got.Name)
+ assert.True(t, got.IsDir)
+ // The fast path lists the directory itself - check the entries
+ // it listed were accounted.
+ assert.Positive(t, accounting.GlobalStats().Listed(0), "expected the fast path listing to be accounted")
+ })
+
+ t.Run("DirWithTrailingSlash", func(t *testing.T) {
+ got, err := operations.StatJSON(ctx, f, "sub/", &operations.ListJSONOpt{})
+ require.NoError(t, err)
+ require.NotNil(t, got)
+ assert.Equal(t, "sub", got.Path)
+ assert.True(t, got.IsDir)
+ })
+
+ t.Run("File", func(t *testing.T) {
+ got, err := operations.StatJSON(ctx, f, "sub/file1", &operations.ListJSONOpt{})
+ require.NoError(t, err)
+ require.NotNil(t, got)
+ assert.Equal(t, "sub/file1", got.Path)
+ assert.False(t, got.IsDir)
+ })
+
+ t.Run("NotFound", func(t *testing.T) {
+ got, err := operations.StatJSON(ctx, f, "notfound", &operations.ListJSONOpt{})
+ require.NoError(t, err)
+ assert.Nil(t, got)
+ })
+}