"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: fix operations/stat for directories wth large parent dirs
Nick Craig-Wood0a44cbff3739
diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.goindex 8e714efbf..0b2bd5a16 100644--- a/fs/operations/lsjson.go+++ b/fs/operations/lsjson.go@@ -319,6 +319,38 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)//// Remove trailing / as rclone listings won't have themremote = strings.TrimRight(remote, "/")++ // For bucket-based backends with ListP, try listing the+ // directory itself first using ListP. This is much cheaper than+ // listing the parent when the parent has many entries.+ //+ // Bucket-based backends don't have directory IDs or other+ // metadata that would be lost by returning a synthetic entry.+ //+ // If the listing returns empty we fall through to listing the+ // parent to distinguish between an empty directory and a+ // non-existent one.+ features := fsrc.Features()+ if features.BucketBased && features.ListP != nil {+ errDirFound := errors.New("directory found")+ err = features.ListP(ctx, remote, func(entries fs.DirEntries) error {+ accounting.Stats(ctx).Listed(int64(len(entries)))+ if len(entries) > 0 {+ return errDirFound+ }+ return nil+ })+ if err == errDirFound {+ return lj.entry(ctx, fs.NewDir(remote, time.Now()))+ } else if err == fs.ErrorDirNotFound {+ return nil, nil+ }+ // Fall through to parent listing+ // - on other errors (err != nil)+ // - empty listing (err == nil)+ }++ // List the parent to find the directory entry with proper metadata.parent := path.Dir(remote)if parent == "." || parent == "/" {parent = ""diff --git a/fs/operations/lsjson_test.go b/fs/operations/lsjson_test.goindex 75d4728ba..e32aeefd4 100644--- a/fs/operations/lsjson_test.go+++ b/fs/operations/lsjson_test.go@@ -2,11 +2,14 @@ package operations_testimport ("context"+ "io""sort"+ "strings""testing""time""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fstest""github.com/stretchr/testify/assert"@@ -404,3 +407,58 @@ func TestStatJSON(t *testing.T) {assert.True(t, err != nil || f.Features().BucketBased, "Need an error for non bucket based backends")})}++// TestStatJSONMemory tests StatJSON against the memory backend.+//+// The memory backend is bucket based and implements ListP, so this+// exercises the fast path in StatJSON which lists the target directory+// itself rather than its (potentially huge) parent. It also checks that+// the entries listed on that path are accounted in the Listed stats.+func TestStatJSONMemory(t *testing.T) {+ ctx := context.Background()+ f, err := fs.NewFs(ctx, ":memory:")+ require.NoError(t, err)++ // Check the memory backend triggers the fast path+ require.True(t, f.Features().BucketBased, "memory backend should be bucket based")+ require.NotNil(t, f.Features().ListP, "memory backend should implement ListP")++ // Put a file into a subdirectory+ _, err = operations.Rcat(ctx, f, "sub/file1", io.NopCloser(strings.NewReader("hello")), t1, nil)+ require.NoError(t, err)++ t.Run("Dir", func(t *testing.T) {+ accounting.GlobalStats().ResetCounters()+ got, err := operations.StatJSON(ctx, f, "sub", &operations.ListJSONOpt{})+ require.NoError(t, err)+ require.NotNil(t, got)+ assert.Equal(t, "sub", got.Path)+ assert.Equal(t, "sub", got.Name)+ assert.True(t, got.IsDir)+ // The fast path lists the directory itself - check the entries+ // it listed were accounted.+ assert.Positive(t, accounting.GlobalStats().Listed(0), "expected the fast path listing to be accounted")+ })++ t.Run("DirWithTrailingSlash", func(t *testing.T) {+ got, err := operations.StatJSON(ctx, f, "sub/", &operations.ListJSONOpt{})+ require.NoError(t, err)+ require.NotNil(t, got)+ assert.Equal(t, "sub", got.Path)+ assert.True(t, got.IsDir)+ })++ t.Run("File", func(t *testing.T) {+ got, err := operations.StatJSON(ctx, f, "sub/file1", &operations.ListJSONOpt{})+ require.NoError(t, err)+ require.NotNil(t, got)+ assert.Equal(t, "sub/file1", got.Path)+ assert.False(t, got.IsDir)+ })++ t.Run("NotFound", func(t *testing.T) {+ got, err := operations.StatJSON(ctx, f, "notfound", &operations.ListJSONOpt{})+ require.NoError(t, err)+ assert.Nil(t, got)+ })+}