"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
dropbox: add ListP interface - #4788
Nick Craig-Wood979c6a573df3
diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.goindex f67a9e261..25cfe2e9d 100644--- a/backend/dropbox/dropbox.go+++ b/backend/dropbox/dropbox.go@@ -47,6 +47,7 @@ import ("github.com/rclone/rclone/fs/config/obscure""github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/lib/batcher""github.com/rclone/rclone/lib/encoder"@@ -834,7 +835,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {// listSharedFolders lists all available shared folders mounted and not mounted// we'll need the id later so we have to return them in original format-func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {+func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) error) (err error) {started := falsevar res *sharing.ListFoldersResultfor {@@ -847,7 +848,7 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, errreturn shouldRetry(ctx, err)})if err != nil {- return nil, err+ return err}started = true} else {@@ -859,15 +860,15 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, errreturn shouldRetry(ctx, err)})if err != nil {- return nil, fmt.Errorf("list continue: %w", err)+ return fmt.Errorf("list continue: %w", err)}}for _, entry := range res.Entries {leaf := f.opt.Enc.ToStandardName(entry.Name)d := fs.NewDir(leaf, time.Time{}).SetID(entry.SharedFolderId)- entries = append(entries, d)+ err = callback(d)if err != nil {- return nil, err+ return err}}if res.Cursor == "" {@@ -875,21 +876,25 @@ func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err}}- return entries, nil+ return nil}// findSharedFolder find the id for a given shared folder name// somewhat annoyingly there is no endpoint to query a shared folder by it's name// so our only option is to iterate over all shared foldersfunc (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {- entries, err := f.listSharedFolders(ctx)- if err != nil {- return "", err- }- for _, entry := range entries {+ errFoundFile := errors.New("found file")+ err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {if entry.(*fs.Dir).Remote() == name {- return entry.(*fs.Dir).ID(), nil+ id = entry.(*fs.Dir).ID()+ return errFoundFile}+ return nil+ })+ if errors.Is(err, errFoundFile) {+ return id, nil+ } else if err != nil {+ return "", err}return "", fs.ErrorDirNotFound}@@ -908,7 +913,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {// listReceivedFiles lists shared the user as access to (note this means individual// files not files contained in shared folders)-func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, err error) {+func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) error) (err error) {started := falsevar res *sharing.ListFilesResultfor {@@ -921,7 +926,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, errreturn shouldRetry(ctx, err)})if err != nil {- return nil, err+ return err}started = true} else {@@ -933,7 +938,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, errreturn shouldRetry(ctx, err)})if err != nil {- return nil, fmt.Errorf("list continue: %w", err)+ return fmt.Errorf("list continue: %w", err)}}for _, entry := range res.Entries {@@ -946,26 +951,33 @@ func (f *Fs) listReceivedFiles(ctx context.Context) (entries fs.DirEntries, errmodTime: *entry.TimeInvited,}if err != nil {- return nil, err+ return err+ }+ err = callback(o)+ if err != nil {+ return err}- entries = append(entries, o)}if res.Cursor == "" {break}}- return entries, nil+ return nil}func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {- files, err := f.listReceivedFiles(ctx)- if err != nil {- return nil, err- }- for _, entry := range files {+ errFoundFile := errors.New("found file")+ err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {if entry.(*Object).remote == name {- return entry.(*Object), nil+ o = entry.(*Object)+ return errFoundFile}+ return nil+ })+ if errors.Is(err, errFoundFile) {+ return o, nil+ } else if err != nil {+ return nil, err}return nil, fs.ErrorObjectNotFound}@@ -980,11 +992,37 @@ func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err er// This should return ErrDirNotFound if the directory isn't// found.func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {+ return list.WithListP(ctx, dir, f)+}++// ListP lists the objects and directories of the Fs starting+// from dir non recursively into out.+//+// dir should be "" to start from the root, and should not+// have trailing slashes.+//+// This should return ErrDirNotFound if the directory isn't+// found.+//+// It should call callback for each tranche of entries read.+// These need not be returned in any particular order. If+// callback returns an error then the listing will stop+// immediately.+func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {+ list := list.NewHelper(callback)if f.opt.SharedFiles {- return f.listReceivedFiles(ctx)+ err := f.listReceivedFiles(ctx, list.Add)+ if err != nil {+ return err+ }+ return list.Flush()}if f.opt.SharedFolders {- return f.listSharedFolders(ctx)+ err := f.listSharedFolders(ctx, list.Add)+ if err != nil {+ return err+ }+ return list.Flush()}root := f.slashRoot@@ -1014,7 +1052,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err eerr = fs.ErrorDirNotFound}}- return nil, err+ return err}started = true} else {@@ -1026,7 +1064,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err ereturn shouldRetry(ctx, err)})if err != nil {- return nil, fmt.Errorf("list continue: %w", err)+ return fmt.Errorf("list continue: %w", err)}}for _, entry := range res.Entries {@@ -1051,14 +1089,20 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err eremote := path.Join(dir, leaf)if folderInfo != nil {d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id)- entries = append(entries, d)+ err = list.Add(d)+ if err != nil {+ return err+ }} else if fileInfo != nil {o, err := f.newObjectWithInfo(ctx, remote, fileInfo)if err != nil {- return nil, err+ return err}if o.(*Object).exportType.listable() {- entries = append(entries, o)+ err = list.Add(o)+ if err != nil {+ return err+ }}}}@@ -1066,7 +1110,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err ebreak}}- return entries, nil+ return list.Flush()}// Put the object@@ -2087,6 +2131,7 @@ var (_ fs.Mover = (*Fs)(nil)_ fs.PublicLinker = (*Fs)(nil)_ fs.DirMover = (*Fs)(nil)+ _ fs.ListPer = (*Fs)(nil)_ fs.Abouter = (*Fs)(nil)_ fs.Shutdowner = &Fs{}_ fs.Object = (*Object)(nil)