"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: show directory operations in the --progress display
Nick Craig-Woodf998f5bb999b
diff --git a/fs/operations/operations.go b/fs/operations/operations.goindex b3b1ba5df..656e052af 100644--- a/fs/operations/operations.go+++ b/fs/operations/operations.go@@ -1057,12 +1057,16 @@ func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error {}// Mkdir makes a destination directory or container-func Mkdir(ctx context.Context, f fs.Fs, dir string) error {+func Mkdir(ctx context.Context, f fs.Fs, dir string) (err error) {+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "making directory")+ defer func() {+ tr.Done(ctx, err)+ }()if SkipDestructive(ctx, fs.LogDirName(f, dir), "make directory") {return nil}fs.Infof(fs.LogDirName(f, dir), "Making directory")- err := f.Mkdir(ctx, dir)+ err = f.Mkdir(ctx, dir)if err != nil {err = fs.CountError(ctx, err)return err@@ -1079,6 +1083,10 @@ func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadatif do == nil {return nil, Mkdir(ctx, f, dir)}+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "making directory")+ defer func() {+ tr.Done(ctx, err)+ }()logName := fs.LogDirName(f, dir)if SkipDestructive(ctx, logName, "make directory") {return nil, nil@@ -1132,6 +1140,11 @@ func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) (// TryRmdir removes a container but not if not empty. It doesn't// count errors but may return one.func TryRmdir(ctx context.Context, f fs.Fs, dir string) error {+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "removing directory")+ defer func() {+ // Pass nil error to Done as TryRmdir doesn't count errors+ tr.Done(ctx, nil)+ }()accounting.Stats(ctx).DeletedDirs(1)if SkipDestructive(ctx, fs.LogDirName(f, dir), "remove directory") {return nil@@ -2685,6 +2698,15 @@ func dirName(f fs.Fs, dst fs.Directory, dir string) any {return f}+// Return the best way of describing the directory as a DirEntry for+// the progress display, describing the root directory as the Fs.+func dirTransferEntry(f fs.Fs, dst fs.Directory, dir string) fs.DirEntry {+ if dst != nil && dst.Remote() != "" {+ return dst+ }+ return fs.NewDir(fmt.Sprint(fs.LogDirName(f, dir)), time.Time{})+}+// CopyDirMetadata copies the src directory to dst or f if nil. If dst is nil then it uses// dir as the name of the new directory.//@@ -2693,6 +2715,12 @@ func dirName(f fs.Fs, dst fs.Directory, dir string) any {func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory, err error) {ci := fs.GetConfig(ctx)logName := dirName(f, dst, dir)+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, dst, dir), "updating metadata")+ defer func() {+ // Count the error before Done so it is only counted once+ err = fs.CountError(ctx, err)+ tr.Done(ctx, err)+ }()if SkipDestructive(ctx, logName, "update directory metadata") {return nil, nil}@@ -2754,6 +2782,12 @@ func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, mfs.Debugf(logName, "Skipping set directory modification time as --no-update-dir-modtime is set")return nil, nil}+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, dst, dir), "setting modtime")+ defer func() {+ // Count the error before Done so it is only counted once+ err = fs.CountError(ctx, err)+ tr.Done(ctx, err)+ }()if SkipDestructive(ctx, logName, "set directory modification time") {return nil, nil}diff --git a/fs/operations/operations_internal_test.go b/fs/operations/operations_internal_test.goindex fd85a9819..331481db1 100644--- a/fs/operations/operations_internal_test.go+++ b/fs/operations/operations_internal_test.go@@ -10,7 +10,9 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/object"+ "github.com/rclone/rclone/fstest/mockfs""github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require")func TestSizeDiffers(t *testing.T) {@@ -41,3 +43,25 @@ func TestSizeDiffers(t *testing.T) {assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize))}}++func TestDirTransferEntry(t *testing.T) {+ ctx := context.Background()+ f, err := mockfs.NewFs(ctx, "mock", "", nil)+ require.NoError(t, err)+ dst := fs.NewDir("existing dir", time.Time{})+ root := fs.NewDir("", time.Time{})++ for _, test := range []struct {+ dst fs.Directory+ dir string+ want string+ }{+ {nil, "new dir", "new dir"},+ {nil, "", fmt.Sprint(f)},+ {dst, "ignored", "existing dir"},+ {root, "", fmt.Sprint(f)},+ } {+ got := dirTransferEntry(f, test.dst, test.dir)+ assert.Equal(t, test.want, got.Remote(), fmt.Sprintf("dst=%v, dir=%q", test.dst, test.dir))+ }+}diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.goindex 727972c30..0b6081d57 100644--- a/fs/operations/operations_test.go+++ b/fs/operations/operations_test.go@@ -37,6 +37,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/filter"+ "github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/object"@@ -66,9 +67,11 @@ func TestMkdir(t *testing.T) {ctx := context.Background()r := fstest.NewRun(t)+ checks := accounting.GlobalStats().GetChecks()err := operations.Mkdir(ctx, r.Fremote, "")require.NoError(t, err)fstest.CheckListing(t, r.Fremote, []fstest.Item{})+ assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "Mkdir should be counted as a check")err = operations.Mkdir(ctx, r.Fremote, "")require.NoError(t, err)@@ -1961,17 +1964,22 @@ func TestSetDirModTime(t *testing.T) {ci.NoUpdateDirModTime = false// First try with the directory not existing - should return an error+ errs := accounting.GlobalStats().GetErrors()newDst, err = operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)require.Error(t, err)require.Nil(t, newDst)+ assert.Equal(t, errs+1, accounting.GlobalStats().GetErrors(), "the error should be counted")+ assert.True(t, fserrors.IsCounted(err), "the returned error should be marked as counted")// Then try with the directory existingrequire.NoError(t, r.Fremote.Mkdir(ctx, name))existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)+ checks := accounting.GlobalStats().GetChecks()newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)require.NoError(t, err)require.NotNil(t, newDst)+ assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "SetDirModTime should be counted as a check")// Check the returned directory and one read from the listing// The modtime will only be correct on newDst if it had a SetModTime methoddiff --git a/fs/sync/sync.go b/fs/sync/sync.goindex d53ac741c..096b43a8f 100644--- a/fs/sync/sync.go+++ b/fs/sync/sync.go@@ -1217,7 +1217,6 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {_, err = operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)}if err != nil {- err = fs.CountError(ctx, err)fs.Errorf(item.dir, "Failed to update directory timestamp or metadata: %v", err)errCount.Add(err)}