Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: show directory operations in the --progress display

Nick Craig-Woodf998f5bb999b
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index b3b1ba5df..656e052af 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1057,12 +1057,16 @@ func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error {
}
// Mkdir makes a destination directory or container
-func Mkdir(ctx context.Context, f fs.Fs, dir string) error {
+func Mkdir(ctx context.Context, f fs.Fs, dir string) (err error) {
+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "making directory")
+ defer func() {
+ tr.Done(ctx, err)
+ }()
if SkipDestructive(ctx, fs.LogDirName(f, dir), "make directory") {
return nil
}
fs.Infof(fs.LogDirName(f, dir), "Making directory")
- err := f.Mkdir(ctx, dir)
+ err = f.Mkdir(ctx, dir)
if err != nil {
err = fs.CountError(ctx, err)
return err
@@ -1079,6 +1083,10 @@ func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadat
if do == nil {
return nil, Mkdir(ctx, f, dir)
}
+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "making directory")
+ defer func() {
+ tr.Done(ctx, err)
+ }()
logName := fs.LogDirName(f, dir)
if SkipDestructive(ctx, logName, "make directory") {
return nil, nil
@@ -1132,6 +1140,11 @@ func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) (
// TryRmdir removes a container but not if not empty. It doesn't
// count errors but may return one.
func TryRmdir(ctx context.Context, f fs.Fs, dir string) error {
+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, nil, dir), "removing directory")
+ defer func() {
+ // Pass nil error to Done as TryRmdir doesn't count errors
+ tr.Done(ctx, nil)
+ }()
accounting.Stats(ctx).DeletedDirs(1)
if SkipDestructive(ctx, fs.LogDirName(f, dir), "remove directory") {
return nil
@@ -2685,6 +2698,15 @@ func dirName(f fs.Fs, dst fs.Directory, dir string) any {
return f
}
+// Return the best way of describing the directory as a DirEntry for
+// the progress display, describing the root directory as the Fs.
+func dirTransferEntry(f fs.Fs, dst fs.Directory, dir string) fs.DirEntry {
+ if dst != nil && dst.Remote() != "" {
+ return dst
+ }
+ return fs.NewDir(fmt.Sprint(fs.LogDirName(f, dir)), time.Time{})
+}
+
// CopyDirMetadata copies the src directory to dst or f if nil. If dst is nil then it uses
// dir as the name of the new directory.
//
@@ -2693,6 +2715,12 @@ func dirName(f fs.Fs, dst fs.Directory, dir string) any {
func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory, err error) {
ci := fs.GetConfig(ctx)
logName := dirName(f, dst, dir)
+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, dst, dir), "updating metadata")
+ defer func() {
+ // Count the error before Done so it is only counted once
+ err = fs.CountError(ctx, err)
+ tr.Done(ctx, err)
+ }()
if SkipDestructive(ctx, logName, "update directory metadata") {
return nil, nil
}
@@ -2754,6 +2782,12 @@ func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, m
fs.Debugf(logName, "Skipping set directory modification time as --no-update-dir-modtime is set")
return nil, nil
}
+ tr := accounting.Stats(ctx).NewCheckingTransferNoHistory(dirTransferEntry(f, dst, dir), "setting modtime")
+ defer func() {
+ // Count the error before Done so it is only counted once
+ err = fs.CountError(ctx, err)
+ tr.Done(ctx, err)
+ }()
if SkipDestructive(ctx, logName, "set directory modification time") {
return nil, nil
}
diff --git a/fs/operations/operations_internal_test.go b/fs/operations/operations_internal_test.go
index fd85a9819..331481db1 100644
--- a/fs/operations/operations_internal_test.go
+++ b/fs/operations/operations_internal_test.go
@@ -10,7 +10,9 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/fstest/mockfs"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestSizeDiffers(t *testing.T) {
@@ -41,3 +43,25 @@ func TestSizeDiffers(t *testing.T) {
assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize))
}
}
+
+func TestDirTransferEntry(t *testing.T) {
+ ctx := context.Background()
+ f, err := mockfs.NewFs(ctx, "mock", "", nil)
+ require.NoError(t, err)
+ dst := fs.NewDir("existing dir", time.Time{})
+ root := fs.NewDir("", time.Time{})
+
+ for _, test := range []struct {
+ dst fs.Directory
+ dir string
+ want string
+ }{
+ {nil, "new dir", "new dir"},
+ {nil, "", fmt.Sprint(f)},
+ {dst, "ignored", "existing dir"},
+ {root, "", fmt.Sprint(f)},
+ } {
+ got := dirTransferEntry(f, test.dst, test.dir)
+ assert.Equal(t, test.want, got.Remote(), fmt.Sprintf("dst=%v, dir=%q", test.dst, test.dir))
+ }
+}
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 727972c30..0b6081d57 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -37,6 +37,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/filter"
+ "github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
@@ -66,9 +67,11 @@ func TestMkdir(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
+ checks := accounting.GlobalStats().GetChecks()
err := operations.Mkdir(ctx, r.Fremote, "")
require.NoError(t, err)
fstest.CheckListing(t, r.Fremote, []fstest.Item{})
+ assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "Mkdir should be counted as a check")
err = operations.Mkdir(ctx, r.Fremote, "")
require.NoError(t, err)
@@ -1961,17 +1964,22 @@ func TestSetDirModTime(t *testing.T) {
ci.NoUpdateDirModTime = false
// First try with the directory not existing - should return an error
+ errs := accounting.GlobalStats().GetErrors()
newDst, err = operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)
require.Error(t, err)
require.Nil(t, newDst)
+ assert.Equal(t, errs+1, accounting.GlobalStats().GetErrors(), "the error should be counted")
+ assert.True(t, fserrors.IsCounted(err), "the returned error should be marked as counted")
// Then try with the directory existing
require.NoError(t, r.Fremote.Mkdir(ctx, name))
existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)
+ checks := accounting.GlobalStats().GetChecks()
newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)
require.NoError(t, err)
require.NotNil(t, newDst)
+ assert.Equal(t, checks+1, accounting.GlobalStats().GetChecks(), "SetDirModTime should be counted as a check")
// Check the returned directory and one read from the listing
// The modtime will only be correct on newDst if it had a SetModTime method
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index d53ac741c..096b43a8f 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -1217,7 +1217,6 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
_, err = operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)
}
if err != nil {
- err = fs.CountError(ctx, err)
fs.Errorf(item.dir, "Failed to update directory timestamp or metadata: %v", err)
errCount.Add(err)
}