Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: speed up checksum by hashing files in parallel - fixes #9727

Teddy Tennanta182fcbce8b7
diff --git a/fs/operations/check.go b/fs/operations/check.go
index d4d1eb3fd..683c2626f 100644
--- a/fs/operations/check.go
+++ b/fs/operations/check.go
@@ -481,24 +481,16 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,
return
}
- var err error
tr := accounting.Stats(ctx).NewCheckingTransfer(obj, "hashing")
- defer tr.Done(ctx, err)
if !sumFound {
- err = errors.New("sum not found")
+ err := errors.New("sum not found")
_ = fs.CountError(ctx, err)
fs.Errorf(obj, "%v", err)
c.differences.Add(1)
c.srcFilesMissing.Add(1)
c.report(obj, c.opt.MissingOnSrc, '-')
- return
- }
-
- if !download {
- var objHash string
- objHash, err = obj.Hash(ctx, hashType)
- c.matchSum(ctx, sumHash, objHash, obj, err, hashType)
+ tr.Done(ctx, nil)
return
}
@@ -512,16 +504,21 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,
)
defer func() {
c.matchSum(ctx, sumHash, objHash, obj, err, hashType)
+ tr.Done(ctx, nil)
<-c.tokens // get the token back to free up a slot
c.wg.Done()
}()
+ if !download {
+ objHash, err = obj.Hash(ctx, hashType)
+ return
+ }
if in, err = Open(ctx, obj); err != nil {
return
}
- tr := accounting.Stats(ctx).NewTransfer(obj, nil)
- in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer
+ dlTr := accounting.Stats(ctx).NewTransfer(obj, nil)
+ in = dlTr.Account(ctx, in).WithBuffer() // account and buffer the transfer
defer func() {
- tr.Done(ctx, nil) // will close the stream
+ dlTr.Done(ctx, nil) // will close the stream
}()
hashVals, err2 := hash.StreamTypes(in, hash.NewHashSet(hashType))
if err2 != nil {
diff --git a/fs/operations/check_test.go b/fs/operations/check_test.go
index e6c309277..f843a8cd3 100644
--- a/fs/operations/check_test.go
+++ b/fs/operations/check_test.go
@@ -8,7 +8,9 @@ import (
"io"
"sort"
"strings"
+ "sync"
"testing"
+ "time"
"github.com/rclone/rclone/cmd/bisync/bilib"
"github.com/rclone/rclone/fs"
@@ -16,6 +18,8 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
+ "github.com/rclone/rclone/fstest/mockfs"
+ "github.com/rclone/rclone/fstest/mockobject"
"github.com/rclone/rclone/lib/readers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -541,6 +545,120 @@ func TestCheckSumDownload(t *testing.T) {
testCheckSum(t, true)
}
+// hashProbe records how many hashing operations are running at once. Each
+// operation blocks until want of them are in flight, or until the deadline
+// expires so a serial implementation finishes instead of deadlocking.
+type hashProbe struct {
+ mu sync.Mutex
+ inFlight int
+ max int
+ want int
+ once sync.Once
+ reached chan struct{}
+ timedOut chan struct{}
+}
+
+func newHashProbe(t *testing.T, want int, deadline time.Duration) *hashProbe {
+ p := &hashProbe{
+ want: want,
+ reached: make(chan struct{}),
+ timedOut: make(chan struct{}),
+ }
+ timer := time.AfterFunc(deadline, func() { close(p.timedOut) })
+ t.Cleanup(func() { timer.Stop() })
+ return p
+}
+
+func (p *hashProbe) enter() {
+ p.mu.Lock()
+ p.inFlight++
+ p.max = max(p.max, p.inFlight)
+ full := p.inFlight >= p.want
+ p.mu.Unlock()
+ if full {
+ p.once.Do(func() { close(p.reached) })
+ }
+ select {
+ case <-p.reached:
+ case <-p.timedOut:
+ }
+}
+
+func (p *hashProbe) leave() {
+ p.mu.Lock()
+ p.inFlight--
+ p.mu.Unlock()
+}
+
+func (p *hashProbe) maxInFlight() int {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.max
+}
+
+// probeObject is an object whose hashing is instrumented by a hashProbe
+type probeObject struct {
+ *mockobject.ContentMockObject
+ probe *hashProbe
+}
+
+func (o *probeObject) Hash(ctx context.Context, ht hash.Type) (string, error) {
+ o.probe.enter()
+ defer o.probe.leave()
+ return o.ContentMockObject.Hash(ctx, ht)
+}
+
+func (o *probeObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+ o.probe.enter()
+ defer o.probe.leave()
+ return o.ContentMockObject.Open(ctx, options...)
+}
+
+func testCheckSumConcurrency(t *testing.T, download bool) {
+ const (
+ checkers = 2
+ numFiles = 4
+ sumFile = "test.sum"
+ )
+ hashType := hash.MD5
+
+ ctx, ci := fs.AddConfig(context.Background())
+ ci.Checkers = checkers
+ probe := newHashProbe(t, checkers, 10*time.Second)
+
+ f, err := mockfs.NewFs(ctx, "checkSum", "checkSum", nil)
+ require.NoError(t, err)
+ fsrc := f.(*mockfs.Fs)
+ fsrc.SetHashes(hash.NewHashSet(hashType))
+
+ sums := &bytes.Buffer{}
+ for i := range numFiles {
+ remote := fmt.Sprintf("file%d", i)
+ o := mockobject.New(remote).WithContent([]byte("contents of "+remote), mockobject.SeekModeNone)
+ sum, err := o.Hash(ctx, hashType)
+ require.NoError(t, err)
+ _, _ = fmt.Fprintf(sums, "%s %s\n", sum, remote)
+ fsrc.AddObject(&probeObject{ContentMockObject: o, probe: probe})
+ }
+
+ f, err = mockfs.NewFs(ctx, "sums", "sums", nil)
+ require.NoError(t, err)
+ fsum := f.(*mockfs.Fs)
+ fsum.AddObject(mockobject.New(sumFile).WithContent(sums.Bytes(), mockobject.SeekModeNone))
+
+ opt := operations.CheckOpt{Combined: new(bytes.Buffer)}
+ require.NoError(t, operations.CheckSum(ctx, fsrc, fsum, sumFile, hashType, &opt, download))
+ assert.Equal(t, checkers, probe.maxInFlight(), "hashing was not run --checkers at a time")
+}
+
+func TestCheckSumConcurrency(t *testing.T) {
+ testCheckSumConcurrency(t, false)
+}
+
+func TestCheckSumDownloadConcurrency(t *testing.T) {
+ testCheckSumConcurrency(t, true)
+}
+
func TestApplyTransforms(t *testing.T) {
var (
hashType = hash.MD5