"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
hasher: keep cached hash when only the modtime changes - fixes #6176
Aniket6aed73d2d370
diff --git a/backend/hasher/hasher_internal_test.go b/backend/hasher/hasher_internal_test.goindex 6c6746a6c..d3883a1ac 100644--- a/backend/hasher/hasher_internal_test.go+++ b/backend/hasher/hasher_internal_test.go@@ -122,6 +122,49 @@ func (f *Fs) testUpdateStoresHash(t *testing.T) {_ = operations.Purge(ctx, f, dirName)}+func (f *Fs) testSetModTimeKeepsHash(t *testing.T) {+ ctx := context.Background()++ const dirName = "setmodtime_hash_1"+ const fileName = dirName + "/file_setmodtime_1"+ const longTime = fs.ModTimeNotSupported+ hashType := f.keepHashes.GetOne()++ // upload a file to hasher via Put+ src := putFile(ctx, t, f, fileName, "setmodtime content")+ dst, err := f.NewObject(ctx, fileName)+ require.NoError(t, err)+ require.NotNil(t, dst)+ _ = src++ // verify hash was stored after Put+ if f.opt.MaxAge <= 0 {+ t.Skip("hash cache is disabled")+ }+ hash1, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)+ require.NoError(t, err)+ require.NotEmpty(t, hash1)++ // change only the modification time, as operations does when it+ // syncs the modtime of an otherwise unchanged destination file+ newTime := fstest.Time("2002-03-04T05:06:07.499999999Z")+ require.NoError(t, dst.SetModTime(ctx, newTime))++ // the cached hash must survive a pure modtime change+ hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)+ assert.NoError(t, err)+ assert.Equal(t, hash1, hash2, "hash should be kept after SetModTime")++ // and it must be found via the fingerprint-checked lookup too+ dst2, err := f.NewObject(ctx, fileName)+ require.NoError(t, err)+ hashVal, err := dst2.Hash(ctx, hashType)+ assert.NoError(t, err)+ assert.Equal(t, hash1, hashVal, "hash should be returned after SetModTime")++ _ = operations.Purge(ctx, f, dirName)+}+// InternalTest dispatches all internal testsfunc (f *Fs) InternalTest(t *testing.T) {if !kv.Supported() {@@ -129,6 +172,7 @@ func (f *Fs) InternalTest(t *testing.T) {}t.Run("UploadFromCrypt", f.testUploadFromCrypt)t.Run("UpdateStoresHash", f.testUpdateStoresHash)+ t.Run("SetModTimeKeepsHash", f.testSetModTimeKeepsHash)}var _ fstests.InternalTester = (*Fs)(nil)diff --git a/backend/hasher/kv.go b/backend/hasher/kv.goindex c51c13b0e..98c4738d1 100644--- a/backend/hasher/kv.go+++ b/backend/hasher/kv.go@@ -206,6 +206,34 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {return err}+// kvUpdateFingerprint: rewrite the fingerprint of an existing record+type kvUpdateFingerprint struct {+ key string+ oldFp string+ newFp string+ age time.Duration+}++func (op *kvUpdateFingerprint) Do(ctx context.Context, b kv.Bucket) error {+ data := b.Get([]byte(op.key))+ if len(data) == 0 {+ return nil+ }+ var r hashRecord+ if err := r.decode(op.key, data); err != nil {+ return nil+ }+ if r.Fp != op.oldFp || time.Since(r.Created) > op.age {+ return nil+ }+ r.Fp = op.newFp+ data, err := r.encode(op.key)+ if err != nil {+ return err+ }+ return b.Put([]byte(op.key), data)+}+// kvDump: dump the database.// Note: long dump can cause concurrent operations to fail.type kvDump struct {diff --git a/backend/hasher/object.go b/backend/hasher/object.goindex 4e33a53d0..d6f7307f9 100644--- a/backend/hasher/object.go+++ b/backend/hasher/object.go@@ -56,6 +56,23 @@ func (o *Object) putHashes(ctx context.Context, rawHashes hashMap) error {return o.f.putRawHashes(ctx, key, fp, hashes)}+// updateFingerprint rewrites the fingerprint of the cached hash+// record for remote from oldFp to newFp, keeping the hashes. It is+// used when only the modification time of an object changes, so the+// cached hashes remain valid under the new fingerprint.+func (f *Fs) updateFingerprint(ctx context.Context, remote, oldFp, newFp string) error {+ if oldFp == "" || newFp == "" || oldFp == newFp || f.opt.MaxAge <= 0 {+ return nil+ }+ key := path.Join(f.Fs.Root(), remote)+ return f.db.Do(true, &kvUpdateFingerprint{+ key: key,+ oldFp: oldFp,+ newFp: newFp,+ age: time.Duration(f.opt.MaxAge),+ })+}+// set hashes for a path without any validationfunc (f *Fs) putRawHashes(ctx context.Context, key, fp string, hashes operations.HashSums) error {return f.db.Do(true, &kvPut{@@ -196,14 +213,27 @@ func (o *Object) Remove(ctx context.Context) error {}// SetModTime sets the modification time of the file.-// Also prunes the cache entry when modtime changes so that-// touching a file will trigger checksum recalculation even-// on backends that don't provide modTime with fingerprint.+//+// When the fingerprint includes the modtime, the contents have not+// changed, so the cached hashes are carried over to the fingerprint+// for the new modtime. Otherwise the cache entry is pruned when the+// modtime changes so that touching a file will trigger checksum+// recalculation even on backends that don't provide modTime with+// fingerprint.func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {- if mtime != o.Object.ModTime(ctx) {+ if mtime == o.Object.ModTime(ctx) {+ return o.Object.SetModTime(ctx, mtime)+ }+ if !o.f.fpTime {_ = o.f.pruneHash(o.Remote())+ return o.Object.SetModTime(ctx, mtime)}- return o.Object.SetModTime(ctx, mtime)+ oldFp := o.fingerprint(ctx)+ if err := o.Object.SetModTime(ctx, mtime); err != nil {+ return err+ }+ _ = o.f.updateFingerprint(ctx, o.Remote(), oldFp, o.fingerprint(ctx))+ return nil}// Open opens the file for read.