Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

hasher: keep cached hash when only the modtime changes - fixes #6176

Aniket6aed73d2d370
diff --git a/backend/hasher/hasher_internal_test.go b/backend/hasher/hasher_internal_test.go
index 6c6746a6c..d3883a1ac 100644
--- a/backend/hasher/hasher_internal_test.go
+++ b/backend/hasher/hasher_internal_test.go
@@ -122,6 +122,49 @@ func (f *Fs) testUpdateStoresHash(t *testing.T) {
_ = operations.Purge(ctx, f, dirName)
}
+func (f *Fs) testSetModTimeKeepsHash(t *testing.T) {
+ ctx := context.Background()
+
+ const dirName = "setmodtime_hash_1"
+ const fileName = dirName + "/file_setmodtime_1"
+ const longTime = fs.ModTimeNotSupported
+ hashType := f.keepHashes.GetOne()
+
+ // upload a file to hasher via Put
+ src := putFile(ctx, t, f, fileName, "setmodtime content")
+ dst, err := f.NewObject(ctx, fileName)
+ require.NoError(t, err)
+ require.NotNil(t, dst)
+ _ = src
+
+ // verify hash was stored after Put
+ if f.opt.MaxAge <= 0 {
+ t.Skip("hash cache is disabled")
+ }
+ hash1, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+ require.NoError(t, err)
+ require.NotEmpty(t, hash1)
+
+ // change only the modification time, as operations does when it
+ // syncs the modtime of an otherwise unchanged destination file
+ newTime := fstest.Time("2002-03-04T05:06:07.499999999Z")
+ require.NoError(t, dst.SetModTime(ctx, newTime))
+
+ // the cached hash must survive a pure modtime change
+ hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+ assert.NoError(t, err)
+ assert.Equal(t, hash1, hash2, "hash should be kept after SetModTime")
+
+ // and it must be found via the fingerprint-checked lookup too
+ dst2, err := f.NewObject(ctx, fileName)
+ require.NoError(t, err)
+ hashVal, err := dst2.Hash(ctx, hashType)
+ assert.NoError(t, err)
+ assert.Equal(t, hash1, hashVal, "hash should be returned after SetModTime")
+
+ _ = operations.Purge(ctx, f, dirName)
+}
+
// InternalTest dispatches all internal tests
func (f *Fs) InternalTest(t *testing.T) {
if !kv.Supported() {
@@ -129,6 +172,7 @@ func (f *Fs) InternalTest(t *testing.T) {
}
t.Run("UploadFromCrypt", f.testUploadFromCrypt)
t.Run("UpdateStoresHash", f.testUpdateStoresHash)
+ t.Run("SetModTimeKeepsHash", f.testSetModTimeKeepsHash)
}
var _ fstests.InternalTester = (*Fs)(nil)
diff --git a/backend/hasher/kv.go b/backend/hasher/kv.go
index c51c13b0e..98c4738d1 100644
--- a/backend/hasher/kv.go
+++ b/backend/hasher/kv.go
@@ -206,6 +206,34 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
return err
}
+// kvUpdateFingerprint: rewrite the fingerprint of an existing record
+type kvUpdateFingerprint struct {
+ key string
+ oldFp string
+ newFp string
+ age time.Duration
+}
+
+func (op *kvUpdateFingerprint) Do(ctx context.Context, b kv.Bucket) error {
+ data := b.Get([]byte(op.key))
+ if len(data) == 0 {
+ return nil
+ }
+ var r hashRecord
+ if err := r.decode(op.key, data); err != nil {
+ return nil
+ }
+ if r.Fp != op.oldFp || time.Since(r.Created) > op.age {
+ return nil
+ }
+ r.Fp = op.newFp
+ data, err := r.encode(op.key)
+ if err != nil {
+ return err
+ }
+ return b.Put([]byte(op.key), data)
+}
+
// kvDump: dump the database.
// Note: long dump can cause concurrent operations to fail.
type kvDump struct {
diff --git a/backend/hasher/object.go b/backend/hasher/object.go
index 4e33a53d0..d6f7307f9 100644
--- a/backend/hasher/object.go
+++ b/backend/hasher/object.go
@@ -56,6 +56,23 @@ func (o *Object) putHashes(ctx context.Context, rawHashes hashMap) error {
return o.f.putRawHashes(ctx, key, fp, hashes)
}
+// updateFingerprint rewrites the fingerprint of the cached hash
+// record for remote from oldFp to newFp, keeping the hashes. It is
+// used when only the modification time of an object changes, so the
+// cached hashes remain valid under the new fingerprint.
+func (f *Fs) updateFingerprint(ctx context.Context, remote, oldFp, newFp string) error {
+ if oldFp == "" || newFp == "" || oldFp == newFp || f.opt.MaxAge <= 0 {
+ return nil
+ }
+ key := path.Join(f.Fs.Root(), remote)
+ return f.db.Do(true, &kvUpdateFingerprint{
+ key: key,
+ oldFp: oldFp,
+ newFp: newFp,
+ age: time.Duration(f.opt.MaxAge),
+ })
+}
+
// set hashes for a path without any validation
func (f *Fs) putRawHashes(ctx context.Context, key, fp string, hashes operations.HashSums) error {
return f.db.Do(true, &kvPut{
@@ -196,14 +213,27 @@ func (o *Object) Remove(ctx context.Context) error {
}
// SetModTime sets the modification time of the file.
-// Also prunes the cache entry when modtime changes so that
-// touching a file will trigger checksum recalculation even
-// on backends that don't provide modTime with fingerprint.
+//
+// When the fingerprint includes the modtime, the contents have not
+// changed, so the cached hashes are carried over to the fingerprint
+// for the new modtime. Otherwise the cache entry is pruned when the
+// modtime changes so that touching a file will trigger checksum
+// recalculation even on backends that don't provide modTime with
+// fingerprint.
func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
- if mtime != o.Object.ModTime(ctx) {
+ if mtime == o.Object.ModTime(ctx) {
+ return o.Object.SetModTime(ctx, mtime)
+ }
+ if !o.f.fpTime {
_ = o.f.pruneHash(o.Remote())
+ return o.Object.SetModTime(ctx, mtime)
}
- return o.Object.SetModTime(ctx, mtime)
+ oldFp := o.fingerprint(ctx)
+ if err := o.Object.SetModTime(ctx, mtime); err != nil {
+ return err
+ }
+ _ = o.f.updateFingerprint(ctx, o.Remote(), oldFp, o.fingerprint(ctx))
+ return nil
}
// Open opens the file for read.