"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
hasher: fix Update not storing hashes in bolt DB after file replacement
Nick Craig-Wood7c803db8f347
diff --git a/backend/hasher/hasher_internal_test.go b/backend/hasher/hasher_internal_test.goindex e289d8f0d..6c6746a6c 100644--- a/backend/hasher/hasher_internal_test.go+++ b/backend/hasher/hasher_internal_test.go@@ -69,12 +69,66 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) {_ = operations.Purge(ctx, f, dirName)}+func (f *Fs) testUpdateStoresHash(t *testing.T) {+ // make a temporary local remote+ tempRoot, err := fstest.LocalRemote()+ require.NoError(t, err)+ defer func() {+ _ = os.RemoveAll(tempRoot)+ }()++ // make a temporary crypt remote as source+ ctx := context.Background()+ pass := obscure.MustObscure("crypt")+ remote := fmt.Sprintf(`:crypt,remote="%s",password="%s":`, tempRoot, pass)+ cryptFs, err := fs.NewFs(ctx, remote)+ require.NoError(t, err)++ const dirName = "update_hash_1"+ const fileName = dirName + "/file_update_1"+ const longTime = fs.ModTimeNotSupported+ hashType := f.keepHashes.GetOne()++ // upload initial file to hasher via Put+ src1 := putFile(ctx, t, cryptFs, fileName, "initial content")+ in1, err := src1.Open(ctx)+ require.NoError(t, err)+ dst, err := f.Put(ctx, in1, src1)+ require.NoError(t, err)+ require.NotNil(t, dst)++ // verify hash was stored after Put+ var hash1 string+ if f.opt.MaxAge > 0 {+ hash1, err = f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)+ assert.NoError(t, err)+ assert.NotEmpty(t, hash1)+ }++ // update the file with new content via Update (this is what sync does for replacements)+ src2 := putFile(ctx, t, cryptFs, fileName, "updated content")+ in2, err := src2.Open(ctx)+ require.NoError(t, err)+ err = dst.Update(ctx, in2, src2)+ require.NoError(t, err)++ // verify hash was stored after Update and is different from the original+ if f.opt.MaxAge > 0 {+ hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)+ assert.NoError(t, err)+ assert.NotEmpty(t, hash2, "hash should be stored after Update")+ assert.NotEqual(t, hash1, hash2, "hash should change after Update with different content")+ }+ _ = operations.Purge(ctx, f, dirName)+}+// InternalTest dispatches all internal testsfunc (f *Fs) InternalTest(t *testing.T) {if !kv.Supported() {t.Skip("hasher is not supported on this OS")}t.Run("UploadFromCrypt", f.testUploadFromCrypt)+ t.Run("UpdateStoresHash", f.testUpdateStoresHash)}var _ fstests.InternalTester = (*Fs)(nil)diff --git a/backend/hasher/object.go b/backend/hasher/object.goindex 1003807c8..4e33a53d0 100644--- a/backend/hasher/object.go+++ b/backend/hasher/object.go@@ -131,10 +131,62 @@ func (o *Object) updateHashes(ctx context.Context) error {return nil}+// wrapInHashingReader wraps in with a hashing reader if needed and+// returns the (possibly wrapped) reader. After the transfer completes+// successfully, call the returned storeHashes function to store the+// computed hashes.+func (f *Fs) wrapInHashingReader(ctx context.Context, in io.Reader, src fs.ObjectInfo) (wrapIn io.Reader, storeHashes func(o *Object)) {+ var (+ common hash.Set+ rehash bool+ hashes hashMap+ )+ if fsrc := src.Fs(); fsrc != nil {+ common = fsrc.Hashes().Overlap(f.keepHashes)+ // Rehash if source does not have all required hashes or hashing is slow+ rehash = fsrc.Features().SlowHash || common != f.keepHashes+ }++ wrapIn = in+ if rehash {+ r, err := f.newHashingReader(ctx, in, func(sums hashMap) {+ hashes = sums+ })+ fs.Debugf(src, "Rehash in-fly due to incomplete or slow source set %v (err: %v)", common, err)+ if err == nil {+ wrapIn = r+ } else {+ rehash = false+ }+ }++ storeHashes = func(o *Object) {+ if !rehash {+ hashes = hashMap{}+ for _, ht := range common.Array() {+ if h, e := src.Hash(ctx, ht); e == nil && h != "" {+ hashes[ht] = h+ }+ }+ }+ if len(hashes) > 0 {+ err := o.putHashes(ctx, hashes)+ fs.Debugf(o, "Applied %d source hashes, err: %v", len(hashes), err)+ }+ }+ return wrapIn, storeHashes+}+// Update the object with the given data, time and size.func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {+ wrapIn, storeHashes := o.f.wrapInHashingReader(ctx, in, src)_ = o.f.pruneHash(src.Remote())- return o.Object.Update(ctx, in, src, options...)+ err := o.Object.Update(ctx, wrapIn, src, options...)+ if err != nil {+ return err+ }+ storeHashes(o)+ return nil}// Remove an object.@@ -189,51 +241,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (r io.ReadC// Put data into the remote path with given modTime and sizefunc (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {- var (- o fs.Object- common hash.Set- rehash bool- hashes hashMap- )- if fsrc := src.Fs(); fsrc != nil {- common = fsrc.Hashes().Overlap(f.keepHashes)- // Rehash if source does not have all required hashes or hashing is slow- rehash = fsrc.Features().SlowHash || common != f.keepHashes- }-- wrapIn := in- if rehash {- r, err := f.newHashingReader(ctx, in, func(sums hashMap) {- hashes = sums- })- fs.Debugf(src, "Rehash in-fly due to incomplete or slow source set %v (err: %v)", common, err)- if err == nil {- wrapIn = r- } else {- rehash = false- }- }-+ wrapIn, storeHashes := f.wrapInHashingReader(ctx, in, src)_ = f.pruneHash(src.Remote())oResult, err := f.Fs.Put(ctx, wrapIn, src, options...)- o, err = f.wrapObject(oResult, err)+ o, err := f.wrapObject(oResult, err)if err != nil {return nil, err}-- if !rehash {- hashes = hashMap{}- for _, ht := range common.Array() {- if h, e := src.Hash(ctx, ht); e == nil && h != "" {- hashes[ht] = h- }- }- }- if len(hashes) > 0 {- err := o.(*Object).putHashes(ctx, hashes)- fs.Debugf(o, "Applied %d source hashes, err: %v", len(hashes), err)- }- return o, err+ storeHashes(o.(*Object))+ return o, nil}type hashingReader struct {