"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
azureblob: fix data corruption bug #7590
Nick Craig-Wood03295bbc3ce8
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.goindex 22dc063c9..36bd50e30 100644--- a/backend/azureblob/azureblob.go+++ b/backend/azureblob/azureblob.go@@ -8,6 +8,7 @@ import ("context""crypto/md5""encoding/base64"+ "encoding/binary""encoding/hex""encoding/json""errors"@@ -1985,34 +1986,21 @@ func (rs *readSeekCloser) Close() error {return nil}-// increment the array as LSB binary-func increment(xs *[8]byte) {- for i, digit := range xs {- newDigit := digit + 1- xs[i] = newDigit- if newDigit >= digit {- // exit if no carry- break- }- }-}-// record chunk number and id for Closetype azBlock struct {- chunkNumber int+ chunkNumber uint64id string}// Implements the fs.ChunkWriter interfacetype azChunkWriter struct {- chunkSize int64- size int64- f *Fs- ui uploadInfo- blocksMu sync.Mutex // protects the below- blocks []azBlock // list of blocks for finalize- binaryBlockID [8]byte // block counter as LSB first 8 bytes- o *Object+ chunkSize int64+ size int64+ f *Fs+ ui uploadInfo+ blocksMu sync.Mutex // protects the below+ blocks []azBlock // list of blocks for finalize+ o *Object}// OpenChunkWriter returns the chunk size and a ChunkWriter@@ -2100,13 +2088,14 @@ func (w *azChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, readertransactionalMD5 := md5sum[:]// increment the blockID and save the blocks for finalize- increment(&w.binaryBlockID)- blockID := base64.StdEncoding.EncodeToString(w.binaryBlockID[:])+ var binaryBlockID [8]byte // block counter as LSB first 8 bytes+ binary.LittleEndian.PutUint64(binaryBlockID[:], uint64(chunkNumber))+ blockID := base64.StdEncoding.EncodeToString(binaryBlockID[:])// Save the blockID for the commitw.blocksMu.Lock()w.blocks = append(w.blocks, azBlock{- chunkNumber: chunkNumber,+ chunkNumber: uint64(chunkNumber),id: blockID,})w.blocksMu.Unlock()@@ -2171,9 +2160,20 @@ func (w *azChunkWriter) Close(ctx context.Context) (err error) {return w.blocks[i].chunkNumber < w.blocks[j].chunkNumber})- // Create a list of block IDs+ // Create and check a list of block IDsblockIDs := make([]string, len(w.blocks))for i := range w.blocks {+ if w.blocks[i].chunkNumber != uint64(i) {+ return fmt.Errorf("internal error: expecting chunkNumber %d but got %d", i, w.blocks[i].chunkNumber)+ }+ chunkBytes, err := base64.StdEncoding.DecodeString(w.blocks[i].id)+ if err != nil {+ return fmt.Errorf("internal error: bad block ID: %w", err)+ }+ chunkNumber := binary.LittleEndian.Uint64(chunkBytes)+ if w.blocks[i].chunkNumber != chunkNumber {+ return fmt.Errorf("internal error: expecting decoded chunkNumber %d but got %d", w.blocks[i].chunkNumber, chunkNumber)+ }blockIDs[i] = w.blocks[i].id}diff --git a/backend/azureblob/azureblob_internal_test.go b/backend/azureblob/azureblob_internal_test.goindex 1871fa265..2479c3624 100644--- a/backend/azureblob/azureblob_internal_test.go+++ b/backend/azureblob/azureblob_internal_test.go@@ -17,21 +17,3 @@ func (f *Fs) InternalTest(t *testing.T) {enabled = f.Features().GetTierassert.True(t, enabled)}--func TestIncrement(t *testing.T) {- for _, test := range []struct {- in [8]byte- want [8]byte- }{- {[8]byte{0, 0, 0, 0}, [8]byte{1, 0, 0, 0}},- {[8]byte{0xFE, 0, 0, 0}, [8]byte{0xFF, 0, 0, 0}},- {[8]byte{0xFF, 0, 0, 0}, [8]byte{0, 1, 0, 0}},- {[8]byte{0, 1, 0, 0}, [8]byte{1, 1, 0, 0}},- {[8]byte{0xFF, 0xFF, 0xFF, 0xFE}, [8]byte{0, 0, 0, 0xFF}},- {[8]byte{0xFF, 0xFF, 0xFF, 0xFF}, [8]byte{0, 0, 0, 0, 1}},- {[8]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, [8]byte{0, 0, 0, 0, 0, 0, 0}},- } {- increment(&test.in)- assert.Equal(t, test.want, test.in)- }-}