"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fstests: write chunks of the size the chunk writer asks for
Roman Butorc79a1d9d4971
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.goindex a0fd66254..1c08ea3d9 100644--- a/fstest/fstests/fstests.go+++ b/fstest/fstests/fstests.go@@ -838,6 +838,10 @@ func Run(t *testing.T, opt *Opt) {// TestFsOpenChunkWriter tests writing in chunks to fs// then reads back the contents and check if they match+ //+ // The chunks are the size the chunk writer asks for, the+ // last one short, as a multi-thread copy writes them+ //// go test -v -run 'TestIntegration/FsMkdir/FsOpenChunkWriter't.Run("FsOpenChunkWriter", func(t *testing.T) {skipIfNotOk(t)@@ -849,35 +853,64 @@ func Run(t *testing.T, opt *Opt) {size1MB := 1 * 1024 * 1024totalSize := int64(size5MBs*2 + size1MB)+ // Ask for small chunks where the backend lets us so the+ // file takes several of them+ if setUploadChunkSizer, ok := f.(SetUploadChunkSizer); ok {+ oldChunkSize, err := setUploadChunkSizer.SetUploadChunkSize(fs.SizeSuffix(size5MBs))+ if err != nil {+ t.Logf("Can't set the chunk size to %v: %v", fs.SizeSuffix(size5MBs), err)+ } else {+ defer func() {+ _, err := setUploadChunkSizer.SetUploadChunkSize(oldChunkSize)+ assert.NoError(t, err)+ }()+ }+ }+path := "writer-at-subdir/writer-at-file"- objSrc := object.NewStaticObjectInfo(path+"-WRONG-REMOTE", file1.ModTime, totalSize, true, nil, nil)- _, out, err := openChunkWriter(ctx, path, objSrc, &fs.ChunkOption{- ChunkSize: int64(size5MBs),- })+ open := func(size int64) (fs.ChunkWriterInfo, fs.ChunkWriter, error) {+ objSrc := object.NewStaticObjectInfo(path+"-WRONG-REMOTE", file1.ModTime, size, true, nil, nil)+ return openChunkWriter(ctx, path, objSrc, &fs.ChunkOption{+ ChunkSize: int64(size5MBs),+ })+ }+ info, out, err := open(totalSize)if errors.Is(err, fs.ErrorFileTooSmall) {t.Skipf("file too small for multipart upload: %v", err)}require.NoError(t, err)+ require.Greater(t, info.ChunkSize, int64(0), "chunk writer returned no chunk size")++ // A backend whose chunk size couldn't be made small takes the+ // file in fewer than three chunks, so start again with a file+ // of two full chunks and a short one - unless the chunks are+ // too big to hold in memory+ if 2*info.ChunkSize >= totalSize && info.ChunkSize <= int64(128*fs.Mebi) {+ require.NoError(t, out.Abort(ctx))+ t.Logf("Chunk size %v leaves the file in fewer than three chunks, writing %v instead", fs.SizeSuffix(info.ChunkSize), fs.SizeSuffix(2*info.ChunkSize+int64(size1MB)))+ totalSize = 2*info.ChunkSize + int64(size1MB)+ info, out, err = open(totalSize)+ require.NoError(t, err)+ }- contents1 := random.String(size5MBs)- contents2 := random.String(size5MBs)- contents3 := random.String(size1MB)+ // Cut the file into chunks of the size the writer asked for+ var contents []string+ for offset := int64(0); offset < totalSize; offset += info.ChunkSize {+ contents = append(contents, random.String(int(min(info.ChunkSize, totalSize-offset))))+ }- var n int64- n, err = out.WriteChunk(ctx, 1, strings.NewReader(contents2))- assert.NoError(t, err)- assert.Equal(t, int64(size5MBs), n)- n, err = out.WriteChunk(ctx, 2, strings.NewReader(contents3))- assert.NoError(t, err)- assert.Equal(t, int64(size1MB), n)- n, err = out.WriteChunk(ctx, 0, strings.NewReader(contents1))- assert.NoError(t, err)- assert.Equal(t, int64(size5MBs), n)+ // Write the chunks out of order, chunk 0 last+ for i := 1; i <= len(contents); i++ {+ chunkNumber := i % len(contents)+ n, err := out.WriteChunk(ctx, chunkNumber, strings.NewReader(contents[chunkNumber]))+ assert.NoError(t, err)+ assert.Equal(t, int64(len(contents[chunkNumber])), n)+ }assert.NoError(t, out.Close(ctx))obj := fstest.NewObject(ctx, t, f, path)- originalContents := contents1 + contents2 + contents3+ originalContents := strings.Join(contents, "")fileContents := ReadObject(ctx, t, obj, -1)isEqual := originalContents == fileContentsassert.True(t, isEqual, "contents of file differ")