"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
dropbox: fix chunked uploads of truncated files never finishing - fixes #9704
Nick Craig-Woodbff17664ad2d
diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.goindex 85164bd66..3a48c2310 100644--- a/backend/dropbox/dropbox.go+++ b/backend/dropbox/dropbox.go@@ -2111,6 +2111,13 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *fif err != nil {return nil, err}+ if size >= 0 {+ // Check for sources which truncate early+ expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))+ if in.BytesRead() < expected {+ return nil, fmt.Errorf("expected %d bytes in input, but only read %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)+ }+ }if appendArg.Close {break}diff --git a/backend/dropbox/dropbox_internal_test.go b/backend/dropbox/dropbox_internal_test.goindex 8d65f2229..8c8a9a2f4 100644--- a/backend/dropbox/dropbox_internal_test.go+++ b/backend/dropbox/dropbox_internal_test.go@@ -2,6 +2,7 @@ package dropboximport ("context"+ "errors""io""net/http""net/http/httptest"@@ -13,6 +14,7 @@ import ("github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files""github.com/rclone/rclone/fs""github.com/rclone/rclone/fstest/fstests"+ "github.com/rclone/rclone/lib/batcher""github.com/rclone/rclone/lib/pacer""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -141,6 +143,98 @@ func TestPaperExportRemote(t *testing.T) {assert.Equal(t, "document.md", legacy.Remote())}+// uploadSessionClient is a mock files.ContextClient which records the+// chunked upload calls made to it+type uploadSessionClient struct {+ files.ContextClient+ appends int // number of UploadSessionAppendV2Context calls+ maxAppends int // fail the append after this many calls to stop runaway loops+ bytesWritten int64 // bytes received by UploadSessionAppendV2Context+ finishCalled bool // set if UploadSessionFinishContext was called+}++var errTooManyAppends = errors.New("too many appends - upload looping?")++func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {+ return &files.UploadSessionStartResult{SessionId: "session"}, nil+}++func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {+ // the real client fails the request if the context is cancelled+ if err := ctx.Err(); err != nil {+ return err+ }+ c.appends+++ if c.appends > c.maxAppends {+ return errTooManyAppends+ }+ n, err := io.Copy(io.Discard, content)+ if err != nil {+ return err+ }+ c.bytesWritten += n+ return nil+}++func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {+ c.finishCalled = true+ return &files.FileMetadata{}, nil+}++// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked+func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {+ ctx := context.Background()+ f := &Fs{+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),+ srv: srv,+ }+ f.opt.ChunkSize = chunkSize+ batcherOptions := defaultBatcherOptions+ batcherOptions.Mode = "off"+ var err error+ f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)+ require.NoError(t, err)+ return f+}++func TestUploadChunkedEarlyEOF(t *testing.T) {+ ctx := context.Background()++ t.Run("MultiChunk", func(t *testing.T) {+ // The declared size spans 4 chunks but the source ends after 1.5+ client := &uploadSessionClient{maxAppends: 8}+ f := newUploadTestFs(t, client, 100)+ o := &Object{fs: f, remote: "test.bin"}+ _, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)+ require.Error(t, err)+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)+ assert.False(t, client.finishCalled, "must not commit a truncated upload")+ })++ t.Run("SingleChunk", func(t *testing.T) {+ // The declared size fits in one chunk but the source ends early+ client := &uploadSessionClient{maxAppends: 8}+ f := newUploadTestFs(t, client, 500)+ o := &Object{fs: f, remote: "test.bin"}+ _, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)+ require.Error(t, err)+ assert.ErrorIs(t, err, io.ErrUnexpectedEOF)+ assert.False(t, client.finishCalled, "must not commit a truncated upload")+ })++ t.Run("Complete", func(t *testing.T) {+ // A source which supplies exactly the declared size uploads OK+ client := &uploadSessionClient{maxAppends: 8}+ f := newUploadTestFs(t, client, 100)+ o := &Object{fs: f, remote: "test.bin"}+ entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)+ require.NoError(t, err)+ require.NotNil(t, entry)+ assert.True(t, client.finishCalled)+ assert.Equal(t, int64(250), client.bytesWritten)+ })+}+func (f *Fs) importPaperForTest(t *testing.T) {content := `# test doc