"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
webdav: fix memory leak uploading to Infinite Scale - fixes #10131
jxj6e01cb6e22d5
diff --git a/backend/webdav/tus-uploader.go b/backend/webdav/tus-uploader.goindex ca89085e9..cb74f511b 100644--- a/backend/webdav/tus-uploader.go+++ b/backend/webdav/tus-uploader.go@@ -20,16 +20,9 @@ type Uploader struct {upload *Uploadoffset int64aborted bool- uploadSubs []chan Upload- notifyChan chan booloverridePatchMethod bool}-// NotifyUploadProgress subscribes to progress updates.-func (u *Uploader) NotifyUploadProgress(c chan Upload) {- u.uploadSubs = append(u.uploadSubs, c)-}-func (f *Fs) shouldRetryChunk(ctx context.Context, resp *http.Response, err error, newOff *int64) (bool, error) {if resp == nil {return true, err@@ -156,36 +149,15 @@ func (u *Uploader) UploadChunk(ctx context.Context, cnt int, options ...fs.OpenOu.upload.updateProgress(u.offset)- u.notifyChan <- true-return nil}-// Waits for a signal to broadcast to all subscribers-func (u *Uploader) broadcastProgress() {- for range u.notifyChan {- for _, c := range u.uploadSubs {- c <- *u.upload- }- }-}-// NewUploader creates a new Uploader.func NewUploader(f *Fs, url string, upload *Upload, offset int64) *Uploader {- notifyChan := make(chan bool)-- uploader := &Uploader{- f,- url,- upload,- offset,- false,- nil,- notifyChan,- false,+ return &Uploader{+ fs: f,+ url: url,+ upload: upload,+ offset: offset,}-- go uploader.broadcastProgress()-- return uploader}diff --git a/backend/webdav/tus-uploader_test.go b/backend/webdav/tus-uploader_test.gonew file mode 100644index 000000000..694886d04--- /dev/null+++ b/backend/webdav/tus-uploader_test.go@@ -0,0 +1,113 @@+package webdav++import (+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "runtime"+ "strconv"+ "strings"+ "sync/atomic"+ "testing"+ "time"+ "weak"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func newTusTestUploader(t *testing.T, server *httptest.Server, contents string) *Uploader {+ t.Helper()+ f := &Fs{+ opt: Options{ChunkSize: 4},+ srv: rest.NewClient(server.Client()),+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(0))),+ }+ // Wrap the reader so NewUpload buffers it, as it does for transfer readers.+ upload := NewUpload(io.LimitReader(strings.NewReader(contents), int64(len(contents))), int64(len(contents)), nil, "")+ require.NotNil(t, upload)+ return NewUploader(f, server.URL, upload, 0)+}++func tusTestHandler(t *testing.T, contents string, fail *atomic.Bool) http.HandlerFunc {+ t.Helper()+ return func(w http.ResponseWriter, r *http.Request) {+ assert.Equal(t, "PATCH", r.Method)+ assert.Equal(t, "1.0.0", r.Header.Get("Tus-Resumable"))+ if fail != nil && fail.Load() {+ w.WriteHeader(http.StatusConflict)+ return+ }+ offset, err := strconv.Atoi(r.Header.Get("Upload-Offset"))+ if !assert.NoError(t, err) {+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ body, err := io.ReadAll(r.Body)+ assert.NoError(t, err)+ end := min(offset+4, len(contents))+ assert.Equal(t, contents[offset:end], string(body))+ w.Header().Set("Upload-Offset", strconv.Itoa(end))+ w.WriteHeader(http.StatusNoContent)+ }+}++func TestTusUploadReleasesBuffer(t *testing.T) {+ for _, tc := range []struct {+ name string+ contents string+ fail bool+ }{+ {name: "Success", contents: "abcdefghij"},+ {name: "Error", contents: "abcdefghij", fail: true},+ {name: "Empty"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ var fail atomic.Bool+ fail.Store(tc.fail)+ server := httptest.NewServer(tusTestHandler(t, tc.contents, &fail))+ defer server.Close()++ released := func() weak.Pointer[Upload] {+ uploader := newTusTestUploader(t, server, tc.contents)+ err := uploader.Upload(context.Background())+ if tc.fail {+ require.ErrorIs(t, err, ErrOffsetMismatch)+ } else {+ require.NoError(t, err)+ assert.True(t, uploader.upload.Finished())+ require.NoError(t, uploader.Upload(context.Background()))+ }+ ref := weak.Make(uploader.upload)+ runtime.KeepAlive(uploader)+ return ref+ }()++ require.Eventually(t, func() bool {+ runtime.GC()+ return released.Value() == nil+ }, 5*time.Second, 10*time.Millisecond, "completed upload still retains its buffered contents")+ })+ }+}++func TestTusUploadRetry(t *testing.T) {+ const contents = "abcdefghij"+ var fail atomic.Bool+ fail.Store(true)+ server := httptest.NewServer(tusTestHandler(t, contents, &fail))+ defer server.Close()++ uploader := newTusTestUploader(t, server, contents)+ require.ErrorIs(t, uploader.Upload(context.Background()), ErrOffsetMismatch)+ assert.Zero(t, uploader.upload.Offset())+ fail.Store(false)+ require.NoError(t, uploader.Upload(context.Background()))+ assert.Equal(t, int64(len(contents)), uploader.upload.Offset())+ assert.True(t, uploader.upload.Finished())+ require.NoError(t, uploader.Upload(context.Background()))+}