"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pikpak: share upload chunk buffers with the global memory pool
Nick Craig-Woodb6b3a0a485f6
diff --git a/backend/pikpak/multipart.go b/backend/pikpak/multipart.goindex ad453b00a..f6c25e0ca 100644--- a/backend/pikpak/multipart.go+++ b/backend/pikpak/multipart.go@@ -7,7 +7,6 @@ import ("sort""strings""sync"- "time""github.com/aws/aws-sdk-go-v2/aws""github.com/aws/aws-sdk-go-v2/service/s3"@@ -18,38 +17,11 @@ import ("github.com/rclone/rclone/fs/chunksize""github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/lib/atexit"+ "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/pacer"- "github.com/rclone/rclone/lib/pool""golang.org/x/sync/errgroup")-const (- bufferSize = 1024 * 1024 // default size of the pages used in the reader- bufferCacheSize = 64 // max number of buffers to keep in cache- bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long-)--// bufferPool is a global pool of buffers-var (- bufferPool *pool.Pool- bufferPoolOnce sync.Once-)--// get a buffer pool-func getPool() *pool.Pool {- bufferPoolOnce.Do(func() {- ci := fs.GetConfig(context.Background())- // Initialise the buffer pool when used- bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap)- })- return bufferPool-}--// NewRW gets a pool.RW using the multipart pool-func NewRW() *pool.RW {- return pool.NewRW(getPool())-}-// Upload does a multipart upload in parallelfunc (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {// make concurrency machinery@@ -80,7 +52,7 @@ func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {for partNum := int64(0); !finished; partNum++ {// Get a block of memory from the pool and token which limits concurrency.tokens.Get()- rw := NewRW()+ rw := multipart.NewRW()if acc != nil {rw.SetAccounting(acc.AccountRead)}