"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
pikpak: use pooled memory for the gcid hash buffer and spool unknown-size uploads to disk
Nick Craig-Wood2ed0c688f6e2
diff --git a/backend/pikpak/helper.go b/backend/pikpak/helper.goindex 9bc8ea041..ee0327553 100644--- a/backend/pikpak/helper.go+++ b/backend/pikpak/helper.go@@ -1,7 +1,6 @@package pikpakimport (- "bytes""context""crypto/md5""crypto/sha1"@@ -23,6 +22,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/rest")@@ -307,7 +307,9 @@ func readGcid(in io.Reader, size, threshold int64) (gcid string, out io.Reader,cleanup = func() {}// don't cache small files on disk to reduce wear of the disk- if size > threshold {+ //+ // files of unknown size go to disk as they may not fit in memory+ if size < 0 || size > threshold {var tempFile *os.File// create the cache file@@ -340,13 +342,16 @@ func readGcid(in io.Reader, size, threshold int64) (gcid string, out io.Reader,// replace the already read source with a reader of our cached fileout = tempFile} else {- buf := &bytes.Buffer{}- teeReader := io.TeeReader(in, buf)+ rw := multipart.NewRW()+ cleanup = func() {+ _ = rw.Close()+ }+ teeReader := io.TeeReader(in, rw)if gcid, err = calcGcid(teeReader, size); err != nil {return}- out = buf+ out = rw}return}diff --git a/backend/pikpak/helper_internal_test.go b/backend/pikpak/helper_internal_test.gonew file mode 100644index 000000000..1384a756d--- /dev/null+++ b/backend/pikpak/helper_internal_test.go@@ -0,0 +1,46 @@+package pikpak++import (+ "bytes"+ "io"+ "os"+ "testing"++ "github.com/rclone/rclone/lib/pool"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// TestReadGcid checks readGcid hashes and replays its input from a pooled+// buffer for small known sizes, and from a temp file for large or unknown+// sizes, releasing the buffer on cleanup.+func TestReadGcid(t *testing.T) {+ content := bytes.Repeat([]byte("pikpak"), 1024)+ want, err := calcGcid(bytes.NewReader(content), int64(len(content)))+ require.NoError(t, err)++ for _, test := range []struct {+ name string+ size int64+ threshold int64+ wantFile bool+ }{+ {"memory", int64(len(content)), int64(len(content)), false},+ {"file", int64(len(content)), int64(len(content)) - 1, true},+ {"unknown size", -1, int64(len(content)), true},+ } {+ t.Run(test.name, func(t *testing.T) {+ inUse := pool.Global().InUse()+ gcid, out, cleanup, err := readGcid(bytes.NewReader(content), test.size, test.threshold)+ require.NoError(t, err)+ assert.Equal(t, want, gcid)+ _, isFile := out.(*os.File)+ assert.Equal(t, test.wantFile, isFile, "expected temp file %v", test.wantFile)+ got, err := io.ReadAll(out)+ require.NoError(t, err)+ assert.Equal(t, content, got)+ cleanup()+ assert.Equal(t, inUse, pool.Global().InUse(), "buffer should be returned to the pool")+ })+ }+}diff --git a/backend/pikpak/pikpak.go b/backend/pikpak/pikpak.goindex df5c3f499..a765efa06 100644--- a/backend/pikpak/pikpak.go+++ b/backend/pikpak/pikpak.go@@ -224,7 +224,7 @@ Fill in for rclone to use a non root folder as its starting point.Advanced: true,}, {Name: "hash_memory_limit",- Help: "Files bigger than this will be cached on disk to calculate hash if required.",+ Help: "Files bigger than this will be cached on disk to calculate hash if required.\n\nFiles of unknown size are always cached on disk.",Default: fs.SizeSuffix(10 * 1024 * 1024),Advanced: true,}, {