"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
compress: stop buffering the compressibility check output in memory
Nick Craig-Woodda868b06f8e0
diff --git a/backend/compress/compress.go b/backend/compress/compress.goindex 34cf1e9ee..3de158897 100644--- a/backend/compress/compress.go+++ b/backend/compress/compress.go@@ -1125,6 +1125,15 @@ func (o *Object) Remove(ctx context.Context) error {return objErr}+// countingDiscard is an io.Writer which discards its input and counts+// the bytes written+type countingDiscard int64++func (c *countingDiscard) Write(p []byte) (int, error) {+ *c += countingDiscard(len(p))+ return len(p), nil+}+// ReadCloserWrapper combines a Reader and a Closer to a ReadClosertype ReadCloserWrapper struct {io.Readerdiff --git a/backend/compress/gzip_handler.go b/backend/compress/gzip_handler.goindex a5abbf7c8..16a0920e0 100644--- a/backend/compress/gzip_handler.go+++ b/backend/compress/gzip_handler.go@@ -2,7 +2,6 @@ package compressimport ("bufio"- "bytes""context""crypto/md5""encoding/hex"@@ -23,7 +22,7 @@ type gzipModeHandler struct{}// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds// the configured thresholdfunc (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {- var b bytes.Buffer+ var b countingDiscardvar n int64w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)if err != nil {@@ -37,7 +36,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (boolif err != nil {return false, err}- ratio := float64(n) / float64(b.Len())+ ratio := float64(n) / float64(b)return ratio > minCompressionRatio, nil}diff --git a/backend/compress/rcat_internal_test.go b/backend/compress/rcat_internal_test.goindex 616ec7e38..b4b5428ba 100644--- a/backend/compress/rcat_internal_test.go+++ b/backend/compress/rcat_internal_test.go@@ -3,6 +3,7 @@ package compressimport ("bytes""context"+ "crypto/rand""io""testing""time"@@ -112,3 +113,25 @@ func TestRcat(t *testing.T) {})}}++// TestIsCompressible checks the compressibility heuristic on data at+// both ends of the scale for every compression mode which has one.+func TestIsCompressible(t *testing.T) {+ compressible := bytes.Repeat([]byte("compress me "), 4096)+ incompressible := make([]byte, len(compressible))+ _, err := io.ReadFull(rand.Reader, incompressible)+ require.NoError(t, err)+ for name, handler := range map[string]compressionModeHandler{+ "gzip": &gzipModeHandler{},+ "zstd": &zstdModeHandler{},+ } {+ t.Run(name, func(t *testing.T) {+ ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)+ require.NoError(t, err)+ assert.True(t, ok)+ ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)+ require.NoError(t, err)+ assert.False(t, ok)+ })+ }+}diff --git a/backend/compress/zstd_handler.go b/backend/compress/zstd_handler.goindex 43914427b..7cf8d7617 100644--- a/backend/compress/zstd_handler.go+++ b/backend/compress/zstd_handler.go@@ -2,7 +2,6 @@ package compressimport ("bufio"- "bytes""context""crypto/md5""encoding/hex"@@ -23,7 +22,7 @@ type zstdModeHandler struct{}// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds// the configured thresholdfunc (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {- var b bytes.Buffer+ var b countingDiscardvar n int64w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))if err != nil {@@ -37,7 +36,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (boolif err != nil {return false, err}- ratio := float64(n) / float64(b.Len())+ ratio := float64(n) / float64(b)return ratio > minCompressionRatio, nil}