Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

compress: stop buffering the compressibility check output in memory

Nick Craig-Woodda868b06f8e0
diff --git a/backend/compress/compress.go b/backend/compress/compress.go
index 34cf1e9ee..3de158897 100644
--- a/backend/compress/compress.go
+++ b/backend/compress/compress.go
@@ -1125,6 +1125,15 @@ func (o *Object) Remove(ctx context.Context) error {
return objErr
}
+// countingDiscard is an io.Writer which discards its input and counts
+// the bytes written
+type countingDiscard int64
+
+func (c *countingDiscard) Write(p []byte) (int, error) {
+ *c += countingDiscard(len(p))
+ return len(p), nil
+}
+
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
type ReadCloserWrapper struct {
io.Reader
diff --git a/backend/compress/gzip_handler.go b/backend/compress/gzip_handler.go
index a5abbf7c8..16a0920e0 100644
--- a/backend/compress/gzip_handler.go
+++ b/backend/compress/gzip_handler.go
@@ -2,7 +2,6 @@ package compress
import (
"bufio"
- "bytes"
"context"
"crypto/md5"
"encoding/hex"
@@ -23,7 +22,7 @@ type gzipModeHandler struct{}
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
// the configured threshold
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
- var b bytes.Buffer
+ var b countingDiscard
var n int64
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
if err != nil {
@@ -37,7 +36,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
if err != nil {
return false, err
}
- ratio := float64(n) / float64(b.Len())
+ ratio := float64(n) / float64(b)
return ratio > minCompressionRatio, nil
}
diff --git a/backend/compress/rcat_internal_test.go b/backend/compress/rcat_internal_test.go
index 616ec7e38..b4b5428ba 100644
--- a/backend/compress/rcat_internal_test.go
+++ b/backend/compress/rcat_internal_test.go
@@ -3,6 +3,7 @@ package compress
import (
"bytes"
"context"
+ "crypto/rand"
"io"
"testing"
"time"
@@ -112,3 +113,25 @@ func TestRcat(t *testing.T) {
})
}
}
+
+// TestIsCompressible checks the compressibility heuristic on data at
+// both ends of the scale for every compression mode which has one.
+func TestIsCompressible(t *testing.T) {
+ compressible := bytes.Repeat([]byte("compress me "), 4096)
+ incompressible := make([]byte, len(compressible))
+ _, err := io.ReadFull(rand.Reader, incompressible)
+ require.NoError(t, err)
+ for name, handler := range map[string]compressionModeHandler{
+ "gzip": &gzipModeHandler{},
+ "zstd": &zstdModeHandler{},
+ } {
+ t.Run(name, func(t *testing.T) {
+ ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
+ require.NoError(t, err)
+ assert.True(t, ok)
+ ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
+ require.NoError(t, err)
+ assert.False(t, ok)
+ })
+ }
+}
diff --git a/backend/compress/zstd_handler.go b/backend/compress/zstd_handler.go
index 43914427b..7cf8d7617 100644
--- a/backend/compress/zstd_handler.go
+++ b/backend/compress/zstd_handler.go
@@ -2,7 +2,6 @@ package compress
import (
"bufio"
- "bytes"
"context"
"crypto/md5"
"encoding/hex"
@@ -23,7 +22,7 @@ type zstdModeHandler struct{}
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
// the configured threshold
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
- var b bytes.Buffer
+ var b countingDiscard
var n int64
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
if err != nil {
@@ -37,7 +36,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
if err != nil {
return false, err
}
- ratio := float64(n) / float64(b.Len())
+ ratio := float64(n) / float64(b)
return ratio > minCompressionRatio, nil
}