"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
backend/compress: add zstd compression
Alex8f74e7d3317a
diff --git a/backend/compress/compress.go b/backend/compress/compress.goindex 0cc56bc62..961d3898f 100644--- a/backend/compress/compress.go+++ b/backend/compress/compress.go@@ -2,10 +2,8 @@package compressimport (- "bufio""bytes""context"- "crypto/md5""encoding/base64""encoding/binary""encoding/hex"@@ -46,6 +44,7 @@ const (minCompressionRatio = 1.1gzFileExt = ".gz"+ zstdFileExt = ".zst"metaFileExt = ".json"uncompressedFileExt = ".bin")@@ -54,6 +53,7 @@ const (const (Uncompressed = 0Gzip = 2+ Zstd = 4)var nameRegexp = regexp.MustCompile(`^(.+?)\.([A-Za-z0-9-_]{11})$`)@@ -66,6 +66,10 @@ func init() {Value: "gzip",Help: "Standard gzip compression with fastest parameters.",},+ {+ Value: "zstd",+ Help: "Zstandard compression — fast modern algorithm offering adjustable speed-to-compression tradeoffs.",+ },}// Register our remote@@ -87,17 +91,23 @@ func init() {Examples: compressionModeOptions,}, {Name: "level",- Help: `GZIP compression level (-2 to 9).--Generally -1 (default, equivalent to 5) is recommended.-Levels 1 to 9 increase compression at the cost of speed. Going past 6-generally offers very little return.--Level -2 uses Huffman encoding only. Only use if you know what you-are doing.-Level 0 turns off compression.`,- Default: sgzip.DefaultCompression,- Advanced: true,+ Help: `GZIP (levels -2 to 9):+- -2 — Huffman encoding only. Only use if you know what you're doing.+- -1 (default) — recommended; equivalent to level 5.+- 0 — turns off compression.+- 1–9 — increase compression at the cost of speed. Going past 6 generally offers very little return.++ZSTD (levels 0 to 4):+- 0 — turns off compression entirely.+- 1 — fastest compression with the lowest ratio.+- 2 (default) — good balance of speed and compression.+- 3 — better compression, but uses about 2–3x more CPU than the default.+- 4 — best possible compression ratio (highest CPU cost).++Notes:+- Choose GZIP for wide compatibility; ZSTD for better speed/ratio tradeoffs.+- Negative gzip levels: -2 = Huffman-only, -1 = default (≈ level 5).`,+ Required: true,}, {Name: "ram_cache_limit",Help: `Some remotes don't allow the upload of files with unknown size.@@ -112,6 +122,47 @@ this limit will be cached on disk.`,})}+// compressionModeHandler defines the interface for handling different compression modes+type compressionModeHandler interface {+ // processFileNameGetFileExtension returns the file extension for the given compression mode+ processFileNameGetFileExtension(compressionMode int) string++ // newObjectGetOriginalSize returns the original file size from the metadata+ newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error)++ // isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds+ // the configured threshold+ isCompressible(r io.Reader, compressionMode int) (bool, error)++ // putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata+ putCompress(+ ctx context.Context,+ f *Fs,+ in io.Reader,+ src fs.ObjectInfo,+ options []fs.OpenOption,+ mimeType string,+ ) (fs.Object, *ObjectMetadata, error)++ // openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method+ openGetReadCloser(+ ctx context.Context,+ o *Object,+ offset int64,+ limit int64,+ cr chunkedreader.ChunkedReader,+ closer io.Closer,+ options ...fs.OpenOption,+ ) (rc io.ReadCloser, err error)++ // putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm+ putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error)++ // This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.+ // Warning: This function panics if cmeta is not of the expected type.+ newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata+}+// Options defines the configuration for this backendtype Options struct {Remote string `config:"remote"`@@ -125,12 +176,13 @@ type Options struct {// Fs represents a wrapped fs.Fstype Fs struct {fs.Fs- wrapper fs.Fs- name string- root string- opt Options- mode int // compression mode id- features *fs.Features // optional features+ wrapper fs.Fs+ name string+ root string+ opt Options+ mode int // compression mode id+ features *fs.Features // optional features+ modeHandler compressionModeHandler // compression mode handler}// NewFs constructs an Fs from the path, container:path@@ -167,13 +219,28 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,return nil, fmt.Errorf("failed to make remote %s:%q to wrap: %w", wName, remotePath, err)}+ compressionMode := compressionModeFromName(opt.CompressionMode)+ var modeHandler compressionModeHandler++ switch compressionMode {+ case Gzip:+ modeHandler = &gzipModeHandler{}+ case Zstd:+ modeHandler = &zstdModeHandler{}+ case Uncompressed:+ modeHandler = &uncompressedModeHandler{}+ default:+ modeHandler = &unknownModeHandler{}+ }+// Create the wrapping fsf := &Fs{- Fs: wrappedFs,- name: name,- root: rpath,- opt: *opt,- mode: compressionModeFromName(opt.CompressionMode),+ Fs: wrappedFs,+ name: name,+ root: rpath,+ opt: *opt,+ mode: compressionMode,+ modeHandler: modeHandler,}// Correct root if definitely pointing to a fileif err == fs.ErrorIsFile {@@ -215,10 +282,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,return f, err}+// compressionModeFromName converts a compression mode name to its int representation.func compressionModeFromName(name string) int {switch name {case "gzip":return Gzip+ case "zstd":+ return Zstddefault:return Uncompressed}@@ -242,7 +312,7 @@ func base64ToInt64(str string) (int64, error) {// Processes a file name for a compressed file. Returns the original file name, the extension, and the size of the original file.// Returns -2 for the original size if the file is uncompressed.-func processFileName(compressedFileName string) (origFileName string, extension string, origSize int64, err error) {+func processFileName(compressedFileName string, modeHandler compressionModeHandler) (origFileName string, extension string, origSize int64, err error) {// Separate the filename and size from the extensionextensionPos := strings.LastIndex(compressedFileName, ".")if extensionPos == -1 {@@ -261,7 +331,8 @@ func processFileName(compressedFileName string) (origFileName string, extensionif err != nil {return "", "", 0, errors.New("could not decode size")}- return match[1], gzFileExt, size, nil+ ext := modeHandler.processFileNameGetFileExtension(compressionModeFromName(compressedFileName[extensionPos+1:]))+ return match[1], ext, size, nil}// Generates the file name for a metadata file@@ -286,11 +357,15 @@ func unwrapMetadataFile(filename string) (string, bool) {// makeDataName generates the file name for a data file with specified compression modefunc makeDataName(remote string, size int64, mode int) (newRemote string) {- if mode != Uncompressed {+ switch mode {+ case Gzip:newRemote = remote + "." + int64ToBase64(size) + gzFileExt- } else {+ case Zstd:+ newRemote = remote + "." + int64ToBase64(size) + zstdFileExt+ default:newRemote = remote + uncompressedFileExt}+return newRemote}@@ -304,7 +379,7 @@ func (f *Fs) dataName(remote string, size int64, compressed bool) (name string)// addData parses an object and adds it to the DirEntriesfunc (f *Fs) addData(entries *fs.DirEntries, o fs.Object) {- origFileName, _, size, err := processFileName(o.Remote())+ origFileName, _, size, err := processFileName(o.Remote(), f.modeHandler)if err != nil {fs.Errorf(o, "Error on parsing file name: %v", err)return@@ -427,8 +502,12 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {if err != nil {return nil, fmt.Errorf("error decoding metadata: %w", err)}+ size, err := f.modeHandler.newObjectGetOriginalSize(meta)+ if err != nil {+ return nil, fmt.Errorf("error reading metadata: %w", err)+ }// Create our Object- o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))+ o, err := f.Fs.NewObject(ctx, makeDataName(remote, size, meta.Mode))if err != nil {return nil, err}@@ -437,7 +516,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {// checkCompressAndType checks if an object is compressible and determines it's mime type// returns a multireader with the bytes that were read to determine mime type-func checkCompressAndType(in io.Reader) (newReader io.Reader, compressible bool, mimeType string, err error) {+func checkCompressAndType(in io.Reader, compressionMode int, modeHandler compressionModeHandler) (newReader io.Reader, compressible bool, mimeType string, err error) {in, wrap := accounting.UnWrap(in)buf := make([]byte, heuristicBytes)n, err := in.Read(buf)@@ -446,7 +525,7 @@ func checkCompressAndType(in io.Reader) (newReader io.Reader, compressible bool,return nil, false, "", err}mime := mimetype.Detect(buf)- compressible, err = isCompressible(bytes.NewReader(buf))+ compressible, err = modeHandler.isCompressible(bytes.NewReader(buf), compressionMode)if err != nil {return nil, false, "", err}@@ -454,26 +533,6 @@ func checkCompressAndType(in io.Reader) (newReader io.Reader, compressible bool,return wrap(in), compressible, mime.String(), nil}-// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds-// the configured threshold-func isCompressible(r io.Reader) (bool, error) {- var b bytes.Buffer- w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)- if err != nil {- return false, err- }- n, err := io.Copy(w, r)- if err != nil {- return false, err- }- err = w.Close()- if err != nil {- return false, err- }- ratio := float64(n) / float64(b.Len())- return ratio > minCompressionRatio, nil-}-// verifyObjectHash verifies the Objects hashfunc (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.MultiHasher, ht hash.Type) error {srcHash := hasher.Sums()[ht]@@ -494,9 +553,9 @@ func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.Multype putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)-type compressionResult struct {+type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {err error- meta sgzip.GzipMetadata+ meta T}// replicating some of operations.Rcat functionality because we want to support remotes without streaming@@ -537,106 +596,18 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modreturn nil, fmt.Errorf("failed to write temporary local file: %w", err)}if _, err = tempFile.Seek(0, 0); err != nil {- return nil, err+ return nil, fmt.Errorf("failed to seek temporary local file: %w", err)}finfo, err := tempFile.Stat()if err != nil {- return nil, err+ return nil, fmt.Errorf("failed to stat temporary local file: %w", err)}return f.Fs.Put(ctx, tempFile, object.NewStaticObjectInfo(dstFileName, modTime, finfo.Size(), false, nil, f.Fs))}// Put a compressed version of a file. Returns a wrappable object and metadata.func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, mimeType string) (fs.Object, *ObjectMetadata, error) {- // Unwrap reader accounting- in, wrap := accounting.UnWrap(in)-- // Add the metadata hasher- metaHasher := md5.New()- in = io.TeeReader(in, metaHasher)-- // Compress the file- pipeReader, pipeWriter := io.Pipe()- results := make(chan compressionResult)- go func() {- gz, err := sgzip.NewWriterLevel(pipeWriter, f.opt.CompressionLevel)- if err != nil {- results <- compressionResult{err: err, meta: sgzip.GzipMetadata{}}- return- }- _, err = io.Copy(gz, in)- gzErr := gz.Close()- if gzErr != nil {- fs.Errorf(nil, "Failed to close compress: %v", gzErr)- if err == nil {- err = gzErr- }- }- closeErr := pipeWriter.Close()- if closeErr != nil {- fs.Errorf(nil, "Failed to close pipe: %v", closeErr)- if err == nil {- err = closeErr- }- }- results <- compressionResult{err: err, meta: gz.MetaData()}- }()- wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize)) // Probably no longer needed as sgzip has it's own buffering-- // Find a hash the destination supports to compute a hash of- // the compressed data.- ht := f.Fs.Hashes().GetOne()- var hasher *hash.MultiHasher- var err error- if ht != hash.None {- // unwrap the accounting again- wrappedIn, wrap = accounting.UnWrap(wrappedIn)- hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))- if err != nil {- return nil, nil, err- }- // add the hasher and re-wrap the accounting- wrappedIn = io.TeeReader(wrappedIn, hasher)- wrappedIn = wrap(wrappedIn)- }-- // Transfer the data- o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)- //o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))- if err != nil {- if o != nil {- removeErr := o.Remove(ctx)- if removeErr != nil {- fs.Errorf(o, "Failed to remove partially transferred object: %v", err)- }- }- return nil, nil, err- }- // Check whether we got an error during compression- result := <-results- err = result.err- if err != nil {- if o != nil {- removeErr := o.Remove(ctx)- if removeErr != nil {- fs.Errorf(o, "Failed to remove partially compressed object: %v", err)- }- }- return nil, nil, err- }-- // Generate metadata- meta := newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)-- // Check the hashes of the compressed data if we were comparing them- if ht != hash.None && hasher != nil {- err = f.verifyObjectHash(ctx, o, hasher, ht)- if err != nil {- return nil, nil, err- }- }-- return o, meta, nil+ return f.modeHandler.putCompress(ctx, f, in, src, options, mimeType)}// Put an uncompressed version of a file. Returns a wrappable object and metadata.@@ -680,7 +651,8 @@ func (f *Fs) putUncompress(ctx context.Context, in io.Reader, src fs.ObjectInfo,if err != nil {return nil, nil, err}- return o, newMetadata(o.Size(), Uncompressed, sgzip.GzipMetadata{}, hex.EncodeToString(sum), mimeType), nil++ return f.modeHandler.putUncompressGetNewMetadata(o, Uncompressed, hex.EncodeToString(sum), mimeType, sum)}// This function will write a metadata struct to a metadata Object for an src. Returns a wrappable metadata object.@@ -751,7 +723,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .o, err := f.NewObject(ctx, src.Remote())if err == fs.ErrorObjectNotFound {// Get our file compressibility- in, compressible, mimeType, err := checkCompressAndType(in)+ in, compressible, mimeType, err := checkCompressAndType(in, f.mode, f.modeHandler)if err != nil {return nil, err}@@ -771,7 +743,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt}found := err == nil- in, compressible, mimeType, err := checkCompressAndType(in)+ in, compressible, mimeType, err := checkCompressAndType(in, f.mode, f.modeHandler)if err != nil {return nil, err}@@ -1090,11 +1062,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, duration fs.Duration// ObjectMetadata describes the metadata for an Object.type ObjectMetadata struct {- Mode int // Compression mode of the file.- Size int64 // Size of the object.- MD5 string // MD5 hash of the file.- MimeType string // Mime type of the file- CompressionMetadata sgzip.GzipMetadata+ Mode int // Compression mode of the file.+ Size int64 // Size of the object.+ MD5 string // MD5 hash of the file.+ MimeType string // Mime type of the file+ CompressionMetadataGzip *sgzip.GzipMetadata // Metadata for Gzip compression+ CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression}// Object with external metadata@@ -1107,17 +1080,6 @@ type Object struct {meta *ObjectMetadata // Metadata struct for this object (nil if not loaded)}-// This function generates a metadata object-func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mimeType string) *ObjectMetadata {- meta := new(ObjectMetadata)- meta.Size = size- meta.Mode = mode- meta.CompressionMetadata = cmeta- meta.MD5 = md5- meta.MimeType = mimeType- return meta-}-// This function will read the metadata from a metadata object.func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) {// Open our meradata object@@ -1165,7 +1127,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opreturn o.mo, o.mo.Update(ctx, in, src, options...)}- in, compressible, mimeType, err := checkCompressAndType(in)+ in, compressible, mimeType, err := checkCompressAndType(in, o.meta.Mode, o.f.modeHandler)if err != nil {return err}@@ -1278,7 +1240,7 @@ func (o *Object) String() string {// Remote returns the remote pathfunc (o *Object) Remote() string {- origFileName, _, _, err := processFileName(o.Object.Remote())+ origFileName, _, _, err := processFileName(o.Object.Remote(), o.f.modeHandler)if err != nil {fs.Errorf(o.f, "Could not get remote path for: %s", o.Object.Remote())return o.Object.Remote()@@ -1381,7 +1343,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Readreturn o.Object.Open(ctx, options...)}// Get offset and limit from OpenOptions, pass the rest to the underlying remote- var openOptions = []fs.OpenOption{&fs.SeekOption{Offset: 0}}var offset, limit int64 = 0, -1for _, option := range options {switch x := option.(type) {@@ -1389,31 +1350,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Readoffset = x.Offsetcase *fs.RangeOption:offset, limit = x.Decode(o.Size())- default:- openOptions = append(openOptions, option)}}// Get a chunkedreader for the wrapped objectchunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize, chunkStreams)- // Get file handle- var file io.Reader- if offset != 0 {- file, err = sgzip.NewReaderAt(chunkedReader, &o.meta.CompressionMetadata, offset)- } else {- file, err = sgzip.NewReader(chunkedReader)- }- if err != nil {- return nil, err- }-- var fileReader io.Reader- if limit != -1 {- fileReader = io.LimitReader(file, limit)- } else {- fileReader = file- }- // Return a ReadCloser- return ReadCloserWrapper{Reader: fileReader, Closer: chunkedReader}, nil+ var retCloser io.Closer = chunkedReader+ return o.f.modeHandler.openGetReadCloser(ctx, o, offset, limit, chunkedReader, retCloser, options...)}// ObjectInfo describes a wrapped fs.ObjectInfo for being the sourcediff --git a/backend/compress/compress_test.go b/backend/compress/compress_test.goindex 8fc954dfa..000d70330 100644--- a/backend/compress/compress_test.go+++ b/backend/compress/compress_test.go@@ -48,7 +48,27 @@ func TestRemoteGzip(t *testing.T) {opt.ExtraConfig = []fstests.ExtraConfigItem{{Name: name, Key: "type", Value: "compress"},{Name: name, Key: "remote", Value: tempdir},- {Name: name, Key: "compression_mode", Value: "gzip"},+ {Name: name, Key: "mode", Value: "gzip"},+ {Name: name, Key: "level", Value: "-1"},+ }+ opt.QuickTestOK = true+ fstests.Run(t, &opt)+}++// TestRemoteZstd tests ZSTD compression+func TestRemoteZstd(t *testing.T) {+ if *fstest.RemoteName != "" {+ t.Skip("Skipping as -remote set")+ }+ tempdir := filepath.Join(os.TempDir(), "rclone-compress-test-zstd")+ name := "TestCompressZstd"+ opt := defaultOpt+ opt.RemoteName = name + ":"+ opt.ExtraConfig = []fstests.ExtraConfigItem{+ {Name: name, Key: "type", Value: "compress"},+ {Name: name, Key: "remote", Value: tempdir},+ {Name: name, Key: "mode", Value: "zstd"},+ {Name: name, Key: "level", Value: "2"},}opt.QuickTestOK = truefstests.Run(t, &opt)diff --git a/backend/compress/gzip_handler.go b/backend/compress/gzip_handler.gonew file mode 100644index 000000000..a5abbf7c8--- /dev/null+++ b/backend/compress/gzip_handler.go@@ -0,0 +1,207 @@+package compress++import (+ "bufio"+ "bytes"+ "context"+ "crypto/md5"+ "encoding/hex"+ "errors"+ "io"++ "github.com/buengese/sgzip"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/chunkedreader"+ "github.com/rclone/rclone/fs/hash"+)++// gzipModeHandler implements compressionModeHandler for gzip+type gzipModeHandler struct{}++// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds+// the configured threshold+func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {+ var b bytes.Buffer+ var n int64+ w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)+ if err != nil {+ return false, err+ }+ n, err = io.Copy(w, r)+ if err != nil {+ return false, err+ }+ err = w.Close()+ if err != nil {+ return false, err+ }+ ratio := float64(n) / float64(b.Len())+ return ratio > minCompressionRatio, nil+}++// newObjectGetOriginalSize returns the original file size from the metadata+func (g *gzipModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {+ if meta.CompressionMetadataGzip == nil {+ return 0, errors.New("missing gzip metadata")+ }+ return meta.CompressionMetadataGzip.Size, nil+}++// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method+func (g *gzipModeHandler) openGetReadCloser(+ ctx context.Context,+ o *Object,+ offset int64,+ limit int64,+ cr chunkedreader.ChunkedReader,+ closer io.Closer,+ options ...fs.OpenOption,+) (rc io.ReadCloser, err error) {+ var file io.Reader++ if offset != 0 {+ file, err = sgzip.NewReaderAt(cr, o.meta.CompressionMetadataGzip, offset)+ } else {+ file, err = sgzip.NewReader(cr)+ }+ if err != nil {+ return nil, err+ }++ var fileReader io.Reader+ if limit != -1 {+ fileReader = io.LimitReader(file, limit)+ } else {+ fileReader = file+ }+ // Return a ReadCloser+ return ReadCloserWrapper{Reader: fileReader, Closer: closer}, nil+}++// processFileNameGetFileExtension returns the file extension for the given compression mode+func (g *gzipModeHandler) processFileNameGetFileExtension(compressionMode int) string {+ if compressionMode == Gzip {+ return gzFileExt+ }++ return ""+}++// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata+func (g *gzipModeHandler) putCompress(+ ctx context.Context,+ f *Fs,+ in io.Reader,+ src fs.ObjectInfo,+ options []fs.OpenOption,+ mimeType string,+) (fs.Object, *ObjectMetadata, error) {+ // Unwrap reader accounting+ in, wrap := accounting.UnWrap(in)++ // Add the metadata hasher+ metaHasher := md5.New()+ in = io.TeeReader(in, metaHasher)++ // Compress the file+ pipeReader, pipeWriter := io.Pipe()++ resultsGzip := make(chan compressionResult[sgzip.GzipMetadata])+ go func() {+ gz, err := sgzip.NewWriterLevel(pipeWriter, f.opt.CompressionLevel)+ if err != nil {+ resultsGzip <- compressionResult[sgzip.GzipMetadata]{err: err, meta: sgzip.GzipMetadata{}}+ close(resultsGzip)+ return+ }+ _, err = io.Copy(gz, in)+ gzErr := gz.Close()+ if gzErr != nil && err == nil {+ err = gzErr+ }+ closeErr := pipeWriter.Close()+ if closeErr != nil && err == nil {+ err = closeErr+ }+ resultsGzip <- compressionResult[sgzip.GzipMetadata]{err: err, meta: gz.MetaData()}+ close(resultsGzip)+ }()++ wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize)) // Probably no longer needed as sgzip has it's own buffering++ // Find a hash the destination supports to compute a hash of+ // the compressed data.+ ht := f.Fs.Hashes().GetOne()+ var hasher *hash.MultiHasher+ var err error+ if ht != hash.None {+ // unwrap the accounting again+ wrappedIn, wrap = accounting.UnWrap(wrappedIn)+ hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))+ if err != nil {+ return nil, nil, err+ }+ // add the hasher and re-wrap the accounting+ wrappedIn = io.TeeReader(wrappedIn, hasher)+ wrappedIn = wrap(wrappedIn)+ }++ // Transfer the data+ o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)+ if err != nil {+ if o != nil {+ if removeErr := o.Remove(ctx); removeErr != nil {+ fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)+ }+ }+ return nil, nil, err+ }+ // Check whether we got an error during compression+ result := <-resultsGzip+ if result.err != nil {+ if o != nil {+ if removeErr := o.Remove(ctx); removeErr != nil {+ fs.Errorf(o, "Failed to remove partially compressed object: %v", removeErr)+ }+ }+ return nil, nil, result.err+ }++ // Generate metadata+ meta := g.newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)++ // Check the hashes of the compressed data if we were comparing them+ if ht != hash.None && hasher != nil {+ err = f.verifyObjectHash(ctx, o, hasher, ht)+ if err != nil {+ return nil, nil, err+ }+ }+ return o, meta, nil+}++// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm+func (g *gzipModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {+ return o, g.newMetadata(o.Size(), mode, sgzip.GzipMetadata{}, hex.EncodeToString(sum), mimeType), nil+}++// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.+// Warning: This function panics if cmeta is not of the expected type.+func (g *gzipModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {+ meta, ok := cmeta.(sgzip.GzipMetadata)+ if !ok {+ panic("invalid cmeta type: expected sgzip.GzipMetadata")+ }++ objMeta := new(ObjectMetadata)+ objMeta.Size = size+ objMeta.Mode = mode+ objMeta.CompressionMetadataGzip = &meta+ objMeta.CompressionMetadataZstd = nil+ objMeta.MD5 = md5+ objMeta.MimeType = mimeType++ return objMeta+}diff --git a/backend/compress/szstd_helper.go b/backend/compress/szstd_helper.gonew file mode 100644index 000000000..97f591f5f--- /dev/null+++ b/backend/compress/szstd_helper.go@@ -0,0 +1,327 @@+package compress++import (+ "context"+ "errors"+ "io"+ "runtime"+ "sync"++ szstd "github.com/a1ex3/zstd-seekable-format-go/pkg"+ "github.com/klauspost/compress/zstd"+)++const szstdChunkSize int = 1 << 20 // 1 MiB chunk size++// SzstdMetadata holds metadata for szstd compressed files.+type SzstdMetadata struct {+ BlockSize int // BlockSize is the size of the blocks in the zstd file+ Size int64 // Size is the uncompressed size of the file+ BlockData []uint32 // BlockData is the block data for the zstd file, used for seeking+}++// SzstdWriter is a writer that compresses data in szstd format.+type SzstdWriter struct {+ enc *zstd.Encoder+ w szstd.ConcurrentWriter+ metadata SzstdMetadata+ mu sync.Mutex+}++// NewWriterSzstd creates a new szstd writer with the specified options.+// It initializes the szstd writer with a zstd encoder and returns a pointer to the SzstdWriter.+// The writer can be used to write data in chunks, and it will automatically handle block sizes and metadata.+func NewWriterSzstd(w io.Writer, opts ...zstd.EOption) (*SzstdWriter, error) {+ encoder, err := zstd.NewWriter(nil, opts...)+ if err != nil {+ return nil, err+ }++ sw, err := szstd.NewWriter(w, encoder)+ if err != nil {+ if err := encoder.Close(); err != nil {+ return nil, err+ }+ return nil, err+ }++ return &SzstdWriter{+ enc: encoder,+ w: sw,+ metadata: SzstdMetadata{+ BlockSize: szstdChunkSize,+ Size: 0,+ },+ }, nil+}++// Write writes data to the szstd writer in chunks of szstdChunkSize.+// It handles the block size and metadata updates automatically.+func (w *SzstdWriter) Write(p []byte) (int, error) {+ if len(p) == 0 {+ return 0, nil+ }++ if w.metadata.BlockData == nil {+ numBlocks := (len(p) + w.metadata.BlockSize - 1) / w.metadata.BlockSize+ w.metadata.BlockData = make([]uint32, 1, numBlocks+1)+ w.metadata.BlockData[0] = 0+ }++ start := 0+ total := len(p)++ var writerFunc szstd.FrameSource = func() ([]byte, error) {+ if start >= total {+ return nil, nil+ }++ end := min(start+w.metadata.BlockSize, total)+ chunk := p[start:end]+ size := end - start++ w.mu.Lock()+ w.metadata.Size += int64(size)+ w.mu.Unlock()++ start = end+ return chunk, nil+ }++ // write sizes of compressed blocks in the callback+ err := w.w.WriteMany(context.Background(), writerFunc,+ szstd.WithWriteCallback(func(size uint32) {+ w.mu.Lock()+ lastOffset := w.metadata.BlockData[len(w.metadata.BlockData)-1]+ w.metadata.BlockData = append(w.metadata.BlockData, lastOffset+size)+ w.mu.Unlock()+ }),+ )+ if err != nil {+ return 0, err+ }++ return total, nil+}++// Close closes the SzstdWriter and its underlying encoder.+func (w *SzstdWriter) Close() error {+ if err := w.w.Close(); err != nil {+ return err+ }+ if err := w.enc.Close(); err != nil {+ return err+ }++ return nil+}++// GetMetadata returns the metadata of the szstd writer.+func (w *SzstdWriter) GetMetadata() SzstdMetadata {+ return w.metadata+}++// SzstdReaderAt is a reader that allows random access in szstd compressed data.+type SzstdReaderAt struct {+ r szstd.Reader+ decoder *zstd.Decoder+ metadata *SzstdMetadata+ pos int64+ mu sync.Mutex+}++// NewReaderAtSzstd creates a new SzstdReaderAt at the specified io.ReadSeeker.+func NewReaderAtSzstd(rs io.ReadSeeker, meta *SzstdMetadata, offset int64, opts ...zstd.DOption) (*SzstdReaderAt, error) {+ decoder, err := zstd.NewReader(nil, opts...)+ if err != nil {+ return nil, err+ }++ r, err := szstd.NewReader(rs, decoder)+ if err != nil {+ decoder.Close()+ return nil, err+ }++ sr := &SzstdReaderAt{+ r: r,+ decoder: decoder,+ metadata: meta,+ pos: 0,+ }++ // Set initial position to the provided offset+ if _, err := sr.Seek(offset, io.SeekStart); err != nil {+ if err := sr.Close(); err != nil {+ return nil, err+ }+ return nil, err+ }++ return sr, nil+}++// Seek sets the offset for the next Read.+func (s *SzstdReaderAt) Seek(offset int64, whence int) (int64, error) {+ s.mu.Lock()+ defer s.mu.Unlock()++ pos, err := s.r.Seek(offset, whence)+ if err == nil {+ s.pos = pos+ }+ return pos, err+}++func (s *SzstdReaderAt) Read(p []byte) (int, error) {+ s.mu.Lock()+ defer s.mu.Unlock()++ n, err := s.r.Read(p)+ if err == nil {+ s.pos += int64(n)+ }+ return n, err+}++// ReadAt reads data at the specified offset.+func (s *SzstdReaderAt) ReadAt(p []byte, off int64) (int, error) {+ if off < 0 {+ return 0, errors.New("invalid offset")+ }+ if off >= s.metadata.Size {+ return 0, io.EOF+ }++ endOff := min(off+int64(len(p)), s.metadata.Size)++ // Find all blocks covered by the range+ type blockInfo struct {+ index int // Block index+ offsetInBlock int64 // Offset within the block for starting reading+ bytesToRead int64 // How many bytes to read from this block+ }++ var blocks []blockInfo+ uncompressedOffset := int64(0)+ currentOff := off++ for i := 0; i < len(s.metadata.BlockData)-1; i++ {+ blockUncompressedEnd := min(uncompressedOffset+int64(s.metadata.BlockSize), s.metadata.Size)++ if currentOff < blockUncompressedEnd && endOff > uncompressedOffset {+ offsetInBlock := max(0, currentOff-uncompressedOffset)+ bytesToRead := min(blockUncompressedEnd-uncompressedOffset-offsetInBlock, endOff-currentOff)++ blocks = append(blocks, blockInfo{+ index: i,+ offsetInBlock: offsetInBlock,+ bytesToRead: bytesToRead,+ })++ currentOff += bytesToRead+ if currentOff >= endOff {+ break+ }+ }+ uncompressedOffset = blockUncompressedEnd+ }++ if len(blocks) == 0 {+ return 0, io.EOF+ }++ // Parallel block decoding+ type decodeResult struct {+ index int+ data []byte+ err error+ }++ resultCh := make(chan decodeResult, len(blocks))+ var wg sync.WaitGroup+ sem := make(chan struct{}, runtime.NumCPU())++ for _, block := range blocks {+ wg.Add(1)+ go func(block blockInfo) {+ defer wg.Done()+ sem <- struct{}{}+ defer func() { <-sem }()++ startOffset := int64(s.metadata.BlockData[block.index])+ endOffset := int64(s.metadata.BlockData[block.index+1])+ compressedSize := endOffset - startOffset++ compressed := make([]byte, compressedSize)+ _, err := s.r.ReadAt(compressed, startOffset)+ if err != nil && err != io.EOF {+ resultCh <- decodeResult{index: block.index, err: err}+ return+ }++ decoded, err := s.decoder.DecodeAll(compressed, nil)+ if err != nil {+ resultCh <- decodeResult{index: block.index, err: err}+ return+ }++ resultCh <- decodeResult{index: block.index, data: decoded, err: nil}+ }(block)+ }++ go func() {+ wg.Wait()+ close(resultCh)+ }()++ // Collect results in block index order+ totalRead := 0+ results := make(map[int]decodeResult)+ expected := len(blocks)+ minIndex := blocks[0].index++ for res := range resultCh {+ results[res.index] = res+ for {+ if result, ok := results[minIndex]; ok {+ if result.err != nil {+ return 0, result.err+ }+ // find the corresponding blockInfo+ var blk blockInfo+ for _, b := range blocks {+ if b.index == result.index {+ blk = b+ break+ }+ }++ start := blk.offsetInBlock+ end := start + blk.bytesToRead+ copy(p[totalRead:totalRead+int(blk.bytesToRead)], result.data[start:end])+ totalRead += int(blk.bytesToRead)+ minIndex+++ if minIndex-blocks[0].index >= len(blocks) {+ break+ }+ } else {+ break+ }+ }+ if len(results) == expected && minIndex-blocks[0].index >= len(blocks) {+ break+ }+ }++ return totalRead, nil+}++// Close closes the SzstdReaderAt and underlying decoder.+func (s *SzstdReaderAt) Close() error {+ if err := s.r.Close(); err != nil {+ return err+ }+ s.decoder.Close()+ return nil+}diff --git a/backend/compress/uncompressed_handler.go b/backend/compress/uncompressed_handler.gonew file mode 100644index 000000000..c1fc0c794--- /dev/null+++ b/backend/compress/uncompressed_handler.go@@ -0,0 +1,65 @@+package compress++import (+ "context"+ "fmt"+ "io"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/chunkedreader"+)++// uncompressedModeHandler implements compressionModeHandler for uncompressed files+type uncompressedModeHandler struct{}++// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds+// the configured threshold+func (u *uncompressedModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {+ return false, nil+}++// newObjectGetOriginalSize returns the original file size from the metadata+func (u *uncompressedModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {+ return 0, nil+}++// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method+func (u *uncompressedModeHandler) openGetReadCloser(+ ctx context.Context,+ o *Object,+ offset int64,+ limit int64,+ cr chunkedreader.ChunkedReader,+ closer io.Closer,+ options ...fs.OpenOption,+) (rc io.ReadCloser, err error) {+ return o.Object.Open(ctx, options...)+}++// processFileNameGetFileExtension returns the file extension for the given compression mode+func (u *uncompressedModeHandler) processFileNameGetFileExtension(compressionMode int) string {+ return ""+}++// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata+func (u *uncompressedModeHandler) putCompress(+ ctx context.Context,+ f *Fs,+ in io.Reader,+ src fs.ObjectInfo,+ options []fs.OpenOption,+ mimeType string,+) (fs.Object, *ObjectMetadata, error) {+ return nil, nil, fmt.Errorf("unsupported compression mode %d", f.mode)+}++// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm+func (u *uncompressedModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {+ return nil, nil, fmt.Errorf("unsupported compression mode %d", Uncompressed)+}++// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.+// Warning: This function panics if cmeta is not of the expected type.+func (u *uncompressedModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {+ return nil+}diff --git a/backend/compress/unknown_handler.go b/backend/compress/unknown_handler.gonew file mode 100644index 000000000..f8d9df589--- /dev/null+++ b/backend/compress/unknown_handler.go@@ -0,0 +1,65 @@+package compress++import (+ "context"+ "fmt"+ "io"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/chunkedreader"+)++// unknownModeHandler implements compressionModeHandler for unknown compression types+type unknownModeHandler struct{}++// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds+// the configured threshold+func (unk *unknownModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {+ return false, fmt.Errorf("unknown compression mode %d", compressionMode)+}++// newObjectGetOriginalSize returns the original file size from the metadata+func (unk *unknownModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {+ return 0, nil+}++// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method+func (unk *unknownModeHandler) openGetReadCloser(+ ctx context.Context,+ o *Object,+ offset int64,+ limit int64,+ cr chunkedreader.ChunkedReader,+ closer io.Closer,+ options ...fs.OpenOption,+) (rc io.ReadCloser, err error) {+ return nil, fmt.Errorf("unknown compression mode %d", o.meta.Mode)+}++// processFileNameGetFileExtension returns the file extension for the given compression mode+func (unk *unknownModeHandler) processFileNameGetFileExtension(compressionMode int) string {+ return ""+}++// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata+func (unk *unknownModeHandler) putCompress(+ ctx context.Context,+ f *Fs,+ in io.Reader,+ src fs.ObjectInfo,+ options []fs.OpenOption,+ mimeType string,+) (fs.Object, *ObjectMetadata, error) {+ return nil, nil, fmt.Errorf("unknown compression mode %d", f.mode)+}++// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm+func (unk *unknownModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {+ return nil, nil, fmt.Errorf("unknown compression mode")+}++// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.+// Warning: This function panics if cmeta is not of the expected type.+func (unk *unknownModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {+ return nil+}diff --git a/backend/compress/zstd_handler.go b/backend/compress/zstd_handler.gonew file mode 100644index 000000000..43914427b--- /dev/null+++ b/backend/compress/zstd_handler.go@@ -0,0 +1,192 @@+package compress++import (+ "bufio"+ "bytes"+ "context"+ "crypto/md5"+ "encoding/hex"+ "errors"+ "io"++ "github.com/klauspost/compress/zstd"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/chunkedreader"+ "github.com/rclone/rclone/fs/hash"+)++// zstdModeHandler implements compressionModeHandler for zstd+type zstdModeHandler struct{}++// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds+// the configured threshold+func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {+ var b bytes.Buffer+ var n int64+ w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))+ if err != nil {+ return false, err+ }+ n, err = io.Copy(w, r)+ if err != nil {+ return false, err+ }+ err = w.Close()+ if err != nil {+ return false, err+ }+ ratio := float64(n) / float64(b.Len())+ return ratio > minCompressionRatio, nil+}++// newObjectGetOriginalSize returns the original file size from the metadata+func (z *zstdModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {+ if meta.CompressionMetadataZstd == nil {+ return 0, errors.New("missing zstd metadata")+ }+ return meta.CompressionMetadataZstd.Size, nil+}++// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method+func (z *zstdModeHandler) openGetReadCloser(+ ctx context.Context,+ o *Object,+ offset int64,+ limit int64,+ cr chunkedreader.ChunkedReader,+ closer io.Closer,+ options ...fs.OpenOption,+) (rc io.ReadCloser, err error) {+ var file io.Reader++ if offset != 0 {+ file, err = NewReaderAtSzstd(cr, o.meta.CompressionMetadataZstd, offset)+ } else {+ file, err = zstd.NewReader(cr)+ }+ if err != nil {+ return nil, err+ }++ var fileReader io.Reader+ if limit != -1 {+ fileReader = io.LimitReader(file, limit)+ } else {+ fileReader = file+ }+ // Return a ReadCloser+ return ReadCloserWrapper{Reader: fileReader, Closer: closer}, nil+}++// processFileNameGetFileExtension returns the file extension for the given compression mode+func (z *zstdModeHandler) processFileNameGetFileExtension(compressionMode int) string {+ if compressionMode == Zstd {+ return zstdFileExt+ }++ return ""+}++// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata+func (z *zstdModeHandler) putCompress(+ ctx context.Context,+ f *Fs,+ in io.Reader,+ src fs.ObjectInfo,+ options []fs.OpenOption,+ mimeType string,+) (fs.Object, *ObjectMetadata, error) {+ // Unwrap reader accounting+ in, wrap := accounting.UnWrap(in)++ // Add the metadata hasher+ metaHasher := md5.New()+ in = io.TeeReader(in, metaHasher)++ // Compress the file+ pipeReader, pipeWriter := io.Pipe()++ resultsZstd := make(chan compressionResult[SzstdMetadata])+ go func() {+ writer, err := NewWriterSzstd(pipeWriter, zstd.WithEncoderLevel(zstd.EncoderLevel(f.opt.CompressionLevel)))+ if err != nil {+ resultsZstd <- compressionResult[SzstdMetadata]{err: err}+ close(resultsZstd)+ return+ }+ _, err = io.Copy(writer, in)+ if wErr := writer.Close(); wErr != nil && err == nil {+ err = wErr+ }+ if cErr := pipeWriter.Close(); cErr != nil && err == nil {+ err = cErr+ }++ resultsZstd <- compressionResult[SzstdMetadata]{err: err, meta: writer.GetMetadata()}+ close(resultsZstd)+ }()++ wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize))++ ht := f.Fs.Hashes().GetOne()+ var hasher *hash.MultiHasher+ var err error+ if ht != hash.None {+ wrappedIn, wrap = accounting.UnWrap(wrappedIn)+ hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))+ if err != nil {+ return nil, nil, err+ }+ wrappedIn = io.TeeReader(wrappedIn, hasher)+ wrappedIn = wrap(wrappedIn)+ }++ o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)+ if err != nil {+ return nil, nil, err+ }++ result := <-resultsZstd+ if result.err != nil {+ if o != nil {+ _ = o.Remove(ctx)+ }+ return nil, nil, result.err+ }++ // Build metadata using uncompressed size for filename+ meta := z.newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)+ if ht != hash.None && hasher != nil {+ err = f.verifyObjectHash(ctx, o, hasher, ht)+ if err != nil {+ return nil, nil, err+ }+ }+ return o, meta, nil+}++// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm+func (z *zstdModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {+ return o, z.newMetadata(o.Size(), mode, SzstdMetadata{}, hex.EncodeToString(sum), mimeType), nil+}++// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.+// Warning: This function panics if cmeta is not of the expected type.+func (z *zstdModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {+ meta, ok := cmeta.(SzstdMetadata)+ if !ok {+ panic("invalid cmeta type: expected SzstdMetadata")+ }++ objMeta := new(ObjectMetadata)+ objMeta.Size = size+ objMeta.Mode = mode+ objMeta.CompressionMetadataGzip = nil+ objMeta.CompressionMetadataZstd = &meta+ objMeta.MD5 = md5+ objMeta.MimeType = mimeType++ return objMeta+}diff --git a/docs/content/compress.md b/docs/content/compress.mdindex 9fe8d518a..e4c090eec 100644--- a/docs/content/compress.md+++ b/docs/content/compress.md@@ -23,6 +23,7 @@ To use this remote, all you need to do is specify another remote and acompression mode to use:```text+$ rclone configCurrent remotes:Name Type@@ -30,7 +31,6 @@ Name Typeremote_to_press sometypee) Edit existing remote-$ rclone confign) New remoted) Delete remoter) Rename remote@@ -39,45 +39,74 @@ s) Set configuration passwordq) Quit confige/n/d/r/c/s/q> nname> compress++Option Storage.+Type of storage to configure.+Choose a number from below, or type in your own value....- 8 / Compress a remote- \ "compress"+12 / Compress a remote+ \ (compress)...Storage> compress-** See help for compress backend at: https://rclone.org/compress/ **+Option remote.Remote to compress.-Enter a string value. Press Enter for the default ("").+Enter a value.remote> remote_to_press:subdir++Option mode.Compression mode.-Enter a string value. Press Enter for the default ("gzip").-Choose a number from below, or type in your own value- 1 / Gzip compression balanced for speed and compression strength.- \ "gzip"-compression_mode> gzip-Edit advanced config? (y/n)+Choose a number from below, or type in your own value of type string.+Press Enter for the default (gzip).+ 1 / Standard gzip compression with fastest parameters.+ \ (gzip)+ 2 / Zstandard compression — fast modern algorithm offering adjustable speed-to-compression tradeoffs.+ \ (zstd)+mode> gzip++Option level.+GZIP (levels -2 to 9):+- -2 — Huffman encoding only. Only use if you know what you're doing.+- -1 (default) — recommended; equivalent to level 5.+- 0 — turns off compression.+- 1–9 — increase compression at the cost of speed. Going past 6 generally offers very little return.++ZSTD (levels 0 to 4):+- 0 — turns off compression entirely.+- 1 — fastest compression with the lowest ratio.+- 2 (default) — good balance of speed and compression.+- 3 — better compression, but uses about 2–3x more CPU than the default.+- 4 — best possible compression ratio (highest CPU cost).++Notes:+- Choose GZIP for wide compatibility; ZSTD for better speed/ratio tradeoffs.+- Negative gzip levels: -2 = Huffman-only, -1 = default (≈ level 5).+Enter a value.+level> -1++Edit advanced config?y) Yesn) No (default)y/n> n-Remote config----------------------[compress]-type = compress-remote = remote_to_press:subdir-compression_mode = gzip---------------------++Configuration complete.+Options:+- type: compress+- remote: remote_to_press:subdir+- mode: gzip+- level: -1+Keep this "compress" remote?y) Yes this is OK (default)e) Edit this remoted) Delete this remotey/e/d> y```-### Compression Modes+### Compression Algorithms-Currently only gzip compression is supported. It provides a decent balance-between speed and size and is well supported by other applications. Compression-strength can further be configured via an advanced setting where 0 is no-compression and 9 is strongest compression.+- **GZIP** – a well-established and widely adopted algorithm that strikes a solid balance between compression speed and ratio. It supports compression levels from -2 to 9, with the default -1 (roughly equivalent to level 5) offering an effective middle ground for most scenarios.++- **Zstandard (zstd)** – a modern, high-performance algorithm that offers precise control over the trade-off between speed and compression efficiency. Compression levels range from 0 (no compression) to 4 (maximum compression).### File types@@ -124,29 +153,38 @@ Properties:- Examples:- "gzip"- Standard gzip compression with fastest parameters.--### Advanced options--Here are the Advanced options specific to compress (Compress a remote).+ - "zstd"+ - Zstandard compression — fast modern algorithm offering adjustable speed-to-compression tradeoffs.#### --compress-level-GZIP compression level (-2 to 9).--Generally -1 (default, equivalent to 5) is recommended.-Levels 1 to 9 increase compression at the cost of speed. Going past 6-generally offers very little return.--Level -2 uses Huffman encoding only. Only use if you know what you-are doing.-Level 0 turns off compression.+GZIP (levels -2 to 9):+- -2 — Huffman encoding only. Only use if you know what you're doing.+- -1 (default) — recommended; equivalent to level 5.+- 0 — turns off compression.+- 1–9 — increase compression at the cost of speed. Going past 6 generally offers very little return.++ZSTD (levels 0 to 4):+- 0 — turns off compression entirely.+- 1 — fastest compression with the lowest ratio.+- 2 (default) — good balance of speed and compression.+- 3 — better compression, but uses about 2–3x more CPU than the default.+- 4 — best possible compression ratio (highest CPU cost).++Notes:+- Choose GZIP for wide compatibility; ZSTD for better speed/ratio tradeoffs.+- Negative gzip levels: -2 = Huffman-only, -1 = default (≈ level 5).Properties:- Config: level- Env Var: RCLONE_COMPRESS_LEVEL-- Type: int-- Default: -1+- Type: string+- Required: true++### Advanced options++Here are the Advanced options specific to compress (Compress a remote).#### --compress-ram-cache-limitdiff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yamlindex eba583e38..a3bc56081 100644--- a/fstest/test_all/config.yaml+++ b/fstest/test_all/config.yaml@@ -133,6 +133,9 @@ backends:- backend: "compress"remote: "TestCompress:"fastlist: false+ - backend: "compress"+ remote: "TestCompressZstd:"+ fastlist: false# - backend: "compress"# remote: "TestCompressSwift:"# fastlist: falsediff --git a/go.mod b/go.modindex d29278d89..9a526e04e 100644--- a/go.mod+++ b/go.mod@@ -1,6 +1,6 @@module github.com/rclone/rclone-go 1.24.0+go 1.24.4require (bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5@@ -11,6 +11,7 @@ require (github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358github.com/Files-com/files-sdk-go/v3 v3.2.242github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd+ github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0github.com/a8m/tree v0.0.0-20240104212747-2c8764a5f17egithub.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3github.com/abbot/go-http-auth v0.4.0@@ -173,6 +174,7 @@ require (github.com/gofrs/flock v0.12.1 // indirectgithub.com/gogo/protobuf v1.3.2 // indirectgithub.com/golang-jwt/jwt/v5 v5.3.0 // indirect+ github.com/google/btree v1.1.3 // indirectgithub.com/google/s2a-go v0.1.9 // indirectgithub.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirectgithub.com/googleapis/gax-go/v2 v2.15.0 // indirectdiff --git a/go.sum b/go.sumindex 88e176a4b..ab9b93c18 100644--- a/go.sum+++ b/go.sum@@ -90,6 +90,8 @@ github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUgithub.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=+github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 h1:iLDOF0rdGTrol/q8OfPIIs5kLD8XvA2q75o6Uq/tgak=+github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0/go.mod h1:DrEWcQJjz7t5iF2duaiyhg4jyoF0kxOD6LtECNGkZ/Q=github.com/a8m/tree v0.0.0-20240104212747-2c8764a5f17e h1:KMVieI1/Ub++GYfnhyFPoGE3g5TUiG4srE3TMGr5nM4=github.com/a8m/tree v0.0.0-20240104212747-2c8764a5f17e/go.mod h1:j5astEcUkZQX8lK+KKlQ3NRQ50f4EE8ZjyZpCz3mrH4=github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 h1:hhdWprfSpFbN7lz3W1gM40vOgvSh1WCSMxYD6gGB4Hs=@@ -344,6 +346,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWekgithub.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=+github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=+github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=