"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
local: remove fadvise calls that cause spinlock contention
Patrick Farrell7ca667d35d6b
diff --git a/backend/local/fadvise_other.go b/backend/local/fadvise_other.godeleted file mode 100644index c07a7d01c..000000000--- a/backend/local/fadvise_other.go+++ /dev/null@@ -1,12 +0,0 @@-//go:build !linux--package local--import (- "io"- "os"-)--func newFadviseReadCloser(o *Object, f *os.File, offset, limit int64) io.ReadCloser {- return f-}diff --git a/backend/local/fadvise_unix.go b/backend/local/fadvise_unix.godeleted file mode 100644index f463e017c..000000000--- a/backend/local/fadvise_unix.go+++ /dev/null@@ -1,165 +0,0 @@-//go:build linux--package local--import (- "io"- "os"-- "github.com/rclone/rclone/fs"- "golang.org/x/sys/unix"-)--// fadvise provides means to automate freeing pages in kernel page cache for-// a given file descriptor as the file is sequentially processed (read or-// written).-//-// When copying a file to a remote backend all the file content is read by-// kernel and put to page cache to make future reads faster.-// This causes memory pressure visible in both memory usage and CPU consumption-// and can even cause OOM errors in applications consuming large amounts memory.-//-// In case of an upload to a remote backend, there is no benefits from caching.-//-// fadvise would orchestrate calling POSIX_FADV_DONTNEED-//-// POSIX_FADV_DONTNEED attempts to free cached pages associated-// with the specified region. This is useful, for example, while-// streaming large files. A program may periodically request the-// kernel to free cached data that has already been used, so that-// more useful cached pages are not discarded instead.-//-// Requests to discard partial pages are ignored. It is-// preferable to preserve needed data than discard unneeded data.-// If the application requires that data be considered for-// discarding, then offset and len must be page-aligned.-//-// The implementation may attempt to write back dirty pages in-// the specified region, but this is not guaranteed. Any-// unwritten dirty pages will not be freed. If the application-// wishes to ensure that dirty pages will be released, it should-// call fsync(2) or fdatasync(2) first.-type fadvise struct {- o *Object- fd int- lastPos int64- curPos int64- windowSize int64-- freePagesCh chan offsetLength- doneCh chan struct{}-}--type offsetLength struct {- offset int64- length int64-}--const (- defaultAllowPages = 32- defaultWorkerQueueSize = 64-)--func newFadvise(o *Object, fd int, offset int64) *fadvise {- f := &fadvise{- o: o,- fd: fd,- lastPos: offset,- curPos: offset,- windowSize: int64(os.Getpagesize()) * defaultAllowPages,-- freePagesCh: make(chan offsetLength, defaultWorkerQueueSize),- doneCh: make(chan struct{}),- }- go f.worker()-- return f-}--// sequential configures readahead strategy in Linux kernel.-//-// Under Linux, POSIX_FADV_NORMAL sets the readahead window to the-// default size for the backing device; POSIX_FADV_SEQUENTIAL doubles-// this size, and POSIX_FADV_RANDOM disables file readahead entirely.-func (f *fadvise) sequential(limit int64) bool {- l := int64(0)- if limit > 0 {- l = limit- }- if err := unix.Fadvise(f.fd, f.curPos, l, unix.FADV_SEQUENTIAL); err != nil {- fs.Debugf(f.o, "fadvise sequential failed on file descriptor %d: %s", f.fd, err)- return false- }-- return true-}--func (f *fadvise) next(n int) {- f.curPos += int64(n)- f.freePagesIfNeeded()-}--func (f *fadvise) freePagesIfNeeded() {- if f.curPos >= f.lastPos+f.windowSize {- f.freePages()- }-}--func (f *fadvise) freePages() {- f.freePagesCh <- offsetLength{f.lastPos, f.curPos - f.lastPos}- f.lastPos = f.curPos-}--func (f *fadvise) worker() {- for p := range f.freePagesCh {- if err := unix.Fadvise(f.fd, p.offset, p.length, unix.FADV_DONTNEED); err != nil {- fs.Debugf(f.o, "fadvise dontneed failed on file descriptor %d: %s", f.fd, err)- }- }-- close(f.doneCh)-}--func (f *fadvise) wait() {- close(f.freePagesCh)- <-f.doneCh-}--type fadviseReadCloser struct {- *fadvise- inner io.ReadCloser-}--// newFadviseReadCloser wraps os.File so that reading from that file would-// remove already consumed pages from kernel page cache.-// In addition to that it instructs kernel to double the readahead window to-// make sequential reads faster.-// See also fadvise.-func newFadviseReadCloser(o *Object, f *os.File, offset, limit int64) io.ReadCloser {- r := fadviseReadCloser{- fadvise: newFadvise(o, int(f.Fd()), offset),- inner: f,- }-- // If syscall failed it's likely that the subsequent syscalls to that- // file descriptor would also fail. In that case return the provided os.File- // pointer.- if !r.sequential(limit) {- r.wait()- return f- }-- return r-}--func (f fadviseReadCloser) Read(p []byte) (n int, err error) {- n, err = f.inner.Read(p)- f.next(n)- return-}--func (f fadviseReadCloser) Close() error {- f.freePages()- f.wait()- return f.inner.Close()-}diff --git a/backend/local/local.go b/backend/local/local.goindex da21c483f..e5ad22ab9 100644--- a/backend/local/local.go+++ b/backend/local/local.go@@ -1168,7 +1168,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {var fd *os.Filefd, err = file.Open(o.path)if fd != nil {- in = newFadviseReadCloser(o, fd, 0, 0)+ in = fd}} else {in, err = o.openTranslatedLink(0, -1)@@ -1374,7 +1374,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Readif err != nil {return}- wrappedFd := readers.NewLimitedReadCloser(newFadviseReadCloser(o, fd, offset, limit), limit)+ wrappedFd := readers.NewLimitedReadCloser(fd, limit)if offset != 0 {// seek the object_, err = fd.Seek(offset, io.SeekStart)