"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve nfs: fix writing files via Finder on macOS - fixes #7503
nielash5638a3841fdd
diff --git a/cmd/mountlib/mount.md b/cmd/mountlib/mount.mdindex 9d367ca1e..6b3ccc620 100644--- a/cmd/mountlib/mount.md+++ b/cmd/mountlib/mount.md@@ -263,6 +263,10 @@ This method spins up an NFS server using [serve nfs](/commands/rclone_serve_nfs/it to the specified mountpoint. If you run this in background mode using |--daemon|, you will need tosend SIGTERM signal to the rclone process using |kill| command to stop the mount.+Note that `--nfs-cache-handle-limit` controls the maximum number of cached file handles stored by the `nfsmount` caching handler.+This should not be set too low or you may experience errors when trying to access files. The default is 1000000,+but consider lowering this limit if the server's system resource usage causes problems.+#### macFUSE NotesIf installing macFUSE using [dmg packages](https://github.com/osxfuse/osxfuse/releases) fromdiff --git a/cmd/nfsmount/nfsmount.go b/cmd/nfsmount/nfsmount.goindex e515e995a..610c2f655 100644--- a/cmd/nfsmount/nfsmount.go+++ b/cmd/nfsmount/nfsmount.go@@ -22,7 +22,8 @@ import ()var (- sudo = false+ sudo = false+ nfsServerOpt nfs.Options)func init() {@@ -33,10 +34,11 @@ func init() {mountlib.AddRc(name, mount)cmdFlags := cmd.Flags()flags.BoolVarP(cmdFlags, &sudo, "sudo", "", sudo, "Use sudo to run the mount command as root.", "")+ nfs.AddFlags(cmdFlags, &nfsServerOpt)}func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (asyncerrors <-chan error, unmount func() error, err error) {- s, err := nfs.NewServer(context.Background(), VFS, &nfs.Options{})+ s, err := nfs.NewServer(context.Background(), VFS, &nfsServerOpt)if err != nil {return}diff --git a/cmd/serve/nfs/filesystem.go b/cmd/serve/nfs/filesystem.goindex 25916e0dd..9d3db34d9 100644--- a/cmd/serve/nfs/filesystem.go+++ b/cmd/serve/nfs/filesystem.go@@ -22,27 +22,35 @@ type FS struct {// ReadDir implements read dirfunc (f *FS) ReadDir(path string) (dir []os.FileInfo, err error) {+ fs.Debugf("nfs", "ReadDir %v", path)return f.vfs.ReadDir(path)}// Create implements creating new filesfunc (f *FS) Create(filename string) (billy.File, error) {+ fs.Debugf("nfs", "Create %v", filename)return f.vfs.Create(filename)}// Open opens a filefunc (f *FS) Open(filename string) (billy.File, error) {- return f.vfs.Open(filename)+ file, err := f.vfs.Open(filename)+ fs.Debugf("nfs", "Open %v file: %v err: %v", filename, file, err)+ return file, err}// OpenFile opens a filefunc (f *FS) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) {- return f.vfs.OpenFile(filename, flag, perm)+ file, err := f.vfs.OpenFile(filename, flag, perm)+ fs.Debugf("nfs", "OpenFile %v flag: %v perm: %v file: %v err: %v", filename, flag, perm, file, err)+ return file, err}// Stat gets the file statfunc (f *FS) Stat(filename string) (os.FileInfo, error) {- return f.vfs.Stat(filename)+ node, err := f.vfs.Stat(filename)+ fs.Debugf("nfs", "Stat %v node: %v err: %v", filename, node, err)+ return node, err}// Rename renames a file@@ -85,7 +93,9 @@ func (f *FS) MkdirAll(filename string, perm os.FileMode) error {// Lstat gets the stats for symlinkfunc (f *FS) Lstat(filename string) (os.FileInfo, error) {- return f.vfs.Stat(filename)+ node, err := f.vfs.Stat(filename)+ fs.Debugf("nfs", "Lstat %v node: %v err: %v", filename, node, err)+ return node, err}// Symlink is not supported over NFSdiff --git a/cmd/serve/nfs/handler.go b/cmd/serve/nfs/handler.goindex 7d9a8e4d1..9cfa82999 100644--- a/cmd/serve/nfs/handler.go+++ b/cmd/serve/nfs/handler.go@@ -5,22 +5,32 @@ package nfsimport ("context"+ "fmt""net""github.com/go-git/go-billy/v5"+ "github.com/rclone/rclone/fs""github.com/rclone/rclone/vfs""github.com/willscott/go-nfs"nfshelper "github.com/willscott/go-nfs/helpers")// NewBackendAuthHandler creates a handler for the provided filesystem-func NewBackendAuthHandler(vfs *vfs.VFS) nfs.Handler {- return &BackendAuthHandler{vfs}+func NewBackendAuthHandler(vfs *vfs.VFS, opt *Options) nfs.Handler {+ handler := &BackendAuthHandler{+ vfs: vfs,+ opt: opt,+ }+ handler.opt.HandleLimit = handler.opt.Limit()+ handler.cache = cacheHelper(handler, handler.HandleLimit())+ return handler}// BackendAuthHandler returns a NFS backing that exposes a given file system in response to all mount requests.type BackendAuthHandler struct {- vfs *vfs.VFS+ vfs *vfs.VFS+ opt *Options+ cache nfs.Handler}// Mount backs Mount RPC Requests, allowing for access control policies.@@ -50,17 +60,17 @@ func (h *BackendAuthHandler) FSStat(ctx context.Context, f billy.Filesystem, s *// ToHandle handled by CachingHandlerfunc (h *BackendAuthHandler) ToHandle(f billy.Filesystem, s []string) []byte {- return []byte{}+ return h.cache.ToHandle(f, s)}// FromHandle handled by CachingHandler-func (h *BackendAuthHandler) FromHandle([]byte) (billy.Filesystem, []string, error) {- return nil, []string{}, nil+func (h *BackendAuthHandler) FromHandle(b []byte) (billy.Filesystem, []string, error) {+ return h.cache.FromHandle(b)}// HandleLimit handled by cachingHandlerfunc (h *BackendAuthHandler) HandleLimit() int {- return -1+ return h.opt.HandleLimit}// InvalidateHandle is called on removes or renames@@ -68,8 +78,136 @@ func (h *BackendAuthHandler) InvalidateHandle(billy.Filesystem, []byte) error {return nil}-func newHandler(vfs *vfs.VFS) nfs.Handler {- handler := NewBackendAuthHandler(vfs)- cacheHelper := nfshelper.NewCachingHandler(handler, 1024)+func newHandler(vfs *vfs.VFS, opt *Options) nfs.Handler {+ handler := NewBackendAuthHandler(vfs, opt)+ nfs.SetLogger(&LogIntercepter{Level: nfs.DebugLevel})+ return handler+}++func cacheHelper(handler nfs.Handler, limit int) nfs.Handler {+ cacheHelper := nfshelper.NewCachingHandler(handler, limit)return cacheHelper}++// Limit overrides the --nfs-cache-handle-limit value if out-of-range+func (o *Options) Limit() int {+ if o.HandleLimit < 0 {+ return 1000000+ }+ if o.HandleLimit <= 5 {+ return 5+ }+ return o.HandleLimit+}++// LogIntercepter intercepts noisy go-nfs logs and reroutes them to DEBUG+type LogIntercepter struct {+ Level nfs.LogLevel+}++// Intercept intercepts go-nfs logs and calls fs.Debugf instead+func (l *LogIntercepter) Intercept(args ...interface{}) {+ args = append([]interface{}{"[NFS DEBUG] "}, args...)+ argsS := fmt.Sprint(args...)+ fs.Debugf(nil, "%v", argsS)+}++// Interceptf intercepts go-nfs logs and calls fs.Debugf instead+func (l *LogIntercepter) Interceptf(format string, args ...interface{}) {+ fs.Debugf(nil, "[NFS DEBUG] "+format, args...)+}++// Debug reroutes go-nfs Debug messages to Intercept+func (l *LogIntercepter) Debug(args ...interface{}) {+ l.Intercept(args...)+}++// Debugf reroutes go-nfs Debugf messages to Interceptf+func (l *LogIntercepter) Debugf(format string, args ...interface{}) {+ l.Interceptf(format, args...)+}++// Error reroutes go-nfs Error messages to Intercept+func (l *LogIntercepter) Error(args ...interface{}) {+ l.Intercept(args...)+}++// Errorf reroutes go-nfs Errorf messages to Interceptf+func (l *LogIntercepter) Errorf(format string, args ...interface{}) {+ l.Interceptf(format, args...)+}++// Fatal reroutes go-nfs Fatal messages to Intercept+func (l *LogIntercepter) Fatal(args ...interface{}) {+ l.Intercept(args...)+}++// Fatalf reroutes go-nfs Fatalf messages to Interceptf+func (l *LogIntercepter) Fatalf(format string, args ...interface{}) {+ l.Interceptf(format, args...)+}++// GetLevel returns the nfs.LogLevel+func (l *LogIntercepter) GetLevel() nfs.LogLevel {+ return l.Level+}++// Info reroutes go-nfs Info messages to Intercept+func (l *LogIntercepter) Info(args ...interface{}) {+ l.Intercept(args...)+}++// Infof reroutes go-nfs Infof messages to Interceptf+func (l *LogIntercepter) Infof(format string, args ...interface{}) {+ l.Interceptf(format, args...)+}++// Panic reroutes go-nfs Panic messages to Intercept+func (l *LogIntercepter) Panic(args ...interface{}) {+ l.Intercept(args...)+}++// Panicf reroutes go-nfs Panicf messages to Interceptf+func (l *LogIntercepter) Panicf(format string, args ...interface{}) {+ l.Interceptf(format, args...)+}++// ParseLevel parses the nfs.LogLevel+func (l *LogIntercepter) ParseLevel(level string) (nfs.LogLevel, error) {+ return nfs.Log.ParseLevel(level)+}++// Print reroutes go-nfs Print messages to Intercept+func (l *LogIntercepter) Print(args ...interface{}) {+ l.Intercept(args...)+}++// Printf reroutes go-nfs Printf messages to Intercept+func (l *LogIntercepter) Printf(format string, args ...interface{}) {+ l.Interceptf(format, args...)+}++// SetLevel sets the nfs.LogLevel+func (l *LogIntercepter) SetLevel(level nfs.LogLevel) {+ l.Level = level+}++// Trace reroutes go-nfs Trace messages to Intercept+func (l *LogIntercepter) Trace(args ...interface{}) {+ l.Intercept(args...)+}++// Tracef reroutes go-nfs Tracef messages to Interceptf+func (l *LogIntercepter) Tracef(format string, args ...interface{}) {+ l.Interceptf(format, args...)+}++// Warn reroutes go-nfs Warn messages to Intercept+func (l *LogIntercepter) Warn(args ...interface{}) {+ l.Intercept(args...)+}++// Warnf reroutes go-nfs Warnf messages to Interceptf+func (l *LogIntercepter) Warnf(format string, args ...interface{}) {+ l.Interceptf(format, args...)+}diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.goindex 7a67f679e..4ef481878 100644--- a/cmd/serve/nfs/nfs.go+++ b/cmd/serve/nfs/nfs.go@@ -25,15 +25,17 @@ import (// Options contains options for the NFS Servertype Options struct {- ListenAddr string // Port to listen on+ ListenAddr string // Port to listen on+ HandleLimit int // max file handles cached by go-nfs CachingHandler}var opt Options-// AddFlags adds flags for the sftp+// AddFlags adds flags for serve nfs (and nfsmount)func AddFlags(flagSet *pflag.FlagSet, Opt *Options) {rc.AddOption("nfs", &Opt)flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to", "")+ flags.IntVarP(flagSet, &Opt.HandleLimit, "nfs-cache-handle-limit", "", 1000000, "max file handles cached simultaneously (min 5)", "")}func init() {@@ -72,7 +74,9 @@ NFS mount over local network, you need to specify the listening address and portModifying files through NFS protocol requires VFS caching. Usually you will need to specify ` + "`--vfs-cache-mode`" + `in order to be able to write to the mountpoint (full is recommended). If you don't specify VFS cache mode,-the mount will be read-only.+the mount will be read-only. Note also that ` + "`--nfs-cache-handle-limit`" + ` controls the maximum number of cached file handles stored by the caching handler.+This should not be set too low or you may experience errors when trying to access files. The default is ` + "`1000000`" + `, but consider lowering this limit if+the server's system resource usage causes problems.To serve NFS over the network use following command:diff --git a/cmd/serve/nfs/server.go b/cmd/serve/nfs/server.goindex c66e687a9..a7396cdb0 100644--- a/cmd/serve/nfs/server.go+++ b/cmd/serve/nfs/server.go@@ -36,7 +36,7 @@ func NewServer(ctx context.Context, vfs *vfs.VFS, opt *Options) (s *Server, errctx: ctx,opt: *opt,}- s.handler = newHandler(vfs)+ s.handler = newHandler(vfs, opt)s.listener, err = net.Listen("tcp", s.opt.ListenAddr)if err != nil {fs.Errorf(nil, "NFS server failed to listen: %v\n", err)