"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: add context parameter to New() for config propagation
Nick Craig-Wood7b8994ab32e6
diff --git a/backend/archive/base/base.go b/backend/archive/base/base.goindex 7f52a07b6..4e33857fc 100644--- a/backend/archive/base/base.go+++ b/backend/archive/base/base.go@@ -36,7 +36,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs// FIXME vfs cache?// FIXME could factor out ReadFileHandle and just use that rather than the full VFSfs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)- VFS := vfs.New(wrappedFs, nil)+ VFS := vfs.New(ctx, wrappedFs, nil)node, err := VFS.Stat(remote)if err != nil {return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)diff --git a/backend/archive/squashfs/squashfs.go b/backend/archive/squashfs/squashfs.goindex 2610db276..0d6425862 100644--- a/backend/archive/squashfs/squashfs.go+++ b/backend/archive/squashfs/squashfs.go@@ -50,7 +50,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)vfsOpt := vfscommon.OptvfsOpt.ReadWait = 0- VFS := vfs.New(wrappedFs, &vfsOpt)+ VFS := vfs.New(ctx, wrappedFs, &vfsOpt)node, err := VFS.Stat(remote)if err != nil {return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)diff --git a/backend/archive/zip/zip.go b/backend/archive/zip/zip.goindex 509b4317e..17511b76b 100644--- a/backend/archive/zip/zip.go+++ b/backend/archive/zip/zip.go@@ -52,7 +52,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)vfsOpt := vfscommon.OptvfsOpt.ReadWait = 0- VFS := vfs.New(wrappedFs, &vfsOpt)+ VFS := vfs.New(ctx, wrappedFs, &vfsOpt)node, err := VFS.Stat(remote)if err != nil {return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.goindex 7a7974ba4..f1abd16c0 100644--- a/cmd/mountlib/mount.go+++ b/cmd/mountlib/mount.go@@ -369,7 +369,7 @@ func (m *MountPoint) Mount() (mountDaemon *os.Process, err error) {}}- m.VFS = vfs.New(m.Fs, &m.VFSOpt)+ m.VFS = vfs.New(context.Background(), m.Fs, &m.VFSOpt)m.ErrChan, m.UnmountFn, err = m.MountFn(m.VFS, m.MountPoint, &m.MountOpt)if err != nil {diff --git a/cmd/nfsmount/nfsmount_test.go b/cmd/nfsmount/nfsmount_test.goindex a8402e901..746ba9f62 100644--- a/cmd/nfsmount/nfsmount_test.go+++ b/cmd/nfsmount/nfsmount_test.go@@ -40,7 +40,7 @@ func TestMount(t *testing.T) {nfs.Opt.HandleCacheDir = t.TempDir()require.NoError(t, nfs.Opt.HandleCache.Set(cacheType))// Check we can create a handler- _, err := nfs.NewHandler(context.Background(), vfs.New(object.MemoryFs, nil), &nfs.Opt)+ _, err := nfs.NewHandler(context.Background(), vfs.New(context.Background(), object.MemoryFs, nil), &nfs.Opt)if errors.Is(err, nfs.ErrorSymlinkCacheNotSupported) || errors.Is(err, nfs.ErrorSymlinkCacheNoPermission) {t.Skip(err.Error() + ": run with: go test -c && sudo setcap cap_dac_read_search+ep ./nfsmount.test && ./nfsmount.test -test.v")}diff --git a/cmd/serve/dlna/cds_test.go b/cmd/serve/dlna/cds_test.goindex cc9b9fca2..2a254d5d9 100644--- a/cmd/serve/dlna/cds_test.go+++ b/cmd/serve/dlna/cds_test.go@@ -16,7 +16,7 @@ func TestMediaWithResources(t *testing.T) {fs, err := localBackend.NewFs(context.Background(), "testdatafiles", "testdata/files", configmap.New())require.NoError(t, err)- myvfs := vfs.New(fs, nil)+ myvfs := vfs.New(context.Background(), fs, nil){rootNode, err := myvfs.Stat("")require.NoError(t, err)diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.goindex 97728c0ba..84068049d 100644--- a/cmd/serve/dlna/dlna.go+++ b/cmd/serve/dlna/dlna.go@@ -203,7 +203,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.OptwaitChan: make(chan struct{}),httpListenAddr: opt.ListenAddr,f: f,- vfs: vfs.New(f, vfsOpt),+ vfs: vfs.New(ctx, f, vfsOpt),}s.services = map[string]UPnPService{diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.goindex e5b9897d8..401477780 100644--- a/cmd/serve/ftp/ftp.go+++ b/cmd/serve/ftp/ftp.go@@ -203,7 +203,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Optd.proxy = proxy.New(ctx, proxyOpt, vfsOpt)d.userPass = make(map[string]string, 16)} else {- d.globalVFS = vfs.New(f, vfsOpt)+ d.globalVFS = vfs.New(ctx, f, vfsOpt)}d.useTLS = d.opt.TLSKey != ""diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.goindex 25b538598..d4cadeeb7 100644--- a/cmd/serve/http/http.go+++ b/cmd/serve/http/http.go@@ -188,7 +188,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt// override auths.opt.Auth.CustomAuthFn = s.auth} else {- s._vfs = vfs.New(f, vfsOpt)+ s._vfs = vfs.New(ctx, f, vfsOpt)}s.server, err = libhttp.NewServer(ctx,diff --git a/cmd/serve/nfs/cache_test.go b/cmd/serve/nfs/cache_test.goindex e38c06934..7eb55b657 100644--- a/cmd/serve/nfs/cache_test.go+++ b/cmd/serve/nfs/cache_test.go@@ -122,7 +122,7 @@ func TestCache(t *testing.T) {for _, cacheType := range []handleCache{cacheMemory, cacheDisk, cacheSymlink} {t.Run(cacheType.String(), func(t *testing.T) {h := &Handler{- vfs: vfs.New(object.MemoryFs, nil),+ vfs: vfs.New(context.Background(), object.MemoryFs, nil),billyFS: billyFS,}h.vfs.Opt.MetadataExtension = ".metadata"diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.goindex 020c92a60..ed4148253 100644--- a/cmd/serve/nfs/nfs.go+++ b/cmd/serve/nfs/nfs.go@@ -94,7 +94,7 @@ func init() {if err != nil {return nil, err}- VFS := vfs.New(f, &vfsOpt)+ VFS := vfs.New(ctx, f, &vfsOpt)// Read optsvar opt = Opt // set default optserr = configstruct.SetAny(in, &opt)@@ -112,7 +112,7 @@ func Run(command *cobra.Command, args []string) {cmd.CheckArgs(1, 1, command, args)f = cmd.NewFsSrc(args)cmd.Run(false, true, command, func() error {- s, err := NewServer(context.Background(), vfs.New(f, &vfscommon.Opt), &Opt)+ s, err := NewServer(context.Background(), vfs.New(context.Background(), f, &vfscommon.Opt), &Opt)if err != nil {return err}diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.goindex efe2fb459..b1f5dd8dc 100644--- a/cmd/serve/proxy/proxy.go+++ b/cmd/serve/proxy/proxy.go@@ -257,7 +257,7 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error)// need to in memory. An attacker would find it easier to go// after the unencrypted password in memory most likely.entry := cacheEntry{- vfs: vfs.New(f, &p.vfsOpt),+ vfs: vfs.New(p.ctx, f, &p.vfsOpt),pwHash: sha256.Sum256([]byte(auth)),}return entry, true, nildiff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.goindex 0d690b642..f059ca062 100644--- a/cmd/serve/s3/server.go+++ b/cmd/serve/s3/server.go@@ -94,7 +94,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Optw.handler = proxyAuthMiddleware(w.handler, w)w.handler = authPairMiddleware(w.handler, w)} else {- w._vfs = vfs.New(f, vfsOpt)+ w._vfs = vfs.New(ctx, f, vfsOpt)if len(opt.AuthKey) > 0 {w.faker.AddAuthKeys(authList)diff --git a/cmd/serve/sftp/connection.go b/cmd/serve/sftp/connection.goindex a9282fc5b..b869eaad6 100644--- a/cmd/serve/sftp/connection.go+++ b/cmd/serve/sftp/connection.go@@ -358,7 +358,7 @@ func serveStdio(f fs.Fs) error {stdin: os.Stdin,stdout: os.Stdout,}- handlers := newVFSHandler(vfs.New(f, &vfscommon.Opt))+ handlers := newVFSHandler(vfs.New(context.Background(), f, &vfscommon.Opt))return serveChannel(sshChannel, handlers, "stdio")}diff --git a/cmd/serve/sftp/server.go b/cmd/serve/sftp/server.goindex 17889503d..91fbc2bb4 100644--- a/cmd/serve/sftp/server.go+++ b/cmd/serve/sftp/server.go@@ -55,7 +55,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Optif proxy.Opt.AuthProxy != "" {s.proxy = proxy.New(ctx, proxyOpt, vfsOpt)} else {- s.vfs = vfs.New(f, vfsOpt)+ s.vfs = vfs.New(ctx, f, vfsOpt)}err := s.configure()if err != nil {diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.goindex 48f46550a..a674a6955 100644--- a/cmd/serve/webdav/webdav.go+++ b/cmd/serve/webdav/webdav.go@@ -263,7 +263,7 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt// override authw.opt.Auth.CustomAuthFn = w.auth} else {- w._vfs = vfs.New(f, vfsOpt)+ w._vfs = vfs.New(ctx, f, vfsOpt)}w.server, err = libhttp.NewServer(ctx,diff --git a/vfs/file_test.go b/vfs/file_test.goindex 2a4b5fd47..ff561d2b0 100644--- a/vfs/file_test.go+++ b/vfs/file_test.go@@ -202,7 +202,7 @@ func TestFileOpenReadUnknownSize(t *testing.T) {assert.Equal(t, int64(-1), testObj.Size())// create a VFS from that mockfs- vfs := New(f, nil)+ vfs := New(context.Background(), f, nil)defer cleanupVFS(t, vfs)// find the filediff --git a/vfs/rc_test.go b/vfs/rc_test.goindex 9c70d526a..d2105d0b4 100644--- a/vfs/rc_test.go+++ b/vfs/rc_test.go@@ -48,7 +48,7 @@ func TestRcGetVFS(t *testing.T) {opt := vfscommon.Optopt.NoModTime = true- vfs3 := New(r.Fremote, &opt)+ vfs3 := New(context.Background(), r.Fremote, &opt)defer vfs3.Shutdown()vfs, err = getVFS(in)diff --git a/vfs/vfs.go b/vfs/vfs.goindex 3ed8aa3d7..9006b798c 100644--- a/vfs/vfs.go+++ b/vfs/vfs.go@@ -38,6 +38,7 @@ import ("github.com/go-git/go-billy/v5""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/cache"+ "github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/log""github.com/rclone/rclone/fs/rc""github.com/rclone/rclone/fs/walk"@@ -196,10 +197,18 @@ var ()// New creates a new VFS and root directory. If opt is nil, then-// DefaultOpt will be used-func New(f fs.Fs, opt *vfscommon.Options) *VFS {+// DefaultOpt will be used.+//+// The ctx passed in is not used for cancellation but is used to find+// the config in the context (if any) and filter config in the context+// (if any).+func New(ctx context.Context, f fs.Fs, opt *vfscommon.Options) *VFS {fsDir := fs.NewDir("", time.Now())- ctx, cancel := context.WithCancel(context.Background())+ // Strip the ctx of any cancellation but copy the config across+ newCtx := context.Background()+ newCtx = fs.CopyConfig(newCtx, ctx)+ newCtx = filter.CopyConfig(newCtx, ctx)+ ctx, cancel := context.WithCancel(newCtx)vfs := &VFS{f: f,ctx: ctx,diff --git a/vfs/vfs_case_test.go b/vfs/vfs_case_test.goindex 26acd2210..7e634ad4a 100644--- a/vfs/vfs_case_test.go+++ b/vfs/vfs_case_test.go@@ -36,12 +36,12 @@ func TestCaseSensitivity(t *testing.T) {// Create a case-Sensitive and case-INsensitive VFSoptCS := vfscommon.OptoptCS.CaseInsensitive = false- vfsCS := New(r.Fremote, &optCS)+ vfsCS := New(context.Background(), r.Fremote, &optCS)defer cleanupVFS(t, vfsCS)optCI := vfscommon.OptoptCI.CaseInsensitive = true- vfsCI := New(r.Fremote, &optCI)+ vfsCI := New(context.Background(), r.Fremote, &optCI)defer cleanupVFS(t, vfsCI)// Run basic checks that must pass on VFS of any type.@@ -180,7 +180,7 @@ func TestUnicodeNormalization(t *testing.T) {// Create VFSopt := vfscommon.Opt- vfs := New(r.Fremote, &opt)+ vfs := New(context.Background(), r.Fremote, &opt)defer cleanupVFS(t, vfs)// assert that both files are found under NFD-normalized namesdiff --git a/vfs/vfs_test.go b/vfs/vfs_test.goindex 30e10a920..96d81fdb9 100644--- a/vfs/vfs_test.go+++ b/vfs/vfs_test.go@@ -47,7 +47,7 @@ func cleanupVFS(t *testing.T, vfs *VFS) {// Create a new VFSfunc newTestVFSOpt(t *testing.T, opt *vfscommon.Options) (r *fstest.Run, vfs *VFS) {r = fstest.NewRun(t)- vfs = New(r.Fremote, opt)+ vfs = New(context.Background(), r.Fremote, opt)t.Cleanup(func() {cleanupVFS(t, vfs)})@@ -144,7 +144,7 @@ func TestVFSNew(t *testing.T) {// Check that we get the same VFS if we ask for it again with// the same options- vfs2 := New(r.Fremote, nil)+ vfs2 := New(context.Background(), r.Fremote, nil)assert.Equal(t, fmt.Sprintf("%p", vfs), fmt.Sprintf("%p", vfs2))checkActiveCacheEntries(1)diff --git a/vfs/vfstest/submount.go b/vfs/vfstest/submount.goindex ac6400cdf..62fc78fb3 100644--- a/vfs/vfstest/submount.go+++ b/vfs/vfstest/submount.go@@ -41,7 +41,7 @@ func (r *Run) startMountSubProcess() {// If testing the VFS we don't start a subprocess, we just use// the VFS directlyif r.useVFS {- vfs := vfs.New(r.fremote, r.vfsOpt)+ vfs := vfs.New(context.Background(), r.fremote, r.vfsOpt)r.os = vfsOs{vfs}return}