Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

vfs: add context parameter to New() for config propagation

Nick Craig-Wood7b8994ab32e6
diff --git a/backend/archive/base/base.go b/backend/archive/base/base.go
index 7f52a07b6..4e33857fc 100644
--- a/backend/archive/base/base.go
+++ b/backend/archive/base/base.go
@@ -36,7 +36,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs
// FIXME vfs cache?
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
- VFS := vfs.New(wrappedFs, nil)
+ VFS := vfs.New(ctx, wrappedFs, nil)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
diff --git a/backend/archive/squashfs/squashfs.go b/backend/archive/squashfs/squashfs.go
index 2610db276..0d6425862 100644
--- a/backend/archive/squashfs/squashfs.go
+++ b/backend/archive/squashfs/squashfs.go
@@ -50,7 +50,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
vfsOpt := vfscommon.Opt
vfsOpt.ReadWait = 0
- VFS := vfs.New(wrappedFs, &vfsOpt)
+ VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
diff --git a/backend/archive/zip/zip.go b/backend/archive/zip/zip.go
index 509b4317e..17511b76b 100644
--- a/backend/archive/zip/zip.go
+++ b/backend/archive/zip/zip.go
@@ -52,7 +52,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
vfsOpt := vfscommon.Opt
vfsOpt.ReadWait = 0
- VFS := vfs.New(wrappedFs, &vfsOpt)
+ VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go
index 7a7974ba4..f1abd16c0 100644
--- a/cmd/mountlib/mount.go
+++ b/cmd/mountlib/mount.go
@@ -369,7 +369,7 @@ func (m *MountPoint) Mount() (mountDaemon *os.Process, err error) {
}
}
- m.VFS = vfs.New(m.Fs, &m.VFSOpt)
+ m.VFS = vfs.New(context.Background(), m.Fs, &m.VFSOpt)
m.ErrChan, m.UnmountFn, err = m.MountFn(m.VFS, m.MountPoint, &m.MountOpt)
if err != nil {
diff --git a/cmd/nfsmount/nfsmount_test.go b/cmd/nfsmount/nfsmount_test.go
index a8402e901..746ba9f62 100644
--- a/cmd/nfsmount/nfsmount_test.go
+++ b/cmd/nfsmount/nfsmount_test.go
@@ -40,7 +40,7 @@ func TestMount(t *testing.T) {
nfs.Opt.HandleCacheDir = t.TempDir()
require.NoError(t, nfs.Opt.HandleCache.Set(cacheType))
// Check we can create a handler
- _, err := nfs.NewHandler(context.Background(), vfs.New(object.MemoryFs, nil), &nfs.Opt)
+ _, err := nfs.NewHandler(context.Background(), vfs.New(context.Background(), object.MemoryFs, nil), &nfs.Opt)
if errors.Is(err, nfs.ErrorSymlinkCacheNotSupported) || errors.Is(err, nfs.ErrorSymlinkCacheNoPermission) {
t.Skip(err.Error() + ": run with: go test -c && sudo setcap cap_dac_read_search+ep ./nfsmount.test && ./nfsmount.test -test.v")
}
diff --git a/cmd/serve/dlna/cds_test.go b/cmd/serve/dlna/cds_test.go
index cc9b9fca2..2a254d5d9 100644
--- a/cmd/serve/dlna/cds_test.go
+++ b/cmd/serve/dlna/cds_test.go
@@ -16,7 +16,7 @@ func TestMediaWithResources(t *testing.T) {
fs, err := localBackend.NewFs(context.Background(), "testdatafiles", "testdata/files", configmap.New())
require.NoError(t, err)
- myvfs := vfs.New(fs, nil)
+ myvfs := vfs.New(context.Background(), fs, nil)
{
rootNode, err := myvfs.Stat("")
require.NoError(t, err)
diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go
index 97728c0ba..84068049d 100644
--- a/cmd/serve/dlna/dlna.go
+++ b/cmd/serve/dlna/dlna.go
@@ -203,7 +203,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
waitChan: make(chan struct{}),
httpListenAddr: opt.ListenAddr,
f: f,
- vfs: vfs.New(f, vfsOpt),
+ vfs: vfs.New(ctx, f, vfsOpt),
}
s.services = map[string]UPnPService{
diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go
index e5b9897d8..401477780 100644
--- a/cmd/serve/ftp/ftp.go
+++ b/cmd/serve/ftp/ftp.go
@@ -203,7 +203,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
d.proxy = proxy.New(ctx, proxyOpt, vfsOpt)
d.userPass = make(map[string]string, 16)
} else {
- d.globalVFS = vfs.New(f, vfsOpt)
+ d.globalVFS = vfs.New(ctx, f, vfsOpt)
}
d.useTLS = d.opt.TLSKey != ""
diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index 25b538598..d4cadeeb7 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -188,7 +188,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
// override auth
s.opt.Auth.CustomAuthFn = s.auth
} else {
- s._vfs = vfs.New(f, vfsOpt)
+ s._vfs = vfs.New(ctx, f, vfsOpt)
}
s.server, err = libhttp.NewServer(ctx,
diff --git a/cmd/serve/nfs/cache_test.go b/cmd/serve/nfs/cache_test.go
index e38c06934..7eb55b657 100644
--- a/cmd/serve/nfs/cache_test.go
+++ b/cmd/serve/nfs/cache_test.go
@@ -122,7 +122,7 @@ func TestCache(t *testing.T) {
for _, cacheType := range []handleCache{cacheMemory, cacheDisk, cacheSymlink} {
t.Run(cacheType.String(), func(t *testing.T) {
h := &Handler{
- vfs: vfs.New(object.MemoryFs, nil),
+ vfs: vfs.New(context.Background(), object.MemoryFs, nil),
billyFS: billyFS,
}
h.vfs.Opt.MetadataExtension = ".metadata"
diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.go
index 020c92a60..ed4148253 100644
--- a/cmd/serve/nfs/nfs.go
+++ b/cmd/serve/nfs/nfs.go
@@ -94,7 +94,7 @@ func init() {
if err != nil {
return nil, err
}
- VFS := vfs.New(f, &vfsOpt)
+ VFS := vfs.New(ctx, f, &vfsOpt)
// Read opts
var opt = Opt // set default opts
err = configstruct.SetAny(in, &opt)
@@ -112,7 +112,7 @@ func Run(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
f = cmd.NewFsSrc(args)
cmd.Run(false, true, command, func() error {
- s, err := NewServer(context.Background(), vfs.New(f, &vfscommon.Opt), &Opt)
+ s, err := NewServer(context.Background(), vfs.New(context.Background(), f, &vfscommon.Opt), &Opt)
if err != nil {
return err
}
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.go
index efe2fb459..b1f5dd8dc 100644
--- a/cmd/serve/proxy/proxy.go
+++ b/cmd/serve/proxy/proxy.go
@@ -257,7 +257,7 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error)
// need to in memory. An attacker would find it easier to go
// after the unencrypted password in memory most likely.
entry := cacheEntry{
- vfs: vfs.New(f, &p.vfsOpt),
+ vfs: vfs.New(p.ctx, f, &p.vfsOpt),
pwHash: sha256.Sum256([]byte(auth)),
}
return entry, true, nil
diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go
index 0d690b642..f059ca062 100644
--- a/cmd/serve/s3/server.go
+++ b/cmd/serve/s3/server.go
@@ -94,7 +94,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
w.handler = proxyAuthMiddleware(w.handler, w)
w.handler = authPairMiddleware(w.handler, w)
} else {
- w._vfs = vfs.New(f, vfsOpt)
+ w._vfs = vfs.New(ctx, f, vfsOpt)
if len(opt.AuthKey) > 0 {
w.faker.AddAuthKeys(authList)
diff --git a/cmd/serve/sftp/connection.go b/cmd/serve/sftp/connection.go
index a9282fc5b..b869eaad6 100644
--- a/cmd/serve/sftp/connection.go
+++ b/cmd/serve/sftp/connection.go
@@ -358,7 +358,7 @@ func serveStdio(f fs.Fs) error {
stdin: os.Stdin,
stdout: os.Stdout,
}
- handlers := newVFSHandler(vfs.New(f, &vfscommon.Opt))
+ handlers := newVFSHandler(vfs.New(context.Background(), f, &vfscommon.Opt))
return serveChannel(sshChannel, handlers, "stdio")
}
diff --git a/cmd/serve/sftp/server.go b/cmd/serve/sftp/server.go
index 17889503d..91fbc2bb4 100644
--- a/cmd/serve/sftp/server.go
+++ b/cmd/serve/sftp/server.go
@@ -55,7 +55,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
if proxy.Opt.AuthProxy != "" {
s.proxy = proxy.New(ctx, proxyOpt, vfsOpt)
} else {
- s.vfs = vfs.New(f, vfsOpt)
+ s.vfs = vfs.New(ctx, f, vfsOpt)
}
err := s.configure()
if err != nil {
diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go
index 48f46550a..a674a6955 100644
--- a/cmd/serve/webdav/webdav.go
+++ b/cmd/serve/webdav/webdav.go
@@ -263,7 +263,7 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
// override auth
w.opt.Auth.CustomAuthFn = w.auth
} else {
- w._vfs = vfs.New(f, vfsOpt)
+ w._vfs = vfs.New(ctx, f, vfsOpt)
}
w.server, err = libhttp.NewServer(ctx,
diff --git a/vfs/file_test.go b/vfs/file_test.go
index 2a4b5fd47..ff561d2b0 100644
--- a/vfs/file_test.go
+++ b/vfs/file_test.go
@@ -202,7 +202,7 @@ func TestFileOpenReadUnknownSize(t *testing.T) {
assert.Equal(t, int64(-1), testObj.Size())
// create a VFS from that mockfs
- vfs := New(f, nil)
+ vfs := New(context.Background(), f, nil)
defer cleanupVFS(t, vfs)
// find the file
diff --git a/vfs/rc_test.go b/vfs/rc_test.go
index 9c70d526a..d2105d0b4 100644
--- a/vfs/rc_test.go
+++ b/vfs/rc_test.go
@@ -48,7 +48,7 @@ func TestRcGetVFS(t *testing.T) {
opt := vfscommon.Opt
opt.NoModTime = true
- vfs3 := New(r.Fremote, &opt)
+ vfs3 := New(context.Background(), r.Fremote, &opt)
defer vfs3.Shutdown()
vfs, err = getVFS(in)
diff --git a/vfs/vfs.go b/vfs/vfs.go
index 3ed8aa3d7..9006b798c 100644
--- a/vfs/vfs.go
+++ b/vfs/vfs.go
@@ -38,6 +38,7 @@ import (
"github.com/go-git/go-billy/v5"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/walk"
@@ -196,10 +197,18 @@ var (
)
// New creates a new VFS and root directory. If opt is nil, then
-// DefaultOpt will be used
-func New(f fs.Fs, opt *vfscommon.Options) *VFS {
+// DefaultOpt will be used.
+//
+// The ctx passed in is not used for cancellation but is used to find
+// the config in the context (if any) and filter config in the context
+// (if any).
+func New(ctx context.Context, f fs.Fs, opt *vfscommon.Options) *VFS {
fsDir := fs.NewDir("", time.Now())
- ctx, cancel := context.WithCancel(context.Background())
+ // Strip the ctx of any cancellation but copy the config across
+ newCtx := context.Background()
+ newCtx = fs.CopyConfig(newCtx, ctx)
+ newCtx = filter.CopyConfig(newCtx, ctx)
+ ctx, cancel := context.WithCancel(newCtx)
vfs := &VFS{
f: f,
ctx: ctx,
diff --git a/vfs/vfs_case_test.go b/vfs/vfs_case_test.go
index 26acd2210..7e634ad4a 100644
--- a/vfs/vfs_case_test.go
+++ b/vfs/vfs_case_test.go
@@ -36,12 +36,12 @@ func TestCaseSensitivity(t *testing.T) {
// Create a case-Sensitive and case-INsensitive VFS
optCS := vfscommon.Opt
optCS.CaseInsensitive = false
- vfsCS := New(r.Fremote, &optCS)
+ vfsCS := New(context.Background(), r.Fremote, &optCS)
defer cleanupVFS(t, vfsCS)
optCI := vfscommon.Opt
optCI.CaseInsensitive = true
- vfsCI := New(r.Fremote, &optCI)
+ vfsCI := New(context.Background(), r.Fremote, &optCI)
defer cleanupVFS(t, vfsCI)
// Run basic checks that must pass on VFS of any type.
@@ -180,7 +180,7 @@ func TestUnicodeNormalization(t *testing.T) {
// Create VFS
opt := vfscommon.Opt
- vfs := New(r.Fremote, &opt)
+ vfs := New(context.Background(), r.Fremote, &opt)
defer cleanupVFS(t, vfs)
// assert that both files are found under NFD-normalized names
diff --git a/vfs/vfs_test.go b/vfs/vfs_test.go
index 30e10a920..96d81fdb9 100644
--- a/vfs/vfs_test.go
+++ b/vfs/vfs_test.go
@@ -47,7 +47,7 @@ func cleanupVFS(t *testing.T, vfs *VFS) {
// Create a new VFS
func newTestVFSOpt(t *testing.T, opt *vfscommon.Options) (r *fstest.Run, vfs *VFS) {
r = fstest.NewRun(t)
- vfs = New(r.Fremote, opt)
+ vfs = New(context.Background(), r.Fremote, opt)
t.Cleanup(func() {
cleanupVFS(t, vfs)
})
@@ -144,7 +144,7 @@ func TestVFSNew(t *testing.T) {
// Check that we get the same VFS if we ask for it again with
// the same options
- vfs2 := New(r.Fremote, nil)
+ vfs2 := New(context.Background(), r.Fremote, nil)
assert.Equal(t, fmt.Sprintf("%p", vfs), fmt.Sprintf("%p", vfs2))
checkActiveCacheEntries(1)
diff --git a/vfs/vfstest/submount.go b/vfs/vfstest/submount.go
index ac6400cdf..62fc78fb3 100644
--- a/vfs/vfstest/submount.go
+++ b/vfs/vfstest/submount.go
@@ -41,7 +41,7 @@ func (r *Run) startMountSubProcess() {
// If testing the VFS we don't start a subprocess, we just use
// the VFS directly
if r.useVFS {
- vfs := vfs.New(r.fremote, r.vfsOpt)
+ vfs := vfs.New(context.Background(), r.fremote, r.vfsOpt)
r.os = vfsOs{vfs}
return
}