"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve nfs: make metadata files have special file handles
Nick Craig-Wood205667143c80
diff --git a/cmd/nfsmount/nfsmount_test.go b/cmd/nfsmount/nfsmount_test.goindex ba6f350dc..a8402e901 100644--- a/cmd/nfsmount/nfsmount_test.go+++ b/cmd/nfsmount/nfsmount_test.go@@ -11,6 +11,8 @@ import ("testing""github.com/rclone/rclone/cmd/serve/nfs"+ "github.com/rclone/rclone/fs/object"+ "github.com/rclone/rclone/vfs""github.com/rclone/rclone/vfs/vfscommon""github.com/rclone/rclone/vfs/vfstest""github.com/stretchr/testify/require"@@ -38,7 +40,7 @@ func TestMount(t *testing.T) {nfs.Opt.HandleCacheDir = t.TempDir()require.NoError(t, nfs.Opt.HandleCache.Set(cacheType))// Check we can create a handler- _, err := nfs.NewHandler(context.Background(), nil, &nfs.Opt)+ _, err := nfs.NewHandler(context.Background(), vfs.New(object.MemoryFs, nil), &nfs.Opt)if errors.Is(err, nfs.ErrorSymlinkCacheNotSupported) || errors.Is(err, nfs.ErrorSymlinkCacheNoPermission) {t.Skip(err.Error() + ": run with: go test -c && sudo setcap cap_dac_read_search+ep ./nfsmount.test && ./nfsmount.test -test.v")}diff --git a/cmd/serve/nfs/cache.go b/cmd/serve/nfs/cache.goindex 32c6cc16c..278d923dc 100644--- a/cmd/serve/nfs/cache.go+++ b/cmd/serve/nfs/cache.go@@ -3,6 +3,7 @@package nfsimport (+ "bytes""crypto/md5""encoding/hex""errors"@@ -30,6 +31,15 @@ var (ErrorSymlinkCacheNoPermission = errors.New("symlink cache must be run as root or with CAP_DAC_READ_SEARCH"))+// Metadata files have the file handle of their source file with this+// suffixed so we can look them up directly from the file handle.+//+// Note that this is 4 bytes - using a non multiple of 4 will cause+// the Linux NFS client not to be able to read any files.+//+// The value is big endian 0x00000001+var metadataSuffix = []byte{0x00, 0x00, 0x00, 0x01}+// Cache controls the file handle cache implementationtype Cache interface {// ToHandle takes a file and represents it with an opaque handle to reference it.@@ -77,7 +87,9 @@ type diskHandler struct {write func(fh []byte, cachePath string, fullPath string) ([]byte, error)read func(fh []byte, cachePath string) ([]byte, error)remove func(fh []byte, cachePath string) error- handleType int32 //nolint:unused // used by the symlink cache+ suffix func(fh []byte) []byte // returns nil for no suffix or the suffix+ handleType int32 //nolint:unused // used by the symlink cache+ metadata string // extension for metadata}// Create a new disk handler@@ -102,6 +114,8 @@ func newDiskHandler(h *Handler) (dh *diskHandler, err error) {write: dh.diskCacheWrite,read: dh.diskCacheRead,remove: dh.diskCacheRemove,+ suffix: dh.diskCacheSuffix,+ metadata: h.vfs.Opt.MetadataExtension,}fs.Infof("nfs", "Storing handle cache in %q", dh.cacheDir)return dh, nil@@ -124,6 +138,17 @@ func (dh *diskHandler) handleToPath(fh []byte) (cachePath string) {return cachePath}+// Return true if name represents a metadata file+//+// It returns the underlying path+func (dh *diskHandler) isMetadataFile(name string) (rawName string, found bool) {+ if dh.metadata == "" {+ return name, false+ }+ rawName, found = strings.CutSuffix(name, dh.metadata)+ return rawName, found+}+// ToHandle takes a file and represents it with an opaque handle to reference it.// In stateless nfs (when it's serving a unix fs) this can be the device + inode// but we can generalize with a stateful local cache of handed out IDs.@@ -131,6 +156,8 @@ func (dh *diskHandler) ToHandle(f billy.Filesystem, splitPath []string) (fh []bydh.mu.Lock()defer dh.mu.Unlock()fullPath := path.Join(splitPath...)+ // metadata file has file handle of original file+ fullPath, isMetadataFile := dh.isMetadataFile(fullPath)fh = hashPath(fullPath)cachePath := dh.handleToPath(fh)cacheDir := filepath.Dir(cachePath)@@ -144,6 +171,10 @@ func (dh *diskHandler) ToHandle(f billy.Filesystem, splitPath []string) (fh []byfs.Errorf("nfs", "Couldn't create cache file handle: %v", err)return fh}+ // metadata file handle is suffixed with metadataSuffix+ if isMetadataFile {+ fh = append(fh, metadataSuffix...)+ }return fh}@@ -152,18 +183,43 @@ func (dh *diskHandler) diskCacheWrite(fh []byte, cachePath string, fullPath strireturn fh, os.WriteFile(cachePath, []byte(fullPath), 0600)}-var errStaleHandle = &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale}+var (+ errStaleHandle = &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale}+)++// Test to see if a fh is a metadata handle and if so return the underlying handle+func (dh *diskHandler) isMetadataHandle(fh []byte) (isMetadata bool, newFh []byte, err error) {+ if dh.metadata == "" {+ return false, fh, nil+ }+ suffix := dh.suffix(fh)+ if len(suffix) == 0 {+ // OK+ return false, fh, nil+ } else if bytes.Equal(suffix, metadataSuffix) {+ return true, fh[:len(fh)-len(suffix)], nil+ }+ fs.Errorf("nfs", "Bad file handle suffix %X", suffix)+ return false, nil, errStaleHandle+}// FromHandle converts from an opaque handle to the file it representsfunc (dh *diskHandler) FromHandle(fh []byte) (f billy.Filesystem, splitPath []string, err error) {dh.mu.RLock()defer dh.mu.RUnlock()+ isMetadata, fh, err := dh.isMetadataHandle(fh)+ if err != nil {+ return nil, nil, err+ }cachePath := dh.handleToPath(fh)fullPathBytes, err := dh.read(fh, cachePath)if err != nil {fs.Errorf("nfs", "Stale handle %q: %v", cachePath, err)return nil, nil, errStaleHandle}+ if isMetadata {+ fullPathBytes = append(fullPathBytes, []byte(dh.metadata)...)+ }splitPath = strings.Split(string(fullPathBytes), "/")return dh.billyFS, splitPath, nil}@@ -177,8 +233,16 @@ func (dh *diskHandler) diskCacheRead(fh []byte, cachePath string) ([]byte, errorfunc (dh *diskHandler) InvalidateHandle(f billy.Filesystem, fh []byte) error {dh.mu.Lock()defer dh.mu.Unlock()+ isMetadata, fh, err := dh.isMetadataHandle(fh)+ if err != nil {+ return err+ }+ if isMetadata {+ // Can't invalidate a metadata handle as it is synthetic+ return nil+ }cachePath := dh.handleToPath(fh)- err := dh.remove(fh, cachePath)+ err = dh.remove(fh, cachePath)if err != nil {fs.Errorf("nfs", "Failed to remove handle %q: %v", cachePath, err)}@@ -190,6 +254,14 @@ func (dh *diskHandler) diskCacheRemove(fh []byte, cachePath string) error {return os.Remove(cachePath)}+// Return a suffix for the file handle or nil+func (dh *diskHandler) diskCacheSuffix(fh []byte) []byte {+ if len(fh) <= md5.Size {+ return nil+ }+ return fh[md5.Size:]+}+// HandleLimit exports how many file handles can be safely stored by this cache.func (dh *diskHandler) HandleLimit() int {return math.MaxIntdiff --git a/cmd/serve/nfs/cache_test.go b/cmd/serve/nfs/cache_test.goindex 03100d34d..6941fde17 100644--- a/cmd/serve/nfs/cache_test.go+++ b/cmd/serve/nfs/cache_test.go@@ -5,10 +5,13 @@ package nfsimport ("context""fmt"+ "strings""sync""testing""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/object"+ "github.com/rclone/rclone/vfs""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -18,6 +21,8 @@ const testSymlinkCache = "go test -c && sudo setcap cap_dac_read_search+ep ./nfs// Check basic CRUD operationsfunc testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {+ isMetadata := strings.HasSuffix(fileName, ".metadata")+// Check reading a non existent handle returns an error_, _, err := c.FromHandle([]byte{10})assert.Error(t, err)@@ -26,6 +31,11 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {splitPath := []string{"dir", fileName}fh := c.ToHandle(h.billyFS, splitPath)assert.True(t, len(fh) > 0)+ if isMetadata {+ assert.Equal(t, metadataSuffix, fh[len(fh)-len(metadataSuffix):])+ } else {+ assert.NotEqual(t, metadataSuffix, fh[len(fh)-len(metadataSuffix):])+ }// Read the handle backnewFs, newSplitPath, err := c.FromHandle(fh)@@ -43,8 +53,13 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {// Check the handle is gone and returning stale handle error_, _, err = c.FromHandle(fh)- require.Error(t, err)- assert.Equal(t, errStaleHandle, err)+ if !isMetadata {+ require.Error(t, err)+ assert.Equal(t, errStaleHandle, err)+ } else {+ // Can't invalidate metadata handles+ require.NoError(t, err)+ }}// Thrash the cache operations in parallel on different files@@ -113,8 +128,10 @@ func TestCache(t *testing.T) {cacheType := cacheTypet.Run(cacheType.String(), func(t *testing.T) {h := &Handler{+ vfs: vfs.New(object.MemoryFs, nil),billyFS: billyFS,}+ h.vfs.Opt.MetadataExtension = ".metadata"h.opt.HandleLimit = 1000h.opt.HandleCache = cacheTypeh.opt.HandleCacheDir = t.TempDir()@@ -151,6 +168,10 @@ func TestCache(t *testing.T) {t.Run("ThrashSame", func(t *testing.T) {testCacheThrashSame(t, h, c)})+ // Metadata file handles only supported on non memory+ t.Run("CRUDMetadata", func(t *testing.T) {+ testCacheCRUD(t, h, c, "file.metadata")+ })}})}diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.goindex ad94c5a49..6e6878e3c 100644--- a/cmd/serve/nfs/nfs.go+++ b/cmd/serve/nfs/nfs.go@@ -169,6 +169,12 @@ Where |$PORT| is the same port number used in the |serve nfs| commandand |$HOSTNAME| is the network address of the machine that |serve nfs|was run on.+If |--vfs-metadata-extension| is in use then for the |--nfs-cache-type disk|+and |--nfs-cache-type cache| the metadata files will have the file+handle of their parent file suffixed with |0x00, 0x00, 0x00, 0x01|.+This means they can be looked up directly from the parent file handle+is desired.+This command is only available on Unix platforms.`, "|", "`") + vfs.Help(),diff --git a/cmd/serve/nfs/symlink_cache_linux.go b/cmd/serve/nfs/symlink_cache_linux.goindex 8002824ff..806de6685 100644--- a/cmd/serve/nfs/symlink_cache_linux.go+++ b/cmd/serve/nfs/symlink_cache_linux.go@@ -82,6 +82,7 @@ func (dh *diskHandler) makeSymlinkCache() error {dh.read = dh.symlinkCacheReaddh.write = dh.symlinkCacheWritedh.remove = dh.symlinkCacheRemove+ dh.suffix = dh.symlinkCacheSuffixreturn nil}@@ -208,3 +209,15 @@ func (dh *diskHandler) symlinkCacheRemove(fh []byte, cachePath string) error {return os.Remove(cachePath)}++// Return a suffix for the file handle or nil+func (dh *diskHandler) symlinkCacheSuffix(fh []byte) []byte {+ if len(fh) < 4 {+ return nil+ }+ length := int(binary.BigEndian.Uint32(fh[:4])) + 4+ if len(fh) <= length {+ return nil+ }+ return fh[length:]+}