Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve nfs: make metadata files have special file handles

Nick Craig-Wood205667143c80
diff --git a/cmd/nfsmount/nfsmount_test.go b/cmd/nfsmount/nfsmount_test.go
index ba6f350dc..a8402e901 100644
--- a/cmd/nfsmount/nfsmount_test.go
+++ b/cmd/nfsmount/nfsmount_test.go
@@ -11,6 +11,8 @@ import (
"testing"
"github.com/rclone/rclone/cmd/serve/nfs"
+ "github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/rclone/rclone/vfs/vfstest"
"github.com/stretchr/testify/require"
@@ -38,7 +40,7 @@ func TestMount(t *testing.T) {
nfs.Opt.HandleCacheDir = t.TempDir()
require.NoError(t, nfs.Opt.HandleCache.Set(cacheType))
// Check we can create a handler
- _, err := nfs.NewHandler(context.Background(), nil, &nfs.Opt)
+ _, err := nfs.NewHandler(context.Background(), vfs.New(object.MemoryFs, nil), &nfs.Opt)
if errors.Is(err, nfs.ErrorSymlinkCacheNotSupported) || errors.Is(err, nfs.ErrorSymlinkCacheNoPermission) {
t.Skip(err.Error() + ": run with: go test -c && sudo setcap cap_dac_read_search+ep ./nfsmount.test && ./nfsmount.test -test.v")
}
diff --git a/cmd/serve/nfs/cache.go b/cmd/serve/nfs/cache.go
index 32c6cc16c..278d923dc 100644
--- a/cmd/serve/nfs/cache.go
+++ b/cmd/serve/nfs/cache.go
@@ -3,6 +3,7 @@
package nfs
import (
+ "bytes"
"crypto/md5"
"encoding/hex"
"errors"
@@ -30,6 +31,15 @@ var (
ErrorSymlinkCacheNoPermission = errors.New("symlink cache must be run as root or with CAP_DAC_READ_SEARCH")
)
+// Metadata files have the file handle of their source file with this
+// suffixed so we can look them up directly from the file handle.
+//
+// Note that this is 4 bytes - using a non multiple of 4 will cause
+// the Linux NFS client not to be able to read any files.
+//
+// The value is big endian 0x00000001
+var metadataSuffix = []byte{0x00, 0x00, 0x00, 0x01}
+
// Cache controls the file handle cache implementation
type Cache interface {
// ToHandle takes a file and represents it with an opaque handle to reference it.
@@ -77,7 +87,9 @@ type diskHandler struct {
write func(fh []byte, cachePath string, fullPath string) ([]byte, error)
read func(fh []byte, cachePath string) ([]byte, error)
remove func(fh []byte, cachePath string) error
- handleType int32 //nolint:unused // used by the symlink cache
+ suffix func(fh []byte) []byte // returns nil for no suffix or the suffix
+ handleType int32 //nolint:unused // used by the symlink cache
+ metadata string // extension for metadata
}
// Create a new disk handler
@@ -102,6 +114,8 @@ func newDiskHandler(h *Handler) (dh *diskHandler, err error) {
write: dh.diskCacheWrite,
read: dh.diskCacheRead,
remove: dh.diskCacheRemove,
+ suffix: dh.diskCacheSuffix,
+ metadata: h.vfs.Opt.MetadataExtension,
}
fs.Infof("nfs", "Storing handle cache in %q", dh.cacheDir)
return dh, nil
@@ -124,6 +138,17 @@ func (dh *diskHandler) handleToPath(fh []byte) (cachePath string) {
return cachePath
}
+// Return true if name represents a metadata file
+//
+// It returns the underlying path
+func (dh *diskHandler) isMetadataFile(name string) (rawName string, found bool) {
+ if dh.metadata == "" {
+ return name, false
+ }
+ rawName, found = strings.CutSuffix(name, dh.metadata)
+ return rawName, found
+}
+
// ToHandle takes a file and represents it with an opaque handle to reference it.
// In stateless nfs (when it's serving a unix fs) this can be the device + inode
// but we can generalize with a stateful local cache of handed out IDs.
@@ -131,6 +156,8 @@ func (dh *diskHandler) ToHandle(f billy.Filesystem, splitPath []string) (fh []by
dh.mu.Lock()
defer dh.mu.Unlock()
fullPath := path.Join(splitPath...)
+ // metadata file has file handle of original file
+ fullPath, isMetadataFile := dh.isMetadataFile(fullPath)
fh = hashPath(fullPath)
cachePath := dh.handleToPath(fh)
cacheDir := filepath.Dir(cachePath)
@@ -144,6 +171,10 @@ func (dh *diskHandler) ToHandle(f billy.Filesystem, splitPath []string) (fh []by
fs.Errorf("nfs", "Couldn't create cache file handle: %v", err)
return fh
}
+ // metadata file handle is suffixed with metadataSuffix
+ if isMetadataFile {
+ fh = append(fh, metadataSuffix...)
+ }
return fh
}
@@ -152,18 +183,43 @@ func (dh *diskHandler) diskCacheWrite(fh []byte, cachePath string, fullPath stri
return fh, os.WriteFile(cachePath, []byte(fullPath), 0600)
}
-var errStaleHandle = &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale}
+var (
+ errStaleHandle = &nfs.NFSStatusError{NFSStatus: nfs.NFSStatusStale}
+)
+
+// Test to see if a fh is a metadata handle and if so return the underlying handle
+func (dh *diskHandler) isMetadataHandle(fh []byte) (isMetadata bool, newFh []byte, err error) {
+ if dh.metadata == "" {
+ return false, fh, nil
+ }
+ suffix := dh.suffix(fh)
+ if len(suffix) == 0 {
+ // OK
+ return false, fh, nil
+ } else if bytes.Equal(suffix, metadataSuffix) {
+ return true, fh[:len(fh)-len(suffix)], nil
+ }
+ fs.Errorf("nfs", "Bad file handle suffix %X", suffix)
+ return false, nil, errStaleHandle
+}
// FromHandle converts from an opaque handle to the file it represents
func (dh *diskHandler) FromHandle(fh []byte) (f billy.Filesystem, splitPath []string, err error) {
dh.mu.RLock()
defer dh.mu.RUnlock()
+ isMetadata, fh, err := dh.isMetadataHandle(fh)
+ if err != nil {
+ return nil, nil, err
+ }
cachePath := dh.handleToPath(fh)
fullPathBytes, err := dh.read(fh, cachePath)
if err != nil {
fs.Errorf("nfs", "Stale handle %q: %v", cachePath, err)
return nil, nil, errStaleHandle
}
+ if isMetadata {
+ fullPathBytes = append(fullPathBytes, []byte(dh.metadata)...)
+ }
splitPath = strings.Split(string(fullPathBytes), "/")
return dh.billyFS, splitPath, nil
}
@@ -177,8 +233,16 @@ func (dh *diskHandler) diskCacheRead(fh []byte, cachePath string) ([]byte, error
func (dh *diskHandler) InvalidateHandle(f billy.Filesystem, fh []byte) error {
dh.mu.Lock()
defer dh.mu.Unlock()
+ isMetadata, fh, err := dh.isMetadataHandle(fh)
+ if err != nil {
+ return err
+ }
+ if isMetadata {
+ // Can't invalidate a metadata handle as it is synthetic
+ return nil
+ }
cachePath := dh.handleToPath(fh)
- err := dh.remove(fh, cachePath)
+ err = dh.remove(fh, cachePath)
if err != nil {
fs.Errorf("nfs", "Failed to remove handle %q: %v", cachePath, err)
}
@@ -190,6 +254,14 @@ func (dh *diskHandler) diskCacheRemove(fh []byte, cachePath string) error {
return os.Remove(cachePath)
}
+// Return a suffix for the file handle or nil
+func (dh *diskHandler) diskCacheSuffix(fh []byte) []byte {
+ if len(fh) <= md5.Size {
+ return nil
+ }
+ return fh[md5.Size:]
+}
+
// HandleLimit exports how many file handles can be safely stored by this cache.
func (dh *diskHandler) HandleLimit() int {
return math.MaxInt
diff --git a/cmd/serve/nfs/cache_test.go b/cmd/serve/nfs/cache_test.go
index 03100d34d..6941fde17 100644
--- a/cmd/serve/nfs/cache_test.go
+++ b/cmd/serve/nfs/cache_test.go
@@ -5,10 +5,13 @@ package nfs
import (
"context"
"fmt"
+ "strings"
"sync"
"testing"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/vfs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -18,6 +21,8 @@ const testSymlinkCache = "go test -c && sudo setcap cap_dac_read_search+ep ./nfs
// Check basic CRUD operations
func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {
+ isMetadata := strings.HasSuffix(fileName, ".metadata")
+
// Check reading a non existent handle returns an error
_, _, err := c.FromHandle([]byte{10})
assert.Error(t, err)
@@ -26,6 +31,11 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {
splitPath := []string{"dir", fileName}
fh := c.ToHandle(h.billyFS, splitPath)
assert.True(t, len(fh) > 0)
+ if isMetadata {
+ assert.Equal(t, metadataSuffix, fh[len(fh)-len(metadataSuffix):])
+ } else {
+ assert.NotEqual(t, metadataSuffix, fh[len(fh)-len(metadataSuffix):])
+ }
// Read the handle back
newFs, newSplitPath, err := c.FromHandle(fh)
@@ -43,8 +53,13 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {
// Check the handle is gone and returning stale handle error
_, _, err = c.FromHandle(fh)
- require.Error(t, err)
- assert.Equal(t, errStaleHandle, err)
+ if !isMetadata {
+ require.Error(t, err)
+ assert.Equal(t, errStaleHandle, err)
+ } else {
+ // Can't invalidate metadata handles
+ require.NoError(t, err)
+ }
}
// Thrash the cache operations in parallel on different files
@@ -113,8 +128,10 @@ func TestCache(t *testing.T) {
cacheType := cacheType
t.Run(cacheType.String(), func(t *testing.T) {
h := &Handler{
+ vfs: vfs.New(object.MemoryFs, nil),
billyFS: billyFS,
}
+ h.vfs.Opt.MetadataExtension = ".metadata"
h.opt.HandleLimit = 1000
h.opt.HandleCache = cacheType
h.opt.HandleCacheDir = t.TempDir()
@@ -151,6 +168,10 @@ func TestCache(t *testing.T) {
t.Run("ThrashSame", func(t *testing.T) {
testCacheThrashSame(t, h, c)
})
+ // Metadata file handles only supported on non memory
+ t.Run("CRUDMetadata", func(t *testing.T) {
+ testCacheCRUD(t, h, c, "file.metadata")
+ })
}
})
}
diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.go
index ad94c5a49..6e6878e3c 100644
--- a/cmd/serve/nfs/nfs.go
+++ b/cmd/serve/nfs/nfs.go
@@ -169,6 +169,12 @@ Where |$PORT| is the same port number used in the |serve nfs| command
and |$HOSTNAME| is the network address of the machine that |serve nfs|
was run on.
+If |--vfs-metadata-extension| is in use then for the |--nfs-cache-type disk|
+and |--nfs-cache-type cache| the metadata files will have the file
+handle of their parent file suffixed with |0x00, 0x00, 0x00, 0x01|.
+This means they can be looked up directly from the parent file handle
+is desired.
+
This command is only available on Unix platforms.
`, "|", "`") + vfs.Help(),
diff --git a/cmd/serve/nfs/symlink_cache_linux.go b/cmd/serve/nfs/symlink_cache_linux.go
index 8002824ff..806de6685 100644
--- a/cmd/serve/nfs/symlink_cache_linux.go
+++ b/cmd/serve/nfs/symlink_cache_linux.go
@@ -82,6 +82,7 @@ func (dh *diskHandler) makeSymlinkCache() error {
dh.read = dh.symlinkCacheRead
dh.write = dh.symlinkCacheWrite
dh.remove = dh.symlinkCacheRemove
+ dh.suffix = dh.symlinkCacheSuffix
return nil
}
@@ -208,3 +209,15 @@ func (dh *diskHandler) symlinkCacheRemove(fh []byte, cachePath string) error {
return os.Remove(cachePath)
}
+
+// Return a suffix for the file handle or nil
+func (dh *diskHandler) symlinkCacheSuffix(fh []byte) []byte {
+ if len(fh) < 4 {
+ return nil
+ }
+ length := int(binary.BigEndian.Uint32(fh[:4])) + 4
+ if len(fh) <= length {
+ return nil
+ }
+ return fh[length:]
+}