"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
vfs: add --vfs-metadata-extension to expose metadata sidecar files
Nick Craig-Wood533c6438f3c0
diff --git a/vfs/dir.go b/vfs/dir.goindex d54e01e92..12d9591a0 100644--- a/vfs/dir.go+++ b/vfs/dir.go@@ -2,6 +2,7 @@ package vfsimport ("context"+ "encoding/json""fmt""os""path"@@ -15,6 +16,7 @@ import ("github.com/rclone/rclone/fs/dirtree""github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/log"+ "github.com/rclone/rclone/fs/object""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/vfs/vfscommon"@@ -817,6 +819,51 @@ func (d *Dir) readDir() error {return d._readDir()}+// jsonErrorf formats the string according to a format specifier and+// returns the resulting string as a JSON blob with key "error"+func jsonErrorf(format string, a ...any) []byte {+ errMsg := fmt.Sprintf(format, a...)+ jsonBlob, _ := json.MarshalIndent(map[string]string{"error": errMsg}, "", "\t")+ return jsonBlob+}++// stat a single metadata item in the directory+//+// Returns true if it is a metadata name+func (d *Dir) statMetadata(leaf, baseLeaf string) (metaNode Node, err error) {+ // Find original file - note that this is recursing into stat()+ node, err := d.stat(baseLeaf)+ if err != nil {+ return node, err+ }+ // Read the metadata from the original entry into a JSON dump+ entry := node.DirEntry()+ var metadataDump []byte+ if entry != nil {+ metadata, err := fs.GetMetadata(context.TODO(), entry)+ if err != nil {+ metadataDump = jsonErrorf("failed to read metadata: %v", err)+ } else if metadata == nil {+ metadataDump = []byte("{}") // no metadata to read+ } else {+ metadataDump, err = json.MarshalIndent(metadata, "", "\t")+ if err != nil {+ metadataDump = jsonErrorf("failed to write metadata: %v", err)+ }+ }+ } else {+ metadataDump = []byte("{}") // no metadata yet when an object is being written+ }+ // Make a memory based file with metadataDump in+ remote := path.Join(d.path, leaf)+ o := object.NewMemoryObject(remote, entry.ModTime(context.TODO()), metadataDump)+ f := newFile(d, d.path, o, leaf)+ // Base the metadata inode number off the real file inode number+ // to keep it constant+ f.inode = node.Inode() ^ (1 << 63)+ return f, nil+}+// stat a single item in the directory//// returns ENOENT if not found.@@ -824,22 +871,38 @@ func (d *Dir) readDir() error {// contains files with names that differ only by case.func (d *Dir) stat(leaf string) (Node, error) {d.mu.Lock()- defer d.mu.Unlock()err := d._readDir()if err != nil {+ d.mu.Unlock()return nil, err}item, ok := d.items[leaf]+ d.mu.Unlock()++ // Look for a metadata file+ if !ok {+ if baseLeaf, found := d.vfs.isMetadataFile(leaf); found {+ node, err := d.statMetadata(leaf, baseLeaf)+ if err != nil {+ return nil, err+ }+ // Add metadata file to directory as virtual object+ d.addObject(node)+ return node, nil+ }+ }ci := fs.GetConfig(context.TODO())normUnicode := !ci.NoUnicodeNormalizationnormCase := ci.IgnoreCaseSync || d.vfs.Opt.CaseInsensitiveif !ok && (normUnicode || normCase) {leafNormalized := operations.ToNormal(leaf, normUnicode, normCase) // this handles both case and unicode normalization+ d.mu.Lock()for name, node := range d.items {if operations.ToNormal(name, normUnicode, normCase) == leafNormalized {if ok {// duplicate normalized match is an error+ d.mu.Unlock()return nil, fmt.Errorf("duplicate filename %q detected with case/unicode normalization settings", leaf)}// found a normalized match@@ -847,6 +910,7 @@ func (d *Dir) stat(leaf string) (Node, error) {item = node}}+ d.mu.Unlock()}if !ok {diff --git a/vfs/dir_test.go b/vfs/dir_test.goindex 7bcbba0f6..8205b33da 100644--- a/vfs/dir_test.go+++ b/vfs/dir_test.go@@ -2,6 +2,7 @@ package vfsimport ("context"+ "encoding/json""fmt""os""runtime"@@ -692,3 +693,72 @@ func TestDirEntryModTimeInvalidation(t *testing.T) {t.Error("ModTime not invalidated")}}++func TestDirMetadataExtension(t *testing.T) {+ r, vfs, dir, _ := dirCreate(t)+ root, err := vfs.Root()+ require.NoError(t, err)+ features := r.Fremote.Features()++ checkListing(t, dir, []string{"file1,14,false"})+ checkListing(t, root, []string{"dir,0,true"})++ node, err := vfs.Stat("dir/file1")+ require.NoError(t, err)+ require.True(t, node.IsFile())++ node, err = vfs.Stat("dir")+ require.NoError(t, err)+ require.True(t, node.IsDir())++ // Check metadata files do not exist+ _, err = vfs.Stat("dir/file1.metadata")+ require.Error(t, err, ENOENT)+ _, err = vfs.Stat("dir.metadata")+ require.Error(t, err, ENOENT)++ // Configure metadata extension+ vfs.Opt.MetadataExtension = ".metadata"++ // Check metadata for file does exist+ node, err = vfs.Stat("dir/file1.metadata")+ require.NoError(t, err)+ require.True(t, node.IsFile())+ size := node.Size()+ assert.Greater(t, size, int64(1))+ modTime := node.ModTime()++ // ...and is now in the listing+ checkListing(t, dir, []string{"file1,14,false", fmt.Sprintf("file1.metadata,%d,false", size)})++ // ...and is a JSON blob with correct "mtime" key+ blob, err := vfs.ReadFile("dir/file1.metadata")+ require.NoError(t, err)+ var metadata map[string]string+ err = json.Unmarshal(blob, &metadata)+ require.NoError(t, err)+ if features.ReadMetadata {+ assert.Equal(t, modTime.Format(time.RFC3339Nano), metadata["mtime"])+ }++ // Check metadata for dir does exist+ node, err = vfs.Stat("dir.metadata")+ require.NoError(t, err)+ require.True(t, node.IsFile())+ size = node.Size()+ assert.Greater(t, size, int64(1))+ modTime = node.ModTime()++ // ...and is now in the listing+ checkListing(t, root, []string{"dir,0,true", fmt.Sprintf("dir.metadata,%d,false", size)})++ // ...and is a JSON blob with correct "mtime" key+ blob, err = vfs.ReadFile("dir.metadata")+ require.NoError(t, err)+ clear(metadata)+ err = json.Unmarshal(blob, &metadata)+ require.NoError(t, err)+ if features.ReadDirMetadata {+ assert.Equal(t, modTime.Format(time.RFC3339Nano), metadata["mtime"])+ }+}diff --git a/vfs/vfs.go b/vfs/vfs.goindex 01124ecc7..ecde90b19 100644--- a/vfs/vfs.go+++ b/vfs/vfs.go@@ -900,3 +900,15 @@ func (vfs *VFS) Symlink(oldname, newname string) error {_, err := vfs.CreateSymlink(oldname, newname)return err}++// Return true if name represents a metadata file+//+// It returns the underlying path+func (vfs *VFS) isMetadataFile(name string) (rawName string, found bool) {+ ext := vfs.Opt.MetadataExtension+ if ext == "" {+ return name, false+ }+ rawName, found = strings.CutSuffix(name, ext)+ return rawName, found+}diff --git a/vfs/vfs.md b/vfs/vfs.mdindex dde8f9f30..5b5496c66 100644--- a/vfs/vfs.md+++ b/vfs/vfs.md@@ -423,3 +423,43 @@ and compute the total used space itself._WARNING._ Contrary to `rclone size`, this flag ignores filters so that theresult is accurate. However, this is very inefficient and may cost lots of APIcalls resulting in extra charges. Use it as a last resort and only with caching.++### VFS Metadata++If you use the `--vfs-metadata-extension` flag you can get the VFS to+expose files which contain the [metadata](/docs/#metadata) as a JSON+blob. These files will not appear in the directory listing, but can be+`stat`-ed and opened and once they have been they **will** appear in+directory listings until the directory cache expires.++Note that some backends won't create metadata unless you pass in the+`--metadata` flag.++For example, using `rclone mount` with `--metadata --vfs-metadata-extension .metadata`+we get++```+$ ls -l /mnt/+total 1048577+-rw-rw-r-- 1 user user 1073741824 Mar 3 16:03 1G++$ cat /mnt/1G.metadata+{+ "atime": "2025-03-04T17:34:22.317069787Z",+ "btime": "2025-03-03T16:03:37.708253808Z",+ "gid": "1000",+ "mode": "100664",+ "mtime": "2025-03-03T16:03:39.640238323Z",+ "uid": "1000"+}++$ ls -l /mnt/+total 1048578+-rw-rw-r-- 1 user user 1073741824 Mar 3 16:03 1G+-rw-rw-r-- 1 user user 185 Mar 3 16:03 1G.metadata+```++If the file has no metadata it will be returned as `{}` and if there+is an error reading the metadata the error will be returned as+`{"error":"error string"}`.+diff --git a/vfs/vfs_test.go b/vfs/vfs_test.goindex 2bd35ee1d..f175e0efc 100644--- a/vfs/vfs_test.go+++ b/vfs/vfs_test.go@@ -487,3 +487,17 @@ func TestFillInMissingSizes(t *testing.T) {})}}++func TestVFSIsMetadataFile(t *testing.T) {+ _, vfs := newTestVFS(t)++ rawName, found := vfs.isMetadataFile("leaf.metadata")+ assert.Equal(t, "leaf.metadata", rawName)+ assert.Equal(t, false, found)++ vfs.Opt.MetadataExtension = ".metadata"++ rawName, found = vfs.isMetadataFile("leaf.metadata")+ assert.Equal(t, "leaf", rawName)+ assert.Equal(t, true, found)+}diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.goindex 650aedd97..7f222edac 100644--- a/vfs/vfscommon/options.go+++ b/vfs/vfscommon/options.go@@ -165,6 +165,11 @@ var OptionsInfo = fs.Options{{Default: getGID(),Help: "Override the gid field set by the filesystem (not supported on Windows)",Groups: "VFS",+}, {+ Name: "vfs_metadata_extension",+ Default: "",+ Help: "Set the extension to read metadata from.",+ Groups: "VFS",}}func init() {@@ -204,6 +209,7 @@ type Options struct {UsedIsSize bool `config:"vfs_used_is_size"` // if true, use the `rclone size` algorithm for Used sizeFastFingerprint bool `config:"vfs_fast_fingerprint"` // if set use fast fingerprintsDiskSpaceTotalSize fs.SizeSuffix `config:"vfs_disk_space_total_size"`+ MetadataExtension string `config:"vfs_metadata_extension"` // if set respond to files with this extension with metadata}// Opt is the default options modified by the environment variables and command line flags