"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
rc: add vfs/queue to show the status of the upload queue
Nick Craig-Wood59acb9dfa951
diff --git a/vfs/rc.go b/vfs/rc.goindex 6e84c26f9..12b7df051 100644--- a/vfs/rc.go+++ b/vfs/rc.go@@ -437,3 +437,51 @@ func rcStats(ctx context.Context, in rc.Params) (out rc.Params, err error) {}return vfs.Stats(), nil}++func init() {+ rc.Add(rc.Call{+ Path: "vfs/queue",+ Title: "Queue info for a VFS.",+ Help: strings.ReplaceAll(`+This returns info about the upload queue for the selected VFS.++This is only useful if |--vfs-cache-mode| > off. If you call it when+the |--vfs-cache-mode| is off, it will return an empty result.++ {+ "queued": // an array of files queued for upload+ [+ {+ "name": "file", // string: name (full path) of the file,+ "id": 123, // integer: id of this item in the queue,+ "size": 79, // integer: size of the file in bytes+ "expiry": 1.5 // float: time until file is eligible for transfer, lowest goes first+ "tries": 1, // integer: number of times we have tried to upload+ "delay": 5.0, // float: seconds between upload attempts+ "uploading": false, // boolean: true if item is being uploaded+ },+ ],+ }++The |expiry| time is the time until the file is elegible for being+uploaded in floating point seconds. This may go negative. As rclone+only transfers |--transfers| files at once, only the lowest+|--transfers| expiry times will have |uploading| as |true|. So there+may be files with negative expiry times for which |uploading| is+|false|.++`, "|", "`") + getVFSHelp,+ Fn: rcQueue,+ })+}++func rcQueue(ctx context.Context, in rc.Params) (out rc.Params, err error) {+ vfs, err := getVFS(in)+ if err != nil {+ return nil, err+ }+ if vfs.cache == nil {+ return nil, nil+ }+ return vfs.cache.Queue(), nil+}diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.goindex 3d34dd82a..dbd408cae 100644--- a/vfs/vfscache/cache.go+++ b/vfs/vfscache/cache.go@@ -170,6 +170,13 @@ func (c *Cache) Stats() (out rc.Params) {return out}+// Queue returns info about the Cache+func (c *Cache) Queue() (out rc.Params) {+ out = make(rc.Params)+ out["queue"] = c.writeback.Queue()+ return out+}+// createDir creates a directory path, along with any necessary parentsfunc createDir(dir string) error {return file.MkdirAll(dir, 0700)diff --git a/vfs/vfscache/cache_test.go b/vfs/vfscache/cache_test.goindex 61357c2f9..20ebee2e7 100644--- a/vfs/vfscache/cache_test.go+++ b/vfs/vfscache/cache_test.go@@ -14,6 +14,7 @@ import ("github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fstest""github.com/rclone/rclone/lib/diskusage"+ "github.com/rclone/rclone/vfs/vfscache/writeback""github.com/rclone/rclone/vfs/vfscommon""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -727,3 +728,16 @@ func TestCacheStats(t *testing.T) {assert.Equal(t, 0, out["uploadsInProgress"])assert.Equal(t, 0, out["uploadsQueued"])}++func TestCacheQueue(t *testing.T) {+ _, c := newTestCache(t)++ out := c.Queue()++ // We've checked the contents of queue in the writeback tests+ // Just check it is present here+ queue, found := out["queue"]+ require.True(t, found)+ _, ok := queue.([]writeback.QueueInfo)+ require.True(t, ok)+}diff --git a/vfs/vfscache/writeback/writeback.go b/vfs/vfscache/writeback/writeback.goindex 33c5616c2..cf3380d4f 100644--- a/vfs/vfscache/writeback/writeback.go+++ b/vfs/vfscache/writeback/writeback.go@@ -6,6 +6,7 @@ import ("container/heap""context""errors"+ "sort""sync""sync/atomic""time"@@ -466,3 +467,47 @@ func (wb *WriteBack) Stats() (uploadsInProgress, uploadsQueued int) {defer wb.mu.Unlock()return wb.uploads, len(wb.items)}++// QueueInfo is information about an item queued for upload, returned+// by Queue+type QueueInfo struct {+ Name string `json:"name"` // name (full path) of the file,+ ID Handle `json:"id"` // id of queue item+ Size int64 `json:"size"` // integer size of the file in bytes+ Expiry float64 `json:"expiry"` // seconds from now which the file is eligible for transfer, oldest goes first+ Tries int `json:"tries"` // number of times we have tried to upload+ Delay float64 `json:"delay"` // delay between upload attempts (s)+ Uploading bool `json:"uploading"` // true if item is being uploaded+}++// Queue return info about the current upload queue+func (wb *WriteBack) Queue() []QueueInfo {+ wb.mu.Lock()+ defer wb.mu.Unlock()++ items := make([]QueueInfo, 0, len(wb.lookup))+ now := time.Now()++ // Lookup all the items in no particular order+ for _, wbItem := range wb.lookup {+ items = append(items, QueueInfo{+ Name: wbItem.name,+ ID: wbItem.id,+ Size: wbItem.size,+ Expiry: wbItem.expiry.Sub(now).Seconds(),+ Tries: wbItem.tries,+ Delay: wbItem.delay.Seconds(),+ Uploading: wbItem.uploading,+ })+ }++ // Sort by Uploading first then Expiry+ sort.Slice(items, func(i, j int) bool {+ if items[i].Uploading != items[j].Uploading {+ return items[i].Uploading+ }+ return items[i].Expiry < items[j].Expiry+ })++ return items+}diff --git a/vfs/vfscache/writeback/writeback_test.go b/vfs/vfscache/writeback/writeback_test.goindex 21d8bae7e..b862e283e 100644--- a/vfs/vfscache/writeback/writeback_test.go+++ b/vfs/vfscache/writeback/writeback_test.go@@ -13,6 +13,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/vfs/vfscommon""github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require")func newTestWriteBack(t *testing.T) (wb *WriteBack, cancel func()) {@@ -493,6 +494,57 @@ func TestWriteBackGetStats(t *testing.T) {}+func TestWriteBackQueue(t *testing.T) {+ wb, cancel := newTestWriteBack(t)+ defer cancel()++ pi := newPutItem(t)++ id := wb.Add(0, "one", 10, true, pi.put)++ queue := wb.Queue()+ require.Equal(t, 1, len(queue))+ assert.Greater(t, queue[0].Expiry, 0.0)+ assert.Less(t, queue[0].Expiry, 1.0)+ queue[0].Expiry = 0.0+ assert.Equal(t, []QueueInfo{+ {+ Name: "one",+ Size: 10,+ Expiry: 0.0,+ Tries: 0,+ Delay: 0.1,+ Uploading: false,+ ID: id,+ },+ }, queue)++ <-pi.started++ queue = wb.Queue()+ require.Equal(t, 1, len(queue))+ assert.Less(t, queue[0].Expiry, 0.0)+ assert.Greater(t, queue[0].Expiry, -1.0)+ queue[0].Expiry = 0.0+ assert.Equal(t, []QueueInfo{+ {+ Name: "one",+ Size: 10,+ Expiry: 0.0,+ Tries: 1,+ Delay: 0.1,+ Uploading: true,+ ID: id,+ },+ }, queue)++ pi.finish(nil) // transfer successful+ waitUntilNoTransfers(t, wb)++ queue = wb.Queue()+ assert.Equal(t, []QueueInfo{}, queue)+}+// Test queuing more than fs.Config.Transfersfunc TestWriteBackMaxQueue(t *testing.T) {ctx := context.Background()