"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
yandex: add app_folder option to support cloud_api:disk.app_folder OAuth scope - Fixes #9848
Sanjay Kanth A13084df67c7e
diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.goindex 412a84da3..20711a3c9 100644--- a/backend/yandex/yandex.go+++ b/backend/yandex/yandex.go@@ -100,6 +100,20 @@ normally enough to stop them, at the cost of slowing down uploads.Yandex support recommend a value of 1.5s - 3s.`,Default: fs.Duration(0),Advanced: true,+ }, {+ Name: "app_folder",+ Help: `Use the application folder as the root.++If you registered your own OAuth application with Yandex with only+the "Application Folder" permission (cloud_api:disk.app_folder)+instead of full disk access, then rclone needs to address paths with+the "app:/" prefix instead of the usual "disk:/" prefix, otherwise+all requests fail with a 403 Forbidden error.++Set this to true if your OAuth token only has access to the+application folder.`,+ Default: false,+ Advanced: true,}}...),})}@@ -111,18 +125,20 @@ type Options struct {Enc encoder.MultiEncoder `config:"encoding"`SpoofUserAgent bool `config:"spoof_ua"`UploadWait fs.Duration `config:"upload_wait"`+ AppFolder bool `config:"app_folder"`}// Fs represents a remote yandextype Fs struct {- name string- root string // root path- opt Options // parsed options- ci *fs.ConfigInfo // global config- features *fs.Features // optional features- srv *rest.Client // the connection to the yandex server- pacer *fs.Pacer // pacer for API calls- diskRoot string // root path with "disk:/" container name+ name string+ root string // root path+ opt Options // parsed options+ ci *fs.ConfigInfo // global config+ features *fs.Features // optional features+ srv *rest.Client // the connection to the yandex server+ pacer *fs.Pacer // pacer for API calls+ diskRoot string // root path with the container prefix, e.g. "disk:/" or "app:/"+ container string // container prefix in use, e.g. "disk:" or "app:"}// Object describes a swift object@@ -209,13 +225,21 @@ func errorHandler(resp *http.Response) error {func (f *Fs) setRoot(root string) {//Set root pathf.root = strings.Trim(root, "/")+ //Set the container prefix. This is "disk:" normally, or "app:" if the+ //OAuth token only has access to the application folder (see the+ //app_folder option).+ if f.opt.AppFolder {+ f.container = "app:"+ } else {+ f.container = "disk:"+ }//Set disk root path.- //Adding "disk:" to root path as all paths on disk start with it+ //Adding the container prefix to root path as all paths on disk start with itvar diskRoot stringif f.root == "" {- diskRoot = "disk:/"+ diskRoot = f.container + "/"} else {- diskRoot = "disk:/" + f.root + "/"+ diskRoot = f.container + "/" + f.root + "/"}f.diskRoot = diskRoot}@@ -494,9 +518,13 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {NoResponse: true,}- // If creating a directory with a : use (undocumented) disk: prefix- if strings.ContainsRune(path, ':') {- path = "disk:" + path+ if f.opt.AppFolder {+ // Bare relative paths are not resolved under the app folder root,+ // unlike the disk root, so the "app:" prefix is always required.+ path = f.container + path+ } else if strings.ContainsRune(path, ':') {+ // If creating a directory with a : use (undocumented) disk: prefix+ path = f.container + path}opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(path))@@ -518,8 +546,8 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {func (f *Fs) mkDirs(ctx context.Context, path string) (err error) {//trim filename from path//dirString := strings.TrimSuffix(path, filepath.Base(path))- //trim "disk:" from path- dirString := strings.TrimPrefix(path, "disk:")+ //trim the container prefix, e.g. "disk:" or "app:", from path+ dirString := strings.TrimPrefix(path, f.container)if dirString == "" {return nil}@@ -808,7 +836,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string//fmt.Printf("Move src: %s (FullPath: %s), dst: %s (FullPath: %s)\n", srcRemote, srcPath, dstRemote, dstPath)// Refuse to move to or from the root- if srcPath == "disk:/" || dstPath == "disk:/" {+ if srcPath == srcFs.container+"/" || dstPath == f.container+"/" {fs.Debugf(src, "DirMove error: Can't move root")return errors.New("can't move root directory")}diff --git a/backend/yandex/yandex_internal_test.go b/backend/yandex/yandex_internal_test.gonew file mode 100644index 000000000..5eea4e0b2--- /dev/null+++ b/backend/yandex/yandex_internal_test.go@@ -0,0 +1,88 @@+package yandex++import (+ "context"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// TestSetRoot checks that setRoot addresses paths with the "disk:/" prefix+// by default, and with the "app:/" prefix when app_folder is set.+func TestSetRoot(t *testing.T) {+ for _, test := range []struct {+ appFolder bool+ root string+ wantDiskRoot string+ wantFilePath string+ }{+ {false, "", "disk:/", "disk:/file.txt"},+ {false, "backups", "disk:/backups/", "disk:/backups/file.txt"},+ {true, "", "app:/", "app:/file.txt"},+ {true, "backups", "app:/backups/", "app:/backups/file.txt"},+ } {+ f := &Fs{opt: Options{AppFolder: test.appFolder}}+ f.setRoot(test.root)+ assert.Equal(t, test.wantDiskRoot, f.diskRoot, "diskRoot for root=%q app_folder=%v", test.root, test.appFolder)+ assert.Equal(t, test.wantFilePath, f.filePath("file.txt"), "filePath for root=%q app_folder=%v", test.root, test.appFolder)+ }+}++// TestCreateDir checks the path CreateDir sends to the API, both for the+// default disk:/ root and for the app:/ root used when app_folder is set.+// Yandex Disk does not resolve bare relative paths under the app folder+// root, so those must be sent with an explicit "app:/" prefix, but the+// default disk:/ behaviour (relying on the API's implicit disk root for+// bare relative paths, only prefixing when the path itself contains a ':')+// must stay unchanged for existing users.+func TestCreateDir(t *testing.T) {+ for _, test := range []struct {+ appFolder bool+ path string+ wantPath string+ }{+ {false, "/backups/", "/backups/"},+ {false, "/foo:bar/", "disk:/foo:bar/"},+ {true, "/backups/", "app:/backups/"},+ {true, "/foo:bar/", "app:/foo:bar/"},+ } {+ var gotPath string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotPath = r.URL.Query().Get("path")+ w.WriteHeader(http.StatusCreated)+ }))++ ctx := context.Background()+ f := &Fs{+ opt: Options{AppFolder: test.appFolder},+ srv: rest.NewClient(server.Client()).SetRoot(server.URL),+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),+ }+ f.setRoot("")++ err := f.CreateDir(ctx, test.path)+ require.NoError(t, err, "path=%q app_folder=%v", test.path, test.appFolder)+ assert.Equal(t, test.wantPath, gotPath, "path=%q app_folder=%v", test.path, test.appFolder)+ server.Close()+ }+}++// TestDirMoveRoot checks that DirMove refuses to move a directory to the+// container root, both for the default disk:/ root and for the app:/ root+// used when app_folder is set.+func TestDirMoveRoot(t *testing.T) {+ for _, appFolder := range []bool{false, true} {+ f := &Fs{opt: Options{AppFolder: appFolder}}+ f.setRoot("")++ err := f.DirMove(context.Background(), f, "somedir", "")+ assert.EqualError(t, err, "can't move root directory", "app_folder=%v", appFolder)+ }+}diff --git a/docs/content/yandex.md b/docs/content/yandex.mdindex f59c5eed8..d1558dc28 100644--- a/docs/content/yandex.md+++ b/docs/content/yandex.md@@ -125,6 +125,16 @@ are replaced.Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),as they can't be used in JSON strings.+### Application folder++If you registered your own OAuth application with Yandex and only granted+it the "Application Folder" permission (`cloud_api:disk.app_folder`)+instead of full disk access, then you must set the `app_folder` option to+`true`. Yandex Disk requires paths to be addressed with an `app:/` prefix+instead of the usual `disk:/` prefix when using this restricted scope, and+rclone will return a 403 Forbidden error on every request if this option+isn't set correctly for the token in use.+<!-- autogenerated options start - DO NOT EDIT - instead edit fs.RegInfo in backend/yandex/yandex.go and run make backenddocs to verify --> <!-- markdownlint-disable-line line-length -->### Standard options