"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
onedrive: authenticate SharePoint downloads, uploads and copies after tempauth retirement
Nick Craig-Wood84bdebae77aa
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.goindex eb253c7eb..5524dc0d4 100644--- a/backend/onedrive/onedrive.go+++ b/backend/onedrive/onedrive.go@@ -41,6 +41,7 @@ import ("github.com/rclone/rclone/lib/oauthutil""github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/rest"+ "golang.org/x/oauth2")const (@@ -844,19 +845,33 @@ type Options struct {// Fs represents a remote OneDrivetype Fs struct {- name string // name of this remote- root string // the path we are working on- opt Options // parsed options- ci *fs.ConfigInfo // global config- features *fs.Features // optional features- srv *rest.Client // the connection to the OneDrive server- unAuth *rest.Client // no authentication connection to the OneDrive server- dirCache *dircache.DirCache // Map of directory path to directory id- pacer *fs.Pacer // pacer for API calls- tokenRenewer *oauthutil.Renew // renew the token on expiry- driveID string // ID to use for querying Microsoft Graph- driveType string // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive- hashType hash.Type // type of the hash we are using+ name string // name of this remote+ root string // the path we are working on+ opt Options // parsed options+ ci *fs.ConfigInfo // global config+ features *fs.Features // optional features+ srv *rest.Client // the connection to the OneDrive server+ unAuth *rest.Client // no authentication connection to the OneDrive server+ dirCache *dircache.DirCache // Map of directory path to directory id+ pacer *fs.Pacer // pacer for API calls+ tokenRenewer *oauthutil.Renew // renew the token on expiry+ tokenSource *oauthutil.TokenSource // source of the API token+ sharePoint *sharePointClients // authenticated clients for SharePoint hosts+ driveID string // ID to use for querying Microsoft Graph+ driveType string // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive+ hashType hash.Type // type of the hash we are using+}++// sharePointClients holds the clients authenticated for SharePoint hosts+type sharePointClients struct {+ mu sync.Mutex+ hosts map[string]sharePointAuth+}++// sharePointAuth is a client authenticated for a SharePoint host+type sharePointAuth struct {+ client *rest.Client+ tokenSource oauth2.TokenSource}// Object describes a OneDrive object@@ -1076,6 +1091,154 @@ func errorHandler(resp *http.Response) error {return errResponse}+// errPreauthRetired explains an authentication failure on a URL which+// SharePoint used to return pre-authenticated.+var errPreauthRetired = errors.New("SharePoint refused to authenticate a request - Microsoft is retiring pre-authenticated (tempauth) URLs, see https://rclone.org/onedrive/#pre-authenticated-urls")++// sharePointDomains are the domains of SharePoint Online in the+// different Microsoft clouds+var sharePointDomains = []string{".sharepoint.com", ".sharepoint.us", ".sharepoint-mil.us", ".sharepoint.cn", ".sharepoint.de"}++// stripTempauth removes the tempauth parameter from the URLs the API+// returns, which simulates the retirement of pre-authenticated URLs+// in tests.+var stripTempauth = false++// returnedURL processes a URL returned by the API before use+func returnedURL(rawURL string) string {+ if !stripTempauth {+ return rawURL+ }+ u, err := url.Parse(rawURL)+ if err != nil {+ return rawURL+ }+ q := u.Query()+ q.Del("tempauth")+ u.RawQuery = q.Encode()+ return u.String()+}++// urlAuth says how a URL returned by the API is authenticated+type urlAuth int++const (+ urlAuthNone urlAuth = iota // call without authentication+ urlAuthAPI // call with the API token+ urlAuthSharePoint // call with a token for the SharePoint host+)++// apiHost returns the host the API calls made with f.srv go to.+func (f *Fs) apiHost() string {+ endpoint := graphAPIEndpoint[f.opt.Region]+ if f.opt.TenantURL != "" {+ endpoint = f.opt.TenantURL+ }+ u, err := url.Parse(endpoint)+ if err != nil || u.Scheme != "https" {+ return ""+ }+ return u.Host+}++// urlAuthFor returns how to authenticate a URL returned by the API,+// such as an upload session or an async job monitor.+//+// Pre-authenticated URLs fail if they are sent an Authorization+// header and tokens must not be sent to other hosts, so only HTTPS+// URLs without tempauth on the API host or a SharePoint host are+// authenticated.+func (f *Fs) urlAuthFor(rawURL string) urlAuth {+ u, err := url.Parse(rawURL)+ if err != nil || u.Scheme != "https" || u.Query().Has("tempauth") {+ return urlAuthNone+ }+ if apiHost := f.apiHost(); apiHost != "" && strings.EqualFold(u.Host, apiHost) {+ return urlAuthAPI+ }+ if isSharePointHost(u.Hostname()) {+ return urlAuthSharePoint+ }+ return urlAuthNone+}++// isSharePointHost returns true if host is in a SharePoint Online domain.+func isSharePointHost(host string) bool {+ host = strings.ToLower(host)+ for _, domain := range sharePointDomains {+ if strings.HasSuffix(host, domain) {+ return true+ }+ }+ return false+}++// checkSharePointRedirect stops the SharePoint token being sent to a+// host which isn't SharePoint, as the token is added to every request+// the client makes.+func checkSharePointRedirect(req *http.Request, via []*http.Request) error {+ if len(via) >= 10 {+ return errors.New("stopped after 10 redirects")+ }+ if req.URL.Scheme != "https" || !isSharePointHost(req.URL.Hostname()) {+ return fmt.Errorf("refusing to send a SharePoint token on a redirect to %s://%s", req.URL.Scheme, req.URL.Host)+ }+ return nil+}++// clientForURL returns the client to call a URL returned by the API with.+func (f *Fs) clientForURL(ctx context.Context, rawURL string) (*rest.Client, error) {+ switch f.urlAuthFor(rawURL) {+ case urlAuthAPI:+ return f.srv, nil+ case urlAuthSharePoint:+ u, _ := url.Parse(rawURL)+ return f.sharePointClient(ctx, u.Host)+ }+ return f.unAuth, nil+}++// sharePointClient returns a client authenticated for the SharePoint+// host, which upload session and async job monitor URLs need once+// they are no longer pre-authenticated.+//+// The token is got from the existing credentials, which Entra ID+// allows for SharePoint as the user has consented to the Graph file+// scopes.+func (f *Fs) sharePointClient(ctx context.Context, host string) (*rest.Client, error) {+ if f.tokenSource == nil || f.sharePoint == nil {+ return nil, fmt.Errorf("%w: no credentials to get a SharePoint token with", errPreauthRetired)+ }+ host = strings.ToLower(host)+ f.sharePoint.mu.Lock()+ defer f.sharePoint.mu.Unlock()+ sp, ok := f.sharePoint.hosts[host]+ if !ok {+ tokenSource := f.tokenSource.ScopedTokenSource([]string{"https://" + host + "/.default"})+ client := oauth2.NewClient(oauthutil.Context(ctx, fshttp.NewClient(ctx)), tokenSource)+ client.CheckRedirect = checkSharePointRedirect+ sp = sharePointAuth{client: rest.NewClient(client), tokenSource: tokenSource}+ f.sharePoint.hosts[host] = sp+ fs.Debugf(f, "Using a SharePoint token for %s", host)+ }+ // Get the token now so failures are reported before the client is used+ if _, err := sp.tokenSource.Token(); err != nil {+ return nil, fmt.Errorf("%w: failed to get a SharePoint token: %w", errPreauthRetired, err)+ }+ return sp.client, nil+}++// checkPreauthRetired adds an explanation to a 401 error from a URL+// returned by the API which needs a SharePoint token, as these were+// pre-authenticated until Microsoft retired them.+func (f *Fs) checkPreauthRetired(rawURL string, resp *http.Response, err error) error {+ if err == nil || resp == nil || resp.StatusCode != http.StatusUnauthorized ||+ f.driveType == driveTypePersonal || errors.Is(err, errPreauthRetired) || f.urlAuthFor(rawURL) != urlAuthSharePoint {+ return err+ }+ return fmt.Errorf("%w: %w", errPreauthRetired, err)+}+func checkUploadChunkSize(cs fs.SizeSuffix) error {const minChunkSize = fs.SizeSuffixBaseif cs%chunkSizeMultiple != 0 {@@ -1144,16 +1307,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eci := fs.GetConfig(ctx)f := &Fs{- name: name,- root: root,- opt: *opt,- ci: ci,- driveID: opt.DriveID,- driveType: opt.DriveType,- srv: rest.NewClient(oAuthClient).SetRoot(rootURL),- unAuth: rest.NewClient(client).SetRoot(rootURL),- pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),- hashType: QuickXorHashType,+ name: name,+ root: root,+ opt: *opt,+ ci: ci,+ driveID: opt.DriveID,+ driveType: opt.DriveType,+ srv: rest.NewClient(oAuthClient).SetRoot(rootURL),+ tokenSource: ts,+ sharePoint: &sharePointClients{hosts: make(map[string]sharePointAuth)},+ unAuth: rest.NewClient(client).SetRoot(rootURL),+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),+ hashType: QuickXorHashType,}f.features = (&fs.Features{CaseInsensitive: true,@@ -1725,17 +1890,25 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {var resp *http.Responsevar err errorvar body []byte+ var redirected boolerr = f.pacer.Call(func() (bool, error) {req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)if err != nil {return false, err}- // The monitor URL needs no authentication, but it should still- // go through the configured transport and honour ctx- resp, err = f.unAuth.Do(req)+ client, err := f.clientForURL(ctx, location)+ if err != nil {+ return false, err+ }+ resp, err = client.Do(req)if err != nil {return fserrors.ShouldRetry(err), err}+ redirected = resp.Request.URL.String() != req.URL.String()+ if resp.StatusCode == http.StatusUnauthorized {+ _ = resp.Body.Close()+ return false, f.checkPreauthRetired(location, resp, fmt.Errorf("async job monitor: HTTP error %s", resp.Status))+ }body, err = rest.ReadBody(resp)return fserrors.ShouldRetry(err), err})@@ -1748,6 +1921,11 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {if err != nil {return fmt.Errorf("async status result not JSON: %q: %w", body, err)}+ // An authenticated monitor redirects to the new item when the+ // job has finished instead of returning its status+ if status.Status == "" && redirected {+ status.Status = "completed"+ }switch status.Status {case "failed":@@ -1847,7 +2025,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj}// read location header- location := resp.Header.Get("Location")+ location := returnedURL(resp.Header.Get("Location"))if location == "" {return nil, errors.New("didn't receive location header in copy response")}@@ -1858,6 +2036,10 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Objfs.Debugf(src, "Server-side copy failed - file not shared between drives")return nil, fs.ErrorCantCopy}+ if errors.Is(err, errPreauthRetired) {+ fs.Debugf(src, "Server-side copy can't be monitored - copying another way: %v", err)+ return nil, fs.ErrorCantCopy+ }if err != nil {return nil, err}@@ -2532,16 +2714,23 @@ func (o *Object) openContentPrefer(ctx context.Context, options ...fs.OpenOptionreturn o.openWithRedirect(ctx, &opts)}-// openWithRedirect downloads via /content style endpoints that 302 to a preauthenticated URL.+// openWithRedirect downloads via /content style endpoints that 302 to+// a download URL, which is pre-authenticated or needs a SharePoint+// token.func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.ReadCloser, err error) {var resp *http.Response- // Make a note of the redirect target as we need to call it without Auth+ // Make a note of the redirect target as it needs different authvar redirectReq *http.Requestopts.CheckRedirect = func(req *http.Request, via []*http.Request) error {if len(via) >= 10 {return errors.New("stopped after 10 redirects")}- // Preauthenticated download URLs must not carry the Graph Authorization header.+ if stripTempauth {+ if u, err := url.Parse(returnedURL(req.URL.String())); err == nil {+ req.URL = u+ }+ }+ // Download URLs must not carry the Graph Authorization header.req.Header.Del("Authorization")// Keep Prefer on the redirect when forcing infected download; some SharePoint// endpoints honor it only on the final download request.@@ -2580,17 +2769,24 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.Rreturn nil, err}if redirectReq != nil {+ redirectURL := redirectReq.URL.String()+ var client *rest.Client+ client, err = o.fs.clientForURL(ctx, redirectURL)+ if err != nil {+ return nil, err+ }err = o.fs.pacer.Call(func() (bool, error) {- resp, err = o.fs.unAuth.Do(redirectReq)+ resp, err = client.Do(redirectReq)if err != nil {return shouldRetry(ctx, resp, err)}- // unAuth.Do does not check status; treat non-2xx as failure so a malware+ // Do does not check status; treat non-2xx as failure so a malware// JSON body is not written out as file content.if resp.StatusCode < 200 || resp.StatusCode > 299 {body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))_ = resp.Body.Close()err = fmt.Errorf("HTTP error %d (%s) %s", resp.StatusCode, resp.Status, strings.TrimSpace(string(body)))+ err = o.fs.checkPreauthRetired(redirectURL, resp, err)if strings.Contains(string(body), "malwareDetected") || resp.Header.Get("X-Virus-Infected") != "" {err = malwareDownloadError(o.fs.opt.AVOverride, err)}@@ -2652,12 +2848,16 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er}var info api.UploadFragmentResponsevar resp *http.Response+ client, err := o.fs.clientForURL(ctx, url)+ if err != nil {+ return 0, err+ }err = o.fs.pacer.Call(func() (bool, error) {- resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)+ resp, err = client.CallJSON(ctx, &opts, nil, &info)return shouldRetry(ctx, resp, err)})if err != nil {- return 0, err+ return 0, o.fs.checkPreauthRetired(url, resp, err)}if len(info.NextExpectedRanges) != 1 {return 0, fmt.Errorf("bad number of ranges in upload position: %v", info.NextExpectedRanges)@@ -2681,6 +2881,10 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, tovar resp *http.Responsevar body []byteskip := int64(0)+ client, err := o.fs.clientForURL(ctx, url)+ if err != nil {+ return nil, err+ }err = o.fs.pacer.Call(func() (bool, error) {toSend := chunkSize - skipopts := rest.Opts{@@ -2692,7 +2896,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, toOptions: options,}_, _ = chunk.Seek(skip, io.SeekStart)- resp, err = o.fs.unAuth.Call(ctx, &opts)+ resp, err = client.Call(ctx, &opts)if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)pos, posErr := o.getPosition(ctx, url)@@ -2731,7 +2935,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to}return false, nil})- return info, err+ return info, o.fs.checkPreauthRetired(url, resp, err)}// cancelUploadSession cancels an upload session@@ -2742,11 +2946,15 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err errorNoResponse: true,}var resp *http.Response+ client, err := o.fs.clientForURL(ctx, url)+ if err != nil {+ return err+ }err = o.fs.pacer.Call(func() (bool, error) {- resp, err = o.fs.unAuth.Call(ctx, &opts)+ resp, err = client.Call(ctx, &opts)return shouldRetry(ctx, resp, err)})- return+ return o.fs.checkPreauthRetired(url, resp, err)}// uploadMultipart uploads a file using multipart upload@@ -2764,7 +2972,20 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objecif err != nil {return nil, err}- uploadURL := session.UploadURL+ uploadURL := returnedURL(session.UploadURL)++ // Check a session which needs SharePoint authentication can get it+ // before reading any data, so it can be uploaded in a single part+ // instead if it can't.+ if o.fs.urlAuthFor(uploadURL) == urlAuthSharePoint {+ if _, err = o.getPosition(ctx, uploadURL); err != nil {+ if !errors.Is(err, errPreauthRetired) || size >= int64(maxSinglePartSize) {+ return nil, err+ }+ fs.Debugf(o, "Uploading in a single part as the upload session can't be used: %v", err)+ return o.uploadSinglepart(ctx, in, src, options...)+ }+ }// Cancel the session if something went wrongdefer atexit.OnError(&err, func() {diff --git a/backend/onedrive/onedrive_internal_test.go b/backend/onedrive/onedrive_internal_test.goindex de576c531..397a7ef51 100644--- a/backend/onedrive/onedrive_internal_test.go+++ b/backend/onedrive/onedrive_internal_test.go@@ -3,6 +3,7 @@ package onedriveimport ("context""encoding/json"+ "flag""fmt""net/http""net/http/httptest"@@ -44,6 +45,10 @@ const (// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_invite?view=odsp-graph-online#http-request-1)+func init() {+ flag.BoolVar(&stripTempauth, "strip-tempauth", false, "Remove tempauth from URLs the API returns to simulate the retirement of pre-authenticated URLs")+}+// TestMain drives the testsfunc TestMain(m *testing.M) {fstest.TestMain(m)@@ -818,3 +823,49 @@ func TestWaitForJobCancel(t *testing.T) {t.Fatal("waitForJob did not return after the context was cancelled")}}++// TestWaitForJobUnauthorized checks that a monitor URL which refuses+// the request fails straight away instead of polling until the timeout.+func TestWaitForJobUnauthorized(t *testing.T) {+ var polls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ polls.Add(1)+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusUnauthorized)+ _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": "unauthenticated", "message": "no token"}})+ }))+ t.Cleanup(server.Close)+ f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())+ f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))+ f.driveType = driveTypeBusiness++ err := f.waitForJob(context.Background(), server.URL+"/monitor", &Object{remote: "file.txt"})+ require.Error(t, err)+ assert.Contains(t, err.Error(), "401")+ assert.Equal(t, int32(1), polls.Load())+}++// TestWaitForJobRedirectCompleted checks that a monitor which+// redirects to the new item, as it does when authenticated, is taken+// to mean the job has finished.+func TestWaitForJobRedirectCompleted(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/monitor":+ http.Redirect(w, r, "/item", http.StatusSeeOther)+ case "/item":+ w.Header().Set("Content-Type", "application/json")+ _, _ = fmt.Fprint(w, `{"id":"newid","name":"file.txt","size":5,"file":{}}`)+ default:+ t.Errorf("unexpected request %s", r.URL)+ }+ }))+ t.Cleanup(server.Close)+ f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())+ f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))++ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt", hasMetaData: true})+ require.NoError(t, err)+}diff --git a/backend/onedrive/upload_internal_test.go b/backend/onedrive/upload_internal_test.goindex aae575466..0150e62e7 100644--- a/backend/onedrive/upload_internal_test.go+++ b/backend/onedrive/upload_internal_test.go@@ -4,6 +4,7 @@ import ("bytes""context""encoding/json"+ "errors""fmt""io""net/http"@@ -13,11 +14,14 @@ import ("time""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/object"+ "github.com/rclone/rclone/lib/oauthutil""github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/rest""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "golang.org/x/oauth2")// fragmentServer is a fake OneDrive upload session which records the@@ -66,6 +70,8 @@ func (s *fragmentServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {}w.WriteHeader(http.StatusAccepted)_, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)+ case r.Method == "DELETE" && r.URL.Path == "/upload":+ w.WriteHeader(http.StatusNoContent)default:s.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)w.WriteHeader(http.StatusNotFound)@@ -118,3 +124,298 @@ func TestUploadMultipartSkip(t *testing.T) {assert.Equal(t, 2, s.fragments)assert.Equal(t, 3, s.attempts, "expected one 416, a partial re-send and the final fragment")}++// TestURLAuthFor checks that tokens are only sent to HTTPS URLs on+// the API host or a SharePoint host which aren't pre-authenticated.+func TestURLAuthFor(t *testing.T) {+ const tenant = "https://contoso-my.sharepoint.com/_api"+ for _, tc := range []struct {+ name string+ region string+ tenantURL string+ url string+ want urlAuth+ }{+ {"graph", "global", "", "https://graph.microsoft.com/v1.0/drives/x/items/y/uploadSession", urlAuthAPI},+ {"graph case", "global", "", "https://Graph.Microsoft.com/v1.0/monitor/abc", urlAuthAPI},+ {"graph tempauth", "global", "", "https://graph.microsoft.com/v1.0/upload?tempauth=abc", urlAuthNone},+ {"graph http", "global", "", "http://graph.microsoft.com/v1.0/upload", urlAuthNone},+ {"graph other region", "global", "", "https://graph.microsoft.us/v1.0/upload", urlAuthNone},+ {"graph us", "us", "", "https://graph.microsoft.us/v1.0/upload", urlAuthAPI},+ {"graph lookalike", "global", "", "https://graph.microsoft.com.example.com/upload", urlAuthNone},+ {"sharepoint tempauth", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x&tempauth=abc", urlAuthNone},+ {"sharepoint", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x", urlAuthSharePoint},+ {"sharepoint case", "global", "", "https://Contoso.SharePoint.com/_api/v2.0/monitor/abc", urlAuthSharePoint},+ {"sharepoint us", "us", "", "https://contoso.sharepoint.us/_api/v2.0/upload", urlAuthSharePoint},+ {"sharepoint cn", "cn", "", "https://contoso.sharepoint.cn/_api/v2.0/upload", urlAuthSharePoint},+ {"sharepoint http", "global", "", "http://contoso-my.sharepoint.com/_api/v2.0/upload", urlAuthNone},+ {"sharepoint lookalike", "global", "", "https://contoso.sharepoint.com.example.com/upload", urlAuthNone},+ {"sharepoint suffix only", "global", "", "https://evilsharepoint.com/upload", urlAuthNone},+ {"personal", "global", "", "https://api.onedrive.com/rup/abc", urlAuthNone},+ {"personal upload", "global", "", "https://my.microsoftpersonalcontent.com/personal/abc/_api/v2.0/drive/items/x/uploadSession", urlAuthNone},+ {"unparseable", "global", "", "https://graph.microsoft.com/%zz", urlAuthNone},+ {"tenant", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/monitor/abc", urlAuthAPI},+ {"tenant tempauth", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/upload?tempauth=abc", urlAuthNone},+ {"tenant graph", "global", tenant, "https://graph.microsoft.com/v1.0/upload", urlAuthNone},+ {"tenant other sharepoint", "global", tenant, "https://contoso.sharepoint.com/_api/v2.0/upload", urlAuthSharePoint},+ } {+ t.Run(tc.name, func(t *testing.T) {+ f := &Fs{}+ f.opt.Region = tc.region+ f.opt.TenantURL = tc.tenantURL+ assert.Equal(t, tc.want, f.urlAuthFor(tc.url))+ })+ }+}++// TestCheckSharePointRedirect checks the SharePoint token is only+// sent on redirects to SharePoint over HTTPS.+func TestCheckSharePointRedirect(t *testing.T) {+ for _, tc := range []struct {+ url string+ ok bool+ }{+ {"https://contoso-my.sharepoint.com/download.aspx", true},+ {"https://205807-ipv4v6.gr.global.aa-rt.sharepoint.com/download.aspx?tempauth=x", true},+ {"https://contoso.sharepoint.us/download.aspx", true},+ {"http://contoso-my.sharepoint.com/download.aspx", false},+ {"https://contoso.sharepoint.com.example.com/download.aspx", false},+ {"https://graph.microsoft.com/v1.0/me", false},+ {"https://example.com/", false},+ } {+ req, err := http.NewRequest("GET", tc.url, nil)+ require.NoError(t, err)+ err = checkSharePointRedirect(req, nil)+ if tc.ok {+ assert.NoError(t, err, tc.url)+ } else {+ assert.Error(t, err, tc.url)+ }+ }+ req, err := http.NewRequest("GET", "https://contoso-my.sharepoint.com/", nil)+ require.NoError(t, err)+ assert.Error(t, checkSharePointRedirect(req, make([]*http.Request, 10)), "too many redirects")+}++// TestCheckPreauthRetired checks which 401 errors get the explanation.+func TestCheckPreauthRetired(t *testing.T) {+ const sharePointURL = "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x"+ errBase := errors.New("HTTP error 401")+ for _, tc := range []struct {+ name string+ driveType string+ url string+ status int+ want bool+ }{+ {"sharepoint 401", driveTypeBusiness, sharePointURL, http.StatusUnauthorized, true},+ {"library 401", driveTypeSharepoint, sharePointURL, http.StatusUnauthorized, true},+ {"personal 401", driveTypePersonal, sharePointURL, http.StatusUnauthorized, false},+ {"sharepoint 403", driveTypeBusiness, sharePointURL, http.StatusForbidden, false},+ {"tempauth 401", driveTypeBusiness, sharePointURL + "&tempauth=abc", http.StatusUnauthorized, false},+ {"graph 401", driveTypeBusiness, "https://graph.microsoft.com/v1.0/upload", http.StatusUnauthorized, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ f := &Fs{driveType: tc.driveType}+ f.opt.Region = "global"+ err := f.checkPreauthRetired(tc.url, &http.Response{StatusCode: tc.status}, errBase)+ assert.ErrorIs(t, err, errBase)+ assert.Equal(t, tc.want, errors.Is(err, errPreauthRetired), "err = %v", err)+ })+ }+ f := &Fs{driveType: driveTypeBusiness}+ assert.NoError(t, f.checkPreauthRetired(sharePointURL, &http.Response{StatusCode: http.StatusUnauthorized}, nil))+ assert.ErrorIs(t, f.checkPreauthRetired(sharePointURL, nil, errBase), errBase)+}++// TestSharePointClient checks that the client for a SharePoint host+// is authenticated with a token for that host got from the existing+// refresh token, which is fetched once and not stored.+func TestSharePointClient(t *testing.T) {+ var scopes []string+ var auths []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/token":+ _ = r.ParseForm()+ assert.Equal(t, "refresh_token", r.PostForm.Get("grant_type"))+ assert.Equal(t, "refresh", r.PostForm.Get("refresh_token"))+ scopes = append(scopes, r.PostForm.Get("scope"))+ w.Header().Set("Content-Type", "application/json")+ _, _ = fmt.Fprint(w, `{"access_token":"sharepoint_token","token_type":"Bearer","expires_in":3600}`)+ case "/upload":+ auths = append(auths, r.Header.Get("Authorization"))+ w.Header().Set("Content-Type", "application/json")+ _, _ = fmt.Fprint(w, `{"nextExpectedRanges":["0-"]}`)+ case "/redirect":+ http.Redirect(w, r, "/upload", http.StatusFound)+ default:+ t.Errorf("unexpected request %s %s", r.Method, r.URL)+ }+ }))+ t.Cleanup(server.Close)++ storedToken := `{"access_token":"graph_token","token_type":"Bearer","refresh_token":"refresh","expiry":"2099-01-01T00:00:00Z"}`+ m := configmap.Simple{"token": storedToken}+ ctx := context.Background()+ _, ts, err := oauthutil.NewClientWithBaseClient(ctx, "remote", m, &oauthutil.Config{+ ClientID: "client_id",+ ClientSecret: "client_secret",+ TokenURL: server.URL + "/token",+ }, server.Client())+ require.NoError(t, err)+ f := &Fs{tokenSource: ts, sharePoint: &sharePointClients{hosts: make(map[string]sharePointAuth)}}++ for range 2 {+ client, err := f.sharePointClient(ctx, "Contoso-my.sharepoint.com")+ require.NoError(t, err)+ _, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/upload"})+ require.NoError(t, err)+ }+ assert.Equal(t, []string{"https://contoso-my.sharepoint.com/.default"}, scopes, "token should be fetched once for the host")+ assert.Equal(t, []string{"Bearer sharepoint_token", "Bearer sharepoint_token"}, auths)+ assert.Equal(t, storedToken, m["token"], "the stored token must not change")++ // The token isn't sent on a redirect to a host which isn't SharePoint+ client, err := f.sharePointClient(ctx, "contoso-my.sharepoint.com")+ require.NoError(t, err)+ _, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/redirect"})+ assert.ErrorContains(t, err, "refusing to send a SharePoint token")+ assert.Len(t, auths, 2)++ // Without credentials there is no client and the error explains why+ _, err = (&Fs{}).sharePointClient(ctx, "contoso-my.sharepoint.com")+ assert.ErrorIs(t, err, errPreauthRetired)+}++// TestUploadSessionFallback checks that an upload session which needs+// SharePoint authentication which can't be got is replaced by a single+// part upload before any data is read.+func TestUploadSessionFallback(t *testing.T) {+ content := bytes.Repeat([]byte("onedrive"), 1024)+ var requests []string+ var uploaded []byte+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests = append(requests, r.Method+" "+r.URL.Path)+ w.Header().Set("Content-Type", "application/json")+ switch {+ case r.Method == "POST" && r.URL.Path == "/root/createUploadSession":+ _, _ = fmt.Fprint(w, `{"uploadUrl":"https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x","nextExpectedRanges":["0-"]}`)+ case r.Method == "PUT" && r.URL.Path == "/root/content":+ var err error+ uploaded, err = io.ReadAll(r.Body)+ require.NoError(t, err)+ _, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))+ case r.Method == "PATCH":+ _, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))+ default:+ t.Errorf("unexpected request %s %s", r.Method, r.URL)+ w.WriteHeader(http.StatusNotFound)+ }+ }))+ t.Cleanup(server.Close)+ client := rest.NewClient(server.Client()).SetRoot(server.URL)+ f := &Fs{+ ci: fs.GetConfig(context.Background()),+ srv: client,+ unAuth: client,+ driveType: driveTypeBusiness,+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),+ }+ f.opt.Region = "global"+ f.opt.ChunkSize = fs.SizeSuffix(len(content) / 2)+ o := &Object{fs: f}++ src := object.NewStaticObjectInfo("remote", time.Now(), int64(len(content)), true, nil, nil)+ _, err := o.uploadMultipart(context.Background(), bytes.NewReader(content), src)+ require.NoError(t, err)+ assert.Equal(t, content, uploaded)+ assert.Equal(t, []string{"POST /root/createUploadSession", "PUT /root/content"}, requests[:2])+}++// redirectTransport sends every request to the test server, so URLs+// on other hosts can be tested.+type redirectTransport struct {+ server *httptest.Server+}++func (t redirectTransport) RoundTrip(req *http.Request) (*http.Response, error) {+ req = req.Clone(req.Context())+ req.URL.Scheme = "http"+ req.URL.Host = t.server.Listener.Addr().String()+ return t.server.Client().Transport.RoundTrip(req)+}++// newDownloadTest returns an Object which downloads content from a+// server which handles the /content call with contentHandler and+// serves download.aspx with Range support, recording the+// Authorization header of each download.+func newDownloadTest(t *testing.T, content []byte, contentHandler http.HandlerFunc) (o *Object, auths *[]string) {+ auths = new([]string)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/items/fake/content":+ contentHandler(w, r)+ case "/download.aspx":+ *auths = append(*auths, r.Header.Get("Authorization"))+ http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(content))+ default:+ t.Errorf("unexpected request %s %s", r.Method, r.URL)+ w.WriteHeader(http.StatusNotFound)+ }+ }))+ t.Cleanup(server.Close)+ transport := redirectTransport{server: server}+ sharePointToken := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "sharepoint_token"})+ sharePointHTTP := &http.Client{Transport: &oauth2.Transport{Source: sharePointToken, Base: transport}}+ f := &Fs{+ ci: fs.GetConfig(context.Background()),+ srv: rest.NewClient(server.Client()).SetRoot(server.URL),+ unAuth: rest.NewClient(&http.Client{Transport: transport}),+ driveType: driveTypeBusiness,+ tokenSource: &oauthutil.TokenSource{},+ sharePoint: &sharePointClients{hosts: map[string]sharePointAuth{+ "contoso-my.sharepoint.com": {client: rest.NewClient(sharePointHTTP), tokenSource: sharePointToken},+ }},+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),+ }+ f.opt.Region = "global"+ return &Object{fs: f, id: "fake", size: int64(len(content))}, auths+}++// TestOpenRedirectAuth checks that downloads are redirected to a+// pre-authenticated URL without authentication and to a SharePoint+// URL with a SharePoint token.+func TestOpenRedirectAuth(t *testing.T) {+ content := []byte("0123456789abcdefghij")+ for _, test := range []struct {+ name string+ location string+ wantAuth string+ }{+ {"tempauth", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x&tempauth=v1.secret", ""},+ {"sharepoint", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", "Bearer sharepoint_token"},+ } {+ t.Run(test.name, func(t *testing.T) {+ o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {+ http.Redirect(w, r, test.location, http.StatusFound)+ })+ in, err := o.Open(context.Background(), &fs.RangeOption{Start: 5, End: 9})+ require.NoError(t, err)+ got, err := io.ReadAll(in)+ require.NoError(t, err)+ require.NoError(t, in.Close())+ assert.Equal(t, "56789", string(got))+ assert.Equal(t, []string{test.wantAuth}, *auths)+ })+ }++ // Without credentials the error explains why+ o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {+ http.Redirect(w, r, "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", http.StatusFound)+ })+ o.fs.tokenSource = nil+ _, err := o.Open(context.Background())+ assert.ErrorIs(t, err, errPreauthRetired)+ assert.Empty(t, *auths)+}diff --git a/docs/content/onedrive.md b/docs/content/onedrive.mdindex d0f5f2d0f..12e6cacfd 100644--- a/docs/content/onedrive.md+++ b/docs/content/onedrive.md@@ -1226,6 +1226,28 @@ rclone cleanup remote:path/subdir # unconditionally remove all old## Troubleshooting+### Pre-authenticated URLs++Microsoft is [retiring pre-authenticated (tempauth) URLs](https://devblogs.microsoft.com/microsoft365dev/pre-authenticated-tempauth-urls-are-retiring-move-to-microsoft-entra-id-tokens-before-april-1-2027/)+in OneDrive for Business and SharePoint from April 1, 2027.++Rclone uses these URLs when downloading files, when uploading files+bigger than `--onedrive-upload-cutoff` in chunks and when doing+server-side copies. When SharePoint returns them without+pre-authentication rclone authenticates them with a SharePoint token+which it gets using the existing login, so no reconfiguration is+needed. OneDrive Personal is not affected.++If rclone can't get a SharePoint token, or SharePoint refuses it, you+will see the error `SharePoint refused to authenticate a request`. In+this case rclone uploads files smaller than 250 MiB in a single+request instead and copies files by downloading and uploading them,+but bigger files can't be uploaded and downloads may fail.++SharePoint does not accept tokens from the client credentials flow+which were got with a client secret, so this affects+`client_credentials` users.+### Excessive throttling or blocked on SharePointIf you experience excessive throttling or is being blocked on SharePoint then